From 13f98a7897b793e071bf569ef6d964de06b0cbf9 Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Wed, 16 Jul 2025 20:56:13 -0400 Subject: [PATCH 01/18] fix proxy type --- src/logger.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/logger.ts b/src/logger.ts index 4d25a1c6..16c9a276 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,13 +1,13 @@ -import colors from 'colors'; -import { bootstrap } from 'global-agent'; -import { ProxyAgent, setGlobalDispatcher } from 'undici'; -import yargs from 'yargs-parser'; +import colors from "colors"; +import { bootstrap } from "global-agent"; +import { ProxyAgent, setGlobalDispatcher } from "undici"; +import yargs from "yargs-parser"; export const logger = console; // When the proxy env var of flag is specified, initiate the proxy const { httpProxy = process.env.http_proxy } = yargs(process.argv.slice(2)); -if (httpProxy) { +if (httpProxy && typeof httpProxy === "string") { logger.info(colors.green(`Initializing proxy: ${httpProxy}`)); // Use global-agent, which overrides `request` based requests From 7f7373f7d19f2731ad17e3ff9d5b8810d42288cf Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Wed, 16 Jul 2025 21:02:35 -0400 Subject: [PATCH 02/18] format --- README.md | 1646 ----------------------------------------------------- 1 file changed, 1646 deletions(-) diff --git a/README.md b/README.md index 4a243c5a..e69de29b 100644 --- a/README.md +++ b/README.md @@ -1,1646 +0,0 @@ -# Transcend CLI - - - - -## Table of Contents - -- [Changelog](#changelog) -- [Overview](#overview) -- [Installation](#installation) -- [transcend.yml](#transcendyml) -- [Usage](#usage) - - [`transcend request approve`](#transcend-request-approve) - - [`transcend request upload`](#transcend-request-upload) - - [`transcend request download-files`](#transcend-request-download-files) - - [`transcend request cancel`](#transcend-request-cancel) - - [`transcend request restart`](#transcend-request-restart) - - [`transcend request notify-additional-time`](#transcend-request-notify-additional-time) - - [`transcend request mark-silent`](#transcend-request-mark-silent) - - [`transcend request enricher-restart`](#transcend-request-enricher-restart) - - [`transcend request reject-unverified-identifiers`](#transcend-request-reject-unverified-identifiers) - - [`transcend request export`](#transcend-request-export) - - [`transcend request skip-preflight-jobs`](#transcend-request-skip-preflight-jobs) - - [`transcend request system mark-request-data-silos-completed`](#transcend-request-system-mark-request-data-silos-completed) - - [`transcend request system retry-request-data-silos`](#transcend-request-system-retry-request-data-silos) - - [`transcend request system skip-request-data-silos`](#transcend-request-system-skip-request-data-silos) - - [`transcend request preflight pull-identifiers`](#transcend-request-preflight-pull-identifiers) - - [`transcend request preflight push-identifiers`](#transcend-request-preflight-push-identifiers) - - [`transcend request cron pull-identifiers`](#transcend-request-cron-pull-identifiers) - - [`transcend request cron mark-identifiers-completed`](#transcend-request-cron-mark-identifiers-completed) - - [`transcend consent build-xdi-sync-endpoint`](#transcend-consent-build-xdi-sync-endpoint) - - [`transcend consent pull-consent-metrics`](#transcend-consent-pull-consent-metrics) - - [`transcend consent pull-consent-preferences`](#transcend-consent-pull-consent-preferences) - - [`transcend consent update-consent-manager`](#transcend-consent-update-consent-manager) - - [`transcend consent upload-consent-preferences`](#transcend-consent-upload-consent-preferences) - - [`transcend consent upload-cookies-from-csv`](#transcend-consent-upload-cookies-from-csv) - - [`transcend consent upload-data-flows-from-csv`](#transcend-consent-upload-data-flows-from-csv) - - [`transcend consent upload-preferences`](#transcend-consent-upload-preferences) - - [`transcend inventory pull`](#transcend-inventory-pull) - - [Scopes](#scopes) - - [Usage](#usage-1) - - [`transcend inventory push`](#transcend-inventory-push) - - [Scopes](#scopes-1) - - [Usage](#usage-2) - - [CI Integration](#ci-integration) - - [Dynamic Variables](#dynamic-variables) - - [`transcend inventory scan-packages`](#transcend-inventory-scan-packages) - - [`transcend inventory discover-silos`](#transcend-inventory-discover-silos) - - [Usage](#usage-3) - - [`transcend inventory pull-datapoints`](#transcend-inventory-pull-datapoints) - - [`transcend inventory pull-unstructured-discovery-files`](#transcend-inventory-pull-unstructured-discovery-files) - - [`transcend inventory derive-data-silos-from-data-flows`](#transcend-inventory-derive-data-silos-from-data-flows) - - [`transcend inventory derive-data-silos-from-data-flows-cross-instance`](#transcend-inventory-derive-data-silos-from-data-flows-cross-instance) - - [`transcend inventory consent-manager-service-json-to-yml`](#transcend-inventory-consent-manager-service-json-to-yml) - - [`transcend inventory consent-managers-to-business-entities`](#transcend-inventory-consent-managers-to-business-entities) - - [`transcend admin generate-api-keys`](#transcend-admin-generate-api-keys) - - [Usage](#usage-4) - - [`transcend migration sync-ot`](#transcend-migration-sync-ot) -- [Prompt Manager](#prompt-manager) -- [Proxy usage](#proxy-usage) - - - -## Changelog - -To stay up to date on breaking changes to the CLI between major version updates, please refer to [CHANGELOG.md](CHANGELOG.md). - -## Overview - -A command line interface that allows you to programatically interact with the Transcend. - -## Installation - -This package is distributed through npm, and assumes an installation of [npm and Node](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm). - -```sh -npm install --global @transcend-io/cli -transcend --help -``` - -You can also run the CLI using npx: - -```sh -npx @transcend-io/cli -- transcend --help -``` - -Note - -_The CLI commands which interact with Transcend's API will default to using Transcend's EU backend. To use these commands with the US backend, you will need to add the flag --transcendUrl=https://api.us.transcend.io. You can also set the environment variable `TRANSCEND_API_URL=https://api.us.transcend.io`_ - -## transcend.yml - -Within your git repositories, you can define a file `transcend.yml`. This file allows you define part of your Data Map in code. Using the CLI, you can sync that configuration back to the Transcend Admin Dashboard (https://app.transcend.io/privacy-requests/connected-services). - -You can find various examples for your `transcend.yml` file in the [examples/](./examples/) folder. If you are looking for a starting point to copy and paste, [simple.yml](./examples/simple.yml) is a good place to start. This file is annotated with links and documentations that new members of your team can use if they come across the file. - -The API for this YAML file can be found in [./src/codecs.ts](./src/codecs.ts) under the variable named "TranscendInput". The shape of the YAML file will be type-checked every time a command is run. - -By default, your editor or IDE should recognize `transcend.yml` and validate it against our latest published [JSON schema](./transcend-yml-schema-latest.json). This is dependent on whether your editor uses [yaml-language-server](https://github.com/redhat-developer/yaml-language-server), such as through the [VS Code YAML extension](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml). - -Your editor will use the latest version's schema. To pin the `transcend.yml` schema to a previous major version, include this at the top of your file (and change `v4` to your target major version): - -```yml -# yaml-language-server: $schema=https://raw.githubusercontent.com/transcend-io/cli/main/transcend-yml-schema-v4.json -``` - -The structure of `transcend.yml` looks something like the following: - -```yaml -# Manage at: https://app.transcend.io/infrastructure/api-keys -# See https://docs.transcend.io/docs/authentication -# Define API keys that may be shared across data silos -# in the data map. When creating new data silos through the YAML -# CLI, it is possible to specify which API key should be associated -# with the newly created data silo. -api-keys: - - title: Webhook Key - - title: Analytics Key - -# Manage at: https://app.transcend.io/privacy-requests/identifiers -# See https://docs.transcend.io/docs/identity-enrichment -# Define enricher or pre-flight check webhooks that will be executed -# prior to privacy request workflows. Some examples may include: -# - identity enrichment: look up additional identifiers for that user. -# i.e. map an email address to a user ID -# - fraud check: auto-cancel requests if the user is flagged for fraudulent behavior -# - customer check: auto-cancel request for some custom business criteria -enrichers: - - title: Basic Identity Enrichment - description: Enrich an email address to the userId and phone number - url: https://example.acme.com/transcend-enrichment-webhook - input-identifier: email - output-identifiers: - - userId - - phone - - myUniqueIdentifier - - title: Fraud Check - description: Ensure the email address is not marked as fraudulent - url: https://example.acme.com/transcend-fraud-check - input-identifier: email - output-identifiers: - - email - privacy-actions: - - ERASURE - -# Manage at: https://app.transcend.io/privacy-requests/connected-services -# See https://docs.transcend.io/docs/the-data-map#data-silos -# Define the data silos in your data map. A data silo can be a database, -# or a web service that may use a collection of different data stores under the hood. -data-silos: - # Note: title is the only required top-level field for a data silo - - title: Redshift Data Warehouse - description: The mega-warehouse that contains a copy over all SQL backed databases - integrationName: server - url: https://example.acme.com/transcend-webhook - api-key-title: Webhook Key - data-subjects: - - customer - - employee - - newsletter-subscriber - - b2b-contact - identity-keys: - - email - - userId - deletion-dependencies: - - Identity Service - owners: - - alice@transcend.io - datapoints: - - title: Webhook Notification - key: _global - privacy-actions: - - ACCESS - - ERASURE - - SALE_OPT_OUT - - title: User Model - description: The centralized user model user - key: users - privacy-actions: - - ACCESS - fields: - - key: firstName - title: First Name - description: The first name of the user, inputted during onboarding - - key: email - title: Email - description: The email address of the user -``` - -## Usage - - - -### `transcend request approve` - -```txt -USAGE - transcend request approve (--auth value) (--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD) [--origins PRIVACY_CENTER|ADMIN_DASHBOARD|API|SHOPIFY] [--silentModeBefore value] [--createdAtBefore value] [--createdAtAfter value] [--transcendUrl value] [--concurrency value] - transcend request approve --help - -Bulk approve a set of privacy requests from the DSR Automation -> Incoming Requests tab. - -FLAGS - --auth The Transcend API key. Requires scopes: "Request Approval and Communication", "View Incoming Requests", "Manage Request Compilation" - --actions The request actions to approve [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--origins] The request origins to approve [PRIVACY_CENTER|ADMIN_DASHBOARD|API|SHOPIFY, separator = ,] - [--silentModeBefore] Any requests made before this date should be marked as silent mode - [--createdAtBefore] Approve requests that were submitted before this time - [--createdAtAfter] Approve requests that were submitted after this time - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--concurrency] The concurrency to use when uploading requests in parallel [default = 50] - -h --help Print help information and exit -``` - -### `transcend request upload` - -```txt -USAGE - transcend request upload (--auth value) [--file value] [--transcendUrl value] [--cacheFilepath value] [--requestReceiptFolder value] [--sombraAuth value] [--concurrency value] [--attributes value] [--isTest] [--isSilent] [--skipSendingReceipt] [--emailIsVerified] [--skipFilterStep] [--dryRun] [--debug] [--defaultPhoneCountryCode value] - transcend request upload --help - -Upload a set of requests from a CSV. - -This command prompts you to map the shape of the CSV to the shape of the Transcend API. There is no requirement for the shape of the incoming CSV, as the script will handle the mapping process. - -The script will also produce a JSON cache file that allows for the mappings to be preserved between runs. - -FLAGS - --auth The Transcend API key. Requires scopes: "Submit New Data Subject Request", "View Identity Verification Settings", "View Global Attributes" - [--file] Path to the CSV file of requests to upload [default = ./requests.csv] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--cacheFilepath] The path to the JSON file encoding the metadata used to map the CSV shape to Transcend API [default = ./transcend-privacy-requests-cache.json] - [--requestReceiptFolder] The path to the folder where receipts of each upload are stored [default = ./privacy-request-upload-receipts] - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - [--concurrency] The concurrency to use when uploading requests in parallel [default = 50] - [--attributes] Tag all of the requests with the following attributes. Format: key1:value1;value2,key2:value3;value4 [default = Tags:transcend-cli] - [--isTest] Flag whether the requests being uploaded are test requests or regular requests [default = false] - [--isSilent/--noIsSilent] Flag whether the requests being uploaded should be submitted in silent mode [default = true] - [--skipSendingReceipt] Flag whether to skip sending of the receipt email [default = false] - [--emailIsVerified/--noEmailIsVerified] Indicate whether the email address being uploaded is pre-verified. Set to false to send a verification email [default = true] - [--skipFilterStep] When true, skip the interactive step to filter down the CSV [default = false] - [--dryRun] When true, perform a dry run of the upload instead of calling the API to submit the requests [default = false] - [--debug] Debug logging [default = false] - [--defaultPhoneCountryCode] When uploading phone numbers, if the phone number is missing a country code, assume this country code [default = 1] - -h --help Print help information and exit -``` - -### `transcend request download-files` - -```txt -USAGE - transcend request download-files (--auth value) [--sombraAuth value] [--concurrency value] [--requestIds value]... [--statuses REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED] [--folderPath value] [--createdAtBefore value] [--createdAtAfter value] [--approveAfterDownload] [--transcendUrl value] - transcend request download-files --help - -Download the files associated with a Data Subject Access Request (DSAR) from DSR Automation -> Incoming Requests tab. - -FLAGS - --auth The Transcend API key. Requires scopes: "View the Request Compilation", "View Incoming Requests", "Request Approval and Communication" - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - [--concurrency] The concurrency to use when downloading requests in parallel [default = 10] - [--requestIds]... Specify the specific request IDs to download [separator = ,] - [--statuses] The request statuses to download. Comma-separated list. Defaults to APPROVING,DOWNLOADABLE. [REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED, separator = ,] - [--folderPath] The folder to download files to [default = ./dsr-files] - [--createdAtBefore] Download requests that were submitted before this time - [--createdAtAfter] Download requests that were submitted after this time - [--approveAfterDownload] If the request is in status=APPROVING, approve the request after its downloaded [default = false] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -### `transcend request cancel` - -```txt -USAGE - transcend request cancel (--auth value) (--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD) [--statuses REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED] [--requestIds value]... [--silentModeBefore value] [--createdAtBefore value] [--createdAtAfter value] [--cancellationTitle value] [--transcendUrl value] [--concurrency value] - transcend request cancel --help - -Bulk cancel a set of privacy requests from the DSR Automation -> Incoming Requests tab. - -FLAGS - --auth The Transcend API key. Requires scopes: "View Incoming Requests", "Request Approval and Communication" - --actions The request actions to cancel [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--statuses] The request statuses to cancel. Comma-separated list. [REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED, separator = ,] - [--requestIds]... Specify the specific request IDs to cancel [separator = ,] - [--silentModeBefore] Any requests made before this date should be marked as silent mode for canceling to skip email sending - [--createdAtBefore] Cancel requests that were submitted before this time - [--createdAtAfter] Cancel requests that were submitted after this time - [--cancellationTitle] The title of the email template that should be sent to the requests upon cancelation [default = Request Canceled] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--concurrency] The concurrency to use when uploading requests in parallel [default = 50] - -h --help Print help information and exit -``` - -### `transcend request restart` - -```txt -USAGE - transcend request restart (--auth value) (--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD) (--statuses REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED) [--transcendUrl value] [--requestReceiptFolder value] [--sombraAuth value] [--concurrency value] [--requestIds value]... [--emailIsVerified] [--createdAt value] [--silentModeBefore value] [--createdAtBefore value] [--createdAtAfter value] [--sendEmailReceipt] [--copyIdentifiers] [--skipWaitingPeriod] - transcend request restart --help - -Bulk update a set of privacy requests based on a set of request filters. - -FLAGS - --auth The Transcend API key. Requires scopes: "Submit New Data Subject Request", "View the Request Compilation" - --actions The request actions to restart [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - --statuses The request statuses to restart [REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED, separator = ,] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--requestReceiptFolder] The path to the folder where receipts of each upload are stored [default = ./privacy-request-upload-receipts] - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - [--concurrency] The concurrency to use when uploading requests in parallel [default = 15] - [--requestIds]... Specify the specific request IDs to restart [separator = ,] - [--emailIsVerified/--noEmailIsVerified] Indicate whether the primary email address is verified. Set to false to send a verification email [default = true] - [--createdAt] Restart requests that were submitted before a specific date - [--silentModeBefore] Requests older than this date should be marked as silent mode - [--createdAtBefore] Restart requests that were submitted before this time - [--createdAtAfter] Restart requests that were submitted after this time - [--sendEmailReceipt] Send email receipts to the restarted requests [default = false] - [--copyIdentifiers] Copy over all enriched identifiers from the initial request [default = false] - [--skipWaitingPeriod] Skip queued state of request and go straight to compiling [default = false] - -h --help Print help information and exit -``` - -### `transcend request notify-additional-time` - -```txt -USAGE - transcend request notify-additional-time (--auth value) (--createdAtBefore value) [--createdAtAfter value] [--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD] [--daysLeft value] [--days value] [--requestIds value]... [--emailTemplate value] [--transcendUrl value] [--concurrency value] - transcend request notify-additional-time --help - -Bulk notify a set of privacy requests from the DSR Automation -> Incoming Requests tab that more time is needed to complete the request. Note any request in silent mode will not be emailed. - -FLAGS - --auth The Transcend API key. Requires scopes: "View Incoming Requests", "Request Approval and Communication" - --createdAtBefore Notify requests that are open but submitted before this time - [--createdAtAfter] Notify requests that are open but submitted after this time - [--actions] The request actions to notify [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--daysLeft] Only notify requests that have less than this number of days until they are considered expired [default = 10] - [--days] The number of days to adjust the expiration of the request to [default = 45] - [--requestIds]... Specify the specific request IDs to notify [separator = ,] - [--emailTemplate] The title of the email template that should be sent to the requests [default = Additional Time Needed] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--concurrency] The concurrency to use when uploading requests in parallel [default = 50] - -h --help Print help information and exit -``` - -### `transcend request mark-silent` - -```txt -USAGE - transcend request mark-silent (--auth value) (--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD) [--statuses REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED] [--requestIds value]... [--createdAtBefore value] [--createdAtAfter value] [--transcendUrl value] [--concurrency value] - transcend request mark-silent --help - -Bulk update a set of privacy requests from the DSR Automation -> Incoming Requests tab to be in silent mode. - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Request Compilation" - --actions The request actions to mark silent [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--statuses] The request statuses to mark silent. Comma-separated list. Defaults to REQUEST_MADE,WAITING,ENRICHING,COMPILING,DELAYED,APPROVING,SECONDARY,SECONDARY_APPROVING. [REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED, separator = ,] - [--requestIds]... Specify the specific request IDs to mark silent [separator = ,] - [--createdAtBefore] Mark silent requests that were submitted before this time - [--createdAtAfter] Mark silent requests that were submitted after this time - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--concurrency] The concurrency to use when uploading requests in parallel [default = 50] - -h --help Print help information and exit -``` - -### `transcend request enricher-restart` - -```txt -USAGE - transcend request enricher-restart (--auth value) (--enricherId value) [--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD] [--requestEnricherStatuses QUEUED|WAITING|SKIPPED|ERROR|RESOLVED|ACTION_REQUIRED|REMOTE_PROCESSING|WAITING_ON_DEPENDENCIES|POLLING] [--transcendUrl value] [--concurrency value] [--requestIds value]... [--createdAtBefore value] [--createdAtAfter value] - transcend request enricher-restart --help - -Bulk restart a particular enricher across a series of DSRs. - -The API key needs the following scopes: -- Manage Request Compilation - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Request Compilation" - --enricherId The ID of the enricher to restart - [--actions] The request action to restart [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--requestEnricherStatuses] The request enricher statuses to restart [QUEUED|WAITING|SKIPPED|ERROR|RESOLVED|ACTION_REQUIRED|REMOTE_PROCESSING|WAITING_ON_DEPENDENCIES|POLLING, separator = ,] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--concurrency] The concurrency to use when uploading requests in parallel [default = 15] - [--requestIds]... Specify the specific request IDs to restart [separator = ,] - [--createdAtBefore] Restart requests that were submitted before this time - [--createdAtAfter] Restart requests that were submitted after this time - -h --help Print help information and exit -``` - -### `transcend request reject-unverified-identifiers` - -```txt -USAGE - transcend request reject-unverified-identifiers (--auth value) (--identifierNames value)... [--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD] [--transcendUrl value] - transcend request reject-unverified-identifiers --help - -Bulk clear out any request identifiers that are unverified. - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Request Compilation" - --identifierNames... The names of identifiers to clear out [separator = ,] - [--actions] The request action to restart [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -### `transcend request export` - -```txt -USAGE - transcend request export (--auth value) [--sombraAuth value] [--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD] [--statuses REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED] [--transcendUrl value] [--file value] [--concurrency value] [--createdAtBefore value] [--createdAtAfter value] [--showTests] [--pageLimit value] - transcend request export --help - -Export privacy requests and request identifiers to a CSV file. - -FLAGS - --auth The Transcend API key. Requires scopes: "View Incoming Requests", "View the Request Compilation" - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - [--actions] The request actions to export [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--statuses] The request statuses to export [REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED, separator = ,] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--file] Path to the CSV file where identifiers will be written to [default = ./transcend-request-export.csv] - [--concurrency] The concurrency to use when uploading requests in parallel [default = 100] - [--createdAtBefore] Pull requests that were submitted before this time - [--createdAtAfter] Pull requests that were submitted after this time - [--showTests/--noShowTests] Filter for test requests or production requests - when not provided, pulls both - [--pageLimit] The page limit to use when pulling in pages of requests [default = 100] - -h --help Print help information and exit -``` - -### `transcend request skip-preflight-jobs` - -```txt -USAGE - transcend request skip-preflight-jobs (--auth value) (--enricherIds value)... [--transcendUrl value] - transcend request skip-preflight-jobs --help - -This command allows for bulk skipping preflight checks. - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Request Compilation" - --enricherIds... The ID of the enrichers to skip privacy request jobs for [separator = ,] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -### `transcend request system mark-request-data-silos-completed` - -```txt -USAGE - transcend request system mark-request-data-silos-completed (--auth value) (--dataSiloId value) [--file value] [--transcendUrl value] - transcend request system mark-request-data-silos-completed --help - -This command takes in a CSV of Request IDs as well as a Data Silo ID and marks all associated privacy request jobs as completed. -This command is useful with the "Bulk Response" UI. The CSV is expected to have 1 column named "Request Id". - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Request Compilation" - --dataSiloId The ID of the data silo to pull in - [--file] Path to the CSV file where identifiers will be written to. The CSV is expected to have 1 column named "Request Id". [default = ./request-identifiers.csv] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -### `transcend request system retry-request-data-silos` - -```txt -USAGE - transcend request system retry-request-data-silos (--auth value) (--dataSiloId value) (--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD) [--transcendUrl value] - transcend request system retry-request-data-silos --help - -This command allows for bulk restarting a set of data silos jobs for open privacy requests. This is equivalent to clicking the "Wipe and Retry" button for a particular data silo across a set of privacy requests. - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Request Compilation" - --dataSiloId The ID of the data silo to pull in - --actions The request actions to restart [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -### `transcend request system skip-request-data-silos` - -```txt -USAGE - transcend request system skip-request-data-silos (--auth value) (--dataSiloId value) [--transcendUrl value] (--statuses REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED) [--status SKIPPED|RESOLVED] - transcend request system skip-request-data-silos --help - -This command allows for bulk skipping all open privacy request jobs for a particular data silo. This command is useful if you want to disable a data silo and then clear out any active privacy requests that are still queued up for that data silo. - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Request Compilation" - --dataSiloId The ID of the data silo to skip privacy request jobs for - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - --statuses The request statuses to skip [REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED, separator = ,] - [--status] The status to set the request data silo job to [SKIPPED|RESOLVED, default = SKIPPED] - -h --help Print help information and exit -``` - -### `transcend request preflight pull-identifiers` - -```txt -USAGE - transcend request preflight pull-identifiers (--auth value) [--sombraAuth value] [--transcendUrl value] [--file value] [--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD] [--concurrency value] - transcend request preflight pull-identifiers --help - -This command pulls down the set of privacy requests that are currently pending manual enrichment. - -This is useful for the following workflow: - -1. Pull identifiers to CSV: - transcend request preflight pull-identifiers --file=./enrichment-requests.csv -2. Fill out the CSV with additional identifiers -3. Push updated back to Transcend - transcend request preflight push-identifiers --file=./enrichment-requests.csv - -FLAGS - --auth The Transcend API key. Requires scopes: "View Incoming Requests", "View the Request Compilation" - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--file] Path to the CSV file where requests will be written to [default = ./manual-enrichment-identifiers.csv] - [--actions] The request actions to pull for [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--concurrency] The concurrency to use when uploading requests in parallel [default = 100] - -h --help Print help information and exit -``` - -### `transcend request preflight push-identifiers` - -```txt -USAGE - transcend request preflight push-identifiers (--auth value) (--enricherId value) [--sombraAuth value] [--transcendUrl value] [--file value] [--markSilent] [--concurrency value] - transcend request preflight push-identifiers --help - -This command push up a set of identifiers for a set of requests pending manual enrichment. - -This is useful for the following workflow: - -1. Pull identifiers to CSV: - transcend request preflight pull-identifiers --file=./enrichment-requests.csv -2. Fill out the CSV with additional identifiers -3. Push updated back to Transcend - transcend request preflight push-identifiers --file=./enrichment-requests.csv - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Request Identity Verification", "Manage Request Compilation" - --enricherId The ID of the Request Enricher to upload to - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--file] Path to the CSV file where requests will be written to [default = ./manual-enrichment-identifiers.csv] - [--markSilent] When true, set requests into silent mode before enriching [default = false] - [--concurrency] The concurrency to use when uploading requests in parallel [default = 100] - -h --help Print help information and exit -``` - -### `transcend request cron pull-identifiers` - -```txt -USAGE - transcend request cron pull-identifiers (--auth value) (--dataSiloId value) (--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD) [--file value] [--transcendUrl value] [--sombraAuth value] [--pageLimit value] [--skipRequestCount] [--chunkSize value] - transcend request cron pull-identifiers --help - -If you are using the cron job integration, you can run this command to pull the outstanding identifiers for the data silo to a CSV. - -For large datasets, the output will be automatically split into multiple CSV files to avoid file system size limits. Use the --chunkSize parameter to control the maximum number of rows per file. - -Read more at https://docs.transcend.io/docs/integrations/cron-job-integration. - -FLAGS - --auth The Transcend API key. This key must be associated with the data silo(s) being operated on. No scopes are required for this command. - --dataSiloId The ID of the data silo to pull in - --actions The request actions to restart [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--file] Path to the CSV file where identifiers will be written to [default = ./cron-identifiers.csv] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - [--pageLimit] The page limit to use when pulling in pages of identifiers [default = 100] - [--skipRequestCount] Whether to skip the count of all outstanding requests. This is required to render the progress bar, but can take a long time to run if you have a large number of outstanding requests to process. In that case, we recommend setting skipRequestCount=true so that you can still proceed with fetching the identifiers [default = false] - [--chunkSize] Maximum number of rows per CSV file. For large datasets, the output will be automatically split into multiple files to avoid file system size limits. Each file will contain at most this many rows [default = 10000] - -h --help Print help information and exit -``` - -### `transcend request cron mark-identifiers-completed` - -```txt -USAGE - transcend request cron mark-identifiers-completed (--auth value) (--dataSiloId value) [--file value] [--transcendUrl value] [--sombraAuth value] - transcend request cron mark-identifiers-completed --help - -This command takes the output of "transcend request cron pull-identifiers" and notifies Transcend that all of the requests in the CSV have been processed. -This is used in the workflow like: - -1. Pull identifiers to CSV: - transcend request cron pull-identifiers --auth=$TRANSCEND_API_KEY --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f --actions=ERASURE --file=./outstanding-requests.csv -2. Run your process to operate on that CSV of requests. -3. Notify Transcend of completion - transcend request cron mark-identifiers-completed --auth=$TRANSCEND_API_KEY --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f --file=./outstanding-requests.csv - -Read more at https://docs.transcend.io/docs/integrations/cron-job-integration. - -FLAGS - --auth The Transcend API key. This key must be associated with the data silo(s) being operated on. No scopes are required for this command. - --dataSiloId The ID of the data silo to pull in - [--file] Path to the CSV file where identifiers will be written to [default = ./cron-identifiers.csv] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - -h --help Print help information and exit -``` - -### `transcend consent build-xdi-sync-endpoint` - -```txt -USAGE - transcend consent build-xdi-sync-endpoint (--auth value) (--xdiLocation value) [--file value] [--removeIpAddresses] [--domainBlockList value] [--xdiAllowedCommands value] [--transcendUrl value] - transcend consent build-xdi-sync-endpoint --help - -This command allows for building of the XDI Sync Endpoint across a set of Transcend accounts. - -FLAGS - --auth The Transcend API key. Requires scopes: "View Consent Manager" - --xdiLocation The location of the XDI that will be loaded by the generated sync endpoint - [--file] The HTML file path where the sync endpoint should be written [default = ./sync-endpoint.html] - [--removeIpAddresses/--noRemoveIpAddresses] When true, remove IP addresses from the domain list [default = true] - [--domainBlockList] The set of domains that should be excluded from the sync endpoint. Comma-separated list. [default = localhost] - [--xdiAllowedCommands] The allowed set of XDI commands [default = ConsentManager:Sync] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -### `transcend consent pull-consent-metrics` - -```txt -USAGE - transcend consent pull-consent-metrics (--auth value) (--start value) [--end value] [--folder value] [--bin value] [--transcendUrl value] - transcend consent pull-consent-metrics --help - -This command allows for pulling consent manager metrics for a Transcend account, or a set of Transcend accounts. - -By default, the consent metrics will be written to a folder named `consent-metrics` within the directory where you run the command. You can override the location that these CSVs are written to using the flag `--folder=./my-folder/`. This folder will contain a set of CSV files: - -- `CONSENT_CHANGES_TIMESERIES_optIn.csv` -> this is a feed containing the number of explicit opt in events that happen - these are calls to `airgap.setConsent(event, { SaleOfInfo: true });` -- `CONSENT_CHANGES_TIMESERIES_optOut.csv` -> this is a feed containing the number of explicit opt out events that happen - these are calls to `airgap.setConsent(event, { SaleOfInfo: false });` -- `CONSENT_SESSIONS_BY_REGIME_Default.csv` -> this contains the number of sessions detected for the bin period -- `PRIVACY_SIGNAL_TIMESERIES_DNT.csv` -> the number of DNT signals detected. -- `PRIVACY_SIGNAL_TIMESERIES_GPC.csv` -> the number of GPC signals detected. - -FLAGS - --auth The Transcend API key. Requires scopes: "View Consent Manager" - --start The start date to pull metrics from - [--end] The end date to pull metrics until - [--folder] The folder to save metrics to [default = ./consent-metrics/] - [--bin] The bin metric when pulling data (1h or 1d) [default = 1d] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -### `transcend consent pull-consent-preferences` - -```txt -USAGE - transcend consent pull-consent-preferences (--auth value) (--partition value) [--sombraAuth value] [--file value] [--transcendUrl value] [--timestampBefore value] [--timestampAfter value] [--identifiers value]... [--concurrency value] - transcend consent pull-consent-preferences --help - -This command allows for pull of consent preferences from the Managed Consent Database. - -FLAGS - --auth The Transcend API key. Requires scopes: "View Managed Consent Database Admin API" - --partition The partition key to download consent preferences to - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - [--file] Path to the CSV file to save preferences to [default = ./preferences.csv] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--timestampBefore] Filter for consents updated this time - [--timestampAfter] Filter for consents updated after this time - [--identifiers]... Filter for specific identifiers [separator = ,] - [--concurrency] The concurrency to use when downloading consents in parallel [default = 100] - -h --help Print help information and exit -``` - -Each row in the CSV will include: - -| Argument | Description | Type | Default | Required | -| -------------- | --------------------------------------------------------------------------------------------------------- | ------------------------- | ------- | -------- | -| userId | Unique ID identifying the user that the preferences ar efor | string | N/A | true | -| timestamp | Timestamp for when consent was collected for that user | string - timestamp | N/A | true | -| purposes | JSON map from consent purpose name -> boolean indicating whether user has opted in or out of that purpose | {[k in string]: boolean } | {} | true | -| airgapVersion | Version of airgap where consent was collected | string | N/A | false | -| [purpose name] | Each consent purpose from `purposes` is also included in a column | boolean | N/A | false | -| tcf | IAB TCF string | string - TCF | N/A | false | -| gpp | IAB GPP string | string - GPP | N/A | false | - -### `transcend consent update-consent-manager` - -```txt -USAGE - transcend consent update-consent-manager (--auth value) (--bundleTypes PRODUCTION|TEST) [--deploy] [--transcendUrl value] - transcend consent update-consent-manager --help - -This command allows for updating Consent Manager to latest version. The Consent Manager bundle can also be deployed using this command. - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Consent Manager Developer Settings" - --bundleTypes The bundle types to deploy. Defaults to PRODUCTION,TEST. [PRODUCTION|TEST, separator = ,] - [--deploy] When true, deploy the Consent Manager after updating the version [default = false] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -### `transcend consent upload-consent-preferences` - -```txt -USAGE - transcend consent upload-consent-preferences (--base64EncryptionKey value) (--base64SigningKey value) (--partition value) [--file value] [--consentUrl value] [--concurrency value] - transcend consent upload-consent-preferences --help - -This command allows for updating of consent preferences to the Managed Consent Database. - -FLAGS - --base64EncryptionKey The encryption key used to encrypt the userId - --base64SigningKey The signing key used to prove authentication of consent request - --partition The partition key to download consent preferences to - [--file] The file to pull consent preferences from [default = ./preferences.csv] - [--consentUrl] URL of the Transcend consent backend. Use https://consent.us.transcend.io for US hosting [default = https://consent.transcend.io] - [--concurrency] The concurrency to use when uploading requests in parallel [default = 100] - -h --help Print help information and exit -``` - -Each row in the CSV must include: - -| Argument | Description | Type | Default | Required | -| --------- | --------------------------------------------------------------------------------------------------------- | ------------------------- | ------- | -------- | -| userId | Unique ID identifying the user that the preferences ar efor | string | N/A | true | -| timestamp | Timestamp for when consent was collected for that user | string - timestamp | N/A | true | -| purposes | JSON map from consent purpose name -> boolean indicating whether user has opted in or out of that purpose | {[k in string]: boolean } | {} | false | -| confirmed | Whether consent preferences have been explicitly confirmed or inferred | boolean | true | false | -| updated | Has the consent been updated (including no-change confirmation) since default resolution | boolean | N/A | false | -| prompted | Whether or not the UI has been shown to the end-user (undefined in older versions of airgap.js) | boolean | N/A | false | -| gpp | IAB GPP string | string - GPP | N/A | false | - -An sample CSV can be found [here](./examples/preference-upload.csv). - -### `transcend consent upload-cookies-from-csv` - -```txt -USAGE - transcend consent upload-cookies-from-csv (--auth value) (--trackerStatus LIVE|NEEDS_REVIEW) [--file value] [--transcendUrl value] - transcend consent upload-cookies-from-csv --help - -Upload cookies from CSV. This command allows for uploading of cookies from CSV. - -Step 1) Download the CSV of cookies that you want to edit from the Admin Dashboard under [Consent Management -> Cookies](https://app.transcend.io/consent-manager/cookies). You can download cookies from both the "Triage" and "Approved" tabs. - -Step 2) You can edit the contents of the CSV file as needed. You may adjust the "Purpose" column, adjust the "Notes" column, add "Owners" and "Teams" or even add custom columns with additional metadata. - -Step 3) Upload the modified CSV file back into the dashboard with this command. - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Data Flows" - --trackerStatus The status of the cookies you will upload. [LIVE|NEEDS_REVIEW] - [--file] Path to the CSV file to upload [default = ./cookies.csv] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -### `transcend consent upload-data-flows-from-csv` - -```txt -USAGE - transcend consent upload-data-flows-from-csv (--auth value) (--trackerStatus LIVE|NEEDS_REVIEW) [--file value] [--classifyService] [--transcendUrl value] - transcend consent upload-data-flows-from-csv --help - -Upload data flows from CSV. This command allows for uploading of data flows from CSV. - -Step 1) Download the CSV of data flows that you want to edit from the Admin Dashboard under [Consent Management -> Data Flows](https://app.transcend.io/consent-manager/data-flows). You can download data flows from both the "Triage" and "Approved" tabs. - -Step 2) You can edit the contents of the CSV file as needed. You may adjust the "Purpose" column, adjust the "Notes" column, add "Owners" and "Teams" or even add custom columns with additional metadata. - -Step 3) Upload the modified CSV file back into the dashboard with this command. - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Data Flows" - --trackerStatus The status of the data flows you will upload. [LIVE|NEEDS_REVIEW] - [--file] Path to the CSV file to upload [default = ./data-flows.csv] - [--classifyService] When true, automatically assign the service for a data flow based on the domain that is specified [default = false] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -### `transcend consent upload-preferences` - -```txt -USAGE - transcend consent upload-preferences (--auth value) (--partition value) [--sombraAuth value] [--consentUrl value] [--file value] [--directory value] [--dryRun] [--skipExistingRecordCheck] [--receiptFileDir value] [--skipWorkflowTriggers] [--forceTriggerWorkflows] [--skipConflictUpdates] [--isSilent] [--attributes value] [--receiptFilepath value] [--concurrency value] - transcend consent upload-preferences --help - -Upload preference management data to your Preference Store. - -This command prompts you to map the shape of the CSV to the shape of the Transcend API. There is no requirement for the shape of the incoming CSV, as the script will handle the mapping process. - -The script will also produce a JSON cache file that allows for the mappings to be preserved between runs. - -FLAGS - --auth The Transcend API key. Requires scopes: "Modify User Stored Preferences", "View Managed Consent Database Admin API", "View Preference Store Settings" - --partition The partition key to download consent preferences to - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - [--consentUrl] URL of the Transcend consent backend. Use https://consent.us.transcend.io for US hosting [default = https://consent.transcend.io] - [--file] Path to the CSV file to load preferences from - [--directory] Path to the directory of CSV files to load preferences from - [--dryRun] Whether to do a dry run only - will write results to receiptFilepath without updating Transcend [default = false] - [--skipExistingRecordCheck] Whether to skip the check for existing records. SHOULD ONLY BE USED FOR INITIAL UPLOAD [default = false] - [--receiptFileDir] Directory path where the response receipts should be saved [default = ./receipts] - [--skipWorkflowTriggers] Whether to skip workflow triggers when uploading to preference store [default = false] - [--forceTriggerWorkflows] Whether to force trigger workflows for existing consent records [default = false] - [--skipConflictUpdates] Whether to skip uploading of any records where the preference store and file have a hard conflict [default = false] - [--isSilent/--noIsSilent] Whether to skip sending emails in workflows [default = true] - [--attributes] Attributes to add to any DSR request if created. Comma-separated list of key:value pairs. [default = Tags:transcend-cli,Source:transcend-cli] - [--receiptFilepath] Store resulting, continuing where left off [default = ./preference-management-upload-receipts.json] - [--concurrency] The concurrency to use when uploading in parallel [default = 10] - -h --help Print help information and exit -``` - -A sample CSV can be found [here](./examples/cli-upload-preferences-example.csv). In this example, `Sales` and `Marketing` are two custom Purposes, and `SalesCommunications` and `MarketingCommunications` are Preference Topics. During the interactive CLI prompt, you can map these columns to the slugs stored in Transcend! - -Upload consent preferences to partition key `4d1c5daa-90b7-4d18-aa40-f86a43d2c726`: - -```sh -transcend consent upload-preferences --auth=$TRANSCEND_API_KEY --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 -``` - -Upload consent preferences with additional options: - -```sh -transcend consent upload-preferences --auth=$TRANSCEND_API_KEY --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 --file=./preferences.csv --dryRun=true --skipWorkflowTriggers=true --skipConflictUpdates=true --isSilent=false --attributes="Tags:transcend-cli,Source:transcend-cli" --receiptFilepath=./preference-management-upload-receipts.json -``` - -Specifying the backend URL, needed for US hosted backend infrastructure: - -```sh -transcend consent upload-preferences --auth=$TRANSCEND_API_KEY --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 --consentUrl=https://consent.us.transcend.io -``` - -### `transcend inventory pull` - -```txt -USAGE - transcend inventory pull (--auth value) [--resources all|apiKeys|customFields|templates|dataSilos|enrichers|dataFlows|businessEntities|actions|dataSubjects|identifiers|cookies|consentManager|partitions|prompts|promptPartials|promptGroups|agents|agentFunctions|agentFiles|vendors|dataCategories|processingPurposes|actionItems|actionItemCollections|teams|privacyCenters|policies|messages|assessments|assessmentTemplates|purposes] [--file value] [--transcendUrl value] [--dataSiloIds value]... [--integrationNames value]... [--trackerStatuses LIVE|NEEDS_REVIEW] [--pageSize value] [--skipDatapoints] [--skipSubDatapoints] [--includeGuessedCategories] [--debug] - transcend inventory pull --help - -Generates a transcend.yml by pulling the configuration from your Transcend instance. - -The API key needs various scopes depending on the resources being pulled (see the CLI's README for more details). - -This command can be helpful if you are looking to: - -- Copy your data into another instance -- Generate a transcend.yml file as a starting point to maintain parts of your data inventory in code. - -FLAGS - --auth The Transcend API key. The scopes required will vary depending on the operation performed. If in doubt, the Full Admin scope will always work. - [--resources] The different resource types to pull in. Defaults to dataSilos,enrichers,templates,apiKeys. [all|apiKeys|customFields|templates|dataSilos|enrichers|dataFlows|businessEntities|actions|dataSubjects|identifiers|cookies|consentManager|partitions|prompts|promptPartials|promptGroups|agents|agentFunctions|agentFiles|vendors|dataCategories|processingPurposes|actionItems|actionItemCollections|teams|privacyCenters|policies|messages|assessments|assessmentTemplates|purposes, separator = ,] - [--file] Path to the YAML file to pull into [default = ./transcend.yml] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--dataSiloIds]... The UUIDs of the data silos that should be pulled into the YAML file [separator = ,] - [--integrationNames]... The types of integrations to pull down [separator = ,] - [--trackerStatuses] The statuses of consent manager trackers to pull down. Defaults to all statuses. [LIVE|NEEDS_REVIEW, separator = ,] - [--pageSize] The page size to use when paginating over the API [default = 50] - [--skipDatapoints] When true, skip pulling in datapoints alongside data silo resource [default = false] - [--skipSubDatapoints] When true, skip pulling in subDatapoints alongside data silo resource [default = false] - [--includeGuessedCategories] When true, included guessed data categories that came from the content classifier [default = false] - [--debug] Set to true to include debug logs while pulling the configuration [default = false] - -h --help Print help information and exit -``` - -#### Scopes - -The API key permissions for this command vary based on the `resources` argument: - -| Resource | Description | Scopes | Link | -| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| apiKeys | API Key definitions assigned to Data Silos. API keys cannot be created through the CLI, but you can map API key usage to Data Silos. | View API Keys | [Developer Tools -> API keys](https://app.transcend.io/infrastructure/api-keys) | -| customFields | Custom field definitions that define extra metadata for each table in the Admin Dashboard. | View Global Attributes | [Custom Fields](https://app.transcend.io/infrastructure/attributes) | -| templates | Email templates. Only template titles can be created and mapped to other resources. | View Email Templates | [DSR Automation -> Email Templates](https://app.transcend.io/privacy-requests/email-templates) | -| dataSilos | The Data Silo/Integration definitions. | View Data Map, View Data Subject Request Settings | [Data Inventory -> Data Silos](https://app.transcend.io/data-map/data-inventory/) and [Infrastucture -> Integrations](https://app.transcend.io/infrastructure/integrationsdata-silos) | -| enrichers | The Privacy Request enricher configurations. | View Identity Verification Settings | [DSR Automation -> Identifiers](https://app.transcend.io/privacy-requests/identifiers) | -| dataFlows | Consent Manager Data Flow definitions. | View Data Flows | [Consent Management -> Data Flows](https://app.transcend.io/consent-manager/data-flows/approved) | -| businessEntities | The business entities in the data inventory. | View Data Inventory | [Data Inventory -> Business Entities](https://app.transcend.io/data-map/data-inventory/business-entities) | -| actions | The Privacy Request action settings. | View Data Subject Request Settings | [DSR Automation -> Request Settings](https://app.transcend.io/privacy-requests/settings) | -| dataSubjects | The Privacy Request data subject settings. | View Data Subject Request Settings | [DSR Automation -> Request Settings](https://app.transcend.io/privacy-requests/settings) | -| identifiers | The Privacy Request identifier configurations. | View Identity Verification Settings | [DSR Automation -> Identifiers](https://app.transcend.io/privacy-requests/identifiers) | -| cookies | Consent Manager Cookie definitions. | View Data Flows | [Consent Management -> Cookies](https://app.transcend.io/consent-manager/cookies/approved) | -| consentManager | Consent Manager general settings, including domain list. | View Consent Manager | [Consent Management -> Developer Settings](https://app.transcend.io/consent-manager/developer-settings) | -| partitions | The partitions in the account (often representative of separate data controllers). | View Consent Manager | [Consent Management -> Developer Settings -> Advanced Settings](https://app.transcend.io/consent-manager/developer-settings/advanced-settings) | -| prompts | The Transcend AI prompts | View Prompts | [Prompt Manager -> Browse](https://app.transcend.io/prompts/browse) | -| promptPartials | The Transcend AI prompt partials | View Prompts | [Prompt Manager -> Partials](https://app.transcend.io/prompts/partialss) | -| promptGroups | The Transcend AI prompt groups | View Prompts | [Prompt Manager -> Groups](https://app.transcend.io/prompts/groups) | -| agents | The agents in the prompt manager. | View Prompts | [Prompt Manager -> Agents](https://app.transcend.io/prompts/agents) | -| agentFunctions | The agent functions in the prompt manager. | View Prompts | [Prompt Manager -> Agent Functions](https://app.transcend.io/prompts/agent-functions) | -| agentFiles | The agent files in the prompt manager. | View Prompts | [Prompt Manager -> Agent Files](https://app.transcend.io/prompts/agent-files) | -| vendors | The vendors in the data inventory. | View Data Inventory | [Data Inventory -> Vendors](https://app.transcend.io/data-map/data-inventory/vendors) | -| dataCategories | The data categories in the data inventory. | View Data Inventory | [Data Inventory -> Data Categories](https://app.transcend.io/data-map/data-inventory/data-categories) | -| processingPurposes | The processing purposes in the data inventory. | View Data Inventory | [Data Inventory -> Processing Purposes](https://app.transcend.io/data-map/data-inventory/purposes) | -| actionItems | Onboarding related action items | View All Action Items | [Action Items](https://app.transcend.io/action-items/all) | -| actionItemCollections | Onboarding related action item group names | View All Action Items | [Action Items](https://app.transcend.io/action-items/all) | -| teams | Team definitions of users and scope groupings | View Scopes | [Administration -> Teams](https://app.transcend.io/admin/teams) | -| privacyCenters | The privacy center configurations. | View Privacy Center Layout | [Privacy Center](https://app.transcend.io/privacy-center/general-settings) | -| policies | The privacy center policies. | View Policies | [Privacy Center -> Policies](https://app.transcend.io/privacy-center/policies) | -| messages | Message definitions used across consent, privacy center, email templates and more. | View Internationalization Messages | [Privacy Center -> Messages](https://app.transcend.io/privacy-center/messages-internationalization), [Consent Management -> Display Settings -> Messages](https://app.transcend.io/consent-manager/display-settings/messages) | -| assessments | Assessment responses. | View Assessments | [Assessments -> Assessments](https://app.transcend.io/assessments/groups) | -| assessmentTemplates | Assessment template configurations. | View Assessments | [Assessment -> Templates](https://app.transcend.io/assessments/form-templates) | -| purposes | Consent purposes and related preference management topics. | View Consent Manager, View Preference Store Settings | [Consent Management -> Regional Experiences -> Purposes](https://app.transcend.io/consent-manager/regional-experiences/purposes) | - -#### Usage - -```sh -# Writes out file to ./transcend.yml -transcend inventory pull --auth=$TRANSCEND_API_KEY -``` - -An alternative file destination can be specified: - -```sh -# Writes out file to ./custom/location.yml -transcend inventory pull --auth=$TRANSCEND_API_KEY --file=./custom/location.yml -``` - -Or a specific data silo(s) can be pulled in: - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY ---dataSiloIds=710fec3c-7bcc-4c9e-baff-bf39f9bec43e -``` - -Or a specific types of data silo(s) can be pulled in: - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY --integrationNames=salesforce,snowflake -``` - -Specifying the resource types to pull in (the following resources are the default resources): - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY --resources=apiKeys,templates,dataSilos,enrichers -``` - -Pull in data flow and cookie resources, filtering for specific tracker statuses (see [this example](./examples/data-flows-cookies.yml)): - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY --resources=dataFlows,cookies --trackerStatuses=NEEDS_REVIEW,LIVE -``` - -Pull in data silos without any datapoint/table information - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY --resources=dataSilos --skipDatapoints=true -``` - -Pull in data silos and datapoints without any subdatapoint/column information - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY --resources=dataSilos --skipSubDatapoints=true -``` - -Pull in data silos and datapoints with guessed data categories instead of just approved data categories - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY --resources=dataSilos --includeGuessedCategories=true -``` - -Pull in attribute definitions only (see [this example](./examples/attributes.yml)): - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY --resources=attributes -``` - -Pull in business entities only (see [this example](./examples/business-entities.yml)): - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY --resources=businessEntities -``` - -Pull in enrichers and identifiers (see [this example](./examples/enrichers.yml)): - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY --resources=enrichers,identifiers -``` - -Pull in onboarding action items (see [this example](./examples/action-items.yml)): - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY --resources=actionItems,actionItemCollections -``` - -Pull in consent manager domain list (see [this example](./examples/consent-manager-domains.yml)): - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY --resources=consentManager -``` - -Pull in identifier configurations (see [this example](./examples/identifiers.yml)): - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY --resources=identifiers -``` - -Pull in request actions configurations (see [this example](./examples/actions.yml)): - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY --resources=actions -``` - -Pull in consent manager purposes and preference management topics (see [this example](./examples/purposes.yml)): - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY --resources=purposes -``` - -Pull in data subject configurations (see [this example](./examples/data-subjects.yml)): - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY --resources=dataSubjects -``` - -Pull in assessments and assessment templates. - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY --resources=assessments,assessmentTemplates -``` - -Pull everything: - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY --resources=all -``` - -Pull in configuration files across multiple instances - -```sh -transcend admin generate-api-keys --email=test@transcend.io --password=$TRANSCEND_PASSWORD --scopes="View Consent Manager" --apiKeyTitle="CLI Usage Cross Instance Sync" --file=./transcend-api-keys.json -transcend inventory pull --auth=./transcend-api-keys.json --resources=consentManager --file=./transcend/ -``` - -Note: This command will overwrite the existing transcend.yml file that you have locally. - -### `transcend inventory push` - -```txt -USAGE - transcend inventory push (--auth value) [--file value] [--transcendUrl value] [--pageSize value] [--variables value] [--publishToPrivacyCenter] [--classifyService] [--deleteExtraAttributeValues] - transcend inventory push --help - -Given a transcend.yml file, sync the contents up to your Transcend instance. - -FLAGS - --auth The Transcend API key. The scopes required will vary depending on the operation performed. If in doubt, the Full Admin scope will always work. - [--file] Path to the YAML file to push from [default = ./transcend.yml] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--pageSize] The page size to use when paginating over the API [default = 50] - [--variables] The variables to template into the YAML file when pushing configuration. Comma-separated list of key:value pairs. [default = ""] - [--publishToPrivacyCenter] When true, publish the configuration to the Privacy Center [default = false] - [--classifyService] When true, automatically assign the service for a data flow based on the domain that is specified [default = false] - [--deleteExtraAttributeValues] When true and syncing attributes, delete any extra attributes instead of just upserting [default = false] - -h --help Print help information and exit -``` - -#### Scopes - -The scopes for `transcend inventory push` are the same as the scopes for [`transcend inventory pull`](#transcend-inventory-pull). - -#### Usage - -```sh -# Looks for file at ./transcend.yml -transcend inventory push --auth=$TRANSCEND_API_KEY -``` - -An alternative file destination can be specified: - -```sh -# Looks for file at custom location ./custom/location.yml -transcend inventory push --auth=$TRANSCEND_API_KEY --file=./custom/location.yml -``` - -Push a single .yml file configuration into multiple Transcend instances. This uses the output of [`transcend admin generate-api-keys`](#transcend-admin-generate-api-keys). - -```sh -transcend admin generate-api-keys --email=test@transcend.io --password=$TRANSCEND_PASSWORD \ - --scopes="View Email Templates,View Data Map" --apiKeyTitle="CLI Usage Cross Instance Sync" --file=./transcend-api-keys.json -transcend inventory pull --auth=$TRANSCEND_API_KEY -transcend inventory push --auth=./transcend-api-keys.json -``` - -Push multiple .yml file configurations into multiple Transcend instances. This uses the output of [`transcend admin generate-api-keys`](#transcend-admin-generate-api-keys). - -```sh -transcend admin generate-api-keys --email=test@transcend.io --password=$TRANSCEND_PASSWORD \ - --scopes="View Email Templates,View Data Map" --apiKeyTitle="CLI Usage Cross Instance Sync" --file=./transcend-api-keys.json -transcend inventory pull --auth=./transcend-api-keys.json --file=./transcend/ -# -transcend inventory push --auth=./transcend-api-keys.json --file=./transcend/ -``` - -Apply service classifier to all data flows. - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY --resources=dataFlows -transcend inventory push --auth=$TRANSCEND_API_KEY --resources=dataFlows --classifyService=true -``` - -Push up attributes, deleting any attributes that are not specified in the transcend.yml file - -```sh -transcend inventory pull --auth=$TRANSCEND_API_KEY --resources=attributes -transcend inventory push --auth=$TRANSCEND_API_KEY --resources=attributes --deleteExtraAttributeValues=true -``` - -Some things to note about this sync process: - -1. Any field that is defined in your .yml file will be synced up to app.transcend.io. If any change was made on the Admin Dashboard, it will be overwritten. -2. If you omit a field from the .yml file, this field will not be synced. This gives you the ability to define as much or as little configuration in your transcend.yml file as you would like, and let the remainder of fields be labeled through the Admin Dashboard -3. If you define new data subjects, identifiers, data silos or datapoints that were not previously defined on the Admin Dashboard, the CLI will create these new resources automatically. -4. Currently, this CLI does not handle deleting or renaming of resources. If you need to delete or rename a data silo, identifier, enricher or API key, you should make the change on the Admin Dashboard. -5. The only resources that this CLI will not auto-generate are: - -- a) Data silo owners: If you assign an email address to a data silo, you must first make sure that user is invited into your Transcend instance (https://app.transcend.io/admin/users). -- b) API keys: This CLI will not create new API keys. You will need to first create the new API keys on the Admin Dashboard (https://app.transcend.io/infrastructure/api-keys). You can then list out the titles of the API keys that you generated in your transcend.yml file, after which the CLI is capable of updating that API key to be able to respond to different data silos in your Data Map - -#### CI Integration - -Once you have a workflow for creating your transcend.yml file, you will want to integrate your `transcend inventory push` command on your CI. - -Below is an example of how to set this up using a Github action: - -```yaml -name: Transcend Data Map Syncing -# See https://app.transcend.io/privacy-requests/connected-services - -on: - push: - branches: - - 'main' - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Setup Node.js - uses: actions/setup-node@v2 - with: - node-version: '16' - - - name: Install Transcend CLI - run: npm install --global @transcend-io/cli - - # If you have a script that generates your transcend.yml file from - # an ORM or infrastructure configuration, add that step here - # Leave this step commented out if you want to manage your transcend.yml manually - # - name: Generate transcend.yml - # run: ./scripts/generate_transcend_yml.py - - - name: Push Transcend config - run: transcend inventory push --auth=${{ secrets.TRANSCEND_API_KEY }} -``` - -#### Dynamic Variables - -If you are using this CLI to sync your Data Map between multiple Transcend instances, you may find the need to make minor modifications to your configurations between environments. The most notable difference would be the domain where your webhook URLs are hosted on. - -The `transcend inventory push` command takes in a parameter `variables`. This is a CSV of `key:value` pairs. - -```sh -transcend inventory push --auth=$TRANSCEND_API_KEY --variables=domain:acme.com,stage:staging -``` - -This command could fill out multiple parameters in a YAML file like [./examples/multi-instance.yml](./examples/multi-instance.yml), copied below: - -```yml -api-keys: - - title: Webhook Key -enrichers: - - title: Basic Identity Enrichment - description: Enrich an email address to the userId and phone number - # The data silo webhook URL is the same in each environment, - # except for the base domain in the webhook URL. - url: https://example.<>/transcend-enrichment-webhook - input-identifier: email - output-identifiers: - - userId - - phone - - myUniqueIdentifier - - title: Fraud Check - description: Ensure the email address is not marked as fraudulent - url: https://example.<>/transcend-fraud-check - input-identifier: email - output-identifiers: - - email - privacy-actions: - - ERASURE -data-silos: - - title: Redshift Data Warehouse - integrationName: server - description: The mega-warehouse that contains a copy over all SQL backed databases - <> - url: https://example.<>/transcend-webhook - api-key-title: Webhook Key -``` - -### `transcend inventory scan-packages` - -```txt -USAGE - transcend inventory scan-packages (--auth value) [--scanPath value] [--ignoreDirs value]... [--repositoryName value] [--transcendUrl value] - transcend inventory scan-packages --help - -Transcend scans packages and dependencies for the following frameworks: - -- package.json -- requirements.txt & setup.py -- Podfile -- Package.resolved -- build.gradle -- pubspec.yaml -- Gemfile & .gemspec -- composer.json - -This command will scan the folder you point at to look for any of these files. Once found, the build file will be parsed in search of dependencies. Those code packages and dependencies will be uploaded to Transcend. The information uploaded to Transcend is: - -- repository name -- package names -- dependency names and versions -- package descriptions - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Code Scanning" - [--scanPath] File path in the project to scan [default = ./] - [--ignoreDirs]... List of directories to ignore in scan [separator = ,] - [--repositoryName] Name of the git repository that the package should be tied to - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -### `transcend inventory discover-silos` - -```txt -USAGE - transcend inventory discover-silos (--scanPath value) (--dataSiloId value) (--auth value) [--fileGlobs value] [--ignoreDirs value] [--transcendUrl value] - transcend inventory discover-silos --help - -We support scanning for new data silos in JavaScript, Python, Gradle, and CocoaPods projects. - -To get started, add a data silo for the corresponding project type with the "silo discovery" plugin enabled. For example, if you want to scan a JavaScript project, add a package.json data silo. Then, specify the data silo ID in the "--dataSiloId" parameter. - -FLAGS - --scanPath File path in the project to scan - --dataSiloId The UUID of the corresponding data silo - --auth The Transcend API key. This key must be associated with the data silo(s) being operated on. Requires scopes: "Manage Assigned Data Inventory" - [--fileGlobs] You can pass a glob syntax pattern(s) to specify additional file paths to scan. Comma-separated list of globs. [default = ""] - [--ignoreDirs] Comma-separated list of directories to ignore. [default = ""] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -#### Usage - -Scan a JavaScript package.json: - -```sh -transcend inventory discover-silos --scanPath=./myJavascriptProject --auth=$TRANSCEND_API_KEY ---dataSiloId=445ee241-5f2a-477b-9948-2a3682a43d0e -``` - -Here are some examples of a [Podfile](./examples/code-scanning/test-cocoa-pods/Podfile) and [Gradle file](./examples/code-scanning/test-gradle/build.gradle). These are scanned like: - -```sh -transcend inventory discover-silos --scanPath=./examples/ --auth=$TRANSCEND_API_KEY ---dataSiloId=b6776589-0b7d-466f-8aad-4378ffd3a321 -``` - -This call will look for all the package.json files that in the scan path `./myJavascriptProject`, parse each of the dependencies into their individual package names, and send it to our Transcend backend for classification. These classifications can then be viewed [here](https://app.transcend.io/data-map/data-inventory/silo-discovery/triage). The process is the same for scanning requirements.txt, podfiles and build.gradle files. - -### `transcend inventory pull-datapoints` - -```txt -USAGE - transcend inventory pull-datapoints (--auth value) [--file value] [--transcendUrl value] [--dataSiloIds value]... [--includeAttributes] [--includeGuessedCategories] [--parentCategories FINANCIAL|HEALTH|CONTACT|LOCATION|DEMOGRAPHIC|ID|ONLINE_ACTIVITY|USER_PROFILE|SOCIAL_MEDIA|CONNECTION|TRACKING|DEVICE|SURVEY|OTHER|UNSPECIFIED|NOT_PERSONAL_DATA|INTEGRATION_IDENTIFIER] [--subCategories value]... - transcend inventory pull-datapoints --help - -Export the datapoints from your Data Inventory into a CSV. - -FLAGS - --auth The Transcend API key. Requires scopes: "View Data Inventory" - [--file] The file to save datapoints to [default = ./datapoints.csv] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--dataSiloIds]... List of data silo IDs to filter by [separator = ,] - [--includeAttributes] Whether to include attributes in the output [default = false] - [--includeGuessedCategories] Whether to include guessed categories in the output [default = false] - [--parentCategories] List of parent categories to filter by [FINANCIAL|HEALTH|CONTACT|LOCATION|DEMOGRAPHIC|ID|ONLINE_ACTIVITY|USER_PROFILE|SOCIAL_MEDIA|CONNECTION|TRACKING|DEVICE|SURVEY|OTHER|UNSPECIFIED|NOT_PERSONAL_DATA|INTEGRATION_IDENTIFIER, separator = ,] - [--subCategories]... List of subcategories to filter by [separator = ,] - -h --help Print help information and exit -``` - -### `transcend inventory pull-unstructured-discovery-files` - -```txt -USAGE - transcend inventory pull-unstructured-discovery-files (--auth value) [--file value] [--transcendUrl value] [--dataSiloIds value]... [--subCategories value]... [--status MANUALLY_ADDED|CORRECTED|VALIDATED|CLASSIFIED|REJECTED] [--includeEncryptedSnippets] - transcend inventory pull-unstructured-discovery-files --help - -This command allows for pulling Unstructured Discovery into a CSV. - -FLAGS - --auth The Transcend API key. Requires scopes: "View Data Inventory" - [--file] The file to save datapoints to [default = ./unstructured-discovery-files.csv] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--dataSiloIds]... List of data silo IDs to filter by [separator = ,] - [--subCategories]... List of data categories to filter by [separator = ,] - [--status] List of classification statuses to filter by [MANUALLY_ADDED|CORRECTED|VALIDATED|CLASSIFIED|REJECTED, separator = ,] - [--includeEncryptedSnippets] Whether to include encrypted snippets of the entries classified [default = false] - -h --help Print help information and exit -``` - -### `transcend inventory derive-data-silos-from-data-flows` - -```txt -USAGE - transcend inventory derive-data-silos-from-data-flows (--auth value) (--dataFlowsYmlFolder value) (--dataSilosYmlFolder value) [--ignoreYmls value]... [--transcendUrl value] - transcend inventory derive-data-silos-from-data-flows --help - -Given a folder of data flow transcend.yml configurations, convert those configurations to set of data silo transcend.yml configurations. - -FLAGS - --auth The Transcend API key. No scopes are required for this command. - --dataFlowsYmlFolder The folder that contains data flow yml files - --dataSilosYmlFolder The folder that contains data silo yml files - [--ignoreYmls]... The set of yml files that should be skipped when uploading [separator = ,] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -### `transcend inventory derive-data-silos-from-data-flows-cross-instance` - -```txt -USAGE - transcend inventory derive-data-silos-from-data-flows-cross-instance (--auth value) (--dataFlowsYmlFolder value) [--output value] [--ignoreYmls value]... [--transcendUrl value] - transcend inventory derive-data-silos-from-data-flows-cross-instance --help - -Given a folder of data flow transcend.yml configurations, convert those configurations to a single transcend.yml configurations of all related data silos. - -FLAGS - --auth The Transcend API key. No scopes are required for this command. - --dataFlowsYmlFolder The folder that contains data flow yml files - [--output] The output transcend.yml file containing the data silo configurations [default = ./transcend.yml] - [--ignoreYmls]... The set of yml files that should be skipped when uploading [separator = ,] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -### `transcend inventory consent-manager-service-json-to-yml` - -```txt -USAGE - transcend inventory consent-manager-service-json-to-yml [--file value] [--output value] - transcend inventory consent-manager-service-json-to-yml --help - -Import the services from an airgap.js file into a Transcend instance. - -Step 1) Run `await airgap.getMetadata()` on a site with airgap -Step 2) Right click on the printed object, and click `Copy object` -Step 3) Place output of file in a file named `services.json` -Step 4) Run `transcend consent consent-manager-service-json-to-yml --file=./services.json --output=./transcend.yml` -Step 5) Run `transcend inventory push --auth=$TRANSCEND_API_KEY --file=./transcend.yml --classifyService=true` - -FLAGS - [--file] Path to the services.json file, output of await airgap.getMetadata() [default = ./services.json] - [--output] Path to the output transcend.yml to write to [default = ./transcend.yml] - -h --help Print help information and exit -``` - -### `transcend inventory consent-managers-to-business-entities` - -```txt -USAGE - transcend inventory consent-managers-to-business-entities (--consentManagerYmlFolder value) [--output value] - transcend inventory consent-managers-to-business-entities --help - -This command allows for converting a folder or Consent Manager transcend.yml files into a single transcend.yml file where each consent manager configuration is a Business Entity in the data inventory. - -FLAGS - --consentManagerYmlFolder Path to the folder of Consent Manager transcend.yml files to combine - [--output] Path to the output transcend.yml with business entity configuration [default = ./combined-business-entities.yml] - -h --help Print help information and exit -``` - -### `transcend admin generate-api-keys` - -```txt -USAGE - transcend admin generate-api-keys (--email value) (--password value) (--apiKeyTitle value) (--file value) (--scopes value)... [--deleteExistingApiKey] [--createNewApiKey] [--parentOrganizationId value] [--transcendUrl value] - transcend admin generate-api-keys --help - -This command allows for creating API keys across multiple Transcend instances. This is useful for customers that are managing many Transcend instances and need to regularly create, cycle or delete API keys across all of their instances. - -Unlike the other commands that rely on API key authentication, this command relies upon username/password authentication. This command will spit out the API keys into a JSON file, and that JSON file can be used in subsequent CLI commands. - -Authentication requires your email and password for the Transcend account. This command will only generate API keys for Transcend instances where you have the permission to "Manage API Keys". - -FLAGS - --email The email address that you use to log into Transcend - --password The password for your account login - --apiKeyTitle The title of the API key being generated or destroyed - --file The file where API keys should be written to - --scopes... The list of scopes that should be given to the API key [separator = ,] - [--deleteExistingApiKey/--noDeleteExistingApiKey] When true, if an API key exists with the specified apiKeyTitle, the existing API key is deleted [default = true] - [--createNewApiKey/--noCreateNewApiKey] When true, new API keys will be created. Set to false if you simply want to delete all API keys with a title [default = true] - [--parentOrganizationId] Filter for only a specific organization by ID, returning all child accounts associated with that organization - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -#### Usage - -```sh -transcend admin generate-api-keys --email=test@transcend.io --password=$TRANSCEND_PASSWORD \ - --scopes="View Email Templates,View Data Map" --apiKeyTitle="CLI Usage Cross Instance Sync" -file=./working/auth.json -``` - -Specifying the backend URL, needed for US hosted backend infrastructure. - -```sh -transcend admin generate-api-keys --email=test@transcend.io --password=$TRANSCEND_PASSWORD \ - --scopes="View Email Templates,View Data Map" --apiKeyTitle="CLI Usage Cross Instance Sync" -file=./working/auth.json \ - --transcendUrl=https://api.us.transcend.io -``` - -Filter for only a specific organization by ID, returning all child accounts associated with that organization. Can use the following GQL query on the [EU GraphQL Playground](https://api.us.transcend.io/graphql) or [US GraphQL Playground](https://api.us.transcend.io/graphql). - -```gql -query { - user { - organization { - id - parentOrganizationId - } - } -} -``` - -```sh -transcend admin generate-api-keys --email=test@transcend.io --password=$TRANSCEND_PASSWORD \ - --scopes="View Email Templates,View Data Map" --apiKeyTitle="CLI Usage Cross Instance Sync" -file=./working/auth.json \ - --parentOrganizationId=7098bb38-070d-4f26-8fa4-1b61b9cdef77 -``` - -Delete all API keys with a certain title. - -```sh -transcend admin generate-api-keys --email=test@transcend.io --password=$TRANSCEND_PASSWORD \ - --scopes="View Email Templates,View Data Map" --apiKeyTitle="CLI Usage Cross Instance Sync" -file=./working/auth.json \ - --createNewApiKey=false -``` - -Throw error if an API key already exists with that title, default behavior is to delete the existing API key and create a new one with that same title. - -```sh -transcend admin generate-api-keys --email=test@transcend.io --password=$TRANSCEND_PASSWORD \ - --scopes="View Email Templates,View Data Map" --apiKeyTitle="CLI Usage Cross Instance Sync" -file=./working/auth.json \ - --deleteExistingApiKey=false -``` - -### `transcend migration sync-ot` - -```txt -USAGE - transcend migration sync-ot [--hostname value] [--oneTrustAuth value] [--source oneTrust|file] [--transcendAuth value] [--transcendUrl value] [--file value] [--resource assessments] [--dryRun] [--debug] - transcend migration sync-ot --help - -Pulls resources from a OneTrust and syncs them to a Transcend instance. For now, it only supports retrieving OneTrust Assessments. - -This command can be helpful if you are looking to: -- Pull resources from your OneTrust account. -- Migrate your resources from your OneTrust account to Transcend. - -OneTrust authentication requires an OAuth Token with scope for accessing the assessment endpoints. -If syncing the resources to Transcend, you will also need to generate an API key on the Transcend Admin Dashboard. - -FLAGS - [--hostname] The domain of the OneTrust environment from which to pull the resource - [--oneTrustAuth] The OAuth access token with the scopes necessary to access the OneTrust Public APIs - [--source] Whether to read the assessments from OneTrust or from a file [oneTrust|file, default = oneTrust] - [--transcendAuth] The Transcend API key. Requires scopes: "Manage Assessments" - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--file] Path to the file to pull the resource into. Must be a json file! - [--resource] The resource to pull from OneTrust. For now, only assessments is supported [assessments, default = assessments] - [--dryRun] Whether to export the resource to a file rather than sync to Transcend [default = false] - [--debug] Whether to print detailed logs in case of error [default = false] - -h --help Print help information and exit -``` - - - -## Prompt Manager - -If you are integrating Transcend's Prompt Manager into your code, it may look like: - -```ts -import { TranscendPromptManager } from '@transcend-io/cli'; -import { - ChatCompletionMessage, - PromptRunProductArea, -} from '@transcend-io/privacy-types'; -import * as t from 'io-ts'; - -/** - * Example prompt integration - */ -export async function main(): Promise { - // Instantiate the Transcend Prompt Manager instance - const promptManager = new TranscendPromptManager({ - // API key - transcendApiKey: process.env.TRANSCEND_API_KEY, - // Define the prompts that are stored in Transcend - prompts: { - test: { - // identify by ID - id: '30bcaa79-889a-4af3-842d-2e8ba443d36d', - // no runtime variables - paramCodec: t.type({}), - // response is list of strings - outputCodec: t.array(t.string), - }, - json: { - // identify by title - title: 'test', - // one runtime variable "test" - paramCodec: t.type({ test: t.string }), - // runtime is json object - outputCodec: t.record(t.string, t.string), - // response is stored in atg - extractFromTag: 'json', - }, - predictProductLine: { - // identify by title - title: 'Predict Product Line', - // runtime parameter for slack channel name - paramCodec: t.type({ - slackChannelName: t.string, - }), - // response is specific JSON shape - outputCodec: t.type({ - product: t.union([t.string, t.null]), - clarification: t.union([t.string, t.null]), - }), - // response is stored in atg - extractFromTag: 'json', - }, - }, - // Optional arguments - // transcendUrl: 'https://api.us.transcend.io', // defaults to 'https://api.transcend.io' - // requireApproval: false, // defaults to true - // cacheDuration: 1000 * 60 * 60, // defaults to undefined, no cache - // defaultVariables: { myVariable: 'this is custom', other: [{ name: 'custom' }] }, // defaults to {} - // handlebarsOptions: { helpers, templates }, // defaults to {} - }); - - // Fetch the prompt from Transcend and template any variables - // in this case, we template the slack channel name in the LLM prompt - const systemPrompt = await promptManager.compilePrompt( - 'predictProductLine', - { - slackChannelName: channelName, - }, - ); - - // Parameters to pass to the LLM - const input: ChatCompletionMessage[] = [ - { - role: 'system', - content: systemPrompt, - }, - { - role: 'user', - content: input, - }, - ]; - const largeLanguageModel = { - name: 'gpt-4', - client: 'openai' as const, - }; - const temperature = 1; - const topP = 1; - const maxTokensToSample = 1000; - - // Run prompt against LLM - let response: string; - const t0 = new Date().getTime(); - try { - response = await openai.createCompletion(input, { - temperature, - top_p: topP, - max_tokens: maxTokensToSample, - }); - } catch (err) { - // report error upon failure - await promptManager.reportPromptRunError('predictProductLine', { - promptRunMessages: input, - duration: new Date().getTime() - t0, - temperature, - topP, - error: err.message, - maxTokensToSample, - largeLanguageModel, - }); - } - const t1 = new Date().getTime(); - - // Parsed response as JSON and do not report to Transcend - // const parsedResponse = promptManager.parseAiResponse( - // 'predictProductLine', - // response, - // ); - - // Parsed response as JSON and report output to Transcend - const parsedResponse = await promptManager.reportAndParsePromptRun( - 'predictProductLine', - { - promptRunMessages: [ - ...input, - { - role: 'assistant', - content: response, - }, - ], - duration: t1 - t0, - temperature, - topP, - maxTokensToSample, - largeLanguageModel, - // Optional parameters - // name, // unique identifier for this run - // productArea, // Transcend product area that the prompt relates to - // runByEmployeeEmail, // Employee email that is executing the request - // promptGroupId, // The prompt group being reported - }, - ); -} -``` - -## Proxy usage - -If you are trying to use the CLI inside a corporate firewall and need to send traffic through a proxy, you can do so via the `http_proxy` environment variable,with a command like `http_proxy=http://localhost:5051 transcend inventory pull --auth=$TRANSCEND_API_KEY`. From 5e6716c8de6634eeea2e541d8748358d6a4adb21 Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Wed, 16 Jul 2025 21:03:11 -0400 Subject: [PATCH 03/18] format --- eslint.config.js | 36 +- scripts/buildPathfinderJsonSchema.ts | 18 +- scripts/buildTranscendJsonSchema.ts | 20 +- src/codecs.ts | 244 +++---- src/commands/admin/generate-api-keys/impl.ts | 28 +- .../consent/build-xdi-sync-endpoint/impl.ts | 20 +- .../consent/pull-consent-metrics/impl.ts | 78 +-- .../consent/upload-preferences/impl.ts | 50 +- .../impl.ts | 30 +- .../impl.ts | 30 +- .../impl.ts | 74 +-- .../derive-data-silos-from-data-flows/impl.ts | 38 +- .../inventory/pull-datapoints/impl.ts | 52 +- .../pull-unstructured-discovery-files/impl.ts | 50 +- src/commands/inventory/pull/impl.ts | 62 +- src/commands/inventory/push/impl.ts | 86 +-- src/commands/inventory/scan-packages/impl.ts | 36 +- src/commands/migration/sync-ot/impl.ts | 44 +- .../request/cron/pull-identifiers/impl.ts | 30 +- .../request/cron/pull-profiles/impl.ts | 58 +- src/commands/request/export/impl.ts | 20 +- src/constants.ts | 78 +-- src/lib/ai/TranscendPromptManager.ts | 132 ++-- src/lib/ai/filterNullishValuesFromObject.ts | 10 +- src/lib/ai/getGitFilesThatChanged.ts | 28 +- src/lib/ai/removeLinks.ts | 2 +- .../api-keys/generateCrossAccountApiKeys.ts | 68 +- src/lib/api-keys/listDirectories.ts | 6 +- src/lib/api-keys/listFiles.ts | 10 +- src/lib/api-keys/validateTranscendAuth.ts | 18 +- src/lib/bluebird-replace.ts | 6 +- src/lib/code-scanning/constants.ts | 6 +- .../code-scanning/findCodePackagesInFolder.ts | 38 +- src/lib/code-scanning/findFilesToScan.ts | 26 +- .../code-scanning/integrations/cocoaPods.ts | 42 +- .../integrations/composerJson.ts | 26 +- src/lib/code-scanning/integrations/gemfile.ts | 42 +- src/lib/code-scanning/integrations/gradle.ts | 70 +- .../integrations/javascriptPackageJson.ts | 28 +- src/lib/code-scanning/integrations/pubspec.ts | 42 +- .../integrations/pythonRequirementsTxt.ts | 30 +- src/lib/code-scanning/integrations/swift.ts | 20 +- .../consent-manager/buildXdiSyncEndpoint.ts | 36 +- .../consentManagersToBusinessEntities.ts | 26 +- src/lib/consent-manager/createConsentToken.ts | 18 +- .../consent-manager/dataFlowsToDataSilos.ts | 38 +- src/lib/consent-manager/uploadConsents.ts | 72 +-- .../consent-manager/uploadCookiesFromCsv.ts | 46 +- .../consent-manager/uploadDataFlowsFromCsv.ts | 44 +- src/lib/cron/markCronIdentifierCompleted.ts | 12 +- .../cron/markRequestDataSiloIdsCompleted.ts | 28 +- ...ChunkedCustomSiloOutstandingIdentifiers.ts | 40 +- src/lib/cron/pullCronPageOfIdentifiers.ts | 16 +- .../pullCustomSiloOutstandingIdentifiers.ts | 40 +- src/lib/cron/pushCronIdentifiersFromCsv.ts | 50 +- src/lib/cron/writeCsv.ts | 32 +- src/lib/data-inventory/pullAllDatapoints.ts | 82 +-- ...UnstructuredSubDataPointRecommendations.ts | 38 +- src/lib/graphql/fetchAllAssessments.ts | 10 +- src/lib/graphql/fetchAllRequestIdentifiers.ts | 28 +- src/lib/graphql/fetchAllRequests.ts | 38 +- src/lib/graphql/fetchApiKeys.ts | 44 +- src/lib/graphql/fetchCatalogs.ts | 14 +- src/lib/graphql/fetchDataSubjects.ts | 42 +- src/lib/graphql/fetchIdentifiers.ts | 44 +- src/lib/graphql/fetchPrompts.ts | 12 +- src/lib/graphql/fetchRequestDataSilo.ts | 26 +- src/lib/graphql/formatAttributeValues.ts | 6 +- src/lib/graphql/gqls/consentManager.ts | 2 +- src/lib/graphql/loginUser.ts | 16 +- src/lib/graphql/makeGraphQLRequest.ts | 40 +- src/lib/graphql/pullTranscendConfiguration.ts | 438 ++++++------- src/lib/graphql/syncActionItemCollections.ts | 55 +- src/lib/graphql/syncActionItems.ts | 74 +-- src/lib/graphql/syncAgentFiles.ts | 44 +- src/lib/graphql/syncAgentFunctions.ts | 48 +- src/lib/graphql/syncAgents.ts | 42 +- src/lib/graphql/syncAttribute.ts | 32 +- src/lib/graphql/syncBusinessEntities.ts | 50 +- src/lib/graphql/syncCodePackages.ts | 86 +-- .../graphql/syncConfigurationToTranscend.ts | 206 +++--- src/lib/graphql/syncConsentManager.ts | 54 +- src/lib/graphql/syncCookies.ts | 28 +- src/lib/graphql/syncDataCategories.ts | 48 +- src/lib/graphql/syncDataFlows.ts | 50 +- src/lib/graphql/syncDataSilos.ts | 228 +++---- src/lib/graphql/syncDataSubject.ts | 12 +- src/lib/graphql/syncEnrichers.ts | 42 +- src/lib/graphql/syncIdentifier.ts | 12 +- src/lib/graphql/syncIntlMessages.ts | 30 +- src/lib/graphql/syncPartitions.ts | 30 +- src/lib/graphql/syncPolicies.ts | 36 +- src/lib/graphql/syncPrivacyCenter.ts | 22 +- src/lib/graphql/syncProcessingPurposes.ts | 58 +- src/lib/graphql/syncPromptGroups.ts | 62 +- src/lib/graphql/syncPromptPartials.ts | 58 +- src/lib/graphql/syncPrompts.ts | 38 +- src/lib/graphql/syncRepositories.ts | 52 +- .../graphql/syncSoftwareDevelopmentKits.ts | 74 +-- src/lib/graphql/syncTeams.ts | 50 +- src/lib/graphql/syncVendors.ts | 44 +- src/lib/helpers/inquirer.ts | 24 +- src/lib/helpers/parseVariablesFromString.ts | 8 +- .../manual-enrichment/enrichPrivacyRequest.ts | 40 +- .../pullManualEnrichmentIdentifiersToCsv.ts | 46 +- .../pushManualEnrichmentIdentifiersFromCsv.ts | 26 +- src/lib/mergeTranscendInputs.ts | 4 +- .../oneTrust/helpers/convertToEmptyStrings.ts | 8 +- .../helpers/parseCliSyncOtArguments.ts | 74 +-- .../helpers/syncOneTrustAssessmentToDisk.ts | 18 +- .../syncOneTrustAssessmentToTranscend.ts | 26 +- .../syncOneTrustAssessmentsFromFile.ts | 38 +- .../syncOneTrustAssessmentsFromOneTrust.ts | 66 +- .../tests/convertToEmptyStrings.test.ts | 50 +- .../checkIfPendingPreferenceUpdatesAreNoOp.ts | 18 +- ...IfPendingPreferenceUpdatesCauseConflict.ts | 18 +- .../getPreferenceUpdatesFromRow.ts | 44 +- .../getPreferencesForIdentifiers.ts | 42 +- .../parsePreferenceAndPurposeValuesFromCsv.ts | 66 +- .../parsePreferenceIdentifiersFromCsv.ts | 48 +- .../parsePreferenceManagementCsv.ts | 60 +- .../parsePreferenceTimestampsFromCsv.ts | 34 +- ...kIfPendingPreferenceUpdatesAreNoOp.test.ts | 240 +++---- ...dingPreferenceUpdatesCauseConflict.test.ts | 246 +++---- .../tests/getPreferenceUpdatesFromRow.test.ts | 510 +++++++-------- ...ferenceManagementPreferencesInteractive.ts | 96 +-- src/lib/readTranscendYaml.ts | 22 +- src/lib/requests/approvePrivacyRequests.ts | 24 +- src/lib/requests/bulkRestartRequests.ts | 74 +-- src/lib/requests/bulkRetryEnrichers.ts | 36 +- src/lib/requests/cancelPrivacyRequests.ts | 34 +- src/lib/requests/constants.ts | 44 +- .../requests/downloadPrivacyRequestFiles.ts | 34 +- src/lib/requests/filterRows.ts | 26 +- .../getFileMetadataForPrivacyRequests.ts | 42 +- src/lib/requests/getUniqueValuesForColumn.ts | 8 +- src/lib/requests/mapColumnsToAttributes.ts | 26 +- src/lib/requests/mapColumnsToIdentifiers.ts | 28 +- src/lib/requests/mapCsvColumnsToApi.ts | 28 +- src/lib/requests/mapCsvRowsToRequestInputs.ts | 100 +-- src/lib/requests/mapEnumValues.ts | 22 +- src/lib/requests/mapRequestEnumValues.ts | 66 +- src/lib/requests/markSilentPrivacyRequests.ts | 24 +- .../notifyPrivacyRequestsAdditionalTime.ts | 34 +- src/lib/requests/pullPrivacyRequests.ts | 63 +- src/lib/requests/readCsv.ts | 20 +- .../removeUnverifiedRequestIdentifiers.ts | 28 +- src/lib/requests/retryRequestDataSilos.ts | 28 +- src/lib/requests/skipPreflightJobs.ts | 32 +- src/lib/requests/skipRequestDataSilos.ts | 32 +- src/lib/requests/splitCsvToList.ts | 2 +- src/lib/requests/streamPrivacyRequestFiles.ts | 28 +- src/lib/requests/submitPrivacyRequest.ts | 32 +- .../tests/mapCsvRowsToRequestInputs.test.ts | 4 +- src/lib/requests/tests/readCsv.test.ts | 72 +-- .../requests/uploadPrivacyRequestsFromCsv.ts | 124 ++-- .../tests/findCodePackagesInFolder.test.ts | 604 +++++++++--------- src/lib/tests/getGitFilesThatChanged.test.ts | 26 +- src/lib/tests/readTranscendYaml.test.ts | 40 +- src/logger.ts | 10 +- 160 files changed, 4277 insertions(+), 4265 deletions(-) diff --git a/eslint.config.js b/eslint.config.js index f8bff766..af2ce018 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -1,7 +1,7 @@ // @ts-check -import eslint from "@eslint/js"; -import eslintPluginUnicorn from "eslint-plugin-unicorn"; -import tseslint from "typescript-eslint"; +import eslint from '@eslint/js'; +import eslintPluginUnicorn from 'eslint-plugin-unicorn'; +import tseslint from 'typescript-eslint'; /** * @type {import('typescript-eslint').Config} @@ -25,34 +25,34 @@ const eslintConfig = tseslint.config( }, { rules: { - "unicorn/filename-case": [ - "error", + 'unicorn/filename-case': [ + 'error', { - case: "kebabCase", + case: 'kebabCase', }, ], - "unicorn/no-nested-ternary": "off", + 'unicorn/no-nested-ternary': 'off', }, }, { // CJS compatibility for src directory since it's also a library - files: ["src/**/*"], + files: ['src/**/*'], // The CLI does not require CJS compatibility - ignores: ["src/bin/**/*", "src/commands/**/*"], + ignores: ['src/bin/**/*', 'src/commands/**/*'], rules: { // Ban top-level await - "unicorn/prefer-top-level-await": "off", - "no-restricted-syntax": [ - "error", + 'unicorn/prefer-top-level-await': 'off', + 'no-restricted-syntax': [ + 'error', { // Matches await expressions at the top level (direct child of Program) - selector: "Program > ExpressionStatement > AwaitExpression", - message: "Top-level await is not allowed in CJS-compatible files.", + selector: 'Program > ExpressionStatement > AwaitExpression', + message: 'Top-level await is not allowed in CJS-compatible files.', }, ], // Ban default exports - "no-restricted-exports": [ - "error", + 'no-restricted-exports': [ + 'error', { restrictDefaultExports: { direct: true, @@ -66,8 +66,8 @@ const eslintConfig = tseslint.config( }, }, { - ignores: ["dist", "examples"], - } + ignores: ['dist', 'examples'], + }, ); export default eslintConfig; diff --git a/scripts/buildPathfinderJsonSchema.ts b/scripts/buildPathfinderJsonSchema.ts index 5db31b12..ba22da89 100644 --- a/scripts/buildPathfinderJsonSchema.ts +++ b/scripts/buildPathfinderJsonSchema.ts @@ -12,16 +12,16 @@ * @see https://github.com/SchemaStore/schemastore */ -import { writeFileSync } from "node:fs"; -import { join } from "node:path"; -import { toJsonSchema } from "@transcend-io/type-utils"; -import { PathfinderPolicy } from "../src/codecs"; +import { writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { toJsonSchema } from '@transcend-io/type-utils'; +import { PathfinderPolicy } from '../src/codecs'; const schemaDefaults = { - $schema: "http://json-schema.org/draft-07/schema#", - $id: "https://raw.githubusercontent.com/transcend-io/cli/main/pathfinder-policy-yml-schema.json", - title: "pathfinder.yml", - description: "Policies for the Transcend Pathfinder AI governance proxy.", + $schema: 'http://json-schema.org/draft-07/schema#', + $id: 'https://raw.githubusercontent.com/transcend-io/cli/main/pathfinder-policy-yml-schema.json', + title: 'pathfinder.yml', + description: 'Policies for the Transcend Pathfinder AI governance proxy.', }; // Build the JSON schema from io-ts codec @@ -30,6 +30,6 @@ const jsonSchema = { ...toJsonSchema(PathfinderPolicy, true), }; -const schemaFilePath = join(process.cwd(), "pathfinder-policy-yml-schema.json"); +const schemaFilePath = join(process.cwd(), 'pathfinder-policy-yml-schema.json'); writeFileSync(schemaFilePath, `${JSON.stringify(jsonSchema, null, 2)}\n`); diff --git a/scripts/buildTranscendJsonSchema.ts b/scripts/buildTranscendJsonSchema.ts index 3087a3fa..9c00ec6c 100644 --- a/scripts/buildTranscendJsonSchema.ts +++ b/scripts/buildTranscendJsonSchema.ts @@ -12,24 +12,24 @@ * @see https://github.com/SchemaStore/schemastore */ -import { writeFileSync } from "node:fs"; -import { join } from "node:path"; -import { toJsonSchema } from "@transcend-io/type-utils"; -import * as packageJson from "../package.json"; -import { TranscendInput } from "../src/codecs"; +import { writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { toJsonSchema } from '@transcend-io/type-utils'; +import * as packageJson from '../package.json'; +import { TranscendInput } from '../src/codecs'; -const majorVersion = packageJson.version.split(".")[0]; +const majorVersion = packageJson.version.split('.')[0]; // Create a major version JSON schema definition, and update the latest JSON schema definition. -for (const key of [`v${majorVersion}`, "latest"]) { +for (const key of [`v${majorVersion}`, 'latest']) { const fileName = `transcend-yml-schema-${key}.json`; const schemaDefaults = { - $schema: "http://json-schema.org/draft-07/schema#", + $schema: 'http://json-schema.org/draft-07/schema#', $id: `https://raw.githubusercontent.com/transcend-io/cli/main/${fileName}`, - title: "transcend.yml", + title: 'transcend.yml', description: - "Define personal data schema and Transcend config as code with the Transcend CLI.", + 'Define personal data schema and Transcend config as code with the Transcend CLI.', }; // Build the JSON schema from io-ts codec diff --git a/src/codecs.ts b/src/codecs.ts index 7fcf8b4f..c1d8342c 100644 --- a/src/codecs.ts +++ b/src/codecs.ts @@ -5,8 +5,8 @@ import { InitialViewState, OnConsentExpiry, UserPrivacySignalEnum, -} from "@transcend-io/airgap.js-types"; -import { LanguageKey } from "@transcend-io/internationalization"; +} from '@transcend-io/airgap.js-types'; +import { LanguageKey } from '@transcend-io/internationalization'; import { ActionItemCode, ActionItemPriorityOverride, @@ -58,12 +58,12 @@ import { UnknownRequestPolicy, UnstructuredSubDataPointRecommendationStatus, UspapiOption, -} from "@transcend-io/privacy-types"; -import { applyEnum, valuesOf } from "@transcend-io/type-utils"; -import * as t from "io-ts"; -import { OpenAIRouteName, PathfinderPolicyName } from "./enums"; -import { buildAIIntegrationType } from "./lib/helpers/buildAIIntegrationType"; -import { buildEnabledRouteType } from "./lib/helpers/buildEnabledRouteType"; +} from '@transcend-io/privacy-types'; +import { applyEnum, valuesOf } from '@transcend-io/type-utils'; +import * as t from 'io-ts'; +import { OpenAIRouteName, PathfinderPolicyName } from './enums'; +import { buildAIIntegrationType } from './lib/helpers/buildAIIntegrationType'; +import { buildEnabledRouteType } from './lib/helpers/buildEnabledRouteType'; /** * Input to define email templates that can be used to communicate to end-users @@ -126,11 +126,11 @@ export const TeamInput = t.intersection([ }), t.partial({ /** SSO department for automated provisioning */ - "sso-department": t.string, + 'sso-department': t.string, /** SSO group name for automated provisioning */ - "sso-group": t.string, + 'sso-group': t.string, /** SSO title mapping for automated provisioning */ - "sso-title": t.string, + 'sso-title': t.string, /** List of user emails on the team */ users: t.array(t.string), /** List of scopes that the team should have */ @@ -162,7 +162,7 @@ export const EnricherInput = t.intersection([ * The names of the identifiers that can be resolved by this enricher. * i.e. email -> [userId, phone, advertisingId] */ - "output-identifiers": t.array(t.string), + 'output-identifiers': t.array(t.string), }), t.partial({ /** Internal description for why the enricher is needed */ @@ -176,7 +176,7 @@ export const EnricherInput = t.intersection([ * Whenever a privacy request contains this identifier, the webhook will * be called with the value of that identifier as input */ - "input-identifier": t.string, + 'input-identifier': t.string, /** * A regular expression that can be used to match on for cancelation */ @@ -199,16 +199,16 @@ export const EnricherInput = t.intersection([ phoneNumbers: t.array(t.string), /** The list of regions that should trigger the preflight check */ regionList: t.array( - valuesOf({ ...IsoCountryCode, ...IsoCountrySubdivisionCode }) + valuesOf({ ...IsoCountryCode, ...IsoCountrySubdivisionCode }), ), /** * Specify which data subjects the enricher should run for */ - "data-subjects": t.array(t.string), + 'data-subjects': t.array(t.string), /** Headers to include in the webhook */ headers: t.array(WebhookHeader), /** The privacy actions that the enricher should run against */ - "privacy-actions": t.array(valuesOf(RequestAction)), + 'privacy-actions': t.array(valuesOf(RequestAction)), }), ]); @@ -355,7 +355,7 @@ export const AgentInput = t.intersection([ /** Whether the agent has retrieval enabled */ retrievalEnabled: t.boolean, /** Large language model powering the agent */ - "large-language-model": t.type({ + 'large-language-model': t.type({ /** Name of the model */ name: t.string, /** Client of the model */ @@ -383,11 +383,11 @@ export const AgentInput = t.intersection([ /** * The names of the functions that the agent has access to */ - "agent-functions": t.array(t.string), + 'agent-functions': t.array(t.string), /** * The names of the files that the agent has access to for retrieval */ - "agent-files": t.array(t.string), + 'agent-files': t.array(t.string), }), ]); @@ -646,19 +646,19 @@ export const FieldInput = t.intersection([ * * @see https://github.com/transcend-io/privacy-types/blob/main/src/objects.ts */ - "guessed-categories": t.array(DataCategoryGuessInput), + 'guessed-categories': t.array(DataCategoryGuessInput), /** * When true, this subdatapoint should be revealed in a data access request. * When false, this field should be redacted */ - "access-request-visibility-enabled": t.boolean, + 'access-request-visibility-enabled': t.boolean, /** * When true, this subdatapoint should be redacted during an erasure request. * There normally is a choice of enabling hard deletion or redaction at the * datapoint level, but if redaction is enabled, this column can be used * to define which fields should be redacted. */ - "erasure-request-redaction-enabled": t.boolean, + 'erasure-request-redaction-enabled': t.boolean, /** Attributes tagged to subdatapoint */ attributes: t.array(AttributePreview), }), @@ -701,21 +701,21 @@ export const DatapointInput = t.intersection([ * * @see https://docs.transcend.io/docs/privacy-requests/connecting-data-silos/saas-tools#configuring-an-integration */ - "data-collection-tag": t.string, + 'data-collection-tag': t.string, /** * The SQL queries that should be run for that datapoint in a privacy request. * * @see https://github.com/transcend-io/privacy-types/blob/main/src/actions.ts */ - "privacy-action-queries": t.partial( - applyEnum(RequestActionObjectResolver, () => t.string) + 'privacy-action-queries': t.partial( + applyEnum(RequestActionObjectResolver, () => t.string), ), /** * The types of privacy actions that this datapoint can implement * * @see https://github.com/transcend-io/privacy-types/blob/main/src/actions.ts */ - "privacy-actions": t.array(valuesOf(RequestActionObjectResolver)), + 'privacy-actions': t.array(valuesOf(RequestActionObjectResolver)), /** * Provide field-level metadata for this datapoint. * This is often the column metadata @@ -742,30 +742,30 @@ export type DatapointInput = t.TypeOf; export const PromptAVendorEmailSettings = t.partial({ /** The email address of the user to notify when a promptAPerson integration */ - "notify-email-address": t.string, + 'notify-email-address': t.string, /** * The frequency with which we should be sending emails for this data silo, in milliseconds. */ - "send-frequency": t.number, + 'send-frequency': t.number, /** * The type of emails to send for this data silo, i.e. send an email for each DSR, across all open DSRs, * or per profile in a DSR. */ - "send-type": valuesOf(PromptAVendorEmailSendType), + 'send-type': valuesOf(PromptAVendorEmailSendType), /** * Indicates whether prompt-a-vendor emails should include a list of identifiers * in addition to a link to the bulk processing UI. */ - "include-identifiers-attachment": t.boolean, + 'include-identifiers-attachment': t.boolean, /** * Indicates what kind of link to generate as part of the emails sent out for this Prompt-a-Vendor silo. */ - "completion-link-type": valuesOf(PromptAVendorEmailCompletionLinkType), + 'completion-link-type': valuesOf(PromptAVendorEmailCompletionLinkType), /** * The frequency with which we should retry sending emails for this data silo, in milliseconds. * Needs to be a string because the number can be larger than the MAX_INT */ - "manual-work-retry-frequency": t.string, + 'manual-work-retry-frequency': t.string, }); /** Type override */ @@ -975,11 +975,11 @@ export const ActionInput = t.intersection([ regionDetectionMethod: valuesOf(RegionDetectionMethod), /** The list of regions to show in the form */ regionList: t.array( - valuesOf({ ...IsoCountryCode, ...IsoCountrySubdivisionCode }) + valuesOf({ ...IsoCountryCode, ...IsoCountrySubdivisionCode }), ), /** The list of regions NOT to show in the form */ regionBlockList: t.array( - valuesOf({ ...IsoCountryCode, ...IsoCountrySubdivisionCode }) + valuesOf({ ...IsoCountryCode, ...IsoCountrySubdivisionCode }), ), }), ]); @@ -1128,7 +1128,7 @@ export const ConsentManageExperienceInput = t.intersection([ t.partial({ countrySubDivision: valuesOf(IsoCountrySubdivisionCode), country: valuesOf(IsoCountryCode), - }) + }), ), /** How to handle consent expiry */ onConsentExpiry: valuesOf(OnConsentExpiry), @@ -1145,14 +1145,14 @@ export const ConsentManageExperienceInput = t.intersection([ t.type({ /** Slug of purpose */ trackingType: t.string, - }) + }), ), /** Purposes that are opted out by default in a particular experience */ optedOutPurposes: t.array( t.type({ /** Slug of purpose */ trackingType: t.string, - }) + }), ), /** * Browser languages that define this regional experience @@ -1344,13 +1344,13 @@ export const DataSiloInput = t.intersection([ }), t.partial({ /** For prompt a person or database integrations, the underlying integration name */ - "outer-type": t.string, + 'outer-type': t.string, /** A description for that data silo */ description: t.string, /** The webhook URL to notify for data privacy requests */ url: t.string, /** The title of the API key that will be used to respond to privacy requests */ - "api-key-title": t.string, + 'api-key-title': t.string, /** Custom headers to include in outbound webhook */ headers: t.array(WebhookHeader), /** @@ -1358,18 +1358,18 @@ export const DataSiloInput = t.intersection([ * This field can be omitted, and the default assumption will be that the system may potentially * contain PII for any potential data subject type. */ - "data-subjects": t.array(t.string), + 'data-subjects': t.array(t.string), /** * When this data silo implements a privacy request, these are the identifiers * that should be looked up within this system. */ - "identity-keys": t.array(t.string), + 'identity-keys': t.array(t.string), /** * When a data erasure request is being performed, this data silo should not be deleted from * until all of the following data silos were deleted first. This list can contain other internal * systems defined in this file, as well as any of the SaaS tools connected in your Transcend instance. */ - "deletion-dependencies": t.array(t.string), + 'deletion-dependencies': t.array(t.string), /** * The email addresses of the employees within your company that are the go-to individuals * for managing this data silo @@ -1396,7 +1396,7 @@ export const DataSiloInput = t.intersection([ /** * Configure email notification settings for privacy requests */ - "email-settings": PromptAVendorEmailSettings, + 'email-settings': PromptAVendorEmailSettings, /** Country of data silo hosting */ country: valuesOf(IsoCountryCode), /** Sub-division of data silo hosting */ @@ -1484,13 +1484,13 @@ export type ActionItemInput = t.TypeOf; export const AssessmentRuleInput = t.intersection([ t.type({ /** The reference id of the question whose answer is compared by this rule */ - "depends-on-question-reference-id": t.string, + 'depends-on-question-reference-id': t.string, /** The operator to use when comparing the question answer to the operands */ - "comparison-operator": valuesOf(ComparisonOperator), + 'comparison-operator': valuesOf(ComparisonOperator), }), t.partial({ /** The values to compare the question answer to */ - "comparison-operands": t.array(t.string), + 'comparison-operands': t.array(t.string), }), ]); @@ -1499,28 +1499,28 @@ export type AssessmentRuleInput = t.TypeOf; export interface AssessmentNestedRuleInput { /** The operator to use when comparing the nested rules */ - "logic-operator": LogicOperator; + 'logic-operator': LogicOperator; /** The rules to evaluate and be compared with to other using the LogicOperator */ rules?: AssessmentRuleInput[]; /** The nested rules to add one more level of nesting to the rules. They are also compared to each other. */ - "nested-rules"?: AssessmentNestedRuleInput[]; + 'nested-rules'?: AssessmentNestedRuleInput[]; } export const AssessmentNestedRuleInput: t.RecursiveType< t.Type -> = t.recursion("AssessmentNestedRuleInput", (self) => +> = t.recursion('AssessmentNestedRuleInput', (self) => t.intersection([ t.type({ /** The operator to use when comparing the nested rules */ - "logic-operator": valuesOf(LogicOperator), + 'logic-operator': valuesOf(LogicOperator), }), t.partial({ /** The rules to evaluate and be compared with to other using the LogicOperator */ rules: t.array(AssessmentRuleInput), /** The nested rules to add one more level of nesting to the rules. They are also compared to each other. */ - "nested-rules": t.array(self), + 'nested-rules': t.array(self), }), - ]) + ]), ); export const AssessmentDisplayLogicInput = t.intersection([ @@ -1532,7 +1532,7 @@ export const AssessmentDisplayLogicInput = t.intersection([ /** The rule to evaluate */ rule: AssessmentRuleInput, /** The nested rule to evaluate */ - "nested-rule": AssessmentNestedRuleInput, + 'nested-rule': AssessmentNestedRuleInput, }), ]); @@ -1543,11 +1543,11 @@ export type AssessmentDisplayLogicInput = t.TypeOf< export const RiskAssignmentInput = t.partial({ /** The risk level to assign to the question */ - "risk-level": t.string, + 'risk-level': t.string, /** The risk matrix column to assign to a question. */ - "risk-matrix-column": t.string, + 'risk-matrix-column': t.string, /** The risk matrix row to assign to a question. */ - "risk-matrix-row": t.string, + 'risk-matrix-row': t.string, }); /** Type override */ @@ -1556,17 +1556,17 @@ export type RiskAssignmentInput = t.TypeOf; export const RiskLogicInput = t.intersection([ t.type({ /** The values to compare */ - "comparison-operands": t.array(t.string), + 'comparison-operands': t.array(t.string), /** The operator */ - "comparison-operator": valuesOf(ComparisonOperator), + 'comparison-operator': valuesOf(ComparisonOperator), }), t.partial({ /** The risk level to assign to the question */ - "risk-level": t.string, + 'risk-level': t.string, /** The risk matrix column to assign to a question. */ - "risk-matrix-column": t.string, + 'risk-matrix-column': t.string, /** The risk matrix row to assign to a question. */ - "risk-matrix-row": t.string, + 'risk-matrix-row': t.string, }), ]); @@ -1592,41 +1592,41 @@ export const AssessmentSectionQuestionInput = t.intersection([ }), t.partial({ /** The sub-type of the assessment question */ - "sub-type": valuesOf(AssessmentQuestionSubType), + 'sub-type': valuesOf(AssessmentQuestionSubType), /** The question placeholder */ placeholder: t.string, /** The question description */ description: t.string, /** Whether an answer is required */ - "is-required": t.boolean, + 'is-required': t.boolean, /** Used to identify the question within a form or template so it can be referenced in conditional logic. */ - "reference-id": t.string, + 'reference-id': t.string, /** Display logic for the question */ - "display-logic": AssessmentDisplayLogicInput, + 'display-logic': AssessmentDisplayLogicInput, /** Risk logic for the question */ - "risk-logic": t.array(RiskLogicInput), + 'risk-logic': t.array(RiskLogicInput), /** Risk category titles for the question */ - "risk-categories": t.array(t.string), + 'risk-categories': t.array(t.string), /** Risk framework titles for the question */ - "risk-framework": t.string, + 'risk-framework': t.string, /** Answer options for the question */ - "answer-options": t.array(AssessmentAnswerOptionInput), + 'answer-options': t.array(AssessmentAnswerOptionInput), /** The selected answers to the assessments */ - "selected-answers": t.array(t.string), + 'selected-answers': t.array(t.string), /** Allowed MIME types for the question */ - "allowed-mime-types": t.array(t.string), + 'allowed-mime-types': t.array(t.string), /** Allow selecting other options */ - "allow-select-other": t.boolean, + 'allow-select-other': t.boolean, /** Sync model for the question */ - "sync-model": valuesOf(AssessmentSyncModel), + 'sync-model': valuesOf(AssessmentSyncModel), /** Sync column for the question */ - "sync-column": valuesOf(AssessmentSyncColumn), + 'sync-column': valuesOf(AssessmentSyncColumn), /** Attribute key / custom field name for the question */ - "attribute-key": t.string, + 'attribute-key': t.string, /** Require risk evaluation for the question */ - "require-risk-evaluation": t.boolean, + 'require-risk-evaluation': t.boolean, /** Require risk matrix evaluation for the question */ - "require-risk-matrix-evaluation": t.boolean, + 'require-risk-matrix-evaluation': t.boolean, }), ]); @@ -1646,11 +1646,11 @@ export const AssessmentSectionInput = t.intersection([ /** Email address of those assigned */ assignees: t.array(t.string), /** Email address of those externally assigned */ - "external-assignees": t.array(t.string), + 'external-assignees': t.array(t.string), /** Status of section */ status: t.string, /** Whether assessment is reviewed */ - "is-reviewed": t.boolean, + 'is-reviewed': t.boolean, }), ]); @@ -1661,7 +1661,7 @@ export const AssessmentRetentionScheduleInput = t.type({ /** The retention schedule type */ type: valuesOf(RetentionScheduleType), /** The duration of the retention schedule in days */ - "duration-days": t.number, + 'duration-days': t.number, /** The operation to perform on the retention schedule */ operand: valuesOf(RetentionScheduleOperation), }); @@ -1690,15 +1690,15 @@ export const AssessmentTemplateInput = t.intersection([ /** Whether the template is in a locked status */ locked: t.boolean, /** ID of parent template this was cloned from */ - "parent-id": t.string, + 'parent-id': t.string, /** Whether the template is archived */ archived: t.boolean, /** The date that the assessment was created */ - "created-at": t.string, + 'created-at': t.string, /** The names of the custom fields associated to this assessment template */ - "attribute-keys": t.array(t.string), + 'attribute-keys': t.array(t.string), /** The retention schedule configuration */ - "retention-schedule": AssessmentRetentionScheduleInput, + 'retention-schedule': AssessmentRetentionScheduleInput, /** The titles of the email templates used in the assessment template */ templates: t.array(t.string), }), @@ -1736,7 +1736,7 @@ export const AssessmentInput = t.intersection([ /** The emails of the transcend users assigned to the assessment */ assignees: t.array(t.string), /** The emails of the external emails assigned to the assessment */ - "external-assignees": t.array(t.string), + 'external-assignees': t.array(t.string), /** The emails of the assessment reviewers */ reviewers: t.array(t.string), /** Whether the assessment is in a locked status */ @@ -1748,25 +1748,25 @@ export const AssessmentInput = t.intersection([ /** * Whether the form title is an internal label only, and the group title should be used in communications with assignees */ - "title-is-internal": t.boolean, + 'title-is-internal': t.boolean, /** The date that the assessment is due */ - "due-date": t.string, + 'due-date': t.string, /** The date that the assessment was created */ - "created-at": t.string, + 'created-at': t.string, /** The date that the assessment was assigned at */ - "assigned-at": t.string, + 'assigned-at': t.string, /** The date that the assessment was submitted at */ - "submitted-at": t.string, + 'submitted-at': t.string, /** The date that the assessment was approved at */ - "approved-at": t.string, + 'approved-at': t.string, /** The date that the assessment was rejected at */ - "rejected-at": t.string, + 'rejected-at': t.string, /** The linked data inventory resources */ resources: t.array(AssessmentResourceInput), /** The linked data inventory synced rows */ rows: t.array(AssessmentResourceInput), /** The assessment retention schedule */ - "retention-schedule": AssessmentRetentionScheduleInput, + 'retention-schedule': AssessmentRetentionScheduleInput, /** The assessment custom fields */ attributes: t.array(AttributePreview), }), @@ -1798,9 +1798,9 @@ export const ConsentPreferenceTopic = t.intersection([ }), t.partial({ /** Default value */ - "default-configuration": t.string, + 'default-configuration': t.string, /** Whether the preference topic is shown in privacy center */ - "show-in-privacy-center": t.boolean, + 'show-in-privacy-center': t.boolean, /** The options when type is single or multi select */ options: t.array(ConsentPreferenceTopicOptionValue), }), @@ -1822,23 +1822,23 @@ export const ConsentPurpose = t.intersection([ /** Description of purpose */ description: t.string, /** Whether purpose is active */ - "is-active": t.boolean, + 'is-active': t.boolean, /** Whether purpose is configurable */ configurable: t.boolean, /** Display order of purpose for privacy center */ - "display-order": t.number, + 'display-order': t.number, /** Whether purpose is shown in privacy center */ - "show-in-privacy-center": t.boolean, + 'show-in-privacy-center': t.boolean, /** Whether purpose is show in consent manger */ - "show-in-consent-manager": t.boolean, + 'show-in-consent-manager': t.boolean, /** The preference topics configured for the purpose */ - "preference-topics": t.array(ConsentPreferenceTopic), + 'preference-topics': t.array(ConsentPreferenceTopic), /** Authentication level for purpose on privacy center */ - "auth-level": valuesOf(PreferenceStoreAuthLevel), + 'auth-level': valuesOf(PreferenceStoreAuthLevel), /** Opt out signals that should instantly opt out of this purpose */ - "opt-out-signals": t.array(valuesOf(UserPrivacySignalEnum)), + 'opt-out-signals': t.array(valuesOf(UserPrivacySignalEnum)), /** Default consent value */ - "default-consent": valuesOf(DefaultConsentOption), + 'default-consent': valuesOf(DefaultConsentOption), }), ]); @@ -1849,15 +1849,15 @@ export const TranscendInput = t.partial({ /** * Action items */ - "action-items": t.array(ActionItemInput), + 'action-items': t.array(ActionItemInput), /** * Action item collections */ - "action-item-collections": t.array(ActionItemCollectionInput), + 'action-item-collections': t.array(ActionItemCollectionInput), /** * API key definitions */ - "api-keys": t.array(ApiKeyInput), + 'api-keys': t.array(ApiKeyInput), /** Team definitions */ teams: t.array(TeamInput), /** @@ -1875,7 +1875,7 @@ export const TranscendInput = t.partial({ /** * Business entity definitions */ - "business-entities": t.array(BusinessEntityInput), + 'business-entities': t.array(BusinessEntityInput), /** * Vendor definitions */ @@ -1883,15 +1883,15 @@ export const TranscendInput = t.partial({ /** * Data categories definitions */ - "data-categories": t.array(DataCategoryInput), + 'data-categories': t.array(DataCategoryInput), /** * Vendor definitions */ - "processing-purposes": t.array(ProcessingPurposeInput), + 'processing-purposes': t.array(ProcessingPurposeInput), /** * Data subject definitions */ - "data-subjects": t.array(DataSubjectInput), + 'data-subjects': t.array(DataSubjectInput), /** * Action definitions */ @@ -1903,11 +1903,11 @@ export const TranscendInput = t.partial({ /** * Data silo definitions */ - "data-silos": t.array(DataSiloInput), + 'data-silos': t.array(DataSiloInput), /** * Data flow definitions */ - "data-flows": t.array(DataFlowInput), + 'data-flows': t.array(DataFlowInput), /** * Cookie definitions */ @@ -1915,7 +1915,7 @@ export const TranscendInput = t.partial({ /** * Consent manager definition */ - "consent-manager": ConsentManagerInput, + 'consent-manager': ConsentManagerInput, /** * Prompt definitions */ @@ -1923,11 +1923,11 @@ export const TranscendInput = t.partial({ /** * Prompt partial definitions */ - "prompt-partials": t.array(PromptPartialInput), + 'prompt-partials': t.array(PromptPartialInput), /** * Prompt group definitions */ - "prompt-groups": t.array(PromptGroupInput), + 'prompt-groups': t.array(PromptGroupInput), /** * Agent definitions */ @@ -1935,15 +1935,15 @@ export const TranscendInput = t.partial({ /** * Agent function definitions */ - "agent-functions": t.array(AgentFunctionInput), + 'agent-functions': t.array(AgentFunctionInput), /** * Agent file definitions */ - "agent-files": t.array(AgentFileInput), + 'agent-files': t.array(AgentFileInput), /** * The privacy center configuration */ - "privacy-center": PrivacyCenterInput, + 'privacy-center': PrivacyCenterInput, /** * The policies configuration */ @@ -1957,7 +1957,7 @@ export const TranscendInput = t.partial({ /** * The full list of assessment templates */ - "assessment-templates": t.array(AssessmentTemplateInput), + 'assessment-templates': t.array(AssessmentTemplateInput), /** * The full list of assessment results */ @@ -1992,7 +1992,7 @@ export type StoredApiKey = t.TypeOf; export const DataFlowCsvInput = t.intersection([ t.type({ /** The value of the data flow (host or regex) */ - "Connections Made To": t.string, + 'Connections Made To': t.string, /** The type of the data flow */ Type: valuesOf(DataFlowScope), /** The CSV of purposes mapped to that data flow */ @@ -2058,7 +2058,7 @@ export const ConsentManagerServiceMetadata = t.type({ name: t.string, /** Allowed purposes */ trackingPurposes: t.array(t.string), - }) + }), ), /** Data Flows */ dataFlows: t.array( @@ -2069,7 +2069,7 @@ export const ConsentManagerServiceMetadata = t.type({ type: valuesOf(DataFlowScope), /** Allowed purposes */ trackingPurposes: t.array(t.string), - }) + }), ), }); diff --git a/src/commands/admin/generate-api-keys/impl.ts b/src/commands/admin/generate-api-keys/impl.ts index f9c0d278..0288e38e 100644 --- a/src/commands/admin/generate-api-keys/impl.ts +++ b/src/commands/admin/generate-api-keys/impl.ts @@ -1,17 +1,17 @@ -import { writeFileSync } from "node:fs"; -import { ScopeName, TRANSCEND_SCOPES } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { keyBy } from "lodash-es"; -import type { LocalContext } from "../../../context"; -import { generateCrossAccountApiKeys } from "../../../lib/api-keys"; -import { logger } from "../../../logger"; +import { writeFileSync } from 'node:fs'; +import { ScopeName, TRANSCEND_SCOPES } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { keyBy } from 'lodash-es'; +import type { LocalContext } from '../../../context'; +import { generateCrossAccountApiKeys } from '../../../lib/api-keys'; +import { logger } from '../../../logger'; const SCOPES_BY_TITLE = keyBy( Object.entries(TRANSCEND_SCOPES).map(([name, value]) => ({ ...value, name, })), - "title" + 'title', ); const SCOPE_TITLES = Object.keys(SCOPES_BY_TITLE); @@ -41,25 +41,25 @@ export async function generateApiKeys( createNewApiKey, parentOrganizationId, transcendUrl, - }: GenerateApiKeysCommandFlags + }: GenerateApiKeysCommandFlags, ): Promise { // Validate scopes const splitScopes = scopes.map((x) => x.trim()); const invalidScopes = splitScopes.filter( - (scopeTitle) => !SCOPES_BY_TITLE[scopeTitle] + (scopeTitle) => !SCOPES_BY_TITLE[scopeTitle], ); if (invalidScopes.length > 0) { logger.error( colors.red( - `Failed to parse scopes:"${invalidScopes.join(",")}".\n` + - `Expected one of: \n${SCOPE_TITLES.join("\n")}` - ) + `Failed to parse scopes:"${invalidScopes.join(',')}".\n` + + `Expected one of: \n${SCOPE_TITLES.join('\n')}`, + ), ); process.exit(1); } const scopeNames = splitScopes.map( - (scopeTitle) => SCOPES_BY_TITLE[scopeTitle].name as ScopeName + (scopeTitle) => SCOPES_BY_TITLE[scopeTitle].name as ScopeName, ); // Upload privacy requests diff --git a/src/commands/consent/build-xdi-sync-endpoint/impl.ts b/src/commands/consent/build-xdi-sync-endpoint/impl.ts index 2b98351a..b42f26bb 100644 --- a/src/commands/consent/build-xdi-sync-endpoint/impl.ts +++ b/src/commands/consent/build-xdi-sync-endpoint/impl.ts @@ -1,9 +1,9 @@ -import { writeFileSync } from "node:fs"; -import colors from "colors"; -import type { LocalContext } from "../../../context"; -import { validateTranscendAuth } from "../../../lib/api-keys"; -import { buildXdiSyncEndpoint as buildXdiSyncEndpointHelper } from "../../../lib/consent-manager"; -import { logger } from "../../../logger"; +import { writeFileSync } from 'node:fs'; +import colors from 'colors'; +import type { LocalContext } from '../../../context'; +import { validateTranscendAuth } from '../../../lib/api-keys'; +import { buildXdiSyncEndpoint as buildXdiSyncEndpointHelper } from '../../../lib/consent-manager'; +import { logger } from '../../../logger'; interface BuildXdiSyncEndpointCommandFlags { auth: string; @@ -25,7 +25,7 @@ export async function buildXdiSyncEndpoint( domainBlockList, xdiAllowedCommands, transcendUrl, - }: BuildXdiSyncEndpointCommandFlags + }: BuildXdiSyncEndpointCommandFlags, ): Promise { // Parse authentication as API key or path to list of API keys const apiKeyOrList = await validateTranscendAuth(auth); @@ -45,9 +45,9 @@ export async function buildXdiSyncEndpoint( `Successfully constructed sync endpoint for sync groups: ${JSON.stringify( syncGroups, null, - 2 - )}` - ) + 2, + )}`, + ), ); // Write to disk diff --git a/src/commands/consent/pull-consent-metrics/impl.ts b/src/commands/consent/pull-consent-metrics/impl.ts index a7078c79..0b1c58b2 100644 --- a/src/commands/consent/pull-consent-metrics/impl.ts +++ b/src/commands/consent/pull-consent-metrics/impl.ts @@ -1,17 +1,17 @@ -import fs, { existsSync, mkdirSync } from "node:fs"; -import { join } from "node:path"; -import colors from "colors"; -import { ADMIN_DASH_INTEGRATIONS } from "../../../constants"; -import type { LocalContext } from "../../../context"; -import { validateTranscendAuth } from "../../../lib/api-keys"; -import { mapSeries } from "../../../lib/bluebird-replace"; -import { pullConsentManagerMetrics } from "../../../lib/consent-manager"; -import { writeCsv } from "../../../lib/cron"; +import fs, { existsSync, mkdirSync } from 'node:fs'; +import { join } from 'node:path'; +import colors from 'colors'; +import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; +import type { LocalContext } from '../../../context'; +import { validateTranscendAuth } from '../../../lib/api-keys'; +import { mapSeries } from '../../../lib/bluebird-replace'; +import { pullConsentManagerMetrics } from '../../../lib/consent-manager'; +import { writeCsv } from '../../../lib/cron'; import { buildTranscendGraphQLClient, ConsentManagerMetricBin, -} from "../../../lib/graphql"; -import { logger } from "../../../logger"; +} from '../../../lib/graphql'; +import { logger } from '../../../logger'; interface PullConsentMetricsCommandFlags { auth: string; @@ -31,7 +31,7 @@ export async function pullConsentMetrics( folder, bin, transcendUrl, - }: PullConsentMetricsCommandFlags + }: PullConsentMetricsCommandFlags, ): Promise { // Parse authentication as API key or path to list of API keys const apiKeyOrList = await validateTranscendAuth(auth); @@ -40,8 +40,8 @@ export async function pullConsentMetrics( if (fs.existsSync(folder) && !fs.lstatSync(folder).isDirectory()) { logger.error( colors.red( - 'The provided argument "folder" was passed a file. expected: folder="./consent-metrics/"' - ) + 'The provided argument "folder" was passed a file. expected: folder="./consent-metrics/"', + ), ); process.exit(1); } @@ -53,9 +53,9 @@ export async function pullConsentMetrics( colors.red( `Failed to parse argument "bin" with value "${bin}"\n` + `Expected one of: \n${Object.values(ConsentManagerMetricBin).join( - "\n" - )}` - ) + '\n', + )}`, + ), ); process.exit(1); } @@ -66,16 +66,16 @@ export async function pullConsentMetrics( if (Number.isNaN(startDate.getTime())) { logger.error( colors.red( - `Start date provided is invalid date. Got --start="${start}" expected --start="01/01/2023"` - ) + `Start date provided is invalid date. Got --start="${start}" expected --start="01/01/2023"`, + ), ); process.exit(1); } if (Number.isNaN(endDate.getTime())) { logger.error( colors.red( - `End date provided is invalid date. Got --end="${end}" expected --end="01/01/2023"` - ) + `End date provided is invalid date. Got --end="${end}" expected --end="01/01/2023"`, + ), ); process.exit(1); } @@ -83,8 +83,8 @@ export async function pullConsentMetrics( logger.error( colors.red( `Got a start date "${startDate.toISOString()}" that was larger than the end date "${endDate.toISOString()}". ` + - "Start date must be before end date." - ) + 'Start date must be before end date.', + ), ); process.exit(1); } @@ -96,12 +96,12 @@ export async function pullConsentMetrics( logger.info( colors.magenta( - `Pulling consent metrics from start=${startDate.toString()} to end=${endDate.toISOString()} with bin size "${bin}"` - ) + `Pulling consent metrics from start=${startDate.toString()} to end=${endDate.toISOString()} with bin size "${bin}"`, + ), ); // Sync to Disk - if (typeof apiKeyOrList === "string") { + if (typeof apiKeyOrList === 'string') { try { // Create a GraphQL client const client = buildTranscendGraphQLClient(transcendUrl, apiKeyOrList); @@ -118,20 +118,20 @@ export async function pullConsentMetrics( for (const { points, name } of metrics) { const file = join(folder, `${metricName}_${name}.csv`); logger.info( - colors.magenta(`Writing configuration to file "${file}"...`) + colors.magenta(`Writing configuration to file "${file}"...`), ); writeCsv( file, points.map(({ key, value }) => ({ timestamp: key, value, - })) + })), ); } } } catch (error) { logger.error( - colors.red(`An error occurred syncing the schema: ${error.message}`) + colors.red(`An error occurred syncing the schema: ${error.message}`), ); process.exit(1); } @@ -139,8 +139,8 @@ export async function pullConsentMetrics( // Indicate success logger.info( colors.green( - `Successfully synced consent metrics to disk in folder "${folder}"! View at ${ADMIN_DASH_INTEGRATIONS}` - ) + `Successfully synced consent metrics to disk in folder "${folder}"! View at ${ADMIN_DASH_INTEGRATIONS}`, + ), ); } else { const encounteredErrors: string[] = []; @@ -150,8 +150,8 @@ export async function pullConsentMetrics( }] `; logger.info( colors.magenta( - `~~~\n\n${prefix}Attempting to pull consent metrics...\n\n~~~` - ) + `~~~\n\n${prefix}Attempting to pull consent metrics...\n\n~~~`, + ), ); // Create a GraphQL client @@ -175,20 +175,20 @@ export async function pullConsentMetrics( for (const { points, name } of metrics) { const file = join(subFolder, `${metricName}_${name}.csv`); logger.info( - colors.magenta(`Writing configuration to file "${file}"...`) + colors.magenta(`Writing configuration to file "${file}"...`), ); writeCsv( file, points.map(({ key, value }) => ({ timestamp: key, value, - })) + })), ); } } logger.info( - colors.green(`${prefix}Successfully pulled configuration!`) + colors.green(`${prefix}Successfully pulled configuration!`), ); } catch { logger.error(colors.red(`${prefix}Failed to sync configuration.`)); @@ -200,9 +200,9 @@ export async function pullConsentMetrics( logger.info( colors.red( `Sync encountered errors for "${encounteredErrors.join( - "," - )}". View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}` - ) + ',', + )}". View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}`, + ), ); process.exit(1); diff --git a/src/commands/consent/upload-preferences/impl.ts b/src/commands/consent/upload-preferences/impl.ts index 5d2f61c9..d8c64d03 100644 --- a/src/commands/consent/upload-preferences/impl.ts +++ b/src/commands/consent/upload-preferences/impl.ts @@ -1,11 +1,11 @@ -import { readdirSync } from "node:fs"; -import { basename, join } from "node:path"; -import colors from "colors"; -import type { LocalContext } from "../../../context"; -import { map } from "../../../lib/bluebird-replace"; -import { uploadPreferenceManagementPreferencesInteractive } from "../../../lib/preference-management"; -import { splitCsvToList } from "../../../lib/requests"; -import { logger } from "../../../logger"; +import { readdirSync } from 'node:fs'; +import { basename, join } from 'node:path'; +import colors from 'colors'; +import type { LocalContext } from '../../../context'; +import { map } from '../../../lib/bluebird-replace'; +import { uploadPreferenceManagementPreferencesInteractive } from '../../../lib/preference-management'; +import { splitCsvToList } from '../../../lib/requests'; +import { logger } from '../../../logger'; interface UploadPreferencesCommandFlags { auth: string; @@ -33,7 +33,7 @@ export async function uploadPreferences( partition, sombraAuth, consentUrl, - file = "", + file = '', directory, dryRun, skipExistingRecordCheck, @@ -44,13 +44,13 @@ export async function uploadPreferences( isSilent, attributes, concurrency, - }: UploadPreferencesCommandFlags + }: UploadPreferencesCommandFlags, ): Promise { if (!!directory && !!file) { logger.error( colors.red( - "Cannot provide both a directory and a file. Please provide only one." - ) + 'Cannot provide both a directory and a file. Please provide only one.', + ), ); process.exit(1); } @@ -58,8 +58,8 @@ export async function uploadPreferences( if (!file && !directory) { logger.error( colors.red( - "A file or directory must be provided. Please provide one using --file=./preferences.csv or --directory=./preferences" - ) + 'A file or directory must be provided. Please provide one using --file=./preferences.csv or --directory=./preferences', + ), ); process.exit(1); } @@ -69,11 +69,11 @@ export async function uploadPreferences( if (directory) { try { const filesInDirectory = readdirSync(directory); - const csvFiles = filesInDirectory.filter((file) => file.endsWith(".csv")); + const csvFiles = filesInDirectory.filter((file) => file.endsWith('.csv')); if (csvFiles.length === 0) { logger.error( - colors.red(`No CSV files found in directory: ${directory}`) + colors.red(`No CSV files found in directory: ${directory}`), ); process.exit(1); } @@ -88,8 +88,8 @@ export async function uploadPreferences( } else { try { // Verify file exists and is a CSV - if (!file.endsWith(".csv")) { - logger.error(colors.red("File must be a CSV file")); + if (!file.endsWith('.csv')) { + logger.error(colors.red('File must be a CSV file')); process.exit(1); } files.push(file); @@ -102,23 +102,23 @@ export async function uploadPreferences( logger.info( colors.green( - `Processing ${files.length} consent preferences files for partition: ${partition}` - ) + `Processing ${files.length} consent preferences files for partition: ${partition}`, + ), ); - logger.debug(`Files to process: ${files.join(", ")}`); + logger.debug(`Files to process: ${files.join(', ')}`); if (skipExistingRecordCheck) { logger.info( colors.bgYellow( - `Skipping existing record check: ${skipExistingRecordCheck}` - ) + `Skipping existing record check: ${skipExistingRecordCheck}`, + ), ); } await map( files, async (filePath) => { - const fileName = basename(filePath).replace(".csv", ""); + const fileName = basename(filePath).replace('.csv', ''); await uploadPreferenceManagementPreferencesInteractive({ receiptFilepath: join(receiptFileDir, `${fileName}-receipts.json`), auth, @@ -135,6 +135,6 @@ export async function uploadPreferences( forceTriggerWorkflows, }); }, - { concurrency } + { concurrency }, ); } diff --git a/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts b/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts index ad41b2e9..28ab8b64 100644 --- a/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts +++ b/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts @@ -1,19 +1,19 @@ -import { existsSync, readFileSync } from "node:fs"; +import { existsSync, readFileSync } from 'node:fs'; import { ConsentTrackerStatus, DataFlowScope, -} from "@transcend-io/privacy-types"; -import { decodeCodec } from "@transcend-io/type-utils"; -import colors from "colors"; -import * as t from "io-ts"; +} from '@transcend-io/privacy-types'; +import { decodeCodec } from '@transcend-io/type-utils'; +import colors from 'colors'; +import * as t from 'io-ts'; import { ConsentManagerServiceMetadata, CookieInput, DataFlowInput, -} from "../../../codecs"; -import type { LocalContext } from "../../../context"; -import { writeTranscendYaml } from "../../../lib/readTranscendYaml"; -import { logger } from "../../../logger"; +} from '../../../codecs'; +import type { LocalContext } from '../../../context'; +import { writeTranscendYaml } from '../../../lib/readTranscendYaml'; +import { logger } from '../../../logger'; interface ConsentManagerServiceJsonToYmlCommandFlags { file: string; @@ -22,7 +22,7 @@ interface ConsentManagerServiceJsonToYmlCommandFlags { export function consentManagerServiceJsonToYml( this: LocalContext, - { file, output }: ConsentManagerServiceJsonToYmlCommandFlags + { file, output }: ConsentManagerServiceJsonToYmlCommandFlags, ): void { // Ensure files exist if (!existsSync(file)) { @@ -33,7 +33,7 @@ export function consentManagerServiceJsonToYml( // Read in each consent manager configuration const services = decodeCodec( t.array(ConsentManagerServiceMetadata), - readFileSync(file, "utf-8") + readFileSync(file, 'utf-8'), ); // Create data flows and cookie configurations @@ -41,7 +41,7 @@ export function consentManagerServiceJsonToYml( const cookies: CookieInput[] = []; for (const service of services) { for (const dataFlow of service.dataFlows.filter( - ({ type }) => type !== DataFlowScope.CSP + ({ type }) => type !== DataFlowScope.CSP, )) { dataFlows.push({ value: dataFlow.value, @@ -62,13 +62,13 @@ export function consentManagerServiceJsonToYml( // write to disk writeTranscendYaml(output, { - "data-flows": dataFlows, + 'data-flows': dataFlows, cookies, }); logger.info( colors.green( - `Successfully wrote ${dataFlows.length} data flows and ${cookies.length} cookies to file "${output}"` - ) + `Successfully wrote ${dataFlows.length} data flows and ${cookies.length} cookies to file "${output}"`, + ), ); } diff --git a/src/commands/inventory/consent-managers-to-business-entities/impl.ts b/src/commands/inventory/consent-managers-to-business-entities/impl.ts index a504d375..e379557a 100644 --- a/src/commands/inventory/consent-managers-to-business-entities/impl.ts +++ b/src/commands/inventory/consent-managers-to-business-entities/impl.ts @@ -1,14 +1,14 @@ -import { existsSync, lstatSync } from "node:fs"; -import { join } from "node:path"; -import colors from "colors"; -import type { LocalContext } from "../../../context"; -import { listFiles } from "../../../lib/api-keys"; -import { consentManagersToBusinessEntities as consentManagersToBusinessEntitiesHelper } from "../../../lib/consent-manager"; +import { existsSync, lstatSync } from 'node:fs'; +import { join } from 'node:path'; +import colors from 'colors'; +import type { LocalContext } from '../../../context'; +import { listFiles } from '../../../lib/api-keys'; +import { consentManagersToBusinessEntities as consentManagersToBusinessEntitiesHelper } from '../../../lib/consent-manager'; import { readTranscendYaml, writeTranscendYaml, -} from "../../../lib/readTranscendYaml"; -import { logger } from "../../../logger"; +} from '../../../lib/readTranscendYaml'; +import { logger } from '../../../logger'; interface ConsentManagersToBusinessEntitiesCommandFlags { consentManagerYmlFolder: string; @@ -20,7 +20,7 @@ export function consentManagersToBusinessEntities( { consentManagerYmlFolder, output, - }: ConsentManagersToBusinessEntitiesCommandFlags + }: ConsentManagersToBusinessEntitiesCommandFlags, ): void { // Ensure folder is passed if ( @@ -28,15 +28,15 @@ export function consentManagersToBusinessEntities( !lstatSync(consentManagerYmlFolder).isDirectory() ) { logger.error( - colors.red(`Folder does not exist: "${consentManagerYmlFolder}"`) + colors.red(`Folder does not exist: "${consentManagerYmlFolder}"`), ); process.exit(1); } // Read in each consent manager configuration const inputs = listFiles(consentManagerYmlFolder).map((directory) => { - const { "consent-manager": consentManager } = readTranscendYaml( - join(consentManagerYmlFolder, directory) + const { 'consent-manager': consentManager } = readTranscendYaml( + join(consentManagerYmlFolder, directory), ); return { name: directory, input: consentManager }; }); @@ -46,12 +46,12 @@ export function consentManagersToBusinessEntities( // write to disk writeTranscendYaml(output, { - "business-entities": businessEntities, + 'business-entities': businessEntities, }); logger.info( colors.green( - `Successfully wrote ${businessEntities.length} business entities to file "${output}"` - ) + `Successfully wrote ${businessEntities.length} business entities to file "${output}"`, + ), ); } diff --git a/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts b/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts index 0002d8f0..9b29f0ba 100644 --- a/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts +++ b/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts @@ -1,20 +1,20 @@ -import { existsSync, lstatSync } from "node:fs"; -import { join } from "node:path"; -import colors from "colors"; -import { difference } from "lodash-es"; -import { DataFlowInput } from "../../../codecs"; -import type { LocalContext } from "../../../context"; -import { listFiles } from "../../../lib/api-keys"; -import { dataFlowsToDataSilos } from "../../../lib/consent-manager/dataFlowsToDataSilos"; +import { existsSync, lstatSync } from 'node:fs'; +import { join } from 'node:path'; +import colors from 'colors'; +import { difference } from 'lodash-es'; +import { DataFlowInput } from '../../../codecs'; +import type { LocalContext } from '../../../context'; +import { listFiles } from '../../../lib/api-keys'; +import { dataFlowsToDataSilos } from '../../../lib/consent-manager/dataFlowsToDataSilos'; import { buildTranscendGraphQLClient, fetchAndIndexCatalogs, -} from "../../../lib/graphql"; +} from '../../../lib/graphql'; import { readTranscendYaml, writeTranscendYaml, -} from "../../../lib/readTranscendYaml"; -import { logger } from "../../../logger"; +} from '../../../lib/readTranscendYaml'; +import { logger } from '../../../logger'; interface DeriveDataSilosFromDataFlowsCrossInstanceCommandFlags { auth: string; @@ -32,14 +32,14 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( output, ignoreYmls = [], transcendUrl, - }: DeriveDataSilosFromDataFlowsCrossInstanceCommandFlags + }: DeriveDataSilosFromDataFlowsCrossInstanceCommandFlags, ): Promise { // Ensure folder is passed to dataFlowsYmlFolder if (!dataFlowsYmlFolder) { logger.error( colors.red( - "Missing required arg: --dataFlowsYmlFolder=./working/data-flows/" - ) + 'Missing required arg: --dataFlowsYmlFolder=./working/data-flows/', + ), ); process.exit(1); } @@ -54,13 +54,13 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( } // Ignore the data flows in these yml files - const instancesToIgnore = ignoreYmls.map((x) => x.split(".")[0]); + const instancesToIgnore = ignoreYmls.map((x) => x.split('.')[0]); // Map over each data flow yml file and convert to data silo configurations const dataSiloInputs = listFiles(dataFlowsYmlFolder).map((directory) => { // read in the data flows for a specific instance - const { "data-flows": dataFlows = [] } = readTranscendYaml( - join(dataFlowsYmlFolder, directory) + const { 'data-flows': dataFlows = [] } = readTranscendYaml( + join(dataFlowsYmlFolder, directory), ); // map the data flows to data silos @@ -69,13 +69,13 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( { serviceToSupportedIntegration, serviceToTitle, - } + }, ); return { adTechDataSilos, siteTechDataSilos, - organizationName: directory.split(".")[0], + organizationName: directory.split('.')[0], }; }); @@ -88,7 +88,7 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( } of dataSiloInputs) { const allDataSilos = [...adTechDataSilos, ...siteTechDataSilos]; for (const dataSilo of allDataSilos) { - const service = dataSilo["outer-type"] || dataSilo.integrationName; + const service = dataSilo['outer-type'] || dataSilo.integrationName; // create mapping to instance if (!serviceToInstance[service]) { serviceToInstance[service] = []; @@ -103,9 +103,9 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( ...new Set( dataSiloInputs.flatMap(({ adTechDataSilos }) => adTechDataSilos.map( - (silo) => silo["outer-type"] || silo.integrationName - ) - ) + (silo) => silo['outer-type'] || silo.integrationName, + ), + ), ), ]; @@ -115,12 +115,12 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( ...new Set( dataSiloInputs.flatMap(({ siteTechDataSilos }) => siteTechDataSilos.map( - (silo) => silo["outer-type"] || silo.integrationName - ) - ) + (silo) => silo['outer-type'] || silo.integrationName, + ), + ), ), ], - adTechIntegrations + adTechIntegrations, ); // Mapping from service name to list of @@ -128,9 +128,9 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( for (const { adTechDataSilos, siteTechDataSilos } of dataSiloInputs) { const allDataSilos = [...adTechDataSilos, ...siteTechDataSilos]; for (const dataSilo of allDataSilos) { - const service = dataSilo["outer-type"] || dataSilo.integrationName; + const service = dataSilo['outer-type'] || dataSilo.integrationName; const foundOnDomain = dataSilo.attributes?.find( - (attribute) => attribute.key === "Found On Domain" + (attribute) => attribute.key === 'Found On Domain', ); // create mapping to instance if (!serviceToFoundOnDomain[service]) { @@ -154,25 +154,25 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( title: serviceToTitle[service], ...(serviceToSupportedIntegration[service] ? { integrationName: service } - : { integrationName: "promptAPerson", "outer-type": service }), + : { integrationName: 'promptAPerson', 'outer-type': service }), attributes: [ { - key: "Tech Type", - values: ["Ad Tech"], + key: 'Tech Type', + values: ['Ad Tech'], }, { - key: "Business Units", + key: 'Business Units', values: difference( serviceToInstance[service] || [], - instancesToIgnore + instancesToIgnore, ), }, { - key: "Found On Domain", + key: 'Found On Domain', values: serviceToFoundOnDomain[service] || [], }, ], - }) + }), ); // Log output @@ -182,6 +182,6 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( // Write to yaml writeTranscendYaml(output, { - "data-silos": dataSilos, + 'data-silos': dataSilos, }); } diff --git a/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts b/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts index 5c6e642e..27bb7b51 100644 --- a/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts +++ b/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts @@ -1,19 +1,19 @@ -import { existsSync, lstatSync } from "node:fs"; -import { join } from "node:path"; -import colors from "colors"; -import { DataFlowInput } from "../../../codecs"; -import type { LocalContext } from "../../../context"; -import { listFiles } from "../../../lib/api-keys"; -import { dataFlowsToDataSilos } from "../../../lib/consent-manager/dataFlowsToDataSilos"; +import { existsSync, lstatSync } from 'node:fs'; +import { join } from 'node:path'; +import colors from 'colors'; +import { DataFlowInput } from '../../../codecs'; +import type { LocalContext } from '../../../context'; +import { listFiles } from '../../../lib/api-keys'; +import { dataFlowsToDataSilos } from '../../../lib/consent-manager/dataFlowsToDataSilos'; import { buildTranscendGraphQLClient, fetchAndIndexCatalogs, -} from "../../../lib/graphql"; +} from '../../../lib/graphql'; import { readTranscendYaml, writeTranscendYaml, -} from "../../../lib/readTranscendYaml"; -import { logger } from "../../../logger"; +} from '../../../lib/readTranscendYaml'; +import { logger } from '../../../logger'; interface DeriveDataSilosFromDataFlowsCommandFlags { auth: string; @@ -31,14 +31,14 @@ export async function deriveDataSilosFromDataFlows( dataSilosYmlFolder, ignoreYmls = [], transcendUrl, - }: DeriveDataSilosFromDataFlowsCommandFlags + }: DeriveDataSilosFromDataFlowsCommandFlags, ): Promise { // Ensure folder is passed to dataFlowsYmlFolder if (!dataFlowsYmlFolder) { logger.error( colors.red( - "Missing required arg: --dataFlowsYmlFolder=./working/data-flows/" - ) + 'Missing required arg: --dataFlowsYmlFolder=./working/data-flows/', + ), ); process.exit(1); } @@ -56,8 +56,8 @@ export async function deriveDataSilosFromDataFlows( if (!dataSilosYmlFolder) { logger.error( colors.red( - "Missing required arg: --dataSilosYmlFolder=./working/data-silos/" - ) + 'Missing required arg: --dataSilosYmlFolder=./working/data-silos/', + ), ); process.exit(1); } @@ -79,8 +79,8 @@ export async function deriveDataSilosFromDataFlows( // List of each data flow yml file for (const directory of listFiles(dataFlowsYmlFolder)) { // read in the data flows for a specific instance - const { "data-flows": dataFlows = [] } = readTranscendYaml( - join(dataFlowsYmlFolder, directory) + const { 'data-flows': dataFlows = [] } = readTranscendYaml( + join(dataFlowsYmlFolder, directory), ); // map the data flows to data silos @@ -89,7 +89,7 @@ export async function deriveDataSilosFromDataFlows( { serviceToSupportedIntegration, serviceToTitle, - } + }, ); // combine and write to yml file @@ -98,7 +98,7 @@ export async function deriveDataSilosFromDataFlows( logger.log(`Ad Tech Services: ${adTechDataSilos.length}`); logger.log(`Site Tech Services: ${siteTechDataSilos.length}`); writeTranscendYaml(join(dataSilosYmlFolder, directory), { - "data-silos": ignoreYmls.includes(directory) ? [] : dataSilos, + 'data-silos': ignoreYmls.includes(directory) ? [] : dataSilos, }); } } diff --git a/src/commands/inventory/pull-datapoints/impl.ts b/src/commands/inventory/pull-datapoints/impl.ts index fa968a75..a76b656a 100644 --- a/src/commands/inventory/pull-datapoints/impl.ts +++ b/src/commands/inventory/pull-datapoints/impl.ts @@ -1,12 +1,12 @@ -import { DataCategoryType } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { groupBy, uniq } from "lodash-es"; -import { ADMIN_DASH_DATAPOINTS } from "../../../constants"; -import type { LocalContext } from "../../../context"; -import { writeCsv } from "../../../lib/cron"; -import { pullAllDatapoints } from "../../../lib/data-inventory"; -import { buildTranscendGraphQLClient } from "../../../lib/graphql"; -import { logger } from "../../../logger"; +import { DataCategoryType } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { groupBy, uniq } from 'lodash-es'; +import { ADMIN_DASH_DATAPOINTS } from '../../../constants'; +import type { LocalContext } from '../../../context'; +import { writeCsv } from '../../../lib/cron'; +import { pullAllDatapoints } from '../../../lib/data-inventory'; +import { buildTranscendGraphQLClient } from '../../../lib/graphql'; +import { logger } from '../../../logger'; interface PullDatapointsCommandFlags { auth: string; @@ -30,7 +30,7 @@ export async function pullDatapoints( includeGuessedCategories, parentCategories, subCategories = [], - }: PullDatapointsCommandFlags + }: PullDatapointsCommandFlags, ): Promise { try { // Create a GraphQL client @@ -48,28 +48,28 @@ export async function pullDatapoints( let headers: string[] = []; const inputs = dataPoints.map((point) => { const result = { - "Property ID": point.id, - "Data Silo": point.dataSilo.title, + 'Property ID': point.id, + 'Data Silo': point.dataSilo.title, Object: point.dataPoint.name, - "Object Path": point.dataPoint.path.join("."), + 'Object Path': point.dataPoint.path.join('.'), Property: point.name, - "Property Description": point.description, - "Data Categories": point.categories + 'Property Description': point.description, + 'Data Categories': point.categories .map((category) => `${category.category}:${category.name}`) - .join(", "), - "Guessed Category": point.pendingCategoryGuesses?.[0] + .join(', '), + 'Guessed Category': point.pendingCategoryGuesses?.[0] ? `${point.pendingCategoryGuesses[0].category.category}:${point.pendingCategoryGuesses[0].category.name}` - : "", - "Processing Purposes": point.purposes + : '', + 'Processing Purposes': point.purposes .map((purpose) => `${purpose.purpose}:${purpose.name}`) - .join(", "), + .join(', '), ...Object.entries( groupBy( point.attributeValues || [], - ({ attributeKey }) => attributeKey.name - ) + ({ attributeKey }) => attributeKey.name, + ), ).reduce>((accumulator, [key, values]) => { - accumulator[key] = values.map((value) => value.name).join(","); + accumulator[key] = values.map((value) => value.name).join(','); return accumulator; }, {}), }; @@ -79,7 +79,7 @@ export async function pullDatapoints( writeCsv(file, inputs, headers); } catch (error) { logger.error( - colors.red(`An error occurred syncing the datapoints: ${error.message}`) + colors.red(`An error occurred syncing the datapoints: ${error.message}`), ); process.exit(1); } @@ -87,7 +87,7 @@ export async function pullDatapoints( // Indicate success logger.info( colors.green( - `Successfully synced datapoints to disk at ${file}! View at ${ADMIN_DASH_DATAPOINTS}` - ) + `Successfully synced datapoints to disk at ${file}! View at ${ADMIN_DASH_DATAPOINTS}`, + ), ); } diff --git a/src/commands/inventory/pull-unstructured-discovery-files/impl.ts b/src/commands/inventory/pull-unstructured-discovery-files/impl.ts index 001ed4b0..9c6026f3 100644 --- a/src/commands/inventory/pull-unstructured-discovery-files/impl.ts +++ b/src/commands/inventory/pull-unstructured-discovery-files/impl.ts @@ -1,11 +1,11 @@ -import type { UnstructuredSubDataPointRecommendationStatus } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { uniq } from "lodash-es"; -import type { LocalContext } from "../../../context"; -import { writeCsv } from "../../../lib/cron"; -import { pullUnstructuredSubDataPointRecommendations } from "../../../lib/data-inventory"; -import { buildTranscendGraphQLClient } from "../../../lib/graphql"; -import { logger } from "../../../logger"; +import type { UnstructuredSubDataPointRecommendationStatus } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { uniq } from 'lodash-es'; +import type { LocalContext } from '../../../context'; +import { writeCsv } from '../../../lib/cron'; +import { pullUnstructuredSubDataPointRecommendations } from '../../../lib/data-inventory'; +import { buildTranscendGraphQLClient } from '../../../lib/graphql'; +import { logger } from '../../../logger'; interface PullUnstructuredDiscoveryFilesCommandFlags { auth: string; @@ -27,7 +27,7 @@ export async function pullUnstructuredDiscoveryFiles( subCategories, status, includeEncryptedSnippets, - }: PullUnstructuredDiscoveryFilesCommandFlags + }: PullUnstructuredDiscoveryFilesCommandFlags, ): Promise { try { // Create a GraphQL client @@ -42,24 +42,24 @@ export async function pullUnstructuredDiscoveryFiles( logger.info( colors.magenta( - `Writing unstructured discovery files to file "${file}"...` - ) + `Writing unstructured discovery files to file "${file}"...`, + ), ); let headers: string[] = []; const inputs = entries.map((entry) => { const result = { - "Entry ID": entry.id, - "Data Silo ID": entry.dataSiloId, - "Object Path ID": entry.scannedObjectPathId, - "Object ID": entry.scannedObjectId, + 'Entry ID': entry.id, + 'Data Silo ID': entry.dataSiloId, + 'Object Path ID': entry.scannedObjectPathId, + 'Object ID': entry.scannedObjectId, ...(includeEncryptedSnippets - ? { Entry: entry.name, "Context Snippet": entry.contextSnippet } + ? { Entry: entry.name, 'Context Snippet': entry.contextSnippet } : {}), - "Data Category": `${entry.dataSubCategory.category}:${entry.dataSubCategory.name}`, - "Classification Status": entry.status, - "Confidence Score": entry.confidence, - "Classification Method": entry.classificationMethod, - "Classifier Version": entry.classifierVersion, + 'Data Category': `${entry.dataSubCategory.category}:${entry.dataSubCategory.name}`, + 'Classification Status': entry.status, + 'Confidence Score': entry.confidence, + 'Classification Method': entry.classificationMethod, + 'Classifier Version': entry.classifierVersion, }; headers = uniq([...headers, ...Object.keys(result)]); return result; @@ -68,8 +68,8 @@ export async function pullUnstructuredDiscoveryFiles( } catch (error) { logger.error( colors.red( - `An error occurred syncing the unstructured discovery files: ${error.message}` - ) + `An error occurred syncing the unstructured discovery files: ${error.message}`, + ), ); process.exit(1); } @@ -77,7 +77,7 @@ export async function pullUnstructuredDiscoveryFiles( // Indicate success logger.info( colors.green( - `Successfully synced unstructured discovery files to disk at ${file}!` - ) + `Successfully synced unstructured discovery files to disk at ${file}!`, + ), ); } diff --git a/src/commands/inventory/pull/impl.ts b/src/commands/inventory/pull/impl.ts index 799d574b..a6802e91 100644 --- a/src/commands/inventory/pull/impl.ts +++ b/src/commands/inventory/pull/impl.ts @@ -1,26 +1,26 @@ -import fs from "node:fs"; -import { join } from "node:path"; -import { ConsentTrackerStatus } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { ADMIN_DASH_INTEGRATIONS } from "../../../constants"; -import type { LocalContext } from "../../../context"; -import { TranscendPullResource } from "../../../enums"; -import { validateTranscendAuth } from "../../../lib/api-keys"; -import { mapSeries } from "../../../lib/bluebird-replace"; +import fs from 'node:fs'; +import { join } from 'node:path'; +import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; +import type { LocalContext } from '../../../context'; +import { TranscendPullResource } from '../../../enums'; +import { validateTranscendAuth } from '../../../lib/api-keys'; +import { mapSeries } from '../../../lib/bluebird-replace'; import { buildTranscendGraphQLClient, pullTranscendConfiguration, -} from "../../../lib/graphql"; -import { writeTranscendYaml } from "../../../lib/readTranscendYaml"; -import { logger } from "../../../logger"; +} from '../../../lib/graphql'; +import { writeTranscendYaml } from '../../../lib/readTranscendYaml'; +import { logger } from '../../../logger'; import { DEFAULT_CONSENT_TRACKER_STATUSES, DEFAULT_TRANSCEND_PULL_RESOURCES, -} from "./command"; +} from './command'; interface PullCommandFlags { auth: string; - resources?: (TranscendPullResource | "all")[]; + resources?: (TranscendPullResource | 'all')[]; file: string; transcendUrl: string; dataSiloIds?: string[]; @@ -48,17 +48,17 @@ export async function pull( skipSubDatapoints, includeGuessedCategories, debug, - }: PullCommandFlags + }: PullCommandFlags, ): Promise { // Parse authentication as API key or path to list of API keys const apiKeyOrList = await validateTranscendAuth(auth); - const resourcesToPull: TranscendPullResource[] = resources.includes("all") + const resourcesToPull: TranscendPullResource[] = resources.includes('all') ? Object.values(TranscendPullResource) : (resources as TranscendPullResource[]); // Sync to Disk - if (typeof apiKeyOrList === "string") { + if (typeof apiKeyOrList === 'string') { try { // Create a GraphQL client const client = buildTranscendGraphQLClient(transcendUrl, apiKeyOrList); @@ -82,8 +82,8 @@ export async function pull( colors.red( `An error occurred syncing the schema: ${ debug ? error.stack : error.message - }` - ) + }`, + ), ); process.exit(1); } @@ -91,13 +91,13 @@ export async function pull( // Indicate success logger.info( colors.green( - `Successfully synced yaml file to disk at ${file}! View at ${ADMIN_DASH_INTEGRATIONS}` - ) + `Successfully synced yaml file to disk at ${file}! View at ${ADMIN_DASH_INTEGRATIONS}`, + ), ); } else { if (!fs.lstatSync(file).isDirectory()) { throw new Error( - "File is expected to be a folder when passing in a list of API keys to pull from. e.g. --file=./working/" + 'File is expected to be a folder when passing in a list of API keys to pull from. e.g. --file=./working/', ); } @@ -108,8 +108,8 @@ export async function pull( }] `; logger.info( colors.magenta( - `~~~\n\n${prefix}Attempting to pull configuration...\n\n~~~` - ) + `~~~\n\n${prefix}Attempting to pull configuration...\n\n~~~`, + ), ); // Create a GraphQL client @@ -130,18 +130,18 @@ export async function pull( const filePath = join(file, `${apiKey.organizationName}.yml`); logger.info( - colors.magenta(`Writing configuration to file "${filePath}"...`) + colors.magenta(`Writing configuration to file "${filePath}"...`), ); writeTranscendYaml(filePath, configuration); logger.info( - colors.green(`${prefix}Successfully pulled configuration!`) + colors.green(`${prefix}Successfully pulled configuration!`), ); } catch (error) { logger.error( colors.red( - `${prefix}Failed to sync configuration. - ${error.message}` - ) + `${prefix}Failed to sync configuration. - ${error.message}`, + ), ); encounteredErrors.push(apiKey.organizationName); } @@ -151,9 +151,9 @@ export async function pull( logger.info( colors.red( `Sync encountered errors for "${encounteredErrors.join( - "," - )}". View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}` - ) + ',', + )}". View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}`, + ), ); process.exit(1); diff --git a/src/commands/inventory/push/impl.ts b/src/commands/inventory/push/impl.ts index d95b7633..3ba81b28 100644 --- a/src/commands/inventory/push/impl.ts +++ b/src/commands/inventory/push/impl.ts @@ -1,19 +1,19 @@ -import { existsSync, lstatSync } from "node:fs"; -import { join } from "node:path"; -import colors from "colors"; -import { TranscendInput } from "../../../codecs"; -import { ADMIN_DASH_INTEGRATIONS } from "../../../constants"; -import type { LocalContext } from "../../../context"; -import { listFiles, validateTranscendAuth } from "../../../lib/api-keys"; -import { mapSeries } from "../../../lib/bluebird-replace"; +import { existsSync, lstatSync } from 'node:fs'; +import { join } from 'node:path'; +import colors from 'colors'; +import { TranscendInput } from '../../../codecs'; +import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; +import type { LocalContext } from '../../../context'; +import { listFiles, validateTranscendAuth } from '../../../lib/api-keys'; +import { mapSeries } from '../../../lib/bluebird-replace'; import { buildTranscendGraphQLClient, syncConfigurationToTranscend, -} from "../../../lib/graphql"; -import { parseVariablesFromString } from "../../../lib/helpers/parseVariablesFromString"; -import { mergeTranscendInputs } from "../../../lib/mergeTranscendInputs"; -import { readTranscendYaml } from "../../../lib/readTranscendYaml"; -import { logger } from "../../../logger"; +} from '../../../lib/graphql'; +import { parseVariablesFromString } from '../../../lib/helpers/parseVariablesFromString'; +import { mergeTranscendInputs } from '../../../lib/mergeTranscendInputs'; +import { readTranscendYaml } from '../../../lib/readTranscendYaml'; +import { logger } from '../../../logger'; /** * Sync configuration to Transcend @@ -57,14 +57,14 @@ async function syncConfiguration({ publishToPrivacyCenter, classifyService, deleteExtraAttributeValues, - } + }, ); return !encounteredError; } catch (error) { logger.error( colors.red( - `An unexpected error occurred syncing the schema: ${error.message}` - ) + `An unexpected error occurred syncing the schema: ${error.message}`, + ), ); return false; } @@ -84,7 +84,7 @@ interface PushCommandFlags { export async function push( this: LocalContext, { - file = "./transcend.yml", + file = './transcend.yml', transcendUrl, auth, variables, @@ -92,7 +92,7 @@ export async function push( publishToPrivacyCenter, classifyService, deleteExtraAttributeValues, - }: PushCommandFlags + }: PushCommandFlags, ): Promise { // Parse authentication as API key or path to list of API keys const apiKeyOrList = await validateTranscendAuth(auth); @@ -105,11 +105,11 @@ export async function push( fileList = Array.isArray(apiKeyOrList) && lstatSync(file).isDirectory() ? listFiles(file).map((filePath) => join(file, filePath)) - : file.split(","); + : file.split(','); // Ensure at least one file is parsed if (fileList.length === 0) { - throw new Error("No file specified!"); + throw new Error('No file specified!'); } const transcendInputs = fileList.map((filePath) => { @@ -119,8 +119,8 @@ export async function push( } else { logger.error( colors.red( - `The file path does not exist on disk: ${filePath}. You can specify the filepath using --file=./examples/transcend.yml` - ) + `The file path does not exist on disk: ${filePath}. You can specify the filepath using --file=./examples/transcend.yml`, + ), ); process.exit(1); } @@ -131,20 +131,20 @@ export async function push( logger.info(colors.green(`Successfully read in "${filePath}"`)); return { content: newContents, - name: filePath.split("/").pop()!.replace(".yml", ""), + name: filePath.split('/').pop()!.replace('.yml', ''), }; } catch (error) { logger.error( colors.red( - `The shape of your yaml file is invalid with the following errors: ${error.message}` - ) + `The shape of your yaml file is invalid with the following errors: ${error.message}`, + ), ); process.exit(1); } }); // process a single API key - if (typeof apiKeyOrList === "string") { + if (typeof apiKeyOrList === 'string') { // if passed multiple inputs, merge them together const [base, ...rest] = transcendInputs.map(({ content }) => content); const contents = mergeTranscendInputs(base, ...rest); @@ -164,8 +164,8 @@ export async function push( if (!success) { logger.info( colors.red( - `Sync encountered errors. View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}` - ) + `Sync encountered errors. View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}`, + ), ); process.exit(1); @@ -177,12 +177,12 @@ export async function push( transcendInputs.length !== apiKeyOrList.length ) { throw new Error( - "Expected list of yml files to be equal to the list of API keys." + + 'Expected list of yml files to be equal to the list of API keys.' + `Got ${transcendInputs.length} YML file${ - transcendInputs.length === 1 ? "" : "s" + transcendInputs.length === 1 ? '' : 's' } and ${apiKeyOrList.length} API key${ - apiKeyOrList.length === 1 ? "" : "s" - }` + apiKeyOrList.length === 1 ? '' : 's' + }`, ); } @@ -193,8 +193,8 @@ export async function push( }] `; logger.info( colors.magenta( - `~~~\n\n${prefix}Attempting to push configuration...\n\n~~~` - ) + `~~~\n\n${prefix}Attempting to push configuration...\n\n~~~`, + ), ); // use the merged contents if 1 yml passed, else use the contents that map to that organization @@ -202,15 +202,15 @@ export async function push( transcendInputs.length === 1 ? transcendInputs[0].content : transcendInputs.find( - (input) => input.name === apiKey.organizationName + (input) => input.name === apiKey.organizationName, )?.content; // Throw error if cannot find a yml file matching that organization name if (!useContents) { logger.error( colors.red( - `${prefix}Failed to find transcend.yml file for organization: "${apiKey.organizationName}".` - ) + `${prefix}Failed to find transcend.yml file for organization: "${apiKey.organizationName}".`, + ), ); encounteredErrors.push(apiKey.organizationName); return; @@ -228,7 +228,7 @@ export async function push( if (success) { logger.info( - colors.green(`${prefix}Successfully pushed configuration!`) + colors.green(`${prefix}Successfully pushed configuration!`), ); } else { logger.error(colors.red(`${prefix}Failed to sync configuration.`)); @@ -240,9 +240,9 @@ export async function push( logger.info( colors.red( `Sync encountered errors for "${encounteredErrors.join( - "," - )}". View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}` - ) + ',', + )}". View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}`, + ), ); process.exit(1); @@ -252,7 +252,7 @@ export async function push( // Indicate success logger.info( colors.green( - `Successfully synced yaml file to Transcend! View at ${ADMIN_DASH_INTEGRATIONS}` - ) + `Successfully synced yaml file to Transcend! View at ${ADMIN_DASH_INTEGRATIONS}`, + ), ); } diff --git a/src/commands/inventory/scan-packages/impl.ts b/src/commands/inventory/scan-packages/impl.ts index c2198f90..6b6abe12 100644 --- a/src/commands/inventory/scan-packages/impl.ts +++ b/src/commands/inventory/scan-packages/impl.ts @@ -1,17 +1,17 @@ -import { execSync } from "node:child_process"; -import colors from "colors"; -import { ADMIN_DASH } from "../../../constants"; -import type { LocalContext } from "../../../context"; -import { findCodePackagesInFolder } from "../../../lib/code-scanning"; +import { execSync } from 'node:child_process'; +import colors from 'colors'; +import { ADMIN_DASH } from '../../../constants'; +import type { LocalContext } from '../../../context'; +import { findCodePackagesInFolder } from '../../../lib/code-scanning'; import { buildTranscendGraphQLClient, syncCodePackages, -} from "../../../lib/graphql"; -import { logger } from "../../../logger"; +} from '../../../lib/graphql'; +import { logger } from '../../../logger'; const REPO_ERROR = - "A repository name must be provided. " + - "You can specify using --repositoryName=$REPO_NAME or by ensuring the " + + 'A repository name must be provided. ' + + 'You can specify using --repositoryName=$REPO_NAME or by ensuring the ' + 'command "git config --get remote.origin.url" returns the name of the repository'; interface ScanPackagesCommandFlags { @@ -30,20 +30,20 @@ export async function scanPackages( ignoreDirs, repositoryName, transcendUrl, - }: ScanPackagesCommandFlags + }: ScanPackagesCommandFlags, ): Promise { // Ensure repository name is specified let gitRepositoryName = repositoryName; if (!gitRepositoryName) { try { const name = execSync( - `cd ${scanPath} && git config --get remote.origin.url` + `cd ${scanPath} && git config --get remote.origin.url`, ); // Trim and parse the URL - const url = name.toString("utf-8").trim(); - [gitRepositoryName] = url.includes("https:") - ? url.split("/").slice(3).join("/").split(".") - : (url.split(":").pop() || "").split("."); + const url = name.toString('utf-8').trim(); + [gitRepositoryName] = url.includes('https:') + ? url.split('/').slice(3).join('/').split('.') + : (url.split(':').pop() || '').split('.'); if (!gitRepositoryName) { logger.error(colors.red(REPO_ERROR)); process.exit(1); @@ -68,13 +68,13 @@ export async function scanPackages( await syncCodePackages(client, results); const newUrl = new URL(ADMIN_DASH); - newUrl.pathname = "/code-scanning/code-packages"; + newUrl.pathname = '/code-scanning/code-packages'; // Indicate success logger.info( colors.green( `Scan found ${results.length} packages at ${scanPath}! ` + - `View results at '${newUrl.href}'` - ) + `View results at '${newUrl.href}'`, + ), ); } diff --git a/src/commands/migration/sync-ot/impl.ts b/src/commands/migration/sync-ot/impl.ts index 443e91e1..fd8914f6 100644 --- a/src/commands/migration/sync-ot/impl.ts +++ b/src/commands/migration/sync-ot/impl.ts @@ -1,17 +1,17 @@ -import colors from "colors"; -import type { LocalContext } from "../../../context"; +import colors from 'colors'; +import type { LocalContext } from '../../../context'; import { OneTrustFileFormat, OneTrustPullResource, OneTrustPullSource, -} from "../../../enums"; -import { buildTranscendGraphQLClient } from "../../../lib/graphql"; -import { createOneTrustGotInstance } from "../../../lib/oneTrust"; +} from '../../../enums'; +import { buildTranscendGraphQLClient } from '../../../lib/graphql'; +import { createOneTrustGotInstance } from '../../../lib/oneTrust'; import { syncOneTrustAssessmentsFromFile, syncOneTrustAssessmentsFromOneTrust, -} from "../../../lib/oneTrust/helpers"; -import { logger } from "../../../logger"; +} from '../../../lib/oneTrust/helpers'; +import { logger } from '../../../logger'; // Command flag interface interface SyncOtCommandFlags { @@ -39,34 +39,34 @@ export async function syncOt( file, dryRun, debug, - }: SyncOtCommandFlags + }: SyncOtCommandFlags, ): Promise { // Must be able to authenticate to transcend to sync resources to it if (!dryRun && !transcendAuth) { throw new Error( - 'Must specify a "transcendAuth" parameter to sync resources to Transcend. e.g. --transcendAuth=${TRANSCEND_API_KEY}' + 'Must specify a "transcendAuth" parameter to sync resources to Transcend. e.g. --transcendAuth=${TRANSCEND_API_KEY}', ); } // If trying to sync to disk, must specify a file path if (dryRun && !file) { throw new Error( - 'Must set a "file" parameter when "dryRun" is "true". e.g. --file=./oneTrustAssessments.json' + 'Must set a "file" parameter when "dryRun" is "true". e.g. --file=./oneTrustAssessments.json', ); } if (file) { - const splitFile = file.split("."); + const splitFile = file.split('.'); if (splitFile.length < 2) { throw new Error( - 'The "file" parameter has an invalid format. Expected a path with extensions. e.g. --file=./pathToFile.json.' + 'The "file" parameter has an invalid format. Expected a path with extensions. e.g. --file=./pathToFile.json.', ); } if (splitFile.at(-1) !== OneTrustFileFormat.Json) { throw new Error( `Expected the format of the "file" parameters '${file}' to be '${ OneTrustFileFormat.Json - }', but got '${splitFile.at(-1)}'.` + }', but got '${splitFile.at(-1)}'.`, ); } } @@ -76,28 +76,28 @@ export async function syncOt( // must specify the OneTrust hostname if (!hostname) { throw new Error( - 'Missing required parameter "hostname". e.g. --hostname=customer.my.onetrust.com' + 'Missing required parameter "hostname". e.g. --hostname=customer.my.onetrust.com', ); } // must specify the OneTrust auth if (!oneTrustAuth) { throw new Error( - 'Missing required parameter "oneTrustAuth". e.g. --oneTrustAuth=$ONE_TRUST_AUTH_TOKEN' + 'Missing required parameter "oneTrustAuth". e.g. --oneTrustAuth=$ONE_TRUST_AUTH_TOKEN', ); } } else { // if reading the assessments from a file, must specify a file to read from if (!file) { throw new Error( - 'Must specify a "file" parameter to read the OneTrust assessments from. e.g. --source=./oneTrustAssessments.json' + 'Must specify a "file" parameter to read the OneTrust assessments from. e.g. --source=./oneTrustAssessments.json', ); } // Cannot try reading from file and save assessments to a file simultaneously if (dryRun) { throw new Error( - "Cannot read and write to a file simultaneously." + - ` Emit the "source" parameter or set it to ${OneTrustPullSource.OneTrust} if "dryRun" is enabled.` + 'Cannot read and write to a file simultaneously.' + + ` Emit the "source" parameter or set it to ${OneTrustPullSource.OneTrust} if "dryRun" is enabled.`, ); } } @@ -134,7 +134,7 @@ export async function syncOt( throw new Error( `An error occurred syncing the resource ${resource} from OneTrust: ${ debug ? error.stack : error.message - }` + }`, ); } @@ -142,8 +142,8 @@ export async function syncOt( logger.info( colors.green( `Successfully synced OneTrust ${resource} to ${ - dryRun ? `disk at "${file}"` : "Transcend" - }!` - ) + dryRun ? `disk at "${file}"` : 'Transcend' + }!`, + ), ); } diff --git a/src/commands/request/cron/pull-identifiers/impl.ts b/src/commands/request/cron/pull-identifiers/impl.ts index 140efaf2..2f1dd235 100644 --- a/src/commands/request/cron/pull-identifiers/impl.ts +++ b/src/commands/request/cron/pull-identifiers/impl.ts @@ -1,14 +1,14 @@ -import { RequestAction } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { uniq } from "lodash-es"; -import type { LocalContext } from "../../../../context"; +import { RequestAction } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { uniq } from 'lodash-es'; +import type { LocalContext } from '../../../../context'; import { CsvFormattedIdentifier, parseFilePath, pullChunkedCustomSiloOutstandingIdentifiers, writeCsv, -} from "../../../../lib/cron"; -import { logger } from "../../../../logger"; +} from '../../../../lib/cron'; +import { logger } from '../../../../logger'; interface PullIdentifiersCommandFlags { file: string; @@ -34,13 +34,13 @@ export async function pullIdentifiers( pageLimit, skipRequestCount, chunkSize, - }: PullIdentifiersCommandFlags + }: PullIdentifiersCommandFlags, ): Promise { if (skipRequestCount) { logger.info( colors.yellow( - "Skipping request count as requested. This may help speed up the call." - ) + 'Skipping request count as requested. This may help speed up the call.', + ), ); } @@ -51,8 +51,8 @@ export async function pullIdentifiers( ) { logger.error( colors.red( - `Invalid chunk size: "${chunkSize}". Must be a positive integer that is a multiple of ${pageLimit}.` - ) + `Invalid chunk size: "${chunkSize}". Must be a positive integer that is a multiple of ${pageLimit}.`, + ), ); process.exit(1); } @@ -64,16 +64,16 @@ export async function pullIdentifiers( const numberedFileName = `${baseName}-${fileCount}${extension}`; logger.info( colors.blue( - `Saving ${chunk.length} identifiers to file "${numberedFileName}"` - ) + `Saving ${chunk.length} identifiers to file "${numberedFileName}"`, + ), ); const headers = uniq(chunk.flatMap((d) => Object.keys(d))); writeCsv(numberedFileName, chunk, headers); logger.info( colors.green( - `Successfully wrote ${chunk.length} identifiers to file "${numberedFileName}"` - ) + `Successfully wrote ${chunk.length} identifiers to file "${numberedFileName}"`, + ), ); fileCount += 1; return Promise.resolve(); diff --git a/src/commands/request/cron/pull-profiles/impl.ts b/src/commands/request/cron/pull-profiles/impl.ts index 3f800a1d..c6ff6848 100644 --- a/src/commands/request/cron/pull-profiles/impl.ts +++ b/src/commands/request/cron/pull-profiles/impl.ts @@ -1,19 +1,19 @@ -import type { RequestAction } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { uniq } from "lodash-es"; -import type { LocalContext } from "../../../../context"; -import { map } from "../../../../lib/bluebird-replace"; +import type { RequestAction } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { uniq } from 'lodash-es'; +import type { LocalContext } from '../../../../context'; +import { map } from '../../../../lib/bluebird-replace'; import { parseFilePath, pullChunkedCustomSiloOutstandingIdentifiers, writeCsv, type CsvFormattedIdentifier, -} from "../../../../lib/cron"; +} from '../../../../lib/cron'; import { buildTranscendGraphQLClient, fetchRequestFilesForRequest, -} from "../../../../lib/graphql"; -import { logger } from "../../../../logger"; +} from '../../../../lib/graphql'; +import { logger } from '../../../../logger'; interface PullProfilesCommandFlags { file: string; @@ -43,13 +43,13 @@ export async function pullProfiles( skipRequestCount, pageLimit, chunkSize, - }: PullProfilesCommandFlags + }: PullProfilesCommandFlags, ): Promise { if (skipRequestCount) { logger.info( colors.yellow( - "Skipping request count as requested. This may help speed up the call." - ) + 'Skipping request count as requested. This may help speed up the call.', + ), ); } @@ -60,8 +60,8 @@ export async function pullProfiles( ) { logger.error( colors.red( - `Invalid chunk size: "${chunkSize}". Must be a positive integer that is a multiple of ${pageLimit}.` - ) + `Invalid chunk size: "${chunkSize}". Must be a positive integer that is a multiple of ${pageLimit}.`, + ), ); process.exit(1); } @@ -95,24 +95,24 @@ export async function pullProfiles( return results.map(({ fileName, remoteId }) => { if (!remoteId) { throw new Error( - `Failed to find remoteId for ${fileName} request: ${requestId}` + `Failed to find remoteId for ${fileName} request: ${requestId}`, ); } return { RecordId: remoteId, Object: fileName - .replace(".json", "") - .split("/") + .replace('.json', '') + .split('/') .pop() - ?.replace(" Information", ""), + ?.replace(' Information', ''), Comment: - "Customer data deletion request submitted via transcend.io", + 'Customer data deletion request submitted via transcend.io', }; }); }, { concurrency: 10, - } + }, ); allTargetIdentifiersCount += results.flat().length; @@ -124,8 +124,8 @@ export async function pullProfiles( writeCsv(numberedFileName, chunk, headers); logger.info( colors.green( - `Successfully wrote ${chunk.length} identifiers to file "${file}"` - ) + `Successfully wrote ${chunk.length} identifiers to file "${file}"`, + ), ); const targetIdentifiers = results.flat(); @@ -133,14 +133,14 @@ export async function pullProfiles( writeCsv(numberedFileNameTarget, targetIdentifiers, headers2); logger.info( colors.green( - `Successfully wrote ${targetIdentifiers.length} identifiers to file "${fileTarget}"` - ) + `Successfully wrote ${targetIdentifiers.length} identifiers to file "${fileTarget}"`, + ), ); logger.info( colors.blue( - `Processed chunk of ${chunk.length} identifiers, found ${targetIdentifiers.length} target identifiers` - ) + `Processed chunk of ${chunk.length} identifiers, found ${targetIdentifiers.length} target identifiers`, + ), ); fileCount += 1; }; @@ -160,12 +160,12 @@ export async function pullProfiles( logger.info( colors.green( - `Successfully wrote ${allIdentifiersCount} identifiers to file "${file}"` - ) + `Successfully wrote ${allIdentifiersCount} identifiers to file "${file}"`, + ), ); logger.info( colors.green( - `Successfully wrote ${allTargetIdentifiersCount} identifiers to file "${fileTarget}"` - ) + `Successfully wrote ${allTargetIdentifiersCount} identifiers to file "${fileTarget}"`, + ), ); } diff --git a/src/commands/request/export/impl.ts b/src/commands/request/export/impl.ts index c40bb266..1e538445 100644 --- a/src/commands/request/export/impl.ts +++ b/src/commands/request/export/impl.ts @@ -1,10 +1,10 @@ -import type { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { uniq } from "lodash-es"; -import type { LocalContext } from "../../../context"; -import { writeCsv } from "../../../lib/cron"; -import { pullPrivacyRequests } from "../../../lib/requests"; -import { logger } from "../../../logger"; +import type { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { uniq } from 'lodash-es'; +import type { LocalContext } from '../../../context'; +import { writeCsv } from '../../../lib/cron'; +import { pullPrivacyRequests } from '../../../lib/requests'; +import { logger } from '../../../logger'; interface ExportCommandFlags { auth: string; @@ -35,7 +35,7 @@ export async function _export( createdAtBefore, createdAtAfter, showTests, - }: ExportCommandFlags + }: ExportCommandFlags, ): Promise { const { requestsFormattedForCsv } = await pullPrivacyRequests({ transcendUrl, @@ -54,7 +54,7 @@ export async function _export( writeCsv(file, requestsFormattedForCsv, headers); logger.info( colors.green( - `Successfully wrote ${requestsFormattedForCsv.length} requests to file "${file}"` - ) + `Successfully wrote ${requestsFormattedForCsv.length} requests to file "${file}"`, + ), ); } diff --git a/src/constants.ts b/src/constants.ts index 42fd2711..dbead72b 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,11 +1,11 @@ -import { ScopeName } from "@transcend-io/privacy-types"; -import { TranscendInput } from "./codecs"; -import { TranscendPullResource } from "./enums"; +import { ScopeName } from '@transcend-io/privacy-types'; +import { TranscendInput } from './codecs'; +import { TranscendPullResource } from './enums'; -export { description, version } from "../package.json"; -export const name = "transcend"; +export { description, version } from '../package.json'; +export const name = 'transcend'; -export const ADMIN_DASH = "https://app.transcend.io"; +export const ADMIN_DASH = 'https://app.transcend.io'; export const ADMIN_DASH_INTEGRATIONS = `${ADMIN_DASH}/infrastructure/integrations`; export const ADMIN_DASH_DATAPOINTS = `${ADMIN_DASH}/data-map/data-inventory/data-points`; @@ -15,14 +15,14 @@ export const ADMIN_DASH_DATAPOINTS = `${ADMIN_DASH}/data-map/data-inventory/data * TRANSCEND_API_URL=https://api.us.transcend.io transcend ... */ export const DEFAULT_TRANSCEND_API = - process.env.TRANSCEND_API_URL || "https://api.transcend.io"; + process.env.TRANSCEND_API_URL || 'https://api.transcend.io'; /** * Override default transcend API url using * TRANSCEND_CONSENT_API_URL=https://consent.us.transcend.io transcend ... */ export const DEFAULT_TRANSCEND_CONSENT_API = - process.env.TRANSCEND_CONSENT_API_URL || "https://consent.transcend.io"; + process.env.TRANSCEND_CONSENT_API_URL || 'https://consent.transcend.io'; /** * Mapping between resource type and scopes required for cli @@ -133,35 +133,35 @@ export const TR_YML_RESOURCE_TO_FIELD_NAME: Record< TranscendPullResource, keyof TranscendInput > = { - [TranscendPullResource.ApiKeys]: "api-keys", - [TranscendPullResource.Attributes]: "attributes", - [TranscendPullResource.DataFlows]: "data-flows", - [TranscendPullResource.Cookies]: "cookies", - [TranscendPullResource.ConsentManager]: "consent-manager", - [TranscendPullResource.Partitions]: "partitions", - [TranscendPullResource.Actions]: "actions", - [TranscendPullResource.DataSubjects]: "data-subjects", - [TranscendPullResource.BusinessEntities]: "business-entities", - [TranscendPullResource.Identifiers]: "identifiers", - [TranscendPullResource.Enrichers]: "enrichers", - [TranscendPullResource.DataSilos]: "data-silos", - [TranscendPullResource.Templates]: "templates", - [TranscendPullResource.Prompts]: "prompts", - [TranscendPullResource.PromptPartials]: "prompt-partials", - [TranscendPullResource.PromptGroups]: "prompt-groups", - [TranscendPullResource.Agents]: "agents", - [TranscendPullResource.AgentFunctions]: "agent-functions", - [TranscendPullResource.AgentFiles]: "agent-files", - [TranscendPullResource.Vendors]: "vendors", - [TranscendPullResource.DataCategories]: "data-categories", - [TranscendPullResource.ProcessingPurposes]: "processing-purposes", - [TranscendPullResource.ActionItems]: "action-items", - [TranscendPullResource.ActionItemCollections]: "action-item-collections", - [TranscendPullResource.Teams]: "teams", - [TranscendPullResource.Messages]: "messages", - [TranscendPullResource.PrivacyCenters]: "privacy-center", - [TranscendPullResource.Policies]: "policies", - [TranscendPullResource.Assessments]: "assessments", - [TranscendPullResource.AssessmentTemplates]: "assessment-templates", - [TranscendPullResource.Purposes]: "purposes", + [TranscendPullResource.ApiKeys]: 'api-keys', + [TranscendPullResource.Attributes]: 'attributes', + [TranscendPullResource.DataFlows]: 'data-flows', + [TranscendPullResource.Cookies]: 'cookies', + [TranscendPullResource.ConsentManager]: 'consent-manager', + [TranscendPullResource.Partitions]: 'partitions', + [TranscendPullResource.Actions]: 'actions', + [TranscendPullResource.DataSubjects]: 'data-subjects', + [TranscendPullResource.BusinessEntities]: 'business-entities', + [TranscendPullResource.Identifiers]: 'identifiers', + [TranscendPullResource.Enrichers]: 'enrichers', + [TranscendPullResource.DataSilos]: 'data-silos', + [TranscendPullResource.Templates]: 'templates', + [TranscendPullResource.Prompts]: 'prompts', + [TranscendPullResource.PromptPartials]: 'prompt-partials', + [TranscendPullResource.PromptGroups]: 'prompt-groups', + [TranscendPullResource.Agents]: 'agents', + [TranscendPullResource.AgentFunctions]: 'agent-functions', + [TranscendPullResource.AgentFiles]: 'agent-files', + [TranscendPullResource.Vendors]: 'vendors', + [TranscendPullResource.DataCategories]: 'data-categories', + [TranscendPullResource.ProcessingPurposes]: 'processing-purposes', + [TranscendPullResource.ActionItems]: 'action-items', + [TranscendPullResource.ActionItemCollections]: 'action-item-collections', + [TranscendPullResource.Teams]: 'teams', + [TranscendPullResource.Messages]: 'messages', + [TranscendPullResource.PrivacyCenters]: 'privacy-center', + [TranscendPullResource.Policies]: 'policies', + [TranscendPullResource.Assessments]: 'assessments', + [TranscendPullResource.AssessmentTemplates]: 'assessment-templates', + [TranscendPullResource.Purposes]: 'purposes', }; diff --git a/src/lib/ai/TranscendPromptManager.ts b/src/lib/ai/TranscendPromptManager.ts index 7fed5f5d..86571d25 100644 --- a/src/lib/ai/TranscendPromptManager.ts +++ b/src/lib/ai/TranscendPromptManager.ts @@ -1,28 +1,28 @@ -import type { Handlebars } from "@transcend-io/handlebars-utils"; +import type { Handlebars } from '@transcend-io/handlebars-utils'; import { createHandlebarsWithHelpers, HandlebarsInput, -} from "@transcend-io/handlebars-utils"; +} from '@transcend-io/handlebars-utils'; import { ChatCompletionRole, LargeLanguageModelClient, PromptRunProductArea, PromptStatus, QueueStatus, -} from "@transcend-io/privacy-types"; -import { Secret } from "@transcend-io/secret-value"; +} from '@transcend-io/privacy-types'; +import { Secret } from '@transcend-io/secret-value'; import { apply, decodeCodec, getValues, Optionalize, Requirize, -} from "@transcend-io/type-utils"; -import { GraphQLClient } from "graphql-request"; -import * as t from "io-ts"; -import { chunk, groupBy, keyBy, uniq } from "lodash-es"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { mapSeries } from "../bluebird-replace"; +} from '@transcend-io/type-utils'; +import { GraphQLClient } from 'graphql-request'; +import * as t from 'io-ts'; +import { chunk, groupBy, keyBy, uniq } from 'lodash-es'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { mapSeries } from '../bluebird-replace'; import { Agent, AgentFile, @@ -32,27 +32,27 @@ import { fetchAllAgents, reportPromptRun, ReportPromptRunInput, -} from "../graphql"; +} from '../graphql'; import { fetchAllLargeLanguageModels, LargeLanguageModel, -} from "../graphql/fetchLargeLanguageModels"; +} from '../graphql/fetchLargeLanguageModels'; import { fetchPromptsWithVariables, TranscendPromptsAndVariables, TranscendPromptTemplated, -} from "../graphql/fetchPrompts"; +} from '../graphql/fetchPrompts'; import { fetchAllPromptThreads, PromptThread, -} from "../graphql/fetchPromptThreads"; +} from '../graphql/fetchPromptThreads'; /** * An LLM Prompt definition */ export type TranscendPrompt< TInputParameters extends t.Any, - TOutputCodec extends t.Any + TOutputCodec extends t.Any, > = ( | { /** ID of the prompt */ @@ -108,7 +108,7 @@ export function createRegexForTag(tagName: string): RegExp { */ export function defineTranscendPrompts< TPromptNames extends string, - TPrompts extends Record> + TPrompts extends Record>, >(prompts: TPrompts): TPrompts { return prompts; } @@ -118,24 +118,24 @@ export function defineTranscendPrompts< */ export type GetPromptParamType< TPromptName extends keyof TPrompts, - TPrompts extends Record> -> = t.TypeOf; + TPrompts extends Record>, +> = t.TypeOf; /** * Helper to get the type of the parameter for a given prompt */ export type GetPromptResponseType< TPromptName extends keyof TPrompts, - TPrompts extends Record> -> = t.TypeOf; + TPrompts extends Record>, +> = t.TypeOf; /** * Input for reporting a prompt run */ export interface ReportPromptRunOptions extends Optionalize< - Omit, - "name" | "productArea" + Omit, + 'name' | 'productArea' > { /** The large language model being run. Either the ID of the LLM or the client/name pairing */ largeLanguageModel: @@ -162,7 +162,7 @@ const jsonParseSafe = (object: string): unknown => { */ export class TranscendPromptManager< TPromptNames extends string, - TPrompts extends Record> + TPrompts extends Record>, > { /** Prompt definitions */ public prompts: TPrompts; @@ -260,9 +260,9 @@ export class TranscendPromptManager< this.defaultVariables = defaultVariables; this.graphQLClient = buildTranscendGraphQLClient( transcendUrl, - typeof transcendApiKey === "object" + typeof transcendApiKey === 'object' ? transcendApiKey.release() - : transcendApiKey + : transcendApiKey, ); this.requireApproval = requireApproval; this.cacheDuration = cacheDuration; @@ -285,7 +285,7 @@ export class TranscendPromptManager< .map(({ title }) => title) .filter((x): x is string => !!x); const agentNames = uniq( - promptDefinitions.flatMap(({ agentNames }) => agentNames || []) + promptDefinitions.flatMap(({ agentNames }) => agentNames || []), ); // Fetch prompts and data @@ -297,15 +297,15 @@ export class TranscendPromptManager< fetchAllLargeLanguageModels(this.graphQLClient), fetchAllAgents(this.graphQLClient, { names: agentNames }), ]); - this.agentsByName = keyBy(agents, "name"); - this.agentsByAgentId = keyBy(agents, "agentId"); + this.agentsByName = keyBy(agents, 'name'); + this.agentsByAgentId = keyBy(agents, 'agentId'); this.largeLanguageModels = largeLanguageModels.filter( - (model) => !model.isTranscendHosted + (model) => !model.isTranscendHosted, ); // Lookup prompts by id/title - const promptByTitle = keyBy(response.prompts, "title"); - const promptById = keyBy(response.prompts, "id"); + const promptByTitle = keyBy(response.prompts, 'title'); + const promptById = keyBy(response.prompts, 'id'); // Update variables this.variables = { @@ -313,7 +313,7 @@ export class TranscendPromptManager< response.calculatedVariables.map((v) => [ v.name, v.data ? JSON.parse(v.data) : v.data, - ]) + ]), ), ...this.defaultVariables, }; @@ -335,11 +335,11 @@ export class TranscendPromptManager< const result = id ? promptById[id] : title - ? promptByTitle[title] - : undefined; + ? promptByTitle[title] + : undefined; if (!result) { throw new Error( - `Failed to find prompt with title: "${title}" and id: "${id}"` + `Failed to find prompt with title: "${title}" and id: "${id}"`, ); } return result; @@ -380,7 +380,7 @@ export class TranscendPromptManager< * @returns Large language model configuration */ async getPromptThreadBySlackTs( - ts: string + ts: string, ): Promise { const [thread] = await fetchAllPromptThreads(this.graphQLClient, { slackMessageTs: [ts], @@ -398,10 +398,10 @@ export class TranscendPromptManager< */ async getAgentsByName(names: string[]): Promise { if (names.length === 0) { - throw new Error("Expected at least one name to be provided"); + throw new Error('Expected at least one name to be provided'); } const { hasCache = [], missingCache = [] } = groupBy(names, (name) => - this.agentsByName[name] ? "hasCache" : "missingCache" + this.agentsByName[name] ? 'hasCache' : 'missingCache', ); const cachedAgents = hasCache.map((name) => this.agentsByName[name]); if (missingCache.length === 0) { @@ -439,21 +439,21 @@ export class TranscendPromptManager< * @returns Large language model configuration */ getLargeLanguageModel( - largeLanguageModel: ReportPromptRunOptions["largeLanguageModel"] + largeLanguageModel: ReportPromptRunOptions['largeLanguageModel'], ): LargeLanguageModel { const matching = this.largeLanguageModels.find((model) => - typeof largeLanguageModel === "string" + typeof largeLanguageModel === 'string' ? model.id === largeLanguageModel : model.name === largeLanguageModel.name && - model.client === largeLanguageModel.client + model.client === largeLanguageModel.client, ); if (!matching) { throw new Error( `Failed to find model matching: ${ - typeof largeLanguageModel === "string" + typeof largeLanguageModel === 'string' ? largeLanguageModel : JSON.stringify(largeLanguageModel) - }` + }`, ); } return matching; @@ -466,7 +466,7 @@ export class TranscendPromptManager< * @returns Parsed content */ async getPromptDefinition( - promptName: TPromptName + promptName: TPromptName, ): Promise { // Determine if prompts need to be fetched if ( @@ -484,12 +484,12 @@ export class TranscendPromptManager< // Lookup prompt const { promptContentMap } = this; if (!promptContentMap) { - throw new Error("Expected this.promptContentMap to be defined"); + throw new Error('Expected this.promptContentMap to be defined'); } const promptTemplate = promptContentMap[promptName]; if (!promptTemplate) { throw new Error( - `Expected this.promptContentMap[${promptName}] to be defined` + `Expected this.promptContentMap[${promptName}] to be defined`, ); } return promptTemplate; @@ -504,7 +504,7 @@ export class TranscendPromptManager< */ async compilePrompt( promptName: TPromptName, - parameters: t.TypeOf + parameters: t.TypeOf, ): Promise { // Grab the prompt const promptTemplate = await this.getPromptDefinition(promptName); @@ -519,14 +519,14 @@ export class TranscendPromptManager< promptTemplate.status !== PromptStatus.Approved ) { throw new Error( - `Assessment "${promptTemplate.title}" cannot be used because its in status: "${promptTemplate.status}"` + `Assessment "${promptTemplate.title}" cannot be used because its in status: "${promptTemplate.status}"`, ); } // If prompt is rejected, throw error if (promptTemplate.status === PromptStatus.Rejected) { throw new Error( - `Assessment "${promptTemplate.title}" cannot be used because it's in status: "${promptTemplate.status}"` + `Assessment "${promptTemplate.title}" cannot be used because it's in status: "${promptTemplate.status}"`, ); } @@ -551,8 +551,8 @@ export class TranscendPromptManager< */ parseAiResponse( promptName: TPromptName, - response: string - ): t.TypeOf { + response: string, + ): t.TypeOf { // Look up prompt info const promptInput = this.prompts[promptName]; if (!promptInput) { @@ -569,7 +569,7 @@ export class TranscendPromptManager< return decodeCodec( promptInput.outputCodec, jsonParseSafe(extracted), - false + false, ); } @@ -582,11 +582,11 @@ export class TranscendPromptManager< */ async reportAndParsePromptRun( promptName: TPromptName, - { largeLanguageModel, ...options }: ReportPromptRunOptions + { largeLanguageModel, ...options }: ReportPromptRunOptions, ): Promise< PromptRunResult & { /** Resulting prompt run */ - result: t.TypeOf; + result: t.TypeOf; } > { // Determine if prompts need to be fetched @@ -607,7 +607,7 @@ export class TranscendPromptManager< `@transcend-io/cli-prompt-run-${new Date().toISOString()}`; if (!this.promptContentMap) { - throw new Error("Expected this.promptContentMap to be defined"); + throw new Error('Expected this.promptContentMap to be defined'); } // Look up prompt info const promptInput = this.promptContentMap[promptName]; @@ -617,11 +617,11 @@ export class TranscendPromptManager< // Ensure the first message in `promptRunMessages` is of type=system if (options.promptRunMessages.length === 0) { - throw new Error("promptRunMessages is expected to have length > 0"); + throw new Error('promptRunMessages is expected to have length > 0'); } if (options.promptRunMessages[0].role !== ChatCompletionRole.System) { throw new Error( - `promptRunMessages[0].role is expected to be = ${ChatCompletionRole.System}` + `promptRunMessages[0].role is expected to be = ${ChatCompletionRole.System}`, ); } if ( @@ -630,12 +630,12 @@ export class TranscendPromptManager< throw new Error( `promptRunMessages[${ options.promptRunMessages.length - 1 - }].role is expected to be = ${ChatCompletionRole.Assistant}` + }].role is expected to be = ${ChatCompletionRole.Assistant}`, ); } const response = options.promptRunMessages.at(-1).content; - let parsed: t.TypeOf; + let parsed: t.TypeOf; try { // Parse the response parsed = this.parseAiResponse(promptName, response); @@ -646,7 +646,7 @@ export class TranscendPromptManager< name, error: error.message, status: QueueStatus.Error, - ...(typeof largeLanguageModel === "string" + ...(typeof largeLanguageModel === 'string' ? { largeLanguageModelId: largeLanguageModel } : { largeLanguageModelName: largeLanguageModel.name, @@ -667,7 +667,7 @@ export class TranscendPromptManager< ...options, name, status: QueueStatus.Resolved, - ...(typeof largeLanguageModel === "string" + ...(typeof largeLanguageModel === 'string' ? { largeLanguageModelId: largeLanguageModel } : { largeLanguageModelName: largeLanguageModel.name, @@ -699,7 +699,7 @@ export class TranscendPromptManager< { largeLanguageModel, ...options - }: Requirize + }: Requirize, ): Promise { // Determine if prompts need to be fetched if ( @@ -719,7 +719,7 @@ export class TranscendPromptManager< `@transcend-io/cli-prompt-run-${new Date().toISOString()}`; if (!this.promptContentMap) { - throw new Error("Expected this.promptContentMap to be defined"); + throw new Error('Expected this.promptContentMap to be defined'); } // Look up prompt info const promptInput = this.promptContentMap[promptName]; @@ -729,11 +729,11 @@ export class TranscendPromptManager< // Ensure the first message in `promptRunMessages` is of type=system if (options.promptRunMessages.length === 0) { - throw new Error("promptRunMessages is expected to have length > 0"); + throw new Error('promptRunMessages is expected to have length > 0'); } if (options.promptRunMessages[0].role !== ChatCompletionRole.System) { throw new Error( - `promptRunMessages[0].role is expected to be = ${ChatCompletionRole.System}` + `promptRunMessages[0].role is expected to be = ${ChatCompletionRole.System}`, ); } @@ -742,7 +742,7 @@ export class TranscendPromptManager< ...options, name, status: QueueStatus.Error, - ...(typeof largeLanguageModel === "string" + ...(typeof largeLanguageModel === 'string' ? { largeLanguageModelId: largeLanguageModel } : { largeLanguageModelName: largeLanguageModel.name, diff --git a/src/lib/ai/filterNullishValuesFromObject.ts b/src/lib/ai/filterNullishValuesFromObject.ts index ea36f39c..707ba966 100644 --- a/src/lib/ai/filterNullishValuesFromObject.ts +++ b/src/lib/ai/filterNullishValuesFromObject.ts @@ -1,4 +1,4 @@ -import { ObjByString } from "@transcend-io/type-utils"; +import { ObjByString } from '@transcend-io/type-utils'; /** * Given an object, remove all keys that are null-ish @@ -7,17 +7,17 @@ import { ObjByString } from "@transcend-io/type-utils"; * @returns Object with null-ish values removed */ export function filterNullishValuesFromObject( - object: T + object: T, ): T { return Object.entries(object).reduce( (accumulator, [k, v]) => v !== null && v !== undefined && - v !== "" && + v !== '' && !(Array.isArray(v) && v.length === 0) && - !(typeof v === "object" && Object.keys(v).length === 0) + !(typeof v === 'object' && Object.keys(v).length === 0) ? Object.assign(accumulator, { [k]: v }) : accumulator, - {} as T + {} as T, ); } diff --git a/src/lib/ai/getGitFilesThatChanged.ts b/src/lib/ai/getGitFilesThatChanged.ts index af27c02b..7ccd2034 100644 --- a/src/lib/ai/getGitFilesThatChanged.ts +++ b/src/lib/ai/getGitFilesThatChanged.ts @@ -1,6 +1,6 @@ -import { execSync } from "node:child_process"; -import fastGlob from "fast-glob"; -import { difference } from "lodash-es"; +import { execSync } from 'node:child_process'; +import fastGlob from 'fast-glob'; +import { difference } from 'lodash-es'; /** * Function thats gets the git files that have changed @@ -42,17 +42,17 @@ export function getGitFilesThatChanged({ // Latest commit on base branch. If we are on the base branch, we take the prior commit const latestBasedCommit = execSync( `git ls-remote ${githubRepo} "refs/heads/${baseBranch}" | cut -f 1`, - { encoding: "utf-8" } - ).split("\n")[0]; + { encoding: 'utf-8' }, + ).split('\n')[0]; // This commit - const latestThisCommit = execSync("git rev-parse HEAD", { - encoding: "utf-8", - }).split("\n")[0]; + const latestThisCommit = execSync('git rev-parse HEAD', { + encoding: 'utf-8', + }).split('\n')[0]; // Ensure commits are present if (!latestBasedCommit || !latestThisCommit) { - throw new Error("FAILED TO FIND COMMIT RANGE"); + throw new Error('FAILED TO FIND COMMIT RANGE'); } // Get the diff between the given branch and base branch @@ -60,13 +60,13 @@ export function getGitFilesThatChanged({ `git fetch && git diff --name-only "${ baseBranch || latestBasedCommit }...${latestThisCommit}" -- ${rootDirectory}`, - { encoding: "utf-8" } + { encoding: 'utf-8' }, ); // Filter out block list const changedFiles = difference( - diff.split("\n").filter(Boolean), - fileBlockList + diff.split('\n').filter(Boolean), + fileBlockList, ); // Filter out globs @@ -79,13 +79,13 @@ export function getGitFilesThatChanged({ const fileDiffs: Record = {}; for (const file of filteredChanges) { const contents = execSync(`git show ${latestThisCommit}:${file}`, { - encoding: "utf-8", + encoding: 'utf-8', }); fileDiffs[file] = contents; } // Pull the github repo name - const repoName = githubRepo.split("/").pop()!.split(".")[0]; + const repoName = githubRepo.split('/').pop()!.split('.')[0]; return { changedFiles, diff --git a/src/lib/ai/removeLinks.ts b/src/lib/ai/removeLinks.ts index dcbe2bee..83424a77 100644 --- a/src/lib/ai/removeLinks.ts +++ b/src/lib/ai/removeLinks.ts @@ -6,5 +6,5 @@ */ export function removeLinks(inputString: string): string { const regex = /(https?:\/\/[^\s]+)/g; - return inputString.replaceAll(regex, ""); + return inputString.replaceAll(regex, ''); } diff --git a/src/lib/api-keys/generateCrossAccountApiKeys.ts b/src/lib/api-keys/generateCrossAccountApiKeys.ts index d189b540..7a357386 100644 --- a/src/lib/api-keys/generateCrossAccountApiKeys.ts +++ b/src/lib/api-keys/generateCrossAccountApiKeys.ts @@ -1,9 +1,9 @@ -import { ScopeName } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { StoredApiKey } from "../../codecs"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +import { ScopeName } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { StoredApiKey } from '../../codecs'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { assumeRole, buildTranscendGraphQLClientGeneric, @@ -11,7 +11,7 @@ import { deleteApiKey, fetchAllApiKeys, loginUser, -} from "../graphql"; +} from '../graphql'; export interface ApiKeyGenerateError { /** Name of instance */ @@ -64,14 +64,14 @@ export async function generateCrossAccountApiKeys({ const client = await buildTranscendGraphQLClientGeneric(transcendUrl, {}); // Login the user - logger.info(colors.magenta("Logging in using email and password.")); + logger.info(colors.magenta('Logging in using email and password.')); const { roles, loginCookie } = await loginUser(client, { email, password }); logger.info( colors.green( `Successfully logged in and found ${roles.length} role${ - roles.length === 1 ? "" : "s" - }!` - ) + roles.length === 1 ? '' : 's' + }!`, + ), ); // Filter down by parentOrganizationId @@ -79,7 +79,7 @@ export async function generateCrossAccountApiKeys({ ? roles.filter( (role) => role.organization.id === parentOrganizationId || - role.organization.parentOrganizationId === parentOrganizationId + role.organization.parentOrganizationId === parentOrganizationId, ) : roles; @@ -96,9 +96,9 @@ export async function generateCrossAccountApiKeys({ logger.info( colors.magenta( `Generating API keys with title: ${apiKeyTitle}, scopes: ${scopes.join( - "," - )}.` - ) + ',', + )}.`, + ), ); // Map over each role @@ -110,8 +110,8 @@ export async function generateCrossAccountApiKeys({ // Grab API keys with that title logger.info( colors.magenta( - `Checking if API key already exists in organization "${role.organization.name}" with title: "${apiKeyTitle}".` - ) + `Checking if API key already exists in organization "${role.organization.name}" with title: "${apiKeyTitle}".`, + ), ); // Delete existing API key @@ -119,14 +119,14 @@ export async function generateCrossAccountApiKeys({ if (apiKeyWithTitle && deleteExistingApiKey) { logger.info( colors.yellow( - `Deleting existing API key in "${role.organization.name}" with title: "${apiKeyTitle}".` - ) + `Deleting existing API key in "${role.organization.name}" with title: "${apiKeyTitle}".`, + ), ); await deleteApiKey(client, apiKeyWithTitle.id); logger.info( colors.green( - `Successfully deleted API key in "${role.organization.name}" with title: "${apiKeyTitle}".` - ) + `Successfully deleted API key in "${role.organization.name}" with title: "${apiKeyTitle}".`, + ), ); } else if (apiKeyWithTitle) { // throw error if one exists but not configured to delete @@ -137,8 +137,8 @@ export async function generateCrossAccountApiKeys({ if (createNewApiKey) { logger.info( colors.magenta( - `Creating API key in "${role.organization.name}" with title: "${apiKeyTitle}".` - ) + `Creating API key in "${role.organization.name}" with title: "${apiKeyTitle}".`, + ), ); const { apiKey } = await createApiKey(client, { title: apiKeyTitle, @@ -151,22 +151,22 @@ export async function generateCrossAccountApiKeys({ }); logger.info( colors.green( - `Successfully created API key in "${role.organization.name}" with title: "${apiKeyTitle}".` - ) + `Successfully created API key in "${role.organization.name}" with title: "${apiKeyTitle}".`, + ), ); } else { // Delete only results.push({ organizationName: role.organization.name, organizationId: role.organization.id, - apiKey: "", + apiKey: '', }); } } catch (error) { logger.error( colors.red( - `Failed to create API key in organization "${role.organization.name}"! - ${error.message}` - ) + `Failed to create API key in organization "${role.organization.name}"! - ${error.message}`, + ), ); errors.push({ organizationName: role.organization.name, @@ -178,18 +178,18 @@ export async function generateCrossAccountApiKeys({ logger.info( colors.green( `Successfully created ${results.length} API key${ - results.length === 1 ? "" : "s" - }` - ) + results.length === 1 ? '' : 's' + }`, + ), ); if (errors.length > 0) { logger.error( colors.red( `Failed to create ${errors.length} API key${ - errors.length === 1 ? "" : "s" - }!` - ) + errors.length === 1 ? '' : 's' + }!`, + ), ); } diff --git a/src/lib/api-keys/listDirectories.ts b/src/lib/api-keys/listDirectories.ts index 72d69c8a..fb9a29e0 100644 --- a/src/lib/api-keys/listDirectories.ts +++ b/src/lib/api-keys/listDirectories.ts @@ -1,5 +1,5 @@ -import { readdirSync, statSync } from "node:fs"; -import { join } from "node:path"; +import { readdirSync, statSync } from 'node:fs'; +import { join } from 'node:path'; /** * List the folders in a directory @@ -9,6 +9,6 @@ import { join } from "node:path"; */ export function listDirectories(startDir: string): string[] { return readdirSync(startDir).filter((entryName) => - statSync(join(startDir, entryName)).isDirectory() + statSync(join(startDir, entryName)).isDirectory(), ); } diff --git a/src/lib/api-keys/listFiles.ts b/src/lib/api-keys/listFiles.ts index b3000f77..9c696e91 100644 --- a/src/lib/api-keys/listFiles.ts +++ b/src/lib/api-keys/listFiles.ts @@ -1,4 +1,4 @@ -import { existsSync, readdirSync } from "node:fs"; +import { existsSync, readdirSync } from 'node:fs'; /** * List the files in a directory @@ -18,7 +18,7 @@ import { existsSync, readdirSync } from "node:fs"; export function listFiles( directory: string, validExtensions?: string[], - removeExtensions = false + removeExtensions = false, ): string[] { if (!existsSync(directory)) { return []; @@ -28,11 +28,11 @@ export function listFiles( .filter((fil) => validExtensions ? validExtensions.filter((extension) => fil.endsWith(extension)).length - : true + : true, ) - .filter((fil) => fil.indexOf(".") > 0); + .filter((fil) => fil.indexOf('.') > 0); return removeExtensions - ? files.map((fil) => fil.replace(/\.[^/.]+$/, "")) + ? files.map((fil) => fil.replace(/\.[^/.]+$/, '')) : files; } diff --git a/src/lib/api-keys/validateTranscendAuth.ts b/src/lib/api-keys/validateTranscendAuth.ts index 45761e1e..7d69ba4a 100644 --- a/src/lib/api-keys/validateTranscendAuth.ts +++ b/src/lib/api-keys/validateTranscendAuth.ts @@ -1,9 +1,9 @@ -import { existsSync, readFileSync } from "node:fs"; -import { decodeCodec } from "@transcend-io/type-utils"; -import colors from "colors"; -import * as t from "io-ts"; -import { StoredApiKey } from "../../codecs"; -import { logger } from "../../logger"; +import { existsSync, readFileSync } from 'node:fs'; +import { decodeCodec } from '@transcend-io/type-utils'; +import colors from 'colors'; +import * as t from 'io-ts'; +import { StoredApiKey } from '../../codecs'; +import { logger } from '../../logger'; /** * Determine if the `--auth` parameter is an API key or a path to a JSON @@ -17,8 +17,8 @@ export function validateTranscendAuth(auth: string): string | StoredApiKey[] { if (!auth) { logger.error( colors.red( - "A Transcend API key must be provided. You can specify using --auth=$TRANSCEND_API_KEY" - ) + 'A Transcend API key must be provided. You can specify using --auth=$TRANSCEND_API_KEY', + ), ); process.exit(1); } @@ -26,7 +26,7 @@ export function validateTranscendAuth(auth: string): string | StoredApiKey[] { // Read from disk if (existsSync(auth)) { // validate that file is a list of API keys - return decodeCodec(t.array(StoredApiKey), readFileSync(auth, "utf-8")); + return decodeCodec(t.array(StoredApiKey), readFileSync(auth, 'utf-8')); } // Return as single API key diff --git a/src/lib/bluebird-replace.ts b/src/lib/bluebird-replace.ts index 46c2fa2e..25e44138 100644 --- a/src/lib/bluebird-replace.ts +++ b/src/lib/bluebird-replace.ts @@ -7,7 +7,7 @@ */ export async function mapSeries( array: R[], - iterator: (item: R, index: number, arrayLength: number) => Promise + iterator: (item: R, index: number, arrayLength: number) => Promise, ): Promise { const results = []; for (let index = 0; index < array.length; index += 1) { @@ -30,7 +30,7 @@ export async function map( options: { /** Concurrency level for the Promise.all call */ concurrency?: number; - } = {} + } = {}, ): Promise { const { concurrency = Infinity } = options; const results: U[] = Array.from({ length: array.length }); @@ -46,7 +46,7 @@ export async function map( const promise = iterator( array[currentIndex], currentIndex, - array.length + array.length, ).then((result) => { results[currentIndex] = result; }); diff --git a/src/lib/code-scanning/constants.ts b/src/lib/code-scanning/constants.ts index 092aee78..889bfc1f 100644 --- a/src/lib/code-scanning/constants.ts +++ b/src/lib/code-scanning/constants.ts @@ -1,4 +1,4 @@ -import { CodePackageType } from "@transcend-io/privacy-types"; +import { CodePackageType } from '@transcend-io/privacy-types'; import { cocoaPods, composerJson, @@ -8,8 +8,8 @@ import { pubspec, pythonRequirementsTxt, swift, -} from "./integrations"; -import { CodeScanningConfig } from "./types"; +} from './integrations'; +import { CodeScanningConfig } from './types'; /** * @deprecated TODO: https://transcend.height.app/T-32325 - use code scanning instead diff --git a/src/lib/code-scanning/findCodePackagesInFolder.ts b/src/lib/code-scanning/findCodePackagesInFolder.ts index 9abd8f34..d4bec9cc 100644 --- a/src/lib/code-scanning/findCodePackagesInFolder.ts +++ b/src/lib/code-scanning/findCodePackagesInFolder.ts @@ -1,9 +1,9 @@ -import { getEntries } from "@transcend-io/type-utils"; -import colors from "colors"; -import fastGlob from "fast-glob"; -import { CodePackageInput } from "../../codecs"; -import { logger } from "../../logger"; -import { CODE_SCANNING_CONFIGS } from "./constants"; +import { getEntries } from '@transcend-io/type-utils'; +import colors from 'colors'; +import fastGlob from 'fast-glob'; +import { CodePackageInput } from '../../codecs'; +import { logger } from '../../logger'; +import { CODE_SCANNING_CONFIGS } from './constants'; /** * Helper to scan and discovery all of the code packages within a folder @@ -36,34 +36,34 @@ export async function findCodePackagesInFolder({ ].filter((dir) => dir.length > 0); try { const filesToScan: string[] = await fastGlob( - `${scanPath}/**/${supportedFiles.join("|")}`, + `${scanPath}/**/${supportedFiles.join('|')}`, { ignore: directoriesToIgnore.map( - (dir: string) => `${scanPath}/**/${dir}` + (dir: string) => `${scanPath}/**/${dir}`, ), unique: true, onlyFiles: true, - } + }, ); logger.info( colors.magenta( - `Scanning: ${filesToScan.length} files of type ${codePackageType}` - ) + `Scanning: ${filesToScan.length} files of type ${codePackageType}`, + ), ); const allPackages = filesToScan.flatMap((filePath) => scanFunction(filePath).map((result) => ({ ...result, - relativePath: filePath.replace(`${scanPath}/`, ""), - })) + relativePath: filePath.replace(`${scanPath}/`, ''), + })), ); logger.info( colors.green( `Found: ${allPackages.length} packages and ${ allPackages.flatMap( - ({ softwareDevelopmentKits = [] }) => softwareDevelopmentKits + ({ softwareDevelopmentKits = [] }) => softwareDevelopmentKits, ).length - } sdks` - ) + } sdks`, + ), ); return allPackages.map( @@ -71,14 +71,14 @@ export async function findCodePackagesInFolder({ ...package_, type: codePackageType, repositoryName, - }) + }), ); } catch (error) { throw new Error( - `Error scanning globs ${supportedFiles} with error: ${error}` + `Error scanning globs ${supportedFiles} with error: ${error}`, ); } - }) + }), ); return allCodePackages.flat(); diff --git a/src/lib/code-scanning/findFilesToScan.ts b/src/lib/code-scanning/findFilesToScan.ts index 4ffe20d4..f6535395 100644 --- a/src/lib/code-scanning/findFilesToScan.ts +++ b/src/lib/code-scanning/findFilesToScan.ts @@ -1,6 +1,6 @@ -import fastGlob from "fast-glob"; -import { logger } from "../../logger"; -import { CodeScanningConfig } from "./types"; +import fastGlob from 'fast-glob'; +import { logger } from '../../logger'; +import { CodeScanningConfig } from './types'; export interface SiloDiscoveryRawResults { /** The name of the potential data silo entry */ @@ -37,29 +37,29 @@ export async function findFilesToScan({ }): Promise { const { ignoreDirs: IGNORE_DIRS, supportedFiles, scanFunction } = config; const globsToSupport = - fileGlobs === "" + fileGlobs === '' ? supportedFiles - : supportedFiles.concat(fileGlobs.split(",")); - const directoriesToIgnore = [...ignoreDirs.split(","), ...IGNORE_DIRS].filter( - (dir) => dir.length > 0 + : supportedFiles.concat(fileGlobs.split(',')); + const directoriesToIgnore = [...ignoreDirs.split(','), ...IGNORE_DIRS].filter( + (dir) => dir.length > 0, ); try { const filesToScan: string[] = await fastGlob( - `${scanPath}/**/${globsToSupport.join("|")}`, + `${scanPath}/**/${globsToSupport.join('|')}`, { ignore: directoriesToIgnore.map( - (dir: string) => `${scanPath}/**/${dir}` + (dir: string) => `${scanPath}/**/${dir}`, ), unique: true, onlyFiles: true, - } + }, ); logger.info(`Scanning: ${filesToScan.length} files`); const allPackages = filesToScan.flatMap((filePath: string) => - scanFunction(filePath) + scanFunction(filePath), ); const allSdks = allPackages.flatMap( - (appPackage) => appPackage.softwareDevelopmentKits || [] + (appPackage) => appPackage.softwareDevelopmentKits || [], ); const uniqueDeps = new Set(allSdks.map((sdk) => sdk.name)); const deps = [...uniqueDeps]; @@ -71,7 +71,7 @@ export async function findFilesToScan({ })); } catch (error) { throw new Error( - `Error scanning globs ${findFilesToScan} with error: ${error}` + `Error scanning globs ${findFilesToScan} with error: ${error}`, ); } } diff --git a/src/lib/code-scanning/integrations/cocoaPods.ts b/src/lib/code-scanning/integrations/cocoaPods.ts index 9a2d57c9..30dfebc5 100644 --- a/src/lib/code-scanning/integrations/cocoaPods.ts +++ b/src/lib/code-scanning/integrations/cocoaPods.ts @@ -1,39 +1,39 @@ -import { readFileSync } from "node:fs"; -import { CodePackageType } from "@transcend-io/privacy-types"; -import { findAllWithRegex } from "@transcend-io/type-utils"; -import { CodePackageSdk } from "../../../codecs"; -import { CodeScanningConfig } from "../types"; +import { readFileSync } from 'node:fs'; +import { CodePackageType } from '@transcend-io/privacy-types'; +import { findAllWithRegex } from '@transcend-io/type-utils'; +import { CodePackageSdk } from '../../../codecs'; +import { CodeScanningConfig } from '../types'; const POD_TARGET_REGEX = /target ('|")(.*?)('|")/; const POD_PACKAGE_REGEX = /pod ('|")(.*?)('|")(, ('|")~> (.+?)('|")|)/; export const cocoaPods: CodeScanningConfig = { - supportedFiles: ["Podfile"], - ignoreDirs: ["Pods"], + supportedFiles: ['Podfile'], + ignoreDirs: ['Pods'], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, "utf-8"); + const fileContents = readFileSync(filePath, 'utf-8'); const targets = findAllWithRegex( { - value: new RegExp(POD_TARGET_REGEX, "g"), - matches: ["quote1", "name", "quote2"], + value: new RegExp(POD_TARGET_REGEX, 'g'), + matches: ['quote1', 'name', 'quote2'], }, - fileContents + fileContents, ); const packages = findAllWithRegex( { - value: new RegExp(POD_PACKAGE_REGEX, "g"), + value: new RegExp(POD_PACKAGE_REGEX, 'g'), matches: [ - "quote1", - "name", - "quote2", - "extra", - "quote3", - "version", - "quote4", + 'quote1', + 'name', + 'quote2', + 'extra', + 'quote3', + 'version', + 'quote4', ], }, - fileContents + fileContents, ); const deps: CodePackageSdk[] = targets.map((target, ind) => ({ @@ -44,7 +44,7 @@ export const cocoaPods: CodeScanningConfig = { (package_) => package_.matchIndex > target.matchIndex && (!targets[ind + 1] || - package_.matchIndex < targets[ind + 1].matchIndex) + package_.matchIndex < targets[ind + 1].matchIndex), ) .map((package_) => ({ name: package_.name, diff --git a/src/lib/code-scanning/integrations/composerJson.ts b/src/lib/code-scanning/integrations/composerJson.ts index 96eb782f..3cf697c5 100644 --- a/src/lib/code-scanning/integrations/composerJson.ts +++ b/src/lib/code-scanning/integrations/composerJson.ts @@ -1,39 +1,39 @@ -import { readFileSync } from "node:fs"; -import { dirname } from "node:path"; -import { CodePackageSdk } from "../../../codecs"; -import { CodeScanningConfig } from "../types"; +import { readFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { CodePackageSdk } from '../../../codecs'; +import { CodeScanningConfig } from '../types'; export const composerJson: CodeScanningConfig = { - supportedFiles: ["composer.json"], - ignoreDirs: ["vendor", "node_modules", "cache", "build", "dist"], + supportedFiles: ['composer.json'], + ignoreDirs: ['vendor', 'node_modules', 'cache', 'build', 'dist'], scanFunction: (filePath) => { - const file = readFileSync(filePath, "utf-8"); + const file = readFileSync(filePath, 'utf-8'); const directory = dirname(filePath); const asJson = JSON.parse(file); const { name, description, require: requireDependencies = {}, - "require-dev": requiredDevelopmentDependencies = {}, + 'require-dev': requiredDevelopmentDependencies = {}, } = asJson; return [ { // name of the package - name: name || directory.split("/").pop()!, + name: name || directory.split('/').pop()!, description, softwareDevelopmentKits: [ ...Object.entries(requireDependencies).map( ([name, version]): CodePackageSdk => ({ name, - version: typeof version === "string" ? version : undefined, - }) + version: typeof version === 'string' ? version : undefined, + }), ), ...Object.entries(requiredDevelopmentDependencies).map( ([name, version]): CodePackageSdk => ({ name, - version: typeof version === "string" ? version : undefined, + version: typeof version === 'string' ? version : undefined, isDevDependency: true, - }) + }), ), ], }, diff --git a/src/lib/code-scanning/integrations/gemfile.ts b/src/lib/code-scanning/integrations/gemfile.ts index 1d16ce8d..28b631df 100644 --- a/src/lib/code-scanning/integrations/gemfile.ts +++ b/src/lib/code-scanning/integrations/gemfile.ts @@ -1,9 +1,9 @@ -import { readFileSync } from "node:fs"; -import { dirname } from "node:path"; -import { CodePackageType } from "@transcend-io/privacy-types"; -import { findAllWithRegex } from "@transcend-io/type-utils"; -import { listFiles } from "../../api-keys"; -import { CodeScanningConfig } from "../types"; +import { readFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { CodePackageType } from '@transcend-io/privacy-types'; +import { findAllWithRegex } from '@transcend-io/type-utils'; +import { listFiles } from '../../api-keys'; +import { CodeScanningConfig } from '../types'; const GEM_PACKAGE_REGEX = /gem *('|")(.+?)('|")(, *('|")(.+?)('|")|)/; const GEMFILE_PACKAGE_NAME_REGEX = /spec\.name *= *('|")(.+?)('|")/; @@ -12,17 +12,17 @@ const GEMFILE_PACKAGE_DESCRIPTION_REGEX = const GEMFILE_PACKAGE_SUMMARY_REGEX = /spec\.summary *= *('|")(.+?)('|")/; export const gemfile: CodeScanningConfig = { - supportedFiles: ["Gemfile"], - ignoreDirs: ["bin"], + supportedFiles: ['Gemfile'], + ignoreDirs: ['bin'], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, "utf-8"); + const fileContents = readFileSync(filePath, 'utf-8'); const directory = dirname(filePath); const filesInFolder = listFiles(directory); // parse gemspec file for name - const gemspec = filesInFolder.find((file) => file === ".gemspec"); + const gemspec = filesInFolder.find((file) => file === '.gemspec'); const gemspecContents = gemspec - ? readFileSync(gemspec, "utf-8") + ? readFileSync(gemspec, 'utf-8') : undefined; const gemfileName = gemspecContents ? (GEMFILE_PACKAGE_NAME_REGEX.exec(gemspecContents) || [])[2] @@ -35,23 +35,23 @@ export const gemfile: CodeScanningConfig = { const targets = findAllWithRegex( { - value: new RegExp(GEM_PACKAGE_REGEX, "g"), + value: new RegExp(GEM_PACKAGE_REGEX, 'g'), matches: [ - "quote1", - "name", - "quote2", - "hasVersion", - "quote3", - "version", - "quote4", + 'quote1', + 'name', + 'quote2', + 'hasVersion', + 'quote3', + 'version', + 'quote4', ], }, - fileContents + fileContents, ); return [ { - name: gemfileName || directory.split("/").pop()!, + name: gemfileName || directory.split('/').pop()!, description: gemfileDescription || undefined, type: CodePackageType.RequirementsTxt, softwareDevelopmentKits: targets.map((package_) => ({ diff --git a/src/lib/code-scanning/integrations/gradle.ts b/src/lib/code-scanning/integrations/gradle.ts index aa871cee..70983adf 100644 --- a/src/lib/code-scanning/integrations/gradle.ts +++ b/src/lib/code-scanning/integrations/gradle.ts @@ -1,7 +1,7 @@ -import { readFileSync } from "node:fs"; -import { dirname } from "node:path"; -import { findAllWithRegex } from "@transcend-io/type-utils"; -import { CodeScanningConfig } from "../types"; +import { readFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { findAllWithRegex } from '@transcend-io/type-utils'; +import { CodeScanningConfig } from '../types'; const GRADLE_IMPLEMENTATION_REGEX = /implementation( *)('|")(.+?):(.+?):(.+?|)('|")/; @@ -21,58 +21,58 @@ const GRADLE_APPLICATION_NAME_REGEX = /applicationId( *)"(.+?)"/; * single and double quotes are both recognized */ export const gradle: CodeScanningConfig = { - supportedFiles: ["build.gradle**"], + supportedFiles: ['build.gradle**'], ignoreDirs: [ - "gradle-app.setting", - "gradle-wrapper.jar", - "gradle-wrapper.properties", + 'gradle-app.setting', + 'gradle-wrapper.jar', + 'gradle-wrapper.properties', ], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, "utf-8"); + const fileContents = readFileSync(filePath, 'utf-8'); const directory = dirname(filePath); const targets = findAllWithRegex( { - value: new RegExp(GRADLE_IMPLEMENTATION_REGEX, "g"), - matches: ["space", "quote1", "name", "path", "version", "quote2"], + value: new RegExp(GRADLE_IMPLEMENTATION_REGEX, 'g'), + matches: ['space', 'quote1', 'name', 'path', 'version', 'quote2'], }, - fileContents + fileContents, ); const targetPlugins = findAllWithRegex( { - value: new RegExp(GRADLE_PLUGIN_REGEX, "g"), - matches: ["quote1", "name", "group", "version", "quote2"], + value: new RegExp(GRADLE_PLUGIN_REGEX, 'g'), + matches: ['quote1', 'name', 'group', 'version', 'quote2'], }, - fileContents + fileContents, ); const targetGroups = findAllWithRegex( { - value: new RegExp(GRADLE_IMPLEMENTATION_GROUP_REGEX, "g"), + value: new RegExp(GRADLE_IMPLEMENTATION_GROUP_REGEX, 'g'), matches: [ - "space1", - "quote1", - "group", - "quote2", - "space2", - "space3", - "quote3", - "name", - "quote4", - "space4", - "space5", - "quote5", - "version", - "quote6", + 'space1', + 'quote1', + 'group', + 'quote2', + 'space2', + 'space3', + 'quote3', + 'name', + 'quote4', + 'space4', + 'space5', + 'quote5', + 'version', + 'quote6', ], }, - fileContents + fileContents, ); const applications = findAllWithRegex( { - value: new RegExp(GRADLE_APPLICATION_NAME_REGEX, "g"), - matches: ["space", "name"], + value: new RegExp(GRADLE_APPLICATION_NAME_REGEX, 'g'), + matches: ['space', 'name'], }, - fileContents + fileContents, ); if (applications.length > 1) { throw new Error(`Expected only one applicationId per file: ${filePath}`); @@ -80,7 +80,7 @@ export const gradle: CodeScanningConfig = { return [ { - name: applications[0]?.name || directory.split("/").pop()!, + name: applications[0]?.name || directory.split('/').pop()!, softwareDevelopmentKits: [ ...targets, ...targetGroups, diff --git a/src/lib/code-scanning/integrations/javascriptPackageJson.ts b/src/lib/code-scanning/integrations/javascriptPackageJson.ts index b7464e3c..fe437c32 100644 --- a/src/lib/code-scanning/integrations/javascriptPackageJson.ts +++ b/src/lib/code-scanning/integrations/javascriptPackageJson.ts @@ -1,13 +1,13 @@ -import { readFileSync } from "node:fs"; -import { dirname } from "node:path"; -import { CodePackageSdk } from "../../../codecs"; -import { CodeScanningConfig } from "../types"; +import { readFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { CodePackageSdk } from '../../../codecs'; +import { CodeScanningConfig } from '../types'; export const javascriptPackageJson: CodeScanningConfig = { - supportedFiles: ["package.json"], - ignoreDirs: ["node_modules", "serverless-build", "lambda-build"], + supportedFiles: ['package.json'], + ignoreDirs: ['node_modules', 'serverless-build', 'lambda-build'], scanFunction: (filePath) => { - const file = readFileSync(filePath, "utf-8"); + const file = readFileSync(filePath, 'utf-8'); const directory = dirname(filePath); const asJson = JSON.parse(file); const { @@ -20,27 +20,27 @@ export const javascriptPackageJson: CodeScanningConfig = { return [ { // name of the package - name: name || directory.split("/").pop()!, + name: name || directory.split('/').pop()!, description, softwareDevelopmentKits: [ ...Object.entries(dependencies).map( ([name, version]): CodePackageSdk => ({ name, - version: typeof version === "string" ? version : undefined, - }) + version: typeof version === 'string' ? version : undefined, + }), ), ...Object.entries(devDependencies).map( ([name, version]): CodePackageSdk => ({ name, - version: typeof version === "string" ? version : undefined, + version: typeof version === 'string' ? version : undefined, isDevDependency: true, - }) + }), ), ...Object.entries(optionalDependencies).map( ([name, version]): CodePackageSdk => ({ name, - version: typeof version === "string" ? version : undefined, - }) + version: typeof version === 'string' ? version : undefined, + }), ), ], }, diff --git a/src/lib/code-scanning/integrations/pubspec.ts b/src/lib/code-scanning/integrations/pubspec.ts index d2edf734..0f8e1b72 100644 --- a/src/lib/code-scanning/integrations/pubspec.ts +++ b/src/lib/code-scanning/integrations/pubspec.ts @@ -1,8 +1,8 @@ -import { readFileSync } from "node:fs"; -import { dirname } from "node:path"; -import { CodePackageType } from "@transcend-io/privacy-types"; -import yaml from "js-yaml"; -import { CodeScanningConfig } from "../types"; +import { readFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { CodePackageType } from '@transcend-io/privacy-types'; +import yaml from 'js-yaml'; +import { CodeScanningConfig } from '../types'; /** * Remove YAML comments from a string @@ -12,10 +12,10 @@ import { CodeScanningConfig } from "../types"; */ function removeYAMLComments(yamlString: string): string { return yamlString - .split("\n") + .split('\n') .map((line) => { // Remove inline comments - const commentIndex = line.indexOf("#"); + const commentIndex = line.indexOf('#'); if ( commentIndex !== -1 && // Check if '#' is not inside a string !line.slice(0, Math.max(0, commentIndex)).includes('"') && @@ -26,15 +26,15 @@ function removeYAMLComments(yamlString: string): string { return line; }) .filter((line) => line.length > 0) - .join("\n"); + .join('\n'); } export const pubspec: CodeScanningConfig = { - supportedFiles: ["pubspec.yml"], - ignoreDirs: ["build"], + supportedFiles: ['pubspec.yml'], + ignoreDirs: ['build'], scanFunction: (filePath) => { const directory = dirname(filePath); - const fileContents = readFileSync(filePath, "utf-8"); + const fileContents = readFileSync(filePath, 'utf-8'); const { name, description, @@ -52,30 +52,30 @@ export const pubspec: CodeScanningConfig = { }; return [ { - name: name || directory.split("/").pop()!, + name: name || directory.split('/').pop()!, description, type: CodePackageType.RequirementsTxt, softwareDevelopmentKits: [ ...Object.entries(dependencies).map(([name, version]) => ({ name, version: - typeof version === "string" + typeof version === 'string' ? version - : typeof version === "number" - ? version.toString() - : version?.sdk, + : typeof version === 'number' + ? version.toString() + : version?.sdk, })), ...Object.entries(development_dependencies).map( ([name, version]) => ({ name, version: - typeof version === "string" + typeof version === 'string' ? version - : typeof version === "number" - ? version.toString() - : version?.sdk, + : typeof version === 'number' + ? version.toString() + : version?.sdk, isDevDependency: true, - }) + }), ), ], }, diff --git a/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts b/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts index 97659eab..fa119fe9 100644 --- a/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts +++ b/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts @@ -1,26 +1,26 @@ -import { readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { CodePackageType } from "@transcend-io/privacy-types"; -import { findAllWithRegex } from "@transcend-io/type-utils"; -import { listFiles } from "../../api-keys"; -import { CodeScanningConfig } from "../types"; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { CodePackageType } from '@transcend-io/privacy-types'; +import { findAllWithRegex } from '@transcend-io/type-utils'; +import { listFiles } from '../../api-keys'; +import { CodeScanningConfig } from '../types'; const REQUIREMENTS_PACKAGE_MATCH = /(.+?)(=+)(.+)/; const PACKAGE_NAME = /name *= *('|")(.+?)('|")/; const PACKAGE_DESCRIPTION = /description *= *('|")(.+?)('|")/; export const pythonRequirementsTxt: CodeScanningConfig = { - supportedFiles: ["requirements.txt"], - ignoreDirs: ["build", "lib", "lib64"], + supportedFiles: ['requirements.txt'], + ignoreDirs: ['build', 'lib', 'lib64'], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, "utf-8"); + const fileContents = readFileSync(filePath, 'utf-8'); const directory = dirname(filePath); const filesInFolder = listFiles(directory); // parse setup file for name - const setupFile = filesInFolder.find((file) => file === "setup.py"); + const setupFile = filesInFolder.find((file) => file === 'setup.py'); const setupFileContents = setupFile - ? readFileSync(join(directory, setupFile), "utf-8") + ? readFileSync(join(directory, setupFile), 'utf-8') : undefined; const packageName = setupFileContents ? (PACKAGE_NAME.exec(setupFileContents) || [])[2] @@ -31,15 +31,15 @@ export const pythonRequirementsTxt: CodeScanningConfig = { const targets = findAllWithRegex( { - value: new RegExp(REQUIREMENTS_PACKAGE_MATCH, "g"), - matches: ["name", "equals", "version"], + value: new RegExp(REQUIREMENTS_PACKAGE_MATCH, 'g'), + matches: ['name', 'equals', 'version'], }, - fileContents + fileContents, ); return [ { - name: packageName || directory.split("/").pop()!, + name: packageName || directory.split('/').pop()!, description: packageDescription || undefined, type: CodePackageType.RequirementsTxt, softwareDevelopmentKits: targets.map((package_) => ({ diff --git a/src/lib/code-scanning/integrations/swift.ts b/src/lib/code-scanning/integrations/swift.ts index 410e75bc..df1f1804 100644 --- a/src/lib/code-scanning/integrations/swift.ts +++ b/src/lib/code-scanning/integrations/swift.ts @@ -1,9 +1,9 @@ -import { readFileSync } from "node:fs"; -import { dirname } from "node:path"; -import { CodePackageType } from "@transcend-io/privacy-types"; -import { decodeCodec } from "@transcend-io/type-utils"; -import * as t from "io-ts"; -import { CodeScanningConfig } from "../types"; +import { readFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { CodePackageType } from '@transcend-io/privacy-types'; +import { decodeCodec } from '@transcend-io/type-utils'; +import * as t from 'io-ts'; +import { CodeScanningConfig } from '../types'; const SwiftPackage = t.type({ pins: t.array( @@ -15,22 +15,22 @@ const SwiftPackage = t.type({ revision: t.string, version: t.string, }), - }) + }), ), version: t.number, }); export const swift: CodeScanningConfig = { - supportedFiles: ["Package.resolved"], + supportedFiles: ['Package.resolved'], ignoreDirs: [], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, "utf-8"); + const fileContents = readFileSync(filePath, 'utf-8'); const parsed = decodeCodec(SwiftPackage, fileContents); return [ { - name: dirname(filePath).split("/").pop() || "", // FIXME pull from Package.swift ->> name if possible + name: dirname(filePath).split('/').pop() || '', // FIXME pull from Package.swift ->> name if possible type: CodePackageType.CocoaPods, // FIXME should be swift softwareDevelopmentKits: parsed.pins.map((target) => ({ name: target.identity, diff --git a/src/lib/consent-manager/buildXdiSyncEndpoint.ts b/src/lib/consent-manager/buildXdiSyncEndpoint.ts index af8dcdaa..756bcd2d 100644 --- a/src/lib/consent-manager/buildXdiSyncEndpoint.ts +++ b/src/lib/consent-manager/buildXdiSyncEndpoint.ts @@ -1,11 +1,11 @@ -import colors from "colors"; -import { difference } from "lodash-es"; -import { StoredApiKey } from "../../codecs"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { buildTranscendGraphQLClient, fetchConsentManager } from "../graphql"; -import { domainToHost } from "./domainToHost"; +import colors from 'colors'; +import { difference } from 'lodash-es'; +import { StoredApiKey } from '../../codecs'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { buildTranscendGraphQLClient, fetchConsentManager } from '../graphql'; +import { domainToHost } from './domainToHost'; /** * Sync group configuration mapping @@ -38,8 +38,8 @@ export async function buildXdiSyncEndpoint( xdiLocation, transcendUrl = DEFAULT_TRANSCEND_API, removeIpAddresses = true, - domainBlockList = ["localhost"], - xdiAllowedCommands = "ConsentManager:Sync", + domainBlockList = ['localhost'], + xdiAllowedCommands = 'ConsentManager:Sync', }: { /** The file location where the XDI file is hosted */ xdiLocation: string; @@ -51,7 +51,7 @@ export async function buildXdiSyncEndpoint( domainBlockList?: string[]; /** Allows XDI commands */ xdiAllowedCommands?: string; - } + }, ): Promise<{ /** Sync group configurations */ syncGroups: XdiSyncGroups; @@ -61,7 +61,7 @@ export async function buildXdiSyncEndpoint( // Convert API keys to list const apiKeysAsList = Array.isArray(apiKeys) ? apiKeys - : [{ apiKey: apiKeys, organizationId: "", organizationName: "" }]; + : [{ apiKey: apiKeys, organizationId: '', organizationName: '' }]; // Fetch configuration for each account const consentManagers = await map( @@ -69,8 +69,8 @@ export async function buildXdiSyncEndpoint( async (apiKey) => { logger.info( colors.magenta( - `Pulling consent metadata for organization - ${apiKey.organizationName}` - ) + `Pulling consent metadata for organization - ${apiKey.organizationName}`, + ), ); // Create a GraphQL client @@ -80,7 +80,7 @@ export async function buildXdiSyncEndpoint( const consentManager = await fetchConsentManager(client); return consentManager; }, - { concurrency: 5 } + { concurrency: 5 }, ); // construct the sync groups @@ -91,7 +91,7 @@ export async function buildXdiSyncEndpoint( // take explicit key first consentManager.partition?.partition || // fallback to bundle ID - consentManager.bundleURL.split("/").reverse()[1]; + consentManager.bundleURL.split('/').reverse()[1]; // Ensure that partition exists in the sync groups if (!syncGroups[partitionKey]) { @@ -103,11 +103,11 @@ export async function buildXdiSyncEndpoint( consentManager.configuration.domains .filter( // ignore IP addresses - (domain) => !removeIpAddresses || !IP_ADDRESS_REGEX.test(domain) + (domain) => !removeIpAddresses || !IP_ADDRESS_REGEX.test(domain), ) .map((domain) => domainToHost(domain)), // ignore block list - domainBlockList + domainBlockList, ); // merge existing sync group with hosts for this consent manager syncGroups[partitionKey] = [ diff --git a/src/lib/consent-manager/consentManagersToBusinessEntities.ts b/src/lib/consent-manager/consentManagersToBusinessEntities.ts index 1007b33e..76bb356e 100644 --- a/src/lib/consent-manager/consentManagersToBusinessEntities.ts +++ b/src/lib/consent-manager/consentManagersToBusinessEntities.ts @@ -1,5 +1,5 @@ -import { BusinessEntityInput, ConsentManagerInput } from "../../codecs"; -import { logger } from "../../logger"; +import { BusinessEntityInput, ConsentManagerInput } from '../../codecs'; +import { logger } from '../../logger'; /** * Combine multiple consent manager configurations into a list of business entity configurations @@ -13,19 +13,19 @@ export function consentManagersToBusinessEntities( name: string; /** Consent manager input */ input?: ConsentManagerInput; - }[] + }[], ): BusinessEntityInput[] { // Construct the business entities YAML definition const businessEntities = inputs.map( ({ name, input }): BusinessEntityInput => ({ // Title of Transcend Instance - title: name.replace(".yml", ""), + title: name.replace('.yml', ''), attributes: [ // Sync domain list ...(input?.domains ? [ { - key: "Transcend Domain List", + key: 'Transcend Domain List', values: [...new Set(input.domains)], }, ] @@ -34,17 +34,17 @@ export function consentManagersToBusinessEntities( ...(input?.bundleUrls ? [ { - key: "Airgap Production URL", + key: 'Airgap Production URL', values: [input.bundleUrls.PRODUCTION], }, { - key: "Airgap Test URL", + key: 'Airgap Test URL', values: [input.bundleUrls.TEST], }, { - key: "Airgap XDI URL", + key: 'Airgap XDI URL', values: [ - input.bundleUrls.PRODUCTION.replace("airgap.js", "xdi.js"), + input.bundleUrls.PRODUCTION.replace('airgap.js', 'xdi.js'), ], }, ] @@ -53,20 +53,20 @@ export function consentManagersToBusinessEntities( ...(input?.partition ? [ { - key: "Consent Partition Key", + key: 'Consent Partition Key', values: [input.partition], }, ] : []), ], - }) + }), ); // Log out info on airgap scripts to host - logger.info("\n\n~~~~~~~~~~~\nAirgap scripts to host:"); + logger.info('\n\n~~~~~~~~~~~\nAirgap scripts to host:'); for (const [ind, { attributes, title }] of businessEntities.entries()) { attributes - ?.find((attribute) => attribute.key === "Airgap Production URL") + ?.find((attribute) => attribute.key === 'Airgap Production URL') ?.values?.forEach((url) => { logger.info(`${ind}) ${title} - ${url}`); }); diff --git a/src/lib/consent-manager/createConsentToken.ts b/src/lib/consent-manager/createConsentToken.ts index b628613f..22e7e9aa 100644 --- a/src/lib/consent-manager/createConsentToken.ts +++ b/src/lib/consent-manager/createConsentToken.ts @@ -1,5 +1,5 @@ -import * as crypto from "node:crypto"; -import * as jwt from "jsonwebtoken"; +import * as crypto from 'node:crypto'; +import * as jwt from 'jsonwebtoken'; /** * Function to create a consent manager token @@ -13,16 +13,16 @@ import * as jwt from "jsonwebtoken"; export function createConsentToken( userId: string, base64EncryptionKey: string, - base64SigningKey: string + base64SigningKey: string, ): string { // Read on for where to find these keys - const signingKey = Buffer.from(base64SigningKey, "base64"); - const encryptionKey = Buffer.from(base64EncryptionKey, "base64"); + const signingKey = Buffer.from(base64SigningKey, 'base64'); + const encryptionKey = Buffer.from(base64EncryptionKey, 'base64'); // NIST's AES-KWP implementation { aes 48 } - see https://tools.ietf.org/html/rfc5649 - const encryptionAlgorithm = "id-aes256-wrap-pad"; + const encryptionAlgorithm = 'id-aes256-wrap-pad'; // Initial Value for AES-KWP integrity check - see https://tools.ietf.org/html/rfc5649#section-3 - const iv = Buffer.from("A65959A6", "hex"); + const iv = Buffer.from('A65959A6', 'hex'); // Set up encryption algorithm const cipher = crypto.createCipheriv(encryptionAlgorithm, encryptionKey, iv); @@ -30,7 +30,7 @@ export function createConsentToken( const encryptedIdentifier = Buffer.concat([ cipher.update(userId), cipher.final(), - ]).toString("base64"); + ]).toString('base64'); // Create the JWT content - jwt.sign will add a 'iat' (issued at) field to the payload // If you wanted to add something manually, consider @@ -42,7 +42,7 @@ export function createConsentToken( // Create a JSON web token and HMAC it with SHA-384 const consentToken = jwt.sign(jwtPayload, signingKey, { - algorithm: "HS384", + algorithm: 'HS384', }); return consentToken; diff --git a/src/lib/consent-manager/dataFlowsToDataSilos.ts b/src/lib/consent-manager/dataFlowsToDataSilos.ts index be5a98da..aa89cc2a 100644 --- a/src/lib/consent-manager/dataFlowsToDataSilos.ts +++ b/src/lib/consent-manager/dataFlowsToDataSilos.ts @@ -1,6 +1,6 @@ -import { union } from "lodash-es"; -import { DataFlowInput, DataSiloInput } from "../../codecs"; -import { IndexedCatalogs } from "../graphql"; +import { union } from 'lodash-es'; +import { DataFlowInput, DataSiloInput } from '../../codecs'; +import { IndexedCatalogs } from '../graphql'; /** * Convert data flow configurations into a set of data silo configurations @@ -12,13 +12,13 @@ import { IndexedCatalogs } from "../graphql"; export function dataFlowsToDataSilos( inputs: DataFlowInput[], { - adTechPurposes = ["SaleOfInfo"], + adTechPurposes = ['SaleOfInfo'], serviceToTitle, serviceToSupportedIntegration, }: IndexedCatalogs & { /** List of purposes that are considered "Ad Tech" */ adTechPurposes?: string[]; - } + }, ): { /** List of data silo configurations for site-tech services */ siteTechDataSilos: DataSiloInput[]; @@ -38,13 +38,13 @@ export function dataFlowsToDataSilos( for (const flow of inputs) { // process data flows with services const { service, attributes = [] } = flow; - if (!service || service === "internalService") { + if (!service || service === 'internalService') { continue; } // create mapping to found on domain const foundOnDomain = attributes.find( - (attribute) => attribute.key === "Found on Domain" + (attribute) => attribute.key === 'Found on Domain', ); // Create a list of all domains where the data flow was found @@ -54,8 +54,8 @@ export function dataFlowsToDataSilos( } serviceToFoundOnDomain[service].push( ...foundOnDomain.values.map((v) => - v.replace("https://", "").replace("http://", "") - ) + v.replace('https://', '').replace('http://', ''), + ), ); serviceToFoundOnDomain[service] = [ ...new Set(serviceToFoundOnDomain[service]), @@ -70,7 +70,7 @@ export function dataFlowsToDataSilos( // remove from site tech list if (siteTechIntegrations.includes(service)) { siteTechIntegrations = siteTechIntegrations.filter( - (s) => s !== service + (s) => s !== service, ); } } else if (!adTechIntegrations.includes(service)) { @@ -84,14 +84,14 @@ export function dataFlowsToDataSilos( title: serviceToTitle[service], ...(serviceToSupportedIntegration[service] ? { integrationName: service } - : { integrationName: "promptAPerson", "outer-type": service }), + : { integrationName: 'promptAPerson', 'outer-type': service }), attributes: [ { - key: "Tech Type", - values: ["Ad Tech"], + key: 'Tech Type', + values: ['Ad Tech'], }, { - key: "Found On Domain", + key: 'Found On Domain', values: serviceToFoundOnDomain[service] || [], }, ], @@ -103,18 +103,18 @@ export function dataFlowsToDataSilos( title: serviceToTitle[service], ...(serviceToSupportedIntegration[service] ? { integrationName: service } - : { integrationName: "promptAPerson", outerType: service }), + : { integrationName: 'promptAPerson', outerType: service }), attributes: [ { - key: "Tech Type", - values: ["Site Tech"], + key: 'Tech Type', + values: ['Site Tech'], }, { - key: "Found On Domain", + key: 'Found On Domain', values: serviceToFoundOnDomain[service] || [], }, ], - }) + }), ); return { diff --git a/src/lib/consent-manager/uploadConsents.ts b/src/lib/consent-manager/uploadConsents.ts index 071bd1b8..c682baca 100644 --- a/src/lib/consent-manager/uploadConsents.ts +++ b/src/lib/consent-manager/uploadConsents.ts @@ -1,20 +1,20 @@ -import { ConsentPreferencesBody } from "@transcend-io/airgap.js-types"; -import { decodeCodec } from "@transcend-io/type-utils"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import * as t from "io-ts"; -import { DEFAULT_TRANSCEND_CONSENT_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { createTranscendConsentGotInstance } from "../graphql"; -import { createConsentToken } from "./createConsentToken"; -import type { ConsentPreferenceUpload } from "./types"; +import { ConsentPreferencesBody } from '@transcend-io/airgap.js-types'; +import { decodeCodec } from '@transcend-io/type-utils'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import * as t from 'io-ts'; +import { DEFAULT_TRANSCEND_CONSENT_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { createTranscendConsentGotInstance } from '../graphql'; +import { createConsentToken } from './createConsentToken'; +import type { ConsentPreferenceUpload } from './types'; export const USP_STRING_REGEX = /^[0-9][Y|N]([Y|N])[Y|N]$/; export const PurposeMap = t.record( t.string, - t.union([t.boolean, t.literal("Auto")]) + t.union([t.boolean, t.literal('Auto')]), ); /** @@ -48,15 +48,15 @@ export async function uploadConsents({ // Ensure usp strings are valid const invalidUspStrings = preferences.filter( - (pref) => pref.usp && !USP_STRING_REGEX.test(pref.usp) + (pref) => pref.usp && !USP_STRING_REGEX.test(pref.usp), ); if (invalidUspStrings.length > 0) { throw new Error( `Received invalid usp strings: ${JSON.stringify( invalidUspStrings, null, - 2 - )}` + 2, + )}`, ); } @@ -79,29 +79,29 @@ export async function uploadConsents({ `Received invalid purpose maps: ${JSON.stringify( invalidPurposeMaps, null, - 2 - )}` + 2, + )}`, ); } // Ensure usp or preferences are provided const invalidInputs = preferences.filter( - (pref) => !pref.usp && !pref.purposes + (pref) => !pref.usp && !pref.purposes, ); if (invalidInputs.length > 0) { throw new Error( `Received invalid inputs, expected either purposes or usp to be defined: ${JSON.stringify( invalidInputs, null, - 2 - )}` + 2, + )}`, ); } logger.info( colors.magenta( - `Uploading ${preferences.length} user preferences to partition ${partition}` - ) + `Uploading ${preferences.length} user preferences to partition ${partition}`, + ), ); // Time duration @@ -109,7 +109,7 @@ export async function uploadConsents({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Build a GraphQL client @@ -119,7 +119,7 @@ export async function uploadConsents({ preferences, async ({ userId, - confirmed = "true", + confirmed = 'true', updated, prompted, purposes, @@ -128,7 +128,7 @@ export async function uploadConsents({ const token = createConsentToken( userId, base64EncryptionKey, - base64SigningKey + base64SigningKey, ); // parse usp string @@ -140,14 +140,14 @@ export async function uploadConsents({ token, partition, consent: { - confirmed: confirmed === "true", + confirmed: confirmed === 'true', purposes: purposes ? decodeCodec(PurposeMap, purposes) : consent.usp - ? { SaleOfInfo: saleStatus === "Y" } - : {}, - ...(updated ? { updated: updated === "true" } : {}), - ...(prompted ? { prompted: prompted === "true" } : {}), + ? { SaleOfInfo: saleStatus === 'Y' } + : {}, + ...(updated ? { updated: updated === 'true' } : {}), + ...(prompted ? { prompted: prompted === 'true' } : {}), ...consent, }, } as ConsentPreferencesBody; @@ -155,13 +155,13 @@ export async function uploadConsents({ // Make the request try { await transcendConsentApi - .post("sync", { + .post('sync', { json: input, }) .json(); } catch (error) { try { - const parsed = JSON.parse(error?.response?.body || "{}"); + const parsed = JSON.parse(error?.response?.body || '{}'); if (parsed.error) { logger.error(colors.red(`Error: ${parsed.error}`)); } @@ -171,14 +171,14 @@ export async function uploadConsents({ throw new Error( `Received an error from server: ${ error?.response?.body || error?.message - }` + }`, ); } total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -191,7 +191,7 @@ export async function uploadConsents({ preferences.length } user preferences to partition ${partition} in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); } diff --git a/src/lib/consent-manager/uploadCookiesFromCsv.ts b/src/lib/consent-manager/uploadCookiesFromCsv.ts index f22487d7..1ef399b2 100644 --- a/src/lib/consent-manager/uploadCookiesFromCsv.ts +++ b/src/lib/consent-manager/uploadCookiesFromCsv.ts @@ -1,22 +1,22 @@ -import { ConsentTrackerStatus } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { CookieCsvInput, CookieInput } from "../../codecs"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { buildTranscendGraphQLClient, syncCookies } from "../graphql"; -import { splitCsvToList } from "../requests"; -import { readCsv } from "../requests/readCsv"; +import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { CookieCsvInput, CookieInput } from '../../codecs'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { buildTranscendGraphQLClient, syncCookies } from '../graphql'; +import { splitCsvToList } from '../requests'; +import { readCsv } from '../requests/readCsv'; const OMIT_COLUMNS = new Set([ - "ID", - "Activity", - "Encounters", - "Last Seen At", - "Has Native Do Not Sell/Share Support", - "IAB USP API Support", - "Service Description", - "Website URL", - "Categories of Recipients", + 'ID', + 'Activity', + 'Encounters', + 'Last Seen At', + 'Has Native Do Not Sell/Share Support', + 'IAB USP API Support', + 'Service Description', + 'Website URL', + 'Categories of Recipients', ]); /** @@ -49,7 +49,7 @@ export async function uploadCookiesFromCsv({ // Convert these inputs into a format that the other function can use const validatedCookieInputs = cookieInputs.map( ({ - "Is Regex?": isRegex, + 'Is Regex?': isRegex, Notes, // TODO: https://transcend.height.app/T-26391 - export in CSV // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -61,8 +61,8 @@ export async function uploadCookiesFromCsv({ Name, ...rest }): CookieInput => ({ - ...(typeof isRegex === "string" - ? { isRegex: isRegex.toLowerCase() === "true" } + ...(typeof isRegex === 'string' + ? { isRegex: isRegex.toLowerCase() === 'true' } : {}), name: Name, description: Notes, @@ -83,7 +83,7 @@ export async function uploadCookiesFromCsv({ key, values: splitCsvToList(value), })), - }) + }), ); // Upload the cookies into Transcend dashboard @@ -93,8 +93,8 @@ export async function uploadCookiesFromCsv({ if (!syncedCookies) { logger.error( colors.red( - "Encountered error(s) syncing cookies from CSV, see logs above for more info. " - ) + 'Encountered error(s) syncing cookies from CSV, see logs above for more info. ', + ), ); process.exit(1); } diff --git a/src/lib/consent-manager/uploadDataFlowsFromCsv.ts b/src/lib/consent-manager/uploadDataFlowsFromCsv.ts index e01f1c4a..d79817ce 100644 --- a/src/lib/consent-manager/uploadDataFlowsFromCsv.ts +++ b/src/lib/consent-manager/uploadDataFlowsFromCsv.ts @@ -1,22 +1,22 @@ -import { ConsentTrackerStatus } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { DataFlowCsvInput, DataFlowInput } from "../../codecs"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { buildTranscendGraphQLClient, syncDataFlows } from "../graphql"; -import { splitCsvToList } from "../requests"; -import { readCsv } from "../requests/readCsv"; +import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { DataFlowCsvInput, DataFlowInput } from '../../codecs'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { buildTranscendGraphQLClient, syncDataFlows } from '../graphql'; +import { splitCsvToList } from '../requests'; +import { readCsv } from '../requests/readCsv'; const OMIT_COLUMNS = new Set([ - "ID", - "Activity", - "Encounters", - "Last Seen At", - "Has Native Do Not Sell/Share Support", - "IAB USP API Support", - "Service Description", - "Website URL", - "Categories of Recipients", + 'ID', + 'Activity', + 'Encounters', + 'Last Seen At', + 'Has Native Do Not Sell/Share Support', + 'IAB USP API Support', + 'Service Description', + 'Website URL', + 'Categories of Recipients', ]); /** @@ -61,7 +61,7 @@ export async function uploadDataFlowsFromCsv({ Status, Owners, Teams, - "Connections Made To": value, + 'Connections Made To': value, ...rest }): DataFlowInput => ({ value, @@ -84,22 +84,22 @@ export async function uploadDataFlowsFromCsv({ key, values: splitCsvToList(value), })), - }) + }), ); // Upload the data flows into Transcend dashboard const syncedDataFlows = await syncDataFlows( client, validatedDataFlowInputs, - classifyService + classifyService, ); // Log errors if (!syncedDataFlows) { logger.error( colors.red( - "Encountered error(s) syncing data flows from CSV, see logs above for more info. " - ) + 'Encountered error(s) syncing data flows from CSV, see logs above for more info. ', + ), ); process.exit(1); } diff --git a/src/lib/cron/markCronIdentifierCompleted.ts b/src/lib/cron/markCronIdentifierCompleted.ts index ce1ce7d3..47b70c9b 100644 --- a/src/lib/cron/markCronIdentifierCompleted.ts +++ b/src/lib/cron/markCronIdentifierCompleted.ts @@ -1,5 +1,5 @@ -import type { Got } from "got"; -import * as t from "io-ts"; +import type { Got } from 'got'; +import * as t from 'io-ts'; /** * Minimal set required to mark as completed @@ -22,13 +22,13 @@ export type CronIdentifierPush = t.TypeOf; */ export async function markCronIdentifierCompleted( sombra: Got, - { nonce, identifier }: CronIdentifierPush + { nonce, identifier }: CronIdentifierPush, ): Promise { try { // Make the GraphQL request - await sombra.put("v1/data-silo", { + await sombra.put('v1/data-silo', { headers: { - "x-transcend-nonce": nonce, + 'x-transcend-nonce': nonce, }, json: { profiles: [ @@ -47,7 +47,7 @@ export async function markCronIdentifierCompleted( throw new Error( `Received an error from server: ${ error?.response?.body || error?.message - }` + }`, ); } } diff --git a/src/lib/cron/markRequestDataSiloIdsCompleted.ts b/src/lib/cron/markRequestDataSiloIdsCompleted.ts index f05f1b53..ff0d03d1 100644 --- a/src/lib/cron/markRequestDataSiloIdsCompleted.ts +++ b/src/lib/cron/markRequestDataSiloIdsCompleted.ts @@ -1,15 +1,15 @@ -import { RequestDataSiloStatus } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { RequestDataSiloStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, CHANGE_REQUEST_DATA_SILO_STATUS, fetchRequestDataSilo, makeGraphQLRequest, -} from "../graphql"; +} from '../graphql'; /** * Given a CSV of Request IDs, mark associated RequestDataSilos as completed @@ -46,14 +46,14 @@ export async function markRequestDataSiloIdsCompleted({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Notify Transcend logger.info( colors.magenta( - `Notifying Transcend for data silo "${dataSiloId}" marking "${requestIds.length}" requests as completed.` - ) + `Notifying Transcend for data silo "${dataSiloId}" marking "${requestIds.length}" requests as completed.`, + ), ); let total = 0; @@ -75,7 +75,7 @@ export async function markRequestDataSiloIdsCompleted({ status, }); } catch (error) { - if (!error.message.includes("Client error: Request must be active:")) { + if (!error.message.includes('Client error: Request must be active:')) { throw error; } } @@ -83,7 +83,7 @@ export async function markRequestDataSiloIdsCompleted({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -92,8 +92,8 @@ export async function markRequestDataSiloIdsCompleted({ logger.info( colors.green( - `Successfully notified Transcend in "${totalTime / 1000}" seconds!` - ) + `Successfully notified Transcend in "${totalTime / 1000}" seconds!`, + ), ); return requestIds.length; } diff --git a/src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts b/src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts index 8594279c..607d9134 100644 --- a/src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts +++ b/src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts @@ -1,18 +1,18 @@ -import { RequestAction } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +import { RequestAction } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { buildTranscendGraphQLClient, createSombraGotInstance, fetchRequestDataSiloActiveCount, -} from "../graphql"; +} from '../graphql'; import { CronIdentifier, pullCronPageOfIdentifiers, -} from "./pullCronPageOfIdentifiers"; +} from './pullCronPageOfIdentifiers'; /** * A CSV formatted identifier @@ -72,7 +72,7 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ // Validate savePageSize if (savePageSize % apiPageSize !== 0) { throw new Error( - `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}` + `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}`, ); } @@ -92,12 +92,12 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ logger.info( colors.magenta( `Pulling ${ - skipRequestCount ? "all" : totalRequestCount + skipRequestCount ? 'all' : totalRequestCount } outstanding request identifiers ` + `for data silo: "${dataSiloId}" for requests of types "${actions.join( - '", "' - )}"` - ) + '", "', + )}"`, + ), ); // Time duration @@ -105,7 +105,7 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); const foundRequestIds = new Set(); @@ -144,9 +144,9 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ ({ attributes, ...identifier }) => ({ ...identifier, ...Object.fromEntries( - attributes.map((value) => [value.key, value.values.join(",")]) + attributes.map((value) => [value.key, value.values.join(',')]), ), - }) + }), ); identifiers.push(...identifiersWithAction); @@ -163,8 +163,8 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ if (skipRequestCount) { logger.info( colors.magenta( - `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests` - ) + `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests`, + ), ); } else { progressBar.update(foundRequestIds.size); @@ -187,8 +187,8 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ colors.green( `Successfully pulled ${identifiers.length} outstanding identifiers from ${ foundRequestIds.size - } requests in "${totalTime / 1000}" seconds!` - ) + } requests in "${totalTime / 1000}" seconds!`, + ), ); return { identifiers }; diff --git a/src/lib/cron/pullCronPageOfIdentifiers.ts b/src/lib/cron/pullCronPageOfIdentifiers.ts index 9688fd62..a1e7b9f4 100644 --- a/src/lib/cron/pullCronPageOfIdentifiers.ts +++ b/src/lib/cron/pullCronPageOfIdentifiers.ts @@ -1,7 +1,7 @@ -import { RequestAction } from "@transcend-io/privacy-types"; -import { decodeCodec } from "@transcend-io/type-utils"; -import type { Got } from "got"; -import * as t from "io-ts"; +import { RequestAction } from '@transcend-io/privacy-types'; +import { decodeCodec } from '@transcend-io/type-utils'; +import type { Got } from 'got'; +import * as t from 'io-ts'; export const CronIdentifier = t.type({ /** The identifier value */ @@ -25,7 +25,7 @@ export const CronIdentifier = t.type({ t.type({ key: t.string, values: t.array(t.string), - }) + }), ), }); @@ -56,7 +56,7 @@ export async function pullCronPageOfIdentifiers( limit?: number; /** Page to pull in */ offset?: number; - } + }, ): Promise { try { // Make the GraphQL request @@ -73,14 +73,14 @@ export async function pullCronPageOfIdentifiers( t.type({ items: t.array(CronIdentifier), }), - response + response, ); return items; } catch (error) { throw new Error( `Received an error from server: ${ error?.response?.body || error?.message - }` + }`, ); } } diff --git a/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts b/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts index 8594279c..607d9134 100644 --- a/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts +++ b/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts @@ -1,18 +1,18 @@ -import { RequestAction } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +import { RequestAction } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { buildTranscendGraphQLClient, createSombraGotInstance, fetchRequestDataSiloActiveCount, -} from "../graphql"; +} from '../graphql'; import { CronIdentifier, pullCronPageOfIdentifiers, -} from "./pullCronPageOfIdentifiers"; +} from './pullCronPageOfIdentifiers'; /** * A CSV formatted identifier @@ -72,7 +72,7 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ // Validate savePageSize if (savePageSize % apiPageSize !== 0) { throw new Error( - `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}` + `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}`, ); } @@ -92,12 +92,12 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ logger.info( colors.magenta( `Pulling ${ - skipRequestCount ? "all" : totalRequestCount + skipRequestCount ? 'all' : totalRequestCount } outstanding request identifiers ` + `for data silo: "${dataSiloId}" for requests of types "${actions.join( - '", "' - )}"` - ) + '", "', + )}"`, + ), ); // Time duration @@ -105,7 +105,7 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); const foundRequestIds = new Set(); @@ -144,9 +144,9 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ ({ attributes, ...identifier }) => ({ ...identifier, ...Object.fromEntries( - attributes.map((value) => [value.key, value.values.join(",")]) + attributes.map((value) => [value.key, value.values.join(',')]), ), - }) + }), ); identifiers.push(...identifiersWithAction); @@ -163,8 +163,8 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ if (skipRequestCount) { logger.info( colors.magenta( - `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests` - ) + `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests`, + ), ); } else { progressBar.update(foundRequestIds.size); @@ -187,8 +187,8 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ colors.green( `Successfully pulled ${identifiers.length} outstanding identifiers from ${ foundRequestIds.size - } requests in "${totalTime / 1000}" seconds!` - ) + } requests in "${totalTime / 1000}" seconds!`, + ), ); return { identifiers }; diff --git a/src/lib/cron/pushCronIdentifiersFromCsv.ts b/src/lib/cron/pushCronIdentifiersFromCsv.ts index ca4cab47..72076830 100644 --- a/src/lib/cron/pushCronIdentifiersFromCsv.ts +++ b/src/lib/cron/pushCronIdentifiersFromCsv.ts @@ -1,15 +1,15 @@ -import cliProgress from "cli-progress"; -import colors from "colors"; -import { chunk } from "lodash-es"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map, mapSeries } from "../bluebird-replace"; -import { createSombraGotInstance } from "../graphql"; -import { readCsv } from "../requests"; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { chunk } from 'lodash-es'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map, mapSeries } from '../bluebird-replace'; +import { createSombraGotInstance } from '../graphql'; +import { readCsv } from '../requests'; import { CronIdentifierPush, markCronIdentifierCompleted, -} from "./markCronIdentifierCompleted"; +} from './markCronIdentifierCompleted'; /** * Given a CSV of cron job outputs, mark all requests as completed in Transcend @@ -51,8 +51,8 @@ export async function pushCronIdentifiersFromCsv({ // Notify Transcend logger.info( colors.magenta( - `Notifying Transcend for data silo "${dataSiloId}" marking "${activeResults.length}" identifiers as completed.` - ) + `Notifying Transcend for data silo "${dataSiloId}" marking "${activeResults.length}" identifiers as completed.`, + ), ); // Time duration @@ -60,7 +60,7 @@ export async function pushCronIdentifiersFromCsv({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); let successCount = 0; @@ -73,14 +73,14 @@ export async function pushCronIdentifiersFromCsv({ const totalChunks = chunks.length; const processChunk = async ( items: CronIdentifierPush[], - chunkIndex: number + chunkIndex: number, ): Promise => { logger.info( colors.blue( `Processing chunk ${chunkIndex + 1}/${totalChunks} (${ chunk.length - } items)` - ) + } items)`, + ), ); // Process the items of the chunk concurrently @@ -95,8 +95,8 @@ export async function pushCronIdentifiersFromCsv({ } catch (error) { logger.error( colors.red( - `Error notifying Transcend for identifier "${identifier.identifier}" - ${error?.message}` - ) + `Error notifying Transcend for identifier "${identifier.identifier}" - ${error?.message}`, + ), ); errorCount += 1; } @@ -106,7 +106,7 @@ export async function pushCronIdentifiersFromCsv({ // Sleep between chunks (except for the last chunk) if (sleepSeconds > 0 && chunkIndex < totalChunks - 1) { logger.info( - colors.yellow(`Sleeping for ${sleepSeconds}s before next chunk...`) + colors.yellow(`Sleeping for ${sleepSeconds}s before next chunk...`), ); await new Promise((resolve) => { @@ -126,24 +126,24 @@ export async function pushCronIdentifiersFromCsv({ colors.green( `Successfully notified Transcend for ${successCount} identifiers in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); if (failureCount) { logger.info( colors.magenta( `There were ${failureCount} identifiers that were not in a state to be updated.` + - "They likely have already been resolved." - ) + 'They likely have already been resolved.', + ), ); } if (errorCount) { logger.error( colors.red( - `There were ${errorCount} identifiers that failed to be updated. Please review the logs for more information.` - ) + `There were ${errorCount} identifiers that failed to be updated. Please review the logs for more information.`, + ), ); - throw new Error("Failed to update all identifiers"); + throw new Error('Failed to update all identifiers'); } return activeResults.length; } diff --git a/src/lib/cron/writeCsv.ts b/src/lib/cron/writeCsv.ts index c4e19451..f12f8f8c 100644 --- a/src/lib/cron/writeCsv.ts +++ b/src/lib/cron/writeCsv.ts @@ -1,6 +1,6 @@ -import { appendFileSync, createWriteStream, writeFileSync } from "node:fs"; -import { ObjByString } from "@transcend-io/type-utils"; -import * as fastcsv from "fast-csv"; +import { appendFileSync, createWriteStream, writeFileSync } from 'node:fs'; +import { ObjByString } from '@transcend-io/type-utils'; +import * as fastcsv from 'fast-csv'; /** * Escape a CSV value @@ -9,7 +9,7 @@ import * as fastcsv from "fast-csv"; * @returns Escaped value */ function escapeCsvValue(value: string): string { - if (value.includes('"') || value.includes(",") || value.includes("\n")) { + if (value.includes('"') || value.includes(',') || value.includes('\n')) { return `"${value.replaceAll('"', '""')}"`; } return value; @@ -24,7 +24,7 @@ function escapeCsvValue(value: string): string { export function writeCsvSync( filePath: string, data: ObjByString[], - headers: string[] + headers: string[], ): void { const rows: string[][] = []; @@ -33,8 +33,8 @@ export function writeCsvSync( // Build CSV content with proper escaping const csvContent = rows - .map((row) => row.map(escapeCsvValue).join(",")) - .join("\n"); + .map((row) => row.map(escapeCsvValue).join(',')) + .join('\n'); // Write to file, overwriting existing content writeFileSync(filePath, csvContent); @@ -53,8 +53,8 @@ export function appendCsvSync(filePath: string, data: ObjByString[]): void { // Build CSV content with proper escaping const csvContent = rows - .map((row) => row.map(escapeCsvValue).join(",")) - .join("\n"); + .map((row) => row.map(escapeCsvValue).join(',')) + .join('\n'); // Append to file with leading newline appendFileSync(filePath, `\n${csvContent}`); @@ -70,7 +70,7 @@ export function appendCsvSync(filePath: string, data: ObjByString[]): void { export async function writeCsv( filePath: string, data: ObjByString[], - headers: boolean | string[] = true + headers: boolean | string[] = true, ): Promise { const ws = createWriteStream(filePath); await new Promise((resolve, reject) => { @@ -78,8 +78,8 @@ export async function writeCsv( fastcsv .write(data, { headers, objectMode: true }) .pipe(ws) - .on("error", reject) - .on("end", () => { + .on('error', reject) + .on('end', () => { resolve(true); }); } catch (error) { @@ -100,14 +100,14 @@ export function parseFilePath(filePath: string): { /** Extension of the file */ extension: string; } { - const lastDotIndex = filePath.lastIndexOf("."); + const lastDotIndex = filePath.lastIndexOf('.'); return { baseName: lastDotIndex === -1 ? filePath : filePath.slice(0, Math.max(0, lastDotIndex)), extension: - lastDotIndex === -1 ? ".csv" : filePath.slice(Math.max(0, lastDotIndex)), + lastDotIndex === -1 ? '.csv' : filePath.slice(Math.max(0, lastDotIndex)), }; } @@ -124,7 +124,7 @@ export async function writeLargeCsv( filePath: string, data: ObjByString[], headers: boolean | string[] = true, - chunkSize = 100_000 + chunkSize = 100_000, ): Promise { if (data.length <= chunkSize) { // If data is small enough, write to single file @@ -145,7 +145,7 @@ export async function writeLargeCsv( // Create filename with chunk number and zero-padding const chunkNumber = String(index + 1).padStart( String(totalChunks).length, - "0" + '0', ); const chunkFilePath = `${baseName}_part${chunkNumber}_of_${totalChunks}${extension}`; diff --git a/src/lib/data-inventory/pullAllDatapoints.ts b/src/lib/data-inventory/pullAllDatapoints.ts index ca56995b..9dbbc78d 100644 --- a/src/lib/data-inventory/pullAllDatapoints.ts +++ b/src/lib/data-inventory/pullAllDatapoints.ts @@ -1,22 +1,22 @@ import { SubDataPointDataSubCategoryGuessStatus, type DataCategoryType, -} from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { gql } from "graphql-request"; -import type { GraphQLClient } from "graphql-request"; -import { chunk, keyBy, sortBy, uniq } from "lodash-es"; -import type { DataCategoryInput, ProcessingPurposeInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +} from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { gql } from 'graphql-request'; +import type { GraphQLClient } from 'graphql-request'; +import { chunk, keyBy, sortBy, uniq } from 'lodash-es'; +import type { DataCategoryInput, ProcessingPurposeInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { DATA_SILO_EXPORT, DATAPOINT_EXPORT, makeGraphQLRequest, SUB_DATA_POINTS_COUNT, type DataSiloAttributeValue, -} from "../graphql"; +} from '../graphql'; export interface DataSiloCsvPreview { /** ID of dataSilo */ @@ -99,7 +99,7 @@ async function pullSubDatapoints( }: DatapointFilterOptions & { /** Page size to pull in */ pageSize?: number; - } = {} + } = {}, ): Promise { const subDataPoints: SubDataPointCsvPreview[] = []; @@ -109,7 +109,7 @@ async function pullSubDatapoints( // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Filters @@ -138,7 +138,7 @@ async function pullSubDatapoints( filterBy, }); - logger.info(colors.magenta("[Step 1/3] Pulling in all subdatapoints")); + logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints')); progressBar.start(totalCount, 0); let total = 0; @@ -193,7 +193,7 @@ async function pullSubDatapoints( status classifierVersion }` - : "" + : '' } ${ includeAttributes @@ -203,7 +203,7 @@ async function pullSubDatapoints( } name }` - : "" + : '' } } } @@ -217,7 +217,7 @@ async function pullSubDatapoints( // TODO: https://transcend.height.app/T-40484 - add cursor support // ...(cursor ? { cursor: { id: cursor } } : {}), }, - } + }, ); cursor = nodes.at(-1)?.id; @@ -229,8 +229,8 @@ async function pullSubDatapoints( } catch (error) { logger.error( colors.red( - `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}` - ) + `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`, + ), ); throw error; } @@ -240,14 +240,14 @@ async function pullSubDatapoints( const t1 = Date.now(); const totalTime = t1 - t0; - const sorted = sortBy(subDataPoints, "name"); + const sorted = sortBy(subDataPoints, 'name'); logger.info( colors.green( `Successfully pulled in ${sorted.length} subdatapoints in ${ totalTime / 1000 - } seconds!` - ) + } seconds!`, + ), ); return sorted; } @@ -269,7 +269,7 @@ async function pullDatapoints( dataPointIds: string[]; /** Page size to pull in */ pageSize?: number; - } + }, ): Promise { const dataPoints: DataPointCsvPreview[] = []; @@ -279,13 +279,13 @@ async function pullDatapoints( // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); logger.info( colors.magenta( - `[Step 2/3] Fetching metadata for ${dataPointIds.length} datapoints` - ) + `[Step 2/3] Fetching metadata for ${dataPointIds.length} datapoints`, + ), ); // Group by 100 @@ -317,9 +317,9 @@ async function pullDatapoints( logger.error( colors.red( `An error fetching subdatapoints for IDs ${dataPointIdsGroup.join( - ", " - )}` - ) + ', ', + )}`, + ), ); throw error; } @@ -333,8 +333,8 @@ async function pullDatapoints( colors.green( `Successfully pulled in ${dataPoints.length} dataPoints in ${ totalTime / 1000 - } seconds!` - ) + } seconds!`, + ), ); return dataPoints; } @@ -356,7 +356,7 @@ async function pullDataSilos( dataSiloIds: string[]; /** Page size to pull in */ pageSize?: number; - } + }, ): Promise { const dataSilos: DataSiloCsvPreview[] = []; @@ -366,13 +366,13 @@ async function pullDataSilos( // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); logger.info( colors.magenta( - `[Step 3/3] Fetching metadata for ${dataSiloIds.length} data silos` - ) + `[Step 3/3] Fetching metadata for ${dataSiloIds.length} data silos`, + ), ); // Group by 100 @@ -403,8 +403,8 @@ async function pullDataSilos( } catch (error) { logger.error( colors.red( - `An error fetching data silos for IDs ${dataSiloIdsGroup.join(", ")}` - ) + `An error fetching data silos for IDs ${dataSiloIdsGroup.join(', ')}`, + ), ); throw error; } @@ -418,8 +418,8 @@ async function pullDataSilos( colors.green( `Successfully pulled in ${dataSilos.length} data silos in ${ totalTime / 1000 - } seconds!` - ) + } seconds!`, + ), ); return dataSilos; } @@ -443,7 +443,7 @@ export async function pullAllDatapoints( }: DatapointFilterOptions & { /** Page size to pull in */ pageSize?: number; - } = {} + } = {}, ): Promise< (SubDataPointCsvPreview & { /** Data point information */ @@ -467,14 +467,14 @@ export async function pullAllDatapoints( const dataPoints = await pullDatapoints(client, { dataPointIds, }); - const dataPointById = keyBy(dataPoints, "id"); + const dataPointById = keyBy(dataPoints, 'id'); // The data silo IDs to grab const allDataSiloIds = uniq(subDatapoints.map((point) => point.dataSiloId)); const dataSilos = await pullDataSilos(client, { dataSiloIds: allDataSiloIds, }); - const dataSiloById = keyBy(dataSilos, "id"); + const dataSiloById = keyBy(dataSilos, 'id'); return subDatapoints.map((subDataPoint) => ({ ...subDataPoint, diff --git a/src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts b/src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts index f595d356..5dbad1b6 100644 --- a/src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts +++ b/src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts @@ -1,11 +1,11 @@ -import type { UnstructuredSubDataPointRecommendationStatus } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { gql, type GraphQLClient } from "graphql-request"; -import { sortBy } from "lodash-es"; -import type { DataCategoryInput } from "../../codecs"; -import { logger } from "../../logger"; -import { ENTRY_COUNT, makeGraphQLRequest } from "../graphql"; +import type { UnstructuredSubDataPointRecommendationStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { gql, type GraphQLClient } from 'graphql-request'; +import { sortBy } from 'lodash-es'; +import type { DataCategoryInput } from '../../codecs'; +import { logger } from '../../logger'; +import { ENTRY_COUNT, makeGraphQLRequest } from '../graphql'; interface UnstructuredSubDataPointRecommendationCsvPreview { /** ID of subDatapoint */ @@ -68,7 +68,7 @@ export async function pullUnstructuredSubDataPointRecommendations( }: EntryFilterOptions & { /** Page size to pull in */ pageSize?: number; - } = {} + } = {}, ): Promise { const unstructuredSubDataPointRecommendations: UnstructuredSubDataPointRecommendationCsvPreview[] = []; @@ -79,7 +79,7 @@ export async function pullUnstructuredSubDataPointRecommendations( // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Filters @@ -102,7 +102,7 @@ export async function pullUnstructuredSubDataPointRecommendations( filterBy, }); - logger.info(colors.magenta("[Step 1/3] Pulling in all subdatapoints")); + logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints')); progressBar.start(totalCount, 0); let total = 0; @@ -138,8 +138,8 @@ export async function pullUnstructuredSubDataPointRecommendations( dataSiloId scannedObjectPathId scannedObjectId - ${includeEncryptedSnippets ? "name" : ""} - ${includeEncryptedSnippets ? "contextSnippet" : ""} + ${includeEncryptedSnippets ? 'name' : ''} + ${includeEncryptedSnippets ? 'contextSnippet' : ''} dataSubCategory { name category @@ -158,7 +158,7 @@ export async function pullUnstructuredSubDataPointRecommendations( filterBy: { ...filterBy, }, - } + }, ); cursor = nodes.at(-1)?.id; @@ -170,8 +170,8 @@ export async function pullUnstructuredSubDataPointRecommendations( } catch (error) { logger.error( colors.red( - `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}` - ) + `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`, + ), ); throw error; } @@ -181,14 +181,14 @@ export async function pullUnstructuredSubDataPointRecommendations( const t1 = Date.now(); const totalTime = t1 - t0; - const sorted = sortBy(unstructuredSubDataPointRecommendations, "name"); + const sorted = sortBy(unstructuredSubDataPointRecommendations, 'name'); logger.info( colors.green( `Successfully pulled in ${sorted.length} subdatapoints in ${ totalTime / 1000 - } seconds!` - ) + } seconds!`, + ), ); return sorted; } diff --git a/src/lib/graphql/fetchAllAssessments.ts b/src/lib/graphql/fetchAllAssessments.ts index 4400472d..19319714 100644 --- a/src/lib/graphql/fetchAllAssessments.ts +++ b/src/lib/graphql/fetchAllAssessments.ts @@ -9,10 +9,10 @@ import { ProcessingPurpose, RetentionScheduleOperation, RetentionScheduleType, -} from "@transcend-io/privacy-types"; -import { GraphQLClient } from "graphql-request"; -import { ASSESSMENTS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; +import { ASSESSMENTS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Represents an assessment with various properties and metadata. @@ -340,7 +340,7 @@ const PAGE_SIZE = 20; * @returns All assessments in the organization */ export async function fetchAllAssessments( - client: GraphQLClient + client: GraphQLClient, ): Promise { const assessments: Assessment[] = []; let offset = 0; diff --git a/src/lib/graphql/fetchAllRequestIdentifiers.ts b/src/lib/graphql/fetchAllRequestIdentifiers.ts index 52646f7a..d81ff2c8 100644 --- a/src/lib/graphql/fetchAllRequestIdentifiers.ts +++ b/src/lib/graphql/fetchAllRequestIdentifiers.ts @@ -1,13 +1,13 @@ -import { IdentifierType } from "@transcend-io/privacy-types"; -import { decodeCodec, valuesOf } from "@transcend-io/type-utils"; -import type { Got } from "got"; -import { GraphQLClient } from "graphql-request"; -import * as t from "io-ts"; -import semver from "semver"; -import { SOMBRA_VERSION } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { IdentifierType } from '@transcend-io/privacy-types'; +import { decodeCodec, valuesOf } from '@transcend-io/type-utils'; +import type { Got } from 'got'; +import { GraphQLClient } from 'graphql-request'; +import * as t from 'io-ts'; +import semver from 'semver'; +import { SOMBRA_VERSION } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; -const MIN_SOMBRA_VERSION_TO_DECRYPT = "7.180"; +const MIN_SOMBRA_VERSION_TO_DECRYPT = '7.180'; const RequestIdentifier = t.type({ /** ID of request */ @@ -45,7 +45,7 @@ export async function fetchAllRequestIdentifiers( }: { /** ID of request to filter on */ requestId: string; - } + }, ): Promise { const requestIdentifiers: RequestIdentifier[] = []; let offset = 0; @@ -69,7 +69,7 @@ export async function fetchAllRequestIdentifiers( if (version && semver.lt(version, MIN_SOMBRA_VERSION_TO_DECRYPT)) { throw new Error( - `Please upgrade Sombra to ${MIN_SOMBRA_VERSION_TO_DECRYPT} or greater to use this command.` + `Please upgrade Sombra to ${MIN_SOMBRA_VERSION_TO_DECRYPT} or greater to use this command.`, ); } @@ -80,7 +80,7 @@ export async function fetchAllRequestIdentifiers( .post<{ /** Decrypted identifiers */ identifiers: RequestIdentifier[]; - }>("v1/request-identifiers", { + }>('v1/request-identifiers', { json: { first: PAGE_SIZE, offset, @@ -92,13 +92,13 @@ export async function fetchAllRequestIdentifiers( throw new Error( `Failed to fetch request identifiers: ${ error?.response?.body || error?.message - }` + }`, ); } const { identifiers: nodes } = decodeCodec( RequestIdentifiersResponse, - response + response, ); requestIdentifiers.push(...nodes); diff --git a/src/lib/graphql/fetchAllRequests.ts b/src/lib/graphql/fetchAllRequests.ts index 3d231e8b..44ac99dc 100644 --- a/src/lib/graphql/fetchAllRequests.ts +++ b/src/lib/graphql/fetchAllRequests.ts @@ -1,19 +1,19 @@ -import { LanguageKey } from "@transcend-io/internationalization"; +import { LanguageKey } from '@transcend-io/internationalization'; import { IsoCountryCode, IsoCountrySubdivisionCode, RequestAction, RequestOrigin, RequestStatus, -} from "@transcend-io/privacy-types"; -import { valuesOf } from "@transcend-io/type-utils"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import * as t from "io-ts"; -import { logger } from "../../logger"; -import { REQUESTS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from '@transcend-io/privacy-types'; +import { valuesOf } from '@transcend-io/type-utils'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import * as t from 'io-ts'; +import { logger } from '../../logger'; +import { REQUESTS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export const PrivacyRequest = t.intersection([ t.type({ @@ -53,7 +53,7 @@ export const PrivacyRequest = t.intersection([ id: t.string, attributeKey: t.type({ name: t.string, id: t.string }), name: t.string, - }) + }), ), }), t.partial({ @@ -111,15 +111,15 @@ export async function fetchAllRequests( * at runtime while other filters are applied at the GraphQL level. */ requestIds?: string[]; - } = {} + } = {}, ): Promise { - logger.info(colors.magenta("Fetching requests...")); + logger.info(colors.magenta('Fetching requests...')); // create a new progress bar instance and use shades_classic theme const t0 = Date.now(); const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // read in requests @@ -178,20 +178,20 @@ export async function fetchAllRequests( colors.green( `Completed fetching of ${requests.length} request in "${ totalTime / 1000 - }" seconds.` - ) + }" seconds.`, + ), ); // Filter down requests by request ID let allRequests = requests; if (requestIds && requestIds.length > 0) { allRequests = allRequests.filter((request) => - requestIds.includes(request.id) + requestIds.includes(request.id), ); logger.info( colors.green( - `Filtered down to ${allRequests.length} requests based on ${requestIds.length} provided IDs.` - ) + `Filtered down to ${allRequests.length} requests based on ${requestIds.length} provided IDs.`, + ), ); } diff --git a/src/lib/graphql/fetchApiKeys.ts b/src/lib/graphql/fetchApiKeys.ts index cd7a0bec..64cdb963 100644 --- a/src/lib/graphql/fetchApiKeys.ts +++ b/src/lib/graphql/fetchApiKeys.ts @@ -1,10 +1,10 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { difference, keyBy, uniq } from "lodash-es"; -import { TranscendInput } from "../../codecs"; -import { logger } from "../../logger"; -import { API_KEYS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { difference, keyBy, uniq } from 'lodash-es'; +import { TranscendInput } from '../../codecs'; +import { logger } from '../../logger'; +import { API_KEYS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface ApiKey { /** ID of API key */ @@ -15,7 +15,7 @@ export interface ApiKey { const PAGE_SIZE = 20; -const ADMIN_LINK = "https://app.transcend.io/infrastructure/api-keys"; +const ADMIN_LINK = 'https://app.transcend.io/infrastructure/api-keys'; /** * Fetch all API keys in an organization @@ -26,7 +26,7 @@ const ADMIN_LINK = "https://app.transcend.io/infrastructure/api-keys"; */ export async function fetchAllApiKeys( client: GraphQLClient, - titles?: string[] + titles?: string[], ): Promise { const apiKeys: ApiKey[] = []; let offset = 0; @@ -65,36 +65,36 @@ export async function fetchAllApiKeys( */ export async function fetchApiKeys( { - "api-keys": apiKeyInputs = [], - "data-silos": dataSilos = [], + 'api-keys': apiKeyInputs = [], + 'data-silos': dataSilos = [], }: TranscendInput, client: GraphQLClient, - fetchAll = false + fetchAll = false, ): Promise> { logger.info( colors.magenta( - `Fetching ${fetchAll ? "all" : apiKeyInputs.length} API keys...` - ) + `Fetching ${fetchAll ? 'all' : apiKeyInputs.length} API keys...`, + ), ); const titles = apiKeyInputs.map(({ title }) => title); const expectedApiKeyTitles = uniq( dataSilos - .map((silo) => silo["api-key-title"]) - .filter((x): x is string => !!x) + .map((silo) => silo['api-key-title']) + .filter((x): x is string => !!x), ); const allTitlesExpected = [...expectedApiKeyTitles, ...titles]; const apiKeys = await fetchAllApiKeys( client, - fetchAll ? undefined : [...expectedApiKeyTitles, ...titles] + fetchAll ? undefined : [...expectedApiKeyTitles, ...titles], ); // Create a map - const apiKeysByTitle = keyBy(apiKeys, "title"); + const apiKeysByTitle = keyBy(apiKeys, 'title'); // Determine expected set of apiKeys expected const missingApiKeys = difference( allTitlesExpected, - apiKeys.map(({ title }) => title) + apiKeys.map(({ title }) => title), ); // If there are missing apiKeys, throw an error @@ -102,9 +102,9 @@ export async function fetchApiKeys( logger.info( colors.red( `Failed to find API keys "${missingApiKeys.join( - '", "' - )}"! Make sure these API keys are created at: ${ADMIN_LINK}` - ) + '", "', + )}"! Make sure these API keys are created at: ${ADMIN_LINK}`, + ), ); process.exit(1); } diff --git a/src/lib/graphql/fetchCatalogs.ts b/src/lib/graphql/fetchCatalogs.ts index d8992af2..75502a95 100644 --- a/src/lib/graphql/fetchCatalogs.ts +++ b/src/lib/graphql/fetchCatalogs.ts @@ -1,6 +1,6 @@ -import { GraphQLClient } from "graphql-request"; -import { CATALOGS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { GraphQLClient } from 'graphql-request'; +import { CATALOGS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface Catalog { /** Integration name */ @@ -20,7 +20,7 @@ const PAGE_SIZE = 100; * @returns Integration catalogs */ export async function fetchAllCatalogs( - client: GraphQLClient + client: GraphQLClient, ): Promise { const catalogs: Catalog[] = []; let offset = 0; @@ -45,7 +45,7 @@ export async function fetchAllCatalogs( shouldContinue = nodes.length === PAGE_SIZE; } while (shouldContinue); return catalogs.sort((a, b) => - a.integrationName.localeCompare(b.integrationName) + a.integrationName.localeCompare(b.integrationName), ); } @@ -76,7 +76,7 @@ export async function fetchAndIndexCatalogs(client: GraphQLClient): Promise< catalogs.map>((catalog) => [ catalog.integrationName, catalog.title, - ]) + ]), ); // Create mapping from service name to boolean indicate if service has API integration support @@ -84,7 +84,7 @@ export async function fetchAndIndexCatalogs(client: GraphQLClient): Promise< catalogs.map>((catalog) => [ catalog.integrationName, catalog.hasApiFunctionality, - ]) + ]), ); return { diff --git a/src/lib/graphql/fetchDataSubjects.ts b/src/lib/graphql/fetchDataSubjects.ts index 3d36bc9d..fcfcb0ed 100644 --- a/src/lib/graphql/fetchDataSubjects.ts +++ b/src/lib/graphql/fetchDataSubjects.ts @@ -1,12 +1,12 @@ -import { RequestActionObjectResolver } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { difference, flatten, keyBy, uniq } from "lodash-es"; -import { TranscendInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { CREATE_DATA_SUBJECT, DATA_SUBJECTS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { RequestActionObjectResolver } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { difference, flatten, keyBy, uniq } from 'lodash-es'; +import { TranscendInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { CREATE_DATA_SUBJECT, DATA_SUBJECTS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface DataSubject { /** ID of data subject */ @@ -36,7 +36,7 @@ export interface DataSubject { * @returns List of data subject configurations */ export async function fetchAllDataSubjects( - client: GraphQLClient + client: GraphQLClient, ): Promise { // Fetch all data subjects in the organization const { internalSubjects } = await makeGraphQLRequest<{ @@ -56,18 +56,18 @@ export async function fetchAllDataSubjects( */ export async function ensureAllDataSubjectsExist( { - "data-silos": dataSilos = [], - "data-subjects": dataSubjects = [], + 'data-silos': dataSilos = [], + 'data-subjects': dataSubjects = [], enrichers = [], }: TranscendInput, client: GraphQLClient, - fetchAll = false + fetchAll = false, ): Promise> { // Only need to fetch data subjects if specified in config const expectedDataSubjects = uniq([ - ...flatten(dataSilos.map((silo) => silo["data-subjects"] || []) || []), + ...flatten(dataSilos.map((silo) => silo['data-subjects'] || []) || []), ...flatten( - enrichers.map((enricher) => enricher["data-subjects"] || []) || [] + enrichers.map((enricher) => enricher['data-subjects'] || []) || [], ), ...dataSubjects.map((subject) => subject.type), ]); @@ -77,20 +77,20 @@ export async function ensureAllDataSubjectsExist( // Fetch all data subjects in the organization const internalSubjects = await fetchAllDataSubjects(client); - const dataSubjectByName = keyBy(internalSubjects, "type"); + const dataSubjectByName = keyBy(internalSubjects, 'type'); // Determine expected set of data subjects to create const missingDataSubjects = difference( expectedDataSubjects, - internalSubjects.map(({ type }) => type) + internalSubjects.map(({ type }) => type), ); // If there are missing data subjects, create new ones if (missingDataSubjects.length > 0) { logger.info( colors.magenta( - `Creating ${missingDataSubjects.length} new data subjects...` - ) + `Creating ${missingDataSubjects.length} new data subjects...`, + ), ); await mapSeries(missingDataSubjects, async (dataSubject) => { logger.info(colors.magenta(`Creating data subject ${dataSubject}...`)); @@ -122,7 +122,7 @@ export async function ensureAllDataSubjectsExist( */ export function convertToDataSubjectBlockList( dataSubjectTypes: string[], - allDataSubjects: Record + allDataSubjects: Record, ): string[] { for (const type of dataSubjectTypes) { if (!allDataSubjects[type]) { @@ -144,7 +144,7 @@ export function convertToDataSubjectBlockList( */ export function convertToDataSubjectAllowlist( dataSubjectTypes: string[], - allDataSubjects: Record + allDataSubjects: Record, ): string[] { for (const type of dataSubjectTypes) { if (!allDataSubjects[type]) { diff --git a/src/lib/graphql/fetchIdentifiers.ts b/src/lib/graphql/fetchIdentifiers.ts index d17e0913..9ff7b3ef 100644 --- a/src/lib/graphql/fetchIdentifiers.ts +++ b/src/lib/graphql/fetchIdentifiers.ts @@ -1,12 +1,12 @@ -import { IdentifierType, RequestAction } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { difference, flatten, keyBy, uniq } from "lodash-es"; -import { TranscendInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { CREATE_IDENTIFIER, IDENTIFIERS, NEW_IDENTIFIER_TYPES } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { IdentifierType, RequestAction } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { difference, flatten, keyBy, uniq } from 'lodash-es'; +import { TranscendInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { CREATE_IDENTIFIER, IDENTIFIERS, NEW_IDENTIFIER_TYPES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface Identifier { /** ID of identifier */ @@ -53,7 +53,7 @@ const PAGE_SIZE = 20; * @returns All identifiers in the organization */ export async function fetchAllIdentifiers( - client: GraphQLClient + client: GraphQLClient, ): Promise { const identifiers: Identifier[] = []; let offset = 0; @@ -93,38 +93,40 @@ export async function fetchAllIdentifiers( export async function fetchIdentifiersAndCreateMissing( { enrichers = [], - "data-silos": dataSilos = [], + 'data-silos': dataSilos = [], identifiers = [], }: TranscendInput, client: GraphQLClient, - skipPublish = false + skipPublish = false, ): Promise> { // Grab all existing identifiers const allIdentifiers = await fetchAllIdentifiers(client); // Create a map - const identifiersByName = keyBy(allIdentifiers, "name"); + const identifiersByName = keyBy(allIdentifiers, 'name'); // Determine expected set of identifiers const expectedIdentifiers = uniq([ ...flatten( enrichers.map((enricher) => [ - enricher["input-identifier"], - ...enricher["output-identifiers"], - ]) + enricher['input-identifier'], + ...enricher['output-identifiers'], + ]), ), - ...flatten(dataSilos.map((dataSilo) => dataSilo["identity-keys"])), + ...flatten(dataSilos.map((dataSilo) => dataSilo['identity-keys'])), ...identifiers.map(({ name }) => name), ]).filter((x) => !!x); const missingIdentifiers = difference( expectedIdentifiers, - allIdentifiers.map(({ name }) => name) + allIdentifiers.map(({ name }) => name), ); // If there are missing identifiers, create new ones if (missingIdentifiers.length > 0) { logger.info( - colors.magenta(`Creating ${missingIdentifiers.length} new identifiers...`) + colors.magenta( + `Creating ${missingIdentifiers.length} new identifiers...`, + ), ); const { newIdentifierTypes } = await makeGraphQLRequest<{ /** Query response */ @@ -134,7 +136,7 @@ export async function fetchIdentifiersAndCreateMissing( }[]; }>(client, NEW_IDENTIFIER_TYPES); const nativeTypesRemaining = new Set( - newIdentifierTypes.map(({ name }) => name) + newIdentifierTypes.map(({ name }) => name), ); await mapSeries(missingIdentifiers, async (identifier) => { logger.info(colors.magenta(`Creating identifier ${identifier}...`)); @@ -147,7 +149,7 @@ export async function fetchIdentifiersAndCreateMissing( }>(client, CREATE_IDENTIFIER, { input: { name: identifier, - type: nativeTypesRemaining.has(identifier!) ? identifier : "custom", + type: nativeTypesRemaining.has(identifier!) ? identifier : 'custom', skipPublish, }, }); diff --git a/src/lib/graphql/fetchPrompts.ts b/src/lib/graphql/fetchPrompts.ts index 6d26c475..fbe02686 100644 --- a/src/lib/graphql/fetchPrompts.ts +++ b/src/lib/graphql/fetchPrompts.ts @@ -1,10 +1,10 @@ import { PromptResponseFormat, PromptStatus, -} from "@transcend-io/privacy-types"; -import { GraphQLClient } from "graphql-request"; -import { PROMPTS, PROMPTS_WITH_VARIABLES } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; +import { PROMPTS, PROMPTS_WITH_VARIABLES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface Prompt { /** ID of prompt */ @@ -45,7 +45,7 @@ export async function fetchAllPrompts( ids?: string[]; /** Filter by titles */ titles?: string[]; - } = {} + } = {}, ): Promise { const prompts: Prompt[] = []; let offset = 0; @@ -165,7 +165,7 @@ export async function fetchPromptsWithVariables( promptIds?: string[]; /** Filter by prompt titles */ promptTitles?: string[]; - } = {} + } = {}, ): Promise { const { promptsWithVariables } = await makeGraphQLRequest<{ /** Prompts */ diff --git a/src/lib/graphql/fetchRequestDataSilo.ts b/src/lib/graphql/fetchRequestDataSilo.ts index bd1c3c31..5e01d303 100644 --- a/src/lib/graphql/fetchRequestDataSilo.ts +++ b/src/lib/graphql/fetchRequestDataSilo.ts @@ -1,13 +1,13 @@ import { RequestDataSiloStatus, RequestStatus, -} from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { logger } from "../../logger"; -import { REQUEST_DATA_SILOS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { logger } from '../../logger'; +import { REQUEST_DATA_SILOS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface RequestDataSilo { /** ID of RequestDataSilo */ @@ -44,13 +44,13 @@ export async function fetchRequestDataSilos( requestStatuses?: RequestStatus[]; /** When true, skip logging */ skipLog?: boolean; - } + }, ): Promise { // create a new progress bar instance and use shades_classic theme const t0 = Date.now(); const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); const requestDataSilos: RequestDataSilo[] = []; @@ -101,8 +101,8 @@ export async function fetchRequestDataSilos( colors.green( `Completed fetching of ${ requestDataSilos.length - } request data silos in "${totalTime / 1000}" seconds.` - ) + } request data silos in "${totalTime / 1000}" seconds.`, + ), ); } @@ -126,7 +126,7 @@ export async function fetchRequestDataSilo( requestId: string; /** Data silo ID */ dataSiloId: string; - } + }, ): Promise { const nodes = await fetchRequestDataSilos(client, { requestId, @@ -135,7 +135,7 @@ export async function fetchRequestDataSilo( }); if (nodes.length !== 1) { throw new Error( - `Failed to find RequestDataSilo with requestId:${requestId},dataSiloId:${dataSiloId}` + `Failed to find RequestDataSilo with requestId:${requestId},dataSiloId:${dataSiloId}`, ); } diff --git a/src/lib/graphql/formatAttributeValues.ts b/src/lib/graphql/formatAttributeValues.ts index f3e0a96f..87af79a8 100644 --- a/src/lib/graphql/formatAttributeValues.ts +++ b/src/lib/graphql/formatAttributeValues.ts @@ -1,4 +1,4 @@ -import type { DataSiloAttributeValue } from "./syncDataSilos"; +import type { DataSiloAttributeValue } from './syncDataSilos'; export interface FormattedAttribute { /** Attribute key */ @@ -14,13 +14,13 @@ export interface FormattedAttribute { * @returns formatted attributes */ export function formatAttributeValues( - vals: DataSiloAttributeValue[] + vals: DataSiloAttributeValue[], ): FormattedAttribute[] { const attributes: FormattedAttribute[] = []; vals.map((value) => { let foundKey = attributes.find( - (att) => att.key === value.attributeKey.name + (att) => att.key === value.attributeKey.name, ); if (foundKey === undefined) { diff --git a/src/lib/graphql/gqls/consentManager.ts b/src/lib/graphql/gqls/consentManager.ts index da2af09f..72b4428b 100644 --- a/src/lib/graphql/gqls/consentManager.ts +++ b/src/lib/graphql/gqls/consentManager.ts @@ -1,4 +1,4 @@ -import { gql } from "graphql-request"; +import { gql } from 'graphql-request'; // TODO: https://transcend.height.app/T-27909 - order by createdAt // TODO: https://transcend.height.app/T-27909 - enable optimizations diff --git a/src/lib/graphql/loginUser.ts b/src/lib/graphql/loginUser.ts index 1bd6afb9..cb1cb462 100644 --- a/src/lib/graphql/loginUser.ts +++ b/src/lib/graphql/loginUser.ts @@ -1,6 +1,6 @@ -import { GraphQLClient } from "graphql-request"; -import { ASSUME_ROLE, DETERMINE_LOGIN_METHOD, LOGIN } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { GraphQLClient } from 'graphql-request'; +import { ASSUME_ROLE, DETERMINE_LOGIN_METHOD, LOGIN } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface OrganizationPreview { /** Name of organization */ @@ -37,7 +37,7 @@ export async function loginUser( email: string; /** Password of user */ password: string; - } + }, ): Promise<{ /** Cookie to be used to make subsequent requests */ loginCookie: string; @@ -80,9 +80,9 @@ export async function loginUser( } = res.data; // Get login cookie from response - const loginCookie = res.headers.get("set-cookie"); - if (!loginCookie?.includes("laravel")) { - throw new Error("Failed to get login cookie in response"); + const loginCookie = res.headers.get('set-cookie'); + if (!loginCookie?.includes('laravel')) { + throw new Error('Failed to get login cookie in response'); } return { @@ -107,7 +107,7 @@ export async function assumeRole( email: string; /** Role of user assuming into */ roleId: string; - } + }, ): Promise { const { determineLoginMethod: { loginMethod }, diff --git a/src/lib/graphql/makeGraphQLRequest.ts b/src/lib/graphql/makeGraphQLRequest.ts index a4bfe985..ea13b0f8 100644 --- a/src/lib/graphql/makeGraphQLRequest.ts +++ b/src/lib/graphql/makeGraphQLRequest.ts @@ -1,10 +1,10 @@ -import colors from "colors"; +import colors from 'colors'; import type { GraphQLClient, RequestDocument, Variables, -} from "graphql-request"; -import { logger } from "../../logger"; +} from 'graphql-request'; +import { logger } from '../../logger'; const MAX_RETRIES = 4; @@ -23,11 +23,11 @@ function sleepPromise(sleepTime: number): Promise { } const KNOWN_ERRORS = [ - "syntax error", - "got invalid value", - "Client error", - "cannot affect row a second time", - "GRAPHQL_VALIDATION_FAILED", + 'syntax error', + 'got invalid value', + 'Client error', + 'cannot affect row a second time', + 'GRAPHQL_VALIDATION_FAILED', ]; /** @@ -45,7 +45,7 @@ export async function makeGraphQLRequest( document: RequestDocument, variables?: V, requestHeaders?: Record | string[][] | Headers, - maxRequests = MAX_RETRIES + maxRequests = MAX_RETRIES, ): Promise { let retryCount = 0; @@ -54,13 +54,13 @@ export async function makeGraphQLRequest( const result = await client.request(document, variables, requestHeaders); return result as T; } catch (error) { - if (error.message.includes("API key is invalid")) { + if (error.message.includes('API key is invalid')) { logger.error( colors.red( - "API key is invalid. " + - "Please ensure that the key provided to `transcendAuth` has the proper scope and is not expired, " + - "and that `transcendUrl` corresponds to the correct backend for your organization." - ) + 'API key is invalid. ' + + 'Please ensure that the key provided to `transcendAuth` has the proper scope and is not expired, ' + + 'and that `transcendUrl` corresponds to the correct backend for your organization.', + ), ); process.exit(1); } @@ -70,16 +70,16 @@ export async function makeGraphQLRequest( } // wait for rate limit to resolve - if (error.message.startsWith("Client error: Too many requests")) { + if (error.message.startsWith('Client error: Too many requests')) { const rateLimitResetAt = - error.response.headers?.get("x-ratelimit-reset"); + error.response.headers?.get('x-ratelimit-reset'); const sleepTime = rateLimitResetAt ? new Date(rateLimitResetAt).getTime() - Date.now() + 100 : 1000 * 10; logger.log( colors.yellow( - `DETECTED RATE LIMIT: ${error.message}. Sleeping for ${sleepTime}ms` - ) + `DETECTED RATE LIMIT: ${error.message}. Sleeping for ${sleepTime}ms`, + ), ); await sleepPromise(sleepTime); @@ -91,8 +91,8 @@ export async function makeGraphQLRequest( retryCount += 1; logger.log( colors.yellow( - `REQUEST FAILED: ${error.message}. Retrying ${retryCount}/${maxRequests}...` - ) + `REQUEST FAILED: ${error.message}. Retrying ${retryCount}/${maxRequests}...`, + ), ); } } diff --git a/src/lib/graphql/pullTranscendConfiguration.ts b/src/lib/graphql/pullTranscendConfiguration.ts index 4d9ffdc1..c07bc102 100644 --- a/src/lib/graphql/pullTranscendConfiguration.ts +++ b/src/lib/graphql/pullTranscendConfiguration.ts @@ -1,12 +1,12 @@ -import { LanguageKey } from "@transcend-io/internationalization"; +import { LanguageKey } from '@transcend-io/internationalization'; import { ActionItemCode, ConsentTrackerStatus, RequestAction, -} from "@transcend-io/privacy-types"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { flatten, keyBy, mapValues } from "lodash-es"; +} from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { flatten, keyBy, mapValues } from 'lodash-es'; import { ActionInput, ActionItemCollectionInput, @@ -41,53 +41,53 @@ import { TeamInput, TranscendInput, VendorInput, -} from "../../codecs"; -import { TranscendPullResource } from "../../enums"; -import { logger } from "../../logger"; -import { fetchAllActionItemCollections } from "./fetchAllActionItemCollections"; -import { fetchAllActionItems } from "./fetchAllActionItems"; -import { fetchAllActions } from "./fetchAllActions"; -import { fetchAllAgentFiles } from "./fetchAllAgentFiles"; -import { fetchAllAgentFunctions } from "./fetchAllAgentFunctions"; -import { fetchAllAgents } from "./fetchAllAgents"; -import { fetchAllAssessments } from "./fetchAllAssessments"; -import { fetchAllAssessmentTemplates } from "./fetchAllAssessmentTemplates"; -import { fetchAllAttributes } from "./fetchAllAttributes"; -import { fetchAllBusinessEntities } from "./fetchAllBusinessEntities"; -import { fetchAllCookies } from "./fetchAllCookies"; -import { fetchAllDataCategories } from "./fetchAllDataCategories"; -import { fetchAllDataFlows } from "./fetchAllDataFlows"; -import { fetchAllMessages } from "./fetchAllMessages"; -import { fetchAllPolicies } from "./fetchAllPolicies"; -import { fetchAllPrivacyCenters } from "./fetchAllPrivacyCenters"; -import { fetchAllProcessingPurposes } from "./fetchAllProcessingPurposes"; -import { fetchAllPurposesAndPreferences } from "./fetchAllPurposesAndPreferences"; -import { fetchAllTeams } from "./fetchAllTeams"; -import { fetchAllVendors } from "./fetchAllVendors"; -import { fetchApiKeys } from "./fetchApiKeys"; +} from '../../codecs'; +import { TranscendPullResource } from '../../enums'; +import { logger } from '../../logger'; +import { fetchAllActionItemCollections } from './fetchAllActionItemCollections'; +import { fetchAllActionItems } from './fetchAllActionItems'; +import { fetchAllActions } from './fetchAllActions'; +import { fetchAllAgentFiles } from './fetchAllAgentFiles'; +import { fetchAllAgentFunctions } from './fetchAllAgentFunctions'; +import { fetchAllAgents } from './fetchAllAgents'; +import { fetchAllAssessments } from './fetchAllAssessments'; +import { fetchAllAssessmentTemplates } from './fetchAllAssessmentTemplates'; +import { fetchAllAttributes } from './fetchAllAttributes'; +import { fetchAllBusinessEntities } from './fetchAllBusinessEntities'; +import { fetchAllCookies } from './fetchAllCookies'; +import { fetchAllDataCategories } from './fetchAllDataCategories'; +import { fetchAllDataFlows } from './fetchAllDataFlows'; +import { fetchAllMessages } from './fetchAllMessages'; +import { fetchAllPolicies } from './fetchAllPolicies'; +import { fetchAllPrivacyCenters } from './fetchAllPrivacyCenters'; +import { fetchAllProcessingPurposes } from './fetchAllProcessingPurposes'; +import { fetchAllPurposesAndPreferences } from './fetchAllPurposesAndPreferences'; +import { fetchAllTeams } from './fetchAllTeams'; +import { fetchAllVendors } from './fetchAllVendors'; +import { fetchApiKeys } from './fetchApiKeys'; import { fetchConsentManager, fetchConsentManagerExperiences, fetchConsentManagerTheme, -} from "./fetchConsentManagerId"; +} from './fetchConsentManagerId'; import { convertToDataSubjectAllowlist, fetchAllDataSubjects, -} from "./fetchDataSubjects"; -import { fetchAllIdentifiers } from "./fetchIdentifiers"; -import { fetchAllPromptGroups } from "./fetchPromptGroups"; -import { fetchAllPromptPartials } from "./fetchPromptPartials"; -import { fetchAllPrompts } from "./fetchPrompts"; -import { formatAttributeValues } from "./formatAttributeValues"; +} from './fetchDataSubjects'; +import { fetchAllIdentifiers } from './fetchIdentifiers'; +import { fetchAllPromptGroups } from './fetchPromptGroups'; +import { fetchAllPromptPartials } from './fetchPromptPartials'; +import { fetchAllPrompts } from './fetchPrompts'; +import { formatAttributeValues } from './formatAttributeValues'; import { AssessmentNestedRule, parseAssessmentDisplayLogic, -} from "./parseAssessmentDisplayLogic"; -import { parseAssessmentRiskLogic } from "./parseAssessmentRiskLogic"; -import { fetchEnrichedDataSilos } from "./syncDataSilos"; -import { fetchAllEnrichers } from "./syncEnrichers"; -import { fetchPartitions } from "./syncPartitions"; -import { fetchAllTemplates } from "./syncTemplates"; +} from './parseAssessmentDisplayLogic'; +import { parseAssessmentRiskLogic } from './parseAssessmentRiskLogic'; +import { fetchEnrichedDataSilos } from './syncDataSilos'; +import { fetchAllEnrichers } from './syncEnrichers'; +import { fetchPartitions } from './syncPartitions'; +import { fetchAllTemplates } from './syncTemplates'; export const DEFAULT_TRANSCEND_PULL_RESOURCES = [ TranscendPullResource.DataSilos, @@ -136,11 +136,11 @@ export async function pullTranscendConfiguration( includeGuessedCategories, skipSubDatapoints, trackerStatuses = Object.values(ConsentTrackerStatus), - }: TranscendPullConfigurationInput + }: TranscendPullConfigurationInput, ): Promise { if (dataSiloIds.length > 0 && integrationNames.length > 0) { throw new Error( - "Only 1 of integrationNames OR dataSiloIds can be provided" + 'Only 1 of integrationNames OR dataSiloIds can be provided', ); } @@ -343,21 +343,21 @@ export async function pullTranscendConfiguration( // Save API keys const apiKeyTitles = flatten( - dataSilos.map(([{ apiKeys }]) => apiKeys.map(({ title }) => title)) + dataSilos.map(([{ apiKeys }]) => apiKeys.map(({ title }) => title)), ); const relevantApiKeys = Object.values(apiKeyTitleMap).filter(({ title }) => resources.includes(TranscendPullResource.ApiKeys) ? true - : apiKeyTitles.includes(title) + : apiKeyTitles.includes(title), ); if ( relevantApiKeys.length > 0 && resources.includes(TranscendPullResource.ApiKeys) ) { - result["api-keys"] = relevantApiKeys.map( + result['api-keys'] = relevantApiKeys.map( ({ title }): ApiKeyInput => ({ title, - }) + }), ); } @@ -377,7 +377,7 @@ export async function pullTranscendConfiguration( consentManager && resources.includes(TranscendPullResource.ConsentManager) ) { - result["consent-manager"] = { + result['consent-manager'] = { bundleUrls: { TEST: consentManager.testBundleURL, PRODUCTION: consentManager.bundleURL, @@ -503,30 +503,30 @@ export async function pullTranscendConfiguration( return { title, type, - "sub-type": subType, + 'sub-type': subType, placeholder, description, - "is-required": isRequired, - "reference-id": referenceId, - "display-logic": + 'is-required': isRequired, + 'reference-id': referenceId, + 'display-logic': displayLogicParsed && Object.keys(displayLogicParsed).length > 0 ? { action: displayLogicParsed.action!, rule: displayLogicParsed.rule ? { - "depends-on-question-reference-id": + 'depends-on-question-reference-id': displayLogicParsed.rule .dependsOnQuestionReferenceId, - "comparison-operator": + 'comparison-operator': displayLogicParsed.rule.comparisonOperator, - "comparison-operands": + 'comparison-operands': displayLogicParsed.rule.comparisonOperands, } : undefined, - "nested-rule": displayLogicParsed.nestedRule + 'nested-rule': displayLogicParsed.nestedRule ? { - "logic-operator": + 'logic-operator': displayLogicParsed.nestedRule.logicOperator, rules: /* eslint-disable @typescript-eslint/no-explicit-any */ @@ -536,11 +536,11 @@ export async function pullTranscendConfiguration( .nestedRule as AssessmentNestedRule ).rules || [] ).map((rule) => ({ - "depends-on-question-reference-id": + 'depends-on-question-reference-id': rule.dependsOnQuestionReferenceId, - "comparison-operator": + 'comparison-operator': rule.comparisonOperator, - "comparison-operands": + 'comparison-operands': rule.comparisonOperands, })), /* eslint-enable @typescript-eslint/no-explicit-any */ @@ -548,55 +548,55 @@ export async function pullTranscendConfiguration( : undefined, } : undefined, - "risk-logic": riskLogic.map((logic): RiskLogicInput => { + 'risk-logic': riskLogic.map((logic): RiskLogicInput => { const parsed = parseAssessmentRiskLogic(logic); return { - "risk-level": parsed.riskAssignment?.riskLevelId, - "comparison-operands": parsed.comparisonOperands, - "comparison-operator": parsed.comparisonOperator, + 'risk-level': parsed.riskAssignment?.riskLevelId, + 'comparison-operands': parsed.comparisonOperands, + 'comparison-operator': parsed.comparisonOperator, }; }), - "risk-categories": riskCategories.map(({ title }) => title), - "risk-framework": riskFramework?.title, - "answer-options": answerOptions.map(({ value }) => ({ + 'risk-categories': riskCategories.map(({ title }) => title), + 'risk-framework': riskFramework?.title, + 'answer-options': answerOptions.map(({ value }) => ({ value, })), - "selected-answers": selectedAnswers.map(({ value }) => value), - "allowed-mime-types": allowedMimeTypes, - "allow-select-other": allowSelectOther, - "sync-model": syncModel || undefined, - "sync-column": syncColumn || undefined, - "attribute-key": attributeKey?.name, - "require-risk-evaluation": requireRiskEvaluation, - "require-risk-matrix-evaluation": requireRiskMatrixEvaluation, + 'selected-answers': selectedAnswers.map(({ value }) => value), + 'allowed-mime-types': allowedMimeTypes, + 'allow-select-other': allowSelectOther, + 'sync-model': syncModel || undefined, + 'sync-column': syncColumn || undefined, + 'attribute-key': attributeKey?.name, + 'require-risk-evaluation': requireRiskEvaluation, + 'require-risk-matrix-evaluation': requireRiskMatrixEvaluation, }; - } + }, ), assignees: assignees.map(({ email }) => email), - "external-assignees": externalAssignees.map(({ email }) => email), - "is-reviewed": isReviewed, - }) + 'external-assignees': externalAssignees.map(({ email }) => email), + 'is-reviewed': isReviewed, + }), ), creator: creator?.email, description, status, assignees: assignees.map(({ email }) => email), - "external-assignees": externalAssignees.map(({ email }) => email), + 'external-assignees': externalAssignees.map(({ email }) => email), reviewers: reviewers.map(({ email }) => email), locked: isLocked, archived: isArchived, external: isExternallyCreated, - "title-is-internal": titleIsInternal, - "due-date": dueDate || undefined, - "created-at": createdAt || undefined, - "assigned-at": assignedAt || undefined, - "submitted-at": submittedAt || undefined, - "approved-at": approvedAt || undefined, - "rejected-at": rejectedAt || undefined, - "retention-schedule": retentionSchedule + 'title-is-internal': titleIsInternal, + 'due-date': dueDate || undefined, + 'created-at': createdAt || undefined, + 'assigned-at': assignedAt || undefined, + 'submitted-at': submittedAt || undefined, + 'approved-at': approvedAt || undefined, + 'rejected-at': rejectedAt || undefined, + 'retention-schedule': retentionSchedule ? { type: retentionSchedule.type, - "duration-days": retentionSchedule.durationDays, + 'duration-days': retentionSchedule.durationDays, operand: retentionSchedule.operation, } : undefined, @@ -610,9 +610,9 @@ export async function pullTranscendConfiguration( title: category ? `${category} - ${name}` : purpose - ? `${purpose} - ${name}` - : title || name || type || "", - }) + ? `${purpose} - ${name}` + : title || name || type || '', + }), ), rows: syncedRows.map( ({ resourceType, title, name, category, type, purpose }) => ({ @@ -620,11 +620,11 @@ export async function pullTranscendConfiguration( title: category ? `${category} - ${name}` : purpose - ? `${purpose} - ${name}` - : title || name || type || "", - }) + ? `${purpose} - ${name}` + : title || name || type || '', + }), ), - }) + }), ); } @@ -633,7 +633,7 @@ export async function pullTranscendConfiguration( assessmentTemplates.length > 0 && resources.includes(TranscendPullResource.AssessmentTemplates) ) { - result["assessment-templates"] = assessmentTemplates.map( + result['assessment-templates'] = assessmentTemplates.map( ({ title, description, @@ -679,30 +679,30 @@ export async function pullTranscendConfiguration( return { title, type, - "sub-type": subType, + 'sub-type': subType, placeholder, description, - "is-required": isRequired, - "reference-id": referenceId, - "display-logic": + 'is-required': isRequired, + 'reference-id': referenceId, + 'display-logic': displayLogicParsed && Object.keys(displayLogicParsed).length > 0 ? { action: displayLogicParsed.action!, rule: displayLogicParsed.rule ? { - "depends-on-question-reference-id": + 'depends-on-question-reference-id': displayLogicParsed.rule .dependsOnQuestionReferenceId, - "comparison-operator": + 'comparison-operator': displayLogicParsed.rule.comparisonOperator, - "comparison-operands": + 'comparison-operands': displayLogicParsed.rule.comparisonOperands, } : undefined, - "nested-rule": displayLogicParsed.nestedRule + 'nested-rule': displayLogicParsed.nestedRule ? { - "logic-operator": + 'logic-operator': displayLogicParsed.nestedRule.logicOperator, rules: /* eslint-disable @typescript-eslint/no-explicit-any */ @@ -712,11 +712,11 @@ export async function pullTranscendConfiguration( .nestedRule as AssessmentNestedRule ).rules || [] ).map((rule) => ({ - "depends-on-question-reference-id": + 'depends-on-question-reference-id': rule.dependsOnQuestionReferenceId, - "comparison-operator": + 'comparison-operator': rule.comparisonOperator, - "comparison-operands": + 'comparison-operands': rule.comparisonOperands, })), /* eslint-enable @typescript-eslint/no-explicit-any */ @@ -724,48 +724,48 @@ export async function pullTranscendConfiguration( : undefined, } : undefined, - "risk-logic": riskLogic.map((logic): RiskLogicInput => { + 'risk-logic': riskLogic.map((logic): RiskLogicInput => { const parsed = parseAssessmentRiskLogic(logic); return { - "risk-level": parsed.riskAssignment?.riskLevelId, - "risk-matrix-row": parsed.riskAssignment?.riskMatrixRowId, - "risk-matrix-column": + 'risk-level': parsed.riskAssignment?.riskLevelId, + 'risk-matrix-row': parsed.riskAssignment?.riskMatrixRowId, + 'risk-matrix-column': parsed.riskAssignment?.riskMatrixColumnId, - "comparison-operands": parsed.comparisonOperands, - "comparison-operator": parsed.comparisonOperator, + 'comparison-operands': parsed.comparisonOperands, + 'comparison-operator': parsed.comparisonOperator, }; }), - "risk-categories": riskCategories.map(({ title }) => title), - "risk-framework": riskFramework?.title, - "answer-options": answerOptions.map(({ value }) => ({ + 'risk-categories': riskCategories.map(({ title }) => title), + 'risk-framework': riskFramework?.title, + 'answer-options': answerOptions.map(({ value }) => ({ value, })), - "allowed-mime-types": allowedMimeTypes, - "allow-select-other": allowSelectOther, - "sync-model": syncModel || undefined, - "sync-column": syncColumn || undefined, - "attribute-key": attributeKey?.name, - "require-risk-evaluation": requireRiskEvaluation, - "require-risk-matrix-evaluation": requireRiskMatrixEvaluation, + 'allowed-mime-types': allowedMimeTypes, + 'allow-select-other': allowSelectOther, + 'sync-model': syncModel || undefined, + 'sync-column': syncColumn || undefined, + 'attribute-key': attributeKey?.name, + 'require-risk-evaluation': requireRiskEvaluation, + 'require-risk-matrix-evaluation': requireRiskMatrixEvaluation, }; - } + }, ), - }) + }), ), status, source, creator: creator?.email, locked: isLocked, archived: isArchived, - "created-at": createdAt || undefined, - "retention-schedule": retentionSchedule + 'created-at': createdAt || undefined, + 'retention-schedule': retentionSchedule ? { type: retentionSchedule.type, - "duration-days": retentionSchedule.durationDays, + 'duration-days': retentionSchedule.durationDays, operand: retentionSchedule.operation, } : undefined, - }) + }), ); } @@ -775,7 +775,7 @@ export async function pullTranscendConfiguration( ({ title, content }): PromptInput => ({ title, content, - }) + }), ); } @@ -784,11 +784,11 @@ export async function pullTranscendConfiguration( promptPartials.length > 0 && resources.includes(TranscendPullResource.PromptPartials) ) { - result["prompt-partials"] = promptPartials.map( + result['prompt-partials'] = promptPartials.map( ({ title, content }): PromptPartialInput => ({ title, content, - }) + }), ); } @@ -797,12 +797,12 @@ export async function pullTranscendConfiguration( promptGroups.length > 0 && resources.includes(TranscendPullResource.PromptGroups) ) { - result["prompt-groups"] = promptGroups.map( + result['prompt-groups'] = promptGroups.map( ({ title, description, prompts }): PromptGroupInput => ({ title, description, prompts: prompts.map(({ title }) => title), - }) + }), ); } @@ -820,12 +820,12 @@ export async function pullTranscendConfiguration( }): TeamInput => ({ name, description, - "sso-department": ssoDepartment || undefined, - "sso-group": ssoGroup || undefined, - "sso-title": ssoTitle || undefined, + 'sso-department': ssoDepartment || undefined, + 'sso-group': ssoGroup || undefined, + 'sso-title': ssoTitle || undefined, users: users.map(({ email }) => email), scopes: scopes.map(({ name }) => name), - }) + }), ); } @@ -834,7 +834,7 @@ export async function pullTranscendConfiguration( dataSubjects.length > 0 && resources.includes(TranscendPullResource.DataSubjects) ) { - result["data-subjects"] = dataSubjects.map( + result['data-subjects'] = dataSubjects.map( ({ type, title, @@ -847,7 +847,7 @@ export async function pullTranscendConfiguration( active, adminDashboardDefaultSilentMode, actions: actions.map(({ type }) => type), - }) + }), ); } @@ -858,7 +858,7 @@ export async function pullTranscendConfiguration( title: title?.defaultMessage, content: versions?.[0]?.content?.defaultMessage, disabledLocales, - }) + }), ); } @@ -877,16 +877,16 @@ export async function pullTranscendConfiguration( translations: translations.reduce( (accumulator, { locale, value }) => Object.assign(accumulator, { [locale]: value }), - {} as Record + {} as Record, ), - }) + }), ); } // Save privacy center if (privacyCenters.length > 0) { const privacyCenter = privacyCenters[0]; - result["privacy-center"] = { + result['privacy-center'] = { isDisabled: privacyCenter.isDisabled, showPrivacyRequestButton: privacyCenter.showPrivacyRequestButton, showPolicies: privacyCenter.showPolicies, @@ -914,7 +914,7 @@ export async function pullTranscendConfiguration( businessEntities.length > 0 && resources.includes(TranscendPullResource.BusinessEntities) ) { - result["business-entities"] = businessEntities.map( + result['business-entities'] = businessEntities.map( ({ title, description, @@ -936,7 +936,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }) + }), ); } @@ -963,7 +963,7 @@ export async function pullTranscendConfiguration( waitingPeriod, regionDetectionMethod, regionList: regionList.length > 0 ? regionList : undefined, - }) + }), ); } @@ -1003,7 +1003,7 @@ export async function pullTranscendConfiguration( displayTitle: displayTitle?.defaultMessage, displayDescription: displayDescription?.defaultMessage, displayOrder, - }) + }), ); } @@ -1031,7 +1031,7 @@ export async function pullTranscendConfiguration( codeInterpreterEnabled, retrievalEnabled, prompt: prompt?.title, - "large-language-model": { + 'large-language-model': { name: largeLanguageModel.name, client: largeLanguageModel.client, }, @@ -1041,15 +1041,15 @@ export async function pullTranscendConfiguration( owners && owners.length > 0 ? owners.map(({ email }) => email) : undefined, - "agent-functions": + 'agent-functions': agentFunctions && agentFunctions.length > 0 ? agentFunctions.map(({ name }) => name) : undefined, - "agent-files": + 'agent-files': agentFiles && agentFiles.length > 0 ? agentFiles.map(({ name }) => name) : undefined, - }) + }), ); } @@ -1058,7 +1058,7 @@ export async function pullTranscendConfiguration( actionItems.length > 0 && resources.includes(TranscendPullResource.ActionItems) ) { - result["action-items"] = actionItems.map( + result['action-items'] = actionItems.map( ({ teams, users, @@ -1088,7 +1088,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }) + }), ); } @@ -1097,7 +1097,7 @@ export async function pullTranscendConfiguration( actionItemCollections.length > 0 && resources.includes(TranscendPullResource.ActionItemCollections) ) { - result["action-item-collections"] = actionItemCollections.map( + result['action-item-collections'] = actionItemCollections.map( ({ title, description, @@ -1108,7 +1108,7 @@ export async function pullTranscendConfiguration( description: description || undefined, hidden, productLine, - }) + }), ); } @@ -1117,12 +1117,12 @@ export async function pullTranscendConfiguration( agentFunctions.length > 0 && resources.includes(TranscendPullResource.AgentFunctions) ) { - result["agent-functions"] = agentFunctions.map( + result['agent-functions'] = agentFunctions.map( ({ name, description, parameters }): AgentFunctionInput => ({ name, description, parameters: JSON.stringify(parameters), - }) + }), ); } @@ -1131,14 +1131,14 @@ export async function pullTranscendConfiguration( agentFiles.length > 0 && resources.includes(TranscendPullResource.AgentFiles) ) { - result["agent-files"] = agentFiles.map( + result['agent-files'] = agentFiles.map( ({ name, description, fileId, size, purpose }): AgentFileInput => ({ name, description, fileId, size, purpose, - }) + }), ); } @@ -1180,7 +1180,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }) + }), ); } @@ -1189,7 +1189,7 @@ export async function pullTranscendConfiguration( dataCategories.length > 0 && resources.includes(TranscendPullResource.DataCategories) ) { - result["data-categories"] = dataCategories.map( + result['data-categories'] = dataCategories.map( ({ name, category, @@ -1213,7 +1213,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }) + }), ); } @@ -1222,7 +1222,7 @@ export async function pullTranscendConfiguration( processingPurposes.length > 0 && resources.includes(TranscendPullResource.ProcessingPurposes) ) { - result["processing-purposes"] = processingPurposes.map( + result['processing-purposes'] = processingPurposes.map( ({ name, purpose, @@ -1244,7 +1244,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }) + }), ); } @@ -1253,7 +1253,7 @@ export async function pullTranscendConfiguration( dataFlows.length > 0 && resources.includes(TranscendPullResource.DataFlows) ) { - result["data-flows"] = dataFlows.map( + result['data-flows'] = dataFlows.map( ({ value, type, @@ -1277,7 +1277,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }) + }), ); } @@ -1307,7 +1307,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }) + }), ); } @@ -1333,7 +1333,7 @@ export async function pullTranscendConfiguration( color: color || undefined, description, })), - }) + }), ); } @@ -1359,15 +1359,15 @@ export async function pullTranscendConfiguration( title, description: description || undefined, trackingType, - "default-consent": defaultConsent, + 'default-consent': defaultConsent, configurable, - "show-in-consent-manager": showInConsentManager, - "show-in-privacy-center": showInPrivacyCenter, - "is-active": isActive, - "display-order": displayOrder, - "opt-out-signals": optOutSignals.length > 0 ? optOutSignals : undefined, - "auth-level": authLevel || undefined, - "preference-topics": topics.map( + 'show-in-consent-manager': showInConsentManager, + 'show-in-privacy-center': showInPrivacyCenter, + 'is-active': isActive, + 'display-order': displayOrder, + 'opt-out-signals': optOutSignals.length > 0 ? optOutSignals : undefined, + 'auth-level': authLevel || undefined, + 'preference-topics': topics.map( ({ title, type, @@ -1379,8 +1379,8 @@ export async function pullTranscendConfiguration( title: title.defaultMessage, type, description: displayDescription.defaultMessage, - "default-configuration": defaultConfiguration, - "show-in-privacy-center": showInPrivacyCenter, + 'default-configuration': defaultConfiguration, + 'show-in-privacy-center': showInPrivacyCenter, ...(preferenceOptionValues.length > 0 ? { options: preferenceOptionValues.map(({ title, slug }) => ({ @@ -1389,9 +1389,9 @@ export async function pullTranscendConfiguration( })), } : {}), - }) + }), ), - }) + }), ); } @@ -1428,9 +1428,9 @@ export async function pullTranscendConfiguration( title, url: url || undefined, type, - "input-identifier": inputIdentifier?.name, - "output-identifiers": identifiers.map(({ name }) => name), - "privacy-actions": + 'input-identifier': inputIdentifier?.name, + 'output-identifiers': identifiers.map(({ name }) => name), + 'privacy-actions': Object.values(RequestAction).length === actions.length ? undefined : actions, @@ -1442,8 +1442,8 @@ export async function pullTranscendConfiguration( phoneNumbers && phoneNumbers.length > 0 ? phoneNumbers : undefined, regionList: regionList && regionList.length > 0 ? regionList : undefined, - "data-subjects": dataSubjects.map(({ type }) => type), - }) + 'data-subjects': dataSubjects.map(({ type }) => type), + }), ); } @@ -1452,8 +1452,8 @@ export async function pullTranscendConfiguration( dataSilos.length > 0 && resources.includes(TranscendPullResource.DataSilos) ) { - const indexedDataSubjects = keyBy(dataSubjects, "type"); - result["data-silos"] = dataSilos.map( + const indexedDataSubjects = keyBy(dataSubjects, 'type'); + result['data-silos'] = dataSilos.map( ([ { title, @@ -1486,16 +1486,16 @@ export async function pullTranscendConfiguration( title, description, integrationName: type, - "outer-type": outerType || undefined, + 'outer-type': outerType || undefined, url: url || undefined, - "api-key-title": apiKeys[0]?.title, - "identity-keys": identifiers + 'api-key-title': apiKeys[0]?.title, + 'identity-keys': identifiers .filter(({ isConnected }) => isConnected) .map(({ name }) => name), ...(dependentDataSilos.length > 0 ? { - "deletion-dependencies": dependentDataSilos.map( - ({ title }) => title + 'deletion-dependencies': dependentDataSilos.map( + ({ title }) => title, ), } : {}), @@ -1514,23 +1514,23 @@ export async function pullTranscendConfiguration( country: country || undefined, countrySubDivision: countrySubDivision || undefined, disabled: !isLive, - "data-subjects": + 'data-subjects': subjectBlocklist.length > 0 ? convertToDataSubjectAllowlist( subjectBlocklist.map(({ type }) => type), - indexedDataSubjects + indexedDataSubjects, ) : undefined, ...(catalog.hasAvcFunctionality ? { - "email-settings": { - "notify-email-address": notifyEmailAddress || undefined, - "send-frequency": promptAVendorEmailSendFrequency, - "send-type": promptAVendorEmailSendType, - "include-identifiers-attachment": + 'email-settings': { + 'notify-email-address': notifyEmailAddress || undefined, + 'send-frequency': promptAVendorEmailSendFrequency, + 'send-type': promptAVendorEmailSendType, + 'include-identifiers-attachment': promptAVendorEmailIncludeIdentifiersAttachment, - "completion-link-type": promptAVendorEmailCompletionLinkType, - "manual-work-retry-frequency": manualWorkRetryFrequency, + 'completion-link-type': promptAVendorEmailCompletionLinkType, + 'manual-work-retry-frequency': manualWorkRetryFrequency, }, } : {}), @@ -1550,18 +1550,18 @@ export async function pullTranscendConfiguration( ...(dataPoint.path.length > 0 ? { path: dataPoint.path } : {}), ...(dataPoint.dataCollection?.title ? { - "data-collection-tag": + 'data-collection-tag': dataPoint.dataCollection.title.defaultMessage, } : {}), ...(dataPoint.dbIntegrationQueries.length > 0 ? { - "privacy-action-queries": mapValues( - keyBy(dataPoint.dbIntegrationQueries, "requestType"), + 'privacy-action-queries': mapValues( + keyBy(dataPoint.dbIntegrationQueries, 'requestType'), (databaseIntegrationQuery) => databaseIntegrationQuery.suggestedQuery || databaseIntegrationQuery.query || - undefined + undefined, ), } : {}), @@ -1577,10 +1577,10 @@ export async function pullTranscendConfiguration( ...(includeGuessedCategories && field.pendingCategoryGuesses ? { - "guessed-categories": + 'guessed-categories': field.pendingCategoryGuesses .filter( - (guess) => guess.status === "PENDING" + (guess) => guess.status === 'PENDING', ) .map((guess) => ({ category: { @@ -1594,31 +1594,31 @@ export async function pullTranscendConfiguration( })), } : {}), - "access-request-visibility-enabled": + 'access-request-visibility-enabled': field.accessRequestVisibilityEnabled, - "erasure-request-redaction-enabled": + 'erasure-request-redaction-enabled': field.erasureRequestRedactionEnabled, attributes: field.attributeValues !== undefined && field.attributeValues.length > 0 ? formatAttributeValues(field.attributeValues) : undefined, - }) + }), ) .sort((a, b) => a.key.localeCompare(b.key)), } : {}), - "privacy-actions": dataPoint.actionSettings + 'privacy-actions': dataPoint.actionSettings .filter(({ active }) => active) .map(({ type }) => type), - }) + }), ) .sort((a, b) => [...(a.path ?? []), a.key] - .join(".") - .localeCompare([...(b.path ?? []), b.key].join(".")) + .join('.') + .localeCompare([...(b.path ?? []), b.key].join('.')), ), - }) + }), ); } return result; diff --git a/src/lib/graphql/syncActionItemCollections.ts b/src/lib/graphql/syncActionItemCollections.ts index 5e31b829..1f4db10d 100644 --- a/src/lib/graphql/syncActionItemCollections.ts +++ b/src/lib/graphql/syncActionItemCollections.ts @@ -1,18 +1,18 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { ActionItemCollectionInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { ActionItemCollectionInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { ActionItemCollection, fetchAllActionItemCollections, -} from "./fetchAllActionItemCollections"; +} from './fetchAllActionItemCollections'; import { CREATE_ACTION_ITEM_COLLECTION, UPDATE_ACTION_ITEM_COLLECTION, -} from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new action item collection @@ -23,11 +23,11 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function createActionItemCollection( client: GraphQLClient, - actionItemCollection: ActionItemCollectionInput -): Promise> { + actionItemCollection: ActionItemCollectionInput, +): Promise> { const input = { title: actionItemCollection.title, - description: actionItemCollection.description || "", + description: actionItemCollection.description || '', hidden: actionItemCollection.hidden || false, productLine: actionItemCollection.productLine, }; @@ -54,7 +54,7 @@ export async function createActionItemCollection( export async function updateActionItemCollection( client: GraphQLClient, input: ActionItemCollectionInput, - actionItemCollectionId: string + actionItemCollectionId: string, ): Promise { await makeGraphQLRequest(client, UPDATE_ACTION_ITEM_COLLECTION, { input: { @@ -76,28 +76,27 @@ export async function updateActionItemCollection( */ export async function syncActionItemCollections( client: GraphQLClient, - inputs: ActionItemCollectionInput[] + inputs: ActionItemCollectionInput[], ): Promise { let encounteredError = false; // Fetch existing logger.info( - colors.magenta(`Syncing "${inputs.length}" action item collections...`) + colors.magenta(`Syncing "${inputs.length}" action item collections...`), ); // Fetch existing - const existingActionItemCollections = await fetchAllActionItemCollections( - client - ); + const existingActionItemCollections = + await fetchAllActionItemCollections(client); // Look up by title const collectionByTitle: Record = keyBy( existingActionItemCollections, - "title" + 'title', ); // Create new actionItems const newCollections = inputs.filter( - (input) => !collectionByTitle[input.title] + (input) => !collectionByTitle[input.title], ); // Create new actionItem collections @@ -106,15 +105,15 @@ export async function syncActionItemCollections( await createActionItemCollection(client, input); logger.info( colors.green( - `Successfully created action item collection "${input.title}"!` - ) + `Successfully created action item collection "${input.title}"!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to create action item collection "${input.title}"! - ${error.message}` - ) + `Failed to create action item collection "${input.title}"! - ${error.message}`, + ), ); } }); @@ -128,15 +127,15 @@ export async function syncActionItemCollections( await updateActionItemCollection(client, input, actionItemId); logger.info( colors.green( - `Successfully synced action item collection "${input.title}"!` - ) + `Successfully synced action item collection "${input.title}"!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync action item collection "${input.title}"! - ${error.message}` - ) + `Failed to sync action item collection "${input.title}"! - ${error.message}`, + ), ); } }); diff --git a/src/lib/graphql/syncActionItems.ts b/src/lib/graphql/syncActionItems.ts index c013a2c1..e12d15d9 100644 --- a/src/lib/graphql/syncActionItems.ts +++ b/src/lib/graphql/syncActionItems.ts @@ -1,17 +1,17 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk, keyBy, uniq } from "lodash-es"; -import { ActionItemInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk, keyBy, uniq } from 'lodash-es'; +import { ActionItemInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { ActionItemCollection, fetchAllActionItemCollections, -} from "./fetchAllActionItemCollections"; -import { ActionItem, fetchAllActionItems } from "./fetchAllActionItems"; -import { Attribute, fetchAllAttributes } from "./fetchAllAttributes"; -import { CREATE_ACTION_ITEMS, UPDATE_ACTION_ITEMS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from './fetchAllActionItemCollections'; +import { ActionItem, fetchAllActionItems } from './fetchAllActionItems'; +import { Attribute, fetchAllAttributes } from './fetchAllAttributes'; +import { CREATE_ACTION_ITEMS, UPDATE_ACTION_ITEMS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new actionItem @@ -27,7 +27,7 @@ export async function createActionItems( actionItemCollectionByTitle: Record, // TODO: https://transcend.height.app/T-38961 - insert attributes // eslint-disable-next-line @typescript-eslint/no-unused-vars - attributeKeysByName: Record = {} + attributeKeysByName: Record = {}, ): Promise { // TODO: https://transcend.height.app/T-38961 - insert attributes // const getAttribute = (key: string): string => { @@ -62,7 +62,7 @@ export async function createActionItems( } : {}), collectionIds: actionItem.collections.map( - (collectionTitle) => actionItemCollectionByTitle[collectionTitle].id + (collectionTitle) => actionItemCollectionByTitle[collectionTitle].id, ), })), }); @@ -81,7 +81,7 @@ export async function updateActionItem( client: GraphQLClient, input: ActionItemInput, actionItemId: string, - attributeKeysByName: Record = {} + attributeKeysByName: Record = {}, ): Promise { const getAttribute = (key: string): string => { const existing = attributeKeysByName[key]; @@ -123,11 +123,11 @@ export async function updateActionItem( function actionItemToUniqueCode({ title, collections, -}: Pick): string { +}: Pick): string { return `${title}-${collections .map((c) => c.title) .sort() - .join("-")}`; + .join('-')}`; } /** @@ -139,8 +139,8 @@ function actionItemToUniqueCode({ function actionItemInputToUniqueCode({ title, collections, -}: Pick): string { - return `${title}-${collections.sort().join("-")}`; +}: Pick): string { + return `${title}-${collections.sort().join('-')}`; } /** @@ -152,7 +152,7 @@ function actionItemInputToUniqueCode({ */ export async function syncActionItems( client: GraphQLClient, - inputs: ActionItemInput[] + inputs: ActionItemInput[], ): Promise { let encounteredError = false; // Fetch existing @@ -160,7 +160,7 @@ export async function syncActionItems( // Determine if attributes are syncing const hasAttributes = inputs.some( - (input) => input.attributes && input.attributes.length > 0 + (input) => input.attributes && input.attributes.length > 0, ); // Fetch existing @@ -173,28 +173,28 @@ export async function syncActionItems( // Look up by title const actionItemCollectionByTitle: Record = - keyBy(existingActionItemCollections, "title"); + keyBy(existingActionItemCollections, 'title'); const actionItemByTitle: Record = keyBy( existingActionItems, - actionItemToUniqueCode + actionItemToUniqueCode, ); - const attributeKeysByName = keyBy(attributeKeys, "name"); + const attributeKeysByName = keyBy(attributeKeys, 'name'); const actionItemByCxId: Record = keyBy( existingActionItems.filter((x) => !!x.customerExperienceActionItemIds), - ({ customerExperienceActionItemIds }) => customerExperienceActionItemIds[0] + ({ customerExperienceActionItemIds }) => customerExperienceActionItemIds[0], ); // Ensure all collections exist const missingCollections = uniq( - inputs.flatMap((input) => input.collections) + inputs.flatMap((input) => input.collections), ).filter((collectionTitle) => !actionItemCollectionByTitle[collectionTitle]); if (missingCollections.length > 0) { logger.info( colors.red( `Missing action item collections: "${missingCollections.join( - '", "' - )}" - please create them first!` - ) + '", "', + )}" - please create them first!`, + ), ); return false; } @@ -203,30 +203,30 @@ export async function syncActionItems( const newActionItems = inputs.filter( (input) => !actionItemByTitle[actionItemInputToUniqueCode(input)] && - !actionItemByCxId[input.customerExperienceActionItemId!] + !actionItemByCxId[input.customerExperienceActionItemId!], ); // Create new actionItems if (newActionItems.length > 0) { try { logger.info( - colors.magenta(`Creating "${newActionItems.length}" actionItems...`) + colors.magenta(`Creating "${newActionItems.length}" actionItems...`), ); await createActionItems( client, newActionItems, actionItemCollectionByTitle, - attributeKeysByName + attributeKeysByName, ); logger.info( colors.green( - `Successfully created "${newActionItems.length}" actionItems!` - ) + `Successfully created "${newActionItems.length}" actionItems!`, + ), ); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create action items! - ${error.message}`) + colors.red(`Failed to create action items! - ${error.message}`), ); } } @@ -243,14 +243,14 @@ export async function syncActionItems( try { await updateActionItem(client, input, actionItemId, attributeKeysByName); logger.info( - colors.green(`Successfully synced action item "${input.title}"!`) + colors.green(`Successfully synced action item "${input.title}"!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync action item "${input.title}"! - ${error.message}` - ) + `Failed to sync action item "${input.title}"! - ${error.message}`, + ), ); } }); diff --git a/src/lib/graphql/syncAgentFiles.ts b/src/lib/graphql/syncAgentFiles.ts index c2bd90fa..c83804d3 100644 --- a/src/lib/graphql/syncAgentFiles.ts +++ b/src/lib/graphql/syncAgentFiles.ts @@ -1,12 +1,12 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { AgentFileInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { AgentFile, fetchAllAgentFiles } from "./fetchAllAgentFiles"; -import { CREATE_AGENT_FILE, UPDATE_AGENT_FILES } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { AgentFileInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { AgentFile, fetchAllAgentFiles } from './fetchAllAgentFiles'; +import { CREATE_AGENT_FILE, UPDATE_AGENT_FILES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new agent file @@ -17,8 +17,8 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function createAgentFile( client: GraphQLClient, - agentFile: AgentFileInput -): Promise> { + agentFile: AgentFileInput, +): Promise> { const input = { name: agentFile.name, description: agentFile.description, @@ -50,7 +50,7 @@ export async function createAgentFile( */ export async function updateAgentFiles( client: GraphQLClient, - agentFileIdPairs: [AgentFileInput, string][] + agentFileIdPairs: [AgentFileInput, string][], ): Promise { await makeGraphQLRequest(client, UPDATE_AGENT_FILES, { input: { @@ -75,7 +75,7 @@ export async function updateAgentFiles( */ export async function syncAgentFiles( client: GraphQLClient, - inputs: AgentFileInput[] + inputs: AgentFileInput[], ): Promise { // Fetch existing logger.info(colors.magenta(`Syncing "${inputs.length}" agent files...`)); @@ -88,8 +88,8 @@ export async function syncAgentFiles( // Look up by name const agentFileByName: Record< string, - Pick - > = keyBy(existingAgentFiles, "name"); + Pick + > = keyBy(existingAgentFiles, 'name'); // Create new agent files const newAgentFiles = inputs.filter((input) => !agentFileByName[input.name]); @@ -100,14 +100,14 @@ export async function syncAgentFiles( const newAgentFile = await createAgentFile(client, agentFile); agentFileByName[newAgentFile.name] = newAgentFile; logger.info( - colors.green(`Successfully synced agent file "${agentFile.name}"!`) + colors.green(`Successfully synced agent file "${agentFile.name}"!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync agent file "${agentFile.name}"! - ${error.message}` - ) + `Failed to sync agent file "${agentFile.name}"! - ${error.message}`, + ), ); } }); @@ -117,17 +117,17 @@ export async function syncAgentFiles( logger.info(colors.magenta(`Updating "${inputs.length}" agent files!`)); await updateAgentFiles( client, - inputs.map((input) => [input, agentFileByName[input.name].id]) + inputs.map((input) => [input, agentFileByName[input.name].id]), ); logger.info( - colors.green(`Successfully synced "${inputs.length}" agent files!`) + colors.green(`Successfully synced "${inputs.length}" agent files!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" agent files! - ${error.message}` - ) + `Failed to sync "${inputs.length}" agent files! - ${error.message}`, + ), ); } diff --git a/src/lib/graphql/syncAgentFunctions.ts b/src/lib/graphql/syncAgentFunctions.ts index 53cc09aa..ca4b949e 100644 --- a/src/lib/graphql/syncAgentFunctions.ts +++ b/src/lib/graphql/syncAgentFunctions.ts @@ -1,15 +1,15 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { AgentFunctionInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { AgentFunctionInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { AgentFunction, fetchAllAgentFunctions, -} from "./fetchAllAgentFunctions"; -import { CREATE_AGENT_FUNCTION, UPDATE_AGENT_FUNCTIONS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from './fetchAllAgentFunctions'; +import { CREATE_AGENT_FUNCTION, UPDATE_AGENT_FUNCTIONS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new agent function @@ -20,8 +20,8 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function createAgentFunction( client: GraphQLClient, - agentFunction: AgentFunctionInput -): Promise> { + agentFunction: AgentFunctionInput, +): Promise> { const input = { name: agentFunction.name, description: agentFunction.description, @@ -50,7 +50,7 @@ export async function createAgentFunction( */ export async function updateAgentFunctions( client: GraphQLClient, - agentFunctionIdPairs: [AgentFunctionInput, string][] + agentFunctionIdPairs: [AgentFunctionInput, string][], ): Promise { await makeGraphQLRequest(client, UPDATE_AGENT_FUNCTIONS, { input: { @@ -73,7 +73,7 @@ export async function updateAgentFunctions( */ export async function syncAgentFunctions( client: GraphQLClient, - inputs: AgentFunctionInput[] + inputs: AgentFunctionInput[], ): Promise { // Fetch existing logger.info(colors.magenta(`Syncing "${inputs.length}" agent functions...`)); @@ -86,12 +86,12 @@ export async function syncAgentFunctions( // Look up by name const agentFunctionByName: Record< string, - Pick - > = keyBy(existingAgentFunctions, "name"); + Pick + > = keyBy(existingAgentFunctions, 'name'); // Create new agent functions const newAgentFunctions = inputs.filter( - (input) => !agentFunctionByName[input.name] + (input) => !agentFunctionByName[input.name], ); // Create new agent functions @@ -101,15 +101,15 @@ export async function syncAgentFunctions( agentFunctionByName[newAgentFunction.name] = newAgentFunction; logger.info( colors.green( - `Successfully synced agent function "${agentFunction.name}"!` - ) + `Successfully synced agent function "${agentFunction.name}"!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync agent function "${agentFunction.name}"! - ${error.message}` - ) + `Failed to sync agent function "${agentFunction.name}"! - ${error.message}`, + ), ); } }); @@ -119,17 +119,17 @@ export async function syncAgentFunctions( logger.info(colors.magenta(`Updating "${inputs.length}" agent functions!`)); await updateAgentFunctions( client, - inputs.map((input) => [input, agentFunctionByName[input.name].id]) + inputs.map((input) => [input, agentFunctionByName[input.name].id]), ); logger.info( - colors.green(`Successfully synced "${inputs.length}" agent functions!`) + colors.green(`Successfully synced "${inputs.length}" agent functions!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" agent functions! - ${error.message}` - ) + `Failed to sync "${inputs.length}" agent functions! - ${error.message}`, + ), ); } diff --git a/src/lib/graphql/syncAgents.ts b/src/lib/graphql/syncAgents.ts index 974f39d0..9a1b1659 100644 --- a/src/lib/graphql/syncAgents.ts +++ b/src/lib/graphql/syncAgents.ts @@ -1,12 +1,12 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { AgentInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { Agent, fetchAllAgents } from "./fetchAllAgents"; -import { CREATE_AGENT, UPDATE_AGENTS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { AgentInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { Agent, fetchAllAgents } from './fetchAllAgents'; +import { CREATE_AGENT, UPDATE_AGENTS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new agent @@ -17,16 +17,16 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function createAgent( client: GraphQLClient, - agent: AgentInput -): Promise> { + agent: AgentInput, +): Promise> { const input = { name: agent.name, description: agent.description, codeInterpreterEnabled: agent.codeInterpreterEnabled, retrievalEnabled: agent.retrievalEnabled, promptTitle: agent.prompt, - largeLanguageModelName: agent["large-language-model"].name, - largeLanguageModelClient: agent["large-language-model"].client, + largeLanguageModelName: agent['large-language-model'].name, + largeLanguageModelClient: agent['large-language-model'].client, // TODO: https://transcend.height.app/T-32760 - agentFunction, agentFile // TODO: https://transcend.height.app/T-31994 - owners and teams }; @@ -51,7 +51,7 @@ export async function createAgent( */ export async function updateAgents( client: GraphQLClient, - agentIdParis: [AgentInput, string][] + agentIdParis: [AgentInput, string][], ): Promise { await makeGraphQLRequest(client, UPDATE_AGENTS, { input: { @@ -76,7 +76,7 @@ export async function updateAgents( */ export async function syncAgents( client: GraphQLClient, - inputs: AgentInput[] + inputs: AgentInput[], ): Promise { // Fetch existing logger.info(colors.magenta(`Syncing "${inputs.length}" agents...`)); @@ -89,8 +89,8 @@ export async function syncAgents( // Look up by name const agentByName: Record< string, - Pick - > = keyBy(existingAgents, "name"); + Pick + > = keyBy(existingAgents, 'name'); // Create new agents const newAgents = inputs.filter((input) => !agentByName[input.name]); @@ -104,7 +104,7 @@ export async function syncAgents( } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to sync agent "${agent.name}"! - ${error.message}`) + colors.red(`Failed to sync agent "${agent.name}"! - ${error.message}`), ); } }); @@ -114,15 +114,15 @@ export async function syncAgents( logger.info(colors.magenta(`Updating "${inputs.length}" agents!`)); await updateAgents( client, - inputs.map((input) => [input, agentByName[input.name].id]) + inputs.map((input) => [input, agentByName[input.name].id]), ); logger.info(colors.green(`Successfully synced "${inputs.length}" agents!`)); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" agents ! - ${error.message}` - ) + `Failed to sync "${inputs.length}" agents ! - ${error.message}`, + ), ); } diff --git a/src/lib/graphql/syncAttribute.ts b/src/lib/graphql/syncAttribute.ts index 609da1bd..f902ef43 100644 --- a/src/lib/graphql/syncAttribute.ts +++ b/src/lib/graphql/syncAttribute.ts @@ -1,18 +1,18 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { difference, groupBy, keyBy } from "lodash-es"; -import { AttributeInput } from "../../codecs"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { Attribute } from "./fetchAllAttributes"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { difference, groupBy, keyBy } from 'lodash-es'; +import { AttributeInput } from '../../codecs'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { Attribute } from './fetchAllAttributes'; import { CREATE_ATTRIBUTE, CREATE_ATTRIBUTE_VALUES, DELETE_ATTRIBUTE_VALUE, UPDATE_ATTRIBUTE, UPDATE_ATTRIBUTE_VALUES, -} from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Sync attribute @@ -32,7 +32,7 @@ export async function syncAttribute( existingAttribute?: Attribute; /** When true, delete extra attributes not specified in the list of values */ deleteExtraAttributeValues?: boolean; - } + }, ): Promise { // attribute key input const input = { @@ -72,15 +72,15 @@ export async function syncAttribute( } // upsert attribute values - const existingAttributeMap = keyBy(existingAttribute?.values || [], "name"); + const existingAttributeMap = keyBy(existingAttribute?.values || [], 'name'); const { existingValues = [], newValues = [] } = groupBy( attribute.values || [], (field) => - existingAttributeMap[field.name] ? "existingValues" : "newValues" + existingAttributeMap[field.name] ? 'existingValues' : 'newValues', ); const removedValues = difference( (existingAttribute?.values || []).map(({ name }) => name), - (attribute.values || []).map(({ name }) => name) + (attribute.values || []).map(({ name }) => name), ); // Create new attribute values @@ -108,7 +108,7 @@ export async function syncAttribute( })), }); logger.info( - colors.green(`Updated ${existingValues.length} attribute values`) + colors.green(`Updated ${existingValues.length} attribute values`), ); } @@ -123,10 +123,10 @@ export async function syncAttribute( }, { concurrency: 10, - } + }, ); logger.info( - colors.green(`Deleted ${removedValues.length} attribute values`) + colors.green(`Deleted ${removedValues.length} attribute values`), ); } } diff --git a/src/lib/graphql/syncBusinessEntities.ts b/src/lib/graphql/syncBusinessEntities.ts index 1fa10623..3b6ce91b 100644 --- a/src/lib/graphql/syncBusinessEntities.ts +++ b/src/lib/graphql/syncBusinessEntities.ts @@ -1,15 +1,15 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk, keyBy } from "lodash-es"; -import { BusinessEntityInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk, keyBy } from 'lodash-es'; +import { BusinessEntityInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { BusinessEntity, fetchAllBusinessEntities, -} from "./fetchAllBusinessEntities"; -import { CREATE_BUSINESS_ENTITY, UPDATE_BUSINESS_ENTITIES } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from './fetchAllBusinessEntities'; +import { CREATE_BUSINESS_ENTITY, UPDATE_BUSINESS_ENTITIES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new business entity @@ -20,7 +20,7 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function createBusinessEntity( client: GraphQLClient, - businessEntity: BusinessEntityInput + businessEntity: BusinessEntityInput, ): Promise { const input = { title: businessEntity.title, @@ -55,7 +55,7 @@ export async function createBusinessEntity( */ export async function updateBusinessEntities( client: GraphQLClient, - businessEntityIdParis: [BusinessEntityInput, string][] + businessEntityIdParis: [BusinessEntityInput, string][], ): Promise { const chunkedUpdates = chunk(businessEntityIdParis, 100); await mapSeries(chunkedUpdates, async (chunked) => { @@ -86,11 +86,11 @@ export async function updateBusinessEntities( */ export async function syncBusinessEntities( client: GraphQLClient, - inputs: BusinessEntityInput[] + inputs: BusinessEntityInput[], ): Promise { // Fetch existing logger.info( - colors.magenta(`Syncing "${inputs.length}" business entities...`) + colors.magenta(`Syncing "${inputs.length}" business entities...`), ); let encounteredError = false; @@ -99,11 +99,11 @@ export async function syncBusinessEntities( const existingBusinessEntities = await fetchAllBusinessEntities(client); // Look up by title - const businessEntityByTitle = keyBy(existingBusinessEntities, "title"); + const businessEntityByTitle = keyBy(existingBusinessEntities, 'title'); // Create new business entities const newBusinessEntities = inputs.filter( - (input) => !businessEntityByTitle[input.title] + (input) => !businessEntityByTitle[input.title], ); // Create new business entities @@ -111,20 +111,20 @@ export async function syncBusinessEntities( try { const newBusinessEntity = await createBusinessEntity( client, - businessEntity + businessEntity, ); businessEntityByTitle[newBusinessEntity.title] = newBusinessEntity; logger.info( colors.green( - `Successfully synced business entity "${businessEntity.title}"!` - ) + `Successfully synced business entity "${businessEntity.title}"!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync business entity "${businessEntity.title}"! - ${error.message}` - ) + `Failed to sync business entity "${businessEntity.title}"! - ${error.message}`, + ), ); } }); @@ -132,21 +132,21 @@ export async function syncBusinessEntities( // Update all business entities try { logger.info( - colors.magenta(`Updating "${inputs.length}" business entities!`) + colors.magenta(`Updating "${inputs.length}" business entities!`), ); await updateBusinessEntities( client, - inputs.map((input) => [input, businessEntityByTitle[input.title].id]) + inputs.map((input) => [input, businessEntityByTitle[input.title].id]), ); logger.info( - colors.green(`Successfully synced "${inputs.length}" business entities!`) + colors.green(`Successfully synced "${inputs.length}" business entities!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" business entities ! - ${error.message}` - ) + `Failed to sync "${inputs.length}" business entities ! - ${error.message}`, + ), ); } diff --git a/src/lib/graphql/syncCodePackages.ts b/src/lib/graphql/syncCodePackages.ts index 3e29b388..8d4ddf84 100644 --- a/src/lib/graphql/syncCodePackages.ts +++ b/src/lib/graphql/syncCodePackages.ts @@ -1,19 +1,19 @@ -import { CodePackageType } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk, keyBy, uniq, uniqBy } from "lodash-es"; -import { CodePackageInput, RepositoryInput } from "../../codecs"; -import { logger } from "../../logger"; -import { map, mapSeries } from "../bluebird-replace"; -import { CodePackage, fetchAllCodePackages } from "./fetchAllCodePackages"; -import { CREATE_CODE_PACKAGE, UPDATE_CODE_PACKAGES } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; -import { syncRepositories } from "./syncRepositories"; -import { syncSoftwareDevelopmentKits } from "./syncSoftwareDevelopmentKits"; +import { CodePackageType } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk, keyBy, uniq, uniqBy } from 'lodash-es'; +import { CodePackageInput, RepositoryInput } from '../../codecs'; +import { logger } from '../../logger'; +import { map, mapSeries } from '../bluebird-replace'; +import { CodePackage, fetchAllCodePackages } from './fetchAllCodePackages'; +import { CREATE_CODE_PACKAGE, UPDATE_CODE_PACKAGES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; +import { syncRepositories } from './syncRepositories'; +import { syncSoftwareDevelopmentKits } from './syncSoftwareDevelopmentKits'; const CHUNK_SIZE = 100; -const LOOKUP_SPLIT_KEY = "%%%%"; +const LOOKUP_SPLIT_KEY = '%%%%'; /** * Create a new code package @@ -47,7 +47,7 @@ export async function createCodePackage( teamIds?: string[]; /** Names of teams */ teamNames?: string[]; - } + }, ): Promise { const { createCodePackage: { codePackage }, @@ -61,7 +61,7 @@ export async function createCodePackage( input, }); logger.info( - colors.green(`Successfully created code package "${input.name}"!`) + colors.green(`Successfully created code package "${input.name}"!`), ); return codePackage; } @@ -100,7 +100,7 @@ export async function updateCodePackages( teamIds?: string[]; /** Names of teams */ teamNames?: string[]; - }[] + }[], ): Promise { const { updateCodePackages: { codePackages }, @@ -116,7 +116,7 @@ export async function updateCodePackages( }, }); logger.info( - colors.green(`Successfully updated ${inputs.length} code packages!`) + colors.green(`Successfully updated ${inputs.length} code packages!`), ); return codePackages; } @@ -132,7 +132,7 @@ export async function updateCodePackages( export async function syncCodePackages( client: GraphQLClient, codePackages: CodePackageInput[], - concurrency = 20 + concurrency = 20, ): Promise { let encounteredError = false; const [ @@ -149,34 +149,34 @@ export async function syncCodePackages( softwareDevelopmentKits.map(({ name }) => ({ name, codePackageType: type, - })) + })), ), ({ name, codePackageType }) => - `${name}${LOOKUP_SPLIT_KEY}${codePackageType}` + `${name}${LOOKUP_SPLIT_KEY}${codePackageType}`, ), - concurrency + concurrency, ), // make sure all Repositories exist syncRepositories( client, - uniqBy(codePackages, "repositoryName").map( + uniqBy(codePackages, 'repositoryName').map( ({ repositoryName }) => ({ name: repositoryName, url: `https://github.com/${repositoryName}`, - } as RepositoryInput) - ) + }) as RepositoryInput, + ), ), ]); const softwareDevelopmentKitLookup = keyBy( existingSoftwareDevelopmentKits, ({ name, codePackageType }) => - `${name}${LOOKUP_SPLIT_KEY}${codePackageType}` + `${name}${LOOKUP_SPLIT_KEY}${codePackageType}`, ); const codePackagesLookup = keyBy( existingCodePackages, - ({ name, type }) => `${name}${LOOKUP_SPLIT_KEY}${type}` + ({ name, type }) => `${name}${LOOKUP_SPLIT_KEY}${type}`, ); // Determine which codePackages are new vs existing @@ -194,8 +194,8 @@ export async function syncCodePackages( try { logger.info( colors.magenta( - `Creating "${newCodePackages.length}" new code packages...` - ) + `Creating "${newCodePackages.length}" new code packages...`, + ), ); await map( newCodePackages, @@ -212,11 +212,11 @@ export async function syncCodePackages( ]; if (!sdk) { throw new Error( - `Failed to find SDK with name: "${name}"` + `Failed to find SDK with name: "${name}"`, ); } return sdk.id; - }) + }), ), } : {}), @@ -224,28 +224,28 @@ export async function syncCodePackages( }, { concurrency, - } + }, ); logger.info( colors.green( - `Successfully synced ${newCodePackages.length} code packages!` - ) + `Successfully synced ${newCodePackages.length} code packages!`, + ), ); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create code packages! - ${error.message}`) + colors.red(`Failed to create code packages! - ${error.message}`), ); } // Update existing codePackages const existingCodePackageInputs = mapCodePackagesToExisting.filter( - (x): x is [CodePackageInput, string] => !!x[1] + (x): x is [CodePackageInput, string] => !!x[1], ); logger.info( colors.magenta( - `Updating "${existingCodePackageInputs.length}" code packages...` - ) + `Updating "${existingCodePackageInputs.length}" code packages...`, + ), ); const chunks = chunk(existingCodePackageInputs, CHUNK_SIZE); @@ -267,25 +267,25 @@ export async function syncCodePackages( ]; if (!sdk) { throw new Error( - `Failed to find SDK with name: "${name}"` + `Failed to find SDK with name: "${name}"`, ); } return sdk.id; - }) + }), ), } : {}), id, - }) - ) + }), + ), ); logger.info( - colors.green(`Successfully updated "${chunk.length}" code packages!`) + colors.green(`Successfully updated "${chunk.length}" code packages!`), ); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to update code packages! - ${error.message}`) + colors.red(`Failed to update code packages! - ${error.message}`), ); } }); diff --git a/src/lib/graphql/syncConfigurationToTranscend.ts b/src/lib/graphql/syncConfigurationToTranscend.ts index e5ac1eac..eaa2b390 100644 --- a/src/lib/graphql/syncConfigurationToTranscend.ts +++ b/src/lib/graphql/syncConfigurationToTranscend.ts @@ -1,46 +1,46 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { TranscendInput } from "../../codecs"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { fetchAllActions } from "./fetchAllActions"; -import { fetchAllAttributes } from "./fetchAllAttributes"; -import { fetchApiKeys } from "./fetchApiKeys"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { TranscendInput } from '../../codecs'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { fetchAllActions } from './fetchAllActions'; +import { fetchAllAttributes } from './fetchAllAttributes'; +import { fetchApiKeys } from './fetchApiKeys'; import { ensureAllDataSubjectsExist, fetchAllDataSubjects, -} from "./fetchDataSubjects"; +} from './fetchDataSubjects'; import { fetchIdentifiersAndCreateMissing, Identifier, -} from "./fetchIdentifiers"; -import { syncAction } from "./syncAction"; -import { syncActionItemCollections } from "./syncActionItemCollections"; -import { syncActionItems } from "./syncActionItems"; -import { syncAgentFiles } from "./syncAgentFiles"; -import { syncAgentFunctions } from "./syncAgentFunctions"; -import { syncAgents } from "./syncAgents"; -import { syncAttribute } from "./syncAttribute"; -import { syncBusinessEntities } from "./syncBusinessEntities"; -import { syncConsentManager } from "./syncConsentManager"; -import { syncCookies } from "./syncCookies"; -import { syncDataCategories } from "./syncDataCategories"; -import { syncDataFlows } from "./syncDataFlows"; -import { syncDataSiloDependencies, syncDataSilos } from "./syncDataSilos"; -import { syncDataSubject } from "./syncDataSubject"; -import { syncEnricher } from "./syncEnrichers"; -import { syncIdentifier } from "./syncIdentifier"; -import { syncIntlMessages } from "./syncIntlMessages"; -import { syncPartitions } from "./syncPartitions"; -import { syncPolicies } from "./syncPolicies"; -import { syncPrivacyCenter } from "./syncPrivacyCenter"; -import { syncProcessingPurposes } from "./syncProcessingPurposes"; -import { syncPromptGroups } from "./syncPromptGroups"; -import { syncPromptPartials } from "./syncPromptPartials"; -import { syncPrompts } from "./syncPrompts"; -import { syncTeams } from "./syncTeams"; -import { syncTemplate } from "./syncTemplates"; -import { syncVendors } from "./syncVendors"; +} from './fetchIdentifiers'; +import { syncAction } from './syncAction'; +import { syncActionItemCollections } from './syncActionItemCollections'; +import { syncActionItems } from './syncActionItems'; +import { syncAgentFiles } from './syncAgentFiles'; +import { syncAgentFunctions } from './syncAgentFunctions'; +import { syncAgents } from './syncAgents'; +import { syncAttribute } from './syncAttribute'; +import { syncBusinessEntities } from './syncBusinessEntities'; +import { syncConsentManager } from './syncConsentManager'; +import { syncCookies } from './syncCookies'; +import { syncDataCategories } from './syncDataCategories'; +import { syncDataFlows } from './syncDataFlows'; +import { syncDataSiloDependencies, syncDataSilos } from './syncDataSilos'; +import { syncDataSubject } from './syncDataSubject'; +import { syncEnricher } from './syncEnrichers'; +import { syncIdentifier } from './syncIdentifier'; +import { syncIntlMessages } from './syncIntlMessages'; +import { syncPartitions } from './syncPartitions'; +import { syncPolicies } from './syncPolicies'; +import { syncPrivacyCenter } from './syncPrivacyCenter'; +import { syncProcessingPurposes } from './syncProcessingPurposes'; +import { syncPromptGroups } from './syncPromptGroups'; +import { syncPromptPartials } from './syncPromptPartials'; +import { syncPrompts } from './syncPrompts'; +import { syncTeams } from './syncTeams'; +import { syncTemplate } from './syncTemplates'; +import { syncVendors } from './syncVendors'; const CONCURRENCY = 10; @@ -70,7 +70,7 @@ export async function syncConfigurationToTranscend( deleteExtraAttributeValues?: boolean; /** classify data flow service if missing */ classifyService?: boolean; - } + }, ): Promise { let encounteredError = false; @@ -81,26 +81,26 @@ export async function syncConfigurationToTranscend( attributes, actions, identifiers, - "data-subjects": dataSubjects, - "business-entities": businessEntities, + 'data-subjects': dataSubjects, + 'business-entities': businessEntities, enrichers, cookies, - "consent-manager": consentManager, - "data-silos": dataSilos, - "data-flows": dataFlows, + 'consent-manager': consentManager, + 'data-silos': dataSilos, + 'data-flows': dataFlows, prompts, - "prompt-groups": promptGroups, - "prompt-partials": promptPartials, + 'prompt-groups': promptGroups, + 'prompt-partials': promptPartials, agents, - "agent-functions": agentFunctions, - "agent-files": agentFiles, + 'agent-functions': agentFunctions, + 'agent-files': agentFiles, vendors, - "data-categories": dataCategories, - "processing-purposes": processingPurposes, - "action-items": actionItems, - "action-item-collections": actionItemCollections, + 'data-categories': dataCategories, + 'processing-purposes': processingPurposes, + 'action-items': actionItems, + 'action-item-collections': actionItemCollections, teams, - "privacy-center": privacyCenter, + 'privacy-center': privacyCenter, messages, policies, partitions, @@ -113,7 +113,7 @@ export async function syncConfigurationToTranscend( ? fetchIdentifiersAndCreateMissing( input, client, - !publishToPrivacyCenter + !publishToPrivacyCenter, ) : ({} as Record), // Grab all data subjects in the organization @@ -123,7 +123,7 @@ export async function syncConfigurationToTranscend( // Grab API keys dataSilos && dataSilos - .map((dataSilo) => dataSilo["api-key-title"] || []) + .map((dataSilo) => dataSilo['api-key-title'] || []) .reduce((accumulator, lst) => accumulator + lst.length, 0) > 0 ? fetchApiKeys(input, client) : {}, @@ -131,14 +131,14 @@ export async function syncConfigurationToTranscend( // Sync consent manager if (consentManager) { - logger.info(colors.magenta("Syncing consent manager...")); + logger.info(colors.magenta('Syncing consent manager...')); try { await syncConsentManager(client, consentManager); - logger.info(colors.green("Successfully synced consent manager!")); + logger.info(colors.green('Successfully synced consent manager!')); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to sync consent manager! - ${error.message}`) + colors.red(`Failed to sync consent manager! - ${error.message}`), ); } } @@ -165,7 +165,7 @@ export async function syncConfigurationToTranscend( // Sync email templates if (templates) { logger.info( - colors.magenta(`Syncing "${templates.length}" email templates...`) + colors.magenta(`Syncing "${templates.length}" email templates...`), ); await map( templates, @@ -174,20 +174,20 @@ export async function syncConfigurationToTranscend( try { await syncTemplate(template, client); logger.info( - colors.green(`Successfully synced template "${template.title}"!`) + colors.green(`Successfully synced template "${template.title}"!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync template "${template.title}"! - ${error.message}` - ) + `Failed to sync template "${template.title}"! - ${error.message}`, + ), ); } }, { concurrency: CONCURRENCY, - } + }, ); logger.info(colors.green(`Synced "${templates.length}" email templates!`)); } @@ -196,7 +196,7 @@ export async function syncConfigurationToTranscend( if (businessEntities) { const businessEntitySuccess = await syncBusinessEntities( client, - businessEntities + businessEntities, ); encounteredError = encounteredError || !businessEntitySuccess; } @@ -211,7 +211,7 @@ export async function syncConfigurationToTranscend( if (dataCategories) { const dataCategoriesSuccess = await syncDataCategories( client, - dataCategories + dataCategories, ); encounteredError = encounteredError || !dataCategoriesSuccess; } @@ -220,7 +220,7 @@ export async function syncConfigurationToTranscend( if (processingPurposes) { const processingPurposesSuccess = await syncProcessingPurposes( client, - processingPurposes + processingPurposes, ); encounteredError = encounteredError || !processingPurposesSuccess; } @@ -241,7 +241,7 @@ export async function syncConfigurationToTranscend( if (agentFunctions) { const agentFunctionsSuccess = await syncAgentFunctions( client, - agentFunctions + agentFunctions, ); encounteredError = encounteredError || !agentFunctionsSuccess; } @@ -262,7 +262,7 @@ export async function syncConfigurationToTranscend( if (actionItemCollections) { const actionItemCollectionsSuccess = await syncActionItemCollections( client, - actionItemCollections + actionItemCollections, ); encounteredError = encounteredError || !actionItemCollectionsSuccess; } @@ -276,7 +276,7 @@ export async function syncConfigurationToTranscend( attributes, async (attribute) => { const existing = existingAttributes.find( - (attribute_) => attribute_.name === attribute.name + (attribute_) => attribute_.name === attribute.name, ); logger.info(colors.magenta(`Syncing attribute "${attribute.name}"...`)); @@ -286,20 +286,20 @@ export async function syncConfigurationToTranscend( deleteExtraAttributeValues, }); logger.info( - colors.green(`Successfully synced attribute "${attribute.name}"!`) + colors.green(`Successfully synced attribute "${attribute.name}"!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync attribute "${attribute.name}"! - ${error.message}` - ) + `Failed to sync attribute "${attribute.name}"! - ${error.message}`, + ), ); } }, { concurrency: CONCURRENCY, - } + }, ); logger.info(colors.green(`Synced "${attributes.length}" attributes!`)); } @@ -324,20 +324,20 @@ export async function syncConfigurationToTranscend( dataSubjectsByName, }); logger.info( - colors.green(`Successfully synced enricher "${enricher.title}"!`) + colors.green(`Successfully synced enricher "${enricher.title}"!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync enricher "${enricher.title}"! - ${error.message}` - ) + `Failed to sync enricher "${enricher.title}"! - ${error.message}`, + ), ); } }, { concurrency: CONCURRENCY, - } + }, ); logger.info(colors.green(`Synced "${enrichers.length}" enrichers!`)); } @@ -346,7 +346,7 @@ export async function syncConfigurationToTranscend( if (identifiers) { // Fetch existing logger.info( - colors.magenta(`Syncing "${identifiers.length}" identifiers...`) + colors.magenta(`Syncing "${identifiers.length}" identifiers...`), ); await map( identifiers, @@ -354,12 +354,12 @@ export async function syncConfigurationToTranscend( const existing = identifierByName[identifier.name]; if (!existing) { throw new Error( - `Failed to find identifier with name: ${identifier.type}. Should have been auto-created by cli.` + `Failed to find identifier with name: ${identifier.type}. Should have been auto-created by cli.`, ); } logger.info( - colors.magenta(`Syncing identifier "${identifier.type}"...`) + colors.magenta(`Syncing identifier "${identifier.type}"...`), ); try { await syncIdentifier(client, { @@ -369,20 +369,22 @@ export async function syncConfigurationToTranscend( skipPublish: !publishToPrivacyCenter, }); logger.info( - colors.green(`Successfully synced identifier "${identifier.type}"!`) + colors.green( + `Successfully synced identifier "${identifier.type}"!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync identifier "${identifier.type}"! - ${error.message}` - ) + `Failed to sync identifier "${identifier.type}"! - ${error.message}`, + ), ); } }, { concurrency: CONCURRENCY, - } + }, ); logger.info(colors.green(`Synced "${identifiers.length}" identifiers!`)); } @@ -396,11 +398,11 @@ export async function syncConfigurationToTranscend( actions, async (action) => { const existing = existingActions.find( - (act) => act.type === action.type + (act) => act.type === action.type, ); if (!existing) { throw new Error( - `Failed to find action with type: ${action.type}. Should have already existing in the organization.` + `Failed to find action with type: ${action.type}. Should have already existing in the organization.`, ); } @@ -412,20 +414,20 @@ export async function syncConfigurationToTranscend( skipPublish: !publishToPrivacyCenter, }); logger.info( - colors.green(`Successfully synced action "${action.type}"!`) + colors.green(`Successfully synced action "${action.type}"!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync action "${action.type}"! - ${error.message}` - ) + `Failed to sync action "${action.type}"! - ${error.message}`, + ), ); } }, { concurrency: CONCURRENCY, - } + }, ); logger.info(colors.green(`Synced "${actions.length}" actions!`)); } @@ -434,23 +436,23 @@ export async function syncConfigurationToTranscend( if (dataSubjects) { // Fetch existing logger.info( - colors.magenta(`Syncing "${dataSubjects.length}" data subjects...`) + colors.magenta(`Syncing "${dataSubjects.length}" data subjects...`), ); const existingDataSubjects = await fetchAllDataSubjects(client); await map( dataSubjects, async (dataSubject) => { const existing = existingDataSubjects.find( - (subj) => subj.type === dataSubject.type + (subj) => subj.type === dataSubject.type, ); if (!existing) { throw new Error( - `Failed to find data subject with type: ${dataSubject.type}. Should have already existing in the organization.` + `Failed to find data subject with type: ${dataSubject.type}. Should have already existing in the organization.`, ); } logger.info( - colors.magenta(`Syncing data subject "${dataSubject.type}"...`) + colors.magenta(`Syncing data subject "${dataSubject.type}"...`), ); try { await syncDataSubject(client, { @@ -460,21 +462,21 @@ export async function syncConfigurationToTranscend( }); logger.info( colors.green( - `Successfully synced data subject "${dataSubject.type}"!` - ) + `Successfully synced data subject "${dataSubject.type}"!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync data subject "${dataSubject.type}"! - ${error.message}` - ) + `Failed to sync data subject "${dataSubject.type}"! - ${error.message}`, + ), ); } }, { concurrency: CONCURRENCY, - } + }, ); logger.info(colors.green(`Synced "${dataSubjects.length}" data subjects!`)); } @@ -484,7 +486,7 @@ export async function syncConfigurationToTranscend( const syncedDataFlows = await syncDataFlows( client, dataFlows, - classifyService + classifyService, ); encounteredError = encounteredError || !syncedDataFlows; } @@ -518,15 +520,15 @@ export async function syncConfigurationToTranscend( dataSubjectsByName, apiKeysByTitle: apiKeyTitleMap, pageSize, - } + }, ); if (dataSilos) for (const dataSilo of dataSilos) { // Queue up dependency update - if (dataSilo["deletion-dependencies"]) { + if (dataSilo['deletion-dependencies']) { dependencyUpdates.push([ dataSiloTitleToId[dataSilo.title], - dataSilo["deletion-dependencies"], + dataSilo['deletion-dependencies'], ]); } } diff --git a/src/lib/graphql/syncConsentManager.ts b/src/lib/graphql/syncConsentManager.ts index 2f45a885..5184f7aa 100644 --- a/src/lib/graphql/syncConsentManager.ts +++ b/src/lib/graphql/syncConsentManager.ts @@ -1,22 +1,22 @@ import { InitialViewState, OnConsentExpiry, -} from "@transcend-io/airgap.js-types"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; +} from '@transcend-io/airgap.js-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; import { ConsentManageExperienceInput, ConsentManagerInput, -} from "../../codecs"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { fetchAllPurposes } from "./fetchAllPurposes"; +} from '../../codecs'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { fetchAllPurposes } from './fetchAllPurposes'; import { fetchConsentManagerExperiences, fetchConsentManagerId, -} from "./fetchConsentManagerId"; -import { fetchPrivacyCenterId } from "./fetchPrivacyCenterId"; +} from './fetchConsentManagerId'; +import { fetchPrivacyCenterId } from './fetchPrivacyCenterId'; import { CREATE_CONSENT_EXPERIENCE, CREATE_CONSENT_MANAGER, @@ -30,12 +30,12 @@ import { UPDATE_CONSENT_MANAGER_THEME, UPDATE_CONSENT_MANAGER_VERSION, UPDATE_TOGGLE_USP_API, -} from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; -import { fetchPartitions } from "./syncPartitions"; +} from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; +import { fetchPartitions } from './syncPartitions'; const PURPOSES_LINK = - "https://app.transcend.io/consent-manager/regional-experiences/purposes"; + 'https://app.transcend.io/consent-manager/regional-experiences/purposes'; /** * Sync consent manager experiences up to Transcend @@ -45,15 +45,15 @@ const PURPOSES_LINK = */ export async function syncConsentManagerExperiences( client: GraphQLClient, - experiences: ConsentManageExperienceInput[] + experiences: ConsentManageExperienceInput[], ): Promise { // Fetch existing experiences and const existingExperiences = await fetchConsentManagerExperiences(client); - const experienceLookup = keyBy(existingExperiences, "name"); + const experienceLookup = keyBy(existingExperiences, 'name'); // Fetch existing purposes const purposes = await fetchAllPurposes(client); - const purposeLookup = keyBy(purposes, "trackingType"); + const purposeLookup = keyBy(purposes, 'trackingType'); // Bulk update or create experiences await map( @@ -65,7 +65,7 @@ export async function syncConsentManagerExperiences( if (!existingPurpose) { throw new Error( `Invalid purpose trackingType provided at consentManager.experiences[${ind}].purposes[${ind2}]: ` + - `${purpose.trackingType}. See list of valid purposes ${PURPOSES_LINK}` + `${purpose.trackingType}. See list of valid purposes ${PURPOSES_LINK}`, ); } return existingPurpose.id; @@ -75,7 +75,7 @@ export async function syncConsentManagerExperiences( if (!existingPurpose) { throw new Error( `Invalid purpose trackingType provided at consentManager.experiences[${ind}].optedOutPurposes[${ind2}]: ` + - `${purpose.trackingType}. See list of valid purposes ${PURPOSES_LINK}` + `${purpose.trackingType}. See list of valid purposes ${PURPOSES_LINK}`, ); } return existingPurpose.id; @@ -104,7 +104,7 @@ export async function syncConsentManagerExperiences( }, }); logger.info( - colors.green(`Successfully synced consent experience "${exp.name}"!`) + colors.green(`Successfully synced consent experience "${exp.name}"!`), ); } else { // create new experience @@ -125,13 +125,15 @@ export async function syncConsentManagerExperiences( }, }); logger.info( - colors.green(`Successfully created consent experience "${exp.name}"!`) + colors.green( + `Successfully created consent experience "${exp.name}"!`, + ), ); } }, { concurrency: 10, - } + }, ); } @@ -143,7 +145,7 @@ export async function syncConsentManagerExperiences( */ export async function syncConsentManager( client: GraphQLClient, - consentManager: ConsentManagerInput + consentManager: ConsentManagerInput, ): Promise { let airgapBundleId: string; @@ -152,7 +154,7 @@ export async function syncConsentManager( airgapBundleId = await fetchConsentManagerId(client, 1); } catch (error) { // TODO: https://transcend.height.app/T-23778 - if (error.message.includes("AirgapBundle not found")) { + if (error.message.includes('AirgapBundle not found')) { const privacyCenterId = await fetchPrivacyCenterId(client); const { createConsentManager } = await makeGraphQLRequest<{ @@ -186,11 +188,11 @@ export async function syncConsentManager( if (consentManager.partition) { const partitions = await fetchPartitions(client); const partitionToUpdate = partitions.find( - (part) => part.name === consentManager.partition + (part) => part.name === consentManager.partition, ); if (!partitionToUpdate) { throw new Error( - `Partition "${consentManager.partition}" not found. Please create the partition first.` + `Partition "${consentManager.partition}" not found. Please create the partition first.`, ); } await makeGraphQLRequest(client, UPDATE_CONSENT_MANAGER_PARTITION, { diff --git a/src/lib/graphql/syncCookies.ts b/src/lib/graphql/syncCookies.ts index 2c0dd7a4..7fcdf38b 100644 --- a/src/lib/graphql/syncCookies.ts +++ b/src/lib/graphql/syncCookies.ts @@ -1,13 +1,13 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk } from "lodash-es"; -import { CookieInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { fetchConsentManagerId } from "./fetchConsentManagerId"; -import { UPDATE_OR_CREATE_COOKIES } from "./gqls"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk } from 'lodash-es'; +import { CookieInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { fetchConsentManagerId } from './fetchConsentManagerId'; +import { UPDATE_OR_CREATE_COOKIES } from './gqls'; // import { keyBy } from 'lodash-es'; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const MAX_PAGE_SIZE = 100; @@ -19,7 +19,7 @@ const MAX_PAGE_SIZE = 100; */ export async function updateOrCreateCookies( client: GraphQLClient, - cookieInputs: CookieInput[] + cookieInputs: CookieInput[], ): Promise { const airgapBundleId = await fetchConsentManagerId(client); @@ -64,7 +64,7 @@ export async function updateOrCreateCookies( */ export async function syncCookies( client: GraphQLClient, - cookies: CookieInput[] + cookies: CookieInput[], ): Promise { let encounteredError = false; logger.info(colors.magenta(`Syncing "${cookies.length}" cookies...`)); @@ -73,14 +73,14 @@ export async function syncCookies( const notUnique = cookies.filter( (cookie) => cookies.filter( - (cook) => cookie.name === cook.name && cookie.isRegex === cook.isRegex - ).length > 1 + (cook) => cookie.name === cook.name && cookie.isRegex === cook.isRegex, + ).length > 1, ); if (notUnique.length > 0) { throw new Error( `Failed to upload cookies as there were non-unique entries found: ${notUnique .map(({ name }) => name) - .join(",")}` + .join(',')}`, ); } diff --git a/src/lib/graphql/syncDataCategories.ts b/src/lib/graphql/syncDataCategories.ts index 0b37f92c..2821bf2b 100644 --- a/src/lib/graphql/syncDataCategories.ts +++ b/src/lib/graphql/syncDataCategories.ts @@ -1,15 +1,15 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { DataCategoryInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { DataCategoryInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { DataSubCategory, fetchAllDataCategories, -} from "./fetchAllDataCategories"; -import { CREATE_DATA_SUB_CATEGORY, UPDATE_DATA_SUB_CATEGORIES } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from './fetchAllDataCategories'; +import { CREATE_DATA_SUB_CATEGORY, UPDATE_DATA_SUB_CATEGORIES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new data category @@ -20,8 +20,8 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function createDataCategory( client: GraphQLClient, - dataCategory: DataCategoryInput -): Promise> { + dataCategory: DataCategoryInput, +): Promise> { const input = { name: dataCategory.name, category: dataCategory.category, @@ -49,7 +49,7 @@ export async function createDataCategory( */ export async function updateDataCategories( client: GraphQLClient, - dataCategoryIdPairs: [DataCategoryInput, string][] + dataCategoryIdPairs: [DataCategoryInput, string][], ): Promise { await makeGraphQLRequest(client, UPDATE_DATA_SUB_CATEGORIES, { input: { @@ -72,7 +72,7 @@ export async function updateDataCategories( */ export async function syncDataCategories( client: GraphQLClient, - inputs: DataCategoryInput[] + inputs: DataCategoryInput[], ): Promise { // Fetch existing logger.info(colors.magenta(`Syncing "${inputs.length}" data categories...`)); @@ -85,15 +85,15 @@ export async function syncDataCategories( // Look up by name const dataCategoryByName: Record< string, - Pick + Pick > = keyBy( existingDataCategories, - ({ name, category }) => `${name}:${category}` + ({ name, category }) => `${name}:${category}`, ); // Create new data categories const newDataCategories = inputs.filter( - (input) => !dataCategoryByName[`${input.name}:${input.category}`] + (input) => !dataCategoryByName[`${input.name}:${input.category}`], ); // Create new data categories @@ -105,15 +105,15 @@ export async function syncDataCategories( ] = newDataCategory; logger.info( colors.green( - `Successfully synced data category "${dataCategory.name}"!` - ) + `Successfully synced data category "${dataCategory.name}"!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync data category "${dataCategory.name}"! - ${error.message}` - ) + `Failed to sync data category "${dataCategory.name}"! - ${error.message}`, + ), ); } }); @@ -126,17 +126,17 @@ export async function syncDataCategories( inputs.map((input) => [ input, dataCategoryByName[`${input.name}:${input.category}`].id, - ]) + ]), ); logger.info( - colors.green(`Successfully synced "${inputs.length}" data categories!`) + colors.green(`Successfully synced "${inputs.length}" data categories!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" data categories ! - ${error.message}` - ) + `Failed to sync "${inputs.length}" data categories ! - ${error.message}`, + ), ); } diff --git a/src/lib/graphql/syncDataFlows.ts b/src/lib/graphql/syncDataFlows.ts index b781f6a7..795fa6de 100644 --- a/src/lib/graphql/syncDataFlows.ts +++ b/src/lib/graphql/syncDataFlows.ts @@ -1,14 +1,14 @@ -import { ConsentTrackerStatus } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk } from "lodash-es"; -import { DataFlowInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { fetchAllDataFlows } from "./fetchAllDataFlows"; -import { fetchConsentManagerId } from "./fetchConsentManagerId"; -import { CREATE_DATA_FLOWS, UPDATE_DATA_FLOWS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk } from 'lodash-es'; +import { DataFlowInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { fetchAllDataFlows } from './fetchAllDataFlows'; +import { fetchConsentManagerId } from './fetchConsentManagerId'; +import { CREATE_DATA_FLOWS, UPDATE_DATA_FLOWS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const MAX_PAGE_SIZE = 100; @@ -22,7 +22,7 @@ const MAX_PAGE_SIZE = 100; export async function updateDataFlows( client: GraphQLClient, dataFlowInputs: [DataFlowInput, string][], - classifyService = false + classifyService = false, ): Promise { const airgapBundleId = await fetchConsentManagerId(client); @@ -70,7 +70,7 @@ export async function updateDataFlows( export async function createDataFlows( client: GraphQLClient, dataFlowInputs: DataFlowInput[], - classifyService = false + classifyService = false, ): Promise { const airgapBundleId = await fetchConsentManagerId(client); // TODO: https://transcend.height.app/T-19841 - add with custom purposes @@ -117,7 +117,7 @@ export async function createDataFlows( export async function syncDataFlows( client: GraphQLClient, dataFlows: DataFlowInput[], - classifyService: boolean + classifyService: boolean, ): Promise { let encounteredError = false; logger.info(colors.magenta(`Syncing "${dataFlows.length}" data flows...`)); @@ -127,8 +127,8 @@ export async function syncDataFlows( const notUnique = dataFlows.filter( (dataFlow) => dataFlows.filter( - (flow) => dataFlow.value === flow.value && dataFlow.type === flow.type - ).length > 1 + (flow) => dataFlow.value === flow.value && dataFlow.type === flow.type, + ).length > 1, ); // Throw error to prompt user to de-dupe before uploading @@ -136,13 +136,13 @@ export async function syncDataFlows( throw new Error( `Failed to upload data flows as there were non-unique entries found: ${notUnique .map(({ value }) => value) - .join(",")}` + .join(',')}`, ); } // Fetch existing data flows to determine whether we are creating a new data flow // or updating an existing data flow - logger.info(colors.magenta("Fetching data flows...")); + logger.info(colors.magenta('Fetching data flows...')); const [existingLiveDataFlows, existingInReviewDataFlows] = await Promise.all([ fetchAllDataFlows(client, ConsentTrackerStatus.Live), fetchAllDataFlows(client, ConsentTrackerStatus.NeedsReview), @@ -153,7 +153,7 @@ export async function syncDataFlows( const mapDataFlowsToExisting = dataFlows.map((dataFlow) => [ dataFlow, allDataFlows.find( - (flow) => dataFlow.value === flow.value && dataFlow.type === flow.type + (flow) => dataFlow.value === flow.value && dataFlow.type === flow.type, )?.id, ]); @@ -163,11 +163,11 @@ export async function syncDataFlows( .map(([flow]) => flow as DataFlowInput); try { logger.info( - colors.magenta(`Creating "${newDataFlows.length}" new data flows...`) + colors.magenta(`Creating "${newDataFlows.length}" new data flows...`), ); await createDataFlows(client, newDataFlows, classifyService); logger.info( - colors.green(`Successfully synced ${newDataFlows.length} data flows!`) + colors.green(`Successfully synced ${newDataFlows.length} data flows!`), ); } catch (error) { encounteredError = true; @@ -176,17 +176,17 @@ export async function syncDataFlows( // Update existing data flows const existingDataFlows = mapDataFlowsToExisting.filter( - (x): x is [DataFlowInput, string] => !!x[1] + (x): x is [DataFlowInput, string] => !!x[1], ); try { logger.info( - colors.magenta(`Updating "${existingDataFlows.length}" data flows...`) + colors.magenta(`Updating "${existingDataFlows.length}" data flows...`), ); await updateDataFlows(client, existingDataFlows, classifyService); logger.info( colors.green( - `Successfully updated "${existingDataFlows.length}" data flows!` - ) + `Successfully updated "${existingDataFlows.length}" data flows!`, + ), ); } catch (error) { encounteredError = true; diff --git a/src/lib/graphql/syncDataSilos.ts b/src/lib/graphql/syncDataSilos.ts index 2c47f458..da3b828b 100644 --- a/src/lib/graphql/syncDataSilos.ts +++ b/src/lib/graphql/syncDataSilos.ts @@ -6,24 +6,24 @@ import { PromptAVendorEmailSendType, RequestActionObjectResolver, SubDataPointDataSubCategoryGuessStatus, -} from "@transcend-io/privacy-types"; -import { apply } from "@transcend-io/type-utils"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk, keyBy, sortBy } from "lodash-es"; +} from '@transcend-io/privacy-types'; +import { apply } from '@transcend-io/type-utils'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk, keyBy, sortBy } from 'lodash-es'; import { DataCategoryInput, DataSiloInput, ProcessingPurposeInput, -} from "../../codecs"; -import { logger } from "../../logger"; -import { map, mapSeries } from "../bluebird-replace"; -import { ApiKey } from "./fetchApiKeys"; +} from '../../codecs'; +import { logger } from '../../logger'; +import { map, mapSeries } from '../bluebird-replace'; +import { ApiKey } from './fetchApiKeys'; import { convertToDataSubjectBlockList, DataSubject, -} from "./fetchDataSubjects"; +} from './fetchDataSubjects'; import { CREATE_DATA_SILOS, DATA_POINTS, @@ -33,8 +33,8 @@ import { SUB_DATA_POINTS_WITH_GUESSES, UPDATE_DATA_SILOS, UPDATE_OR_CREATE_DATA_POINT, -} from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface DataSiloAttributeValue { /** Key associated to value */ @@ -93,12 +93,12 @@ export async function fetchAllDataSilos( integrationNames?: string[]; /** GQL query for data silos */ gql?: string; - } + }, ): Promise { logger.info( colors.magenta( - `Fetching ${ids.length === 0 ? "all" : ids.length} Data Silos...` - ) + `Fetching ${ids.length === 0 ? 'all' : ids.length} Data Silos...`, + ), ); const dataSilos: TDataSilo[] = []; @@ -131,13 +131,13 @@ export async function fetchAllDataSilos( logger.info( colors.green( `Found a total of ${dataSilos.length} data silo${ - ids.length > 0 ? ` matching IDs ${ids.join(",")}` : "" + ids.length > 0 ? ` matching IDs ${ids.join(',')}` : '' }s${ integrationNames.length > 0 - ? ` matching integrationNames ${integrationNames.join(",")}` - : "" - }` - ) + ? ` matching integrationNames ${integrationNames.join(',')}` + : '' + }`, + ), ); return dataSilos.sort((a, b) => a.title.localeCompare(b.title)); @@ -266,7 +266,7 @@ export async function fetchAllSubDataPoints( pageSize: number; /** When true, metadata around guessed data categories should be included */ includeGuessedCategories?: boolean; - } + }, ): Promise { const subDataPoints: SubDataPoint[] = []; @@ -277,7 +277,7 @@ export async function fetchAllSubDataPoints( try { if (debug) { logger.log( - colors.magenta(`Pulling in subdatapoints for offset ${offset}`) + colors.magenta(`Pulling in subdatapoints for offset ${offset}`), ); } const { @@ -299,7 +299,7 @@ export async function fetchAllSubDataPoints( dataPoints: [dataPointId], }, offset, - } + }, ); subDataPoints.push(...nodes); @@ -309,20 +309,20 @@ export async function fetchAllSubDataPoints( if (debug) { logger.log( colors.green( - `Pulled in subdatapoints for offset ${offset} for dataPointId=${dataPointId}` - ) + `Pulled in subdatapoints for offset ${offset} for dataPointId=${dataPointId}`, + ), ); } } catch (error) { logger.error( colors.red( - `An error fetching subdatapoints for offset ${offset} for dataPointId=${dataPointId}` - ) + `An error fetching subdatapoints for offset ${offset} for dataPointId=${dataPointId}`, + ), ); throw error; } } while (shouldContinue); - return sortBy(subDataPoints, "name"); + return sortBy(subDataPoints, 'name'); } /** @@ -350,7 +350,7 @@ export async function fetchAllDataPoints( skipSubDatapoints?: boolean; /** When true, metadata around guessed data categories should be included */ includeGuessedCategories?: boolean; - } + }, ): Promise { const dataPoints: DataPointWithSubDataPoint[] = []; @@ -383,8 +383,8 @@ export async function fetchAllDataPoints( if (debug) { logger.info( colors.magenta( - `Fetched ${nodes.length} datapoints at offset: ${offset}` - ) + `Fetched ${nodes.length} datapoints at offset: ${offset}`, + ), ); } @@ -397,8 +397,8 @@ export async function fetchAllDataPoints( if (debug) { logger.info( colors.magenta( - `Fetching subdatapoints for ${node.name} for datapoint offset ${offset}` - ) + `Fetching subdatapoints for ${node.name} for datapoint offset ${offset}`, + ), ); } @@ -410,22 +410,22 @@ export async function fetchAllDataPoints( dataPoints.push({ ...node, subDataPoints: subDataPoints.sort((a, b) => - a.name.localeCompare(b.name) + a.name.localeCompare(b.name), ), }); if (debug) { logger.info( colors.green( - `Successfully fetched subdatapoints for ${node.name}` - ) + `Successfully fetched subdatapoints for ${node.name}`, + ), ); } } catch (error) { logger.error( colors.red( - `An error fetching subdatapoints for ${node.name} datapoint offset ${offset}` - ) + `An error fetching subdatapoints for ${node.name} datapoint offset ${offset}`, + ), ); throw error; } @@ -433,14 +433,14 @@ export async function fetchAllDataPoints( { concurrency: 5, - } + }, ); if (debug) { logger.info( colors.green( - `Fetched all subdatapoints for page of datapoints at offset: ${offset}` - ) + `Fetched all subdatapoints for page of datapoints at offset: ${offset}`, + ), ); } } @@ -587,7 +587,7 @@ export async function fetchEnrichedDataSilos( skipSubDatapoints?: boolean; /** When true, metadata around guessed data categories should be included */ includeGuessedCategories?: boolean; - } + }, ): Promise<[DataSiloEnriched, DataPointWithSubDataPoint[]][]> { const dataSilos: [DataSiloEnriched, DataPointWithSubDataPoint[]][] = []; @@ -605,8 +605,8 @@ export async function fetchEnrichedDataSilos( await mapSeries(silos, async (silo, index) => { logger.info( colors.magenta( - `[${index + 1}/${silos.length}] Fetching data silo - ${silo.title}` - ) + `[${index + 1}/${silos.length}] Fetching data silo - ${silo.title}`, + ), ); const dataPoints = await fetchAllDataPoints(client, silo.id, { @@ -621,8 +621,8 @@ export async function fetchEnrichedDataSilos( colors.green( `[${index + 1}/${ silos.length - }] Successfully fetched datapoint for - ${silo.title}` - ) + }] Successfully fetched datapoint for - ${silo.title}`, + ), ); } @@ -632,8 +632,8 @@ export async function fetchEnrichedDataSilos( logger.info( colors.green( - `Successfully fetched all ${silos.length} data silo configurations` - ) + `Successfully fetched all ${silos.length} data silo configurations`, + ), ); return dataSilos; @@ -661,7 +661,7 @@ export async function syncDataSilos( dataSubjectsByName: Record; /** API key title to API key */ apiKeysByTitle: Record; - } + }, ): Promise<{ /** Whether successfully updated */ success: boolean; @@ -681,20 +681,20 @@ export async function syncDataSilos( }); // Create a mapping of title -> existing silo, if it exists - const existingDataSiloByTitle = keyBy>( + const existingDataSiloByTitle = keyBy>( existingDataSilos, - "title" + 'title', ); // Create new silos that do not exist const newDataSiloInputs = dataSilos.filter( - ({ title }) => !existingDataSiloByTitle[title] + ({ title }) => !existingDataSiloByTitle[title], ); if (newDataSiloInputs.length > 0) { logger.info( colors.magenta( - `Creating "${newDataSiloInputs.length}" data silos that did not exist...` - ) + `Creating "${newDataSiloInputs.length}" data silos that did not exist...`, + ), ); // Batch the creation @@ -706,11 +706,11 @@ export async function syncDataSilos( /** Mutation result */ createDataSilos: { /** New data silos */ - dataSilos: Pick[]; + dataSilos: Pick[]; }; }>(client, CREATE_DATA_SILOS, { input: dependencyUpdateChunk.map((input) => ({ - name: input["outer-type"] || input.integrationName, + name: input['outer-type'] || input.integrationName, title: input.title, country: input.country, countrySubDivision: input.countrySubDivision, @@ -725,8 +725,8 @@ export async function syncDataSilos( logger.info( colors.green( - `Successfully created "${newDataSiloInputs.length}" data silos!` - ) + `Successfully created "${newDataSiloInputs.length}" data silos!`, + ), ); } @@ -737,14 +737,14 @@ export async function syncDataSilos( colors.magenta( `[Batch ${ind + 1}/${chunkedUpdates.length}] Syncing "${ dataSiloUpdateChunk.length - }" data silos` - ) + }" data silos`, + ), ); await makeGraphQLRequest<{ /** Mutation result */ updateDataSilos: { /** New data silos */ - dataSilos: Pick[]; + dataSilos: Pick[]; }; }>(client, UPDATE_DATA_SILOS, { input: { @@ -755,37 +755,37 @@ export async function syncDataSilos( url: input.url, headers: input.headers, description: input.description, - identifiers: input["identity-keys"], + identifiers: input['identity-keys'], isLive: !input.disabled, ownerEmails: input.owners, teamNames: input.teams, // clear out if not specified, otherwise the update needs to be applied after // all data silos are created - dependedOnDataSiloTitles: input["deletion-dependencies"] + dependedOnDataSiloTitles: input['deletion-dependencies'] ? undefined : [], - apiKeyId: input["api-key-title"] - ? apiKeysByTitle[input["api-key-title"]].id + apiKeyId: input['api-key-title'] + ? apiKeysByTitle[input['api-key-title']].id : undefined, - dataSubjectBlockListIds: input["data-subjects"] + dataSubjectBlockListIds: input['data-subjects'] ? convertToDataSubjectBlockList( - input["data-subjects"], - dataSubjectsByName + input['data-subjects'], + dataSubjectsByName, ) : undefined, attributes: input.attributes, businessEntityTitles: input.businessEntityTitles, // AVC settings - notifyEmailAddress: input["email-settings"]?.["notify-email-address"], + notifyEmailAddress: input['email-settings']?.['notify-email-address'], promptAVendorEmailSendFrequency: - input["email-settings"]?.["send-frequency"], - promptAVendorEmailSendType: input["email-settings"]?.["send-type"], + input['email-settings']?.['send-frequency'], + promptAVendorEmailSendType: input['email-settings']?.['send-type'], promptAVendorEmailIncludeIdentifiersAttachment: - input["email-settings"]?.["include-identifiers-attachment"], + input['email-settings']?.['include-identifiers-attachment'], promptAVendorEmailCompletionLinkType: - input["email-settings"]?.["completion-link-type"], + input['email-settings']?.['completion-link-type'], manualWorkRetryFrequency: - input["email-settings"]?.["manual-work-retry-frequency"], + input['email-settings']?.['manual-work-retry-frequency'], })), }, }); @@ -793,8 +793,8 @@ export async function syncDataSilos( colors.green( `[Batch ${ind + 1}/${chunkedUpdates.length}] Synced "${ dataSiloUpdateChunk.length - }" data silos!` - ) + }" data silos!`, + ), ); }); @@ -803,18 +803,18 @@ export async function syncDataSilos( // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); const dataSilosWithDataPoints = dataSilos.filter( - ({ datapoints = [] }) => datapoints.length > 0 + ({ datapoints = [] }) => datapoints.length > 0, ); const totalDataPoints = dataSilos .map(({ datapoints = [] }) => datapoints.length) .reduce((accumulator, count) => accumulator + count, 0); logger.info( colors.magenta( - `Syncing "${totalDataPoints}" datapoints from "${dataSilosWithDataPoints.length}" data silos...` - ) + `Syncing "${totalDataPoints}" datapoints from "${dataSilosWithDataPoints.length}" data silos...`, + ), ); progressBar.start(totalDataPoints, 0); let total = 0; @@ -841,21 +841,21 @@ export async function syncDataSilos( categories: categories ? categories.map((category) => ({ ...category, - name: category.name || "Other", + name: category.name || 'Other', })) : undefined, purposes: purposes ? purposes.map((purpose) => ({ ...purpose, - name: purpose.name || "Other", + name: purpose.name || 'Other', })) : undefined, attributes, accessRequestVisibilityEnabled: - rest["access-request-visibility-enabled"], + rest['access-request-visibility-enabled'], erasureRequestRedactionEnabled: - rest["erasure-request-redaction-enabled"], - }) + rest['erasure-request-redaction-enabled'], + }), ) : undefined; @@ -875,27 +875,27 @@ export async function syncDataSilos( teamNames: datapoint.teams, } : {}), - ...(datapoint["data-collection-tag"] - ? { dataCollectionTag: datapoint["data-collection-tag"] } + ...(datapoint['data-collection-tag'] + ? { dataCollectionTag: datapoint['data-collection-tag'] } : {}), - querySuggestions: datapoint["privacy-action-queries"] - ? Object.entries(datapoint["privacy-action-queries"]).map( + querySuggestions: datapoint['privacy-action-queries'] + ? Object.entries(datapoint['privacy-action-queries']).map( ([key, value]) => ({ requestType: key, suggestedQuery: value, - }) + }), ) : undefined, - enabledActions: datapoint["privacy-actions"] || [], // clear out when not specified + enabledActions: datapoint['privacy-actions'] || [], // clear out when not specified subDataPoints: fields, }; // Ensure no duplicate sub-datapoints are provided const subDataPointsToUpdate = (payload.subDataPoints || []).map( - ({ name }) => name + ({ name }) => name, ); const duplicateDataPoints = subDataPointsToUpdate.filter( - (name, index) => subDataPointsToUpdate.indexOf(name) !== index + (name, index) => subDataPointsToUpdate.indexOf(name) !== index, ); if (duplicateDataPoints.length > 0) { logger.info( @@ -903,9 +903,9 @@ export async function syncDataSilos( `\nCannot update datapoint "${ datapoint.key }" as it has duplicate sub-datapoints with the same name: \n${duplicateDataPoints.join( - "\n" - )}` - ) + '\n', + )}`, + ), ); encounteredError = true; } else { @@ -913,13 +913,13 @@ export async function syncDataSilos( await makeGraphQLRequest( client, UPDATE_OR_CREATE_DATA_POINT, - payload + payload, ); } catch (error) { logger.info( colors.red( - `\nFailed to update datapoint "${datapoint.key}" for data silo "${title}"! - \n${error.message}` - ) + `\nFailed to update datapoint "${datapoint.key}" for data silo "${title}"! - \n${error.message}`, + ), ); encounteredError = true; } @@ -931,7 +931,7 @@ export async function syncDataSilos( }, { concurrency: 10, - } + }, ); progressBar.stop(); @@ -944,8 +944,8 @@ export async function syncDataSilos( dataSilos.length }" data silos and "${totalDataPoints}" datapoints in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); return { success: !encounteredError, @@ -962,13 +962,13 @@ export async function syncDataSilos( */ export async function syncDataSiloDependencies( client: GraphQLClient, - dependencyUpdates: [string, string[]][] + dependencyUpdates: [string, string[]][], ): Promise { let encounteredError = false; logger.info( colors.magenta( - `Syncing "${dependencyUpdates.length}" data silo dependencies...` - ) + `Syncing "${dependencyUpdates.length}" data silo dependencies...`, + ), ); // Batch the updates @@ -976,15 +976,15 @@ export async function syncDataSiloDependencies( await mapSeries(chunkedUpdates, async (dependencyUpdateChunk, ind) => { logger.info( colors.magenta( - `[Batch ${ind}/${dependencyUpdateChunk.length}] Updating "${dependencyUpdateChunk.length}" data silos...` - ) + `[Batch ${ind}/${dependencyUpdateChunk.length}] Updating "${dependencyUpdateChunk.length}" data silos...`, + ), ); try { await makeGraphQLRequest<{ /** Mutation result */ updateDataSilos: { /** New data silos */ - dataSilos: Pick[]; + dataSilos: Pick[]; }; }>(client, UPDATE_DATA_SILOS, { input: { @@ -992,23 +992,23 @@ export async function syncDataSiloDependencies( ([id, dependedOnDataSiloTitles]) => ({ id, dependedOnDataSiloTitles, - }) + }), ), }, }); logger.info( colors.green( `[Batch ${ind + 1}/${dependencyUpdateChunk.length}] ` + - `Synced "${dependencyUpdateChunk.length}" data silos!` - ) + `Synced "${dependencyUpdateChunk.length}" data silos!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( `[Batch ${ind + 1}/${dependencyUpdateChunk.length}] ` + - `Failed to update "${dependencyUpdateChunk.length}" silos! - ${error.message}` - ) + `Failed to update "${dependencyUpdateChunk.length}" silos! - ${error.message}`, + ), ); } }); diff --git a/src/lib/graphql/syncDataSubject.ts b/src/lib/graphql/syncDataSubject.ts index e548d91c..3f55d609 100644 --- a/src/lib/graphql/syncDataSubject.ts +++ b/src/lib/graphql/syncDataSubject.ts @@ -1,7 +1,7 @@ -import { GraphQLClient } from "graphql-request"; -import { DataSubjectInput } from "../../codecs"; -import { TOGGLE_DATA_SUBJECT, UPDATE_DATA_SUBJECT } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { GraphQLClient } from 'graphql-request'; +import { DataSubjectInput } from '../../codecs'; +import { TOGGLE_DATA_SUBJECT, UPDATE_DATA_SUBJECT } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Sync the data subjects @@ -22,7 +22,7 @@ export async function syncDataSubject( dataSubjectId: string; /** When true, skip publishing to privacy center */ skipPublish?: boolean; - } + }, ): Promise { await makeGraphQLRequest(client, UPDATE_DATA_SUBJECT, { input: { @@ -35,7 +35,7 @@ export async function syncDataSubject( }, }); - if (typeof dataSubject.active === "boolean") { + if (typeof dataSubject.active === 'boolean') { await makeGraphQLRequest(client, TOGGLE_DATA_SUBJECT, { input: { id: dataSubjectId, diff --git a/src/lib/graphql/syncEnrichers.ts b/src/lib/graphql/syncEnrichers.ts index f6e557bf..634dff1b 100644 --- a/src/lib/graphql/syncEnrichers.ts +++ b/src/lib/graphql/syncEnrichers.ts @@ -4,13 +4,13 @@ import { IsoCountrySubdivisionCode, PreflightRequestStatus, RequestAction, -} from "@transcend-io/privacy-types"; -import { GraphQLClient } from "graphql-request"; -import { EnricherInput } from "../../codecs"; -import { DataSubject } from "./fetchDataSubjects"; -import { Identifier } from "./fetchIdentifiers"; -import { CREATE_ENRICHER, ENRICHERS, UPDATE_ENRICHER } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; +import { EnricherInput } from '../../codecs'; +import { DataSubject } from './fetchDataSubjects'; +import { Identifier } from './fetchIdentifiers'; +import { CREATE_ENRICHER, ENRICHERS, UPDATE_ENRICHER } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface Enricher { /** ID of enricher */ @@ -65,7 +65,7 @@ const PAGE_SIZE = 20; */ export async function fetchAllEnrichers( client: GraphQLClient, - title?: string + title?: string, ): Promise { const enrichers: Enricher[] = []; let offset = 0; @@ -113,16 +113,16 @@ export async function syncEnricher( identifierByName: Record; /** Lookup data subject by name */ dataSubjectsByName: Record; - } + }, ): Promise { // Whether to continue looping const matches = await fetchAllEnrichers(client, enricher.title); const existingEnricher = matches.find( - ({ title }) => title === enricher.title + ({ title }) => title === enricher.title, ); // Map to data subject Ids - const dataSubjectIds = enricher["data-subjects"]?.map((subject) => { + const dataSubjectIds = enricher['data-subjects']?.map((subject) => { const existing = dataSubjectsByName[subject]; if (!existing) { throw new Error(`Failed to find a data subject with name: ${subject}`); @@ -131,9 +131,9 @@ export async function syncEnricher( }); // If enricher exists, update it, else create new - const inputIdentifier = enricher["input-identifier"]; + const inputIdentifier = enricher['input-identifier']; const actionUpdates = - enricher["privacy-actions"] || Object.values(RequestAction); + enricher['privacy-actions'] || Object.values(RequestAction); if (existingEnricher) { await makeGraphQLRequest(client, UPDATE_ENRICHER, { input: { @@ -144,19 +144,19 @@ export async function syncEnricher( testRegex: enricher.testRegex, lookerQueryTitle: enricher.lookerQueryTitle, expirationDuration: - typeof enricher.expirationDuration === "number" + typeof enricher.expirationDuration === 'number' ? enricher.expirationDuration.toString() : undefined, transitionRequestStatus: enricher.transitionRequestStatus, phoneNumbers: enricher.phoneNumbers, regionList: enricher.regionList, dataSubjectIds, - description: enricher.description || "", + description: enricher.description || '', inputIdentifier: inputIdentifier ? identifierByName[inputIdentifier].id : undefined, - identifiers: enricher["output-identifiers"].map( - (id) => identifierByName[id].id + identifiers: enricher['output-identifiers'].map( + (id) => identifierByName[id].id, ), ...(existingEnricher.type === EnricherType.Sombra ? {} @@ -173,17 +173,17 @@ export async function syncEnricher( testRegex: enricher.testRegex, lookerQueryTitle: enricher.lookerQueryTitle, expirationDuration: - typeof enricher.expirationDuration === "number" + typeof enricher.expirationDuration === 'number' ? enricher.expirationDuration.toString() : undefined, transitionRequestStatus: enricher.transitionRequestStatus, phoneNumbers: enricher.phoneNumbers, dataSubjectIds, regionList: enricher.regionList, - description: enricher.description || "", + description: enricher.description || '', inputIdentifier: identifierByName[inputIdentifier].id, - identifiers: enricher["output-identifiers"].map( - (id) => identifierByName[id].id + identifiers: enricher['output-identifiers'].map( + (id) => identifierByName[id].id, ), actions: actionUpdates, }, diff --git a/src/lib/graphql/syncIdentifier.ts b/src/lib/graphql/syncIdentifier.ts index 35edd820..dbf53249 100644 --- a/src/lib/graphql/syncIdentifier.ts +++ b/src/lib/graphql/syncIdentifier.ts @@ -1,8 +1,8 @@ -import { GraphQLClient } from "graphql-request"; -import { IdentifierInput } from "../../codecs"; -import type { DataSubject } from "./fetchDataSubjects"; -import { UPDATE_IDENTIFIER } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { GraphQLClient } from 'graphql-request'; +import { IdentifierInput } from '../../codecs'; +import type { DataSubject } from './fetchDataSubjects'; +import { UPDATE_IDENTIFIER } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Sync the consent manager @@ -26,7 +26,7 @@ export async function syncIdentifier( identifierId: string; /** When true, skip publishing to privacy center */ skipPublish?: boolean; - } + }, ): Promise { await makeGraphQLRequest(client, UPDATE_IDENTIFIER, { input: { diff --git a/src/lib/graphql/syncIntlMessages.ts b/src/lib/graphql/syncIntlMessages.ts index d879db54..4d048f37 100644 --- a/src/lib/graphql/syncIntlMessages.ts +++ b/src/lib/graphql/syncIntlMessages.ts @@ -1,11 +1,11 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk } from "lodash-es"; -import { IntlMessageInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { UPDATE_INTL_MESSAGES } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk } from 'lodash-es'; +import { IntlMessageInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { UPDATE_INTL_MESSAGES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const MAX_PAGE_SIZE = 100; @@ -17,13 +17,13 @@ const MAX_PAGE_SIZE = 100; */ export async function updateIntlMessages( client: GraphQLClient, - messageInputs: IntlMessageInput[] + messageInputs: IntlMessageInput[], ): Promise { // Batch update messages await mapSeries(chunk(messageInputs, MAX_PAGE_SIZE), async (page) => { await makeGraphQLRequest(client, UPDATE_INTL_MESSAGES, { messages: page.map((message) => ({ - ...(message.id.includes(".") ? {} : { id: message.id }), + ...(message.id.includes('.') ? {} : { id: message.id }), defaultMessage: message.defaultMessage, targetReactIntlId: message.targetReactIntlId, translations: message.translations @@ -46,30 +46,30 @@ export async function updateIntlMessages( */ export async function syncIntlMessages( client: GraphQLClient, - messages: IntlMessageInput[] + messages: IntlMessageInput[], ): Promise { let encounteredError = false; logger.info(colors.magenta(`Syncing "${messages.length}" messages...`)); // Ensure no duplicates are being uploaded const notUnique = messages.filter( - (message) => messages.filter((pol) => message.id === pol.id).length > 1 + (message) => messages.filter((pol) => message.id === pol.id).length > 1, ); if (notUnique.length > 0) { throw new Error( `Failed to upload messages as there were non-unique entries found: ${notUnique .map(({ id }) => id) - .join(",")}` + .join(',')}`, ); } try { logger.info( - colors.magenta(`Upserting "${messages.length}" new messages...`) + colors.magenta(`Upserting "${messages.length}" new messages...`), ); await updateIntlMessages(client, messages); logger.info( - colors.green(`Successfully synced ${messages.length} messages!`) + colors.green(`Successfully synced ${messages.length} messages!`), ); } catch (error) { encounteredError = true; diff --git a/src/lib/graphql/syncPartitions.ts b/src/lib/graphql/syncPartitions.ts index cee07491..88c34c44 100644 --- a/src/lib/graphql/syncPartitions.ts +++ b/src/lib/graphql/syncPartitions.ts @@ -1,12 +1,12 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { difference } from "lodash-es"; -import { PartitionInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { fetchConsentManagerId } from "./fetchConsentManagerId"; -import { CONSENT_PARTITIONS, CREATE_CONSENT_PARTITION } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { difference } from 'lodash-es'; +import { PartitionInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { fetchConsentManagerId } from './fetchConsentManagerId'; +import { CONSENT_PARTITIONS, CREATE_CONSENT_PARTITION } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const PAGE_SIZE = 50; @@ -26,7 +26,7 @@ export interface TranscendPartition { * @returns Partition list */ export async function fetchPartitions( - client: GraphQLClient + client: GraphQLClient, ): Promise { const partitions: TranscendPartition[] = []; let offset = 0; @@ -63,7 +63,7 @@ export async function fetchPartitions( */ export async function syncPartitions( client: GraphQLClient, - partitionInputs: PartitionInput[] + partitionInputs: PartitionInput[], ): Promise { // Grab the bundleId associated with this API key const airgapBundleId = await fetchConsentManagerId(client); @@ -71,7 +71,7 @@ export async function syncPartitions( const partitions = await fetchPartitions(client); const newPartitionNames = difference( partitionInputs.map(({ name }) => name), - partitions.map(({ name }) => name) + partitions.map(({ name }) => name), ); await mapSeries(newPartitionNames, async (name) => { try { @@ -82,13 +82,13 @@ export async function syncPartitions( }, }); logger.info( - colors.green(`Successfully created consent partition: ${name}!`) + colors.green(`Successfully created consent partition: ${name}!`), ); } catch (error) { logger.error( colors.red( - `Failed to create consent partition: ${name}! - ${error.message}` - ) + `Failed to create consent partition: ${name}! - ${error.message}`, + ), ); encounteredError = true; } diff --git a/src/lib/graphql/syncPolicies.ts b/src/lib/graphql/syncPolicies.ts index c60cda73..4ebb1846 100644 --- a/src/lib/graphql/syncPolicies.ts +++ b/src/lib/graphql/syncPolicies.ts @@ -1,13 +1,13 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk, keyBy } from "lodash-es"; -import { PolicyInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { fetchAllPolicies } from "./fetchAllPolicies"; -import { fetchPrivacyCenterId } from "./fetchPrivacyCenterId"; -import { UPDATE_POLICIES } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk, keyBy } from 'lodash-es'; +import { PolicyInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { fetchAllPolicies } from './fetchAllPolicies'; +import { fetchPrivacyCenterId } from './fetchPrivacyCenterId'; +import { UPDATE_POLICIES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const MAX_PAGE_SIZE = 100; @@ -19,7 +19,7 @@ const MAX_PAGE_SIZE = 100; */ export async function updatePolicies( client: GraphQLClient, - policyInputs: [PolicyInput, string | undefined][] + policyInputs: [PolicyInput, string | undefined][], ): Promise { const privacyCenterId = await fetchPrivacyCenterId(client); @@ -62,20 +62,20 @@ export async function updatePolicies( */ export async function syncPolicies( client: GraphQLClient, - policies: PolicyInput[] + policies: PolicyInput[], ): Promise { let encounteredError = false; logger.info(colors.magenta(`Syncing "${policies.length}" policies...`)); // Ensure no duplicates are being uploaded const notUnique = policies.filter( - (policy) => policies.filter((pol) => policy.title === pol.title).length > 1 + (policy) => policies.filter((pol) => policy.title === pol.title).length > 1, ); if (notUnique.length > 0) { throw new Error( `Failed to upload policies as there were non-unique entries found: ${notUnique .map(({ title }) => title) - .join(",")}` + .join(',')}`, ); } @@ -83,19 +83,19 @@ export async function syncPolicies( const existingPolicies = await fetchAllPolicies(client); const policiesById = keyBy( existingPolicies, - ({ title }) => title.defaultMessage + ({ title }) => title.defaultMessage, ); try { logger.info( - colors.magenta(`Upserting "${policies.length}" new policies...`) + colors.magenta(`Upserting "${policies.length}" new policies...`), ); await updatePolicies( client, - policies.map((policy) => [policy, policiesById[policy.title]?.id]) + policies.map((policy) => [policy, policiesById[policy.title]?.id]), ); logger.info( - colors.green(`Successfully synced ${policies.length} policies!`) + colors.green(`Successfully synced ${policies.length} policies!`), ); } catch (error) { encounteredError = true; diff --git a/src/lib/graphql/syncPrivacyCenter.ts b/src/lib/graphql/syncPrivacyCenter.ts index ac8c90ca..4c6ea353 100644 --- a/src/lib/graphql/syncPrivacyCenter.ts +++ b/src/lib/graphql/syncPrivacyCenter.ts @@ -1,10 +1,10 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { PrivacyCenterInput } from "../../codecs"; -import { logger } from "../../logger"; -import { fetchPrivacyCenterId } from "./fetchPrivacyCenterId"; -import { UPDATE_PRIVACY_CENTER } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { PrivacyCenterInput } from '../../codecs'; +import { logger } from '../../logger'; +import { fetchPrivacyCenterId } from './fetchPrivacyCenterId'; +import { UPDATE_PRIVACY_CENTER } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Sync the privacy center @@ -15,10 +15,10 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function syncPrivacyCenter( client: GraphQLClient, - privacyCenter: PrivacyCenterInput + privacyCenter: PrivacyCenterInput, ): Promise { let encounteredError = false; - logger.info(colors.magenta("Syncing privacy center...")); + logger.info(colors.magenta('Syncing privacy center...')); // Grab the privacy center ID const privacyCenterId = await fetchPrivacyCenterId(client); @@ -54,11 +54,11 @@ export async function syncPrivacyCenter( : {}), }, }); - logger.info(colors.green("Successfully synced privacy center!")); + logger.info(colors.green('Successfully synced privacy center!')); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create privacy center! - ${error.message}`) + colors.red(`Failed to create privacy center! - ${error.message}`), ); } diff --git a/src/lib/graphql/syncProcessingPurposes.ts b/src/lib/graphql/syncProcessingPurposes.ts index 52a334cc..b20526dd 100644 --- a/src/lib/graphql/syncProcessingPurposes.ts +++ b/src/lib/graphql/syncProcessingPurposes.ts @@ -1,18 +1,18 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { ProcessingPurposeInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { ProcessingPurposeInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { fetchAllProcessingPurposes, ProcessingPurposeSubCategory, -} from "./fetchAllProcessingPurposes"; +} from './fetchAllProcessingPurposes'; import { CREATE_PROCESSING_PURPOSE_SUB_CATEGORY, UPDATE_PROCESSING_PURPOSE_SUB_CATEGORIES, -} from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new processing purpose @@ -23,8 +23,8 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function createProcessingPurpose( client: GraphQLClient, - processingPurpose: ProcessingPurposeInput -): Promise> { + processingPurpose: ProcessingPurposeInput, +): Promise> { const input = { name: processingPurpose.name, purpose: processingPurpose.purpose, @@ -52,7 +52,7 @@ export async function createProcessingPurpose( */ export async function updateProcessingPurposes( client: GraphQLClient, - processingPurposeIdPairs: [ProcessingPurposeInput, string][] + processingPurposeIdPairs: [ProcessingPurposeInput, string][], ): Promise { await makeGraphQLRequest(client, UPDATE_PROCESSING_PURPOSE_SUB_CATEGORIES, { input: { @@ -62,7 +62,7 @@ export async function updateProcessingPurposes( description: processingPurpose.description, // TODO: https://transcend.height.app/T-31994 - add teams, owners attributes: processingPurpose.attributes, - }) + }), ), }, }); @@ -77,11 +77,11 @@ export async function updateProcessingPurposes( */ export async function syncProcessingPurposes( client: GraphQLClient, - inputs: ProcessingPurposeInput[] + inputs: ProcessingPurposeInput[], ): Promise { // Fetch existing logger.info( - colors.magenta(`Syncing "${inputs.length}" processing purposes...`) + colors.magenta(`Syncing "${inputs.length}" processing purposes...`), ); let encounteredError = false; @@ -92,15 +92,15 @@ export async function syncProcessingPurposes( // Look up by name const processingPurposeByName: Record< string, - Pick + Pick > = keyBy( existingProcessingPurposes, - ({ name, purpose }) => `${name}:${purpose}` + ({ name, purpose }) => `${name}:${purpose}`, ); // Create new processing purposes const newProcessingPurposes = inputs.filter( - (input) => !processingPurposeByName[`${input.name}:${input.purpose}`] + (input) => !processingPurposeByName[`${input.name}:${input.purpose}`], ); // Create new processing purposes @@ -108,22 +108,22 @@ export async function syncProcessingPurposes( try { const newProcessingPurpose = await createProcessingPurpose( client, - processingPurpose + processingPurpose, ); processingPurposeByName[ `${newProcessingPurpose.name}:${newProcessingPurpose.purpose}` ] = newProcessingPurpose; logger.info( colors.green( - `Successfully synced processing purpose "${processingPurpose.name}"!` - ) + `Successfully synced processing purpose "${processingPurpose.name}"!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync processing purpose "${processingPurpose.name}"! - ${error.message}` - ) + `Failed to sync processing purpose "${processingPurpose.name}"! - ${error.message}`, + ), ); } }); @@ -131,26 +131,26 @@ export async function syncProcessingPurposes( // Update all processing purposes try { logger.info( - colors.magenta(`Updating "${inputs.length}" processing purposes!`) + colors.magenta(`Updating "${inputs.length}" processing purposes!`), ); await updateProcessingPurposes( client, inputs.map((input) => [ input, processingPurposeByName[`${input.name}:${input.purpose}`].id, - ]) + ]), ); logger.info( colors.green( - `Successfully synced "${inputs.length}" processing purposes!` - ) + `Successfully synced "${inputs.length}" processing purposes!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" processing purposes ! - ${error.message}` - ) + `Failed to sync "${inputs.length}" processing purposes ! - ${error.message}`, + ), ); } diff --git a/src/lib/graphql/syncPromptGroups.ts b/src/lib/graphql/syncPromptGroups.ts index 659249a3..178ba8d1 100644 --- a/src/lib/graphql/syncPromptGroups.ts +++ b/src/lib/graphql/syncPromptGroups.ts @@ -1,13 +1,13 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { PromptGroupInput } from "../../codecs"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { fetchAllPromptGroups } from "./fetchPromptGroups"; -import { fetchAllPrompts } from "./fetchPrompts"; -import { CREATE_PROMPT_GROUP, UPDATE_PROMPT_GROUPS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { PromptGroupInput } from '../../codecs'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { fetchAllPromptGroups } from './fetchPromptGroups'; +import { fetchAllPrompts } from './fetchPrompts'; +import { CREATE_PROMPT_GROUP, UPDATE_PROMPT_GROUPS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface EditPromptGroupInput { /** Title of prompt group */ @@ -27,7 +27,7 @@ export interface EditPromptGroupInput { */ export async function createPromptGroup( client: GraphQLClient, - input: EditPromptGroupInput + input: EditPromptGroupInput, ): Promise { const { createPromptGroup: { promptGroup }, @@ -44,7 +44,7 @@ export async function createPromptGroup( input, }); logger.info( - colors.green(`Successfully created prompt group "${input.title}"!`) + colors.green(`Successfully created prompt group "${input.title}"!`), ); return promptGroup.id; } @@ -57,7 +57,7 @@ export async function createPromptGroup( */ export async function updatePromptGroups( client: GraphQLClient, - input: [EditPromptGroupInput, string][] + input: [EditPromptGroupInput, string][], ): Promise { await makeGraphQLRequest(client, UPDATE_PROMPT_GROUPS, { input: { @@ -68,7 +68,7 @@ export async function updatePromptGroups( }, }); logger.info( - colors.green(`Successfully updated ${input.length} prompt groups!`) + colors.green(`Successfully updated ${input.length} prompt groups!`), ); } @@ -83,18 +83,18 @@ export async function updatePromptGroups( export async function syncPromptGroups( client: GraphQLClient, promptGroups: PromptGroupInput[], - concurrency = 20 + concurrency = 20, ): Promise { let encounteredError = false; logger.info( - colors.magenta(`Syncing "${promptGroups.length}" prompt groups...`) + colors.magenta(`Syncing "${promptGroups.length}" prompt groups...`), ); // Index existing prompt groups const existing = await fetchAllPromptGroups(client); const existingPrompts = await fetchAllPrompts(client); - const promptByTitle = keyBy(existingPrompts, "title"); - const promptGroupByTitle = keyBy(existing, "title"); + const promptByTitle = keyBy(existingPrompts, 'title'); + const promptGroupByTitle = keyBy(existing, 'title'); // Determine which promptGroups are new vs existing const mapPromptGroupsToExisting = promptGroups.map((promptInput) => [ @@ -109,8 +109,8 @@ export async function syncPromptGroups( try { logger.info( colors.magenta( - `Creating "${newPromptGroups.length}" new prompt groups...` - ) + `Creating "${newPromptGroups.length}" new prompt groups...`, + ), ); await map( newPromptGroups, @@ -128,29 +128,29 @@ export async function syncPromptGroups( }, { concurrency, - } + }, ); logger.info( colors.green( - `Successfully synced ${newPromptGroups.length} prompt groups!` - ) + `Successfully synced ${newPromptGroups.length} prompt groups!`, + ), ); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create prompt groups! - ${error.message}`) + colors.red(`Failed to create prompt groups! - ${error.message}`), ); } // Update existing promptGroups const existingPromptGroups = mapPromptGroupsToExisting.filter( - (x): x is [PromptGroupInput, string] => !!x[1] + (x): x is [PromptGroupInput, string] => !!x[1], ); try { logger.info( colors.magenta( - `Updating "${existingPromptGroups.length}" prompt groups...` - ) + `Updating "${existingPromptGroups.length}" prompt groups...`, + ), ); await updatePromptGroups( client, @@ -166,17 +166,17 @@ export async function syncPromptGroups( }), }, id, - ]) + ]), ); logger.info( colors.green( - `Successfully updated "${existingPromptGroups.length}" prompt groups!` - ) + `Successfully updated "${existingPromptGroups.length}" prompt groups!`, + ), ); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create prompt groups! - ${error.message}`) + colors.red(`Failed to create prompt groups! - ${error.message}`), ); } diff --git a/src/lib/graphql/syncPromptPartials.ts b/src/lib/graphql/syncPromptPartials.ts index 2668f030..dcdbe2af 100644 --- a/src/lib/graphql/syncPromptPartials.ts +++ b/src/lib/graphql/syncPromptPartials.ts @@ -1,12 +1,12 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { PromptPartialInput } from "../../codecs"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { fetchAllPromptPartials } from "./fetchPromptPartials"; -import { CREATE_PROMPT_PARTIAL, UPDATE_PROMPT_PARTIALS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { PromptPartialInput } from '../../codecs'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { fetchAllPromptPartials } from './fetchPromptPartials'; +import { CREATE_PROMPT_PARTIAL, UPDATE_PROMPT_PARTIALS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Create a new prompt partial @@ -22,7 +22,7 @@ export async function createPromptPartial( title: string; /** Prompt content */ content: string; - } + }, ): Promise { const { createPromptPartial: { promptPartial }, @@ -39,7 +39,7 @@ export async function createPromptPartial( input, }); logger.info( - colors.green(`Successfully created prompt partial "${input.title}"!`) + colors.green(`Successfully created prompt partial "${input.title}"!`), ); return promptPartial.id; } @@ -52,7 +52,7 @@ export async function createPromptPartial( */ export async function updatePromptPartials( client: GraphQLClient, - input: [PromptPartialInput, string][] + input: [PromptPartialInput, string][], ): Promise { await makeGraphQLRequest(client, UPDATE_PROMPT_PARTIALS, { input: { @@ -63,7 +63,7 @@ export async function updatePromptPartials( }, }); logger.info( - colors.green(`Successfully updated ${input.length} prompt partials!`) + colors.green(`Successfully updated ${input.length} prompt partials!`), ); } @@ -78,16 +78,16 @@ export async function updatePromptPartials( export async function syncPromptPartials( client: GraphQLClient, promptPartials: PromptPartialInput[], - concurrency = 20 + concurrency = 20, ): Promise { let encounteredError = false; logger.info( - colors.magenta(`Syncing "${promptPartials.length}" prompt partials...`) + colors.magenta(`Syncing "${promptPartials.length}" prompt partials...`), ); // Index existing prompt partials const existing = await fetchAllPromptPartials(client); - const promptPartialByTitle = keyBy(existing, "title"); + const promptPartialByTitle = keyBy(existing, 'title'); // Determine which promptPartials are new vs existing const mapPromptPartialsToExisting = promptPartials.map((promptInput) => [ @@ -102,8 +102,8 @@ export async function syncPromptPartials( try { logger.info( colors.magenta( - `Creating "${newPromptPartials.length}" new prompt partials...` - ) + `Creating "${newPromptPartials.length}" new prompt partials...`, + ), ); await map( newPromptPartials, @@ -112,45 +112,45 @@ export async function syncPromptPartials( }, { concurrency, - } + }, ); logger.info( colors.green( - `Successfully synced ${newPromptPartials.length} prompt partials!` - ) + `Successfully synced ${newPromptPartials.length} prompt partials!`, + ), ); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create prompt partials! - ${error.message}`) + colors.red(`Failed to create prompt partials! - ${error.message}`), ); } // Update existing promptPartials const existingPromptPartials = mapPromptPartialsToExisting.filter( - (x): x is [PromptPartialInput, string] => !!x[1] + (x): x is [PromptPartialInput, string] => !!x[1], ); try { logger.info( colors.magenta( - `Updating "${existingPromptPartials.length}" prompt partials...` - ) + `Updating "${existingPromptPartials.length}" prompt partials...`, + ), ); await updatePromptPartials(client, existingPromptPartials); logger.info( colors.green( - `Successfully updated "${existingPromptPartials.length}" prompt partials!` - ) + `Successfully updated "${existingPromptPartials.length}" prompt partials!`, + ), ); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create prompt partials! - ${error.message}`) + colors.red(`Failed to create prompt partials! - ${error.message}`), ); } logger.info( - colors.green(`Synced "${promptPartials.length}" prompt partials!`) + colors.green(`Synced "${promptPartials.length}" prompt partials!`), ); // Return true upon success diff --git a/src/lib/graphql/syncPrompts.ts b/src/lib/graphql/syncPrompts.ts index 4864b8cf..ba666e5f 100644 --- a/src/lib/graphql/syncPrompts.ts +++ b/src/lib/graphql/syncPrompts.ts @@ -1,12 +1,12 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { PromptInput } from "../../codecs"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { fetchAllPrompts } from "./fetchPrompts"; -import { CREATE_PROMPT, UPDATE_PROMPTS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { PromptInput } from '../../codecs'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { fetchAllPrompts } from './fetchPrompts'; +import { CREATE_PROMPT, UPDATE_PROMPTS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Create a new prompt @@ -22,7 +22,7 @@ export async function createPrompt( title: string; /** Prompt content */ content: string; - } + }, ): Promise { const { createPrompt: { prompt }, @@ -51,7 +51,7 @@ export async function createPrompt( */ export async function updatePrompts( client: GraphQLClient, - input: [PromptInput, string][] + input: [PromptInput, string][], ): Promise { await makeGraphQLRequest(client, UPDATE_PROMPTS, { input: { @@ -75,14 +75,14 @@ export async function updatePrompts( export async function syncPrompts( client: GraphQLClient, prompts: PromptInput[], - concurrency = 20 + concurrency = 20, ): Promise { let encounteredError = false; logger.info(colors.magenta(`Syncing "${prompts.length}" prompts...`)); // Index existing prompts const existing = await fetchAllPrompts(client); - const promptByTitle = keyBy(existing, "title"); + const promptByTitle = keyBy(existing, 'title'); // Determine which prompts are new vs existing const mapPromptsToExisting = prompts.map((promptInput) => [ @@ -96,7 +96,7 @@ export async function syncPrompts( .map(([promptInput]) => promptInput as PromptInput); try { logger.info( - colors.magenta(`Creating "${newPrompts.length}" new prompts...`) + colors.magenta(`Creating "${newPrompts.length}" new prompts...`), ); await map( newPrompts, @@ -105,10 +105,10 @@ export async function syncPrompts( }, { concurrency, - } + }, ); logger.info( - colors.green(`Successfully synced ${newPrompts.length} prompts!`) + colors.green(`Successfully synced ${newPrompts.length} prompts!`), ); } catch (error) { encounteredError = true; @@ -117,15 +117,15 @@ export async function syncPrompts( // Update existing prompts const existingPrompts = mapPromptsToExisting.filter( - (x): x is [PromptInput, string] => !!x[1] + (x): x is [PromptInput, string] => !!x[1], ); try { logger.info( - colors.magenta(`Updating "${existingPrompts.length}" prompts...`) + colors.magenta(`Updating "${existingPrompts.length}" prompts...`), ); await updatePrompts(client, existingPrompts); logger.info( - colors.green(`Successfully updated "${existingPrompts.length}" prompts!`) + colors.green(`Successfully updated "${existingPrompts.length}" prompts!`), ); } catch (error) { encounteredError = true; diff --git a/src/lib/graphql/syncRepositories.ts b/src/lib/graphql/syncRepositories.ts index 61a8aac9..b0d08aed 100644 --- a/src/lib/graphql/syncRepositories.ts +++ b/src/lib/graphql/syncRepositories.ts @@ -1,12 +1,12 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk, keyBy } from "lodash-es"; -import { RepositoryInput } from "../../codecs"; -import { logger } from "../../logger"; -import { map, mapSeries } from "../bluebird-replace"; -import { fetchAllRepositories, Repository } from "./fetchAllRepositories"; -import { CREATE_REPOSITORY, UPDATE_REPOSITORIES } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk, keyBy } from 'lodash-es'; +import { RepositoryInput } from '../../codecs'; +import { logger } from '../../logger'; +import { map, mapSeries } from '../bluebird-replace'; +import { fetchAllRepositories, Repository } from './fetchAllRepositories'; +import { CREATE_REPOSITORY, UPDATE_REPOSITORIES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const CHUNK_SIZE = 100; @@ -34,7 +34,7 @@ export async function createRepository( teamIds?: string[]; /** Team names */ teamNames?: string[]; - } + }, ): Promise { const { createRepository: { repository }, @@ -77,7 +77,7 @@ export async function updateRepositories( teamIds?: string[]; /** Team names */ teamNames?: string[]; - }[] + }[], ): Promise { const { updateRepositories: { repositories }, @@ -93,7 +93,7 @@ export async function updateRepositories( }, }); logger.info( - colors.green(`Successfully updated ${inputs.length} repositories!`) + colors.green(`Successfully updated ${inputs.length} repositories!`), ); return repositories; } @@ -109,7 +109,7 @@ export async function updateRepositories( export async function syncRepositories( client: GraphQLClient, repositories: RepositoryInput[], - concurrency = 20 + concurrency = 20, ): Promise<{ /** The repositories that were upserted */ repositories: Repository[]; @@ -121,7 +121,7 @@ export async function syncRepositories( // Index existing repositories const existing = await fetchAllRepositories(client); - const repositoryByName = keyBy(existing, "name"); + const repositoryByName = keyBy(existing, 'name'); // Determine which repositories are new vs existing const mapRepositoriesToExisting = repositories.map((repoInput) => [ @@ -135,7 +135,9 @@ export async function syncRepositories( .map(([repoInput]) => repoInput as RepositoryInput); try { logger.info( - colors.magenta(`Creating "${newRepositories.length}" new repositories...`) + colors.magenta( + `Creating "${newRepositories.length}" new repositories...`, + ), ); await map( newRepositories, @@ -145,27 +147,27 @@ export async function syncRepositories( }, { concurrency, - } + }, ); logger.info( colors.green( - `Successfully synced ${newRepositories.length} repositories!` - ) + `Successfully synced ${newRepositories.length} repositories!`, + ), ); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create repositories! - ${error.message}`) + colors.red(`Failed to create repositories! - ${error.message}`), ); } // Update existing repositories const existingRepositories = mapRepositoriesToExisting.filter( - (x): x is [RepositoryInput, string] => !!x[1] + (x): x is [RepositoryInput, string] => !!x[1], ); const chunks = chunk(existingRepositories, CHUNK_SIZE); logger.info( - colors.magenta(`Updating "${existingRepositories.length}" repositories...`) + colors.magenta(`Updating "${existingRepositories.length}" repositories...`), ); await mapSeries(chunks, async (chunk) => { @@ -175,18 +177,18 @@ export async function syncRepositories( chunk.map(([input, id]) => ({ ...input, id, - })) + })), ); repos.push(...updatedRepos); logger.info( colors.green( - `Successfully updated "${existingRepositories.length}" repositories!` - ) + `Successfully updated "${existingRepositories.length}" repositories!`, + ), ); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to update repositories! - ${error.message}`) + colors.red(`Failed to update repositories! - ${error.message}`), ); } diff --git a/src/lib/graphql/syncSoftwareDevelopmentKits.ts b/src/lib/graphql/syncSoftwareDevelopmentKits.ts index 227fede5..62059d8c 100644 --- a/src/lib/graphql/syncSoftwareDevelopmentKits.ts +++ b/src/lib/graphql/syncSoftwareDevelopmentKits.ts @@ -1,19 +1,19 @@ -import { CodePackageType } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk, keyBy } from "lodash-es"; -import { SoftwareDevelopmentKitInput } from "../../codecs"; -import { logger } from "../../logger"; -import { map, mapSeries } from "../bluebird-replace"; +import { CodePackageType } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk, keyBy } from 'lodash-es'; +import { SoftwareDevelopmentKitInput } from '../../codecs'; +import { logger } from '../../logger'; +import { map, mapSeries } from '../bluebird-replace'; import { fetchAllSoftwareDevelopmentKits, SoftwareDevelopmentKit, -} from "./fetchAllSoftwareDevelopmentKits"; +} from './fetchAllSoftwareDevelopmentKits'; import { CREATE_SOFTWARE_DEVELOPMENT_KIT, UPDATE_SOFTWARE_DEVELOPMENT_KITS, -} from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const CHUNK_SIZE = 100; @@ -51,7 +51,7 @@ export async function createSoftwareDevelopmentKit( teamIds?: string[]; /** Team names */ teamNames?: string[]; - } + }, ): Promise { const { createSoftwareDevelopmentKit: { softwareDevelopmentKit }, @@ -66,8 +66,8 @@ export async function createSoftwareDevelopmentKit( }); logger.info( colors.green( - `Successfully created software development kit "${input.name}"!` - ) + `Successfully created software development kit "${input.name}"!`, + ), ); return softwareDevelopmentKit; } @@ -106,7 +106,7 @@ export async function updateSoftwareDevelopmentKits( teamIds?: string[]; /** Team names */ teamNames?: string[]; - }[] + }[], ): Promise { const { updateSoftwareDevelopmentKits: { softwareDevelopmentKits }, @@ -123,8 +123,8 @@ export async function updateSoftwareDevelopmentKits( }); logger.info( colors.green( - `Successfully updated ${inputs.length} software development kits!` - ) + `Successfully updated ${inputs.length} software development kits!`, + ), ); return softwareDevelopmentKits; } @@ -140,7 +140,7 @@ export async function updateSoftwareDevelopmentKits( export async function syncSoftwareDevelopmentKits( client: GraphQLClient, softwareDevelopmentKits: SoftwareDevelopmentKitInput[], - concurrency = 20 + concurrency = 20, ): Promise<{ /** The SDKs that were upserted */ softwareDevelopmentKits: SoftwareDevelopmentKit[]; @@ -149,13 +149,13 @@ export async function syncSoftwareDevelopmentKits( }> { let encounteredError = false; const sdks: SoftwareDevelopmentKit[] = []; - logger.info(colors.magenta("Syncing software development kits...")); + logger.info(colors.magenta('Syncing software development kits...')); // Index existing software development kits const existing = await fetchAllSoftwareDevelopmentKits(client); const softwareDevelopmentKitByTitle = keyBy( existing, - ({ name, codePackageType }) => JSON.stringify({ name, codePackageType }) + ({ name, codePackageType }) => JSON.stringify({ name, codePackageType }), ); // Determine which software development kits are new vs existing @@ -168,7 +168,7 @@ export async function syncSoftwareDevelopmentKits( codePackageType: sdkInput.codePackageType, }) ]?.id, - ] + ], ); // Create the new software development kits @@ -178,8 +178,8 @@ export async function syncSoftwareDevelopmentKits( try { logger.info( colors.magenta( - `Creating "${newSoftwareDevelopmentKits.length}" new software development kits...` - ) + `Creating "${newSoftwareDevelopmentKits.length}" new software development kits...`, + ), ); await map( newSoftwareDevelopmentKits, @@ -189,32 +189,32 @@ export async function syncSoftwareDevelopmentKits( }, { concurrency, - } + }, ); logger.info( colors.green( - `Successfully synced ${newSoftwareDevelopmentKits.length} software development kits!` - ) + `Successfully synced ${newSoftwareDevelopmentKits.length} software development kits!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to create software development kits! - ${error.message}` - ) + `Failed to create software development kits! - ${error.message}`, + ), ); } // Update existing software development kits const existingSoftwareDevelopmentKits = mapSoftwareDevelopmentKitsToExisting.filter( - (x): x is [SoftwareDevelopmentKitInput, string] => !!x[1] + (x): x is [SoftwareDevelopmentKitInput, string] => !!x[1], ); const chunks = chunk(existingSoftwareDevelopmentKits, CHUNK_SIZE); logger.info( colors.magenta( - `Updating "${existingSoftwareDevelopmentKits.length}" software development kits...` - ) + `Updating "${existingSoftwareDevelopmentKits.length}" software development kits...`, + ), ); await mapSeries(chunks, async (chunk) => { @@ -225,27 +225,27 @@ export async function syncSoftwareDevelopmentKits( chunk.map(([{ codePackageType, ...input }, id]) => ({ ...input, id, - })) + })), ); sdks.push(...updatedSdks); logger.info( colors.green( - `Successfully updated "${existingSoftwareDevelopmentKits.length}" software development kits!` - ) + `Successfully updated "${existingSoftwareDevelopmentKits.length}" software development kits!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to update software development kits! - ${error.message}` - ) + `Failed to update software development kits! - ${error.message}`, + ), ); } logger.info( colors.green( - `Synced "${softwareDevelopmentKits.length}" software development kits!` - ) + `Synced "${softwareDevelopmentKits.length}" software development kits!`, + ), ); }); diff --git a/src/lib/graphql/syncTeams.ts b/src/lib/graphql/syncTeams.ts index f22199bc..7681bee0 100644 --- a/src/lib/graphql/syncTeams.ts +++ b/src/lib/graphql/syncTeams.ts @@ -1,12 +1,12 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { TeamInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { fetchAllTeams, Team } from "./fetchAllTeams"; -import { CREATE_TEAM, UPDATE_TEAM } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { TeamInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { fetchAllTeams, Team } from './fetchAllTeams'; +import { CREATE_TEAM, UPDATE_TEAM } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new team @@ -17,14 +17,14 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function createTeam( client: GraphQLClient, - team: TeamInput -): Promise> { + team: TeamInput, +): Promise> { const input = { name: team.name, description: team.description, - ssoTitle: team["sso-title"], - ssoDepartment: team["sso-department"], - ssoGroup: team["sso-group"], + ssoTitle: team['sso-title'], + ssoDepartment: team['sso-department'], + ssoGroup: team['sso-group'], scopes: team.scopes, userEmails: team.users, }; @@ -52,8 +52,8 @@ export async function createTeam( export async function updateTeam( client: GraphQLClient, input: TeamInput, - teamId: string -): Promise> { + teamId: string, +): Promise> { const { updateTeam } = await makeGraphQLRequest<{ /** Update team mutation */ updateTeam: { @@ -65,9 +65,9 @@ export async function updateTeam( id: teamId, name: input.name, description: input.description, - ssoTitle: input["sso-title"], - ssoDepartment: input["sso-department"], - ssoGroup: input["sso-group"], + ssoTitle: input['sso-title'], + ssoDepartment: input['sso-department'], + ssoGroup: input['sso-group'], scopes: input.scopes, userEmails: input.users, }, @@ -84,7 +84,7 @@ export async function updateTeam( */ export async function syncTeams( client: GraphQLClient, - inputs: TeamInput[] + inputs: TeamInput[], ): Promise { // Fetch existing logger.info(colors.magenta(`Syncing "${inputs.length}" teams...`)); @@ -95,9 +95,9 @@ export async function syncTeams( const existingTeams = await fetchAllTeams(client); // Look up by name - const teamsByName: Record> = keyBy( + const teamsByName: Record> = keyBy( existingTeams, - "name" + 'name', ); // Create new teams @@ -113,7 +113,7 @@ export async function syncTeams( } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to sync team "${team.name}"! - ${error.message}`) + colors.red(`Failed to sync team "${team.name}"! - ${error.message}`), ); } }); @@ -124,14 +124,14 @@ export async function syncTeams( const newTeam = await updateTeam( client, input, - teamsByName[input.name].id + teamsByName[input.name].id, ); teamsByName[newTeam.name] = newTeam; logger.info(colors.green(`Successfully updated team "${input.name}"!`)); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to sync team "${input.name}"! - ${error.message}`) + colors.red(`Failed to sync team "${input.name}"! - ${error.message}`), ); } }); diff --git a/src/lib/graphql/syncVendors.ts b/src/lib/graphql/syncVendors.ts index cd9e60c4..9b25814e 100644 --- a/src/lib/graphql/syncVendors.ts +++ b/src/lib/graphql/syncVendors.ts @@ -1,12 +1,12 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { VendorInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { fetchAllVendors, Vendor } from "./fetchAllVendors"; -import { CREATE_VENDOR, UPDATE_VENDORS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { VendorInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { fetchAllVendors, Vendor } from './fetchAllVendors'; +import { CREATE_VENDOR, UPDATE_VENDORS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new vendor @@ -17,8 +17,8 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function createVendor( client: GraphQLClient, - vendor: VendorInput -): Promise> { + vendor: VendorInput, +): Promise> { const input = { title: vendor.title, description: vendor.description, @@ -52,7 +52,7 @@ export async function createVendor( */ export async function updateVendors( client: GraphQLClient, - vendorIdParis: [VendorInput, string][] + vendorIdParis: [VendorInput, string][], ): Promise { await makeGraphQLRequest(client, UPDATE_VENDORS, { input: { @@ -83,7 +83,7 @@ export async function updateVendors( */ export async function syncVendors( client: GraphQLClient, - inputs: VendorInput[] + inputs: VendorInput[], ): Promise { // Fetch existing logger.info(colors.magenta(`Syncing "${inputs.length}" vendors...`)); @@ -94,9 +94,9 @@ export async function syncVendors( const existingVendors = await fetchAllVendors(client); // Look up by title - const vendorByTitle: Record> = keyBy( + const vendorByTitle: Record> = keyBy( existingVendors, - "title" + 'title', ); // Create new vendors @@ -108,14 +108,14 @@ export async function syncVendors( const newVendor = await createVendor(client, vendor); vendorByTitle[newVendor.title] = newVendor; logger.info( - colors.green(`Successfully synced vendor "${vendor.title}"!`) + colors.green(`Successfully synced vendor "${vendor.title}"!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync vendor "${vendor.title}"! - ${error.message}` - ) + `Failed to sync vendor "${vendor.title}"! - ${error.message}`, + ), ); } }); @@ -125,17 +125,17 @@ export async function syncVendors( logger.info(colors.magenta(`Updating "${inputs.length}" vendors!`)); await updateVendors( client, - inputs.map((input) => [input, vendorByTitle[input.title].id]) + inputs.map((input) => [input, vendorByTitle[input.title].id]), ); logger.info( - colors.green(`Successfully synced "${inputs.length}" vendors!`) + colors.green(`Successfully synced "${inputs.length}" vendors!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" vendors ! - ${error.message}` - ) + `Failed to sync "${inputs.length}" vendors ! - ${error.message}`, + ), ); } diff --git a/src/lib/helpers/inquirer.ts b/src/lib/helpers/inquirer.ts index 911d90ca..151b8c37 100644 --- a/src/lib/helpers/inquirer.ts +++ b/src/lib/helpers/inquirer.ts @@ -1,7 +1,7 @@ -import { ObjByString } from "@transcend-io/type-utils"; -import inquirer from "inquirer"; -import autoCompletePrompt from "inquirer-autocomplete-prompt"; -import { fuzzySearch } from "../requests"; +import { ObjByString } from '@transcend-io/type-utils'; +import inquirer from 'inquirer'; +import autoCompletePrompt from 'inquirer-autocomplete-prompt'; +import { fuzzySearch } from '../requests'; /** * Inquirer confirm text @@ -20,9 +20,9 @@ export async function inquirerConfirmBoolean({ response: boolean; }>([ { - name: "response", + name: 'response', message, - type: "confirm", + type: 'confirm', }, ]); return response; @@ -45,9 +45,9 @@ export async function inquirerConfirmText({ response: string; }>([ { - name: "response", + name: 'response', message, - type: "text", + type: 'text', validate: (x) => x.trim().length > 0, }, ]); @@ -72,19 +72,19 @@ export async function inquirerAutoComplete({ /** Values to select */ values: string[]; }): Promise { - inquirer.registerPrompt("autocomplete", autoCompletePrompt); + inquirer.registerPrompt('autocomplete', autoCompletePrompt); const { response } = await inquirer.prompt<{ /** confirmation */ response: string; }>([ { - name: "response", + name: 'response', message, - type: "autocomplete", + type: 'autocomplete', default: defaultValue, source: (answersSoFar: ObjByString, input: string) => input - ? values.filter((x) => typeof x === "string" && fuzzySearch(input, x)) + ? values.filter((x) => typeof x === 'string' && fuzzySearch(input, x)) : values, }, ]); diff --git a/src/lib/helpers/parseVariablesFromString.ts b/src/lib/helpers/parseVariablesFromString.ts index e58df764..a187a7f0 100644 --- a/src/lib/helpers/parseVariablesFromString.ts +++ b/src/lib/helpers/parseVariablesFromString.ts @@ -5,16 +5,16 @@ * @returns Variables as object */ export function parseVariablesFromString( - variables: string + variables: string, ): Record { // Parse out the variables - const splitVariables = variables.split(",").filter((x) => !!x); + const splitVariables = variables.split(',').filter((x) => !!x); const variables_: Record = {}; for (const variable of splitVariables) { - const [k, v] = variable.split(":"); + const [k, v] = variable.split(':'); if (!k || !v) { throw new Error( - `Invalid variable: ${variable}. Expected format: key:value` + `Invalid variable: ${variable}. Expected format: key:value`, ); } variables_[k] = v; diff --git a/src/lib/manual-enrichment/enrichPrivacyRequest.ts b/src/lib/manual-enrichment/enrichPrivacyRequest.ts index e8999237..91b1475a 100644 --- a/src/lib/manual-enrichment/enrichPrivacyRequest.ts +++ b/src/lib/manual-enrichment/enrichPrivacyRequest.ts @@ -1,12 +1,12 @@ -import colors from "colors"; -import type { Got } from "got"; -import * as t from "io-ts"; -import { uniq } from "lodash-es"; -import { logger } from "../../logger"; -import { splitCsvToList } from "../requests/splitCsvToList"; +import colors from 'colors'; +import type { Got } from 'got'; +import * as t from 'io-ts'; +import { uniq } from 'lodash-es'; +import { logger } from '../../logger'; +import { splitCsvToList } from '../requests/splitCsvToList'; const ADMIN_URL = - "https://app.transcend.io/privacy-requests/incoming-requests/"; + 'https://app.transcend.io/privacy-requests/incoming-requests/'; /** * Minimal set required to mark as completed */ @@ -28,12 +28,12 @@ export async function enrichPrivacyRequest( sombra: Got, { id: rawId, ...rest }: EnrichPrivacyRequest, enricherId: string, - index?: number + index?: number, ): Promise { if (!rawId) { // error const message = `Request ID must be provided to enricher request.${ - index ? ` Found error in row: ${index}` : "" + index ? ` Found error in row: ${index}` : '' }`; logger.error(colors.red(message)); throw new Error(message); @@ -50,7 +50,7 @@ export async function enrichPrivacyRequest( ? accumulator : Object.assign(accumulator, { [key]: uniq(splitCsvToList(value)).map((value_) => ({ - value: key === "email" ? value_.toLowerCase() : value_, + value: key === 'email' ? value_.toLowerCase() : value_, })), }); }, {}); @@ -58,10 +58,10 @@ export async function enrichPrivacyRequest( // Make the GraphQL request try { await sombra - .post("v1/enrich-identifiers", { + .post('v1/enrich-identifiers', { headers: { - "x-transcend-request-id": id, - "x-transcend-enricher-id": enricherId, + 'x-transcend-request-id': id, + 'x-transcend-enricher-id': enricherId, }, json: { enrichedIdentifiers, @@ -70,19 +70,19 @@ export async function enrichPrivacyRequest( .json(); logger.error( - colors.green(`Successfully enriched request: ${ADMIN_URL}${id}`) + colors.green(`Successfully enriched request: ${ADMIN_URL}${id}`), ); return true; } catch (error) { // skip if already enriched if ( - typeof error.response.body === "string" && - error.response.body.includes("Cannot update a resolved RequestEnricher") + typeof error.response.body === 'string' && + error.response.body.includes('Cannot update a resolved RequestEnricher') ) { logger.warn( colors.magenta( - `Skipped enrichment for request: ${ADMIN_URL}${id}, request is no longer in the enriching phase.` - ) + `Skipped enrichment for request: ${ADMIN_URL}${id}, request is no longer in the enriching phase.`, + ), ); return false; } @@ -90,8 +90,8 @@ export async function enrichPrivacyRequest( // error logger.error( colors.red( - `Failed to enricher identifiers for request with id: ${ADMIN_URL}${id} - ${error.message} - ${error.response.body}` - ) + `Failed to enricher identifiers for request with id: ${ADMIN_URL}${id} - ${error.message} - ${error.response.body}`, + ), ); throw error; } diff --git a/src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts b/src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts index 053e9184..cf022bec 100644 --- a/src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts +++ b/src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts @@ -1,10 +1,10 @@ -import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { groupBy, uniq } from "lodash-es"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { writeCsv } from "../cron/writeCsv"; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { groupBy, uniq } from 'lodash-es'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { writeCsv } from '../cron/writeCsv'; import { buildTranscendGraphQLClient, createSombraGotInstance, @@ -14,7 +14,7 @@ import { PrivacyRequest, RequestEnricher, RequestIdentifier, -} from "../graphql"; +} from '../graphql'; export interface PrivacyRequestWithIdentifiers extends PrivacyRequest { /** Request Enrichers */ @@ -57,9 +57,9 @@ export async function pullManualEnrichmentIdentifiersToCsv({ logger.info( colors.magenta( `Pulling manual enrichment requests, filtered for actions: ${requestActions.join( - "," - )}` - ) + ',', + )}`, + ), ); // Pull all privacy requests @@ -82,7 +82,7 @@ export async function pullManualEnrichmentIdentifiersToCsv({ // Check if manual enrichment exists for that request const hasManualEnrichment = requestEnrichers.filter( - ({ status }) => status === "ACTION_REQUIRED" + ({ status }) => status === 'ACTION_REQUIRED', ); // Save request to queue @@ -92,7 +92,7 @@ export async function pullManualEnrichmentIdentifiersToCsv({ sombra, { requestId: request.id, - } + }, ); savedRequests.push({ ...request, @@ -103,7 +103,7 @@ export async function pullManualEnrichmentIdentifiersToCsv({ }, { concurrency, - } + }, ); const data = savedRequests.map( @@ -116,17 +116,17 @@ export async function pullManualEnrichmentIdentifiersToCsv({ ...request, // flatten identifiers ...Object.fromEntries( - Object.entries(groupBy(requestIdentifiers, "name")).map( - ([key, values]) => [key, values.map(({ value }) => value).join(",")] - ) + Object.entries(groupBy(requestIdentifiers, 'name')).map( + ([key, values]) => [key, values.map(({ value }) => value).join(',')], + ), ), // flatten attributes ...Object.fromEntries( - Object.entries(groupBy(attributeValues, "attributeKey.name")).map( - ([key, values]) => [key, values.map(({ name }) => name).join(",")] - ) + Object.entries(groupBy(attributeValues, 'attributeKey.name')).map( + ([key, values]) => [key, values.map(({ name }) => name).join(',')], + ), ), - }) + }), ); // Write out to CSV @@ -135,8 +135,8 @@ export async function pullManualEnrichmentIdentifiersToCsv({ logger.info( colors.green( - `Successfully wrote ${savedRequests.length} requests to file "${file}"` - ) + `Successfully wrote ${savedRequests.length} requests to file "${file}"`, + ), ); return savedRequests; diff --git a/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts b/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts index 67720d5f..58e5f10c 100644 --- a/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts +++ b/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts @@ -1,18 +1,18 @@ -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, createSombraGotInstance, makeGraphQLRequest, UPDATE_PRIVACY_REQUEST, -} from "../graphql"; -import { readCsv } from "../requests"; +} from '../graphql'; +import { readCsv } from '../requests'; import { enrichPrivacyRequest, EnrichPrivacyRequest, -} from "./enrichPrivacyRequest"; +} from './enrichPrivacyRequest'; /** * Push a CSV of enriched requests back into Transcend @@ -54,7 +54,7 @@ export async function pushManualEnrichmentIdentifiersFromCsv({ // Notify Transcend logger.info( - colors.magenta(`Enriching "${activeResults.length}" privacy requests.`) + colors.magenta(`Enriching "${activeResults.length}" privacy requests.`), ); let successCount = 0; @@ -75,7 +75,7 @@ export async function pushManualEnrichmentIdentifiersFromCsv({ }); logger.info( - colors.magenta(`Mark request as silent mode - ${request.id}`) + colors.magenta(`Mark request as silent mode - ${request.id}`), ); } @@ -83,7 +83,7 @@ export async function pushManualEnrichmentIdentifiersFromCsv({ sombra, request, enricherId, - index + index, ); if (result) { successCount += 1; @@ -94,13 +94,13 @@ export async function pushManualEnrichmentIdentifiersFromCsv({ errorCount += 1; } }, - { concurrency } + { concurrency }, ); logger.info( colors.green( - `Successfully notified Transcend! \n Success count: ${successCount}.` - ) + `Successfully notified Transcend! \n Success count: ${successCount}.`, + ), ); if (skippedCount > 0) { diff --git a/src/lib/mergeTranscendInputs.ts b/src/lib/mergeTranscendInputs.ts index 71c2c8ee..f05ea2a2 100644 --- a/src/lib/mergeTranscendInputs.ts +++ b/src/lib/mergeTranscendInputs.ts @@ -1,5 +1,5 @@ -import { getEntries } from "@transcend-io/type-utils"; -import { TranscendInput } from "../codecs"; +import { getEntries } from '@transcend-io/type-utils'; +import { TranscendInput } from '../codecs'; /** * Combine a set of TranscendInput yaml files into a single yaml diff --git a/src/lib/oneTrust/helpers/convertToEmptyStrings.ts b/src/lib/oneTrust/helpers/convertToEmptyStrings.ts index 5cbff11a..067e325a 100644 --- a/src/lib/oneTrust/helpers/convertToEmptyStrings.ts +++ b/src/lib/oneTrust/helpers/convertToEmptyStrings.ts @@ -36,7 +36,7 @@ export function convertToEmptyStrings(input: T): any { // Handle null/undefined if (input === null || input === undefined) { - return ""; + return ''; } // Handle arrays @@ -45,15 +45,15 @@ export function convertToEmptyStrings(input: T): any { } // Handle objects - if (typeof input === "object") { + if (typeof input === 'object') { return Object.fromEntries( Object.entries(input).map>(([key, value]) => [ key, convertToEmptyStrings(value), - ]) + ]), ); } // Handle primitives - return ""; + return ''; } diff --git a/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts b/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts index 9e7c38f4..5dde3ee2 100644 --- a/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts +++ b/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts @@ -1,11 +1,11 @@ -import colors from "colors"; -import yargs from "yargs-parser"; +import colors from 'colors'; +import yargs from 'yargs-parser'; import { OneTrustFileFormat, OneTrustPullResource, OneTrustPullSource, -} from "../../../enums"; -import { logger } from "../../../logger"; +} from '../../../enums'; +import { logger } from '../../../logger'; const VALID_RESOURCES = Object.values(OneTrustPullResource); @@ -48,21 +48,21 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { source, } = yargs(process.argv.slice(2), { string: [ - "file", - "hostname", - "oneTrustAuth", - "resource", - "dryRun", - "transcendAuth", - "transcendUrl", - "source", + 'file', + 'hostname', + 'oneTrustAuth', + 'resource', + 'dryRun', + 'transcendAuth', + 'transcendUrl', + 'source', ], - boolean: ["debug", "dryRun"], + boolean: ['debug', 'dryRun'], default: { resource: OneTrustPullResource.Assessments, debug: false, dryRun: false, - transcendUrl: "https://api.transcend.io", + transcendUrl: 'https://api.transcend.io', source: OneTrustPullSource.OneTrust, }, }); @@ -71,16 +71,16 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { if (!dryRun && !transcendAuth) { logger.error( colors.red( - 'Must specify a "transcendAuth" parameter to sync resources to Transcend. e.g. --transcendAuth=${TRANSCEND_API_KEY}' - ) + 'Must specify a "transcendAuth" parameter to sync resources to Transcend. e.g. --transcendAuth=${TRANSCEND_API_KEY}', + ), ); return process.exit(1); } if (!dryRun && !transcendUrl) { logger.error( colors.red( - 'Must specify a "transcendUrl" parameter to sync resources to Transcend. e.g. --transcendUrl=https://api.transcend.io' - ) + 'Must specify a "transcendUrl" parameter to sync resources to Transcend. e.g. --transcendUrl=https://api.transcend.io', + ), ); return process.exit(1); } @@ -89,19 +89,19 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { if (dryRun && !file) { logger.error( colors.red( - 'Must set a "file" parameter when "dryRun" is "true". e.g. --file=./oneTrustAssessments.json' - ) + 'Must set a "file" parameter when "dryRun" is "true". e.g. --file=./oneTrustAssessments.json', + ), ); return process.exit(1); } if (file) { - const splitFile = file.split("."); + const splitFile = file.split('.'); if (splitFile.length < 2) { logger.error( colors.red( - 'The "file" parameter has an invalid format. Expected a path with extensions. e.g. --file=./pathToFile.json.' - ) + 'The "file" parameter has an invalid format. Expected a path with extensions. e.g. --file=./pathToFile.json.', + ), ); return process.exit(1); } @@ -110,8 +110,8 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { colors.red( `Expected the format of the "file" parameters '${file}' to be '${ OneTrustFileFormat.Json - }', but got '${splitFile.at(-1)}'.` - ) + }', but got '${splitFile.at(-1)}'.`, + ), ); return process.exit(1); } @@ -123,8 +123,8 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { if (!hostname) { logger.error( colors.red( - 'Missing required parameter "hostname". e.g. --hostname=customer.my.onetrust.com' - ) + 'Missing required parameter "hostname". e.g. --hostname=customer.my.onetrust.com', + ), ); return process.exit(1); } @@ -132,8 +132,8 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { if (!oneTrustAuth) { logger.error( colors.red( - 'Missing required parameter "oneTrustAuth". e.g. --oneTrustAuth=$ONE_TRUST_AUTH_TOKEN' - ) + 'Missing required parameter "oneTrustAuth". e.g. --oneTrustAuth=$ONE_TRUST_AUTH_TOKEN', + ), ); return process.exit(1); } @@ -142,8 +142,8 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { if (!file) { logger.error( colors.red( - 'Must specify a "file" parameter to read the OneTrust assessments from. e.g. --source=./oneTrustAssessments.json' - ) + 'Must specify a "file" parameter to read the OneTrust assessments from. e.g. --source=./oneTrustAssessments.json', + ), ); return process.exit(1); } @@ -152,9 +152,9 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { if (dryRun) { logger.error( colors.red( - "Cannot read and write to a file simultaneously." + - ` Emit the "source" parameter or set it to ${OneTrustPullSource.OneTrust} if "dryRun" is enabled.` - ) + 'Cannot read and write to a file simultaneously.' + + ` Emit the "source" parameter or set it to ${OneTrustPullSource.OneTrust} if "dryRun" is enabled.`, + ), ); return process.exit(1); } @@ -164,9 +164,9 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { logger.error( colors.red( `Received invalid resource value: "${resource}". Allowed: ${VALID_RESOURCES.join( - "," - )}` - ) + ',', + )}`, + ), ); return process.exit(1); } diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToDisk.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToDisk.ts index 885e33d5..ee2a9126 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToDisk.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToDisk.ts @@ -1,8 +1,8 @@ -import fs from "node:fs"; -import { OneTrustEnrichedAssessment } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { logger } from "../../../logger"; -import { oneTrustAssessmentToJson } from "./oneTrustAssessmentToJson"; +import fs from 'node:fs'; +import { OneTrustEnrichedAssessment } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { logger } from '../../../logger'; +import { oneTrustAssessmentToJson } from './oneTrustAssessmentToJson'; /** * Write the assessment to disk at the specified file path. @@ -29,8 +29,8 @@ export const syncOneTrustAssessmentToDisk = ({ colors.magenta( `Writing enriched assessment ${ index + 1 - } of ${total} to file "${file}"...` - ) + } of ${total} to file "${file}"...`, + ), ); if (index === 0) { @@ -41,7 +41,7 @@ export const syncOneTrustAssessmentToDisk = ({ index, total, wrap: false, - }) + }), ); } else { fs.appendFileSync( @@ -51,7 +51,7 @@ export const syncOneTrustAssessmentToDisk = ({ index, total, wrap: false, - }) + }), ); } }; diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts index 80127ee8..e18c1d26 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts @@ -1,13 +1,13 @@ -import { OneTrustEnrichedAssessment } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { ImportOnetrustAssessmentsInput } from "../../../codecs"; -import { logger } from "../../../logger"; +import { OneTrustEnrichedAssessment } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { ImportOnetrustAssessmentsInput } from '../../../codecs'; +import { logger } from '../../../logger'; import { IMPORT_ONE_TRUST_ASSESSMENT_FORMS, makeGraphQLRequest, -} from "../../graphql"; -import { oneTrustAssessmentToJson } from "./oneTrustAssessmentToJson"; +} from '../../graphql'; +import { oneTrustAssessmentToJson } from './oneTrustAssessmentToJson'; export interface AssessmentForm { /** ID of Assessment Form */ @@ -40,9 +40,9 @@ export const syncOneTrustAssessmentToTranscend = async ({ logger.info( colors.magenta( `Writing enriched assessment ${index + 1} ${ - total ? `of ${total} ` : " " - }to Transcend...` - ) + total ? `of ${total} ` : ' ' + }to Transcend...`, + ), ); // convert the OneTrust assessment object into a json record @@ -71,10 +71,10 @@ export const syncOneTrustAssessmentToTranscend = async ({ logger.error( colors.red( `Failed to sync assessment ${index + 1} ${ - total ? `of ${total} ` : " " + total ? `of ${total} ` : ' ' }to Transcend.\n` + - `\tAssessment Title: ${assessment.name}. Template Title: ${assessment.template.name}\n` - ) + `\tAssessment Title: ${assessment.name}. Template Title: ${assessment.template.name}\n`, + ), ); } }; diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts index 3d9b69a4..e2d3ae23 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts @@ -1,11 +1,11 @@ -import { createReadStream } from "node:fs"; -import { OneTrustEnrichedAssessment } from "@transcend-io/privacy-types"; -import { decodeCodec } from "@transcend-io/type-utils"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import JSONStream from "JSONStream"; -import { logger } from "../../../logger"; -import { syncOneTrustAssessmentToTranscend } from "./syncOneTrustAssessmentToTranscend"; +import { createReadStream } from 'node:fs'; +import { OneTrustEnrichedAssessment } from '@transcend-io/privacy-types'; +import { decodeCodec } from '@transcend-io/type-utils'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import JSONStream from 'JSONStream'; +import { logger } from '../../../logger'; +import { syncOneTrustAssessmentToTranscend } from './syncOneTrustAssessmentToTranscend'; /** * Reads assessments from a file and syncs them to Transcend. @@ -26,12 +26,12 @@ export const syncOneTrustAssessmentsFromFile = ({ return new Promise((resolve, reject) => { // Create a readable stream from the file const fileStream = createReadStream(file, { - encoding: "utf-8", + encoding: 'utf-8', highWaterMark: 64 * 1024, // 64KB chunks }); // Create a JSONStream parser to parse the array of OneTrust assessments from the file - const parser = JSONStream.parse("*"); // '*' matches each element in the root array + const parser = JSONStream.parse('*'); // '*' matches each element in the root array let index = 0; @@ -39,7 +39,7 @@ export const syncOneTrustAssessmentsFromFile = ({ fileStream.pipe(parser); // Handle each parsed assessment object - parser.on("data", async (assessment) => { + parser.on('data', async (assessment) => { try { // Pause the stream while processing to avoid overwhelming memory parser.pause(); @@ -47,7 +47,7 @@ export const syncOneTrustAssessmentsFromFile = ({ // Decode and validate the assessment const parsedAssessment = decodeCodec( OneTrustEnrichedAssessment, - assessment + assessment, ); // Sync the assessment to transcend @@ -65,29 +65,29 @@ export const syncOneTrustAssessmentsFromFile = ({ // if failed to parse a line, report error and continue logger.error( colors.red( - `Failed to parse the assessment ${index} from file '${file}': ${error.message}.` - ) + `Failed to parse the assessment ${index} from file '${file}': ${error.message}.`, + ), ); } }); // Handle completion - parser.on("end", () => { + parser.on('end', () => { logger.info(`Finished processing ${index} assessments from file ${file}`); resolve(); }); // Handle stream or parsing errors - parser.on("error", (error) => { + parser.on('error', (error) => { logger.error( - colors.red(`Error parsing file '${file}': ${error.message}`) + colors.red(`Error parsing file '${file}': ${error.message}`), ); reject(error); }); - fileStream.on("error", (error) => { + fileStream.on('error', (error) => { logger.error( - colors.red(`Error reading file '${file}': ${error.message}`) + colors.red(`Error reading file '${file}': ${error.message}`), ); reject(error); }); diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts index d33d0897..c06eb645 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts @@ -4,22 +4,22 @@ import { OneTrustEnrichedAssessment, OneTrustGetRiskResponse, OneTrustGetUserResponse, -} from "@transcend-io/privacy-types"; -import colors from "colors"; -import type { Got } from "got"; -import { GraphQLClient } from "graphql-request"; -import { uniq } from "lodash-es"; -import { logger } from "../../../logger"; -import { map, mapSeries } from "../../bluebird-replace"; +} from '@transcend-io/privacy-types'; +import colors from 'colors'; +import type { Got } from 'got'; +import { GraphQLClient } from 'graphql-request'; +import { uniq } from 'lodash-es'; +import { logger } from '../../../logger'; +import { map, mapSeries } from '../../bluebird-replace'; import { getListOfOneTrustAssessments, getOneTrustAssessment, getOneTrustRisk, getOneTrustUser, -} from "../endpoints"; -import { enrichOneTrustAssessment } from "./enrichOneTrustAssessment"; -import { syncOneTrustAssessmentToDisk } from "./syncOneTrustAssessmentToDisk"; -import { syncOneTrustAssessmentToTranscend } from "./syncOneTrustAssessmentToTranscend"; +} from '../endpoints'; +import { enrichOneTrustAssessment } from './enrichOneTrustAssessment'; +import { syncOneTrustAssessmentToDisk } from './syncOneTrustAssessmentToDisk'; +import { syncOneTrustAssessmentToTranscend } from './syncOneTrustAssessmentToTranscend'; export interface AssessmentForm { /** ID of Assessment Form */ @@ -49,7 +49,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ file?: string; }): Promise => { // fetch the list of all assessments in the OneTrust organization - logger.info("Getting list of all assessments from OneTrust..."); + logger.info('Getting list of all assessments from OneTrust...'); const assessments = await getListOfOneTrustAssessments({ oneTrust }); // a cache of OneTrust users so we avoid requesting already fetched users @@ -62,7 +62,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ length: Math.ceil(assessments.length / BATCH_SIZE), }, (_, index) => - assessments.slice(index * BATCH_SIZE, (index + 1) * BATCH_SIZE) + assessments.slice(index * BATCH_SIZE, (index + 1) * BATCH_SIZE), ); // process each batch and sync the batch right away so it's garbage collected and we don't run out of memory @@ -75,7 +75,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ async (assessment, index) => { const assessmentNumber = BATCH_SIZE * batch + index + 1; logger.info( - `[assessment ${assessmentNumber} of ${assessments.length}]: fetching details...` + `[assessment ${assessmentNumber} of ${assessments.length}]: fetching details...`, ); const { templateName, assessmentId } = assessment; const assessmentDetails = await getOneTrustAssessment({ @@ -87,7 +87,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ let creator = oneTrustCachedUsers[creatorId]; if (!creator) { logger.info( - `[assessment ${assessmentNumber} of ${assessments.length}]: fetching creator...` + `[assessment ${assessmentNumber} of ${assessments.length}]: fetching creator...`, ); try { creator = await getOneTrustUser({ @@ -99,8 +99,8 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ logger.warn( colors.yellow( `[assessment ${assessmentNumber} of ${assessments.length}]: failed to fetch form creator.` + - `\tcreatorId: ${creatorId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}` - ) + `\tcreatorId: ${creatorId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}`, + ), ); } } @@ -110,7 +110,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ let approversDetails: OneTrustGetUserResponse[][] = []; if (approvers.length > 0) { logger.info( - `[assessment ${assessmentNumber} of ${assessments.length}]: fetching approvers...` + `[assessment ${assessmentNumber} of ${assessments.length}]: fetching approvers...`, ); approversDetails = await map( approvers.map(({ id }) => id), @@ -126,13 +126,13 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ logger.warn( colors.yellow( `[assessment ${assessmentNumber} of ${assessments.length}]: failed to fetch a form approver.` + - `\tapproverId: ${userId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}` - ) + `\tapproverId: ${userId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}`, + ), ); return []; } }, - { concurrency: 5 } + { concurrency: 5 }, ); } @@ -140,12 +140,12 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ const { respondents } = assessmentDetails; // if a user is an internal respondents, their 'name' field can't be an email. const internalRespondents = respondents.filter( - (r) => !r.name.includes("@") + (r) => !r.name.includes('@'), ); let respondentsDetails: OneTrustGetUserResponse[][] = []; if (internalRespondents.length > 0) { logger.info( - `[assessment ${assessmentNumber} of ${assessments.length}]: fetching respondents...` + `[assessment ${assessmentNumber} of ${assessments.length}]: fetching respondents...`, ); respondentsDetails = await map( internalRespondents.map(({ id }) => id), @@ -161,13 +161,13 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ logger.warn( colors.yellow( `[assessment ${assessmentNumber} of ${assessments.length}]: failed to fetch a respondent.` + - `\trespondentId: ${userId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}` - ) + `\trespondentId: ${userId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}`, + ), ); return []; } }, - { concurrency: 5 } + { concurrency: 5 }, ); } @@ -176,20 +176,20 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ const riskIds = uniq( assessmentDetails.sections.flatMap((s: OneTrustAssessmentSection) => s.questions.flatMap((q: OneTrustAssessmentQuestion) => - (q.risks ?? []).flatMap((r) => r.riskId) - ) - ) + (q.risks ?? []).flatMap((r) => r.riskId), + ), + ), ); if (riskIds.length > 0) { logger.info( - `[assessment ${assessmentNumber} of ${assessments.length}]: fetching risks...` + `[assessment ${assessmentNumber} of ${assessments.length}]: fetching risks...`, ); riskDetails = await map( riskIds, (riskId) => getOneTrustRisk({ oneTrust, riskId: riskId }), { concurrency: 5, - } + }, ); } @@ -205,7 +205,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ batchEnrichedAssessments.push(enrichedAssessment); }, - { concurrency: BATCH_SIZE } + { concurrency: BATCH_SIZE }, ); // sync assessments in series to avoid concurrency bugs @@ -232,7 +232,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ index: globalIndex, }); } - } + }, ); }); }; diff --git a/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts b/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts index 91024c53..a5de0dd5 100644 --- a/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts +++ b/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts @@ -1,51 +1,51 @@ -import { describe, expect, it } from "vitest"; -import { convertToEmptyStrings } from "../convertToEmptyStrings"; +import { describe, expect, it } from 'vitest'; +import { convertToEmptyStrings } from '../convertToEmptyStrings'; -describe("buildDefaultCodecWrapper", () => { - it("should correctly build a default codec for null", () => { +describe('buildDefaultCodecWrapper', () => { + it('should correctly build a default codec for null', () => { const result = convertToEmptyStrings(null); - expect(result).to.equal(""); + expect(result).to.equal(''); }); - it("should correctly build a default codec for number", () => { + it('should correctly build a default codec for number', () => { const result = convertToEmptyStrings(0); - expect(result).to.equal(""); + expect(result).to.equal(''); }); - it("should correctly build a default codec for boolean", () => { + it('should correctly build a default codec for boolean', () => { const result = convertToEmptyStrings(false); - expect(result).to.equal(""); + expect(result).to.equal(''); }); - it("should correctly build a default codec for undefined", () => { + it('should correctly build a default codec for undefined', () => { const result = convertToEmptyStrings(); - expect(result).to.equal(""); + expect(result).to.equal(''); }); - it("should correctly build a default codec for string", () => { - const result = convertToEmptyStrings("1"); - expect(result).to.equal(""); + it('should correctly build a default codec for string', () => { + const result = convertToEmptyStrings('1'); + expect(result).to.equal(''); }); - it("should correctly build a default codec for an object", () => { - const result = convertToEmptyStrings({ name: "joe" }); - expect(result).to.deep.equal({ name: "" }); + it('should correctly build a default codec for an object', () => { + const result = convertToEmptyStrings({ name: 'joe' }); + expect(result).to.deep.equal({ name: '' }); }); - it("should correctly build a default codec for an array of primitive types", () => { - const result = convertToEmptyStrings(["name", 0, false]); - expect(result).to.deep.equal(["", "", ""]); + it('should correctly build a default codec for an array of primitive types', () => { + const result = convertToEmptyStrings(['name', 0, false]); + expect(result).to.deep.equal(['', '', '']); }); - it("should correctly build a default codec for an array of object", () => { + it('should correctly build a default codec for an array of object', () => { const result = convertToEmptyStrings([ - { name: "john", age: 52 }, - { name: "jane", age: 15, isAdult: true }, + { name: 'john', age: 52 }, + { name: 'jane', age: 15, isAdult: true }, ]); // should default to the array with object if the union contains an array of objects expect(result).to.deep.equal([ - { name: "", age: "" }, - { name: "", age: "", isAdult: "" }, + { name: '', age: '' }, + { name: '', age: '', isAdult: '' }, ]); }); }); diff --git a/src/lib/preference-management/checkIfPendingPreferenceUpdatesAreNoOp.ts b/src/lib/preference-management/checkIfPendingPreferenceUpdatesAreNoOp.ts index 3a822364..e9e464a3 100644 --- a/src/lib/preference-management/checkIfPendingPreferenceUpdatesAreNoOp.ts +++ b/src/lib/preference-management/checkIfPendingPreferenceUpdatesAreNoOp.ts @@ -2,8 +2,8 @@ import { PreferenceQueryResponseItem, PreferenceStorePurposeResponse, PreferenceTopicType, -} from "@transcend-io/privacy-types"; -import { PreferenceTopic } from "../graphql"; +} from '@transcend-io/privacy-types'; +import { PreferenceTopic } from '../graphql'; /** * Check if the pending set of updates are exactly the same as the current consent record. @@ -21,7 +21,7 @@ export function checkIfPendingPreferenceUpdatesAreNoOp({ /** The pending updates */ pendingUpdates: Record< string, - Omit + Omit >; /** The preference topic configurations */ preferenceTopics: PreferenceTopic[]; @@ -31,7 +31,7 @@ export function checkIfPendingPreferenceUpdatesAreNoOp({ ([purposeName, { preferences = [], enabled }]) => { // Ensure the purpose exists const currentPurpose = currentConsentRecord.purposes.find( - (existingPurpose) => existingPurpose.purpose === purposeName + (existingPurpose) => existingPurpose.purpose === purposeName, ); // Ensure purpose.enabled is in sync @@ -53,7 +53,7 @@ export function checkIfPendingPreferenceUpdatesAreNoOp({ // Determine type of preference topic const preferenceTopic = preferenceTopics.find( - (x) => x.slug === topic && x.purpose.trackingType === purposeName + (x) => x.slug === topic && x.purpose.trackingType === purposeName, ); if (!preferenceTopic) { throw new Error(`Could not find preference topic for ${topic}`); @@ -80,18 +80,18 @@ export function checkIfPendingPreferenceUpdatesAreNoOp({ return ( sortedCurrentValues.length === sortedNewValues.length && sortedCurrentValues.every( - (x, index) => x === sortedNewValues[index] + (x, index) => x === sortedNewValues[index], ) ); } default: { throw new Error( - `Unknown preference topic type: ${preferenceTopic.type}` + `Unknown preference topic type: ${preferenceTopic.type}`, ); } } - }) + }), ); - } + }, ); } diff --git a/src/lib/preference-management/checkIfPendingPreferenceUpdatesCauseConflict.ts b/src/lib/preference-management/checkIfPendingPreferenceUpdatesCauseConflict.ts index 06bd39b7..276d170f 100644 --- a/src/lib/preference-management/checkIfPendingPreferenceUpdatesCauseConflict.ts +++ b/src/lib/preference-management/checkIfPendingPreferenceUpdatesCauseConflict.ts @@ -2,8 +2,8 @@ import { PreferenceQueryResponseItem, PreferenceStorePurposeResponse, PreferenceTopicType, -} from "@transcend-io/privacy-types"; -import { PreferenceTopic } from "../graphql"; +} from '@transcend-io/privacy-types'; +import { PreferenceTopic } from '../graphql'; /** * Check if the pending set of updates will result in a change of @@ -22,7 +22,7 @@ export function checkIfPendingPreferenceUpdatesCauseConflict({ /** The pending updates */ pendingUpdates: Record< string, - Omit + Omit >; /** The preference topic configurations */ preferenceTopics: PreferenceTopic[]; @@ -32,7 +32,7 @@ export function checkIfPendingPreferenceUpdatesCauseConflict({ ([purposeName, { preferences = [], enabled }]) => { // Ensure the purpose exists const currentPurpose = currentConsentRecord.purposes.find( - (existingPurpose) => existingPurpose.purpose === purposeName + (existingPurpose) => existingPurpose.purpose === purposeName, ); // If no purpose exists, then it is not a conflict @@ -49,7 +49,7 @@ export function checkIfPendingPreferenceUpdatesCauseConflict({ return !!preferences.find(({ topic, choice }) => { // find matching topic const currentPreference = (currentPurpose.preferences || []).find( - (existingPreference) => existingPreference.topic === topic + (existingPreference) => existingPreference.topic === topic, ); // if no topic exists, no conflict @@ -59,7 +59,7 @@ export function checkIfPendingPreferenceUpdatesCauseConflict({ // Determine type of preference topic const preferenceTopic = preferenceTopics.find( - (x) => x.slug === topic && x.purpose.trackingType === purposeName + (x) => x.slug === topic && x.purpose.trackingType === purposeName, ); if (!preferenceTopic) { throw new Error(`Could not find preference topic for ${topic}`); @@ -84,17 +84,17 @@ export function checkIfPendingPreferenceUpdatesCauseConflict({ return ( sortedCurrentValues.length !== sortedNewValues.length || !sortedCurrentValues.every( - (x, index) => x === sortedNewValues[index] + (x, index) => x === sortedNewValues[index], ) ); } default: { throw new Error( - `Unknown preference topic type: ${preferenceTopic.type}` + `Unknown preference topic type: ${preferenceTopic.type}`, ); } } }); - } + }, ); } diff --git a/src/lib/preference-management/getPreferenceUpdatesFromRow.ts b/src/lib/preference-management/getPreferenceUpdatesFromRow.ts index 7d9a5801..da4b3ac3 100644 --- a/src/lib/preference-management/getPreferenceUpdatesFromRow.ts +++ b/src/lib/preference-management/getPreferenceUpdatesFromRow.ts @@ -1,11 +1,11 @@ import { PreferenceStorePurposeResponse, PreferenceTopicType, -} from "@transcend-io/privacy-types"; -import { apply } from "@transcend-io/type-utils"; -import { PreferenceTopic } from "../graphql"; -import { splitCsvToList } from "../requests"; -import { PurposeRowMapping } from "./codecs"; +} from '@transcend-io/privacy-types'; +import { apply } from '@transcend-io/type-utils'; +import { PreferenceTopic } from '../graphql'; +import { splitCsvToList } from '../requests'; +import { PurposeRowMapping } from './codecs'; /** * Parse an arbitrary object to the Transcend PUT /v1/preference update shape @@ -42,7 +42,7 @@ export function getPreferenceUpdatesFromRow({ purposeSlugs: string[]; /** The preference topics */ preferenceTopics: PreferenceTopic[]; -}): Record> { +}): Record> { // Create a result object to store the parsed preferences const result: Record> = {}; @@ -54,14 +54,14 @@ export function getPreferenceUpdatesFromRow({ // Ensure the purpose is valid if (!purposeSlugs.includes(purpose)) { throw new Error( - `Invalid purpose slug: ${purpose}, expected: ${purposeSlugs.join(", ")}` + `Invalid purpose slug: ${purpose}, expected: ${purposeSlugs.join(', ')}`, ); } // CHeck if parsing a preference or just the top level purpose if (preference) { const preferenceTopic = preferenceTopics.find( - (x) => x.slug === preference && x.purpose.trackingType === purpose + (x) => x.slug === preference && x.purpose.trackingType === purpose, ); if (!preferenceTopic) { const allowedTopics = preferenceTopics @@ -70,8 +70,8 @@ export function getPreferenceUpdatesFromRow({ throw new Error( `Invalid preference slug: ${preference} for purpose: ${purpose}. ` + `Allowed preference slugs for purpose are: ${allowedTopics.join( - "," - )}` + ',', + )}`, ); } @@ -89,14 +89,14 @@ export function getPreferenceUpdatesFromRow({ const rawValue = row[columnName]; const rawMapping = valueMapping[rawValue]; const trimmedMapping = - typeof rawMapping === "string" ? rawMapping.trim() || null : null; + typeof rawMapping === 'string' ? rawMapping.trim() || null : null; // handle each type of preference switch (preferenceTopic.type) { case PreferenceTopicType.Boolean: { - if (typeof rawMapping !== "boolean") { + if (typeof rawMapping !== 'boolean') { throw new TypeError( - `Invalid value for boolean preference: ${preference}, expected boolean, got: ${rawValue}` + `Invalid value for boolean preference: ${preference}, expected boolean, got: ${rawValue}`, ); } result[purpose].preferences.push({ @@ -108,9 +108,9 @@ export function getPreferenceUpdatesFromRow({ break; } case PreferenceTopicType.Select: { - if (typeof rawMapping !== "string" && rawMapping !== null) { + if (typeof rawMapping !== 'string' && rawMapping !== null) { throw new Error( - `Invalid value for select preference: ${preference}, expected string or null, got: ${rawValue}` + `Invalid value for select preference: ${preference}, expected string or null, got: ${rawValue}`, ); } @@ -124,7 +124,7 @@ export function getPreferenceUpdatesFromRow({ `Invalid value for select preference: ${preference}, expected one of: ` + `${preferenceTopic.preferenceOptionValues .map(({ slug }) => slug) - .join(", ")}, got: ${rawValue}` + .join(', ')}, got: ${rawValue}`, ); } @@ -138,9 +138,9 @@ export function getPreferenceUpdatesFromRow({ break; } case PreferenceTopicType.MultiSelect: { - if (typeof rawValue !== "string") { + if (typeof rawValue !== 'string') { throw new TypeError( - `Invalid value for multi select preference: ${preference}, expected string, got: ${rawValue}` + `Invalid value for multi select preference: ${preference}, expected string, got: ${rawValue}`, ); } // Update preferences @@ -150,12 +150,12 @@ export function getPreferenceUpdatesFromRow({ selectValues: splitCsvToList(rawValue) .map((value) => { const result = valueMapping[value]; - if (typeof result !== "string") { + if (typeof result !== 'string') { throw new TypeError( `Invalid value for multi select preference: ${preference}, ` + `expected one of: ${preferenceTopic.preferenceOptionValues .map(({ slug }) => slug) - .join(", ")}, got: ${value}` + .join(', ')}, got: ${value}`, ); } return result; @@ -182,9 +182,9 @@ export function getPreferenceUpdatesFromRow({ // Ensure that enabled is provided return apply(result, (x, purposeName) => { - if (typeof x.enabled !== "boolean") { + if (typeof x.enabled !== 'boolean') { throw new TypeError( - `No mapping provided for purpose.enabled=true/false value: ${purposeName}` + `No mapping provided for purpose.enabled=true/false value: ${purposeName}`, ); } return { diff --git a/src/lib/preference-management/getPreferencesForIdentifiers.ts b/src/lib/preference-management/getPreferencesForIdentifiers.ts index 3f523344..5fd84c25 100644 --- a/src/lib/preference-management/getPreferencesForIdentifiers.ts +++ b/src/lib/preference-management/getPreferencesForIdentifiers.ts @@ -1,12 +1,12 @@ -import { PreferenceQueryResponseItem } from "@transcend-io/privacy-types"; -import { decodeCodec } from "@transcend-io/type-utils"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import type { Got } from "got"; -import * as t from "io-ts"; -import { chunk } from "lodash-es"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { PreferenceQueryResponseItem } from '@transcend-io/privacy-types'; +import { decodeCodec } from '@transcend-io/type-utils'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import type { Got } from 'got'; +import * as t from 'io-ts'; +import { chunk } from 'lodash-es'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; const PreferenceRecordsQueryResponse = t.intersection([ t.type({ @@ -19,10 +19,10 @@ const PreferenceRecordsQueryResponse = t.intersection([ ]); const MSGS = [ - "ENOTFOUND", - "ETIMEDOUT", - "504 Gateway Time-out", - "Task timed out after", + 'ENOTFOUND', + 'ETIMEDOUT', + '504 Gateway Time-out', + 'Task timed out after', ]; /** @@ -48,7 +48,7 @@ export async function getPreferencesForIdentifiers( partitionKey: string; /** Whether to skip logging */ skipLogging?: boolean; - } + }, ): Promise { const results: PreferenceQueryResponseItem[] = []; const groupedIdentifiers = chunk(identifiers, 100); @@ -57,7 +57,7 @@ export async function getPreferencesForIdentifiers( const t0 = Date.now(); const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); if (!skipLogging) { progressBar.start(identifiers.length, 0); @@ -90,28 +90,28 @@ export async function getPreferencesForIdentifiers( break; // Exit loop if successful } catch (error) { attempts += 1; - const message = error?.response?.body || error?.message || ""; + const message = error?.response?.body || error?.message || ''; if ( attempts >= maxAttempts || !MSGS.some((errorMessage) => message.includes(errorMessage)) ) { throw new Error( - `Received an error from server after ${attempts} attempts: ${message}` + `Received an error from server after ${attempts} attempts: ${message}`, ); } logger.warn( colors.yellow( `[RETRYING FAILED REQUEST - Attempt ${attempts}] ` + - `Failed to fetch ${group.length} user preferences from partition ${partitionKey}: ${message}` - ) + `Failed to fetch ${group.length} user preferences from partition ${partitionKey}: ${message}`, + ), ); } } }, { concurrency: 40, - } + }, ); progressBar.stop(); @@ -121,7 +121,7 @@ export async function getPreferencesForIdentifiers( if (!skipLogging) { // Log completion time logger.info( - colors.green(`Completed download in "${totalTime / 1000}" seconds.`) + colors.green(`Completed download in "${totalTime / 1000}" seconds.`), ); } diff --git a/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts b/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts index bd7240f1..c0845f58 100644 --- a/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts +++ b/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts @@ -1,12 +1,12 @@ -import { PreferenceTopicType } from "@transcend-io/privacy-types"; -import colors from "colors"; -import inquirer from "inquirer"; -import { difference, uniq } from "lodash-es"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { PreferenceTopic } from "../graphql"; -import { splitCsvToList } from "../requests"; -import { FileMetadataState } from "./codecs"; +import { PreferenceTopicType } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import inquirer from 'inquirer'; +import { difference, uniq } from 'lodash-es'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { PreferenceTopic } from '../graphql'; +import { splitCsvToList } from '../requests'; +import { FileMetadataState } from './codecs'; /** * Parse out the purpose.enabled and preference values from a CSV file @@ -30,7 +30,7 @@ export async function parsePreferenceAndPurposeValuesFromCsv( preferenceTopics: PreferenceTopic[]; /** Force workflow triggers */ forceTriggerWorkflows: boolean; - } + }, ): Promise { // Determine columns to map const columnNames = uniq(preferences.flatMap((x) => Object.keys(x))); @@ -44,7 +44,7 @@ export async function parsePreferenceAndPurposeValuesFromCsv( if (forceTriggerWorkflows) { return currentState; } - throw new Error("No other columns to process"); + throw new Error('No other columns to process'); } // The purpose and preferences to map to @@ -63,8 +63,8 @@ export async function parsePreferenceAndPurposeValuesFromCsv( if (purposeMapping) { logger.info( colors.magenta( - `Column "${col}" is associated with purpose "${purposeMapping.purpose}"` - ) + `Column "${col}" is associated with purpose "${purposeMapping.purpose}"`, + ), ); } else { const { purposeName } = await inquirer.prompt<{ @@ -72,14 +72,14 @@ export async function parsePreferenceAndPurposeValuesFromCsv( purposeName: string; }>([ { - name: "purposeName", + name: 'purposeName', message: `Choose the purpose that column ${col} is associated with`, - type: "list", + type: 'list', default: purposeNames.find((x) => x.startsWith(purposeSlugs[0])), choices: purposeNames, }, ]); - const [purposeSlug, preferenceSlug] = purposeName.split("->"); + const [purposeSlug, preferenceSlug] = purposeName.split('->'); purposeMapping = { purpose: purposeSlug, preference: preferenceSlug || null, @@ -92,8 +92,8 @@ export async function parsePreferenceAndPurposeValuesFromCsv( if (purposeMapping.valueMapping[value] !== undefined) { logger.info( colors.magenta( - `Value "${value}" is associated with purpose value "${purposeMapping.valueMapping[value]}"` - ) + `Value "${value}" is associated with purpose value "${purposeMapping.valueMapping[value]}"`, + ), ); return; } @@ -104,10 +104,10 @@ export async function parsePreferenceAndPurposeValuesFromCsv( purposeValue: boolean; }>([ { - name: "purposeValue", + name: 'purposeValue', message: `Choose the purpose value for value "${value}" associated with purpose "${purposeMapping.purpose}"`, - type: "confirm", - default: value !== "false", + type: 'confirm', + default: value !== 'false', }, ]); purposeMapping.valueMapping[value] = purposeValue; @@ -116,18 +116,18 @@ export async function parsePreferenceAndPurposeValuesFromCsv( // if preference is not null, this column is for a specific preference if (purposeMapping.preference !== null) { const preferenceTopic = preferenceTopics.find( - (x) => x.slug === purposeMapping.preference + (x) => x.slug === purposeMapping.preference, ); if (!preferenceTopic) { logger.error( colors.red( - `Preference topic "${purposeMapping.preference}" not found` - ) + `Preference topic "${purposeMapping.preference}" not found`, + ), ); return; } const preferenceOptions = preferenceTopic.preferenceOptionValues.map( - ({ slug }) => slug + ({ slug }) => slug, ); if (preferenceTopic.type === PreferenceTopicType.Boolean) { @@ -136,10 +136,10 @@ export async function parsePreferenceAndPurposeValuesFromCsv( preferenceValue: boolean; }>([ { - name: "preferenceValue", + name: 'preferenceValue', message: `Choose the preference value for "${preferenceTopic.slug}" value "${value}" associated with purpose "${purposeMapping.purpose}"`, - type: "confirm", - default: value !== "false", + type: 'confirm', + default: value !== 'false', }, ]); purposeMapping.valueMapping[value] = preferenceValue; @@ -152,10 +152,10 @@ export async function parsePreferenceAndPurposeValuesFromCsv( preferenceValue: boolean; }>([ { - name: "preferenceValue", + name: 'preferenceValue', message: `Choose the preference value for "${preferenceTopic.slug}" value "${value}" associated with purpose "${purposeMapping.purpose}"`, - type: "list", + type: 'list', choices: preferenceOptions, default: preferenceOptions.find((x) => x === value), }, @@ -177,10 +177,10 @@ export async function parsePreferenceAndPurposeValuesFromCsv( preferenceValue: boolean; }>([ { - name: "preferenceValue", + name: 'preferenceValue', message: `Choose the preference value for "${preferenceTopic.slug}" value "${parsedValue}" associated with purpose "${purposeMapping.purpose}"`, - type: "list", + type: 'list', choices: preferenceOptions, default: preferenceOptions.find((x) => x === parsedValue), }, @@ -191,7 +191,7 @@ export async function parsePreferenceAndPurposeValuesFromCsv( } throw new Error( - `Unknown preference topic type: ${preferenceTopic.type}` + `Unknown preference topic type: ${preferenceTopic.type}`, ); } }); diff --git a/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts b/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts index 3487eedc..df9357ce 100644 --- a/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts +++ b/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts @@ -1,9 +1,9 @@ -import colors from "colors"; -import inquirer from "inquirer"; -import { difference, groupBy, uniq } from "lodash-es"; -import { logger } from "../../logger"; -import { inquirerConfirmBoolean } from "../helpers"; -import { FileMetadataState } from "./codecs"; +import colors from 'colors'; +import inquirer from 'inquirer'; +import { difference, groupBy, uniq } from 'lodash-es'; +import { logger } from '../../logger'; +import { inquirerConfirmBoolean } from '../helpers'; +import { FileMetadataState } from './codecs'; /** * Parse identifiers from a CSV list of preferences @@ -17,7 +17,7 @@ import { FileMetadataState } from "./codecs"; */ export async function parsePreferenceIdentifiersFromCsv( preferences: Record[], - currentState: FileMetadataState + currentState: FileMetadataState, ): Promise<{ /** The updated state */ currentState: FileMetadataState; @@ -40,13 +40,13 @@ export async function parsePreferenceIdentifiersFromCsv( identifierName: string; }>([ { - name: "identifierName", + name: 'identifierName', message: - "Choose the column that will be used as the identifier to upload consent preferences by", - type: "list", + 'Choose the column that will be used as the identifier to upload consent preferences by', + type: 'list', default: remainingColumnsForIdentifier.find((col) => - col.toLowerCase().includes("email") + col.toLowerCase().includes('email'), ) || remainingColumnsForIdentifier[0], choices: remainingColumnsForIdentifier, }, @@ -54,7 +54,9 @@ export async function parsePreferenceIdentifiersFromCsv( currentState.identifierColumn = identifierName; } logger.info( - colors.magenta(`Using identifier column "${currentState.identifierColumn}"`) + colors.magenta( + `Using identifier column "${currentState.identifierColumn}"`, + ), ); // Validate that the identifier column is present for all rows and unique @@ -66,13 +68,13 @@ export async function parsePreferenceIdentifiersFromCsv( const message = `The identifier column "${ currentState.identifierColumn }" is missing a value for the following rows: ${identifierColumnsMissing.join( - ", " + ', ', )}`; logger.warn(colors.yellow(message)); // Ask user if they would like to skip rows missing an identifier const skip = await inquirerConfirmBoolean({ - message: "Would you like to skip rows missing an identifier?", + message: 'Would you like to skip rows missing an identifier?', }); if (!skip) { throw new Error(message); @@ -81,24 +83,24 @@ export async function parsePreferenceIdentifiersFromCsv( // Filter out rows missing an identifier const previous = preferences.length; preferences = preferences.filter( - (pref) => pref[currentState.identifierColumn!] + (pref) => pref[currentState.identifierColumn!], ); logger.info( colors.yellow( - `Skipped ${previous - preferences.length} rows missing an identifier` - ) + `Skipped ${previous - preferences.length} rows missing an identifier`, + ), ); } logger.info( colors.magenta( - `The identifier column "${currentState.identifierColumn}" is present for all rows` - ) + `The identifier column "${currentState.identifierColumn}" is present for all rows`, + ), ); // Validate that all identifiers are unique const rowsByUserId = groupBy(preferences, currentState.identifierColumn); const duplicateIdentifiers = Object.entries(rowsByUserId).filter( - ([, rows]) => rows.length > 1 + ([, rows]) => rows.length > 1, ); if (duplicateIdentifiers.length > 0) { const message = `The identifier column "${ @@ -106,13 +108,13 @@ export async function parsePreferenceIdentifiersFromCsv( }" has duplicate values for the following rows: ${duplicateIdentifiers .slice(0, 10) .map(([userId, rows]) => `${userId} (${rows.length})`) - .join("\n")}`; + .join('\n')}`; logger.warn(colors.yellow(message)); // Ask user if they would like to take the most recent update // for each duplicate identifier const skip = await inquirerConfirmBoolean({ - message: "Would you like to automatically take the latest update?", + message: 'Would you like to automatically take the latest update?', }); if (!skip) { throw new Error(message); @@ -122,7 +124,7 @@ export async function parsePreferenceIdentifiersFromCsv( const sorted = rows.sort( (a, b) => new Date(b[currentState.timestampColum!]).getTime() - - new Date(a[currentState.timestampColum!]).getTime() + new Date(a[currentState.timestampColum!]).getTime(), ); return sorted[0]; }) diff --git a/src/lib/preference-management/parsePreferenceManagementCsv.ts b/src/lib/preference-management/parsePreferenceManagementCsv.ts index 13415db4..c0d629f5 100644 --- a/src/lib/preference-management/parsePreferenceManagementCsv.ts +++ b/src/lib/preference-management/parsePreferenceManagementCsv.ts @@ -1,19 +1,19 @@ -import { PersistedState } from "@transcend-io/persisted-state"; -import colors from "colors"; -import type { Got } from "got"; -import * as t from "io-ts"; -import { keyBy } from "lodash-es"; -import { logger } from "../../logger"; -import { PreferenceTopic } from "../graphql"; -import { readCsv } from "../requests"; -import { checkIfPendingPreferenceUpdatesAreNoOp } from "./checkIfPendingPreferenceUpdatesAreNoOp"; -import { checkIfPendingPreferenceUpdatesCauseConflict } from "./checkIfPendingPreferenceUpdatesCauseConflict"; -import { FileMetadataState, PreferenceState } from "./codecs"; -import { getPreferencesForIdentifiers } from "./getPreferencesForIdentifiers"; -import { getPreferenceUpdatesFromRow } from "./getPreferenceUpdatesFromRow"; -import { parsePreferenceAndPurposeValuesFromCsv } from "./parsePreferenceAndPurposeValuesFromCsv"; -import { parsePreferenceIdentifiersFromCsv } from "./parsePreferenceIdentifiersFromCsv"; -import { parsePreferenceTimestampsFromCsv } from "./parsePreferenceTimestampsFromCsv"; +import { PersistedState } from '@transcend-io/persisted-state'; +import colors from 'colors'; +import type { Got } from 'got'; +import * as t from 'io-ts'; +import { keyBy } from 'lodash-es'; +import { logger } from '../../logger'; +import { PreferenceTopic } from '../graphql'; +import { readCsv } from '../requests'; +import { checkIfPendingPreferenceUpdatesAreNoOp } from './checkIfPendingPreferenceUpdatesAreNoOp'; +import { checkIfPendingPreferenceUpdatesCauseConflict } from './checkIfPendingPreferenceUpdatesCauseConflict'; +import { FileMetadataState, PreferenceState } from './codecs'; +import { getPreferencesForIdentifiers } from './getPreferencesForIdentifiers'; +import { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow'; +import { parsePreferenceAndPurposeValuesFromCsv } from './parsePreferenceAndPurposeValuesFromCsv'; +import { parsePreferenceIdentifiersFromCsv } from './parsePreferenceIdentifiersFromCsv'; +import { parsePreferenceTimestampsFromCsv } from './parsePreferenceTimestampsFromCsv'; /** * Parse a file into the cache @@ -48,13 +48,13 @@ export async function parsePreferenceManagementCsvWithCache( /** Wheather to force workflow triggers */ forceTriggerWorkflows: boolean; }, - cache: PersistedState + cache: PersistedState, ): Promise { // Start the timer const t0 = Date.now(); // Get the current metadata - const fileMetadata = cache.getValue("fileMetadata"); + const fileMetadata = cache.getValue('fileMetadata'); // Read in the file logger.info(colors.magenta(`Reading in file: "${file}"`)); @@ -74,20 +74,20 @@ export async function parsePreferenceManagementCsvWithCache( // Validate that all timestamps are present in the file currentState = await parsePreferenceTimestampsFromCsv( preferences, - currentState + currentState, ); fileMetadata[file] = currentState; - await cache.setValue(fileMetadata, "fileMetadata"); + await cache.setValue(fileMetadata, 'fileMetadata'); // Validate that all identifiers are present and unique const result = await parsePreferenceIdentifiersFromCsv( preferences, - currentState + currentState, ); currentState = result.currentState; preferences = result.preferences; fileMetadata[file] = currentState; - await cache.setValue(fileMetadata, "fileMetadata"); + await cache.setValue(fileMetadata, 'fileMetadata'); // Ensure all other columns are mapped to purpose and preference // slug values @@ -98,14 +98,14 @@ export async function parsePreferenceManagementCsvWithCache( preferenceTopics, purposeSlugs, forceTriggerWorkflows, - } + }, ); fileMetadata[file] = currentState; - await cache.setValue(fileMetadata, "fileMetadata"); + await cache.setValue(fileMetadata, 'fileMetadata'); // Grab existing preference store records const identifiers = preferences.map( - (pref) => pref[currentState.identifierColumn!] + (pref) => pref[currentState.identifierColumn!], ); const existingConsentRecords = skipExistingRecordCheck ? [] @@ -113,7 +113,7 @@ export async function parsePreferenceManagementCsvWithCache( identifiers: identifiers.map((x) => ({ value: x })), partitionKey, }); - const consentRecordByIdentifier = keyBy(existingConsentRecords, "userId"); + const consentRecordByIdentifier = keyBy(existingConsentRecords, 'userId'); // Clear out previous updates currentState.pendingConflictUpdates = {}; @@ -138,7 +138,7 @@ export async function parsePreferenceManagementCsvWithCache( if (forceTriggerWorkflows && !currentConsentRecord) { throw new Error( `No existing consent record found for user with id: ${userId}. - When 'forceTriggerWorkflows' is set all the user identifiers should contain a consent record` + When 'forceTriggerWorkflows' is set all the user identifiers should contain a consent record`, ); } // Check if the update can be skipped @@ -179,11 +179,11 @@ export async function parsePreferenceManagementCsvWithCache( // Read in the file fileMetadata[file] = currentState; - await cache.setValue(fileMetadata, "fileMetadata"); + await cache.setValue(fileMetadata, 'fileMetadata'); const t1 = Date.now(); logger.info( colors.green( - `Successfully pre-processed file: "${file}" in ${(t1 - t0) / 1000}s` - ) + `Successfully pre-processed file: "${file}" in ${(t1 - t0) / 1000}s`, + ), ); } diff --git a/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts b/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts index e59ca765..09b2fb48 100644 --- a/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts +++ b/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts @@ -1,10 +1,10 @@ -import colors from "colors"; -import inquirer from "inquirer"; -import { difference, uniq } from "lodash-es"; -import { logger } from "../../logger"; -import { FileMetadataState } from "./codecs"; +import colors from 'colors'; +import inquirer from 'inquirer'; +import { difference, uniq } from 'lodash-es'; +import { logger } from '../../logger'; +import { FileMetadataState } from './codecs'; -export const NONE_PREFERENCE_MAP = "[NONE]"; +export const NONE_PREFERENCE_MAP = '[NONE]'; /** * Parse timestamps from a CSV list of preferences @@ -20,7 +20,7 @@ export const NONE_PREFERENCE_MAP = "[NONE]"; */ export async function parsePreferenceTimestampsFromCsv( preferences: Record[], - currentState: FileMetadataState + currentState: FileMetadataState, ): Promise { // Determine columns to map const columnNames = uniq(preferences.flatMap((x) => Object.keys(x))); @@ -38,16 +38,16 @@ export async function parsePreferenceTimestampsFromCsv( timestampName: string; }>([ { - name: "timestampName", + name: 'timestampName', message: - "Choose the column that will be used as the timestamp of last preference update", - type: "list", + 'Choose the column that will be used as the timestamp of last preference update', + type: 'list', default: remainingColumnsForTimestamp.find((col) => - col.toLowerCase().includes("date") + col.toLowerCase().includes('date'), ) || remainingColumnsForTimestamp.find((col) => - col.toLowerCase().includes("time") + col.toLowerCase().includes('time'), ) || remainingColumnsForTimestamp[0], choices: [...remainingColumnsForTimestamp, NONE_PREFERENCE_MAP], @@ -56,7 +56,7 @@ export async function parsePreferenceTimestampsFromCsv( currentState.timestampColum = timestampName; } logger.info( - colors.magenta(`Using timestamp column "${currentState.timestampColum}"`) + colors.magenta(`Using timestamp column "${currentState.timestampColum}"`), ); // Validate that all rows have valid timestamp @@ -70,14 +70,14 @@ export async function parsePreferenceTimestampsFromCsv( `The timestamp column "${ currentState.timestampColum }" is missing a value for the following rows: ${timestampColumnsMissing.join( - "\n" - )}` + '\n', + )}`, ); } logger.info( colors.magenta( - `The timestamp column "${currentState.timestampColum}" is present for all row` - ) + `The timestamp column "${currentState.timestampColum}" is present for all row`, + ), ); } return currentState; diff --git a/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesAreNoOp.test.ts b/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesAreNoOp.test.ts index baa9f464..7d6bec73 100644 --- a/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesAreNoOp.test.ts +++ b/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesAreNoOp.test.ts @@ -1,15 +1,15 @@ -import { PreferenceTopicType } from "@transcend-io/privacy-types"; -import { describe, expect, it } from "vitest"; -import { PreferenceTopic } from "../../graphql"; -import { checkIfPendingPreferenceUpdatesAreNoOp } from "../index"; +import { PreferenceTopicType } from '@transcend-io/privacy-types'; +import { describe, expect, it } from 'vitest'; +import { PreferenceTopic } from '../../graphql'; +import { checkIfPendingPreferenceUpdatesAreNoOp } from '../index'; const DEFAULT_VALUES = { - userId: "test@transcend.io", - timestamp: "2024-11-30T00:00:15.327Z", - partition: "d9c0b9ca-2253-4418-89d2-88776d654223", + userId: 'test@transcend.io', + timestamp: '2024-11-30T00:00:15.327Z', + partition: 'd9c0b9ca-2253-4418-89d2-88776d654223', system: { - decryptionStatus: "DECRYPTED" as const, - updatedAt: "2024-11-30T00:00:16.506Z", + decryptionStatus: 'DECRYPTED' as const, + updatedAt: '2024-11-30T00:00:16.506Z', }, consentManagement: { usp: null, @@ -22,122 +22,122 @@ const DEFAULT_VALUES = { const PREFERENCE_TOPICS: PreferenceTopic[] = [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference1", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, title: { - defaultMessage: "Boolean Preference 1", - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3", + defaultMessage: 'Boolean Preference 1', + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3', }, displayDescription: { - defaultMessage: "This is a boolean preference for testing.", - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4", + defaultMessage: 'This is a boolean preference for testing.', + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4', }, - defaultConfiguration: "", + defaultConfiguration: '', showInPrivacyCenter: true, }, { - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference2", + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, title: { - defaultMessage: "Boolean Preference 2", - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4", + defaultMessage: 'Boolean Preference 2', + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4', }, displayDescription: { - defaultMessage: "This is another boolean preference for testing.", - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b5", + defaultMessage: 'This is another boolean preference for testing.', + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b5', }, - defaultConfiguration: "", + defaultConfiguration: '', showInPrivacyCenter: true, }, { - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "MultiSelectPreference", + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'MultiSelectPreference', type: PreferenceTopicType.MultiSelect, title: { - defaultMessage: "Multi Select Preference", - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b0", + defaultMessage: 'Multi Select Preference', + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b0', }, displayDescription: { - defaultMessage: "This is a multi-select preference for testing.", - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1", + defaultMessage: 'This is a multi-select preference for testing.', + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1', }, - defaultConfiguration: "", + defaultConfiguration: '', showInPrivacyCenter: true, preferenceOptionValues: [ { - slug: "Value1", + slug: 'Value1', title: { - defaultMessage: "Value 1", - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1", + defaultMessage: 'Value 1', + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1', }, }, { - slug: "Value2", + slug: 'Value2', title: { - defaultMessage: "Value 2", - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2", + defaultMessage: 'Value 2', + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2', }, }, ], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, { - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "SingleSelectPreference", + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'SingleSelectPreference', type: PreferenceTopicType.Select, preferenceOptionValues: [ { - slug: "Value1", + slug: 'Value1', title: { - defaultMessage: "Value 1", - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1", + defaultMessage: 'Value 1', + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1', }, }, { - slug: "Value2", + slug: 'Value2', title: { - defaultMessage: "Value 2", - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2", + defaultMessage: 'Value 2', + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2', }, }, ], title: { - defaultMessage: "Single Select Preference", - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b0", + defaultMessage: 'Single Select Preference', + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b0', }, displayDescription: { - defaultMessage: "This is a single-select preference for testing.", - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1", + defaultMessage: 'This is a single-select preference for testing.', + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1', }, - defaultConfiguration: "", + defaultConfiguration: '', showInPrivacyCenter: true, purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, ]; -describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { - it("should return true for simple purpose comparison", () => { +describe('checkIfPendingPreferenceUpdatesAreNoOp', () => { + it('should return true for simple purpose comparison', () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "SalesOutreach", + purpose: 'SalesOutreach', enabled: false, preferences: [], }, @@ -149,18 +149,18 @@ describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(true); }); - it("should return false for simple purpose comparison", () => { + it('should return false for simple purpose comparison', () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "SalesOutreach", + purpose: 'SalesOutreach', enabled: true, preferences: [], }, @@ -172,22 +172,22 @@ describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); - it("should return true for simple purpose comparison with extra preference", () => { + it('should return true for simple purpose comparison with extra preference', () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "SalesOutreach", + purpose: 'SalesOutreach', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, @@ -202,18 +202,18 @@ describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(true); }); - it("should return false for simple purpose comparison with extra preference in update", () => { + it('should return false for simple purpose comparison with extra preference in update', () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "SalesOutreach", + purpose: 'SalesOutreach', enabled: false, }, ], @@ -223,7 +223,7 @@ describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, @@ -232,36 +232,36 @@ describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); - it("should return true for preferences being same", () => { + it('should return true for preferences being same', () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "Marketing", + purpose: 'Marketing', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], @@ -273,57 +273,57 @@ describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(true); }); - it("should return false for boolean preference changing", () => { + it('should return false for boolean preference changing', () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "Marketing", + purpose: 'Marketing', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: false, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], @@ -335,57 +335,57 @@ describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); - it("should return false for single select preference changing", () => { + it('should return false for single select preference changing', () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "Marketing", + purpose: 'Marketing', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value2", + selectValue: 'Value2', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], @@ -397,57 +397,57 @@ describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); - it("should return false for multi select preference changing", () => { + it('should return false for multi select preference changing', () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "Marketing", + purpose: 'Marketing', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value2"], + selectValues: ['Value2'], }, }, ], @@ -459,28 +459,28 @@ describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); }); diff --git a/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts b/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts index 11fffaf9..f2735fbf 100644 --- a/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts +++ b/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts @@ -1,15 +1,15 @@ -import { PreferenceTopicType } from "@transcend-io/privacy-types"; -import { describe, expect, it } from "vitest"; -import { PreferenceTopic } from "../../graphql"; -import { checkIfPendingPreferenceUpdatesCauseConflict } from "../index"; +import { PreferenceTopicType } from '@transcend-io/privacy-types'; +import { describe, expect, it } from 'vitest'; +import { PreferenceTopic } from '../../graphql'; +import { checkIfPendingPreferenceUpdatesCauseConflict } from '../index'; const DEFAULT_VALUES = { - userId: "test@transcend.io", - timestamp: "2024-11-30T00:00:15.327Z", - partition: "d9c0b9ca-2253-4418-89d2-88776d654223", + userId: 'test@transcend.io', + timestamp: '2024-11-30T00:00:15.327Z', + partition: 'd9c0b9ca-2253-4418-89d2-88776d654223', system: { - decryptionStatus: "DECRYPTED" as const, - updatedAt: "2024-11-30T00:00:16.506Z", + decryptionStatus: 'DECRYPTED' as const, + updatedAt: '2024-11-30T00:00:16.506Z', }, consentManagement: { usp: null, @@ -22,125 +22,125 @@ const DEFAULT_VALUES = { const PREFERENCE_TOPICS: PreferenceTopic[] = [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference1", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, title: { - defaultMessage: "Boolean Preference 1", - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3", + defaultMessage: 'Boolean Preference 1', + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3', }, displayDescription: { - defaultMessage: "This is a boolean preference for testing purposes.", - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4", + defaultMessage: 'This is a boolean preference for testing purposes.', + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', }, { - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference2", + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, title: { - defaultMessage: "Boolean Preference 2", - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4", + defaultMessage: 'Boolean Preference 2', + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4', }, displayDescription: { defaultMessage: - "This is another boolean preference for testing purposes.", - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b5", + 'This is another boolean preference for testing purposes.', + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b5', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', }, { - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "MultiSelectPreference", + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'MultiSelectPreference', type: PreferenceTopicType.MultiSelect, preferenceOptionValues: [ { - slug: "Value1", + slug: 'Value1', title: { - defaultMessage: "Value 1", - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1", + defaultMessage: 'Value 1', + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1', }, }, { - slug: "Value2", + slug: 'Value2', title: { - defaultMessage: "Value 2", - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2", + defaultMessage: 'Value 2', + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2', }, }, ], title: { - defaultMessage: "Multi Select Preference", - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3", + defaultMessage: 'Multi Select Preference', + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3', }, displayDescription: { - defaultMessage: "This is a multi-select preference for testing purposes.", - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4", + defaultMessage: 'This is a multi-select preference for testing purposes.', + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, { - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "SingleSelectPreference", + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'SingleSelectPreference', type: PreferenceTopicType.Select, preferenceOptionValues: [ { - slug: "Value1", + slug: 'Value1', title: { - defaultMessage: "Value 1", - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1", + defaultMessage: 'Value 1', + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1', }, }, { - slug: "Value2", + slug: 'Value2', title: { - defaultMessage: "Value 2", - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2", + defaultMessage: 'Value 2', + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2', }, }, ], title: { - defaultMessage: "Single Select Preference", - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3", + defaultMessage: 'Single Select Preference', + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3', }, displayDescription: { defaultMessage: - "This is a single-select preference for testing purposes.", - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4", + 'This is a single-select preference for testing purposes.', + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', // This preference is associated with the Marketing purpose purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, ]; -describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { - it("should return false for simple purpose comparison", () => { +describe('checkIfPendingPreferenceUpdatesCauseConflict', () => { + it('should return false for simple purpose comparison', () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "SalesOutreach", + purpose: 'SalesOutreach', enabled: false, preferences: [], }, @@ -152,18 +152,18 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); - it("should return false if purpose missing", () => { + it('should return false if purpose missing', () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "SalesOutreach", + purpose: 'SalesOutreach', enabled: false, preferences: [], }, @@ -175,18 +175,18 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); - it("should return true for simple purpose comparison", () => { + it('should return true for simple purpose comparison', () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "SalesOutreach", + purpose: 'SalesOutreach', enabled: true, preferences: [], }, @@ -198,22 +198,22 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(true); }); - it("should return true for simple purpose comparison with extra preference", () => { + it('should return true for simple purpose comparison with extra preference', () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "SalesOutreach", + purpose: 'SalesOutreach', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, @@ -228,18 +228,18 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); - it("should return false for simple purpose comparison with extra preference in update", () => { + it('should return false for simple purpose comparison with extra preference in update', () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "SalesOutreach", + purpose: 'SalesOutreach', enabled: false, }, ], @@ -249,7 +249,7 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, @@ -258,36 +258,36 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); - it("should return false for preferences being same", () => { + it('should return false for preferences being same', () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "Marketing", + purpose: 'Marketing', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], @@ -299,57 +299,57 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); - it("should return true for boolean preference changing", () => { + it('should return true for boolean preference changing', () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "Marketing", + purpose: 'Marketing', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: false, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], @@ -361,57 +361,57 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(true); }); - it("should return true for single select preference changing", () => { + it('should return true for single select preference changing', () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "Marketing", + purpose: 'Marketing', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value2", + selectValue: 'Value2', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], @@ -423,57 +423,57 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(true); }); - it("should return true for multi select preference changing", () => { + it('should return true for multi select preference changing', () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "Marketing", + purpose: 'Marketing', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value2"], + selectValues: ['Value2'], }, }, ], @@ -485,28 +485,28 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(true); }); }); diff --git a/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts b/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts index 587d3353..1c4a3c90 100644 --- a/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts +++ b/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts @@ -1,60 +1,60 @@ -import { PreferenceTopicType } from "@transcend-io/privacy-types"; -import { describe, expect, it } from "vitest"; -import { getPreferenceUpdatesFromRow } from "../index"; +import { PreferenceTopicType } from '@transcend-io/privacy-types'; +import { describe, expect, it } from 'vitest'; +import { getPreferenceUpdatesFromRow } from '../index'; -describe("getPreferenceUpdatesFromRow", () => { - it("should parse boolean updates", () => { +describe('getPreferenceUpdatesFromRow', () => { + it('should parse boolean updates', () => { expect( getPreferenceUpdatesFromRow({ row: { - my_purpose: "true", - has_topic_1: "true", - has_topic_2: "false", + my_purpose: 'true', + has_topic_1: 'true', + has_topic_2: 'false', }, - purposeSlugs: ["Marketing"], + purposeSlugs: ['Marketing'], preferenceTopics: [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference1", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], title: { - defaultMessage: "Marketing Preferences", - id: "12345678-1234-1234-1234-123456789012", + defaultMessage: 'Marketing Preferences', + id: '12345678-1234-1234-1234-123456789012', }, displayDescription: { - defaultMessage: "Enable marketing tracking", - id: "12345678-1234-1234-1234-123456789013", + defaultMessage: 'Enable marketing tracking', + id: '12345678-1234-1234-1234-123456789013', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, { - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference2", + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', title: { - defaultMessage: "Advertising Preferences", - id: "12345678-1234-1234-1234-123456789014", + defaultMessage: 'Advertising Preferences', + id: '12345678-1234-1234-1234-123456789014', }, displayDescription: { - defaultMessage: "Enable advertising tracking", - id: "12345678-1234-1234-1234-123456789015", + defaultMessage: 'Enable advertising tracking', + id: '12345678-1234-1234-1234-123456789015', }, }, ], columnToPurposeName: { my_purpose: { - purpose: "Marketing", + purpose: 'Marketing', preference: null, valueMapping: { true: true, @@ -62,35 +62,35 @@ describe("getPreferenceUpdatesFromRow", () => { }, }, has_topic_1: { - purpose: "Marketing", - preference: "BooleanPreference1", + purpose: 'Marketing', + preference: 'BooleanPreference1', valueMapping: { true: true, false: false, }, }, has_topic_2: { - purpose: "Marketing", - preference: "BooleanPreference2", + purpose: 'Marketing', + preference: 'BooleanPreference2', valueMapping: { true: true, false: false, }, }, }, - }) + }), ).to.deep.equal({ Marketing: { enabled: true, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "BooleanPreference2", + topic: 'BooleanPreference2', choice: { booleanValue: false, }, @@ -100,53 +100,53 @@ describe("getPreferenceUpdatesFromRow", () => { }); }); - it("should parse a single select", () => { + it('should parse a single select', () => { expect( getPreferenceUpdatesFromRow({ row: { - my_purpose: "true", - has_topic_3: "Option 1", + my_purpose: 'true', + has_topic_3: 'Option 1', }, - purposeSlugs: ["Marketing"], + purposeSlugs: ['Marketing'], preferenceTopics: [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "SingleSelectPreference", - defaultConfiguration: "", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'SingleSelectPreference', + defaultConfiguration: '', title: { - defaultMessage: "Single Select Preference", - id: "12345678-1234-1234-1234-123456789010", + defaultMessage: 'Single Select Preference', + id: '12345678-1234-1234-1234-123456789010', }, displayDescription: { - defaultMessage: "Choose one option", - id: "12345678-1234-1234-1234-123456789011", + defaultMessage: 'Choose one option', + id: '12345678-1234-1234-1234-123456789011', }, showInPrivacyCenter: true, type: PreferenceTopicType.Select, preferenceOptionValues: [ { - slug: "Value1", + slug: 'Value1', title: { - defaultMessage: "Option 1", - id: "12345678-1234-1234-1234-123456789016", + defaultMessage: 'Option 1', + id: '12345678-1234-1234-1234-123456789016', }, }, { - slug: "Value2", + slug: 'Value2', title: { - defaultMessage: "Option 2", - id: "12345678-1234-1234-1234-123456789017", + defaultMessage: 'Option 2', + id: '12345678-1234-1234-1234-123456789017', }, }, ], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, ], columnToPurposeName: { my_purpose: { - purpose: "Marketing", + purpose: 'Marketing', preference: null, valueMapping: { true: true, @@ -154,23 +154,23 @@ describe("getPreferenceUpdatesFromRow", () => { }, }, has_topic_3: { - purpose: "Marketing", - preference: "SingleSelectPreference", + purpose: 'Marketing', + preference: 'SingleSelectPreference', valueMapping: { - "Option 1": "Value1", - "Option 2": "Value2", + 'Option 1': 'Value1', + 'Option 2': 'Value2', }, }, }, - }) + }), ).to.deep.equal({ Marketing: { enabled: true, preferences: [ { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, ], @@ -178,54 +178,54 @@ describe("getPreferenceUpdatesFromRow", () => { }); }); - it("should parse a multi select example", () => { + it('should parse a multi select example', () => { expect( getPreferenceUpdatesFromRow({ row: { - my_purpose: "true", - has_topic_4: "Option 2,Option 1", - has_topic_5: "Option 1", + my_purpose: 'true', + has_topic_4: 'Option 2,Option 1', + has_topic_5: 'Option 1', }, - purposeSlugs: ["Marketing"], + purposeSlugs: ['Marketing'], preferenceTopics: [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "MultiSelectPreference", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'MultiSelectPreference', type: PreferenceTopicType.MultiSelect, - defaultConfiguration: "", + defaultConfiguration: '', title: { - defaultMessage: "Multi Select Preference", - id: "12345678-1234-1234-1234-123456789020", + defaultMessage: 'Multi Select Preference', + id: '12345678-1234-1234-1234-123456789020', }, displayDescription: { - defaultMessage: "Choose multiple options", - id: "12345678-1234-1234-1234-123456789021", + defaultMessage: 'Choose multiple options', + id: '12345678-1234-1234-1234-123456789021', }, showInPrivacyCenter: true, preferenceOptionValues: [ { - slug: "Value1", + slug: 'Value1', title: { - defaultMessage: "Option 1", - id: "12345678-1234-1234-1234-123456789018", + defaultMessage: 'Option 1', + id: '12345678-1234-1234-1234-123456789018', }, }, { - slug: "Value2", + slug: 'Value2', title: { - defaultMessage: "Option 2", - id: "12345678-1234-1234-1234-123456789019", + defaultMessage: 'Option 2', + id: '12345678-1234-1234-1234-123456789019', }, }, ], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, ], columnToPurposeName: { my_purpose: { - purpose: "Marketing", + purpose: 'Marketing', preference: null, valueMapping: { true: true, @@ -233,37 +233,37 @@ describe("getPreferenceUpdatesFromRow", () => { }, }, has_topic_4: { - purpose: "Marketing", - preference: "MultiSelectPreference", + purpose: 'Marketing', + preference: 'MultiSelectPreference', valueMapping: { - "Option 1": "Value1", - "Option 2": "Value2", + 'Option 1': 'Value1', + 'Option 2': 'Value2', }, }, has_topic_5: { - purpose: "Marketing", - preference: "MultiSelectPreference", + purpose: 'Marketing', + preference: 'MultiSelectPreference', valueMapping: { - "Option 1": "Value1", - "Option 2": "Value2", + 'Option 1': 'Value1', + 'Option 2': 'Value2', }, }, }, - }) + }), ).to.deep.equal({ Marketing: { enabled: true, preferences: [ { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1"], + selectValues: ['Value1'], }, }, ], @@ -271,128 +271,128 @@ describe("getPreferenceUpdatesFromRow", () => { }); }); - it("should parse boolean, single select, multi select example", () => { + it('should parse boolean, single select, multi select example', () => { expect( getPreferenceUpdatesFromRow({ row: { - my_purpose: "true", - has_topic_1: "true", - has_topic_2: "false", - has_topic_3: "Option 1", - has_topic_4: "Option 2,Option 1", + my_purpose: 'true', + has_topic_1: 'true', + has_topic_2: 'false', + has_topic_3: 'Option 1', + has_topic_4: 'Option 2,Option 1', }, - purposeSlugs: ["Marketing"], + purposeSlugs: ['Marketing'], preferenceTopics: [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference1", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], title: { - defaultMessage: "Boolean Preference 1", - id: "12345678-1234-1234-1234-123456789022", + defaultMessage: 'Boolean Preference 1', + id: '12345678-1234-1234-1234-123456789022', }, displayDescription: { - defaultMessage: "Enable this preference", - id: "12345678-1234-1234-1234-123456789023", + defaultMessage: 'Enable this preference', + id: '12345678-1234-1234-1234-123456789023', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, { - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference2", + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], title: { - defaultMessage: "Boolean Preference 2", - id: "12345678-1234-1234-1234-123456789024", + defaultMessage: 'Boolean Preference 2', + id: '12345678-1234-1234-1234-123456789024', }, displayDescription: { - defaultMessage: "Disable this preference", - id: "12345678-1234-1234-1234-123456789025", + defaultMessage: 'Disable this preference', + id: '12345678-1234-1234-1234-123456789025', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, { - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "MultiSelectPreference", + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'MultiSelectPreference', type: PreferenceTopicType.MultiSelect, - defaultConfiguration: "", + defaultConfiguration: '', title: { - defaultMessage: "Multi Select Preference", - id: "12345678-1234-1234-1234-123456789028", + defaultMessage: 'Multi Select Preference', + id: '12345678-1234-1234-1234-123456789028', }, displayDescription: { - defaultMessage: "Choose multiple options", - id: "12345678-1234-1234-1234-123456789029", + defaultMessage: 'Choose multiple options', + id: '12345678-1234-1234-1234-123456789029', }, showInPrivacyCenter: true, preferenceOptionValues: [ { - slug: "Value1", + slug: 'Value1', title: { - defaultMessage: "Option 1", - id: "12345678-1234-1234-1234-123456789026", + defaultMessage: 'Option 1', + id: '12345678-1234-1234-1234-123456789026', }, }, { - slug: "Value2", + slug: 'Value2', title: { - defaultMessage: "Option 2", - id: "12345678-1234-1234-1234-123456789027", + defaultMessage: 'Option 2', + id: '12345678-1234-1234-1234-123456789027', }, }, ], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, { - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "SingleSelectPreference", + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'SingleSelectPreference', type: PreferenceTopicType.Select, - defaultConfiguration: "", + defaultConfiguration: '', title: { - defaultMessage: "Single Select Preference", - id: "12345678-1234-1234-1234-123456789030", + defaultMessage: 'Single Select Preference', + id: '12345678-1234-1234-1234-123456789030', }, displayDescription: { - defaultMessage: "Choose one option", - id: "12345678-1234-1234-1234-123456789031", + defaultMessage: 'Choose one option', + id: '12345678-1234-1234-1234-123456789031', }, showInPrivacyCenter: true, preferenceOptionValues: [ { - slug: "Value1", + slug: 'Value1', title: { - defaultMessage: "Option 1", - id: "12345678-1234-1234-1234-123456789030", + defaultMessage: 'Option 1', + id: '12345678-1234-1234-1234-123456789030', }, }, { - slug: "Value2", + slug: 'Value2', title: { - defaultMessage: "Option 2", - id: "12345678-1234-1234-1234-123456789031", + defaultMessage: 'Option 2', + id: '12345678-1234-1234-1234-123456789031', }, }, ], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, ], columnToPurposeName: { my_purpose: { - purpose: "Marketing", + purpose: 'Marketing', preference: null, valueMapping: { true: true, @@ -400,65 +400,65 @@ describe("getPreferenceUpdatesFromRow", () => { }, }, has_topic_1: { - purpose: "Marketing", - preference: "BooleanPreference1", + purpose: 'Marketing', + preference: 'BooleanPreference1', valueMapping: { true: true, false: false, }, }, has_topic_2: { - purpose: "Marketing", - preference: "BooleanPreference2", + purpose: 'Marketing', + preference: 'BooleanPreference2', valueMapping: { true: true, false: false, }, }, has_topic_3: { - purpose: "Marketing", - preference: "SingleSelectPreference", + purpose: 'Marketing', + preference: 'SingleSelectPreference', valueMapping: { - "Option 1": "Value1", - "Option 2": "Value2", + 'Option 1': 'Value1', + 'Option 2': 'Value2', }, }, has_topic_4: { - purpose: "Marketing", - preference: "MultiSelectPreference", + purpose: 'Marketing', + preference: 'MultiSelectPreference', valueMapping: { - "Option 1": "Value1", - "Option 2": "Value2", + 'Option 1': 'Value1', + 'Option 2': 'Value2', }, }, }, - }) + }), ).to.deep.equal({ Marketing: { enabled: true, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "BooleanPreference2", + topic: 'BooleanPreference2', choice: { booleanValue: false, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], @@ -466,57 +466,57 @@ describe("getPreferenceUpdatesFromRow", () => { }); }); - it("should error if missing purpose", () => { + it('should error if missing purpose', () => { try { getPreferenceUpdatesFromRow({ row: { - has_topic_1: "true", + has_topic_1: 'true', }, - purposeSlugs: ["Marketing"], + purposeSlugs: ['Marketing'], preferenceTopics: [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference1", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, displayDescription: { - defaultMessage: "Enable this preference", - id: "12345678-1234-1234-1234-123456789032", + defaultMessage: 'Enable this preference', + id: '12345678-1234-1234-1234-123456789032', }, title: { - defaultMessage: "Boolean Preference 1", - id: "12345678-1234-1234-1234-123456789033", + defaultMessage: 'Boolean Preference 1', + id: '12345678-1234-1234-1234-123456789033', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', }, { - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference2", + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, displayDescription: { - defaultMessage: "Disable this preference", - id: "12345678-1234-1234-1234-123456789034", + defaultMessage: 'Disable this preference', + id: '12345678-1234-1234-1234-123456789034', }, title: { - defaultMessage: "Boolean Preference 2", - id: "12345678-1234-1234-1234-123456789035", + defaultMessage: 'Boolean Preference 2', + id: '12345678-1234-1234-1234-123456789035', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', }, ], columnToPurposeName: { has_topic_1: { - purpose: "Marketing", - preference: "BooleanPreference1", + purpose: 'Marketing', + preference: 'BooleanPreference1', valueMapping: { true: true, false: false, @@ -524,63 +524,63 @@ describe("getPreferenceUpdatesFromRow", () => { }, }, }); - expect.fail("Should have thrown"); + expect.fail('Should have thrown'); } catch (error) { - expect(error.message).to.include("No mapping provided"); + expect(error.message).to.include('No mapping provided'); } }); - it("should error if purpose name is not valid", () => { + it('should error if purpose name is not valid', () => { try { getPreferenceUpdatesFromRow({ row: { - has_topic_1: "true", + has_topic_1: 'true', }, - purposeSlugs: ["Marketing", "Advertising"], + purposeSlugs: ['Marketing', 'Advertising'], preferenceTopics: [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference1", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, displayDescription: { - defaultMessage: "Enable this preference", - id: "12345678-1234-1234-1234-123456789036", + defaultMessage: 'Enable this preference', + id: '12345678-1234-1234-1234-123456789036', }, title: { - defaultMessage: "Boolean Preference 1", - id: "12345678-1234-1234-1234-123456789037", + defaultMessage: 'Boolean Preference 1', + id: '12345678-1234-1234-1234-123456789037', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', }, { - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference2", + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, displayDescription: { - defaultMessage: "Disable this preference", - id: "12345678-1234-1234-1234-123456789038", + defaultMessage: 'Disable this preference', + id: '12345678-1234-1234-1234-123456789038', }, title: { - defaultMessage: "Boolean Preference 2", - id: "12345678-1234-1234-1234-123456789039", + defaultMessage: 'Boolean Preference 2', + id: '12345678-1234-1234-1234-123456789039', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', }, ], columnToPurposeName: { has_topic_1: { - purpose: "InvalidPurpose", - preference: "BooleanPreference1", + purpose: 'InvalidPurpose', + preference: 'BooleanPreference1', valueMapping: { true: true, false: false, @@ -588,134 +588,134 @@ describe("getPreferenceUpdatesFromRow", () => { }, }, }); - expect.fail("Should have thrown"); + expect.fail('Should have thrown'); } catch (error) { expect(error.message).to.equal( - "Invalid purpose slug: InvalidPurpose, expected: Marketing, Advertising" + 'Invalid purpose slug: InvalidPurpose, expected: Marketing, Advertising', ); } }); - it("should error if single select option is invalid", () => { + it('should error if single select option is invalid', () => { try { getPreferenceUpdatesFromRow({ row: { - has_topic_1: "true", + has_topic_1: 'true', }, - purposeSlugs: ["Marketing"], + purposeSlugs: ['Marketing'], preferenceTopics: [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "SingleSelectPreference", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'SingleSelectPreference', type: PreferenceTopicType.Select, preferenceOptionValues: [ { - slug: "Value1", + slug: 'Value1', title: { - defaultMessage: "Option 1", - id: "12345678-1234-1234-1234-123456789040", + defaultMessage: 'Option 1', + id: '12345678-1234-1234-1234-123456789040', }, }, { - slug: "Value2", + slug: 'Value2', title: { - defaultMessage: "Option 2", - id: "12345678-1234-1234-1234-123456789041", + defaultMessage: 'Option 2', + id: '12345678-1234-1234-1234-123456789041', }, }, ], title: { - defaultMessage: "Single Select Preference", - id: "12345678-1234-1234-1234-123456789042", + defaultMessage: 'Single Select Preference', + id: '12345678-1234-1234-1234-123456789042', }, displayDescription: { - defaultMessage: "Choose one option", - id: "12345678-1234-1234-1234-123456789043", + defaultMessage: 'Choose one option', + id: '12345678-1234-1234-1234-123456789043', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, ], columnToPurposeName: { has_topic_1: { - purpose: "Marketing", - preference: "SingleSelectPreference", + purpose: 'Marketing', + preference: 'SingleSelectPreference', valueMapping: { - "Option 1": "Value1", - "Option 2": "Value2", + 'Option 1': 'Value1', + 'Option 2': 'Value2', }, }, }, }); - expect.fail("Should have thrown"); + expect.fail('Should have thrown'); } catch (error) { expect(error.message).to.equal( - "Invalid value for select preference: SingleSelectPreference, expected string or null, got: true" + 'Invalid value for select preference: SingleSelectPreference, expected string or null, got: true', ); } }); - it("should error if multi select value is invalid", () => { + it('should error if multi select value is invalid', () => { try { getPreferenceUpdatesFromRow({ row: { - has_topic_1: "true", + has_topic_1: 'true', }, - purposeSlugs: ["Marketing"], + purposeSlugs: ['Marketing'], preferenceTopics: [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "MultiSelectPreference", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'MultiSelectPreference', type: PreferenceTopicType.MultiSelect, preferenceOptionValues: [ { title: { - defaultMessage: "Option 1", - id: "12345678-1234-1234-1234-123456789044", + defaultMessage: 'Option 1', + id: '12345678-1234-1234-1234-123456789044', }, - slug: "Value1", + slug: 'Value1', }, { title: { - defaultMessage: "Option 2", - id: "12345678-1234-1234-1234-123456789045", + defaultMessage: 'Option 2', + id: '12345678-1234-1234-1234-123456789045', }, - slug: "Value2", + slug: 'Value2', }, ], title: { - defaultMessage: "Multi Select Preference", - id: "12345678-1234-1234-1234-123456789046", + defaultMessage: 'Multi Select Preference', + id: '12345678-1234-1234-1234-123456789046', }, displayDescription: { - defaultMessage: "Choose multiple options", - id: "12345678-1234-1234-1234-123456789047", + defaultMessage: 'Choose multiple options', + id: '12345678-1234-1234-1234-123456789047', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, ], columnToPurposeName: { has_topic_1: { - purpose: "Marketing", - preference: "MultiSelectPreference", + purpose: 'Marketing', + preference: 'MultiSelectPreference', valueMapping: { - "Option 1": "Value1", - "Option 2": "Value2", + 'Option 1': 'Value1', + 'Option 2': 'Value2', }, }, }, }); - expect.fail("Should have thrown"); + expect.fail('Should have thrown'); } catch (error) { expect(error.message).to.equal( - "Invalid value for multi select preference: MultiSelectPreference, expected one of: Value1, Value2, got: true" + 'Invalid value for multi select preference: MultiSelectPreference, expected one of: Value1, Value2, got: true', ); } }); diff --git a/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts b/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts index 06da6e1f..0f592e5e 100644 --- a/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts +++ b/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts @@ -1,12 +1,12 @@ -import { PersistedState } from "@transcend-io/persisted-state"; -import { PreferenceUpdateItem } from "@transcend-io/privacy-types"; -import { apply } from "@transcend-io/type-utils"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { chunk } from "lodash-es"; -import { DEFAULT_TRANSCEND_CONSENT_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { PersistedState } from '@transcend-io/persisted-state'; +import { PreferenceUpdateItem } from '@transcend-io/privacy-types'; +import { apply } from '@transcend-io/type-utils'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { chunk } from 'lodash-es'; +import { DEFAULT_TRANSCEND_CONSENT_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, createSombraGotInstance, @@ -14,12 +14,12 @@ import { fetchAllPurposes, PreferenceTopic, Purpose, -} from "../graphql"; -import { parseAttributesFromString } from "../requests"; -import { PreferenceState } from "./codecs"; -import { getPreferenceUpdatesFromRow } from "./getPreferenceUpdatesFromRow"; -import { parsePreferenceManagementCsvWithCache } from "./parsePreferenceManagementCsv"; -import { NONE_PREFERENCE_MAP } from "./parsePreferenceTimestampsFromCsv"; +} from '../graphql'; +import { parseAttributesFromString } from '../requests'; +import { PreferenceState } from './codecs'; +import { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow'; +import { parsePreferenceManagementCsvWithCache } from './parsePreferenceManagementCsv'; +import { NONE_PREFERENCE_MAP } from './parsePreferenceTimestampsFromCsv'; /** * Upload a set of consent preferences @@ -80,13 +80,13 @@ export async function uploadPreferenceManagementPreferencesInteractive({ failingUpdates: {}, pendingUpdates: {}, }); - const failingRequests = preferenceState.getValue("failingUpdates"); - const pendingRequests = preferenceState.getValue("pendingUpdates"); - let fileMetadata = preferenceState.getValue("fileMetadata"); + const failingRequests = preferenceState.getValue('failingUpdates'); + const pendingRequests = preferenceState.getValue('pendingUpdates'); + let fileMetadata = preferenceState.getValue('fileMetadata'); logger.info( colors.magenta( - "Restored cache, there are: \n" + + 'Restored cache, there are: \n' + `${ Object.values(failingRequests).length } failing requests to be retried\n` + @@ -94,12 +94,12 @@ export async function uploadPreferenceManagementPreferencesInteractive({ Object.values(pendingRequests).length } pending requests to be processed\n` + `The following files are stored in cache and will be used:\n${Object.keys( - fileMetadata + fileMetadata, ) .map((x) => x) - .join("\n")}\n` + - `The following file will be processed: ${file}\n` - ) + .join('\n')}\n` + + `The following file will be processed: ${file}\n`, + ), ); // Create GraphQL client to connect to Transcend backend @@ -128,34 +128,34 @@ export async function uploadPreferenceManagementPreferencesInteractive({ skipExistingRecordCheck, forceTriggerWorkflows, }, - preferenceState + preferenceState, ); // Construct the pending updates const pendingUpdates: Record = {}; - fileMetadata = preferenceState.getValue("fileMetadata"); + fileMetadata = preferenceState.getValue('fileMetadata'); const metadata = fileMetadata[file]; logger.info( colors.magenta( `Found ${ Object.entries(metadata.pendingSafeUpdates).length - } safe updates in ${file}` - ) + } safe updates in ${file}`, + ), ); logger.info( colors.magenta( `Found ${ Object.entries(metadata.pendingConflictUpdates).length - } conflict updates in ${file}` - ) + } conflict updates in ${file}`, + ), ); logger.info( colors.magenta( `Found ${ Object.entries(metadata.skippedUpdates).length - } skipped updates in ${file}` - ) + } skipped updates in ${file}`, + ), ); // Update either safe updates only or safe + conflict @@ -193,8 +193,8 @@ export async function uploadPreferenceManagementPreferencesInteractive({ })), }; } - await preferenceState.setValue(pendingUpdates, "pendingUpdates"); - await preferenceState.setValue({}, "failingUpdates"); + await preferenceState.setValue(pendingUpdates, 'pendingUpdates'); + await preferenceState.setValue({}, 'failingUpdates'); // Exist early if dry run if (dryRun) { @@ -202,8 +202,8 @@ export async function uploadPreferenceManagementPreferencesInteractive({ colors.green( `Dry run complete, exiting. ${ Object.values(pendingUpdates).length - } pending updates. Check file: ${receiptFilepath}` - ) + } pending updates. Check file: ${receiptFilepath}`, + ), ); return; } @@ -212,8 +212,8 @@ export async function uploadPreferenceManagementPreferencesInteractive({ colors.magenta( `Uploading ${ Object.values(pendingUpdates).length - } preferences to partition: ${partition}` - ) + } preferences to partition: ${partition}`, + ), ); // Time duration @@ -222,7 +222,7 @@ export async function uploadPreferenceManagementPreferencesInteractive({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Build a GraphQL client @@ -236,7 +236,7 @@ export async function uploadPreferenceManagementPreferencesInteractive({ // Make the request try { await sombra - .put("v1/preferences", { + .put('v1/preferences', { json: { records: currentChunk.map(([, update]) => update), skipWorkflowTriggers, @@ -246,7 +246,7 @@ export async function uploadPreferenceManagementPreferencesInteractive({ .json(); } catch (error) { try { - const parsed = JSON.parse(error?.response?.body || "{}"); + const parsed = JSON.parse(error?.response?.body || '{}'); if (parsed.error) { logger.error(colors.red(`Error: ${parsed.error}`)); } @@ -259,18 +259,18 @@ export async function uploadPreferenceManagementPreferencesInteractive({ currentChunk.length } user preferences to partition ${partition}: ${ error?.response?.body || error?.message - }` - ) + }`, + ), ); - const failingUpdates = preferenceState.getValue("failingUpdates"); + const failingUpdates = preferenceState.getValue('failingUpdates'); for (const [userId, update] of currentChunk) { failingUpdates[userId] = { uploadedAt: new Date().toISOString(), update, - error: error?.response?.body || error?.message || "Unknown error", + error: error?.response?.body || error?.message || 'Unknown error', }; } - await preferenceState.setValue(failingUpdates, "failingUpdates"); + await preferenceState.setValue(failingUpdates, 'failingUpdates'); } total += currentChunk.length; @@ -278,7 +278,7 @@ export async function uploadPreferenceManagementPreferencesInteractive({ }, { concurrency: 40, - } + }, ); progressBar.stop(); @@ -290,7 +290,7 @@ export async function uploadPreferenceManagementPreferencesInteractive({ updatesToRun.length } user preferences to partition ${partition} in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); } diff --git a/src/lib/readTranscendYaml.ts b/src/lib/readTranscendYaml.ts index d31ed407..ea8e645e 100644 --- a/src/lib/readTranscendYaml.ts +++ b/src/lib/readTranscendYaml.ts @@ -1,10 +1,10 @@ -import { readFileSync, writeFileSync } from "node:fs"; -import { decodeCodec, ObjByString } from "@transcend-io/type-utils"; -import yaml from "js-yaml"; -import { TranscendInput } from "../codecs"; +import { readFileSync, writeFileSync } from 'node:fs'; +import { decodeCodec, ObjByString } from '@transcend-io/type-utils'; +import yaml from 'js-yaml'; +import { TranscendInput } from '../codecs'; export const VARIABLE_PARAMETERS_REGEXP = /<>/; -export const VARIABLE_PARAMETERS_NAME = "parameters"; +export const VARIABLE_PARAMETERS_NAME = 'parameters'; /** * Function that replaces variables in a text file. @@ -18,7 +18,7 @@ export const VARIABLE_PARAMETERS_NAME = "parameters"; export function replaceVariablesInYaml( input: string, variables: ObjByString, - extraErrorMessage = "" + extraErrorMessage = '', ): string { let contents = input; // Replace variables @@ -34,7 +34,7 @@ export function replaceVariablesInYaml( throw new Error( `Found variable that was not set: ${name}. Make sure you are passing all parameters through the --${VARIABLE_PARAMETERS_NAME}=${name}:value-for-param flag. -${extraErrorMessage}` +${extraErrorMessage}`, ); } @@ -51,16 +51,16 @@ ${extraErrorMessage}` */ export function readTranscendYaml( filePath: string, - variables: ObjByString = {} + variables: ObjByString = {}, ): TranscendInput { // Read in contents - const fileContents = readFileSync(filePath, "utf-8"); + const fileContents = readFileSync(filePath, 'utf-8'); // Replace variables const replacedVariables = replaceVariablesInYaml( fileContents, variables, - `Also check that there are no extra variables defined in your yaml: ${filePath}` + `Also check that there are no extra variables defined in your yaml: ${filePath}`, ); // Validate shape @@ -75,7 +75,7 @@ export function readTranscendYaml( */ export function writeTranscendYaml( filePath: string, - input: TranscendInput + input: TranscendInput, ): void { writeFileSync(filePath, yaml.dump(decodeCodec(TranscendInput, input))); } diff --git a/src/lib/requests/approvePrivacyRequests.ts b/src/lib/requests/approvePrivacyRequests.ts index bfe2eebe..dd331f2e 100644 --- a/src/lib/requests/approvePrivacyRequests.ts +++ b/src/lib/requests/approvePrivacyRequests.ts @@ -2,19 +2,19 @@ import { RequestAction, RequestOrigin, RequestStatus, -} from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +} from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { APPROVE_PRIVACY_REQUEST, buildTranscendGraphQLClient, fetchAllRequests, makeGraphQLRequest, UPDATE_PRIVACY_REQUEST, -} from "../graphql"; +} from '../graphql'; /** * Approve a set of privacy requests @@ -57,7 +57,7 @@ export async function approvePrivacyRequests({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Pull in the requests @@ -98,7 +98,7 @@ export async function approvePrivacyRequests({ input: { requestId: requestToApprove.id }, }); } catch (error) { - if (error.message.includes("Request must be in an approving state,")) { + if (error.message.includes('Request must be in an approving state,')) { skipped += 1; } } @@ -106,7 +106,7 @@ export async function approvePrivacyRequests({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -119,8 +119,8 @@ export async function approvePrivacyRequests({ colors.green( `Successfully approved ${total} requests in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); return allRequests.length; } diff --git a/src/lib/requests/bulkRestartRequests.ts b/src/lib/requests/bulkRestartRequests.ts index b022421b..07a04e91 100644 --- a/src/lib/requests/bulkRestartRequests.ts +++ b/src/lib/requests/bulkRestartRequests.ts @@ -1,22 +1,22 @@ -import { join } from "node:path"; -import { PersistedState } from "@transcend-io/persisted-state"; -import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import * as t from "io-ts"; -import { difference } from "lodash-es"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { join } from 'node:path'; +import { PersistedState } from '@transcend-io/persisted-state'; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import * as t from 'io-ts'; +import { difference } from 'lodash-es'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, createSombraGotInstance, fetchAllRequestIdentifiers, fetchAllRequests, -} from "../graphql"; -import { SuccessfulRequest } from "./constants"; -import { extractClientError } from "./extractClientError"; -import { restartPrivacyRequest } from "./restartPrivacyRequest"; +} from '../graphql'; +import { SuccessfulRequest } from './constants'; +import { extractClientError } from './extractClientError'; +import { restartPrivacyRequest } from './restartPrivacyRequest'; /** Minimal state we need to keep a list of requests */ const ErrorRequest = t.intersection([ @@ -96,13 +96,13 @@ export async function bulkRestartRequests({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Create a new state file to store the requests from this run const cacheFile = join( requestReceiptFolder, - `tr-request-restart-${new Date().toISOString()}` + `tr-request-restart-${new Date().toISOString()}`, ); const state = new PersistedState(cacheFile, CachedRequestState, { restartedRequests: [], @@ -122,33 +122,33 @@ export async function bulkRestartRequests({ createdAtAfter, }); const requests = allRequests.filter( - (request) => new Date(request.createdAt) < createdAt + (request) => new Date(request.createdAt) < createdAt, ); logger.info(`Found ${requests.length} requests to process`); if (copyIdentifiers) { - logger.info("copyIdentifiers detected - All Identifiers will be copied."); + logger.info('copyIdentifiers detected - All Identifiers will be copied.'); } if (sendEmailReceipt) { - logger.info("sendEmailReceipt detected - Email receipts will be sent."); + logger.info('sendEmailReceipt detected - Email receipts will be sent.'); } if (skipWaitingPeriod) { - logger.info("skipWaitingPeriod detected - Waiting period will be skipped."); + logger.info('skipWaitingPeriod detected - Waiting period will be skipped.'); } // Validate request IDs if (requestIds.length > 0 && requestIds.length !== requests.length) { const missingRequests = difference( requestIds, - requests.map(({ id }) => id) + requests.map(({ id }) => id), ); if (missingRequests.length > 0) { logger.error( colors.red( `Failed to find the following requests by ID: ${missingRequests.join( - "," - )}.` - ) + ',', + )}.`, + ), ); process.exit(1); } @@ -185,11 +185,11 @@ export async function bulkRestartRequests({ skipWaitingPeriod, sendEmailReceipt, emailIsVerified, - } + }, ); // Cache successful upload - const restartedRequests = state.getValue("restartedRequests"); + const restartedRequests = state.getValue('restartedRequests'); restartedRequests.push({ id: requestResponse.id, link: requestResponse.link, @@ -197,16 +197,16 @@ export async function bulkRestartRequests({ coreIdentifier: requestResponse.coreIdentifier, attemptedAt: new Date().toISOString(), }); - await state.setValue(restartedRequests, "restartedRequests"); + await state.setValue(restartedRequests, 'restartedRequests'); } catch (error) { const message = `${error.message} - ${JSON.stringify( error.response?.body, null, - 2 + 2, )}`; const clientError = extractClientError(message); - const failingRequests = state.getValue("failingRequests"); + const failingRequests = state.getValue('failingRequests'); failingRequests.push({ id: request.id, link: request.link, @@ -215,12 +215,12 @@ export async function bulkRestartRequests({ attemptedAt: new Date().toISOString(), error: clientError || message, }); - await state.setValue(failingRequests, "failingRequests"); + await state.setValue(failingRequests, 'failingRequests'); } total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -230,17 +230,17 @@ export async function bulkRestartRequests({ // Log completion time logger.info( colors.green( - `Completed restarting of requests in "${totalTime / 1000}" seconds.` - ) + `Completed restarting of requests in "${totalTime / 1000}" seconds.`, + ), ); // Log errors - if (state.getValue("failingRequests").length > 0) { + if (state.getValue('failingRequests').length > 0) { logger.error( colors.red( - `Encountered "${state.getValue("failingRequests").length}" errors. ` + - `See "${cacheFile}" to review the error messages and inputs.` - ) + `Encountered "${state.getValue('failingRequests').length}" errors. ` + + `See "${cacheFile}" to review the error messages and inputs.`, + ), ); process.exit(1); } diff --git a/src/lib/requests/bulkRetryEnrichers.ts b/src/lib/requests/bulkRetryEnrichers.ts index 7e83e71f..d854fb35 100644 --- a/src/lib/requests/bulkRetryEnrichers.ts +++ b/src/lib/requests/bulkRetryEnrichers.ts @@ -2,19 +2,19 @@ import { RequestAction, RequestEnricherStatus, RequestStatus, -} from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { difference } from "lodash-es"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +} from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { difference } from 'lodash-es'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, fetchAllRequestEnrichers, fetchAllRequests, retryRequestEnricher, -} from "../graphql"; +} from '../graphql'; /** * Restart a bunch of request enrichers @@ -56,13 +56,13 @@ export async function bulkRetryEnrichers({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Find all requests made before createdAt that are in a removing data state const client = buildTranscendGraphQLClient(transcendUrl, auth); - logger.info(colors.magenta("Fetching requests to restart...")); + logger.info(colors.magenta('Fetching requests to restart...')); const requests = await fetchAllRequests(client, { actions: requestActions, @@ -78,15 +78,15 @@ export async function bulkRetryEnrichers({ if (requestIds.length > 0 && requestIds.length !== requests.length) { const missingRequests = difference( requestIds, - requests.map(({ id }) => id) + requests.map(({ id }) => id), ); if (missingRequests.length > 0) { logger.error( colors.red( `Failed to find the following requests by ID: ${missingRequests.join( - "," - )}.` - ) + ',', + )}.`, + ), ); process.exit(1); } @@ -105,7 +105,7 @@ export async function bulkRetryEnrichers({ const requestEnrichersToRestart = requestEnrichers.filter( (requestEnricher) => requestEnricher.enricher.id === enricherId && - requestEnricherStatuses.includes(requestEnricher.status) + requestEnricherStatuses.includes(requestEnricher.status), ); await map(requestEnrichersToRestart, async (requestEnricher) => { await retryRequestEnricher(client, requestEnricher.id); @@ -116,7 +116,7 @@ export async function bulkRetryEnrichers({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -130,7 +130,7 @@ export async function bulkRetryEnrichers({ requests.length } requests and ${totalRestarted} enrichers in "${ totalTime / 1000 - }" seconds.` - ) + }" seconds.`, + ), ); } diff --git a/src/lib/requests/cancelPrivacyRequests.ts b/src/lib/requests/cancelPrivacyRequests.ts index e3b43a6b..815ff5d5 100644 --- a/src/lib/requests/cancelPrivacyRequests.ts +++ b/src/lib/requests/cancelPrivacyRequests.ts @@ -1,9 +1,9 @@ -import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, CANCEL_PRIVACY_REQUEST, @@ -12,7 +12,7 @@ import { makeGraphQLRequest, Template, UPDATE_PRIVACY_REQUEST, -} from "../graphql"; +} from '../graphql'; /** * Cancel a set of privacy requests @@ -70,7 +70,7 @@ export async function cancelPrivacyRequests({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Grab the template with that title @@ -78,14 +78,14 @@ export async function cancelPrivacyRequests({ if (cancellationTitle) { const matchingTemplates = await fetchAllTemplates( client, - cancellationTitle + cancellationTitle, ); const exactTitleMatch = matchingTemplates.find( - (template) => template.title === cancellationTitle + (template) => template.title === cancellationTitle, ); if (!exactTitleMatch) { throw new Error( - `Failed to find a template with title: "${cancellationTitle}"` + `Failed to find a template with title: "${cancellationTitle}"`, ); } cancelationTemplate = exactTitleMatch; @@ -106,9 +106,9 @@ export async function cancelPrivacyRequests({ `Canceling "${allRequests.length}" requests${ cancelationTemplate ? ` Using template: ${cancelationTemplate.title}` - : "" - }.` - ) + : '' + }.`, + ), ); let total = 0; @@ -146,7 +146,7 @@ export async function cancelPrivacyRequests({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -157,8 +157,8 @@ export async function cancelPrivacyRequests({ colors.green( `Successfully canceled ${total} requests in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); return allRequests.length; } diff --git a/src/lib/requests/constants.ts b/src/lib/requests/constants.ts index 3e785dd8..d917b352 100644 --- a/src/lib/requests/constants.ts +++ b/src/lib/requests/constants.ts @@ -1,44 +1,44 @@ -import { LanguageKey } from "@transcend-io/internationalization"; +import { LanguageKey } from '@transcend-io/internationalization'; import { CompletedRequestStatus, IsoCountryCode, IsoCountrySubdivisionCode, RequestAction, -} from "@transcend-io/privacy-types"; -import { applyEnum, valuesOf } from "@transcend-io/type-utils"; -import * as t from "io-ts"; +} from '@transcend-io/privacy-types'; +import { applyEnum, valuesOf } from '@transcend-io/type-utils'; +import * as t from 'io-ts'; -export const NONE = "[NONE]" as const; -export const BULK_APPLY = "[APPLY VALUE TO ALL ROWS]" as const; -export const BLANK = "" as const; +export const NONE = '[NONE]' as const; +export const BULK_APPLY = '[APPLY VALUE TO ALL ROWS]' as const; +export const BLANK = '' as const; /** These are uploaded at the top level of the request */ -export const IDENTIFIER_BLOCK_LIST = ["email", "coreIdentifier"]; +export const IDENTIFIER_BLOCK_LIST = ['email', 'coreIdentifier']; /** * Column names to map */ export enum ColumnName { /** The title of the email column */ - Email = "email", + Email = 'email', /** The title of the core identifier column */ - CoreIdentifier = "coreIdentifier", + CoreIdentifier = 'coreIdentifier', /** The title of the requestType column */ - RequestType = "requestType", + RequestType = 'requestType', /** The title of the subjectType column */ - SubjectType = "subjectType", + SubjectType = 'subjectType', /** The title of the locale column */ - Locale = "locale", + Locale = 'locale', /** The country */ - Country = "country", + Country = 'country', /** The country sub division */ - CountrySubDivision = "countrySubDivision", + CountrySubDivision = 'countrySubDivision', /** The title of the requestStatus column */ - RequestStatus = "requestStatus", + RequestStatus = 'requestStatus', /** The title of the createdAt column */ - CreatedAt = "createdAt", + CreatedAt = 'createdAt', /** The title of the dataSiloIds column */ - DataSiloIds = "dataSiloIds", + DataSiloIds = 'dataSiloIds', } /** These parameters are required in the Transcend DSR API */ @@ -78,17 +78,17 @@ export const CachedFileState = t.type({ /** Mapping between region and country code */ regionToCountry: t.record( t.string, - valuesOf({ ...IsoCountryCode, [NONE]: NONE }) + valuesOf({ ...IsoCountryCode, [NONE]: NONE }), ), /** Mapping between region and country sub division code */ regionToCountrySubDivision: t.record( t.string, - valuesOf({ ...IsoCountrySubdivisionCode, [NONE]: NONE }) + valuesOf({ ...IsoCountrySubdivisionCode, [NONE]: NONE }), ), /** Mapping between request status in import to Transcend request status */ statusToRequestStatus: t.record( t.string, - valuesOf({ ...CompletedRequestStatus, [NONE]: NONE }) + valuesOf({ ...CompletedRequestStatus, [NONE]: NONE }), ), }); @@ -121,7 +121,7 @@ export const CachedRequestState = t.type({ rowIndex: t.number, coreIdentifier: t.string, attemptedAt: t.string, - }) + }), ), }); diff --git a/src/lib/requests/downloadPrivacyRequestFiles.ts b/src/lib/requests/downloadPrivacyRequestFiles.ts index e5771a85..83fc4ee1 100644 --- a/src/lib/requests/downloadPrivacyRequestFiles.ts +++ b/src/lib/requests/downloadPrivacyRequestFiles.ts @@ -1,20 +1,20 @@ -import { existsSync, mkdirSync, writeFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { APPROVE_PRIVACY_REQUEST, buildTranscendGraphQLClient, createSombraGotInstance, fetchAllRequests, makeGraphQLRequest, -} from "../graphql"; -import { getFileMetadataForPrivacyRequests } from "./getFileMetadataForPrivacyRequests"; -import { streamPrivacyRequestFiles } from "./streamPrivacyRequestFiles"; +} from '../graphql'; +import { getFileMetadataForPrivacyRequests } from './getFileMetadataForPrivacyRequests'; +import { streamPrivacyRequestFiles } from './streamPrivacyRequestFiles'; /** * Download a set of privacy requests to disk @@ -81,14 +81,14 @@ export async function downloadPrivacyRequestFiles({ { sombra, concurrency, - } + }, ); // Start timer for download process const t0 = Date.now(); const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); let total = 0; let totalApproved = 0; @@ -134,7 +134,7 @@ export async function downloadPrivacyRequestFiles({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -145,12 +145,12 @@ export async function downloadPrivacyRequestFiles({ colors.green( `Successfully downloaded ${total} requests in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); if (totalApproved > 0) { logger.info( - colors.green(`Approved ${totalApproved} requests in Transcend.`) + colors.green(`Approved ${totalApproved} requests in Transcend.`), ); } return allRequests.length; diff --git a/src/lib/requests/filterRows.ts b/src/lib/requests/filterRows.ts index 9cb62f30..ac289466 100644 --- a/src/lib/requests/filterRows.ts +++ b/src/lib/requests/filterRows.ts @@ -1,10 +1,10 @@ -import { ObjByString } from "@transcend-io/type-utils"; -import colors from "colors"; -import inquirer from "inquirer"; -import { uniq } from "lodash-es"; -import { logger } from "../../logger"; -import { NONE } from "./constants"; -import { getUniqueValuesForColumn } from "./getUniqueValuesForColumn"; +import { ObjByString } from '@transcend-io/type-utils'; +import colors from 'colors'; +import inquirer from 'inquirer'; +import { uniq } from 'lodash-es'; +import { logger } from '../../logger'; +import { NONE } from './constants'; +import { getUniqueValuesForColumn } from './getUniqueValuesForColumn'; /** * Filter a list of CSV rows by column values @@ -30,10 +30,10 @@ export async function filterRows(rows: ObjByString[]): Promise { filterColumnName: string; }>([ { - name: "filterColumnName", + name: 'filterColumnName', message: `If you need to filter the list of requests to import, choose the column to filter on. Currently ${filteredRows.length} rows.`, - type: "list", + type: 'list', default: columnNames, choices: [NONE, ...columnNames], }, @@ -49,16 +49,16 @@ export async function filterRows(rows: ObjByString[]): Promise { valuesToKeep: string[]; }>([ { - name: "valuesToKeep", - message: "Keep rows matching this value", - type: "checkbox", + name: 'valuesToKeep', + message: 'Keep rows matching this value', + type: 'checkbox', default: columnNames, choices: options, }, ]); filteredRows = filteredRows.filter((request) => - valuesToKeep.includes(request[filterColumnName]) + valuesToKeep.includes(request[filterColumnName]), ); } } diff --git a/src/lib/requests/getFileMetadataForPrivacyRequests.ts b/src/lib/requests/getFileMetadataForPrivacyRequests.ts index c74a98d2..0c856642 100644 --- a/src/lib/requests/getFileMetadataForPrivacyRequests.ts +++ b/src/lib/requests/getFileMetadataForPrivacyRequests.ts @@ -1,12 +1,12 @@ -import { TableEncryptionType } from "@transcend-io/privacy-types"; -import { decodeCodec, valuesOf } from "@transcend-io/type-utils"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import type { Got } from "got"; -import * as t from "io-ts"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { PrivacyRequest } from "../graphql"; +import { TableEncryptionType } from '@transcend-io/privacy-types'; +import { decodeCodec, valuesOf } from '@transcend-io/type-utils'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import type { Got } from 'got'; +import * as t from 'io-ts'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { PrivacyRequest } from '../graphql'; export const IntlMessage = t.type({ /** The message key */ @@ -93,7 +93,7 @@ export type RequestFileMetadataResponse = t.TypeOf< * @returns The number of requests canceled */ export async function getFileMetadataForPrivacyRequests( - requests: Pick[], + requests: Pick[], { sombra, concurrency = 5, @@ -105,10 +105,10 @@ export async function getFileMetadataForPrivacyRequests( limit?: number; /** Concurrency limit for approving */ concurrency?: number; - } -): Promise<[Pick, RequestFileMetadata[]][]> { + }, +): Promise<[Pick, RequestFileMetadata[]][]> { logger.info( - colors.magenta(`Pulling file metadata for ${requests.length} requests`) + colors.magenta(`Pulling file metadata for ${requests.length} requests`), ); // Time duration @@ -116,7 +116,7 @@ export async function getFileMetadataForPrivacyRequests( // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Start timer @@ -127,9 +127,9 @@ export async function getFileMetadataForPrivacyRequests( const results = await map( requests, async ( - requestToDownload + requestToDownload, ): Promise< - [Pick, RequestFileMetadata[]] + [Pick, RequestFileMetadata[]] > => { const localResults: RequestFileMetadata[] = []; @@ -149,7 +149,7 @@ export async function getFileMetadataForPrivacyRequests( limit, offset, }, - } + }, ) .json(); response = decodeCodec(RequestFileMetadataResponse, rawResponse); @@ -163,7 +163,7 @@ export async function getFileMetadataForPrivacyRequests( throw new Error( `Received an error from server: ${ error?.response?.body || error?.message - }` + }`, ); } } @@ -172,7 +172,7 @@ export async function getFileMetadataForPrivacyRequests( progressBar.update(total); return [requestToDownload, localResults]; }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -183,8 +183,8 @@ export async function getFileMetadataForPrivacyRequests( colors.green( `Successfully downloaded file metadata ${requests.length} requests in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); return results; diff --git a/src/lib/requests/getUniqueValuesForColumn.ts b/src/lib/requests/getUniqueValuesForColumn.ts index fb4f56ee..99527b67 100644 --- a/src/lib/requests/getUniqueValuesForColumn.ts +++ b/src/lib/requests/getUniqueValuesForColumn.ts @@ -1,5 +1,5 @@ -import { ObjByString } from "@transcend-io/type-utils"; -import { uniq } from "lodash-es"; +import { ObjByString } from '@transcend-io/type-utils'; +import { uniq } from 'lodash-es'; /** * Return the unique set of values for a column in a CSV @@ -10,7 +10,7 @@ import { uniq } from "lodash-es"; */ export function getUniqueValuesForColumn( rows: ObjByString[], - columnName: string + columnName: string, ): string[] { - return uniq(rows.flatMap((row) => row[columnName] || "")); + return uniq(rows.flatMap((row) => row[columnName] || '')); } diff --git a/src/lib/requests/mapColumnsToAttributes.ts b/src/lib/requests/mapColumnsToAttributes.ts index 3697d0da..fa86032b 100644 --- a/src/lib/requests/mapColumnsToAttributes.ts +++ b/src/lib/requests/mapColumnsToAttributes.ts @@ -1,9 +1,9 @@ -import type { PersistedState } from "@transcend-io/persisted-state"; -import type { GraphQLClient } from "graphql-request"; -import inquirer from "inquirer"; -import { AttributeKey } from "../graphql"; -import { CachedFileState } from "./constants"; -import { fuzzyMatchColumns } from "./fuzzyMatchColumns"; +import type { PersistedState } from '@transcend-io/persisted-state'; +import type { GraphQLClient } from 'graphql-request'; +import inquirer from 'inquirer'; +import { AttributeKey } from '../graphql'; +import { CachedFileState } from './constants'; +import { fuzzyMatchColumns } from './fuzzyMatchColumns'; /** * Mapping from attribute name to request input parameter @@ -25,11 +25,11 @@ export async function mapColumnsToAttributes( client: GraphQLClient, columnNames: string[], state: PersistedState, - requestAttributeKeys: AttributeKey[] + requestAttributeKeys: AttributeKey[], ): Promise { // Determine the columns that should be mapped const columnQuestions = requestAttributeKeys.filter( - ({ name }) => !state.getValue("attributeNames", name) + ({ name }) => !state.getValue('attributeNames', name), ); // Skip mapping when everything is mapped @@ -43,20 +43,20 @@ export async function mapColumnsToAttributes( return { name, message: `Choose the column that will be used to map in the attribute: ${name}`, - type: "list", + type: 'list', default: matches[0], choices: matches, }; - }) + }), ); await Promise.all( Object.entries(attributeNameMap).map(([k, v]) => - state.setValue(v, "attributeNames", k) - ) + state.setValue(v, 'attributeNames', k), + ), ); return { - ...state.getValue("attributeNames"), + ...state.getValue('attributeNames'), ...attributeNameMap, }; } diff --git a/src/lib/requests/mapColumnsToIdentifiers.ts b/src/lib/requests/mapColumnsToIdentifiers.ts index 214cc888..d88deaea 100644 --- a/src/lib/requests/mapColumnsToIdentifiers.ts +++ b/src/lib/requests/mapColumnsToIdentifiers.ts @@ -1,9 +1,9 @@ -import type { PersistedState } from "@transcend-io/persisted-state"; -import type { GraphQLClient } from "graphql-request"; -import inquirer from "inquirer"; -import { INITIALIZER, Initializer, makeGraphQLRequest } from "../graphql"; -import { CachedFileState, IDENTIFIER_BLOCK_LIST } from "./constants"; -import { fuzzyMatchColumns } from "./fuzzyMatchColumns"; +import type { PersistedState } from '@transcend-io/persisted-state'; +import type { GraphQLClient } from 'graphql-request'; +import inquirer from 'inquirer'; +import { INITIALIZER, Initializer, makeGraphQLRequest } from '../graphql'; +import { CachedFileState, IDENTIFIER_BLOCK_LIST } from './constants'; +import { fuzzyMatchColumns } from './fuzzyMatchColumns'; /** * Mapping from identifier name to request input parameter @@ -23,7 +23,7 @@ export type IdentifierNameMap = Record; export async function mapColumnsToIdentifiers( client: GraphQLClient, columnNames: string[], - state: PersistedState + state: PersistedState, ): Promise { // Grab the initializer const { initializer } = await makeGraphQLRequest<{ @@ -34,8 +34,8 @@ export async function mapColumnsToIdentifiers( // Determine the columns that should be mapped const columnQuestions = initializer.identifiers.filter( ({ name }) => - !state.getValue("identifierNames", name) && - !IDENTIFIER_BLOCK_LIST.includes(name) + !state.getValue('identifierNames', name) && + !IDENTIFIER_BLOCK_LIST.includes(name), ); // Skip mapping when everything is mapped @@ -49,20 +49,20 @@ export async function mapColumnsToIdentifiers( return { name, message: `Choose the column that will be used to map in the identifier: ${name}`, - type: "list", + type: 'list', default: matches[0], choices: matches, }; - }) + }), ); await Promise.all( Object.entries(identifierNameMap).map(([k, v]) => - state.setValue(v, "identifierNames", k) - ) + state.setValue(v, 'identifierNames', k), + ), ); return { - ...state.getValue("identifierNames"), + ...state.getValue('identifierNames'), ...identifierNameMap, }; } diff --git a/src/lib/requests/mapCsvColumnsToApi.ts b/src/lib/requests/mapCsvColumnsToApi.ts index 76c1b0d1..11547c04 100644 --- a/src/lib/requests/mapCsvColumnsToApi.ts +++ b/src/lib/requests/mapCsvColumnsToApi.ts @@ -1,14 +1,14 @@ -import type { PersistedState } from "@transcend-io/persisted-state"; -import { getEntries, getValues } from "@transcend-io/type-utils"; -import inquirer from "inquirer"; -import { startCase } from "lodash-es"; +import type { PersistedState } from '@transcend-io/persisted-state'; +import { getEntries, getValues } from '@transcend-io/type-utils'; +import inquirer from 'inquirer'; +import { startCase } from 'lodash-es'; import { CachedFileState, CAN_APPLY_IN_BULK, ColumnName, IS_REQUIRED, -} from "./constants"; -import { fuzzyMatchColumns } from "./fuzzyMatchColumns"; +} from './constants'; +import { fuzzyMatchColumns } from './fuzzyMatchColumns'; /** * Mapping from column name to request input parameter @@ -24,11 +24,11 @@ export type ColumnNameMap = Partial>; */ export async function mapCsvColumnsToApi( columnNames: string[], - state: PersistedState + state: PersistedState, ): Promise { // Determine the columns that should be mapped const columnQuestions = getValues(ColumnName).filter( - (name) => !state.getValue("columnNames", name) + (name) => !state.getValue('columnNames', name), ); // Skip mapping when everything is mapped @@ -38,27 +38,27 @@ export async function mapCsvColumnsToApi( : // prompt questions to map columns await inquirer.prompt>>( columnQuestions.map((name) => { - const field = startCase(name.replace("ColumnName", "")); + const field = startCase(name.replace('ColumnName', '')); const matches = fuzzyMatchColumns( columnNames, field, IS_REQUIRED[name], - !!CAN_APPLY_IN_BULK[name] + !!CAN_APPLY_IN_BULK[name], ); return { name, message: `Choose the column that will be used to map in the field: ${field}`, - type: "list", + type: 'list', default: matches[0], choices: matches, }; - }) + }), ); await Promise.all( getEntries(columnNameMap).map(([k, v]) => - state.setValue(v, "columnNames", k) - ) + state.setValue(v, 'columnNames', k), + ), ); return columnNameMap; } diff --git a/src/lib/requests/mapCsvRowsToRequestInputs.ts b/src/lib/requests/mapCsvRowsToRequestInputs.ts index 125ad974..da8ebefa 100644 --- a/src/lib/requests/mapCsvRowsToRequestInputs.ts +++ b/src/lib/requests/mapCsvRowsToRequestInputs.ts @@ -1,5 +1,5 @@ -import { LanguageKey } from "@transcend-io/internationalization"; -import type { PersistedState } from "@transcend-io/persisted-state"; +import { LanguageKey } from '@transcend-io/internationalization'; +import type { PersistedState } from '@transcend-io/persisted-state'; import { CompletedRequestStatus, IdentifierType, @@ -7,23 +7,23 @@ import { IsoCountrySubdivisionCode, NORMALIZE_PHONE_NUMBER, RequestAction, -} from "@transcend-io/privacy-types"; -import { ObjByString, valuesOf } from "@transcend-io/type-utils"; -import * as t from "io-ts"; -import { DateFromISOString } from "io-ts-types"; -import { AttributeKey } from "../graphql"; +} from '@transcend-io/privacy-types'; +import { ObjByString, valuesOf } from '@transcend-io/type-utils'; +import * as t from 'io-ts'; +import { DateFromISOString } from 'io-ts-types'; +import { AttributeKey } from '../graphql'; import { BLANK, BULK_APPLY, CachedFileState, ColumnName, NONE, -} from "./constants"; -import { AttributeNameMap } from "./mapColumnsToAttributes"; -import { IdentifierNameMap } from "./mapColumnsToIdentifiers"; -import { ColumnNameMap } from "./mapCsvColumnsToApi"; -import { ParsedAttributeInput } from "./parseAttributesFromString"; -import { splitCsvToList } from "./splitCsvToList"; +} from './constants'; +import { AttributeNameMap } from './mapColumnsToAttributes'; +import { IdentifierNameMap } from './mapColumnsToIdentifiers'; +import { ColumnNameMap } from './mapCsvColumnsToApi'; +import { ParsedAttributeInput } from './parseAttributesFromString'; +import { splitCsvToList } from './splitCsvToList'; /** * Shape of additional identifiers @@ -42,8 +42,8 @@ export const AttestedExtraIdentifiers = t.record( /** Name of identifier - option for non-custom identifier types */ name: t.string, }), - ]) - ) + ]), + ), ); /** Type override */ @@ -96,7 +96,7 @@ export type PrivacyRequestInput = t.TypeOf; export function normalizeIdentifierValue( identifierValue: string, identifierType: IdentifierType, - defaultPhoneCountryCode: string + defaultPhoneCountryCode: string, ): string { // Lowercase email if (identifierType === IdentifierType.Email) { @@ -106,17 +106,17 @@ export function normalizeIdentifierValue( // Normalize phone number if (identifierType === IdentifierType.Phone) { const normalized = identifierValue - .replace(NORMALIZE_PHONE_NUMBER, "") - .replaceAll(/[()]/g, "") - .replaceAll(/[–]/g, "") - .replaceAll(/[:]/g, "") - .replaceAll(/[‭‬]/g, "") - .replaceAll(/[A-Za-z]/g, ""); + .replace(NORMALIZE_PHONE_NUMBER, '') + .replaceAll(/[()]/g, '') + .replaceAll(/[–]/g, '') + .replaceAll(/[:]/g, '') + .replaceAll(/[‭‬]/g, '') + .replaceAll(/[A-Za-z]/g, ''); return normalized - ? normalized.startsWith("+") + ? normalized.startsWith('+') ? normalized : `+${defaultPhoneCountryCode}${normalized}` - : ""; + : ''; } return identifierValue; } @@ -139,7 +139,7 @@ export function mapCsvRowsToRequestInputs( identifierNameMap, attributeNameMap, requestAttributeKeys, - defaultPhoneCountryCode = "1", // US + defaultPhoneCountryCode = '1', // US }: { /** Default country code */ defaultPhoneCountryCode?: string; @@ -151,23 +151,23 @@ export function mapCsvRowsToRequestInputs( attributeNameMap: AttributeNameMap; /** Request attribute keys */ requestAttributeKeys: AttributeKey[]; - } + }, ): [Record, PrivacyRequestInput][] { // map the CSV to request input const getMappedName = (attribute: ColumnName): string => - state.getValue("columnNames", attribute) || columnNameMap[attribute]!; + state.getValue('columnNames', attribute) || columnNameMap[attribute]!; return requestInputs.map( (input): [Record, PrivacyRequestInput] => { // The extra identifiers to upload for this request const attestedExtraIdentifiers: AttestedExtraIdentifiers = {}; for (const [identifierName, columnName] of Object.entries( - identifierNameMap + identifierNameMap, ) // filter out skipped identifiers .filter(([, columnName]) => columnName !== NONE)) { // Determine the identifier type being specified const identifierType = Object.values(IdentifierType).includes( - identifierName as any // eslint-disable-line @typescript-eslint/no-explicit-any + identifierName as any, // eslint-disable-line @typescript-eslint/no-explicit-any ) ? (identifierName as IdentifierType) : IdentifierType.Custom; @@ -178,7 +178,7 @@ export function mapCsvRowsToRequestInputs( const normalized = normalizeIdentifierValue( identifierValue, identifierType, - defaultPhoneCountryCode + defaultPhoneCountryCode, ); if (normalized) { // Initialize @@ -206,8 +206,8 @@ export function mapCsvRowsToRequestInputs( // Add the attribute const isMulti = requestAttributeKeys.find( - (attribute) => attribute.name === attributeName - )?.type === "MULTI_SELECT"; + (attribute) => attribute.name === attributeName, + )?.type === 'MULTI_SELECT'; attributes.push({ values: isMulti ? splitCsvToList(attributeValueString) @@ -228,24 +228,24 @@ export function mapCsvRowsToRequestInputs( coreIdentifier: input[getMappedName(ColumnName.CoreIdentifier)], requestType: requestTypeColumn === BULK_APPLY - ? state.getValue("requestTypeToRequestAction", BLANK) + ? state.getValue('requestTypeToRequestAction', BLANK) : state.getValue( - "requestTypeToRequestAction", - input[requestTypeColumn] + 'requestTypeToRequestAction', + input[requestTypeColumn], ), subjectType: dataSubjectTypeColumn === BULK_APPLY - ? state.getValue("subjectTypeToSubjectName", BLANK) + ? state.getValue('subjectTypeToSubjectName', BLANK) : state.getValue( - "subjectTypeToSubjectName", - input[dataSubjectTypeColumn] + 'subjectTypeToSubjectName', + input[dataSubjectTypeColumn], ), ...(getMappedName(ColumnName.Locale) !== NONE && input[getMappedName(ColumnName.Locale)] ? { locale: state.getValue( - "languageToLocale", - input[getMappedName(ColumnName.Locale)] + 'languageToLocale', + input[getMappedName(ColumnName.Locale)], ), } : {}), @@ -253,8 +253,8 @@ export function mapCsvRowsToRequestInputs( input[getMappedName(ColumnName.Country)] ? { country: state.getValue( - "regionToCountry", - input[getMappedName(ColumnName.Country)] + 'regionToCountry', + input[getMappedName(ColumnName.Country)], ) as IsoCountryCode, } : {}), @@ -262,21 +262,21 @@ export function mapCsvRowsToRequestInputs( input[getMappedName(ColumnName.CountrySubDivision)] ? { countrySubDivision: state.getValue( - "regionToCountrySubDivision", - input[getMappedName(ColumnName.CountrySubDivision)] + 'regionToCountrySubDivision', + input[getMappedName(ColumnName.CountrySubDivision)], ) as IsoCountrySubdivisionCode, } : {}), ...(getMappedName(ColumnName.RequestStatus) !== NONE && state.getValue( - "statusToRequestStatus", - input[getMappedName(ColumnName.RequestStatus)] + 'statusToRequestStatus', + input[getMappedName(ColumnName.RequestStatus)], ) !== NONE && input[getMappedName(ColumnName.RequestStatus)] ? { status: state.getValue( - "statusToRequestStatus", - input[getMappedName(ColumnName.RequestStatus)] + 'statusToRequestStatus', + input[getMappedName(ColumnName.RequestStatus)], ) as CompletedRequestStatus, } : {}), @@ -290,12 +290,12 @@ export function mapCsvRowsToRequestInputs( input[getMappedName(ColumnName.DataSiloIds)] ? { dataSiloIds: splitCsvToList( - input[getMappedName(ColumnName.DataSiloIds)] + input[getMappedName(ColumnName.DataSiloIds)], ), } : {}), }, ]; - } + }, ); } diff --git a/src/lib/requests/mapEnumValues.ts b/src/lib/requests/mapEnumValues.ts index 4535a8c1..d53b4388 100644 --- a/src/lib/requests/mapEnumValues.ts +++ b/src/lib/requests/mapEnumValues.ts @@ -1,7 +1,7 @@ -import { apply, ObjByString } from "@transcend-io/type-utils"; -import inquirer from "inquirer"; -import autoCompletePrompt from "inquirer-autocomplete-prompt"; -import { fuzzySearch } from "./fuzzyMatchColumns"; +import { apply, ObjByString } from '@transcend-io/type-utils'; +import inquirer from 'inquirer'; +import autoCompletePrompt from 'inquirer-autocomplete-prompt'; +import { fuzzySearch } from './fuzzyMatchColumns'; /** * Map a set of inputs to a set of outputs @@ -14,12 +14,12 @@ import { fuzzySearch } from "./fuzzyMatchColumns"; export async function mapEnumValues( csvInputs: string[], expectedOutputs: TValue[], - cache: Record + cache: Record, ): Promise> { - inquirer.registerPrompt("autocomplete", autoCompletePrompt); + inquirer.registerPrompt('autocomplete', autoCompletePrompt); const inputs = csvInputs - .map((item) => item || "") + .map((item) => item || '') .filter((value) => !cache[value]); if (inputs.length === 0) { return cache; @@ -28,20 +28,20 @@ export async function mapEnumValues( inputs.map((value) => ({ name: value, message: `Map value of: ${value}`, - type: "autocomplete", + type: 'autocomplete', default: expectedOutputs.find((x) => fuzzySearch(value, x)), source: (answersSoFar: ObjByString, input: string) => input ? expectedOutputs.filter( - (x) => typeof x === "string" && fuzzySearch(input, x) + (x) => typeof x === 'string' && fuzzySearch(input, x), ) : expectedOutputs, - })) + })), ); return { ...cache, ...apply(result, (r) => - typeof r === "string" ? r : (Object.values(r)[0] as TValue) + typeof r === 'string' ? r : (Object.values(r)[0] as TValue), ), }; } diff --git a/src/lib/requests/mapRequestEnumValues.ts b/src/lib/requests/mapRequestEnumValues.ts index db5e93f9..baf2ee61 100644 --- a/src/lib/requests/mapRequestEnumValues.ts +++ b/src/lib/requests/mapRequestEnumValues.ts @@ -1,20 +1,20 @@ -import { LanguageKey } from "@transcend-io/internationalization"; -import type { PersistedState } from "@transcend-io/persisted-state"; +import { LanguageKey } from '@transcend-io/internationalization'; +import type { PersistedState } from '@transcend-io/persisted-state'; import { CompletedRequestStatus, IsoCountryCode, IsoCountrySubdivisionCode, RequestAction, -} from "@transcend-io/privacy-types"; -import { ObjByString } from "@transcend-io/type-utils"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { logger } from "../../logger"; -import { DATA_SUBJECTS, DataSubject, makeGraphQLRequest } from "../graphql"; -import { CachedFileState, ColumnName, NONE } from "./constants"; -import { getUniqueValuesForColumn } from "./getUniqueValuesForColumn"; -import { ColumnNameMap } from "./mapCsvColumnsToApi"; -import { mapEnumValues } from "./mapEnumValues"; +} from '@transcend-io/privacy-types'; +import { ObjByString } from '@transcend-io/type-utils'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { logger } from '../../logger'; +import { DATA_SUBJECTS, DataSubject, makeGraphQLRequest } from '../graphql'; +import { CachedFileState, ColumnName, NONE } from './constants'; +import { getUniqueValuesForColumn } from './getUniqueValuesForColumn'; +import { ColumnNameMap } from './mapCsvColumnsToApi'; +import { mapEnumValues } from './mapEnumValues'; /** * Map the values in a CSV to the enum values in Transcend @@ -34,11 +34,11 @@ export async function mapRequestEnumValues( state: PersistedState; /** Mapping of column names */ columnNameMap: ColumnNameMap; - } + }, ): Promise { // Get mapped value const getMappedName = (attribute: ColumnName): string => - state.getValue("columnNames", attribute) || columnNameMap[attribute]!; + state.getValue('columnNames', attribute) || columnNameMap[attribute]!; // Fetch all data subjects in the organization const { internalSubjects } = await makeGraphQLRequest<{ @@ -48,43 +48,43 @@ export async function mapRequestEnumValues( // Map RequestAction logger.info( - colors.magenta("Determining mapping of columns for request action") + colors.magenta('Determining mapping of columns for request action'), ); const requestTypeToRequestAction: Record = await mapEnumValues( getUniqueValuesForColumn(requests, getMappedName(ColumnName.RequestType)), Object.values(RequestAction), - state.getValue("requestTypeToRequestAction") + state.getValue('requestTypeToRequestAction'), ); await state.setValue( requestTypeToRequestAction, - "requestTypeToRequestAction" + 'requestTypeToRequestAction', ); // Map data subject type - logger.info(colors.magenta("Determining mapping of columns for subject")); + logger.info(colors.magenta('Determining mapping of columns for subject')); const subjectTypeToSubjectName: Record = await mapEnumValues( getUniqueValuesForColumn(requests, getMappedName(ColumnName.SubjectType)), internalSubjects.map(({ type }) => type), - state.getValue("subjectTypeToSubjectName") + state.getValue('subjectTypeToSubjectName'), ); - await state.setValue(subjectTypeToSubjectName, "subjectTypeToSubjectName"); + await state.setValue(subjectTypeToSubjectName, 'subjectTypeToSubjectName'); // Map locale - logger.info(colors.magenta("Determining mapping of columns for locale")); + logger.info(colors.magenta('Determining mapping of columns for locale')); const languageToLocale: Record = await mapEnumValues( getUniqueValuesForColumn(requests, getMappedName(ColumnName.Locale)), Object.values(LanguageKey), - state.getValue("languageToLocale") + state.getValue('languageToLocale'), ); - await state.setValue(languageToLocale, "languageToLocale"); + await state.setValue(languageToLocale, 'languageToLocale'); logger.info( - colors.magenta("Determining mapping of columns for request status") + colors.magenta('Determining mapping of columns for request status'), ); // Map request status logger.info( - colors.magenta("Determining mapping of columns for request status") + colors.magenta('Determining mapping of columns for request status'), ); const requestStatusColumn = getMappedName(ColumnName.RequestStatus); const statusToRequestStatus: Record< @@ -96,12 +96,12 @@ export async function mapRequestEnumValues( : await mapEnumValues( getUniqueValuesForColumn(requests, requestStatusColumn), [...Object.values(CompletedRequestStatus), NONE], - state.getValue("statusToRequestStatus") + state.getValue('statusToRequestStatus'), ); - await state.setValue(statusToRequestStatus, "statusToRequestStatus"); + await state.setValue(statusToRequestStatus, 'statusToRequestStatus'); // Map country - logger.info(colors.magenta("Determining mapping of columns for country")); + logger.info(colors.magenta('Determining mapping of columns for country')); const countryColumn = getMappedName(ColumnName.Country); const regionToCountry: Record = countryColumn === NONE @@ -109,13 +109,13 @@ export async function mapRequestEnumValues( : await mapEnumValues( getUniqueValuesForColumn(requests, countryColumn), [...Object.values(IsoCountryCode), NONE], - state.getValue("regionToCountry") + state.getValue('regionToCountry'), ); - await state.setValue(regionToCountry, "regionToCountry"); + await state.setValue(regionToCountry, 'regionToCountry'); // Map country sub division logger.info( - colors.magenta("Determining mapping of columns for country sub division") + colors.magenta('Determining mapping of columns for country sub division'), ); const countrySubDivisionColumn = getMappedName(ColumnName.CountrySubDivision); const regionToCountrySubDivision: Record< @@ -127,10 +127,10 @@ export async function mapRequestEnumValues( : await mapEnumValues( getUniqueValuesForColumn(requests, countrySubDivisionColumn), [...Object.values(IsoCountrySubdivisionCode), NONE], - state.getValue("regionToCountrySubDivision") + state.getValue('regionToCountrySubDivision'), ); await state.setValue( regionToCountrySubDivision, - "regionToCountrySubDivision" + 'regionToCountrySubDivision', ); } diff --git a/src/lib/requests/markSilentPrivacyRequests.ts b/src/lib/requests/markSilentPrivacyRequests.ts index 4a215125..eb6dc7e7 100644 --- a/src/lib/requests/markSilentPrivacyRequests.ts +++ b/src/lib/requests/markSilentPrivacyRequests.ts @@ -1,15 +1,15 @@ -import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, fetchAllRequests, makeGraphQLRequest, UPDATE_PRIVACY_REQUEST, -} from "../graphql"; +} from '../graphql'; /** * Mark a set of privacy requests to be in silent mode @@ -61,7 +61,7 @@ export async function markSilentPrivacyRequests({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Pull in the requests @@ -76,7 +76,7 @@ export async function markSilentPrivacyRequests({ // Notify Transcend logger.info( - colors.magenta(`Marking "${allRequests.length}" as silent mode.`) + colors.magenta(`Marking "${allRequests.length}" as silent mode.`), ); let total = 0; @@ -94,7 +94,7 @@ export async function markSilentPrivacyRequests({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -105,8 +105,8 @@ export async function markSilentPrivacyRequests({ colors.green( `Successfully marked ${total} requests as silent mode in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); return allRequests.length; } diff --git a/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts b/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts index 2fc43ef3..9da01de9 100644 --- a/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts +++ b/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts @@ -1,16 +1,16 @@ -import { RequestAction } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { RequestAction } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, fetchAllRequests, fetchAllTemplates, makeGraphQLRequest, NOTIFY_ADDITIONAL_TIME, -} from "../graphql"; +} from '../graphql'; /** * Mark a set of privacy requests to be in silent mode. @@ -27,7 +27,7 @@ export async function notifyPrivacyRequestsAdditionalTime({ days = 45, daysLeft = 10, createdAtAfter, - emailTemplate = "Additional Time Needed", + emailTemplate = 'Additional Time Needed', concurrency = 100, transcendUrl = DEFAULT_TRANSCEND_API, }: { @@ -63,13 +63,13 @@ export async function notifyPrivacyRequestsAdditionalTime({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Grab the template with that title const matchingTemplates = await fetchAllTemplates(client, emailTemplate); const exactTemplateMatch = matchingTemplates.find( - (template) => template.title === emailTemplate + (template) => template.title === emailTemplate, ); if (!exactTemplateMatch) { throw new Error(`Failed to find a template with title: "${emailTemplate}"`); @@ -88,15 +88,15 @@ export async function notifyPrivacyRequestsAdditionalTime({ // Filter requests by daysLeft allRequests = allRequests.filter( (request) => - typeof request.daysRemaining === "number" && - request.daysRemaining < daysLeft + typeof request.daysRemaining === 'number' && + request.daysRemaining < daysLeft, ); // Notify Transcend logger.info( colors.magenta( - `Notifying "${allRequests.length}" that more time is needed.` - ) + `Notifying "${allRequests.length}" that more time is needed.`, + ), ); let total = 0; @@ -116,7 +116,7 @@ export async function notifyPrivacyRequestsAdditionalTime({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -127,8 +127,8 @@ export async function notifyPrivacyRequestsAdditionalTime({ colors.green( `Successfully marked ${total} requests as silent mode in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); return allRequests.length; } diff --git a/src/lib/requests/pullPrivacyRequests.ts b/src/lib/requests/pullPrivacyRequests.ts index 9cf43a17..5ba70bbb 100644 --- a/src/lib/requests/pullPrivacyRequests.ts +++ b/src/lib/requests/pullPrivacyRequests.ts @@ -1,9 +1,9 @@ -import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { groupBy } from "lodash-es"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { groupBy } from 'lodash-es'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, createSombraGotInstance, @@ -11,7 +11,7 @@ import { fetchAllRequests, PrivacyRequest, RequestIdentifier, -} from "../graphql"; +} from '../graphql'; export interface ExportedPrivacyRequest extends PrivacyRequest { /** Request identifiers */ @@ -67,13 +67,13 @@ export async function pullPrivacyRequests({ const sombra = await createSombraGotInstance(transcendUrl, auth, sombraAuth); // Log date range - let dateRange = ""; + let dateRange = ''; if (createdAtBefore) { dateRange += ` before ${createdAtBefore.toISOString()}`; } if (createdAtAfter) { dateRange += `${ - dateRange ? ", and" : "" + dateRange ? ', and' : '' } after ${createdAtAfter.toISOString()}`; } @@ -83,9 +83,9 @@ export async function pullPrivacyRequests({ `${ actions.length > 0 ? `Pulling requests of type "${actions.join('" , "')}"` - : "Pulling all requests" - }${dateRange}` - ) + : 'Pulling all requests' + }${dateRange}`, + ), ); // fetch the requests @@ -107,7 +107,7 @@ export async function pullPrivacyRequests({ sombra, { requestId: request.id, - } + }, ); return { ...request, @@ -116,11 +116,11 @@ export async function pullPrivacyRequests({ }, { concurrency: pageLimit, - } + }, ); logger.info( - colors.magenta(`Pulled ${requestsWithRequestIdentifiers.length} requests`) + colors.magenta(`Pulled ${requestsWithRequestIdentifiers.length} requests`), ); // Write out to CSV @@ -144,32 +144,35 @@ export async function pullPrivacyRequests({ coreIdentifier, ...request }) => ({ - "Request ID": id, - "Created At": createdAt, + 'Request ID': id, + 'Created At': createdAt, Email: email, - "Core Identifier": coreIdentifier, - "Request Type": type, - "Data Subject Type": subjectType, + 'Core Identifier': coreIdentifier, + 'Request Type': type, + 'Data Subject Type': subjectType, Status: status, Country: country, - "Country Sub Division": countrySubDivision, + 'Country Sub Division': countrySubDivision, Details: details, Origin: origin, - "Silent Mode": isSilent, - "Is Test Request": isTest, + 'Silent Mode': isSilent, + 'Is Test Request': isTest, Language: locale, ...request, ...Object.fromEntries( - Object.entries(groupBy(attributeValues, "attributeKey.name")).map( - ([name, values]) => [name, values.map(({ name }) => name).join(",")] - ) + Object.entries(groupBy(attributeValues, 'attributeKey.name')).map( + ([name, values]) => [name, values.map(({ name }) => name).join(',')], + ), ), ...Object.fromEntries( - Object.entries(groupBy(requestIdentifiers, "name")).map( - ([name, values]) => [name, values.map(({ value }) => value).join(",")] - ) + Object.entries(groupBy(requestIdentifiers, 'name')).map( + ([name, values]) => [ + name, + values.map(({ value }) => value).join(','), + ], + ), ), - }) + }), ); return { requestsWithRequestIdentifiers, requestsFormattedForCsv: data }; diff --git a/src/lib/requests/readCsv.ts b/src/lib/requests/readCsv.ts index fed0f1cd..e949fe64 100644 --- a/src/lib/requests/readCsv.ts +++ b/src/lib/requests/readCsv.ts @@ -1,8 +1,8 @@ -import { readFileSync } from "node:fs"; -import { decodeCodec } from "@transcend-io/type-utils"; -import type { Options } from "csv-parse"; -import { parse } from "csv-parse/sync"; -import * as t from "io-ts"; +import { readFileSync } from 'node:fs'; +import { decodeCodec } from '@transcend-io/type-utils'; +import type { Options } from 'csv-parse'; +import { parse } from 'csv-parse/sync'; +import * as t from 'io-ts'; /** * Read in a CSV and validate its shape @@ -15,10 +15,10 @@ import * as t from "io-ts"; export function readCsv( pathToFile: string, codec: T, - options: Options = { columns: true } + options: Options = { columns: true }, ): t.TypeOf[] { // read file contents and parse - const fileContent = parse(readFileSync(pathToFile, "utf-8"), options); + const fileContent = parse(readFileSync(pathToFile, 'utf-8'), options); // validate codec const data = decodeCodec(t.array(codec), fileContent); @@ -28,10 +28,10 @@ export function readCsv( Object.entries(datum).reduce( (accumulator, [key, value]) => Object.assign(accumulator, { - [key.replaceAll(/[^a-z_.+\-A-Z -~]/g, "")]: value, + [key.replaceAll(/[^a-z_.+\-A-Z -~]/g, '')]: value, }), - {} as T - ) + {} as T, + ), ); return parsed; } diff --git a/src/lib/requests/removeUnverifiedRequestIdentifiers.ts b/src/lib/requests/removeUnverifiedRequestIdentifiers.ts index 2b8b02e3..0d4effc4 100644 --- a/src/lib/requests/removeUnverifiedRequestIdentifiers.ts +++ b/src/lib/requests/removeUnverifiedRequestIdentifiers.ts @@ -1,16 +1,16 @@ -import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, fetchAllRequestIdentifierMetadata, fetchAllRequests, makeGraphQLRequest, REMOVE_REQUEST_IDENTIFIERS, -} from "../graphql"; +} from '../graphql'; /** * Remove a set of unverified request identifier @@ -44,7 +44,7 @@ export async function removeUnverifiedRequestIdentifiers({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Pull in the requests @@ -54,7 +54,7 @@ export async function removeUnverifiedRequestIdentifiers({ }); // Notify Transcend - logger.info(colors.magenta("Fetched requests in preflight/enriching state.")); + logger.info(colors.magenta('Fetched requests in preflight/enriching state.')); let total = 0; let processed = 0; @@ -64,12 +64,12 @@ export async function removeUnverifiedRequestIdentifiers({ async (requestToRestart) => { const requestIdentifiers = await fetchAllRequestIdentifierMetadata( client, - { requestId: requestToRestart.id } + { requestId: requestToRestart.id }, ); const clearOut = requestIdentifiers .filter( ({ isVerifiedAtLeastOnce, name }) => - !isVerifiedAtLeastOnce && identifierNames.includes(name) + !isVerifiedAtLeastOnce && identifierNames.includes(name), ) .map(({ id }) => id); @@ -89,7 +89,7 @@ export async function removeUnverifiedRequestIdentifiers({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -100,8 +100,8 @@ export async function removeUnverifiedRequestIdentifiers({ colors.green( `Successfully cleared out unverified identifiers "${ totalTime / 1000 - }" seconds for ${total} requests, ${processed} identifiers were cleared out!` - ) + }" seconds for ${total} requests, ${processed} identifiers were cleared out!`, + ), ); return allRequests.length; } diff --git a/src/lib/requests/retryRequestDataSilos.ts b/src/lib/requests/retryRequestDataSilos.ts index 478841b5..bdcb9525 100644 --- a/src/lib/requests/retryRequestDataSilos.ts +++ b/src/lib/requests/retryRequestDataSilos.ts @@ -1,16 +1,16 @@ -import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, fetchAllRequests, fetchRequestDataSilo, makeGraphQLRequest, RETRY_REQUEST_DATA_SILO, -} from "../graphql"; +} from '../graphql'; /** * Retry a set of RequestDataSilos @@ -44,7 +44,7 @@ export async function retryRequestDataSilos({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Pull in the requests @@ -56,8 +56,8 @@ export async function retryRequestDataSilos({ // Notify Transcend logger.info( colors.magenta( - `Retrying requests for Data Silo: "${dataSiloId}", restarting "${allRequests.length}" requests.` - ) + `Retrying requests for Data Silo: "${dataSiloId}", restarting "${allRequests.length}" requests.`, + ), ); let total = 0; @@ -80,7 +80,7 @@ export async function retryRequestDataSilos({ }); } catch (error) { // some requests may not have this data silo connected - if (!error.message.includes("Failed to find RequestDataSilo")) { + if (!error.message.includes('Failed to find RequestDataSilo')) { throw error; } skipped += 1; @@ -89,7 +89,7 @@ export async function retryRequestDataSilos({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -100,8 +100,8 @@ export async function retryRequestDataSilos({ colors.green( `Successfully notified Transcend in "${ totalTime / 1000 - }" seconds for ${total} requests, ${skipped} requests were skipped because data silo was not attached to the request!` - ) + }" seconds for ${total} requests, ${skipped} requests were skipped because data silo was not attached to the request!`, + ), ); return allRequests.length; } diff --git a/src/lib/requests/skipPreflightJobs.ts b/src/lib/requests/skipPreflightJobs.ts index 48922c98..893f6259 100644 --- a/src/lib/requests/skipPreflightJobs.ts +++ b/src/lib/requests/skipPreflightJobs.ts @@ -1,19 +1,19 @@ import { RequestEnricherStatus, RequestStatus, -} from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map, mapSeries } from "../bluebird-replace"; +} from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map, mapSeries } from '../bluebird-replace'; import { buildTranscendGraphQLClient, fetchAllRequestEnrichers, fetchAllRequests, makeGraphQLRequest, SKIP_REQUEST_ENRICHER, -} from "../graphql"; +} from '../graphql'; /** * Given an enricher ID, mark all open request enrichers as skipped @@ -52,16 +52,16 @@ export async function skipPreflightJobs({ // Notify Transcend logger.info( colors.magenta( - `Processing enricher: "${enricherIds.join(",")}" fetched "${ + `Processing enricher: "${enricherIds.join(',')}" fetched "${ requests.length - }" in enriching status.` - ) + }" in enriching status.`, + ), ); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); let total = 0; @@ -81,7 +81,7 @@ export async function skipPreflightJobs({ RequestEnricherStatus.Resolved, RequestEnricherStatus.Skipped, // eslint-disable-next-line @typescript-eslint/no-explicit-any - ].includes(enricher.status as any) + ].includes(enricher.status as any), ); // FIXME @@ -98,7 +98,7 @@ export async function skipPreflightJobs({ } catch (error) { if ( !error.message.includes( - "Client error: Cannot skip Request enricher because it has already completed" + 'Client error: Cannot skip Request enricher because it has already completed', ) ) { throw error; @@ -109,7 +109,7 @@ export async function skipPreflightJobs({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -120,8 +120,8 @@ export async function skipPreflightJobs({ colors.green( `Successfully skipped "${totalSkipped}" for "${ requests.length - }" requests in "${totalTime / 1000}" seconds!` - ) + }" requests in "${totalTime / 1000}" seconds!`, + ), ); return requests.length; } diff --git a/src/lib/requests/skipRequestDataSilos.ts b/src/lib/requests/skipRequestDataSilos.ts index 89d0154c..23000495 100644 --- a/src/lib/requests/skipRequestDataSilos.ts +++ b/src/lib/requests/skipRequestDataSilos.ts @@ -1,15 +1,15 @@ -import { RequestStatus } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, CHANGE_REQUEST_DATA_SILO_STATUS, fetchRequestDataSilos, makeGraphQLRequest, -} from "../graphql"; +} from '../graphql'; /** * Given a data silo ID, mark all open request data silos as skipped @@ -21,7 +21,7 @@ export async function skipRequestDataSilos({ dataSiloId, auth, concurrency = 100, - status = "SKIPPED", + status = 'SKIPPED', transcendUrl = DEFAULT_TRANSCEND_API, requestStatuses = [RequestStatus.Compiling, RequestStatus.Secondary], }: { @@ -30,7 +30,7 @@ export async function skipRequestDataSilos({ /** Data Silo ID to pull down jobs for */ dataSiloId: string; /** Status to set */ - status?: "SKIPPED" | "RESOLVED"; + status?: 'SKIPPED' | 'RESOLVED'; /** Upload concurrency */ concurrency?: number; /** API URL for Transcend backend */ @@ -53,14 +53,14 @@ export async function skipRequestDataSilos({ // Notify Transcend logger.info( colors.magenta( - `Processing data silo: "${dataSiloId}" marking "${requestDataSilos.length}" requests as skipped.` - ) + `Processing data silo: "${dataSiloId}" marking "${requestDataSilos.length}" requests as skipped.`, + ), ); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); let total = 0; @@ -77,7 +77,7 @@ export async function skipRequestDataSilos({ status, }); } catch (error) { - if (!error.message.includes("Client error: Request must be active:")) { + if (!error.message.includes('Client error: Request must be active:')) { throw error; } } @@ -85,7 +85,7 @@ export async function skipRequestDataSilos({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -96,8 +96,8 @@ export async function skipRequestDataSilos({ colors.green( `Successfully skipped "${requestDataSilos.length}" requests in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); return requestDataSilos.length; } diff --git a/src/lib/requests/splitCsvToList.ts b/src/lib/requests/splitCsvToList.ts index 7daa7a9c..b937aa77 100644 --- a/src/lib/requests/splitCsvToList.ts +++ b/src/lib/requests/splitCsvToList.ts @@ -10,7 +10,7 @@ */ export function splitCsvToList(value: string): string[] { return value - .split(",") + .split(',') .map((x) => x.trim()) .filter(Boolean); } diff --git a/src/lib/requests/streamPrivacyRequestFiles.ts b/src/lib/requests/streamPrivacyRequestFiles.ts index 76bb964c..35d20a85 100644 --- a/src/lib/requests/streamPrivacyRequestFiles.ts +++ b/src/lib/requests/streamPrivacyRequestFiles.ts @@ -1,8 +1,8 @@ -import colors from "colors"; -import type { Got } from "got"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { RequestFileMetadata } from "./getFileMetadataForPrivacyRequests"; +import colors from 'colors'; +import type { Got } from 'got'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { RequestFileMetadata } from './getFileMetadataForPrivacyRequests'; /** * This function will take in a set of file metadata for privacy requests @@ -28,7 +28,7 @@ export async function streamPrivacyRequestFiles( onFileDownloaded: (metadata: RequestFileMetadata, stream: Buffer) => void; /** Concurrent downloads at once */ concurrency?: number; - } + }, ): Promise { // Loop over each file await map( @@ -37,7 +37,7 @@ export async function streamPrivacyRequestFiles( try { // Construct the stream await sombra - .get("v1/files", { + .get('v1/files', { searchParams: { downloadKey: metadata.downloadKey, }, @@ -47,26 +47,26 @@ export async function streamPrivacyRequestFiles( onFileDownloaded(metadata, fileResponse); }); } catch (error) { - if (error?.response?.body?.includes("fileMetadata#verify")) { + if (error?.response?.body?.includes('fileMetadata#verify')) { logger.error( colors.red( `Failed to pull file for: ${metadata.fileName} (request:${requestId}) - JWT expired. ` + - "This likely means that the file is no longer available. " + - "Try restarting the request from scratch in Transcend Admin Dashboard. " + - "Skipping the download of this file." - ) + 'This likely means that the file is no longer available. ' + + 'Try restarting the request from scratch in Transcend Admin Dashboard. ' + + 'Skipping the download of this file.', + ), ); return; } throw new Error( `Received an error from server: ${ error?.response?.body || error?.message - }` + }`, ); } }, { concurrency, - } + }, ); } diff --git a/src/lib/requests/submitPrivacyRequest.ts b/src/lib/requests/submitPrivacyRequest.ts index f6d875ae..ffb7cba7 100644 --- a/src/lib/requests/submitPrivacyRequest.ts +++ b/src/lib/requests/submitPrivacyRequest.ts @@ -3,13 +3,13 @@ import { IsoCountrySubdivisionCode, RequestAction, RequestStatus, -} from "@transcend-io/privacy-types"; -import { decodeCodec, valuesOf } from "@transcend-io/type-utils"; -import type { Got } from "got"; -import * as t from "io-ts"; -import { uniq } from "lodash-es"; -import { PrivacyRequestInput } from "./mapCsvRowsToRequestInputs"; -import { ParsedAttributeInput } from "./parseAttributesFromString"; +} from '@transcend-io/privacy-types'; +import { decodeCodec, valuesOf } from '@transcend-io/type-utils'; +import type { Got } from 'got'; +import * as t from 'io-ts'; +import { uniq } from 'lodash-es'; +import { PrivacyRequestInput } from './mapCsvRowsToRequestInputs'; +import { ParsedAttributeInput } from './parseAttributesFromString'; export const PrivacyRequestResponse = t.type({ id: t.string, @@ -27,7 +27,7 @@ export const PrivacyRequestResponse = t.type({ t.type({ attributeKey: t.type({ name: t.string }), name: t.string, - }) + }), ), }); @@ -46,7 +46,7 @@ export async function submitPrivacyRequest( sombra: Got, input: PrivacyRequestInput, { - details = "", + details = '', isTest = false, emailIsVerified = true, skipSendingReceipt = false, @@ -65,14 +65,14 @@ export async function submitPrivacyRequest( details?: string; /** Additional attributes to tag the requests with */ additionalAttributes?: ParsedAttributeInput[]; - } = {} + } = {}, ): Promise { // Merge the per-request attributes with the // global attributes const mergedAttributes = [...additionalAttributes]; for (const attribute of input.attributes || []) { const existing = mergedAttributes.find( - (attribute_) => attribute_.key === attribute.key + (attribute_) => attribute_.key === attribute.key, ); if (existing) { existing.values.push(...attribute.values); @@ -86,7 +86,7 @@ export async function submitPrivacyRequest( let response: unknown; try { response = await sombra - .post("v1/data-subject-request", { + .post('v1/data-subject-request', { json: { type: input.requestType, subject: { @@ -110,8 +110,8 @@ export async function submitPrivacyRequest( country: input.country, } : input.countrySubDivision - ? { country: input.countrySubDivision.split("-")[0] } - : {}), + ? { country: input.countrySubDivision.split('-')[0] } + : {}), ...(input.countrySubDivision ? { countrySubDivision: input.countrySubDivision } : {}), @@ -128,7 +128,7 @@ export async function submitPrivacyRequest( throw new Error( `Received an error from server: ${ error?.response?.body || error?.message - }` + }`, ); } @@ -136,7 +136,7 @@ export async function submitPrivacyRequest( t.type({ request: PrivacyRequestResponse, }), - response + response, ); return requestResponse; } diff --git a/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts b/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts index 4f4f4163..1c1523f9 100644 --- a/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts +++ b/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts @@ -1,4 +1,4 @@ -import { describe } from "vitest"; +import { describe } from 'vitest'; // TODO: https://transcend.height.app/T-10772 - mapCsvRowsToRequestInputs test -describe.skip("mapCsvRowsToRequestInputs", () => {}); +describe.skip('mapCsvRowsToRequestInputs', () => {}); diff --git a/src/lib/requests/tests/readCsv.test.ts b/src/lib/requests/tests/readCsv.test.ts index 6a8feef5..b3c3ae84 100644 --- a/src/lib/requests/tests/readCsv.test.ts +++ b/src/lib/requests/tests/readCsv.test.ts @@ -1,54 +1,54 @@ -import { join } from "node:path"; -import * as t from "io-ts"; -import { describe, expect, it } from "vitest"; -import { readCsv } from "../index"; +import { join } from 'node:path'; +import * as t from 'io-ts'; +import { describe, expect, it } from 'vitest'; +import { readCsv } from '../index'; -describe("readCsv", () => { - it("should successfully parse a csv", () => { +describe('readCsv', () => { + it('should successfully parse a csv', () => { expect( - readCsv(join(__dirname, "sample.csv"), t.record(t.string, t.string)) + readCsv(join(__dirname, 'sample.csv'), t.record(t.string, t.string)), ).to.deep.equal([ { - CASL_STATUS: "Undefined", - CONTACT_ID: "949858", - "Data Subject": "Customer", - FIRST_NAME: "Mike", - GDPR_STATUS: "LBI Notice Not Sent", - LAST_NAME: "Farrell", - LINKEDIN_HANDLE: "michaelfarrell", - MOBILE_PHONE: "(860) 906-6012", - OTHER_URL: "", - PERSONAL_EMAIL: "mike@transcend.io", - PHONE: "", - PREFERRED_EMAIL: "mike@transcend.io", - "Primary Company": "Transcend", - "Request Type": "Opt In", - SEARCH_ID: "10749", - SEARCH_NAME: "Transcend - Chief Technology Officer", - SKYPE_HANDLE: "", - STAGE_START_DATE: "2022-11-22", - TITLE: "Chief Technology Officer", - WORK_EMAIL: "", - WORK_PHONE: "", + CASL_STATUS: 'Undefined', + CONTACT_ID: '949858', + 'Data Subject': 'Customer', + FIRST_NAME: 'Mike', + GDPR_STATUS: 'LBI Notice Not Sent', + LAST_NAME: 'Farrell', + LINKEDIN_HANDLE: 'michaelfarrell', + MOBILE_PHONE: '(860) 906-6012', + OTHER_URL: '', + PERSONAL_EMAIL: 'mike@transcend.io', + PHONE: '', + PREFERRED_EMAIL: 'mike@transcend.io', + 'Primary Company': 'Transcend', + 'Request Type': 'Opt In', + SEARCH_ID: '10749', + SEARCH_NAME: 'Transcend - Chief Technology Officer', + SKYPE_HANDLE: '', + STAGE_START_DATE: '2022-11-22', + TITLE: 'Chief Technology Officer', + WORK_EMAIL: '', + WORK_PHONE: '', }, ]); }); - it("throw an error for invalid file", () => { + it('throw an error for invalid file', () => { expect(() => - readCsv(join(__dirname, "sample.csv"), t.type({ notValid: t.string })) - ).to.throw("Failed to decode codec"); + readCsv(join(__dirname, 'sample.csv'), t.type({ notValid: t.string })), + ).to.throw('Failed to decode codec'); }); - it("throw an error for invalid codec", () => { + it('throw an error for invalid codec', () => { expect(() => - readCsv(join(__dirname, "sample.csvs"), t.record(t.string, t.string)) - ).to.throw("ENOENT: no such file or directory, open"); + readCsv(join(__dirname, 'sample.csvs'), t.record(t.string, t.string)), + ).to.throw('ENOENT: no such file or directory, open'); }); - it("throw an error for invalid format", () => { + it('throw an error for invalid format', () => { expect(() => - readCsv(join(__dirname, "readCsv.test.ts"), t.record(t.string, t.string)) + readCsv(join(__dirname, 'readCsv.test.ts'), t.record(t.string, t.string)), ).to.throw(); }); }); diff --git a/src/lib/requests/uploadPrivacyRequestsFromCsv.ts b/src/lib/requests/uploadPrivacyRequestsFromCsv.ts index b897697d..4fc243c5 100644 --- a/src/lib/requests/uploadPrivacyRequestsFromCsv.ts +++ b/src/lib/requests/uploadPrivacyRequestsFromCsv.ts @@ -1,28 +1,28 @@ -import { join } from "node:path"; -import { PersistedState } from "@transcend-io/persisted-state"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import * as t from "io-ts"; -import { uniq } from "lodash-es"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { join } from 'node:path'; +import { PersistedState } from '@transcend-io/persisted-state'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import * as t from 'io-ts'; +import { uniq } from 'lodash-es'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, createSombraGotInstance, fetchAllRequestAttributeKeys, -} from "../graphql"; -import { CachedFileState, CachedRequestState } from "./constants"; -import { extractClientError } from "./extractClientError"; -import { filterRows } from "./filterRows"; -import { mapColumnsToAttributes } from "./mapColumnsToAttributes"; -import { mapColumnsToIdentifiers } from "./mapColumnsToIdentifiers"; -import { mapCsvColumnsToApi } from "./mapCsvColumnsToApi"; -import { mapCsvRowsToRequestInputs } from "./mapCsvRowsToRequestInputs"; -import { mapRequestEnumValues } from "./mapRequestEnumValues"; -import { parseAttributesFromString } from "./parseAttributesFromString"; -import { readCsv } from "./readCsv"; -import { submitPrivacyRequest } from "./submitPrivacyRequest"; +} from '../graphql'; +import { CachedFileState, CachedRequestState } from './constants'; +import { extractClientError } from './extractClientError'; +import { filterRows } from './filterRows'; +import { mapColumnsToAttributes } from './mapColumnsToAttributes'; +import { mapColumnsToIdentifiers } from './mapColumnsToIdentifiers'; +import { mapCsvColumnsToApi } from './mapCsvColumnsToApi'; +import { mapCsvRowsToRequestInputs } from './mapCsvRowsToRequestInputs'; +import { mapRequestEnumValues } from './mapRequestEnumValues'; +import { parseAttributesFromString } from './parseAttributesFromString'; +import { readCsv } from './readCsv'; +import { submitPrivacyRequest } from './submitPrivacyRequest'; /** * Upload a set of privacy requests from CSV @@ -36,7 +36,7 @@ export async function uploadPrivacyRequestsFromCsv({ auth, sombraAuth, concurrency = 100, - defaultPhoneCountryCode = "1", // USA + defaultPhoneCountryCode = '1', // USA transcendUrl = DEFAULT_TRANSCEND_API, attributes = [], emailIsVerified = true, @@ -85,7 +85,7 @@ export async function uploadPrivacyRequestsFromCsv({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Parse out the extra attributes to apply to all requests uploaded @@ -109,8 +109,8 @@ export async function uploadPrivacyRequestsFromCsv({ const requestCacheFile = join( requestReceiptFolder, `tr-request-upload-${new Date().toISOString()}-${file - .split("/") - .pop()}`.replace(".csv", ".json") + .split('/') + .pop()}`.replace('.csv', '.json'), ); const requestState = new PersistedState( requestCacheFile, @@ -119,7 +119,7 @@ export async function uploadPrivacyRequestsFromCsv({ successfulRequests: [], duplicateRequests: [], failingRequests: [], - } + }, ); // Create sombra instance to communicate with @@ -132,13 +132,13 @@ export async function uploadPrivacyRequestsFromCsv({ // Log out an example request if (requestsList.length === 0) { throw new Error( - "No Requests found in list! Ensure the first row of the CSV is a header and the rest are requests." + 'No Requests found in list! Ensure the first row of the CSV is a header and the rest are requests.', ); } if (debug) { const firstRequest = requestsList[0]; logger.info( - colors.magenta(`First request: ${JSON.stringify(firstRequest, null, 2)}`) + colors.magenta(`First request: ${JSON.stringify(firstRequest, null, 2)}`), ); } // Determine what rows in the CSV should be imported @@ -156,13 +156,13 @@ export async function uploadPrivacyRequestsFromCsv({ const identifierNameMap = await mapColumnsToIdentifiers( client, columnNames, - state + state, ); const attributeNameMap = await mapColumnsToAttributes( client, columnNames, state, - requestAttributeKeys + requestAttributeKeys, ); await mapRequestEnumValues(client, filteredRequestList, { state, @@ -198,16 +198,16 @@ export async function uploadPrivacyRequestsFromCsv({ `[${ind + 1}/${requestInputs.length}] Importing: ${JSON.stringify( requestInput, null, - 2 - )}` - ) + 2, + )}`, + ), ); } // Skip on dry run if (dryRun) { logger.info( - colors.magenta("Bailing out on dry run because dryRun is set") + colors.magenta('Bailing out on dry run because dryRun is set'), ); return; } @@ -221,14 +221,14 @@ export async function uploadPrivacyRequestsFromCsv({ details: `Uploaded by Transcend Cli: "tr-request-upload" : ${JSON.stringify( rawRow, null, - 2 + 2, )}`, isTest, emailIsVerified, skipSendingReceipt, isSilent, additionalAttributes: parsedAttributes, - } + }, ); // Log success @@ -237,20 +237,20 @@ export async function uploadPrivacyRequestsFromCsv({ colors.green( `[${ind + 1}/${ requestInputs.length - }] Successfully submitted the test data subject request: "${requestLogId}"` - ) + }] Successfully submitted the test data subject request: "${requestLogId}"`, + ), ); logger.info( colors.green( `[${ind + 1}/${requestInputs.length}] View it at: "${ requestResponse.link - }"` - ) + }"`, + ), ); } // Cache successful upload - const successfulRequests = requestState.getValue("successfulRequests"); + const successfulRequests = requestState.getValue('successfulRequests'); successfulRequests.push({ id: requestResponse.id, link: requestResponse.link, @@ -258,51 +258,51 @@ export async function uploadPrivacyRequestsFromCsv({ coreIdentifier: requestResponse.coreIdentifier, attemptedAt: new Date().toISOString(), }); - await requestState.setValue(successfulRequests, "successfulRequests"); + await requestState.setValue(successfulRequests, 'successfulRequests'); } catch (error) { const message = `${error.message} - ${JSON.stringify( error.response?.body, null, - 2 + 2, )}`; const clientError = extractClientError(message); if ( - clientError === "Client error: You have already made this request." + clientError === 'Client error: You have already made this request.' ) { if (debug) { logger.info( colors.yellow( `[${ind + 1}/${ requestInputs.length - }] Skipping request as it is a duplicate` - ) + }] Skipping request as it is a duplicate`, + ), ); } - const duplicateRequests = requestState.getValue("duplicateRequests"); + const duplicateRequests = requestState.getValue('duplicateRequests'); duplicateRequests.push({ coreIdentifier: requestInput.coreIdentifier, rowIndex: ind, attemptedAt: new Date().toISOString(), }); - await requestState.setValue(duplicateRequests, "duplicateRequests"); + await requestState.setValue(duplicateRequests, 'duplicateRequests'); } else { - const failingRequests = requestState.getValue("failingRequests"); + const failingRequests = requestState.getValue('failingRequests'); failingRequests.push({ ...requestInput, rowIndex: ind, error: clientError || message, attemptedAt: new Date().toISOString(), }); - await requestState.setValue(failingRequests, "failingRequests"); + await requestState.setValue(failingRequests, 'failingRequests'); if (debug) { logger.error(colors.red(clientError || message)); logger.error( colors.red( `[${ind + 1}/${ requestInputs.length - }] Failed to submit request for: "${requestLogId}"` - ) + }] Failed to submit request for: "${requestLogId}"`, + ), ); } } @@ -315,7 +315,7 @@ export async function uploadPrivacyRequestsFromCsv({ }, { concurrency, - } + }, ); progressBar.stop(); @@ -324,30 +324,30 @@ export async function uploadPrivacyRequestsFromCsv({ // Log completion time logger.info( - colors.green(`Completed upload in "${totalTime / 1000}" seconds.`) + colors.green(`Completed upload in "${totalTime / 1000}" seconds.`), ); // Log duplicates - if (requestState.getValue("duplicateRequests").length > 0) { + if (requestState.getValue('duplicateRequests').length > 0) { logger.info( colors.yellow( `Encountered "${ - requestState.getValue("duplicateRequests").length + requestState.getValue('duplicateRequests').length }" duplicate requests. ` + - `See "${requestCacheFile}" to review the core identifiers for these requests.` - ) + `See "${requestCacheFile}" to review the core identifiers for these requests.`, + ), ); } // Log errors - if (requestState.getValue("failingRequests").length > 0) { + if (requestState.getValue('failingRequests').length > 0) { logger.error( colors.red( `Encountered "${ - requestState.getValue("failingRequests").length + requestState.getValue('failingRequests').length }" errors. ` + - `See "${requestCacheFile}" to review the error messages and inputs.` - ) + `See "${requestCacheFile}" to review the error messages and inputs.`, + ), ); process.exit(1); } diff --git a/src/lib/tests/findCodePackagesInFolder.test.ts b/src/lib/tests/findCodePackagesInFolder.test.ts index 0cb4d54f..078f92e3 100644 --- a/src/lib/tests/findCodePackagesInFolder.test.ts +++ b/src/lib/tests/findCodePackagesInFolder.test.ts @@ -1,699 +1,699 @@ -import { join } from "node:path"; -import { describe, expect, it } from "vitest"; -import type { CodePackageInput } from "../../codecs"; -import { findCodePackagesInFolder } from "../code-scanning/findCodePackagesInFolder"; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import type { CodePackageInput } from '../../codecs'; +import { findCodePackagesInFolder } from '../code-scanning/findCodePackagesInFolder'; const expected: CodePackageInput[] = [ { - name: "YourAppTargetName", - type: "COCOA_PODS", + name: 'YourAppTargetName', + type: 'COCOA_PODS', softwareDevelopmentKits: [ { - name: "Braze-iOS-SDK", + name: 'Braze-iOS-SDK', version: undefined, }, { - name: "Branch", + name: 'Branch', version: undefined, }, { - name: "Firebase/Analytics", + name: 'Firebase/Analytics', version: undefined, }, { - name: "Mixpanel", + name: 'Mixpanel', version: undefined, }, { - name: "Amplitude-iOS", - version: "8.0", + name: 'Amplitude-iOS', + version: '8.0', }, { - name: "Google-Mobile-Ads-SDK", + name: 'Google-Mobile-Ads-SDK', version: undefined, }, { - name: "FacebookAdsSDK", + name: 'FacebookAdsSDK', version: undefined, }, { - name: "MoPub-SDK", + name: 'MoPub-SDK', version: undefined, }, { - name: "Alamofire", - version: "5.2", + name: 'Alamofire', + version: '5.2', }, { - name: "SDWebImage", + name: 'SDWebImage', version: undefined, }, { - name: "AppsFlyerFramework", + name: 'AppsFlyerFramework', version: undefined, }, { - name: "Adjust", + name: 'Adjust', version: undefined, }, { - name: "Flurry-iOS-SDK/FlurrySDK", + name: 'Flurry-iOS-SDK/FlurrySDK', version: undefined, }, ], - relativePath: "test-cocoa-pods/Podfile", - repositoryName: "transcend-io/cli", + relativePath: 'test-cocoa-pods/Podfile', + repositoryName: 'transcend-io/cli', }, { - name: "ExampleBootstrap", - type: "COCOA_PODS", + name: 'ExampleBootstrap', + type: 'COCOA_PODS', softwareDevelopmentKits: [ { - name: "ExampleLib", + name: 'ExampleLib', version: undefined, }, { - name: "AppsFlyerFramework", + name: 'AppsFlyerFramework', version: undefined, }, { - name: "Adjust", + name: 'Adjust', version: undefined, }, { - name: "Flurry-iOS-SDK/FlurrySDK", + name: 'Flurry-iOS-SDK/FlurrySDK', version: undefined, }, ], - relativePath: "test-requirements-txt/nested-cocoapods/Podfile", - repositoryName: "transcend-io/cli", + relativePath: 'test-requirements-txt/nested-cocoapods/Podfile', + repositoryName: 'transcend-io/cli', }, { - name: "ExampleBootstrapTests", - type: "COCOA_PODS", + name: 'ExampleBootstrapTests', + type: 'COCOA_PODS', softwareDevelopmentKits: [ { - name: "ExampleLib", + name: 'ExampleLib', version: undefined, }, { - name: "Braze-iOS-SDK", + name: 'Braze-iOS-SDK', version: undefined, }, { - name: "Branch", + name: 'Branch', version: undefined, }, { - name: "Firebase/Analytics", + name: 'Firebase/Analytics', version: undefined, }, { - name: "Mixpanel", + name: 'Mixpanel', version: undefined, }, { - name: "Amplitude-iOS", - version: "8.0", + name: 'Amplitude-iOS', + version: '8.0', }, ], - relativePath: "test-requirements-txt/nested-cocoapods/Podfile", - repositoryName: "transcend-io/cli", + relativePath: 'test-requirements-txt/nested-cocoapods/Podfile', + repositoryName: 'transcend-io/cli', }, { - name: "Acme", - relativePath: "test-requirements-txt/nested-cocoapods-2/Podfile", - repositoryName: "transcend-io/cli", + name: 'Acme', + relativePath: 'test-requirements-txt/nested-cocoapods-2/Podfile', + repositoryName: 'transcend-io/cli', softwareDevelopmentKits: [ { - name: "RZVinyl", + name: 'RZVinyl', version: undefined, }, { - name: "RZTransitions", + name: 'RZTransitions', version: undefined, }, { - name: "SDWebImage", + name: 'SDWebImage', version: undefined, }, { - name: "SwiftLint", + name: 'SwiftLint', version: undefined, }, ], - type: "COCOA_PODS", + type: 'COCOA_PODS', }, { - name: "AcmeTests", - type: "COCOA_PODS", + name: 'AcmeTests', + type: 'COCOA_PODS', softwareDevelopmentKits: [ { - name: "RZVinyl", + name: 'RZVinyl', version: undefined, }, { - name: "iOSSnapshotTestCase", + name: 'iOSSnapshotTestCase', version: undefined, }, { - name: "SnapshotTesting", - version: "1.8.1", + name: 'SnapshotTesting', + version: '1.8.1', }, ], - relativePath: "test-requirements-txt/nested-cocoapods-2/Podfile", - repositoryName: "transcend-io/cli", + relativePath: 'test-requirements-txt/nested-cocoapods-2/Podfile', + repositoryName: 'transcend-io/cli', }, { - name: "NotificationServiceExtension", - relativePath: "test-requirements-txt/nested-cocoapods-2/Podfile", - repositoryName: "transcend-io/cli", + name: 'NotificationServiceExtension', + relativePath: 'test-requirements-txt/nested-cocoapods-2/Podfile', + repositoryName: 'transcend-io/cli', softwareDevelopmentKits: [], - type: "COCOA_PODS", + type: 'COCOA_PODS', }, { - name: "com.yourcompany.yourapp", + name: 'com.yourcompany.yourapp', softwareDevelopmentKits: [ { - name: "androidx.appcompat", - version: "1.2.0", + name: 'androidx.appcompat', + version: '1.2.0', }, { - name: "androidx.constraintlayout", - version: "2.0.4", + name: 'androidx.constraintlayout', + version: '2.0.4', }, { - name: "com.appboy", - version: "14.0.0", + name: 'com.appboy', + version: '14.0.0', }, { - name: "io.branch.sdk.android", - version: "5.0.1", + name: 'io.branch.sdk.android', + version: '5.0.1', }, { - name: "com.google.firebase", - version: "18.0.0", + name: 'com.google.firebase', + version: '18.0.0', }, { - name: "com.google.android.gms", - version: "19.7.0", + name: 'com.google.android.gms', + version: '19.7.0', }, { - name: "com.facebook.android", - version: "6.2.0", + name: 'com.facebook.android', + version: '6.2.0', }, { - name: "com.mixpanel.android", - version: "5.8.7", + name: 'com.mixpanel.android', + version: '5.8.7', }, { - name: "com.amplitude", - version: "2.30.0", + name: 'com.amplitude', + version: '2.30.0', }, { - name: "com.squareup.retrofit2", - version: "2.9.0", + name: 'com.squareup.retrofit2', + version: '2.9.0', }, { - name: "com.squareup.okhttp3", - version: "4.9.0", + name: 'com.squareup.okhttp3', + version: '4.9.0', }, { - name: "com.squareup.picasso", - version: "2.71828", + name: 'com.squareup.picasso', + version: '2.71828', }, { - name: "org.eclipse.jdt.core", - version: "3.28.0", + name: 'org.eclipse.jdt.core', + version: '3.28.0', }, { - name: "com.android.application", + name: 'com.android.application', version: undefined, }, { - name: "com.google.gms.google-services", + name: 'com.google.gms.google-services', version: undefined, }, ], - relativePath: "test-gradle/build.gradle", - type: "GRADLE", - repositoryName: "transcend-io/cli", + relativePath: 'test-gradle/build.gradle', + type: 'GRADLE', + repositoryName: 'transcend-io/cli', }, { - name: "@test-example/test", - description: "Example npm package.", + name: '@test-example/test', + description: 'Example npm package.', softwareDevelopmentKits: [ { - name: "dd-trace", - version: "2.45.1", + name: 'dd-trace', + version: '2.45.1', }, { - name: "fast-csv", - version: "^4.3.6", + name: 'fast-csv', + version: '^4.3.6', }, { - name: "sequelize", - version: "^6.37.3", + name: 'sequelize', + version: '^6.37.3', }, { - name: "sequelize-mock", - version: "^0.10.2", + name: 'sequelize-mock', + version: '^0.10.2', }, { isDevDependency: true, - name: "@types/sequelize", - version: "^4.28.20", + name: '@types/sequelize', + version: '^4.28.20', }, { - name: "typescript", - version: "^5.0.4", + name: 'typescript', + version: '^5.0.4', isDevDependency: true, }, ], - relativePath: "test-package-json/package.json", - type: "PACKAGE_JSON", - repositoryName: "transcend-io/cli", + relativePath: 'test-package-json/package.json', + type: 'PACKAGE_JSON', + repositoryName: 'transcend-io/cli', }, { - name: "@test-example/nested-test", - description: "Example npm nested package.", + name: '@test-example/nested-test', + description: 'Example npm nested package.', softwareDevelopmentKits: [ { - name: "dd-trace", - version: "2.45.1", + name: 'dd-trace', + version: '2.45.1', }, { - name: "fast-csv", - version: "^4.3.6", + name: 'fast-csv', + version: '^4.3.6', }, { - name: "typescript", - version: "^5.0.4", + name: 'typescript', + version: '^5.0.4', isDevDependency: true, }, ], - relativePath: "test-gradle/test-nested-package-json/package.json", - type: "PACKAGE_JSON", - repositoryName: "transcend-io/cli", + relativePath: 'test-gradle/test-nested-package-json/package.json', + type: 'PACKAGE_JSON', + repositoryName: 'transcend-io/cli', }, { - name: "test_requirements_txt", - type: "REQUIREMENTS_TXT", - description: "A sample Python package", + name: 'test_requirements_txt', + type: 'REQUIREMENTS_TXT', + description: 'A sample Python package', softwareDevelopmentKits: [ { - name: "pyarrow", - version: "14.0.1", + name: 'pyarrow', + version: '14.0.1', }, { - name: "cryptography", - version: "41.0.6", + name: 'cryptography', + version: '41.0.6', }, { - name: "Flask", - version: "2.2.5", + name: 'Flask', + version: '2.2.5', }, { - name: "cachetools", - version: "5.3.0", + name: 'cachetools', + version: '5.3.0', }, ], - relativePath: "test-requirements-txt/requirements.txt", - repositoryName: "transcend-io/cli", + relativePath: 'test-requirements-txt/requirements.txt', + repositoryName: 'transcend-io/cli', }, { - name: "test-nested-requirements-txt", - type: "REQUIREMENTS_TXT", + name: 'test-nested-requirements-txt', + type: 'REQUIREMENTS_TXT', description: undefined, softwareDevelopmentKits: [ { - name: "pyarrow", - version: "14.0.1", + name: 'pyarrow', + version: '14.0.1', }, { - name: "pandas", - version: "2.0.3", + name: 'pandas', + version: '2.0.3', }, ], relativePath: - "test-package-json/test-nested-requirements-txt/requirements.txt", - repositoryName: "transcend-io/cli", + 'test-package-json/test-nested-requirements-txt/requirements.txt', + repositoryName: 'transcend-io/cli', }, { - name: "test-gemfile", - type: "GEMFILE", + name: 'test-gemfile', + type: 'GEMFILE', description: undefined, softwareDevelopmentKits: [ { - name: "rails", - version: "~> 6.1.4", + name: 'rails', + version: '~> 6.1.4', }, { - name: "ahoy_matey", + name: 'ahoy_matey', version: undefined, }, { - name: "rack-tracker", + name: 'rack-tracker', version: undefined, }, { - name: "adroll", + name: 'adroll', version: undefined, }, { - name: "google-ads-googleads", + name: 'google-ads-googleads', version: undefined, }, { - name: "facebookads", + name: 'facebookads', version: undefined, }, { - name: "devise", + name: 'devise', version: undefined, }, { - name: "impressionist", + name: 'impressionist', version: undefined, }, { - name: "sidekiq", + name: 'sidekiq', version: undefined, }, { - name: "sidekiq-cron", - version: "~> 1.2", + name: 'sidekiq-cron', + version: '~> 1.2', }, { - name: "byebug", + name: 'byebug', version: undefined, }, { - name: "listen", - version: "~> 3.3", + name: 'listen', + version: '~> 3.3', }, { - name: "capybara", - version: ">= 2.15", + name: 'capybara', + version: '>= 2.15', }, { - name: "selenium-webdriver", + name: 'selenium-webdriver', version: undefined, }, { - name: "webdrivers", + name: 'webdrivers', version: undefined, }, { - name: "bundler-audit", + name: 'bundler-audit', version: undefined, }, ], - relativePath: "test-gemfile/Gemfile", - repositoryName: "transcend-io/cli", + relativePath: 'test-gemfile/Gemfile', + repositoryName: 'transcend-io/cli', }, { - name: "test-nested-gemfile", - type: "GEMFILE", + name: 'test-nested-gemfile', + type: 'GEMFILE', softwareDevelopmentKits: [ { - name: "rails", - version: "~> 6.1.4", + name: 'rails', + version: '~> 6.1.4', }, { - name: "ahoy_matey", + name: 'ahoy_matey', version: undefined, }, { - name: "rack-tracker", + name: 'rack-tracker', version: undefined, }, { - name: "adroll", + name: 'adroll', version: undefined, }, { - name: "google-ads-googleads", + name: 'google-ads-googleads', version: undefined, }, { - name: "facebookads", + name: 'facebookads', version: undefined, }, { - name: "byebug", + name: 'byebug', version: undefined, }, { - name: "listen", - version: "~> 3.3", + name: 'listen', + version: '~> 3.3', }, { - name: "capybara", - version: ">= 2.15", + name: 'capybara', + version: '>= 2.15', }, { - name: "selenium-webdriver", + name: 'selenium-webdriver', version: undefined, }, { - name: "webdrivers", + name: 'webdrivers', version: undefined, }, { - name: "bundler-audit", + name: 'bundler-audit', version: undefined, }, ], description: undefined, - relativePath: "test-gradle/test-nested-gemfile/Gemfile", - repositoryName: "transcend-io/cli", + relativePath: 'test-gradle/test-nested-gemfile/Gemfile', + repositoryName: 'transcend-io/cli', }, { - name: "example", - description: "test example app", - type: "PUBSPEC", + name: 'example', + description: 'test example app', + type: 'PUBSPEC', softwareDevelopmentKits: [ { - name: "flutter", - version: "flutter", + name: 'flutter', + version: 'flutter', }, { - name: "flutter_localizations", - version: "flutter", + name: 'flutter_localizations', + version: 'flutter', }, { - name: "firebase_core", - version: "2.16.0", + name: 'firebase_core', + version: '2.16.0', }, { - name: "firebase_analytics", - version: "10.5.0", + name: 'firebase_analytics', + version: '10.5.0', }, { - name: "firebase_crashlytics", - version: "3.3.6", + name: 'firebase_crashlytics', + version: '3.3.6', }, { - name: "video_player", - version: "2.6.1", + name: 'video_player', + version: '2.6.1', }, { - name: "appsflyer_sdk", - version: "6.12.2", + name: 'appsflyer_sdk', + version: '6.12.2', }, { - name: "isolate", - version: "2.1.1", + name: 'isolate', + version: '2.1.1', }, { - name: "custom_platform_device_id", - version: "1.0.8", + name: 'custom_platform_device_id', + version: '1.0.8', }, { - name: "image_editor", - version: "1.3.0", + name: 'image_editor', + version: '1.3.0', }, { - name: "firebase_remote_config", - version: "4.2.6", + name: 'firebase_remote_config', + version: '4.2.6', }, { - name: "intercom_flutter", - version: "7.8.4", + name: 'intercom_flutter', + version: '7.8.4', }, { - name: "dismissible_page", - version: "1.0.2", + name: 'dismissible_page', + version: '1.0.2', }, { - name: "extended_text", - version: "11.1.0", + name: 'extended_text', + version: '11.1.0', }, { - name: "recaptcha_enterprise_flutter", - version: "18.3.0", + name: 'recaptcha_enterprise_flutter', + version: '18.3.0', }, { - name: "flutter_test", - version: "flutter", + name: 'flutter_test', + version: 'flutter', isDevDependency: true, }, { - name: "test", - version: "1.24.3", + name: 'test', + version: '1.24.3', isDevDependency: true, }, { - name: "lints", - version: "3.0.0", + name: 'lints', + version: '3.0.0', isDevDependency: true, }, { - name: "mocktail", - version: "1.0.1", + name: 'mocktail', + version: '1.0.1', isDevDependency: true, }, ], - relativePath: "test-pubspec/pubspec.yml", - repositoryName: "transcend-io/cli", + relativePath: 'test-pubspec/pubspec.yml', + repositoryName: 'transcend-io/cli', }, { - name: "composer/example", - description: "Example app", + name: 'composer/example', + description: 'Example app', softwareDevelopmentKits: [ { - name: "php", - version: "^7.2.5 || ^8.0", + name: 'php', + version: '^7.2.5 || ^8.0', }, { - name: "composer/ca-bundle", - version: "^1.5", + name: 'composer/ca-bundle', + version: '^1.5', }, { - name: "composer/class-map-generator", - version: "^1.3.3", + name: 'composer/class-map-generator', + version: '^1.3.3', }, { - name: "composer/metadata-minifier", - version: "^1.0", + name: 'composer/metadata-minifier', + version: '^1.0', }, { - name: "composer/semver", - version: "^3.3", + name: 'composer/semver', + version: '^3.3', }, { - name: "composer/spdx-licenses", - version: "^1.5.7", + name: 'composer/spdx-licenses', + version: '^1.5.7', }, { - name: "composer/xdebug-handler", - version: "^2.0.2 || ^3.0.3", + name: 'composer/xdebug-handler', + version: '^2.0.2 || ^3.0.3', }, { - name: "justinrainbow/json-schema", - version: "^5.3", + name: 'justinrainbow/json-schema', + version: '^5.3', }, { - name: "psr/log", - version: "^1.0 || ^2.0 || ^3.0", + name: 'psr/log', + version: '^1.0 || ^2.0 || ^3.0', }, { - name: "seld/jsonlint", - version: "^1.4", + name: 'seld/jsonlint', + version: '^1.4', }, { - name: "seld/phar-utils", - version: "^1.2", + name: 'seld/phar-utils', + version: '^1.2', }, { - name: "symfony/console", - version: "^5.4.35 || ^6.3.12 || ^7.0.3", + name: 'symfony/console', + version: '^5.4.35 || ^6.3.12 || ^7.0.3', }, { - name: "symfony/filesystem", - version: "^5.4.35 || ^6.3.12 || ^7.0.3", + name: 'symfony/filesystem', + version: '^5.4.35 || ^6.3.12 || ^7.0.3', }, { - name: "symfony/finder", - version: "^5.4.35 || ^6.3.12 || ^7.0.3", + name: 'symfony/finder', + version: '^5.4.35 || ^6.3.12 || ^7.0.3', }, { - name: "symfony/process", - version: "^5.4.35 || ^6.3.12 || ^7.0.3", + name: 'symfony/process', + version: '^5.4.35 || ^6.3.12 || ^7.0.3', }, { - name: "react/promise", - version: "^3.2", + name: 'react/promise', + version: '^3.2', }, { - name: "composer/pcre", - version: "^2.2 || ^3.2", + name: 'composer/pcre', + version: '^2.2 || ^3.2', }, { - name: "symfony/polyfill-php73", - version: "^1.24", + name: 'symfony/polyfill-php73', + version: '^1.24', }, { - name: "symfony/polyfill-php80", - version: "^1.24", + name: 'symfony/polyfill-php80', + version: '^1.24', }, { - name: "symfony/polyfill-php81", - version: "^1.24", + name: 'symfony/polyfill-php81', + version: '^1.24', }, { - name: "seld/signal-handler", - version: "^2.0", + name: 'seld/signal-handler', + version: '^2.0', }, { - name: "symfony/phpunit-bridge", - version: "^6.4.3 || ^7.0.1", + name: 'symfony/phpunit-bridge', + version: '^6.4.3 || ^7.0.1', isDevDependency: true, }, { - name: "phpstan/phpstan", - version: "^1.11.8", + name: 'phpstan/phpstan', + version: '^1.11.8', isDevDependency: true, }, { - name: "phpstan/phpstan-phpunit", - version: "^1.4.0", + name: 'phpstan/phpstan-phpunit', + version: '^1.4.0', isDevDependency: true, }, { - name: "phpstan/phpstan-deprecation-rules", - version: "^1.2.0", + name: 'phpstan/phpstan-deprecation-rules', + version: '^1.2.0', isDevDependency: true, }, { - name: "phpstan/phpstan-strict-rules", - version: "^1.6.0", + name: 'phpstan/phpstan-strict-rules', + version: '^1.6.0', isDevDependency: true, }, { - name: "phpstan/phpstan-symfony", - version: "^1.4.0", + name: 'phpstan/phpstan-symfony', + version: '^1.4.0', isDevDependency: true, }, ], - relativePath: "test-php/composer.json", - type: "COMPOSER_JSON", - repositoryName: "transcend-io/cli", + relativePath: 'test-php/composer.json', + type: 'COMPOSER_JSON', + repositoryName: 'transcend-io/cli', }, { - name: "test-swift", - type: "SWIFT", - relativePath: "test-swift/Package.resolved", - repositoryName: "transcend-io/cli", + name: 'test-swift', + type: 'SWIFT', + relativePath: 'test-swift/Package.resolved', + repositoryName: 'transcend-io/cli', softwareDevelopmentKits: [ { - name: "alamofire", - version: "5.8.1", + name: 'alamofire', + version: '5.8.1', }, { - name: "swift-numerics", - version: "1.0.2", + name: 'swift-numerics', + version: '1.0.2', }, ], }, @@ -706,24 +706,24 @@ const expected: CodePackageInput[] = [ * @returns Sorted code packages */ function sortCodePackages( - codePackages: CodePackageInput[] + codePackages: CodePackageInput[], ): CodePackageInput[] { return codePackages .sort((a, b) => a.name.localeCompare(b.name)) .map((c) => ({ ...c, softwareDevelopmentKits: c.softwareDevelopmentKits?.sort((a, b) => - a.name.localeCompare(b.name) + a.name.localeCompare(b.name), ), })); } // not easy to test this but can uncomment to run against current commit -describe("findCodePackagesInFolder", () => { - it("should remove links", async () => { +describe('findCodePackagesInFolder', () => { + it('should remove links', async () => { const result = await findCodePackagesInFolder({ - repositoryName: "transcend-io/cli", - scanPath: join(__dirname, "../../../examples/code-scanning"), + repositoryName: 'transcend-io/cli', + scanPath: join(__dirname, '../../../examples/code-scanning'), }); expect(sortCodePackages(result)).to.deep.equal(sortCodePackages(expected)); }); diff --git a/src/lib/tests/getGitFilesThatChanged.test.ts b/src/lib/tests/getGitFilesThatChanged.test.ts index a90c7116..837126de 100644 --- a/src/lib/tests/getGitFilesThatChanged.test.ts +++ b/src/lib/tests/getGitFilesThatChanged.test.ts @@ -1,22 +1,22 @@ -import { join } from "node:path"; -import { describe, expect, it } from "vitest"; -import { getGitFilesThatChanged } from "../ai/getGitFilesThatChanged"; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { getGitFilesThatChanged } from '../ai/getGitFilesThatChanged'; // not easy to test this but can uncomment to run against current commit -describe.skip("getGitFilesThatChanged", () => { - it("should remove links", () => { +describe.skip('getGitFilesThatChanged', () => { + it('should remove links', () => { expect( getGitFilesThatChanged({ - baseBranch: "main", - githubRepo: "https://github.com/transcend-io/cli.git", - rootDirectory: join(__dirname, "../../"), - }) + baseBranch: 'main', + githubRepo: 'https://github.com/transcend-io/cli.git', + rootDirectory: join(__dirname, '../../'), + }), ).to.deep.equal({ changedFiles: [ - "package.json", - "src/ai/TranscendAiPrompt.ts", - "src/cli-analyze-pull-request.ts", - "src/tests/TranscendAiPrompt.test.ts", + 'package.json', + 'src/ai/TranscendAiPrompt.ts', + 'src/cli-analyze-pull-request.ts', + 'src/tests/TranscendAiPrompt.test.ts', ], fileDiffs: {}, }); diff --git a/src/lib/tests/readTranscendYaml.test.ts b/src/lib/tests/readTranscendYaml.test.ts index 2c632f87..050ae708 100644 --- a/src/lib/tests/readTranscendYaml.test.ts +++ b/src/lib/tests/readTranscendYaml.test.ts @@ -1,18 +1,18 @@ -import { join } from "node:path"; -import { describe, expect, it } from "vitest"; -import { readTranscendYaml } from "../../index"; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { readTranscendYaml } from '../../index'; -const EXAMPLE_DIR = join(__dirname, "..", "..", "..", "examples"); +const EXAMPLE_DIR = join(__dirname, '..', '..', '..', 'examples'); -describe("readTranscendYaml", () => { - it("simple.yml should pass the codec validation for TranscendInput", () => { +describe('readTranscendYaml', () => { + it('simple.yml should pass the codec validation for TranscendInput', () => { expect(() => - readTranscendYaml(join(EXAMPLE_DIR, "simple.yml")) + readTranscendYaml(join(EXAMPLE_DIR, 'simple.yml')), ).to.not.throw(); }); - it("invalid.yml should fail the codec validation for TranscendInput", () => { - expect(() => readTranscendYaml(join(EXAMPLE_DIR, "invalid.yml"))).to + it('invalid.yml should fail the codec validation for TranscendInput', () => { + expect(() => readTranscendYaml(join(EXAMPLE_DIR, 'invalid.yml'))).to .throw(` ".enrichers.0.0.title expected type 'string'", ".enrichers.1.0.output-identifiers expected type 'Array'", ".data-silos.0.0.title expected type 'string'", @@ -22,26 +22,26 @@ describe("readTranscendYaml", () => { ".data-silos.1.1.disabled expected type 'boolean'"`); }); - it("multi-instance.yml should fail when no variables are provided", () => { + it('multi-instance.yml should fail when no variables are provided', () => { expect(() => - readTranscendYaml(join(EXAMPLE_DIR, "multi-instance.yml")) - ).to.throw("Found variable that was not set: domain"); + readTranscendYaml(join(EXAMPLE_DIR, 'multi-instance.yml')), + ).to.throw('Found variable that was not set: domain'); }); - it("multi-instance.yml should be successful when variables are provided", () => { - const result = readTranscendYaml(join(EXAMPLE_DIR, "multi-instance.yml"), { - domain: "acme.com", - stage: "Staging", + it('multi-instance.yml should be successful when variables are provided', () => { + const result = readTranscendYaml(join(EXAMPLE_DIR, 'multi-instance.yml'), { + domain: 'acme.com', + stage: 'Staging', }); expect(result.enrichers![0].url).to.equal( - "https://example.acme.com/transcend-enrichment-webhook" + 'https://example.acme.com/transcend-enrichment-webhook', ); expect(result.enrichers![1].url).to.equal( - "https://example.acme.com/transcend-fraud-check" + 'https://example.acme.com/transcend-fraud-check', ); - expect(result["data-silos"]![0].description).to.equal( - "The mega-warehouse that contains a copy over all SQL backed databases - Staging" + expect(result['data-silos']![0].description).to.equal( + 'The mega-warehouse that contains a copy over all SQL backed databases - Staging', ); }); }); diff --git a/src/logger.ts b/src/logger.ts index 16c9a276..5ffebdb1 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,13 +1,13 @@ -import colors from "colors"; -import { bootstrap } from "global-agent"; -import { ProxyAgent, setGlobalDispatcher } from "undici"; -import yargs from "yargs-parser"; +import colors from 'colors'; +import { bootstrap } from 'global-agent'; +import { ProxyAgent, setGlobalDispatcher } from 'undici'; +import yargs from 'yargs-parser'; export const logger = console; // When the proxy env var of flag is specified, initiate the proxy const { httpProxy = process.env.http_proxy } = yargs(process.argv.slice(2)); -if (httpProxy && typeof httpProxy === "string") { +if (httpProxy && typeof httpProxy === 'string') { logger.info(colors.green(`Initializing proxy: ${httpProxy}`)); // Use global-agent, which overrides `request` based requests From 7877a1ddfe0be9d3a789973743d24bac82921013 Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Wed, 16 Jul 2025 21:38:09 -0400 Subject: [PATCH 04/18] fix scripts --- README.md | 92 +++++++++---------- package.json | 2 +- scripts/buildPathfinderJsonSchema.ts | 9 +- .../{buildReadmeDocs.ts => buildReadme.ts} | 32 ++++--- scripts/buildTranscendJsonSchema.ts | 9 +- 5 files changed, 78 insertions(+), 66 deletions(-) rename scripts/{buildReadmeDocs.ts => buildReadme.ts} (65%) diff --git a/README.md b/README.md index 4a243c5a..30de3733 100644 --- a/README.md +++ b/README.md @@ -10,52 +10,52 @@ - [Installation](#installation) - [transcend.yml](#transcendyml) - [Usage](#usage) - - [`transcend request approve`](#transcend-request-approve) - - [`transcend request upload`](#transcend-request-upload) - - [`transcend request download-files`](#transcend-request-download-files) - - [`transcend request cancel`](#transcend-request-cancel) - - [`transcend request restart`](#transcend-request-restart) - - [`transcend request notify-additional-time`](#transcend-request-notify-additional-time) - - [`transcend request mark-silent`](#transcend-request-mark-silent) - - [`transcend request enricher-restart`](#transcend-request-enricher-restart) - - [`transcend request reject-unverified-identifiers`](#transcend-request-reject-unverified-identifiers) - - [`transcend request export`](#transcend-request-export) - - [`transcend request skip-preflight-jobs`](#transcend-request-skip-preflight-jobs) - - [`transcend request system mark-request-data-silos-completed`](#transcend-request-system-mark-request-data-silos-completed) - - [`transcend request system retry-request-data-silos`](#transcend-request-system-retry-request-data-silos) - - [`transcend request system skip-request-data-silos`](#transcend-request-system-skip-request-data-silos) - - [`transcend request preflight pull-identifiers`](#transcend-request-preflight-pull-identifiers) - - [`transcend request preflight push-identifiers`](#transcend-request-preflight-push-identifiers) - - [`transcend request cron pull-identifiers`](#transcend-request-cron-pull-identifiers) - - [`transcend request cron mark-identifiers-completed`](#transcend-request-cron-mark-identifiers-completed) - - [`transcend consent build-xdi-sync-endpoint`](#transcend-consent-build-xdi-sync-endpoint) - - [`transcend consent pull-consent-metrics`](#transcend-consent-pull-consent-metrics) - - [`transcend consent pull-consent-preferences`](#transcend-consent-pull-consent-preferences) - - [`transcend consent update-consent-manager`](#transcend-consent-update-consent-manager) - - [`transcend consent upload-consent-preferences`](#transcend-consent-upload-consent-preferences) - - [`transcend consent upload-cookies-from-csv`](#transcend-consent-upload-cookies-from-csv) - - [`transcend consent upload-data-flows-from-csv`](#transcend-consent-upload-data-flows-from-csv) - - [`transcend consent upload-preferences`](#transcend-consent-upload-preferences) - - [`transcend inventory pull`](#transcend-inventory-pull) - - [Scopes](#scopes) - - [Usage](#usage-1) - - [`transcend inventory push`](#transcend-inventory-push) - - [Scopes](#scopes-1) - - [Usage](#usage-2) - - [CI Integration](#ci-integration) - - [Dynamic Variables](#dynamic-variables) - - [`transcend inventory scan-packages`](#transcend-inventory-scan-packages) - - [`transcend inventory discover-silos`](#transcend-inventory-discover-silos) - - [Usage](#usage-3) - - [`transcend inventory pull-datapoints`](#transcend-inventory-pull-datapoints) - - [`transcend inventory pull-unstructured-discovery-files`](#transcend-inventory-pull-unstructured-discovery-files) - - [`transcend inventory derive-data-silos-from-data-flows`](#transcend-inventory-derive-data-silos-from-data-flows) - - [`transcend inventory derive-data-silos-from-data-flows-cross-instance`](#transcend-inventory-derive-data-silos-from-data-flows-cross-instance) - - [`transcend inventory consent-manager-service-json-to-yml`](#transcend-inventory-consent-manager-service-json-to-yml) - - [`transcend inventory consent-managers-to-business-entities`](#transcend-inventory-consent-managers-to-business-entities) - - [`transcend admin generate-api-keys`](#transcend-admin-generate-api-keys) - - [Usage](#usage-4) - - [`transcend migration sync-ot`](#transcend-migration-sync-ot) + - [`transcend request approve`](#transcend-request-approve) + - [`transcend request upload`](#transcend-request-upload) + - [`transcend request download-files`](#transcend-request-download-files) + - [`transcend request cancel`](#transcend-request-cancel) + - [`transcend request restart`](#transcend-request-restart) + - [`transcend request notify-additional-time`](#transcend-request-notify-additional-time) + - [`transcend request mark-silent`](#transcend-request-mark-silent) + - [`transcend request enricher-restart`](#transcend-request-enricher-restart) + - [`transcend request reject-unverified-identifiers`](#transcend-request-reject-unverified-identifiers) + - [`transcend request export`](#transcend-request-export) + - [`transcend request skip-preflight-jobs`](#transcend-request-skip-preflight-jobs) + - [`transcend request system mark-request-data-silos-completed`](#transcend-request-system-mark-request-data-silos-completed) + - [`transcend request system retry-request-data-silos`](#transcend-request-system-retry-request-data-silos) + - [`transcend request system skip-request-data-silos`](#transcend-request-system-skip-request-data-silos) + - [`transcend request preflight pull-identifiers`](#transcend-request-preflight-pull-identifiers) + - [`transcend request preflight push-identifiers`](#transcend-request-preflight-push-identifiers) + - [`transcend request cron pull-identifiers`](#transcend-request-cron-pull-identifiers) + - [`transcend request cron mark-identifiers-completed`](#transcend-request-cron-mark-identifiers-completed) + - [`transcend consent build-xdi-sync-endpoint`](#transcend-consent-build-xdi-sync-endpoint) + - [`transcend consent pull-consent-metrics`](#transcend-consent-pull-consent-metrics) + - [`transcend consent pull-consent-preferences`](#transcend-consent-pull-consent-preferences) + - [`transcend consent update-consent-manager`](#transcend-consent-update-consent-manager) + - [`transcend consent upload-consent-preferences`](#transcend-consent-upload-consent-preferences) + - [`transcend consent upload-cookies-from-csv`](#transcend-consent-upload-cookies-from-csv) + - [`transcend consent upload-data-flows-from-csv`](#transcend-consent-upload-data-flows-from-csv) + - [`transcend consent upload-preferences`](#transcend-consent-upload-preferences) + - [`transcend inventory pull`](#transcend-inventory-pull) + - [Scopes](#scopes) + - [Usage](#usage-1) + - [`transcend inventory push`](#transcend-inventory-push) + - [Scopes](#scopes-1) + - [Usage](#usage-2) + - [CI Integration](#ci-integration) + - [Dynamic Variables](#dynamic-variables) + - [`transcend inventory scan-packages`](#transcend-inventory-scan-packages) + - [`transcend inventory discover-silos`](#transcend-inventory-discover-silos) + - [Usage](#usage-3) + - [`transcend inventory pull-datapoints`](#transcend-inventory-pull-datapoints) + - [`transcend inventory pull-unstructured-discovery-files`](#transcend-inventory-pull-unstructured-discovery-files) + - [`transcend inventory derive-data-silos-from-data-flows`](#transcend-inventory-derive-data-silos-from-data-flows) + - [`transcend inventory derive-data-silos-from-data-flows-cross-instance`](#transcend-inventory-derive-data-silos-from-data-flows-cross-instance) + - [`transcend inventory consent-manager-service-json-to-yml`](#transcend-inventory-consent-manager-service-json-to-yml) + - [`transcend inventory consent-managers-to-business-entities`](#transcend-inventory-consent-managers-to-business-entities) + - [`transcend admin generate-api-keys`](#transcend-admin-generate-api-keys) + - [Usage](#usage-4) + - [`transcend migration sync-ot`](#transcend-migration-sync-ot) - [Prompt Manager](#prompt-manager) - [Proxy usage](#proxy-usage) diff --git a/package.json b/package.json index 06a9394f..26b7f71b 100644 --- a/package.json +++ b/package.json @@ -73,7 +73,7 @@ "test": "vitest run", "script:transcend-json-schema": "tsx scripts/buildTranscendJsonSchema.ts && prettier ./transcend-yml-schema-*.json --write", "script:pathfinder-json-schema": "tsx scripts/buildPathfinderJsonSchema.ts && prettier ./pathfinder-policy-yml-schema.json --write", - "docgen": "tsx scripts/buildReadmeDocs.ts" + "docgen": "tsx scripts/buildReadme.ts" }, "tsup": { "entry": [ diff --git a/scripts/buildPathfinderJsonSchema.ts b/scripts/buildPathfinderJsonSchema.ts index ba22da89..2bf96766 100644 --- a/scripts/buildPathfinderJsonSchema.ts +++ b/scripts/buildPathfinderJsonSchema.ts @@ -13,7 +13,7 @@ */ import { writeFileSync } from 'node:fs'; -import { join } from 'node:path'; +import path from 'node:path'; import { toJsonSchema } from '@transcend-io/type-utils'; import { PathfinderPolicy } from '../src/codecs'; @@ -30,6 +30,9 @@ const jsonSchema = { ...toJsonSchema(PathfinderPolicy, true), }; -const schemaFilePath = join(process.cwd(), 'pathfinder-policy-yml-schema.json'); +const schemaFilePath = path.join( + process.cwd(), + 'pathfinder-policy-yml-schema.json', +); -writeFileSync(schemaFilePath, `${JSON.stringify(jsonSchema, null, 2)}\n`); +writeFileSync(schemaFilePath, `${JSON.stringify(jsonSchema, undefined, 2)}\n`); diff --git a/scripts/buildReadmeDocs.ts b/scripts/buildReadme.ts similarity index 65% rename from scripts/buildReadmeDocs.ts rename to scripts/buildReadme.ts index c02b9cb9..0dcc1fe7 100644 --- a/scripts/buildReadmeDocs.ts +++ b/scripts/buildReadme.ts @@ -14,17 +14,20 @@ const documentFiles = new fdir() .crawl('./src/commands') .sync(); -// For each src/commands/**/readme.ts file, create a key-value pair of the command and the exported Markdown documentation -const additionalDocumentation: Record = Object.fromEntries( - await Promise.all( - documentFiles.map(async (file) => { - const command = `transcend ${file.split('/').slice(0, -1).join(' ')}`; - const readme = (await import(`../src/commands/${file}`)).default; - return [command, readme]; - }), - ), +const entriesOfAdditionalDocumentation: [string, string][] = await Promise.all( + documentFiles.map(async (file) => { + const command = `transcend ${file.split('/').slice(0, -1).join(' ')}`; + const { default: readme } = (await import(`../src/commands/${file}`)) as { + default: string; + }; + return [command, readme]; + }), ); +// For each src/commands/**/readme.ts file, create a key-value pair of the command and the exported Markdown documentation +const additionalDocumentation: Record = + Object.fromEntries(entriesOfAdditionalDocumentation); + const helpTextForAllCommands = generateHelpTextForAllCommands( app as Application, ); @@ -48,7 +51,10 @@ const newReadme = readme.replaceAll( fs.writeFileSync('README.md', newReadme); -execSync('doctoc README.md --title "\n## Table of Contents" --maxlevel 5', { - stdio: 'inherit', -}); -execSync('prettier --write README.md', { stdio: 'inherit' }); +execSync( + 'pnpm exec doctoc README.md --title "\n## Table of Contents" --maxlevel 5', + { + stdio: 'inherit', + }, +); +execSync('pnpm exec prettier --write README.md', { stdio: 'inherit' }); diff --git a/scripts/buildTranscendJsonSchema.ts b/scripts/buildTranscendJsonSchema.ts index 9c00ec6c..da9361af 100644 --- a/scripts/buildTranscendJsonSchema.ts +++ b/scripts/buildTranscendJsonSchema.ts @@ -13,7 +13,7 @@ */ import { writeFileSync } from 'node:fs'; -import { join } from 'node:path'; +import path from 'node:path'; import { toJsonSchema } from '@transcend-io/type-utils'; import * as packageJson from '../package.json'; import { TranscendInput } from '../src/codecs'; @@ -35,7 +35,10 @@ for (const key of [`v${majorVersion}`, 'latest']) { // Build the JSON schema from io-ts codec const jsonSchema = { ...schemaDefaults, ...toJsonSchema(TranscendInput) }; - const schemaFilePath = join(process.cwd(), fileName); + const schemaFilePath = path.join(process.cwd(), fileName); - writeFileSync(schemaFilePath, `${JSON.stringify(jsonSchema, null, 2)}\n`); + writeFileSync( + schemaFilePath, + `${JSON.stringify(jsonSchema, undefined, 2)}\n`, + ); } From 369cebb0b4221e4a165401314293cdcd206d8f5d Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Wed, 16 Jul 2025 21:39:36 -0400 Subject: [PATCH 05/18] readTranscendYaml --- src/lib/tests/readTranscendYaml.test.ts | 33 ++++++++++++++++--------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/src/lib/tests/readTranscendYaml.test.ts b/src/lib/tests/readTranscendYaml.test.ts index 050ae708..0b2d9a48 100644 --- a/src/lib/tests/readTranscendYaml.test.ts +++ b/src/lib/tests/readTranscendYaml.test.ts @@ -1,18 +1,24 @@ -import { join } from 'node:path'; +import path from 'node:path'; import { describe, expect, it } from 'vitest'; import { readTranscendYaml } from '../../index'; -const EXAMPLE_DIR = join(__dirname, '..', '..', '..', 'examples'); +const EXAMPLE_DIR = path.join( + import.meta.dirname, + '..', + '..', + '..', + 'examples', +); describe('readTranscendYaml', () => { it('simple.yml should pass the codec validation for TranscendInput', () => { expect(() => - readTranscendYaml(join(EXAMPLE_DIR, 'simple.yml')), + readTranscendYaml(path.join(EXAMPLE_DIR, 'simple.yml')), ).to.not.throw(); }); it('invalid.yml should fail the codec validation for TranscendInput', () => { - expect(() => readTranscendYaml(join(EXAMPLE_DIR, 'invalid.yml'))).to + expect(() => readTranscendYaml(path.join(EXAMPLE_DIR, 'invalid.yml'))).to .throw(` ".enrichers.0.0.title expected type 'string'", ".enrichers.1.0.output-identifiers expected type 'Array'", ".data-silos.0.0.title expected type 'string'", @@ -24,23 +30,26 @@ describe('readTranscendYaml', () => { it('multi-instance.yml should fail when no variables are provided', () => { expect(() => - readTranscendYaml(join(EXAMPLE_DIR, 'multi-instance.yml')), + readTranscendYaml(path.join(EXAMPLE_DIR, 'multi-instance.yml')), ).to.throw('Found variable that was not set: domain'); }); it('multi-instance.yml should be successful when variables are provided', () => { - const result = readTranscendYaml(join(EXAMPLE_DIR, 'multi-instance.yml'), { - domain: 'acme.com', - stage: 'Staging', - }); + const result = readTranscendYaml( + path.join(EXAMPLE_DIR, 'multi-instance.yml'), + { + domain: 'acme.com', + stage: 'Staging', + }, + ); - expect(result.enrichers![0].url).to.equal( + expect(result.enrichers?.[0].url).to.equal( 'https://example.acme.com/transcend-enrichment-webhook', ); - expect(result.enrichers![1].url).to.equal( + expect(result.enrichers?.[1].url).to.equal( 'https://example.acme.com/transcend-fraud-check', ); - expect(result['data-silos']![0].description).to.equal( + expect(result['data-silos']?.[0].description).to.equal( 'The mega-warehouse that contains a copy over all SQL backed databases - Staging', ); }); From eb2a970a190477ed9fef1127ccb133423155abd6 Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Wed, 16 Jul 2025 21:41:43 -0400 Subject: [PATCH 06/18] yml --- src/lib/tests/getGitFilesThatChanged.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/tests/getGitFilesThatChanged.test.ts b/src/lib/tests/getGitFilesThatChanged.test.ts index 837126de..138aaf11 100644 --- a/src/lib/tests/getGitFilesThatChanged.test.ts +++ b/src/lib/tests/getGitFilesThatChanged.test.ts @@ -1,4 +1,4 @@ -import { join } from 'node:path'; +import path from 'node:path'; import { describe, expect, it } from 'vitest'; import { getGitFilesThatChanged } from '../ai/getGitFilesThatChanged'; @@ -9,7 +9,7 @@ describe.skip('getGitFilesThatChanged', () => { getGitFilesThatChanged({ baseBranch: 'main', githubRepo: 'https://github.com/transcend-io/cli.git', - rootDirectory: join(__dirname, '../../'), + rootDirectory: path.join(import.meta.dirname, '../../'), }), ).to.deep.equal({ changedFiles: [ From 09b0968c93dafadf284ab1f4c9ece8fd7ca6492f Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Wed, 16 Jul 2025 21:52:13 -0400 Subject: [PATCH 07/18] import path --- .../consent/pull-consent-metrics/impl.ts | 8 ++++---- .../consent/upload-preferences/impl.ts | 8 ++++---- .../impl.ts | 4 ++-- .../impl.ts | 4 ++-- .../derive-data-silos-from-data-flows/impl.ts | 6 +++--- src/commands/inventory/pull/impl.ts | 4 ++-- src/commands/inventory/push/impl.ts | 4 ++-- src/lib/api-keys/listDirectories.ts | 4 ++-- .../integrations/composerJson.ts | 4 ++-- src/lib/code-scanning/integrations/gemfile.ts | 4 ++-- src/lib/code-scanning/integrations/gradle.ts | 4 ++-- .../integrations/javascriptPackageJson.ts | 4 ++-- src/lib/code-scanning/integrations/pubspec.ts | 4 ++-- .../integrations/pythonRequirementsTxt.ts | 6 +++--- src/lib/code-scanning/integrations/swift.ts | 4 ++-- src/lib/requests/bulkRestartRequests.ts | 4 ++-- .../requests/downloadPrivacyRequestFiles.ts | 8 ++++---- src/lib/requests/tests/readCsv.test.ts | 19 ++++++++++++++----- .../requests/uploadPrivacyRequestsFromCsv.ts | 4 ++-- .../tests/findCodePackagesInFolder.test.ts | 4 ++-- 20 files changed, 60 insertions(+), 51 deletions(-) diff --git a/src/commands/consent/pull-consent-metrics/impl.ts b/src/commands/consent/pull-consent-metrics/impl.ts index 0b1c58b2..4b6c3501 100644 --- a/src/commands/consent/pull-consent-metrics/impl.ts +++ b/src/commands/consent/pull-consent-metrics/impl.ts @@ -1,5 +1,5 @@ import fs, { existsSync, mkdirSync } from 'node:fs'; -import { join } from 'node:path'; +import path from 'node:path'; import colors from 'colors'; import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; import type { LocalContext } from '../../../context'; @@ -116,7 +116,7 @@ export async function pullConsentMetrics( // Write to file for (const [metricName, metrics] of Object.entries(configuration)) { for (const { points, name } of metrics) { - const file = join(folder, `${metricName}_${name}.csv`); + const file = path.join(folder, `${metricName}_${name}.csv`); logger.info( colors.magenta(`Writing configuration to file "${file}"...`), ); @@ -165,7 +165,7 @@ export async function pullConsentMetrics( }); // ensure folder exists for that organization - const subFolder = join(folder, apiKey.organizationName); + const subFolder = path.join(folder, apiKey.organizationName); if (!existsSync(subFolder)) { mkdirSync(subFolder); } @@ -173,7 +173,7 @@ export async function pullConsentMetrics( // Write to file for (const [metricName, metrics] of Object.entries(configuration)) { for (const { points, name } of metrics) { - const file = join(subFolder, `${metricName}_${name}.csv`); + const file = path.join(subFolder, `${metricName}_${name}.csv`); logger.info( colors.magenta(`Writing configuration to file "${file}"...`), ); diff --git a/src/commands/consent/upload-preferences/impl.ts b/src/commands/consent/upload-preferences/impl.ts index d8c64d03..e0e903cf 100644 --- a/src/commands/consent/upload-preferences/impl.ts +++ b/src/commands/consent/upload-preferences/impl.ts @@ -1,5 +1,5 @@ import { readdirSync } from 'node:fs'; -import { basename, join } from 'node:path'; +import path from 'node:path'; import colors from 'colors'; import type { LocalContext } from '../../../context'; import { map } from '../../../lib/bluebird-replace'; @@ -79,7 +79,7 @@ export async function uploadPreferences( } // Add full paths for each CSV file - files.push(...csvFiles.map((file) => join(directory, file))); + files.push(...csvFiles.map((file) => path.join(directory, file))); } catch (error) { logger.error(colors.red(`Failed to read directory: ${directory}`)); logger.error(colors.red((error as Error).message)); @@ -118,9 +118,9 @@ export async function uploadPreferences( await map( files, async (filePath) => { - const fileName = basename(filePath).replace('.csv', ''); + const fileName = path.basename(filePath).replace('.csv', ''); await uploadPreferenceManagementPreferencesInteractive({ - receiptFilepath: join(receiptFileDir, `${fileName}-receipts.json`), + receiptFilepath: path.join(receiptFileDir, `${fileName}-receipts.json`), auth, sombraAuth, file: filePath, diff --git a/src/commands/inventory/consent-managers-to-business-entities/impl.ts b/src/commands/inventory/consent-managers-to-business-entities/impl.ts index e379557a..1913f227 100644 --- a/src/commands/inventory/consent-managers-to-business-entities/impl.ts +++ b/src/commands/inventory/consent-managers-to-business-entities/impl.ts @@ -1,5 +1,5 @@ import { existsSync, lstatSync } from 'node:fs'; -import { join } from 'node:path'; +import path from 'node:path'; import colors from 'colors'; import type { LocalContext } from '../../../context'; import { listFiles } from '../../../lib/api-keys'; @@ -36,7 +36,7 @@ export function consentManagersToBusinessEntities( // Read in each consent manager configuration const inputs = listFiles(consentManagerYmlFolder).map((directory) => { const { 'consent-manager': consentManager } = readTranscendYaml( - join(consentManagerYmlFolder, directory), + path.join(consentManagerYmlFolder, directory), ); return { name: directory, input: consentManager }; }); diff --git a/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts b/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts index 9b29f0ba..c30a0aa8 100644 --- a/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts +++ b/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts @@ -1,5 +1,5 @@ import { existsSync, lstatSync } from 'node:fs'; -import { join } from 'node:path'; +import path from 'node:path'; import colors from 'colors'; import { difference } from 'lodash-es'; import { DataFlowInput } from '../../../codecs'; @@ -60,7 +60,7 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( const dataSiloInputs = listFiles(dataFlowsYmlFolder).map((directory) => { // read in the data flows for a specific instance const { 'data-flows': dataFlows = [] } = readTranscendYaml( - join(dataFlowsYmlFolder, directory), + path.join(dataFlowsYmlFolder, directory), ); // map the data flows to data silos diff --git a/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts b/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts index 27bb7b51..c73286e7 100644 --- a/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts +++ b/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts @@ -1,5 +1,5 @@ import { existsSync, lstatSync } from 'node:fs'; -import { join } from 'node:path'; +import path from 'node:path'; import colors from 'colors'; import { DataFlowInput } from '../../../codecs'; import type { LocalContext } from '../../../context'; @@ -80,7 +80,7 @@ export async function deriveDataSilosFromDataFlows( for (const directory of listFiles(dataFlowsYmlFolder)) { // read in the data flows for a specific instance const { 'data-flows': dataFlows = [] } = readTranscendYaml( - join(dataFlowsYmlFolder, directory), + path.join(dataFlowsYmlFolder, directory), ); // map the data flows to data silos @@ -97,7 +97,7 @@ export async function deriveDataSilosFromDataFlows( logger.log(`Total Services: ${dataSilos.length}`); logger.log(`Ad Tech Services: ${adTechDataSilos.length}`); logger.log(`Site Tech Services: ${siteTechDataSilos.length}`); - writeTranscendYaml(join(dataSilosYmlFolder, directory), { + writeTranscendYaml(path.join(dataSilosYmlFolder, directory), { 'data-silos': ignoreYmls.includes(directory) ? [] : dataSilos, }); } diff --git a/src/commands/inventory/pull/impl.ts b/src/commands/inventory/pull/impl.ts index a6802e91..322edf4d 100644 --- a/src/commands/inventory/pull/impl.ts +++ b/src/commands/inventory/pull/impl.ts @@ -1,5 +1,5 @@ import fs from 'node:fs'; -import { join } from 'node:path'; +import path from 'node:path'; import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; import colors from 'colors'; import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; @@ -128,7 +128,7 @@ export async function pull( trackerStatuses, }); - const filePath = join(file, `${apiKey.organizationName}.yml`); + const filePath = path.join(file, `${apiKey.organizationName}.yml`); logger.info( colors.magenta(`Writing configuration to file "${filePath}"...`), ); diff --git a/src/commands/inventory/push/impl.ts b/src/commands/inventory/push/impl.ts index 3ba81b28..bd490ab3 100644 --- a/src/commands/inventory/push/impl.ts +++ b/src/commands/inventory/push/impl.ts @@ -1,5 +1,5 @@ import { existsSync, lstatSync } from 'node:fs'; -import { join } from 'node:path'; +import path from 'node:path'; import colors from 'colors'; import { TranscendInput } from '../../../codecs'; import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; @@ -104,7 +104,7 @@ export async function push( let fileList: string[]; fileList = Array.isArray(apiKeyOrList) && lstatSync(file).isDirectory() - ? listFiles(file).map((filePath) => join(file, filePath)) + ? listFiles(file).map((filePath) => path.join(file, filePath)) : file.split(','); // Ensure at least one file is parsed diff --git a/src/lib/api-keys/listDirectories.ts b/src/lib/api-keys/listDirectories.ts index fb9a29e0..d2ab5e23 100644 --- a/src/lib/api-keys/listDirectories.ts +++ b/src/lib/api-keys/listDirectories.ts @@ -1,5 +1,5 @@ import { readdirSync, statSync } from 'node:fs'; -import { join } from 'node:path'; +import path from 'node:path'; /** * List the folders in a directory @@ -9,6 +9,6 @@ import { join } from 'node:path'; */ export function listDirectories(startDir: string): string[] { return readdirSync(startDir).filter((entryName) => - statSync(join(startDir, entryName)).isDirectory(), + statSync(path.join(startDir, entryName)).isDirectory(), ); } diff --git a/src/lib/code-scanning/integrations/composerJson.ts b/src/lib/code-scanning/integrations/composerJson.ts index 3cf697c5..55a0b730 100644 --- a/src/lib/code-scanning/integrations/composerJson.ts +++ b/src/lib/code-scanning/integrations/composerJson.ts @@ -1,5 +1,5 @@ import { readFileSync } from 'node:fs'; -import { dirname } from 'node:path'; +import path from 'node:path'; import { CodePackageSdk } from '../../../codecs'; import { CodeScanningConfig } from '../types'; @@ -8,7 +8,7 @@ export const composerJson: CodeScanningConfig = { ignoreDirs: ['vendor', 'node_modules', 'cache', 'build', 'dist'], scanFunction: (filePath) => { const file = readFileSync(filePath, 'utf-8'); - const directory = dirname(filePath); + const directory = path.dirname(filePath); const asJson = JSON.parse(file); const { name, diff --git a/src/lib/code-scanning/integrations/gemfile.ts b/src/lib/code-scanning/integrations/gemfile.ts index 28b631df..2f75dadb 100644 --- a/src/lib/code-scanning/integrations/gemfile.ts +++ b/src/lib/code-scanning/integrations/gemfile.ts @@ -1,5 +1,5 @@ import { readFileSync } from 'node:fs'; -import { dirname } from 'node:path'; +import path from 'node:path'; import { CodePackageType } from '@transcend-io/privacy-types'; import { findAllWithRegex } from '@transcend-io/type-utils'; import { listFiles } from '../../api-keys'; @@ -16,7 +16,7 @@ export const gemfile: CodeScanningConfig = { ignoreDirs: ['bin'], scanFunction: (filePath) => { const fileContents = readFileSync(filePath, 'utf-8'); - const directory = dirname(filePath); + const directory = path.dirname(filePath); const filesInFolder = listFiles(directory); // parse gemspec file for name diff --git a/src/lib/code-scanning/integrations/gradle.ts b/src/lib/code-scanning/integrations/gradle.ts index 70983adf..113dda44 100644 --- a/src/lib/code-scanning/integrations/gradle.ts +++ b/src/lib/code-scanning/integrations/gradle.ts @@ -1,5 +1,5 @@ import { readFileSync } from 'node:fs'; -import { dirname } from 'node:path'; +import path from 'node:path'; import { findAllWithRegex } from '@transcend-io/type-utils'; import { CodeScanningConfig } from '../types'; @@ -29,7 +29,7 @@ export const gradle: CodeScanningConfig = { ], scanFunction: (filePath) => { const fileContents = readFileSync(filePath, 'utf-8'); - const directory = dirname(filePath); + const directory = path.dirname(filePath); const targets = findAllWithRegex( { diff --git a/src/lib/code-scanning/integrations/javascriptPackageJson.ts b/src/lib/code-scanning/integrations/javascriptPackageJson.ts index fe437c32..8eb8aabc 100644 --- a/src/lib/code-scanning/integrations/javascriptPackageJson.ts +++ b/src/lib/code-scanning/integrations/javascriptPackageJson.ts @@ -1,5 +1,5 @@ import { readFileSync } from 'node:fs'; -import { dirname } from 'node:path'; +import path from 'node:path'; import { CodePackageSdk } from '../../../codecs'; import { CodeScanningConfig } from '../types'; @@ -8,7 +8,7 @@ export const javascriptPackageJson: CodeScanningConfig = { ignoreDirs: ['node_modules', 'serverless-build', 'lambda-build'], scanFunction: (filePath) => { const file = readFileSync(filePath, 'utf-8'); - const directory = dirname(filePath); + const directory = path.dirname(filePath); const asJson = JSON.parse(file); const { name, diff --git a/src/lib/code-scanning/integrations/pubspec.ts b/src/lib/code-scanning/integrations/pubspec.ts index 0f8e1b72..4310ba8a 100644 --- a/src/lib/code-scanning/integrations/pubspec.ts +++ b/src/lib/code-scanning/integrations/pubspec.ts @@ -1,5 +1,5 @@ import { readFileSync } from 'node:fs'; -import { dirname } from 'node:path'; +import path from 'node:path'; import { CodePackageType } from '@transcend-io/privacy-types'; import yaml from 'js-yaml'; import { CodeScanningConfig } from '../types'; @@ -33,7 +33,7 @@ export const pubspec: CodeScanningConfig = { supportedFiles: ['pubspec.yml'], ignoreDirs: ['build'], scanFunction: (filePath) => { - const directory = dirname(filePath); + const directory = path.dirname(filePath); const fileContents = readFileSync(filePath, 'utf-8'); const { name, diff --git a/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts b/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts index fa119fe9..2d6177b5 100644 --- a/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts +++ b/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts @@ -1,5 +1,5 @@ import { readFileSync } from 'node:fs'; -import { dirname, join } from 'node:path'; +import path from 'node:path'; import { CodePackageType } from '@transcend-io/privacy-types'; import { findAllWithRegex } from '@transcend-io/type-utils'; import { listFiles } from '../../api-keys'; @@ -14,13 +14,13 @@ export const pythonRequirementsTxt: CodeScanningConfig = { ignoreDirs: ['build', 'lib', 'lib64'], scanFunction: (filePath) => { const fileContents = readFileSync(filePath, 'utf-8'); - const directory = dirname(filePath); + const directory = path.dirname(filePath); const filesInFolder = listFiles(directory); // parse setup file for name const setupFile = filesInFolder.find((file) => file === 'setup.py'); const setupFileContents = setupFile - ? readFileSync(join(directory, setupFile), 'utf-8') + ? readFileSync(path.join(directory, setupFile), 'utf-8') : undefined; const packageName = setupFileContents ? (PACKAGE_NAME.exec(setupFileContents) || [])[2] diff --git a/src/lib/code-scanning/integrations/swift.ts b/src/lib/code-scanning/integrations/swift.ts index df1f1804..740a9859 100644 --- a/src/lib/code-scanning/integrations/swift.ts +++ b/src/lib/code-scanning/integrations/swift.ts @@ -1,5 +1,5 @@ import { readFileSync } from 'node:fs'; -import { dirname } from 'node:path'; +import path from 'node:path'; import { CodePackageType } from '@transcend-io/privacy-types'; import { decodeCodec } from '@transcend-io/type-utils'; import * as t from 'io-ts'; @@ -30,7 +30,7 @@ export const swift: CodeScanningConfig = { return [ { - name: dirname(filePath).split('/').pop() || '', // FIXME pull from Package.swift ->> name if possible + name: path.dirname(filePath).split('/').pop() || '', // FIXME pull from Package.swift ->> name if possible type: CodePackageType.CocoaPods, // FIXME should be swift softwareDevelopmentKits: parsed.pins.map((target) => ({ name: target.identity, diff --git a/src/lib/requests/bulkRestartRequests.ts b/src/lib/requests/bulkRestartRequests.ts index 07a04e91..3590d271 100644 --- a/src/lib/requests/bulkRestartRequests.ts +++ b/src/lib/requests/bulkRestartRequests.ts @@ -1,4 +1,4 @@ -import { join } from 'node:path'; +import path from 'node:path'; import { PersistedState } from '@transcend-io/persisted-state'; import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; import cliProgress from 'cli-progress'; @@ -100,7 +100,7 @@ export async function bulkRestartRequests({ ); // Create a new state file to store the requests from this run - const cacheFile = join( + const cacheFile = path.join( requestReceiptFolder, `tr-request-restart-${new Date().toISOString()}`, ); diff --git a/src/lib/requests/downloadPrivacyRequestFiles.ts b/src/lib/requests/downloadPrivacyRequestFiles.ts index 83fc4ee1..9632c657 100644 --- a/src/lib/requests/downloadPrivacyRequestFiles.ts +++ b/src/lib/requests/downloadPrivacyRequestFiles.ts @@ -1,5 +1,5 @@ import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; -import { dirname, join } from 'node:path'; +import path from 'node:path'; import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; import cliProgress from 'cli-progress'; import colors from 'colors'; @@ -99,7 +99,7 @@ export async function downloadPrivacyRequestFiles({ requestFileMetadata, async ([request, metadata]) => { // Create a new folder to store request files - const requestFolder = join(folderPath, request.id); + const requestFolder = path.join(folderPath, request.id); if (!existsSync(requestFolder)) { mkdirSync(requestFolder); } @@ -111,8 +111,8 @@ export async function downloadPrivacyRequestFiles({ onFileDownloaded: (fil, stream) => { // Ensure a folder exists for the file // filename looks like Health/heartbeat.csv - const filePath = join(requestFolder, fil.fileName); - const folder = dirname(filePath); + const filePath = path.join(requestFolder, fil.fileName); + const folder = path.dirname(filePath); if (!existsSync(folder)) { mkdirSync(folder, { recursive: true }); } diff --git a/src/lib/requests/tests/readCsv.test.ts b/src/lib/requests/tests/readCsv.test.ts index b3c3ae84..033c76fc 100644 --- a/src/lib/requests/tests/readCsv.test.ts +++ b/src/lib/requests/tests/readCsv.test.ts @@ -1,4 +1,4 @@ -import { join } from 'node:path'; +import path from 'node:path'; import * as t from 'io-ts'; import { describe, expect, it } from 'vitest'; import { readCsv } from '../index'; @@ -6,7 +6,7 @@ import { readCsv } from '../index'; describe('readCsv', () => { it('should successfully parse a csv', () => { expect( - readCsv(join(__dirname, 'sample.csv'), t.record(t.string, t.string)), + readCsv(path.join(__dirname, 'sample.csv'), t.record(t.string, t.string)), ).to.deep.equal([ { CASL_STATUS: 'Undefined', @@ -36,19 +36,28 @@ describe('readCsv', () => { it('throw an error for invalid file', () => { expect(() => - readCsv(join(__dirname, 'sample.csv'), t.type({ notValid: t.string })), + readCsv( + path.join(__dirname, 'sample.csv'), + t.type({ notValid: t.string }), + ), ).to.throw('Failed to decode codec'); }); it('throw an error for invalid codec', () => { expect(() => - readCsv(join(__dirname, 'sample.csvs'), t.record(t.string, t.string)), + readCsv( + path.join(__dirname, 'sample.csvs'), + t.record(t.string, t.string), + ), ).to.throw('ENOENT: no such file or directory, open'); }); it('throw an error for invalid format', () => { expect(() => - readCsv(join(__dirname, 'readCsv.test.ts'), t.record(t.string, t.string)), + readCsv( + path.join(__dirname, 'readCsv.test.ts'), + t.record(t.string, t.string), + ), ).to.throw(); }); }); diff --git a/src/lib/requests/uploadPrivacyRequestsFromCsv.ts b/src/lib/requests/uploadPrivacyRequestsFromCsv.ts index 4fc243c5..e9625325 100644 --- a/src/lib/requests/uploadPrivacyRequestsFromCsv.ts +++ b/src/lib/requests/uploadPrivacyRequestsFromCsv.ts @@ -1,4 +1,4 @@ -import { join } from 'node:path'; +import path from 'node:path'; import { PersistedState } from '@transcend-io/persisted-state'; import cliProgress from 'cli-progress'; import colors from 'colors'; @@ -106,7 +106,7 @@ export async function uploadPrivacyRequestsFromCsv({ }); // Create a new state file to store the requests from this run - const requestCacheFile = join( + const requestCacheFile = path.join( requestReceiptFolder, `tr-request-upload-${new Date().toISOString()}-${file .split('/') diff --git a/src/lib/tests/findCodePackagesInFolder.test.ts b/src/lib/tests/findCodePackagesInFolder.test.ts index 078f92e3..b32bfaa5 100644 --- a/src/lib/tests/findCodePackagesInFolder.test.ts +++ b/src/lib/tests/findCodePackagesInFolder.test.ts @@ -1,4 +1,4 @@ -import { join } from 'node:path'; +import path from 'node:path'; import { describe, expect, it } from 'vitest'; import type { CodePackageInput } from '../../codecs'; import { findCodePackagesInFolder } from '../code-scanning/findCodePackagesInFolder'; @@ -723,7 +723,7 @@ describe('findCodePackagesInFolder', () => { it('should remove links', async () => { const result = await findCodePackagesInFolder({ repositoryName: 'transcend-io/cli', - scanPath: join(__dirname, '../../../examples/code-scanning'), + scanPath: path.join(__dirname, '../../../examples/code-scanning'), }); expect(sortCodePackages(result)).to.deep.equal(sortCodePackages(expected)); }); From 33e50dd8924411fbbc7285ed804e2178ea7f19c0 Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Wed, 16 Jul 2025 21:52:44 -0400 Subject: [PATCH 08/18] __dirname to import.meta.dirname --- src/lib/requests/tests/readCsv.test.ts | 11 +++++++---- src/lib/tests/findCodePackagesInFolder.test.ts | 5 ++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/lib/requests/tests/readCsv.test.ts b/src/lib/requests/tests/readCsv.test.ts index 033c76fc..587833a8 100644 --- a/src/lib/requests/tests/readCsv.test.ts +++ b/src/lib/requests/tests/readCsv.test.ts @@ -6,7 +6,10 @@ import { readCsv } from '../index'; describe('readCsv', () => { it('should successfully parse a csv', () => { expect( - readCsv(path.join(__dirname, 'sample.csv'), t.record(t.string, t.string)), + readCsv( + path.join(import.meta.dirname, 'sample.csv'), + t.record(t.string, t.string), + ), ).to.deep.equal([ { CASL_STATUS: 'Undefined', @@ -37,7 +40,7 @@ describe('readCsv', () => { it('throw an error for invalid file', () => { expect(() => readCsv( - path.join(__dirname, 'sample.csv'), + path.join(import.meta.dirname, 'sample.csv'), t.type({ notValid: t.string }), ), ).to.throw('Failed to decode codec'); @@ -46,7 +49,7 @@ describe('readCsv', () => { it('throw an error for invalid codec', () => { expect(() => readCsv( - path.join(__dirname, 'sample.csvs'), + path.join(import.meta.dirname, 'sample.csvs'), t.record(t.string, t.string), ), ).to.throw('ENOENT: no such file or directory, open'); @@ -55,7 +58,7 @@ describe('readCsv', () => { it('throw an error for invalid format', () => { expect(() => readCsv( - path.join(__dirname, 'readCsv.test.ts'), + path.join(import.meta.dirname, 'readCsv.test.ts'), t.record(t.string, t.string), ), ).to.throw(); diff --git a/src/lib/tests/findCodePackagesInFolder.test.ts b/src/lib/tests/findCodePackagesInFolder.test.ts index b32bfaa5..4cd6c622 100644 --- a/src/lib/tests/findCodePackagesInFolder.test.ts +++ b/src/lib/tests/findCodePackagesInFolder.test.ts @@ -723,7 +723,10 @@ describe('findCodePackagesInFolder', () => { it('should remove links', async () => { const result = await findCodePackagesInFolder({ repositoryName: 'transcend-io/cli', - scanPath: path.join(__dirname, '../../../examples/code-scanning'), + scanPath: path.join( + import.meta.dirname, + '../../../examples/code-scanning', + ), }); expect(sortCodePackages(result)).to.deep.equal(sortCodePackages(expected)); }); From 50a07731b5c487925eb514ba5d2aa45d1b2469c0 Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Wed, 16 Jul 2025 22:46:46 -0400 Subject: [PATCH 09/18] type safety --- .../impl.ts | 45 +++++++--------- .../derive-data-silos-from-data-flows/impl.ts | 30 ++++------- .../oneTrust/helpers/convertToEmptyStrings.ts | 7 +-- src/lib/requests/submitPrivacyRequest.ts | 6 ++- .../requests/uploadPrivacyRequestsFromCsv.ts | 52 ++++++++----------- src/lib/tests/codebase.test.ts | 5 +- 6 files changed, 64 insertions(+), 81 deletions(-) diff --git a/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts b/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts index c30a0aa8..2a61680b 100644 --- a/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts +++ b/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts @@ -1,8 +1,6 @@ import { existsSync, lstatSync } from 'node:fs'; import path from 'node:path'; -import colors from 'colors'; import { difference } from 'lodash-es'; -import { DataFlowInput } from '../../../codecs'; import type { LocalContext } from '../../../context'; import { listFiles } from '../../../lib/api-keys'; import { dataFlowsToDataSilos } from '../../../lib/consent-manager/dataFlowsToDataSilos'; @@ -36,12 +34,9 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( ): Promise { // Ensure folder is passed to dataFlowsYmlFolder if (!dataFlowsYmlFolder) { - logger.error( - colors.red( - 'Missing required arg: --dataFlowsYmlFolder=./working/data-flows/', - ), + throw new Error( + 'Missing required arg: --dataFlowsYmlFolder=./working/data-flows/', ); - process.exit(1); } // Ensure folder is passed @@ -49,8 +44,7 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( !existsSync(dataFlowsYmlFolder) || !lstatSync(dataFlowsYmlFolder).isDirectory() ) { - logger.error(colors.red(`Folder does not exist: "${dataFlowsYmlFolder}"`)); - process.exit(1); + throw new Error(`Folder does not exist: "${dataFlowsYmlFolder}"`); } // Ignore the data flows in these yml files @@ -88,11 +82,9 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( } of dataSiloInputs) { const allDataSilos = [...adTechDataSilos, ...siteTechDataSilos]; for (const dataSilo of allDataSilos) { - const service = dataSilo['outer-type'] || dataSilo.integrationName; + const service = dataSilo['outer-type'] ?? dataSilo.integrationName; // create mapping to instance - if (!serviceToInstance[service]) { - serviceToInstance[service] = []; - } + serviceToInstance[service] ??= []; serviceToInstance[service].push(organizationName); serviceToInstance[service] = [...new Set(serviceToInstance[service])]; } @@ -103,7 +95,7 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( ...new Set( dataSiloInputs.flatMap(({ adTechDataSilos }) => adTechDataSilos.map( - (silo) => silo['outer-type'] || silo.integrationName, + (silo) => silo['outer-type'] ?? silo.integrationName, ), ), ), @@ -115,7 +107,7 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( ...new Set( dataSiloInputs.flatMap(({ siteTechDataSilos }) => siteTechDataSilos.map( - (silo) => silo['outer-type'] || silo.integrationName, + (silo) => silo['outer-type'] ?? silo.integrationName, ), ), ), @@ -128,15 +120,13 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( for (const { adTechDataSilos, siteTechDataSilos } of dataSiloInputs) { const allDataSilos = [...adTechDataSilos, ...siteTechDataSilos]; for (const dataSilo of allDataSilos) { - const service = dataSilo['outer-type'] || dataSilo.integrationName; + const service = dataSilo['outer-type'] ?? dataSilo.integrationName; const foundOnDomain = dataSilo.attributes?.find( (attribute) => attribute.key === 'Found On Domain', ); // create mapping to instance - if (!serviceToFoundOnDomain[service]) { - serviceToFoundOnDomain[service] = []; - } - serviceToFoundOnDomain[service].push(...(foundOnDomain?.values || [])); + serviceToFoundOnDomain[service] ??= []; + serviceToFoundOnDomain[service].push(...(foundOnDomain?.values ?? [])); serviceToFoundOnDomain[service] = [ ...new Set(serviceToFoundOnDomain[service]), ]; @@ -163,22 +153,27 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( { key: 'Business Units', values: difference( - serviceToInstance[service] || [], + service in serviceToInstance ? serviceToInstance[service] : [], instancesToIgnore, ), }, { key: 'Found On Domain', - values: serviceToFoundOnDomain[service] || [], + values: + service in serviceToFoundOnDomain + ? serviceToFoundOnDomain[service] + : [], }, ], }), ); // Log output - logger.log(`Total Services: ${dataSilos.length}`); - logger.log(`Ad Tech Services: ${adTechIntegrations.length}`); - logger.log(`Site Tech Services: ${siteTechIntegrations.length}`); + logger.log(`Total Services: ${dataSilos.length.toLocaleString()}`); + logger.log(`Ad Tech Services: ${adTechIntegrations.length.toLocaleString()}`); + logger.log( + `Site Tech Services: ${siteTechIntegrations.length.toLocaleString()}`, + ); // Write to yaml writeTranscendYaml(output, { diff --git a/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts b/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts index c73286e7..0c41c7a6 100644 --- a/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts +++ b/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts @@ -1,7 +1,5 @@ import { existsSync, lstatSync } from 'node:fs'; import path from 'node:path'; -import colors from 'colors'; -import { DataFlowInput } from '../../../codecs'; import type { LocalContext } from '../../../context'; import { listFiles } from '../../../lib/api-keys'; import { dataFlowsToDataSilos } from '../../../lib/consent-manager/dataFlowsToDataSilos'; @@ -35,12 +33,9 @@ export async function deriveDataSilosFromDataFlows( ): Promise { // Ensure folder is passed to dataFlowsYmlFolder if (!dataFlowsYmlFolder) { - logger.error( - colors.red( - 'Missing required arg: --dataFlowsYmlFolder=./working/data-flows/', - ), + throw new Error( + 'Missing required arg: --dataFlowsYmlFolder=./working/data-flows/', ); - process.exit(1); } // Ensure folder is passed @@ -48,18 +43,14 @@ export async function deriveDataSilosFromDataFlows( !existsSync(dataFlowsYmlFolder) || !lstatSync(dataFlowsYmlFolder).isDirectory() ) { - logger.error(colors.red(`Folder does not exist: "${dataFlowsYmlFolder}"`)); - process.exit(1); + throw new Error(`Folder does not exist: "${dataFlowsYmlFolder}"`); } // Ensure folder is passed to dataSilosYmlFolder if (!dataSilosYmlFolder) { - logger.error( - colors.red( - 'Missing required arg: --dataSilosYmlFolder=./working/data-silos/', - ), + throw new Error( + 'Missing required arg: --dataSilosYmlFolder=./working/data-silos/', ); - process.exit(1); } // Ensure folder is passed @@ -67,8 +58,7 @@ export async function deriveDataSilosFromDataFlows( !existsSync(dataSilosYmlFolder) || !lstatSync(dataSilosYmlFolder).isDirectory() ) { - logger.error(colors.red(`Folder does not exist: "${dataSilosYmlFolder}"`)); - process.exit(1); + throw new Error(`Folder does not exist: "${dataSilosYmlFolder}"`); } // Fetch all integrations in the catalog @@ -94,9 +84,11 @@ export async function deriveDataSilosFromDataFlows( // combine and write to yml file const dataSilos = [...adTechDataSilos, ...siteTechDataSilos]; - logger.log(`Total Services: ${dataSilos.length}`); - logger.log(`Ad Tech Services: ${adTechDataSilos.length}`); - logger.log(`Site Tech Services: ${siteTechDataSilos.length}`); + logger.log(`Total Services: ${dataSilos.length.toLocaleString()}`); + logger.log(`Ad Tech Services: ${adTechDataSilos.length.toLocaleString()}`); + logger.log( + `Site Tech Services: ${siteTechDataSilos.length.toLocaleString()}`, + ); writeTranscendYaml(path.join(dataSilosYmlFolder, directory), { 'data-silos': ignoreYmls.includes(directory) ? [] : dataSilos, }); diff --git a/src/lib/oneTrust/helpers/convertToEmptyStrings.ts b/src/lib/oneTrust/helpers/convertToEmptyStrings.ts index 067e325a..40a81162 100644 --- a/src/lib/oneTrust/helpers/convertToEmptyStrings.ts +++ b/src/lib/oneTrust/helpers/convertToEmptyStrings.ts @@ -1,6 +1,3 @@ -/* eslint-disable eslint-comments/disable-enable-pair */ -/* eslint-disable @typescript-eslint/no-explicit-any */ - /** * Converts all primitive values in an object or array to empty strings while maintaining structure * @@ -33,7 +30,7 @@ * // items: ['', '', ''] * // } */ -export function convertToEmptyStrings(input: T): any { +export function convertToEmptyStrings(input?: unknown): unknown { // Handle null/undefined if (input === null || input === undefined) { return ''; @@ -47,7 +44,7 @@ export function convertToEmptyStrings(input: T): any { // Handle objects if (typeof input === 'object') { return Object.fromEntries( - Object.entries(input).map>(([key, value]) => [ + Object.entries(input).map(([key, value]) => [ key, convertToEmptyStrings(value), ]), diff --git a/src/lib/requests/submitPrivacyRequest.ts b/src/lib/requests/submitPrivacyRequest.ts index ffb7cba7..a88a1ac3 100644 --- a/src/lib/requests/submitPrivacyRequest.ts +++ b/src/lib/requests/submitPrivacyRequest.ts @@ -70,7 +70,7 @@ export async function submitPrivacyRequest( // Merge the per-request attributes with the // global attributes const mergedAttributes = [...additionalAttributes]; - for (const attribute of input.attributes || []) { + for (const attribute of input.attributes ?? []) { const existing = mergedAttributes.find( (attribute_) => attribute_.key === attribute.key, ); @@ -127,7 +127,9 @@ export async function submitPrivacyRequest( } catch (error) { throw new Error( `Received an error from server: ${ - error?.response?.body || error?.message + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + (error as { response?: { body: string } }).response?.body || + (error as { message: string }).message }`, ); } diff --git a/src/lib/requests/uploadPrivacyRequestsFromCsv.ts b/src/lib/requests/uploadPrivacyRequestsFromCsv.ts index e9625325..ed403c09 100644 --- a/src/lib/requests/uploadPrivacyRequestsFromCsv.ts +++ b/src/lib/requests/uploadPrivacyRequestsFromCsv.ts @@ -106,11 +106,10 @@ export async function uploadPrivacyRequestsFromCsv({ }); // Create a new state file to store the requests from this run + const filenameWithoutExtension = path.basename(file, path.extname(file)); const requestCacheFile = path.join( requestReceiptFolder, - `tr-request-upload-${new Date().toISOString()}-${file - .split('/') - .pop()}`.replace('.csv', '.json'), + `tr-request-upload-${new Date().toISOString()}-${filenameWithoutExtension}.json`, ); const requestState = new PersistedState( requestCacheFile, @@ -190,12 +189,12 @@ export async function uploadPrivacyRequestsFromCsv({ // The identifier to log, only include personal data if debug mode is on const requestLogId = debug ? `email:${requestInput.email} | coreIdentifier:${requestInput.coreIdentifier}` - : `row:${ind.toString()}`; + : `row:${ind.toLocaleString()}`; if (debug) { logger.info( colors.magenta( - `[${ind + 1}/${requestInputs.length}] Importing: ${JSON.stringify( + `[${(ind + 1).toLocaleString()}/${requestInputs.length.toLocaleString()}] Importing: ${JSON.stringify( requestInput, null, 2, @@ -235,14 +234,12 @@ export async function uploadPrivacyRequestsFromCsv({ if (debug) { logger.info( colors.green( - `[${ind + 1}/${ - requestInputs.length - }] Successfully submitted the test data subject request: "${requestLogId}"`, + `[${(ind + 1).toLocaleString()}/${requestInputs.length.toLocaleString()}] Successfully submitted the test data subject request: "${requestLogId}"`, ), ); logger.info( colors.green( - `[${ind + 1}/${requestInputs.length}] View it at: "${ + `[${(ind + 1).toLocaleString()}/${requestInputs.length.toLocaleString()}] View it at: "${ requestResponse.link }"`, ), @@ -260,8 +257,8 @@ export async function uploadPrivacyRequestsFromCsv({ }); await requestState.setValue(successfulRequests, 'successfulRequests'); } catch (error) { - const message = `${error.message} - ${JSON.stringify( - error.response?.body, + const message = `${(error as { message: string }).message} - ${JSON.stringify( + (error as { response?: { body: string } }).response?.body, null, 2, )}`; @@ -273,9 +270,7 @@ export async function uploadPrivacyRequestsFromCsv({ if (debug) { logger.info( colors.yellow( - `[${ind + 1}/${ - requestInputs.length - }] Skipping request as it is a duplicate`, + `[${(ind + 1).toLocaleString()}/${requestInputs.length.toLocaleString()}] Skipping request as it is a duplicate`, ), ); } @@ -291,17 +286,17 @@ export async function uploadPrivacyRequestsFromCsv({ failingRequests.push({ ...requestInput, rowIndex: ind, + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing error: clientError || message, attemptedAt: new Date().toISOString(), }); await requestState.setValue(failingRequests, 'failingRequests'); if (debug) { + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing logger.error(colors.red(clientError || message)); logger.error( colors.red( - `[${ind + 1}/${ - requestInputs.length - }] Failed to submit request for: "${requestLogId}"`, + `[${(ind + 1).toLocaleString()}/${requestInputs.length.toLocaleString()}] Failed to submit request for: "${requestLogId}"`, ), ); } @@ -324,16 +319,18 @@ export async function uploadPrivacyRequestsFromCsv({ // Log completion time logger.info( - colors.green(`Completed upload in "${totalTime / 1000}" seconds.`), + colors.green( + `Completed upload in "${(totalTime / 1000).toLocaleString()}" seconds.`, + ), ); // Log duplicates if (requestState.getValue('duplicateRequests').length > 0) { logger.info( colors.yellow( - `Encountered "${ - requestState.getValue('duplicateRequests').length - }" duplicate requests. ` + + `Encountered "${requestState + .getValue('duplicateRequests') + .length.toLocaleString()}" duplicate requests. ` + `See "${requestCacheFile}" to review the core identifiers for these requests.`, ), ); @@ -341,14 +338,11 @@ export async function uploadPrivacyRequestsFromCsv({ // Log errors if (requestState.getValue('failingRequests').length > 0) { - logger.error( - colors.red( - `Encountered "${ - requestState.getValue('failingRequests').length - }" errors. ` + - `See "${requestCacheFile}" to review the error messages and inputs.`, - ), + throw new Error( + `Encountered "${requestState + .getValue('failingRequests') + .length.toLocaleString()}" errors. ` + + `See "${requestCacheFile}" to review the error messages and inputs.`, ); - process.exit(1); } } diff --git a/src/lib/tests/codebase.test.ts b/src/lib/tests/codebase.test.ts index 78c2e332..85363c3d 100644 --- a/src/lib/tests/codebase.test.ts +++ b/src/lib/tests/codebase.test.ts @@ -75,7 +75,10 @@ async function checkExport( .relative(process.cwd(), filePath) .replaceAll('\\', '/'); - const module = await import(`../../../${relativePath}`); + const module = (await import(`../../../${relativePath}`)) as Record< + string, + unknown + >; return exportName in module && module[exportName] !== undefined; } catch { From 30ccbdce9c066db0b7bbb748a551e9f939765e0d Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Wed, 16 Jul 2025 23:15:12 -0400 Subject: [PATCH 10/18] rm unused files with knip --- knip.config.ts | 8 + package.json | 9 +- pnpm-lock.yaml | 420 +++++++++++++++--- src/lib/consent-manager/types.ts | 4 +- .../pullCustomSiloOutstandingIdentifiers.ts | 195 -------- src/lib/graphql/formatAttributeValues.ts | 2 +- src/lib/graphql/syncPromptGroups.ts | 6 +- src/lib/graphql/syncPromptPartials.ts | 4 +- src/lib/graphql/syncTeams.ts | 4 +- .../oneTrust/helpers/convertToEmptyStrings.ts | 56 --- src/lib/oneTrust/helpers/index.ts | 1 - .../helpers/parseCliSyncOtArguments.ts | 185 -------- .../syncOneTrustAssessmentToTranscend.ts | 2 +- .../syncOneTrustAssessmentsFromOneTrust.ts | 7 - .../tests/convertToEmptyStrings.test.ts | 51 --- vitest.config.ts | 3 + 16 files changed, 391 insertions(+), 566 deletions(-) create mode 100644 knip.config.ts delete mode 100644 src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts delete mode 100644 src/lib/oneTrust/helpers/convertToEmptyStrings.ts delete mode 100644 src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts delete mode 100644 src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts diff --git a/knip.config.ts b/knip.config.ts new file mode 100644 index 00000000..4e3e2824 --- /dev/null +++ b/knip.config.ts @@ -0,0 +1,8 @@ +import type { KnipConfig } from 'knip'; + +const config: KnipConfig = { + ignore: ['examples/**/*', 'src/commands/**/readme.ts'], + ignoreDependencies: ['doctoc'], +}; + +export default config; diff --git a/package.json b/package.json index 26b7f71b..4da00044 100644 --- a/package.json +++ b/package.json @@ -73,7 +73,8 @@ "test": "vitest run", "script:transcend-json-schema": "tsx scripts/buildTranscendJsonSchema.ts && prettier ./transcend-yml-schema-*.json --write", "script:pathfinder-json-schema": "tsx scripts/buildPathfinderJsonSchema.ts && prettier ./pathfinder-policy-yml-schema.json --write", - "docgen": "tsx scripts/buildReadme.ts" + "docgen": "tsx scripts/buildReadme.ts", + "knip": "knip" }, "tsup": { "entry": [ @@ -105,7 +106,6 @@ "@transcend-io/privacy-types": "^4.124.1", "@transcend-io/secret-value": "^1.2.0", "@transcend-io/type-utils": "^1.8.0", - "JSONStream": "^1.3.5", "cli-progress": "^3.11.2", "colors": "^1.4.0", "csv-parse": "^5.6.0", @@ -121,6 +121,7 @@ "io-ts": "^2.2.21", "io-ts-types": "^0.5.16", "js-yaml": "^4.1.0", + "JSONStream": "^1.3.5", "jsonwebtoken": "^9.0.2", "lodash-es": "^4.17.21", "monocle-ts": "^2.3.13", @@ -135,15 +136,14 @@ "@ianvs/prettier-plugin-sort-imports": "^4.5.1", "@tsconfig/node22": "^22.0.2", "@tsconfig/strictest": "^2.0.5", - "@types/JSONStream": "npm:@types/jsonstream@^0.8.33", "@types/cli-progress": "^3.11.0", - "@types/eslint": "^9.6.1", "@types/fuzzysearch": "^1.0.0", "@types/global-agent": "^2.1.1", "@types/inquirer": "^7.3.1", "@types/inquirer-autocomplete-prompt": "^3.0.0", "@types/js-yaml": "^4.0.5", "@types/json-schema": "^7.0.15", + "@types/JSONStream": "npm:@types/jsonstream@^0.8.33", "@types/jsonwebtoken": "^9", "@types/lodash-es": "^4.17.12", "@types/node": "^22.x", @@ -154,6 +154,7 @@ "eslint": "^9.31.0", "eslint-plugin-unicorn": "^59.0.1", "fdir": "^6.4.6", + "knip": "^5.61.3", "prettier": "^3.6.2", "publint": "^0.3.12", "tsup": "^8.5.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4678d557..c728f4f3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -126,9 +126,6 @@ importers: '@types/cli-progress': specifier: ^3.11.0 version: 3.11.6 - '@types/eslint': - specifier: ^9.6.1 - version: 9.6.1 '@types/fuzzysearch': specifier: ^1.0.0 version: 1.0.2 @@ -170,13 +167,16 @@ importers: version: 2.2.1 eslint: specifier: ^9.31.0 - version: 9.31.0 + version: 9.31.0(jiti@2.4.2) eslint-plugin-unicorn: specifier: ^59.0.1 - version: 59.0.1(eslint@9.31.0) + version: 59.0.1(eslint@9.31.0(jiti@2.4.2)) fdir: specifier: ^6.4.6 version: 6.4.6(picomatch@4.0.2) + knip: + specifier: ^5.61.3 + version: 5.61.3(@types/node@22.16.4)(typescript@5.8.3) prettier: specifier: ^3.6.2 version: 3.6.2 @@ -185,7 +185,7 @@ importers: version: 0.3.12 tsup: specifier: ^8.5.0 - version: 8.5.0(postcss@8.5.6)(tsx@4.20.3)(typescript@5.8.3) + version: 8.5.0(jiti@2.4.2)(postcss@8.5.6)(tsx@4.20.3)(typescript@5.8.3) tsx: specifier: ^4.20.3 version: 4.20.3 @@ -194,13 +194,13 @@ importers: version: 5.8.3 typescript-eslint: specifier: ^8.37.0 - version: 8.37.0(eslint@9.31.0)(typescript@5.8.3) + version: 8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3) vite-tsconfig-paths: specifier: ^5.1.4 - version: 5.1.4(typescript@5.8.3)(vite@7.0.2(@types/node@22.16.4)(tsx@4.20.3)) + version: 5.1.4(typescript@5.8.3)(vite@7.0.2(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3)) vitest: specifier: ^3.2.4 - version: 3.2.4(@types/node@22.16.4)(tsx@4.20.3) + version: 3.2.4(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3) packages: @@ -241,6 +241,15 @@ packages: resolution: {integrity: sha512-jYnje+JyZG5YThjHiF28oT4SIZLnYOcSBb6+SDaFIyzDVSkXQmQQYclJ2R+YxcdmK0AX6x1E5OQNtuh3jHDrUg==} engines: {node: '>=6.9.0'} + '@emnapi/core@1.4.4': + resolution: {integrity: sha512-A9CnAbC6ARNMKcIcrQwq6HeHCjpcBZ5wSx4U01WXCqEKlrzB9F9315WDNHkrs2xbx7YjjSxbUYxuN6EQzpcY2g==} + + '@emnapi/runtime@1.4.4': + resolution: {integrity: sha512-hHyapA4A3gPaDCNfiqyZUStTMqIkKRshqPIuDOXv1hcBnD4U3l8cP0T1HMCfGRxQ6V64TGCcoswChANyOAwbQg==} + + '@emnapi/wasi-threads@1.0.3': + resolution: {integrity: sha512-8K5IFFsQqF9wQNJptGbS6FNKgUTsSRYnTqNCG1vPP8jFdjSv18n2mQfJpkt2Oibo9iBEzcDnDxNwKTzC7svlJw==} + '@esbuild/aix-ppc64@0.25.6': resolution: {integrity: sha512-ShbM/3XxwuxjFiuVBHA+d3j5dyac0aEVVq1oluIDf71hUw0aRF59dV/efUsIwFnR6m8JNM2FjZOzmaZ8yG61kw==} engines: {node: '>=18'} @@ -507,6 +516,9 @@ packages: '@jridgewell/trace-mapping@0.3.29': resolution: {integrity: sha512-uw6guiW/gcAGPDhLmd77/6lW8QLeiV5RUTsAX46Db6oLhGaVj4lhnPwb184s1bkc8kdVg/+h988dro8GRDpmYQ==} + '@napi-rs/wasm-runtime@0.2.12': + resolution: {integrity: sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==} + '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -519,6 +531,101 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@oxc-resolver/binding-android-arm-eabi@11.5.2': + resolution: {integrity: sha512-g3Dh0uN8E1fJAi+m5LxDU1frUz5q4ox/arqXGpEmt+u7wRXBpXnGsxDV/GFB59AmVWbQAiyhVCiM2GymkaxwwQ==} + cpu: [arm] + os: [android] + + '@oxc-resolver/binding-android-arm64@11.5.2': + resolution: {integrity: sha512-bij8HIMXYGsxdxuvycpkgvTfBpj6tv5jKaZ4tcPKPJjewH5WYIaSAT4PJYlAidP/0m8jyPu5GGkslF7/qPUhAg==} + cpu: [arm64] + os: [android] + + '@oxc-resolver/binding-darwin-arm64@11.5.2': + resolution: {integrity: sha512-C2hjujTOPgyi4sgc4UL+JHlEiClTNncLUdwiilMnwjiEcxSe7ubBmeZRENUd9bx8P9DbS1ApaBjwv13ZngrZRw==} + cpu: [arm64] + os: [darwin] + + '@oxc-resolver/binding-darwin-x64@11.5.2': + resolution: {integrity: sha512-Llf2qMBzs4PdbnrA7s3tVjW7MXnjUXepfqQkEXM2klxIggcbtbIESe3KupYHoo0Q0p6hLHwWoadyM32Ho2hLzA==} + cpu: [x64] + os: [darwin] + + '@oxc-resolver/binding-freebsd-x64@11.5.2': + resolution: {integrity: sha512-dKCHhqgKW3eqnJBlgLC03qoDSVeZSZJVcSVpyomu0XrrNha3wVyv6aJjN7A8HnjUCqJDibbZfTtD3/gnsm30eQ==} + cpu: [x64] + os: [freebsd] + + '@oxc-resolver/binding-linux-arm-gnueabihf@11.5.2': + resolution: {integrity: sha512-AMV4MbHdUvwA6oBLk90/gPo3gPMZl9+DHeas8BxRdq/uX1BFQ05s+mhy9ATGElGQsRVVOPya9qczOdb8eAlM6w==} + cpu: [arm] + os: [linux] + + '@oxc-resolver/binding-linux-arm-musleabihf@11.5.2': + resolution: {integrity: sha512-hTCkii4HwQushiD3L86cefvojTY6OSDzcrQZHVaUmrtkL0OQnRT9qUff83lJIQhb94rjaEfQsgUdVl1bvuUK/Q==} + cpu: [arm] + os: [linux] + + '@oxc-resolver/binding-linux-arm64-gnu@11.5.2': + resolution: {integrity: sha512-EXkMvem90Pdw0bw0TlOhAHFAGLopb1LaVwsxF+iSc/zQtuR62kl2jGMQRvsW4NHaF+nUN29H8IYQDzox4gxsRw==} + cpu: [arm64] + os: [linux] + + '@oxc-resolver/binding-linux-arm64-musl@11.5.2': + resolution: {integrity: sha512-UvA2QZB73XPXmFweDRyXyUchN1YnEx+cca7a/ojdhT+stDe0WKMK32y27oabWokJJsZZOd+W40dD7sxjzx7K/g==} + cpu: [arm64] + os: [linux] + + '@oxc-resolver/binding-linux-ppc64-gnu@11.5.2': + resolution: {integrity: sha512-0rllGQIAmeb+vAtmco0PnTzqlMs0DQs+QvHu/8AQAmgrlKBZDJJmRvLqMv6EXgTrLlWxoM0o9oNf7mZ0tEenUQ==} + cpu: [ppc64] + os: [linux] + + '@oxc-resolver/binding-linux-riscv64-gnu@11.5.2': + resolution: {integrity: sha512-kfE5ALnGsxEyz/e6lZbNUyPjZwTIuExTVJLVzjT/RjvaltSZ6J0u/6/CKsVFD3t686yqse1fnXuydUsgAFmuXg==} + cpu: [riscv64] + os: [linux] + + '@oxc-resolver/binding-linux-riscv64-musl@11.5.2': + resolution: {integrity: sha512-O6lbEl+heEd3QS2GOwm+iDGMqEWA18X/b9JNodzEHe2TJeOJAV/5xJ7jQTGA2seoy6/REhW744O35DyPFxZ2aQ==} + cpu: [riscv64] + os: [linux] + + '@oxc-resolver/binding-linux-s390x-gnu@11.5.2': + resolution: {integrity: sha512-6ZASmeqVq+xEQZz/EH+U4j1hPeqVQ8Eo58oYrt9FGJhseowAh6TAOHXe80HAJH6HQTcws1fhS/A7I4hm6NOgZA==} + cpu: [s390x] + os: [linux] + + '@oxc-resolver/binding-linux-x64-gnu@11.5.2': + resolution: {integrity: sha512-MYTtU3sKGZfvOYVpUfFHFcxLGOI8WN5BIQeWgNnNDEBHasthEDnyeNYpj6QbLd3XMz84gGA1G+mKMm/lVUF6hA==} + cpu: [x64] + os: [linux] + + '@oxc-resolver/binding-linux-x64-musl@11.5.2': + resolution: {integrity: sha512-7u1ANU1jkDUbC5ZxGXWDs0OLuUvV3DzqHUI+g41wHdz0iLoVSJ7rR+hl/crHIm4PpFkYbpU+joRslM5OLxeKlw==} + cpu: [x64] + os: [linux] + + '@oxc-resolver/binding-wasm32-wasi@11.5.2': + resolution: {integrity: sha512-2tOsCVH+THg9b9h6MiTymTrveSUWAOaQGj2CPQ4XJncxECsZY6MfxKLul+XsW4KLpstE89KBemRIQi6Il0Twew==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + + '@oxc-resolver/binding-win32-arm64-msvc@11.5.2': + resolution: {integrity: sha512-NmpFIoT86wD2cNAweoEMLKZ4aaGzbYzmeMcYK65Ml9PbH53YXe5XZOXdzVExLKGJ3Rorf055n/67pRRvpIm/sQ==} + cpu: [arm64] + os: [win32] + + '@oxc-resolver/binding-win32-ia32-msvc@11.5.2': + resolution: {integrity: sha512-1EwjnPP5sEKdQl4+3edw+8xMZ79qk7iPXOJRUtdE0jLEdlFmzpnLBfsz54G7mOiQvnc6uR8YePBQb1iCRnysNA==} + cpu: [ia32] + os: [win32] + + '@oxc-resolver/binding-win32-x64-msvc@11.5.2': + resolution: {integrity: sha512-eB8eV8SdO+OpbJJ3dvTgSPOsDsW7SJp+ih5WIBWt7pWMlVbQyjBwDgTI8gGTqg2iwdEEUVqlfivEEs22hKnxRw==} + cpu: [x64] + os: [win32] + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -675,6 +782,9 @@ packages: '@tsconfig/strictest@2.0.5': resolution: {integrity: sha512-ec4tjL2Rr0pkZ5hww65c+EEPYwxOi4Ryv+0MtjeaSQRJyq322Q27eOQiFbuNgw2hpL4hB1/W/HBGk3VKS43osg==} + '@tybys/wasm-util@0.10.0': + resolution: {integrity: sha512-VyyPYFlOMNylG45GoAe0xDoLwWuowvf92F9kySqzYh8vmYm7D2u4iUJKa1tOUpS70Ku13ASrOkS4ScXFsTaCNQ==} + '@types/cacheable-request@6.0.3': resolution: {integrity: sha512-IQ3EbTzGxIigb1I3qPZc1rWJnH0BmSKv5QYTalEwweFvyBDLSAe24zP0le/hyi7ecGfZVlIVAg4BZqb8WBwKqw==} @@ -687,9 +797,6 @@ packages: '@types/deep-eql@4.0.2': resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} - '@types/eslint@9.6.1': - resolution: {integrity: sha512-FXx2pKgId/WyYo2jXw63kk7/+TY7u7AziEJxJAnSFzHlqTAS3Ync6SvgYAN/k4/PQpnnVuzoMuVnByKK2qp0ag==} - '@types/estree@1.0.8': resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} @@ -1414,6 +1521,9 @@ packages: fault@1.0.4: resolution: {integrity: sha512-CJ0HCB5tL5fYTEA7ToAq5+kTwd++Borf1/bifxd9iT70QcXr4MRrO3Llf8Ifs70q+SJcGHFtnIE/Nw6giCtECA==} + fd-package-json@2.0.0: + resolution: {integrity: sha512-jKmm9YtsNXN789RS/0mSzOC1NUq9mkVd65vbSSVsKdjGvYXBuE4oWe2QOEoFeRmJg+lPuZxpmrfFclNhoRMneQ==} + fdir@6.4.6: resolution: {integrity: sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w==} peerDependencies: @@ -1476,6 +1586,11 @@ packages: resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==} engines: {node: '>=0.4.x'} + formatly@0.2.4: + resolution: {integrity: sha512-lIN7GpcvX/l/i24r/L9bnJ0I8Qn01qijWpQpDDvTLL29nKqSaJJu4h20+7VJ6m2CAhQ2/En/GbxDiHCzq/0MyA==} + engines: {node: '>=18.3.0'} + hasBin: true + fp-ts@2.16.10: resolution: {integrity: sha512-vuROzbNVfCmUkZSUbnWSltR1sbheyQbTzug7LB/46fEa1c0EucLeBaCEUE0gF3ZGUGBt9lVUiziGOhhj6K1ORA==} @@ -1826,6 +1941,10 @@ packages: jackspeak@3.4.3: resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} + jiti@2.4.2: + resolution: {integrity: sha512-rg9zJN+G4n2nfJl5MW3BMygZX56zKPNVEYYqq7adpmMh4Jn2QNEwhvQlFy6jPVdcod7txZtKHWnyZiA3a0zP7A==} + hasBin: true + joycon@3.1.1: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} engines: {node: '>=10'} @@ -1891,6 +2010,14 @@ packages: keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + knip@5.61.3: + resolution: {integrity: sha512-8iSz8i8ufIjuUwUKzEwye7ROAW0RzCze7T770bUiz0PKL+SSwbs4RS32fjMztLwcOzSsNPlXdUAeqmkdzXxJ1Q==} + engines: {node: '>=18.18.0'} + hasBin: true + peerDependencies: + '@types/node': '>=18' + typescript: '>=5.0.4' + levn@0.4.1: resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} engines: {node: '>= 0.8.0'} @@ -2140,6 +2267,11 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + napi-postinstall@0.3.0: + resolution: {integrity: sha512-M7NqKyhODKV1gRLdkwE7pDsZP2/SC2a2vHkOYh9MCpKMbWVfyVfUw5MaH83Fv6XMjxr5jryUp3IDDL9rlxsTeA==} + engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} + hasBin: true + natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} @@ -2206,6 +2338,9 @@ packages: resolution: {integrity: sha512-qFOyK5PjiWZd+QQIh+1jhdb9LpxTF0qs7Pm8o5QHYZ0M3vKqSqzsZaEB6oWlxZ+q2sJBMI/Ktgd2N5ZwQoRHfg==} engines: {node: '>= 0.4'} + oxc-resolver@11.5.2: + resolution: {integrity: sha512-mYkOsrgvlm4OLPCgSR2XCMkJ203PwSOASxzHYzW7Kz3GXONVbe2VTpgwL/yBo0igSUwlZWTUKEbRJLscJ6N5QQ==} + p-cancelable@2.1.1: resolution: {integrity: sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==} engines: {node: '>=8'} @@ -2541,6 +2676,10 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} + smol-toml@1.4.1: + resolution: {integrity: sha512-CxdwHXyYTONGHThDbq5XdwbFsuY4wlClRGejfE2NtwUtiHYsP1QtNsHb/hnj31jKYSchztJsaA8pSQoVzkfCFg==} + engines: {node: '>= 18'} + snake-case@3.0.4: resolution: {integrity: sha512-LAOh4z89bGQvl9pFfNF8V146i7o7/CqFPbqzYgP+yYzDIDeS9HaNFtXABamRW+AQzEVODcvE79ljJ+8a9YSdMg==} @@ -2616,6 +2755,10 @@ packages: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} + strip-json-comments@5.0.2: + resolution: {integrity: sha512-4X2FR3UwhNUE9G49aIsJW5hRRR3GXGTBTZRMfv568O60ojM8HcWjV/VxAxCDW3SUND33O6ZY66ZuRcdkj73q2g==} + engines: {node: '>=14.16'} + strip-literal@3.0.0: resolution: {integrity: sha512-TcccoMhJOM3OebGhSBEmp3UZ2SfDMZUEBdRA/9ynfLi8yYajyWX3JiXArcJt4Umh4vISpspkQIY8ZZoCqjbviA==} @@ -2921,6 +3064,10 @@ packages: jsdom: optional: true + walk-up-path@4.0.0: + resolution: {integrity: sha512-3hu+tD8YzSLGuFYtPRb48vdhKMi0KQV5sn+uWr8+7dMEq/2G/dtLrdDinkLjqq5TIbIBjYJ4Ax/n3YiaW7QM8A==} + engines: {node: 20 || >=22} + webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -3005,6 +3152,15 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + zod-validation-error@3.5.3: + resolution: {integrity: sha512-OT5Y8lbUadqVZCsnyFaTQ4/O2mys4tj7PqhdbBCp7McPwvIEKfPtdA6QfPeFQK2/Rz5LgwmAXRJTugBNBi0btw==} + engines: {node: '>=18.0.0'} + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + zwitch@1.0.5: resolution: {integrity: sha512-V50KMwwzqJV0NpZIZFwfOD5/lyny3WlSzRiXgA0G7VUnRlqttta1L6UQIHzd6EuBY/cHGfwTIck7w1yH6Q5zUw==} @@ -3057,6 +3213,22 @@ snapshots: '@babel/helper-string-parser': 7.27.1 '@babel/helper-validator-identifier': 7.27.1 + '@emnapi/core@1.4.4': + dependencies: + '@emnapi/wasi-threads': 1.0.3 + tslib: 2.8.1 + optional: true + + '@emnapi/runtime@1.4.4': + dependencies: + tslib: 2.8.1 + optional: true + + '@emnapi/wasi-threads@1.0.3': + dependencies: + tslib: 2.8.1 + optional: true + '@esbuild/aix-ppc64@0.25.6': optional: true @@ -3135,9 +3307,9 @@ snapshots: '@esbuild/win32-x64@0.25.6': optional: true - '@eslint-community/eslint-utils@4.7.0(eslint@9.31.0)': + '@eslint-community/eslint-utils@4.7.0(eslint@9.31.0(jiti@2.4.2))': dependencies: - eslint: 9.31.0 + eslint: 9.31.0(jiti@2.4.2) eslint-visitor-keys: 3.4.3 '@eslint-community/regexpp@4.12.1': {} @@ -3262,6 +3434,13 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.4 + '@napi-rs/wasm-runtime@0.2.12': + dependencies: + '@emnapi/core': 1.4.4 + '@emnapi/runtime': 1.4.4 + '@tybys/wasm-util': 0.10.0 + optional: true + '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 @@ -3274,6 +3453,65 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.19.1 + '@oxc-resolver/binding-android-arm-eabi@11.5.2': + optional: true + + '@oxc-resolver/binding-android-arm64@11.5.2': + optional: true + + '@oxc-resolver/binding-darwin-arm64@11.5.2': + optional: true + + '@oxc-resolver/binding-darwin-x64@11.5.2': + optional: true + + '@oxc-resolver/binding-freebsd-x64@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-arm-gnueabihf@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-arm-musleabihf@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-arm64-gnu@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-arm64-musl@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-ppc64-gnu@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-riscv64-gnu@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-riscv64-musl@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-s390x-gnu@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-x64-gnu@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-x64-musl@11.5.2': + optional: true + + '@oxc-resolver/binding-wasm32-wasi@11.5.2': + dependencies: + '@napi-rs/wasm-runtime': 0.2.12 + optional: true + + '@oxc-resolver/binding-win32-arm64-msvc@11.5.2': + optional: true + + '@oxc-resolver/binding-win32-ia32-msvc@11.5.2': + optional: true + + '@oxc-resolver/binding-win32-x64-msvc@11.5.2': + optional: true + '@pkgjs/parseargs@0.11.0': optional: true @@ -3411,6 +3649,11 @@ snapshots: '@tsconfig/strictest@2.0.5': {} + '@tybys/wasm-util@0.10.0': + dependencies: + tslib: 2.8.1 + optional: true + '@types/cacheable-request@6.0.3': dependencies: '@types/http-cache-semantics': 4.0.4 @@ -3428,11 +3671,6 @@ snapshots: '@types/deep-eql@4.0.2': {} - '@types/eslint@9.6.1': - dependencies: - '@types/estree': 1.0.8 - '@types/json-schema': 7.0.15 - '@types/estree@1.0.8': {} '@types/fuzzysearch@1.0.2': {} @@ -3503,15 +3741,15 @@ snapshots: '@types/yargs-parser@21.0.3': {} - '@typescript-eslint/eslint-plugin@8.37.0(@typescript-eslint/parser@8.37.0(eslint@9.31.0)(typescript@5.8.3))(eslint@9.31.0)(typescript@5.8.3)': + '@typescript-eslint/eslint-plugin@8.37.0(@typescript-eslint/parser@8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3))(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3)': dependencies: '@eslint-community/regexpp': 4.12.1 - '@typescript-eslint/parser': 8.37.0(eslint@9.31.0)(typescript@5.8.3) + '@typescript-eslint/parser': 8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3) '@typescript-eslint/scope-manager': 8.37.0 - '@typescript-eslint/type-utils': 8.37.0(eslint@9.31.0)(typescript@5.8.3) - '@typescript-eslint/utils': 8.37.0(eslint@9.31.0)(typescript@5.8.3) + '@typescript-eslint/type-utils': 8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3) + '@typescript-eslint/utils': 8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3) '@typescript-eslint/visitor-keys': 8.37.0 - eslint: 9.31.0 + eslint: 9.31.0(jiti@2.4.2) graphemer: 1.4.0 ignore: 7.0.5 natural-compare: 1.4.0 @@ -3520,14 +3758,14 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@8.37.0(eslint@9.31.0)(typescript@5.8.3)': + '@typescript-eslint/parser@8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3)': dependencies: '@typescript-eslint/scope-manager': 8.37.0 '@typescript-eslint/types': 8.37.0 '@typescript-eslint/typescript-estree': 8.37.0(typescript@5.8.3) '@typescript-eslint/visitor-keys': 8.37.0 debug: 4.4.1 - eslint: 9.31.0 + eslint: 9.31.0(jiti@2.4.2) typescript: 5.8.3 transitivePeerDependencies: - supports-color @@ -3550,13 +3788,13 @@ snapshots: dependencies: typescript: 5.8.3 - '@typescript-eslint/type-utils@8.37.0(eslint@9.31.0)(typescript@5.8.3)': + '@typescript-eslint/type-utils@8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3)': dependencies: '@typescript-eslint/types': 8.37.0 '@typescript-eslint/typescript-estree': 8.37.0(typescript@5.8.3) - '@typescript-eslint/utils': 8.37.0(eslint@9.31.0)(typescript@5.8.3) + '@typescript-eslint/utils': 8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3) debug: 4.4.1 - eslint: 9.31.0 + eslint: 9.31.0(jiti@2.4.2) ts-api-utils: 2.1.0(typescript@5.8.3) typescript: 5.8.3 transitivePeerDependencies: @@ -3580,13 +3818,13 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@8.37.0(eslint@9.31.0)(typescript@5.8.3)': + '@typescript-eslint/utils@8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3)': dependencies: - '@eslint-community/eslint-utils': 4.7.0(eslint@9.31.0) + '@eslint-community/eslint-utils': 4.7.0(eslint@9.31.0(jiti@2.4.2)) '@typescript-eslint/scope-manager': 8.37.0 '@typescript-eslint/types': 8.37.0 '@typescript-eslint/typescript-estree': 8.37.0(typescript@5.8.3) - eslint: 9.31.0 + eslint: 9.31.0(jiti@2.4.2) typescript: 5.8.3 transitivePeerDependencies: - supports-color @@ -3604,13 +3842,13 @@ snapshots: chai: 5.2.0 tinyrainbow: 2.0.0 - '@vitest/mocker@3.2.4(vite@7.0.2(@types/node@22.16.4)(tsx@4.20.3))': + '@vitest/mocker@3.2.4(vite@7.0.2(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3))': dependencies: '@vitest/spy': 3.2.4 estree-walker: 3.0.3 magic-string: 0.30.17 optionalDependencies: - vite: 7.0.2(@types/node@22.16.4)(tsx@4.20.3) + vite: 7.0.2(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3) '@vitest/pretty-format@3.2.4': dependencies: @@ -4219,15 +4457,15 @@ snapshots: escape-string-regexp@4.0.0: {} - eslint-plugin-unicorn@59.0.1(eslint@9.31.0): + eslint-plugin-unicorn@59.0.1(eslint@9.31.0(jiti@2.4.2)): dependencies: '@babel/helper-validator-identifier': 7.27.1 - '@eslint-community/eslint-utils': 4.7.0(eslint@9.31.0) + '@eslint-community/eslint-utils': 4.7.0(eslint@9.31.0(jiti@2.4.2)) '@eslint/plugin-kit': 0.2.8 ci-info: 4.3.0 clean-regexp: 1.0.0 core-js-compat: 3.44.0 - eslint: 9.31.0 + eslint: 9.31.0(jiti@2.4.2) esquery: 1.6.0 find-up-simple: 1.0.1 globals: 16.3.0 @@ -4249,9 +4487,9 @@ snapshots: eslint-visitor-keys@4.2.1: {} - eslint@9.31.0: + eslint@9.31.0(jiti@2.4.2): dependencies: - '@eslint-community/eslint-utils': 4.7.0(eslint@9.31.0) + '@eslint-community/eslint-utils': 4.7.0(eslint@9.31.0(jiti@2.4.2)) '@eslint-community/regexpp': 4.12.1 '@eslint/config-array': 0.21.0 '@eslint/config-helpers': 0.3.0 @@ -4286,6 +4524,8 @@ snapshots: minimatch: 3.1.2 natural-compare: 1.4.0 optionator: 0.9.4 + optionalDependencies: + jiti: 2.4.2 transitivePeerDependencies: - supports-color @@ -4358,6 +4598,10 @@ snapshots: dependencies: format: 0.2.2 + fd-package-json@2.0.0: + dependencies: + walk-up-path: 4.0.0 + fdir@6.4.6(picomatch@4.0.2): optionalDependencies: picomatch: 4.0.2 @@ -4421,6 +4665,10 @@ snapshots: format@0.2.2: {} + formatly@0.2.4: + dependencies: + fd-package-json: 2.0.0 + fp-ts@2.16.10: {} fsevents@2.3.3: @@ -4813,6 +5061,8 @@ snapshots: optionalDependencies: '@pkgjs/parseargs': 0.11.0 + jiti@2.4.2: {} + joycon@3.1.1: {} js-tokens@4.0.0: {} @@ -4874,6 +5124,24 @@ snapshots: dependencies: json-buffer: 3.0.1 + knip@5.61.3(@types/node@22.16.4)(typescript@5.8.3): + dependencies: + '@nodelib/fs.walk': 1.2.8 + '@types/node': 22.16.4 + fast-glob: 3.3.3 + formatly: 0.2.4 + jiti: 2.4.2 + js-yaml: 4.1.0 + minimist: 1.2.8 + oxc-resolver: 11.5.2 + picocolors: 1.1.1 + picomatch: 4.0.2 + smol-toml: 1.4.1 + strip-json-comments: 5.0.2 + typescript: 5.8.3 + zod: 3.25.76 + zod-validation-error: 3.5.3(zod@3.25.76) + levn@0.4.1: dependencies: prelude-ls: 1.2.1 @@ -5146,6 +5414,8 @@ snapshots: nanoid@3.3.11: {} + napi-postinstall@0.3.0: {} + natural-compare@1.4.0: {} neo-async@2.6.2: {} @@ -5208,6 +5478,30 @@ snapshots: object-keys: 1.1.1 safe-push-apply: 1.0.0 + oxc-resolver@11.5.2: + dependencies: + napi-postinstall: 0.3.0 + optionalDependencies: + '@oxc-resolver/binding-android-arm-eabi': 11.5.2 + '@oxc-resolver/binding-android-arm64': 11.5.2 + '@oxc-resolver/binding-darwin-arm64': 11.5.2 + '@oxc-resolver/binding-darwin-x64': 11.5.2 + '@oxc-resolver/binding-freebsd-x64': 11.5.2 + '@oxc-resolver/binding-linux-arm-gnueabihf': 11.5.2 + '@oxc-resolver/binding-linux-arm-musleabihf': 11.5.2 + '@oxc-resolver/binding-linux-arm64-gnu': 11.5.2 + '@oxc-resolver/binding-linux-arm64-musl': 11.5.2 + '@oxc-resolver/binding-linux-ppc64-gnu': 11.5.2 + '@oxc-resolver/binding-linux-riscv64-gnu': 11.5.2 + '@oxc-resolver/binding-linux-riscv64-musl': 11.5.2 + '@oxc-resolver/binding-linux-s390x-gnu': 11.5.2 + '@oxc-resolver/binding-linux-x64-gnu': 11.5.2 + '@oxc-resolver/binding-linux-x64-musl': 11.5.2 + '@oxc-resolver/binding-wasm32-wasi': 11.5.2 + '@oxc-resolver/binding-win32-arm64-msvc': 11.5.2 + '@oxc-resolver/binding-win32-ia32-msvc': 11.5.2 + '@oxc-resolver/binding-win32-x64-msvc': 11.5.2 + p-cancelable@2.1.1: {} p-limit@3.1.0: @@ -5298,10 +5592,11 @@ snapshots: possible-typed-array-names@1.1.0: {} - postcss-load-config@6.0.1(postcss@8.5.6)(tsx@4.20.3): + postcss-load-config@6.0.1(jiti@2.4.2)(postcss@8.5.6)(tsx@4.20.3): dependencies: lilconfig: 3.1.3 optionalDependencies: + jiti: 2.4.2 postcss: 8.5.6 tsx: 4.20.3 @@ -5581,6 +5876,8 @@ snapshots: signal-exit@4.1.0: {} + smol-toml@1.4.1: {} + snake-case@3.0.4: dependencies: dot-case: 3.0.4 @@ -5660,6 +5957,8 @@ snapshots: strip-json-comments@3.1.1: {} + strip-json-comments@5.0.2: {} + strip-literal@3.0.0: dependencies: js-tokens: 9.0.1 @@ -5743,7 +6042,7 @@ snapshots: tslib@2.8.1: {} - tsup@8.5.0(postcss@8.5.6)(tsx@4.20.3)(typescript@5.8.3): + tsup@8.5.0(jiti@2.4.2)(postcss@8.5.6)(tsx@4.20.3)(typescript@5.8.3): dependencies: bundle-require: 5.1.0(esbuild@0.25.6) cac: 6.7.14 @@ -5754,7 +6053,7 @@ snapshots: fix-dts-default-cjs-exports: 1.0.1 joycon: 3.1.1 picocolors: 1.1.1 - postcss-load-config: 6.0.1(postcss@8.5.6)(tsx@4.20.3) + postcss-load-config: 6.0.1(jiti@2.4.2)(postcss@8.5.6)(tsx@4.20.3) resolve-from: 5.0.0 rollup: 4.44.2 source-map: 0.8.0-beta.0 @@ -5830,13 +6129,13 @@ snapshots: typed-array-buffer: 1.0.3 typed-array-byte-offset: 1.0.4 - typescript-eslint@8.37.0(eslint@9.31.0)(typescript@5.8.3): + typescript-eslint@8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3): dependencies: - '@typescript-eslint/eslint-plugin': 8.37.0(@typescript-eslint/parser@8.37.0(eslint@9.31.0)(typescript@5.8.3))(eslint@9.31.0)(typescript@5.8.3) - '@typescript-eslint/parser': 8.37.0(eslint@9.31.0)(typescript@5.8.3) + '@typescript-eslint/eslint-plugin': 8.37.0(@typescript-eslint/parser@8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3))(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3) + '@typescript-eslint/parser': 8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3) '@typescript-eslint/typescript-estree': 8.37.0(typescript@5.8.3) - '@typescript-eslint/utils': 8.37.0(eslint@9.31.0)(typescript@5.8.3) - eslint: 9.31.0 + '@typescript-eslint/utils': 8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3) + eslint: 9.31.0(jiti@2.4.2) typescript: 5.8.3 transitivePeerDependencies: - supports-color @@ -5916,13 +6215,13 @@ snapshots: unist-util-stringify-position: 2.0.3 vfile-message: 2.0.4 - vite-node@3.2.4(@types/node@22.16.4)(tsx@4.20.3): + vite-node@3.2.4(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3): dependencies: cac: 6.7.14 debug: 4.4.1 es-module-lexer: 1.7.0 pathe: 2.0.3 - vite: 7.0.2(@types/node@22.16.4)(tsx@4.20.3) + vite: 7.0.2(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3) transitivePeerDependencies: - '@types/node' - jiti @@ -5937,18 +6236,18 @@ snapshots: - tsx - yaml - vite-tsconfig-paths@5.1.4(typescript@5.8.3)(vite@7.0.2(@types/node@22.16.4)(tsx@4.20.3)): + vite-tsconfig-paths@5.1.4(typescript@5.8.3)(vite@7.0.2(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3)): dependencies: debug: 4.4.1 globrex: 0.1.2 tsconfck: 3.1.6(typescript@5.8.3) optionalDependencies: - vite: 7.0.2(@types/node@22.16.4)(tsx@4.20.3) + vite: 7.0.2(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3) transitivePeerDependencies: - supports-color - typescript - vite@7.0.2(@types/node@22.16.4)(tsx@4.20.3): + vite@7.0.2(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3): dependencies: esbuild: 0.25.6 fdir: 6.4.6(picomatch@4.0.2) @@ -5959,13 +6258,14 @@ snapshots: optionalDependencies: '@types/node': 22.16.4 fsevents: 2.3.3 + jiti: 2.4.2 tsx: 4.20.3 - vitest@3.2.4(@types/node@22.16.4)(tsx@4.20.3): + vitest@3.2.4(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3): dependencies: '@types/chai': 5.2.2 '@vitest/expect': 3.2.4 - '@vitest/mocker': 3.2.4(vite@7.0.2(@types/node@22.16.4)(tsx@4.20.3)) + '@vitest/mocker': 3.2.4(vite@7.0.2(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3)) '@vitest/pretty-format': 3.2.4 '@vitest/runner': 3.2.4 '@vitest/snapshot': 3.2.4 @@ -5983,8 +6283,8 @@ snapshots: tinyglobby: 0.2.14 tinypool: 1.1.1 tinyrainbow: 2.0.0 - vite: 7.0.2(@types/node@22.16.4)(tsx@4.20.3) - vite-node: 3.2.4(@types/node@22.16.4)(tsx@4.20.3) + vite: 7.0.2(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3) + vite-node: 3.2.4(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 22.16.4 @@ -6002,6 +6302,8 @@ snapshots: - tsx - yaml + walk-up-path@4.0.0: {} + webidl-conversions@3.0.1: {} webidl-conversions@4.0.2: {} @@ -6109,4 +6411,10 @@ snapshots: yocto-queue@0.1.0: {} + zod-validation-error@3.5.3(zod@3.25.76): + dependencies: + zod: 3.25.76 + + zod@3.25.76: {} + zwitch@1.0.5: {} diff --git a/src/lib/consent-manager/types.ts b/src/lib/consent-manager/types.ts index f538b5f9..265d33c1 100644 --- a/src/lib/consent-manager/types.ts +++ b/src/lib/consent-manager/types.ts @@ -1,6 +1,6 @@ import * as t from 'io-ts'; -export const ConsentPreferenceBase = t.intersection([ +const ConsentPreferenceBase = t.intersection([ t.type({ /** User ID */ userId: t.string, @@ -28,7 +28,7 @@ export const ConsentPreferenceBase = t.intersection([ ]); /** Type override */ -export type ConsentPreferenceBase = t.TypeOf; +type ConsentPreferenceBase = t.TypeOf; export const ConsentPreferenceUpload = t.intersection([ ConsentPreferenceBase, diff --git a/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts b/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts deleted file mode 100644 index 607d9134..00000000 --- a/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts +++ /dev/null @@ -1,195 +0,0 @@ -import { RequestAction } from '@transcend-io/privacy-types'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; -import { - buildTranscendGraphQLClient, - createSombraGotInstance, - fetchRequestDataSiloActiveCount, -} from '../graphql'; -import { - CronIdentifier, - pullCronPageOfIdentifiers, -} from './pullCronPageOfIdentifiers'; - -/** - * A CSV formatted identifier - */ -export type CsvFormattedIdentifier = Record< - string, - string | null | boolean | number ->; - -export interface CronIdentifierWithAction extends CronIdentifier { - /** The request action that the identifier relates to */ - action: RequestAction; -} - -/** - * Pull the set of identifiers outstanding for a cron or AVC integration - * - * This function is designed to be used in a loop, and will call the onSave callback - * with a chunk of identifiers when the savePageSize is reached. - * - * @param options - Options - * @returns The identifiers and identifiers formatted for CSV - */ -export async function pullChunkedCustomSiloOutstandingIdentifiers({ - dataSiloId, - auth, - sombraAuth, - actions, - apiPageSize = 100, - savePageSize = 1000, - onSave, - transcendUrl = DEFAULT_TRANSCEND_API, - skipRequestCount = false, -}: { - /** Transcend API key authentication */ - auth: string; - /** Data Silo ID to pull down jobs for */ - dataSiloId: string; - /** The request actions to fetch */ - actions: RequestAction[]; - /** How many identifiers to pull in a single call to the backend */ - apiPageSize: number; - /** How many identifiers to save at a time (usually to a CSV file, should be a multiple of apiPageSize) */ - savePageSize: number; - /** Callback function called when a chunk of identifiers is ready to be saved */ - onSave: (chunk: CsvFormattedIdentifier[]) => Promise; - /** API URL for Transcend backend */ - transcendUrl?: string; - /** Sombra API key authentication */ - sombraAuth?: string; - /** Skip request count */ - skipRequestCount?: boolean; -}): Promise<{ - /** Raw Identifiers */ - identifiers: CronIdentifierWithAction[]; -}> { - // Validate savePageSize - if (savePageSize % apiPageSize !== 0) { - throw new Error( - `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}`, - ); - } - - // Create sombra instance to communicate with - const sombra = await createSombraGotInstance(transcendUrl, auth, sombraAuth); - - // Create GraphQL client to connect to Transcend backend - const client = buildTranscendGraphQLClient(transcendUrl, auth); - - let totalRequestCount = 0; - if (!skipRequestCount) { - totalRequestCount = await fetchRequestDataSiloActiveCount(client, { - dataSiloId, - }); - } - - logger.info( - colors.magenta( - `Pulling ${ - skipRequestCount ? 'all' : totalRequestCount - } outstanding request identifiers ` + - `for data silo: "${dataSiloId}" for requests of types "${actions.join( - '", "', - )}"`, - ), - ); - - // Time duration - const t0 = Date.now(); - // create a new progress bar instance and use shades_classic theme - const progressBar = new cliProgress.SingleBar( - {}, - cliProgress.Presets.shades_classic, - ); - const foundRequestIds = new Set(); - - // identifiers found in total - const identifiers: CronIdentifierWithAction[] = []; - // current chunk of identifiers to be saved - let currentChunk: CsvFormattedIdentifier[] = []; - - // map over each action - if (!skipRequestCount) { - progressBar.start(totalRequestCount, 0); - } - await mapSeries(actions, async (action) => { - let offset = 0; - let shouldContinue = true; - - // Fetch a page of identifiers - while (shouldContinue) { - const pageIdentifiers = await pullCronPageOfIdentifiers(sombra, { - dataSiloId, - limit: apiPageSize, - offset, - requestType: action, - }); - - const identifiersWithAction: CronIdentifierWithAction[] = - pageIdentifiers.map((identifier) => { - foundRequestIds.add(identifier.requestId); - return { - ...identifier, - action, - }; - }); - - const csvFormattedIdentifiers = identifiersWithAction.map( - ({ attributes, ...identifier }) => ({ - ...identifier, - ...Object.fromEntries( - attributes.map((value) => [value.key, value.values.join(',')]), - ), - }), - ); - - identifiers.push(...identifiersWithAction); - currentChunk.push(...csvFormattedIdentifiers); - - // Check if we've reached the savePageSize and call the onSave callback - if (currentChunk.length >= savePageSize) { - await onSave(currentChunk); - currentChunk = []; - } - - shouldContinue = pageIdentifiers.length === apiPageSize; - offset += apiPageSize; - if (skipRequestCount) { - logger.info( - colors.magenta( - `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests`, - ), - ); - } else { - progressBar.update(foundRequestIds.size); - } - } - }); - - // Save any remaining identifiers in the current chunk - if (currentChunk.length > 0) { - await onSave(currentChunk); - } - - if (!skipRequestCount) { - progressBar.stop(); - } - const t1 = Date.now(); - const totalTime = t1 - t0; - - logger.info( - colors.green( - `Successfully pulled ${identifiers.length} outstanding identifiers from ${ - foundRequestIds.size - } requests in "${totalTime / 1000}" seconds!`, - ), - ); - - return { identifiers }; -} diff --git a/src/lib/graphql/formatAttributeValues.ts b/src/lib/graphql/formatAttributeValues.ts index 87af79a8..642aa64c 100644 --- a/src/lib/graphql/formatAttributeValues.ts +++ b/src/lib/graphql/formatAttributeValues.ts @@ -1,6 +1,6 @@ import type { DataSiloAttributeValue } from './syncDataSilos'; -export interface FormattedAttribute { +interface FormattedAttribute { /** Attribute key */ key: string; /** Attribute values */ diff --git a/src/lib/graphql/syncPromptGroups.ts b/src/lib/graphql/syncPromptGroups.ts index 178ba8d1..c1456c5a 100644 --- a/src/lib/graphql/syncPromptGroups.ts +++ b/src/lib/graphql/syncPromptGroups.ts @@ -9,7 +9,7 @@ import { fetchAllPrompts } from './fetchPrompts'; import { CREATE_PROMPT_GROUP, UPDATE_PROMPT_GROUPS } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -export interface EditPromptGroupInput { +interface EditPromptGroupInput { /** Title of prompt group */ title: string; /** Prompt group description */ @@ -25,7 +25,7 @@ export interface EditPromptGroupInput { * @param input - Prompt input * @returns Prompt group ID */ -export async function createPromptGroup( +async function createPromptGroup( client: GraphQLClient, input: EditPromptGroupInput, ): Promise { @@ -55,7 +55,7 @@ export async function createPromptGroup( * @param client - GraphQL client * @param input - Prompt input */ -export async function updatePromptGroups( +async function updatePromptGroups( client: GraphQLClient, input: [EditPromptGroupInput, string][], ): Promise { diff --git a/src/lib/graphql/syncPromptPartials.ts b/src/lib/graphql/syncPromptPartials.ts index dcdbe2af..bfc13644 100644 --- a/src/lib/graphql/syncPromptPartials.ts +++ b/src/lib/graphql/syncPromptPartials.ts @@ -15,7 +15,7 @@ import { makeGraphQLRequest } from './makeGraphQLRequest'; * @param input - Prompt input * @returns Prompt partial ID */ -export async function createPromptPartial( +async function createPromptPartial( client: GraphQLClient, input: { /** Title of prompt partial */ @@ -50,7 +50,7 @@ export async function createPromptPartial( * @param client - GraphQL client * @param input - Prompt input */ -export async function updatePromptPartials( +async function updatePromptPartials( client: GraphQLClient, input: [PromptPartialInput, string][], ): Promise { diff --git a/src/lib/graphql/syncTeams.ts b/src/lib/graphql/syncTeams.ts index 7681bee0..98c126ad 100644 --- a/src/lib/graphql/syncTeams.ts +++ b/src/lib/graphql/syncTeams.ts @@ -15,7 +15,7 @@ import { makeGraphQLRequest } from './makeGraphQLRequest'; * @param team - Input * @returns Created team */ -export async function createTeam( +async function createTeam( client: GraphQLClient, team: TeamInput, ): Promise> { @@ -49,7 +49,7 @@ export async function createTeam( * @param teamId - ID of team * @returns Updated team */ -export async function updateTeam( +async function updateTeam( client: GraphQLClient, input: TeamInput, teamId: string, diff --git a/src/lib/oneTrust/helpers/convertToEmptyStrings.ts b/src/lib/oneTrust/helpers/convertToEmptyStrings.ts deleted file mode 100644 index 40a81162..00000000 --- a/src/lib/oneTrust/helpers/convertToEmptyStrings.ts +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Converts all primitive values in an object or array to empty strings while maintaining structure - * - * @param input - A primitive value, object, or array to be processed - * @returns The input structure with all primitive values converted to empty strings - * @example - * // Simple primitive - * convertToEmptyStrings(42) // returns '' - * - * // Array - * convertToEmptyStrings([1, 'hello', true]) // returns ['', '', ''] - * - * // Complex object - * convertToEmptyStrings({ - * id: 123, - * name: 'test', - * nested: { - * active: true, - * count: 0 - * }, - * items: [1, 2, 3] - * }) - * // returns { - * // id: '', - * // name: '', - * // nested: { - * // active: '', - * // count: '' - * // }, - * // items: ['', '', ''] - * // } - */ -export function convertToEmptyStrings(input?: unknown): unknown { - // Handle null/undefined - if (input === null || input === undefined) { - return ''; - } - - // Handle arrays - if (Array.isArray(input)) { - return input.map((item) => convertToEmptyStrings(item)); - } - - // Handle objects - if (typeof input === 'object') { - return Object.fromEntries( - Object.entries(input).map(([key, value]) => [ - key, - convertToEmptyStrings(value), - ]), - ); - } - - // Handle primitives - return ''; -} diff --git a/src/lib/oneTrust/helpers/index.ts b/src/lib/oneTrust/helpers/index.ts index bdf24b7f..38d7c8aa 100644 --- a/src/lib/oneTrust/helpers/index.ts +++ b/src/lib/oneTrust/helpers/index.ts @@ -1,4 +1,3 @@ -export * from './parseCliSyncOtArguments'; export * from './syncOneTrustAssessmentToDisk'; export * from './syncOneTrustAssessmentsFromOneTrust'; export * from './syncOneTrustAssessmentsFromFile'; diff --git a/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts b/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts deleted file mode 100644 index 5dde3ee2..00000000 --- a/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts +++ /dev/null @@ -1,185 +0,0 @@ -import colors from 'colors'; -import yargs from 'yargs-parser'; -import { - OneTrustFileFormat, - OneTrustPullResource, - OneTrustPullSource, -} from '../../../enums'; -import { logger } from '../../../logger'; - -const VALID_RESOURCES = Object.values(OneTrustPullResource); - -interface OneTrustCliArguments { - /** The name of the file to write the resources to */ - file: string; - /** The OneTrust hostname to send the requests to */ - hostname?: string; - /** The OAuth Bearer token used to authenticate the requests to OneTrust */ - oneTrustAuth?: string; - /** The Transcend API key to authenticate the requests to Transcend */ - transcendAuth: string; - /** The Transcend URL where to forward requests */ - transcendUrl: string; - /** The resource to pull from OneTrust */ - resource: OneTrustPullResource; - /** Whether to enable debugging while reporting errors */ - debug: boolean; - /** Whether to export the resource into a file rather than push to transcend */ - dryRun: boolean; - /** Where to read the OneTrust resource from */ - source: OneTrustPullSource; -} - -/** - * Parse the command line arguments - * - * @returns the parsed arguments - */ -export const parseCliSyncOtArguments = (): OneTrustCliArguments => { - const { - file, - hostname, - oneTrustAuth, - resource, - debug, - dryRun, - transcendAuth, - transcendUrl, - source, - } = yargs(process.argv.slice(2), { - string: [ - 'file', - 'hostname', - 'oneTrustAuth', - 'resource', - 'dryRun', - 'transcendAuth', - 'transcendUrl', - 'source', - ], - boolean: ['debug', 'dryRun'], - default: { - resource: OneTrustPullResource.Assessments, - debug: false, - dryRun: false, - transcendUrl: 'https://api.transcend.io', - source: OneTrustPullSource.OneTrust, - }, - }); - - // Must be able to authenticate to transcend to sync resources to it - if (!dryRun && !transcendAuth) { - logger.error( - colors.red( - 'Must specify a "transcendAuth" parameter to sync resources to Transcend. e.g. --transcendAuth=${TRANSCEND_API_KEY}', - ), - ); - return process.exit(1); - } - if (!dryRun && !transcendUrl) { - logger.error( - colors.red( - 'Must specify a "transcendUrl" parameter to sync resources to Transcend. e.g. --transcendUrl=https://api.transcend.io', - ), - ); - return process.exit(1); - } - - // If trying to sync to disk, must specify a file path - if (dryRun && !file) { - logger.error( - colors.red( - 'Must set a "file" parameter when "dryRun" is "true". e.g. --file=./oneTrustAssessments.json', - ), - ); - return process.exit(1); - } - - if (file) { - const splitFile = file.split('.'); - if (splitFile.length < 2) { - logger.error( - colors.red( - 'The "file" parameter has an invalid format. Expected a path with extensions. e.g. --file=./pathToFile.json.', - ), - ); - return process.exit(1); - } - if (splitFile.at(-1) !== OneTrustFileFormat.Json) { - logger.error( - colors.red( - `Expected the format of the "file" parameters '${file}' to be '${ - OneTrustFileFormat.Json - }', but got '${splitFile.at(-1)}'.`, - ), - ); - return process.exit(1); - } - } - - // if reading assessments from a OneTrust - if (source === OneTrustPullSource.OneTrust) { - // must specify the OneTrust hostname - if (!hostname) { - logger.error( - colors.red( - 'Missing required parameter "hostname". e.g. --hostname=customer.my.onetrust.com', - ), - ); - return process.exit(1); - } - // must specify the OneTrust auth - if (!oneTrustAuth) { - logger.error( - colors.red( - 'Missing required parameter "oneTrustAuth". e.g. --oneTrustAuth=$ONE_TRUST_AUTH_TOKEN', - ), - ); - return process.exit(1); - } - } else { - // if reading the assessments from a file, must specify a file to read from - if (!file) { - logger.error( - colors.red( - 'Must specify a "file" parameter to read the OneTrust assessments from. e.g. --source=./oneTrustAssessments.json', - ), - ); - return process.exit(1); - } - - // Cannot try reading from file and save assessments to a file simultaneously - if (dryRun) { - logger.error( - colors.red( - 'Cannot read and write to a file simultaneously.' + - ` Emit the "source" parameter or set it to ${OneTrustPullSource.OneTrust} if "dryRun" is enabled.`, - ), - ); - return process.exit(1); - } - } - - if (!VALID_RESOURCES.includes(resource)) { - logger.error( - colors.red( - `Received invalid resource value: "${resource}". Allowed: ${VALID_RESOURCES.join( - ',', - )}`, - ), - ); - return process.exit(1); - } - - return { - file, - ...(hostname && { hostname }), - ...(oneTrustAuth && { oneTrustAuth }), - resource, - debug, - dryRun, - transcendAuth, - transcendUrl, - source, - }; -}; diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts index e18c1d26..38f08f0e 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts @@ -9,7 +9,7 @@ import { } from '../../graphql'; import { oneTrustAssessmentToJson } from './oneTrustAssessmentToJson'; -export interface AssessmentForm { +interface AssessmentForm { /** ID of Assessment Form */ id: string; /** Title of Assessment Form */ diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts index c06eb645..0ed1ee93 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts @@ -21,13 +21,6 @@ import { enrichOneTrustAssessment } from './enrichOneTrustAssessment'; import { syncOneTrustAssessmentToDisk } from './syncOneTrustAssessmentToDisk'; import { syncOneTrustAssessmentToTranscend } from './syncOneTrustAssessmentToTranscend'; -export interface AssessmentForm { - /** ID of Assessment Form */ - id: string; - /** Title of Assessment Form */ - name: string; -} - /** * Reads all the assessments from a OneTrust instance and syncs them to Transcend or to Disk. * diff --git a/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts b/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts deleted file mode 100644 index a5de0dd5..00000000 --- a/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { convertToEmptyStrings } from '../convertToEmptyStrings'; - -describe('buildDefaultCodecWrapper', () => { - it('should correctly build a default codec for null', () => { - const result = convertToEmptyStrings(null); - expect(result).to.equal(''); - }); - - it('should correctly build a default codec for number', () => { - const result = convertToEmptyStrings(0); - expect(result).to.equal(''); - }); - - it('should correctly build a default codec for boolean', () => { - const result = convertToEmptyStrings(false); - expect(result).to.equal(''); - }); - - it('should correctly build a default codec for undefined', () => { - const result = convertToEmptyStrings(); - expect(result).to.equal(''); - }); - - it('should correctly build a default codec for string', () => { - const result = convertToEmptyStrings('1'); - expect(result).to.equal(''); - }); - - it('should correctly build a default codec for an object', () => { - const result = convertToEmptyStrings({ name: 'joe' }); - expect(result).to.deep.equal({ name: '' }); - }); - - it('should correctly build a default codec for an array of primitive types', () => { - const result = convertToEmptyStrings(['name', 0, false]); - expect(result).to.deep.equal(['', '', '']); - }); - - it('should correctly build a default codec for an array of object', () => { - const result = convertToEmptyStrings([ - { name: 'john', age: 52 }, - { name: 'jane', age: 15, isAdult: true }, - ]); - // should default to the array with object if the union contains an array of objects - expect(result).to.deep.equal([ - { name: '', age: '' }, - { name: '', age: '', isAdult: '' }, - ]); - }); -}); diff --git a/vitest.config.ts b/vitest.config.ts index 9ecfec02..497f8153 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,4 +3,7 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ plugins: [tsconfigPaths()], + test: { + include: ['src/**/*.test.ts'], + }, }); From a472eab7a4cefefdd30b36da1bde015b1cf33192 Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Thu, 17 Jul 2025 00:38:44 -0400 Subject: [PATCH 11/18] error handling + type errors --- src/lib/ai/TranscendPromptManager.ts | 24 +++++++---- src/lib/cron/markCronIdentifierCompleted.ts | 12 ++++-- src/lib/graphql/createSombraGotInstance.ts | 12 +++++- src/lib/graphql/fetchCatalogs.ts | 4 +- src/lib/graphql/makeGraphQLRequest.ts | 43 +++++++++++-------- .../manual-enrichment/enrichPrivacyRequest.ts | 23 ++++++---- .../pushManualEnrichmentIdentifiersFromCsv.ts | 16 ++++--- src/lib/requests/skipRequestDataSilos.ts | 9 ++-- src/lib/requests/streamPrivacyRequestFiles.ts | 15 +++++-- src/lib/requests/submitPrivacyRequest.ts | 9 ++-- .../tests/mapCsvRowsToRequestInputs.test.ts | 4 +- 11 files changed, 115 insertions(+), 56 deletions(-) diff --git a/src/lib/ai/TranscendPromptManager.ts b/src/lib/ai/TranscendPromptManager.ts index 86571d25..f1d4db36 100644 --- a/src/lib/ai/TranscendPromptManager.ts +++ b/src/lib/ai/TranscendPromptManager.ts @@ -277,7 +277,9 @@ export class TranscendPromptManager< */ async fetchPromptsAndMetadata(): Promise { // Determine what to fetch - const promptDefinitions = getValues(this.prompts); + const promptDefinitions = getValues>( + this.prompts, + ); const promptIds = promptDefinitions .map(({ id }) => id) .filter((x): x is string => !!x); @@ -285,7 +287,7 @@ export class TranscendPromptManager< .map(({ title }) => title) .filter((x): x is string => !!x); const agentNames = uniq( - promptDefinitions.flatMap(({ agentNames }) => agentNames || []), + promptDefinitions.flatMap(({ agentNames }) => agentNames ?? []), ); // Fetch prompts and data @@ -624,22 +626,28 @@ export class TranscendPromptManager< `promptRunMessages[0].role is expected to be = ${ChatCompletionRole.System}`, ); } - if ( - options.promptRunMessages.at(-1).role !== ChatCompletionRole.Assistant - ) { + + const lastMessage = options.promptRunMessages.at(-1); + if (!lastMessage) { + throw new Error('promptRunMessages is expected to have length > 0'); + } + if (lastMessage.role !== ChatCompletionRole.Assistant) { throw new Error( - `promptRunMessages[${ + `promptRunMessages[${( options.promptRunMessages.length - 1 - }].role is expected to be = ${ChatCompletionRole.Assistant}`, + ).toString()}].role is expected to be = ${ChatCompletionRole.Assistant}`, ); } - const response = options.promptRunMessages.at(-1).content; + const response = lastMessage.content; let parsed: t.TypeOf; try { // Parse the response parsed = this.parseAiResponse(promptName, response); } catch (error) { + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } await reportPromptRun(this.graphQLClient, { productArea: PromptRunProductArea.PromptManager, ...options, diff --git a/src/lib/cron/markCronIdentifierCompleted.ts b/src/lib/cron/markCronIdentifierCompleted.ts index 47b70c9b..6da7c4a6 100644 --- a/src/lib/cron/markCronIdentifierCompleted.ts +++ b/src/lib/cron/markCronIdentifierCompleted.ts @@ -1,5 +1,6 @@ -import type { Got } from 'got'; +import { RequestError, type Got } from 'got'; import * as t from 'io-ts'; +import { isSombraError } from '../graphql'; /** * Minimal set required to mark as completed @@ -41,12 +42,17 @@ export async function markCronIdentifierCompleted( return true; } catch (error) { // handle gracefully - if (error.response?.statusCode === 409) { + if (error instanceof RequestError && error.response?.statusCode === 409) { return false; } + + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + throw new Error( `Received an error from server: ${ - error?.response?.body || error?.message + isSombraError(error) ? error.response.body : error.message }`, ); } diff --git a/src/lib/graphql/createSombraGotInstance.ts b/src/lib/graphql/createSombraGotInstance.ts index 9bd392f0..38d7e36e 100644 --- a/src/lib/graphql/createSombraGotInstance.ts +++ b/src/lib/graphql/createSombraGotInstance.ts @@ -1,8 +1,18 @@ -import got, { Got } from 'got'; +import got, { Got, RequestError, Response } from 'got'; import { buildTranscendGraphQLClient } from './buildTranscendGraphQLClient'; import { ORGANIZATION } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; +interface SombraError extends Omit { + response: Response; +} + +export function isSombraError(error: unknown): error is SombraError { + return ( + error instanceof RequestError && typeof error.response?.body === 'string' + ); +} + /** * Instantiate an instance of got that is capable of making requests * to a sombra gateway. diff --git a/src/lib/graphql/fetchCatalogs.ts b/src/lib/graphql/fetchCatalogs.ts index 75502a95..b5ca8298 100644 --- a/src/lib/graphql/fetchCatalogs.ts +++ b/src/lib/graphql/fetchCatalogs.ts @@ -73,7 +73,7 @@ export async function fetchAndIndexCatalogs(client: GraphQLClient): Promise< // Create mapping from service name to service title const serviceToTitle = Object.fromEntries( - catalogs.map>((catalog) => [ + catalogs.map<[string, string]>((catalog) => [ catalog.integrationName, catalog.title, ]), @@ -81,7 +81,7 @@ export async function fetchAndIndexCatalogs(client: GraphQLClient): Promise< // Create mapping from service name to boolean indicate if service has API integration support const serviceToSupportedIntegration = Object.fromEntries( - catalogs.map>((catalog) => [ + catalogs.map<[string, boolean]>((catalog) => [ catalog.integrationName, catalog.hasApiFunctionality, ]), diff --git a/src/lib/graphql/makeGraphQLRequest.ts b/src/lib/graphql/makeGraphQLRequest.ts index ea13b0f8..3edf1a0b 100644 --- a/src/lib/graphql/makeGraphQLRequest.ts +++ b/src/lib/graphql/makeGraphQLRequest.ts @@ -1,8 +1,10 @@ +/* eslint-disable unicorn/filename-case */ import colors from 'colors'; -import type { - GraphQLClient, - RequestDocument, - Variables, +import { + ClientError, + type GraphQLClient, + type RequestDocument, + type Variables, } from 'graphql-request'; import { logger } from '../../logger'; @@ -40,29 +42,31 @@ const KNOWN_ERRORS = [ * @param maxRequests - Max number of requests * @returns Response */ -export async function makeGraphQLRequest( +export async function makeGraphQLRequest( client: GraphQLClient, document: RequestDocument, - variables?: V, + variables?: Variables, requestHeaders?: Record | string[][] | Headers, maxRequests = MAX_RETRIES, ): Promise { let retryCount = 0; + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition while (true) { try { const result = await client.request(document, variables, requestHeaders); return result as T; } catch (error) { + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + if (error.message.includes('API key is invalid')) { - logger.error( - colors.red( - 'API key is invalid. ' + - 'Please ensure that the key provided to `transcendAuth` has the proper scope and is not expired, ' + - 'and that `transcendUrl` corresponds to the correct backend for your organization.', - ), + throw new Error( + 'API key is invalid. ' + + 'Please ensure that the key provided to `transcendAuth` has the proper scope and is not expired, ' + + 'and that `transcendUrl` corresponds to the correct backend for your organization.', ); - process.exit(1); } if (KNOWN_ERRORS.some((message) => error.message.includes(message))) { @@ -70,15 +74,18 @@ export async function makeGraphQLRequest( } // wait for rate limit to resolve - if (error.message.startsWith('Client error: Too many requests')) { - const rateLimitResetAt = - error.response.headers?.get('x-ratelimit-reset'); + if ( + error instanceof ClientError && + error.message.startsWith('Client error: Too many requests') + ) { + const headers = error.response.headers as Headers | undefined; + const rateLimitResetAt = headers?.get('x-ratelimit-reset'); const sleepTime = rateLimitResetAt ? new Date(rateLimitResetAt).getTime() - Date.now() + 100 : 1000 * 10; logger.log( colors.yellow( - `DETECTED RATE LIMIT: ${error.message}. Sleeping for ${sleepTime}ms`, + `DETECTED RATE LIMIT: ${error.message}. Sleeping for ${sleepTime.toLocaleString()}ms`, ), ); @@ -91,7 +98,7 @@ export async function makeGraphQLRequest( retryCount += 1; logger.log( colors.yellow( - `REQUEST FAILED: ${error.message}. Retrying ${retryCount}/${maxRequests}...`, + `REQUEST FAILED: ${error.message}. Retrying ${retryCount.toLocaleString()}/${maxRequests.toLocaleString()}...`, ), ); } diff --git a/src/lib/manual-enrichment/enrichPrivacyRequest.ts b/src/lib/manual-enrichment/enrichPrivacyRequest.ts index 91b1475a..82baa54a 100644 --- a/src/lib/manual-enrichment/enrichPrivacyRequest.ts +++ b/src/lib/manual-enrichment/enrichPrivacyRequest.ts @@ -1,8 +1,9 @@ import colors from 'colors'; -import type { Got } from 'got'; +import { type Got } from 'got'; import * as t from 'io-ts'; import { uniq } from 'lodash-es'; import { logger } from '../../logger'; +import { isSombraError } from '../graphql'; import { splitCsvToList } from '../requests/splitCsvToList'; const ADMIN_URL = @@ -33,7 +34,7 @@ export async function enrichPrivacyRequest( if (!rawId) { // error const message = `Request ID must be provided to enricher request.${ - index ? ` Found error in row: ${index}` : '' + index ? ` Found error in row: ${index.toLocaleString()}` : '' }`; logger.error(colors.red(message)); throw new Error(message); @@ -42,6 +43,7 @@ export async function enrichPrivacyRequest( const id = rawId.toLowerCase(); // Pull out the identifiers + // eslint-disable-next-line unicorn/no-array-reduce const enrichedIdentifiers = Object.entries(rest).reduce< Record >((accumulator, [key, value]) => { @@ -76,7 +78,7 @@ export async function enrichPrivacyRequest( } catch (error) { // skip if already enriched if ( - typeof error.response.body === 'string' && + isSombraError(error) && error.response.body.includes('Cannot update a resolved RequestEnricher') ) { logger.warn( @@ -87,12 +89,15 @@ export async function enrichPrivacyRequest( return false; } - // error - logger.error( - colors.red( - `Failed to enricher identifiers for request with id: ${ADMIN_URL}${id} - ${error.message} - ${error.response.body}`, - ), - ); + // Error message + let message = `Failed to enricher identifiers for request with id: ${ADMIN_URL}${id}`; + if (error instanceof Error) { + message += ` - ${error.message}`; + } + if (isSombraError(error)) { + message += ` - ${error.response.body}`; + } + logger.error(colors.red(message)); throw error; } } diff --git a/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts b/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts index 58e5f10c..d97395e8 100644 --- a/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts +++ b/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts @@ -54,7 +54,9 @@ export async function pushManualEnrichmentIdentifiersFromCsv({ // Notify Transcend logger.info( - colors.magenta(`Enriching "${activeResults.length}" privacy requests.`), + colors.magenta( + `Enriching "${activeResults.length.toLocaleString()}" privacy requests.`, + ), ); let successCount = 0; @@ -99,17 +101,21 @@ export async function pushManualEnrichmentIdentifiersFromCsv({ logger.info( colors.green( - `Successfully notified Transcend! \n Success count: ${successCount}.`, + `Successfully notified Transcend! \n Success count: ${successCount.toLocaleString()}.`, ), ); if (skippedCount > 0) { - logger.info(colors.magenta(`Skipped count: ${skippedCount}.`)); + logger.info( + colors.magenta(`Skipped count: ${skippedCount.toLocaleString()}.`), + ); } if (errorCount > 0) { - logger.info(colors.red(`Error Count: ${errorCount}.`)); - throw new Error(`Failed to enrich: ${errorCount} requests.`); + logger.info(colors.red(`Error Count: ${errorCount.toLocaleString()}.`)); + throw new Error( + `Failed to enrich: ${errorCount.toLocaleString()} requests.`, + ); } return activeResults.length; diff --git a/src/lib/requests/skipRequestDataSilos.ts b/src/lib/requests/skipRequestDataSilos.ts index 23000495..f1981dcf 100644 --- a/src/lib/requests/skipRequestDataSilos.ts +++ b/src/lib/requests/skipRequestDataSilos.ts @@ -53,7 +53,7 @@ export async function skipRequestDataSilos({ // Notify Transcend logger.info( colors.magenta( - `Processing data silo: "${dataSiloId}" marking "${requestDataSilos.length}" requests as skipped.`, + `Processing data silo: "${dataSiloId}" marking "${requestDataSilos.length.toLocaleString()}" requests as skipped.`, ), ); @@ -77,7 +77,10 @@ export async function skipRequestDataSilos({ status, }); } catch (error) { - if (!error.message.includes('Client error: Request must be active:')) { + if ( + !(error instanceof Error) || + !error.message.includes('Client error: Request must be active:') + ) { throw error; } } @@ -94,7 +97,7 @@ export async function skipRequestDataSilos({ logger.info( colors.green( - `Successfully skipped "${requestDataSilos.length}" requests in "${ + `Successfully skipped "${requestDataSilos.length.toLocaleString()}" requests in "${ totalTime / 1000 }" seconds!`, ), diff --git a/src/lib/requests/streamPrivacyRequestFiles.ts b/src/lib/requests/streamPrivacyRequestFiles.ts index 35d20a85..8659164a 100644 --- a/src/lib/requests/streamPrivacyRequestFiles.ts +++ b/src/lib/requests/streamPrivacyRequestFiles.ts @@ -1,7 +1,8 @@ import colors from 'colors'; -import type { Got } from 'got'; +import { type Got } from 'got'; import { logger } from '../../logger'; import { map } from '../bluebird-replace'; +import { isSombraError } from '../graphql'; import { RequestFileMetadata } from './getFileMetadataForPrivacyRequests'; /** @@ -47,7 +48,10 @@ export async function streamPrivacyRequestFiles( onFileDownloaded(metadata, fileResponse); }); } catch (error) { - if (error?.response?.body?.includes('fileMetadata#verify')) { + if ( + isSombraError(error) && + error.response.body.includes('fileMetadata#verify') + ) { logger.error( colors.red( `Failed to pull file for: ${metadata.fileName} (request:${requestId}) - JWT expired. ` + @@ -58,9 +62,14 @@ export async function streamPrivacyRequestFiles( ); return; } + + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + throw new Error( `Received an error from server: ${ - error?.response?.body || error?.message + isSombraError(error) ? error.response.body : error.message }`, ); } diff --git a/src/lib/requests/submitPrivacyRequest.ts b/src/lib/requests/submitPrivacyRequest.ts index a88a1ac3..54704633 100644 --- a/src/lib/requests/submitPrivacyRequest.ts +++ b/src/lib/requests/submitPrivacyRequest.ts @@ -8,6 +8,7 @@ import { decodeCodec, valuesOf } from '@transcend-io/type-utils'; import type { Got } from 'got'; import * as t from 'io-ts'; import { uniq } from 'lodash-es'; +import { isSombraError } from '../graphql'; import { PrivacyRequestInput } from './mapCsvRowsToRequestInputs'; import { ParsedAttributeInput } from './parseAttributesFromString'; @@ -125,11 +126,13 @@ export async function submitPrivacyRequest( }) .json(); } catch (error) { + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + throw new Error( `Received an error from server: ${ - // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing - (error as { response?: { body: string } }).response?.body || - (error as { message: string }).message + isSombraError(error) ? error.response.body : error.message }`, ); } diff --git a/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts b/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts index 1c1523f9..b101e35c 100644 --- a/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts +++ b/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts @@ -1,4 +1,6 @@ import { describe } from 'vitest'; // TODO: https://transcend.height.app/T-10772 - mapCsvRowsToRequestInputs test -describe.skip('mapCsvRowsToRequestInputs', () => {}); +describe.skip('mapCsvRowsToRequestInputs', () => { + return; +}); From 937c6e4e8ba3b13f1751a809ca98d4963b82990f Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Thu, 17 Jul 2025 01:00:59 -0400 Subject: [PATCH 12/18] more type safety --- src/lib/requests/filterRows.ts | 13 ++++++--- src/lib/requests/getUniqueValuesForColumn.ts | 5 ++-- src/lib/requests/mapRequestEnumValues.ts | 13 ++++++--- src/lib/requests/markSilentPrivacyRequests.ts | 10 ++++--- .../notifyPrivacyRequestsAdditionalTime.ts | 8 +++--- src/lib/requests/pullPrivacyRequests.ts | 4 ++- src/lib/requests/readCsv.ts | 5 ++-- .../removeUnverifiedRequestIdentifiers.ts | 6 +++-- src/lib/requests/restartPrivacyRequest.ts | 6 ++--- src/lib/requests/retryRequestDataSilos.ts | 15 ++++++++--- src/lib/requests/skipPreflightJobs.ts | 27 +++++++++++-------- src/lib/requests/skipRequestDataSilos.ts | 4 +-- 12 files changed, 74 insertions(+), 42 deletions(-) diff --git a/src/lib/requests/filterRows.ts b/src/lib/requests/filterRows.ts index ac289466..184be160 100644 --- a/src/lib/requests/filterRows.ts +++ b/src/lib/requests/filterRows.ts @@ -1,4 +1,3 @@ -import { ObjByString } from '@transcend-io/type-utils'; import colors from 'colors'; import inquirer from 'inquirer'; import { uniq } from 'lodash-es'; @@ -13,7 +12,9 @@ import { getUniqueValuesForColumn } from './getUniqueValuesForColumn'; * @param rows - Rows to filter * @returns Filtered rows */ -export async function filterRows(rows: ObjByString[]): Promise { +export async function filterRows( + rows: Record[], +): Promise[]> { // Determine set of column names const columnNames = uniq(rows.flatMap((x) => Object.keys(x))); @@ -32,7 +33,7 @@ export async function filterRows(rows: ObjByString[]): Promise { { name: 'filterColumnName', - message: `If you need to filter the list of requests to import, choose the column to filter on. Currently ${filteredRows.length} rows.`, + message: `If you need to filter the list of requests to import, choose the column to filter on. Currently ${filteredRows.length.toLocaleString()} rows.`, type: 'list', default: columnNames, choices: [NONE, ...columnNames], @@ -63,6 +64,10 @@ export async function filterRows(rows: ObjByString[]): Promise { } } - logger.info(colors.magenta(`Importing ${filteredRows.length} requests`)); + logger.info( + colors.magenta( + `Importing ${filteredRows.length.toLocaleString()} requests`, + ), + ); return filteredRows; } diff --git a/src/lib/requests/getUniqueValuesForColumn.ts b/src/lib/requests/getUniqueValuesForColumn.ts index 99527b67..55f197dc 100644 --- a/src/lib/requests/getUniqueValuesForColumn.ts +++ b/src/lib/requests/getUniqueValuesForColumn.ts @@ -1,4 +1,3 @@ -import { ObjByString } from '@transcend-io/type-utils'; import { uniq } from 'lodash-es'; /** @@ -9,8 +8,8 @@ import { uniq } from 'lodash-es'; * @returns Unique set of values in that column */ export function getUniqueValuesForColumn( - rows: ObjByString[], + rows: Record[], columnName: string, ): string[] { - return uniq(rows.flatMap((row) => row[columnName] || '')); + return uniq(rows.flatMap((row) => row[columnName] ?? '')); } diff --git a/src/lib/requests/mapRequestEnumValues.ts b/src/lib/requests/mapRequestEnumValues.ts index baf2ee61..21c3800e 100644 --- a/src/lib/requests/mapRequestEnumValues.ts +++ b/src/lib/requests/mapRequestEnumValues.ts @@ -6,7 +6,6 @@ import { IsoCountrySubdivisionCode, RequestAction, } from '@transcend-io/privacy-types'; -import { ObjByString } from '@transcend-io/type-utils'; import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; import { logger } from '../../logger'; @@ -25,7 +24,7 @@ import { mapEnumValues } from './mapEnumValues'; */ export async function mapRequestEnumValues( client: GraphQLClient, - requests: ObjByString[], + requests: Record[], { state, columnNameMap, @@ -37,8 +36,14 @@ export async function mapRequestEnumValues( }, ): Promise { // Get mapped value - const getMappedName = (attribute: ColumnName): string => - state.getValue('columnNames', attribute) || columnNameMap[attribute]!; + const getMappedName = (attribute: ColumnName): string => { + const value = + state.getValue('columnNames', attribute) ?? columnNameMap[attribute]; + if (value === undefined) { + throw new Error(`Column name ${attribute} is not mapped`); + } + return value; + }; // Fetch all data subjects in the organization const { internalSubjects } = await makeGraphQLRequest<{ diff --git a/src/lib/requests/markSilentPrivacyRequests.ts b/src/lib/requests/markSilentPrivacyRequests.ts index eb6dc7e7..ccbaa4c2 100644 --- a/src/lib/requests/markSilentPrivacyRequests.ts +++ b/src/lib/requests/markSilentPrivacyRequests.ts @@ -76,7 +76,9 @@ export async function markSilentPrivacyRequests({ // Notify Transcend logger.info( - colors.magenta(`Marking "${allRequests.length}" as silent mode.`), + colors.magenta( + `Marking "${allRequests.length.toLocaleString()}" as silent mode.`, + ), ); let total = 0; @@ -103,9 +105,11 @@ export async function markSilentPrivacyRequests({ logger.info( colors.green( - `Successfully marked ${total} requests as silent mode in "${ + `Successfully marked ${total.toLocaleString()} requests as silent mode in "${( totalTime / 1000 - }" seconds!`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds!`, ), ); return allRequests.length; diff --git a/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts b/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts index 9da01de9..835ff784 100644 --- a/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts +++ b/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts @@ -95,7 +95,7 @@ export async function notifyPrivacyRequestsAdditionalTime({ // Notify Transcend logger.info( colors.magenta( - `Notifying "${allRequests.length}" that more time is needed.`, + `Notifying "${allRequests.length.toLocaleString()}" that more time is needed.`, ), ); @@ -125,9 +125,11 @@ export async function notifyPrivacyRequestsAdditionalTime({ logger.info( colors.green( - `Successfully marked ${total} requests as silent mode in "${ + `Successfully marked ${total.toLocaleString()} requests as silent mode in "${( totalTime / 1000 - }" seconds!`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds!`, ), ); return allRequests.length; diff --git a/src/lib/requests/pullPrivacyRequests.ts b/src/lib/requests/pullPrivacyRequests.ts index 5ba70bbb..54b36e0e 100644 --- a/src/lib/requests/pullPrivacyRequests.ts +++ b/src/lib/requests/pullPrivacyRequests.ts @@ -120,7 +120,9 @@ export async function pullPrivacyRequests({ ); logger.info( - colors.magenta(`Pulled ${requestsWithRequestIdentifiers.length} requests`), + colors.magenta( + `Pulled ${requestsWithRequestIdentifiers.length.toLocaleString()} requests`, + ), ); // Write out to CSV diff --git a/src/lib/requests/readCsv.ts b/src/lib/requests/readCsv.ts index e949fe64..be0478f2 100644 --- a/src/lib/requests/readCsv.ts +++ b/src/lib/requests/readCsv.ts @@ -15,16 +15,17 @@ import * as t from 'io-ts'; export function readCsv( pathToFile: string, codec: T, - options: Options = { columns: true }, + options: Options = { columns: true } as Options, ): t.TypeOf[] { // read file contents and parse - const fileContent = parse(readFileSync(pathToFile, 'utf-8'), options); + const fileContent: unknown = parse(readFileSync(pathToFile, 'utf8'), options); // validate codec const data = decodeCodec(t.array(codec), fileContent); // remove any special characters from object keys const parsed = data.map((datum) => + // eslint-disable-next-line unicorn/no-array-reduce Object.entries(datum).reduce( (accumulator, [key, value]) => Object.assign(accumulator, { diff --git a/src/lib/requests/removeUnverifiedRequestIdentifiers.ts b/src/lib/requests/removeUnverifiedRequestIdentifiers.ts index 0d4effc4..40c20908 100644 --- a/src/lib/requests/removeUnverifiedRequestIdentifiers.ts +++ b/src/lib/requests/removeUnverifiedRequestIdentifiers.ts @@ -98,9 +98,11 @@ export async function removeUnverifiedRequestIdentifiers({ logger.info( colors.green( - `Successfully cleared out unverified identifiers "${ + `Successfully cleared out unverified identifiers "${( totalTime / 1000 - }" seconds for ${total} requests, ${processed} identifiers were cleared out!`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds for ${total.toLocaleString()} requests, ${processed.toLocaleString()} identifiers were cleared out!`, ), ); return allRequests.length; diff --git a/src/lib/requests/restartPrivacyRequest.ts b/src/lib/requests/restartPrivacyRequest.ts index 1e41f41c..8fd49ff8 100644 --- a/src/lib/requests/restartPrivacyRequest.ts +++ b/src/lib/requests/restartPrivacyRequest.ts @@ -57,9 +57,9 @@ export async function restartPrivacyRequest( ) .map((ri) => ({ ...ri, - type: Object.values(IdentifierType).includes( - ri.name as any, // eslint-disable-line @typescript-eslint/no-explicit-any - ) + type: ( + Object.values(IdentifierType) as string[] + ).includes(ri.name) ? ri.name : IdentifierType.Custom, })), diff --git a/src/lib/requests/retryRequestDataSilos.ts b/src/lib/requests/retryRequestDataSilos.ts index bdcb9525..2a92a5a7 100644 --- a/src/lib/requests/retryRequestDataSilos.ts +++ b/src/lib/requests/retryRequestDataSilos.ts @@ -56,7 +56,7 @@ export async function retryRequestDataSilos({ // Notify Transcend logger.info( colors.magenta( - `Retrying requests for Data Silo: "${dataSiloId}", restarting "${allRequests.length}" requests.`, + `Retrying requests for Data Silo: "${dataSiloId}", restarting "${allRequests.length.toLocaleString()}" requests.`, ), ); @@ -79,6 +79,10 @@ export async function retryRequestDataSilos({ requestDataSiloId: requestDataSilo.id, }); } catch (error) { + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + // some requests may not have this data silo connected if (!error.message.includes('Failed to find RequestDataSilo')) { throw error; @@ -98,9 +102,12 @@ export async function retryRequestDataSilos({ logger.info( colors.green( - `Successfully notified Transcend in "${ - totalTime / 1000 - }" seconds for ${total} requests, ${skipped} requests were skipped because data silo was not attached to the request!`, + `Successfully notified Transcend in "${(totalTime / 1000).toLocaleString( + undefined, + { + maximumFractionDigits: 2, + }, + )}" seconds for ${total.toLocaleString()} requests, ${skipped.toLocaleString()} requests were skipped because data silo was not attached to the request!`, ), ); return allRequests.length; diff --git a/src/lib/requests/skipPreflightJobs.ts b/src/lib/requests/skipPreflightJobs.ts index 893f6259..c9555b99 100644 --- a/src/lib/requests/skipPreflightJobs.ts +++ b/src/lib/requests/skipPreflightJobs.ts @@ -52,9 +52,7 @@ export async function skipPreflightJobs({ // Notify Transcend logger.info( colors.magenta( - `Processing enricher: "${enricherIds.join(',')}" fetched "${ - requests.length - }" in enriching status.`, + `Processing enricher: "${enricherIds.join(',')}" fetched "${requests.length.toLocaleString()}" in enriching status.`, ), ); @@ -77,11 +75,12 @@ export async function skipPreflightJobs({ const requestEnrichersFiltered = requestEnrichers.filter( (enricher) => enricherIds.includes(enricher.enricher.id) && - ![ - RequestEnricherStatus.Resolved, - RequestEnricherStatus.Skipped, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - ].includes(enricher.status as any), + !( + [ + RequestEnricherStatus.Resolved, + RequestEnricherStatus.Skipped, + ] as RequestEnricherStatus[] + ).includes(enricher.status), ); // FIXME @@ -96,6 +95,10 @@ export async function skipPreflightJobs({ }); totalSkipped += 1; } catch (error) { + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + if ( !error.message.includes( 'Client error: Cannot skip Request enricher because it has already completed', @@ -118,9 +121,11 @@ export async function skipPreflightJobs({ logger.info( colors.green( - `Successfully skipped "${totalSkipped}" for "${ - requests.length - }" requests in "${totalTime / 1000}" seconds!`, + `Successfully skipped "${totalSkipped.toLocaleString()}" for "${requests.length.toLocaleString()}" requests in "${( + totalTime / 1000 + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds!`, ), ); return requests.length; diff --git a/src/lib/requests/skipRequestDataSilos.ts b/src/lib/requests/skipRequestDataSilos.ts index f1981dcf..db643a21 100644 --- a/src/lib/requests/skipRequestDataSilos.ts +++ b/src/lib/requests/skipRequestDataSilos.ts @@ -97,9 +97,9 @@ export async function skipRequestDataSilos({ logger.info( colors.green( - `Successfully skipped "${requestDataSilos.length.toLocaleString()}" requests in "${ + `Successfully skipped "${requestDataSilos.length.toLocaleString()}" requests in "${( totalTime / 1000 - }" seconds!`, + ).toLocaleString(undefined, { maximumFractionDigits: 2 })}" seconds!`, ), ); return requestDataSilos.length; From b621cd294b50f1b7cf20f61e4cdeb6cd598e51c8 Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Thu, 17 Jul 2025 01:05:44 -0400 Subject: [PATCH 13/18] remove unnecessary type roots --- package.json | 2 +- pnpm-lock.yaml | 27 ++++++++++++++----- src/lib/@types/fuzzysearch.d.ts | 13 --------- .../@types/inquirer-autocomplete-prompt.d.ts | 3 --- tsconfig.json | 1 - 5 files changed, 22 insertions(+), 24 deletions(-) delete mode 100644 src/lib/@types/fuzzysearch.d.ts delete mode 100644 src/lib/@types/inquirer-autocomplete-prompt.d.ts diff --git a/package.json b/package.json index 4da00044..493a9bbc 100644 --- a/package.json +++ b/package.json @@ -140,7 +140,7 @@ "@types/fuzzysearch": "^1.0.0", "@types/global-agent": "^2.1.1", "@types/inquirer": "^7.3.1", - "@types/inquirer-autocomplete-prompt": "^3.0.0", + "@types/inquirer-autocomplete-prompt": "1.x", "@types/js-yaml": "^4.0.5", "@types/json-schema": "^7.0.15", "@types/JSONStream": "npm:@types/jsonstream@^0.8.33", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c728f4f3..577a1f72 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -136,8 +136,8 @@ importers: specifier: ^7.3.1 version: 7.3.3 '@types/inquirer-autocomplete-prompt': - specifier: ^3.0.0 - version: 3.0.3 + specifier: 1.x + version: 1.3.5 '@types/js-yaml': specifier: ^4.0.5 version: 4.0.9 @@ -809,12 +809,15 @@ packages: '@types/http-cache-semantics@4.0.4': resolution: {integrity: sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==} - '@types/inquirer-autocomplete-prompt@3.0.3': - resolution: {integrity: sha512-OQCW09mEECgvhcppbQRgZSmWskWv58l+WwyUvWB1oxTu3CZj8keYSDZR9U8owUzJ5Zeux5kacN9iVPJLXcoLXg==} + '@types/inquirer-autocomplete-prompt@1.3.5': + resolution: {integrity: sha512-xsydZ63gZt/2vqlqdSJQgxhbZd2NpRO6TawrDu1/IR6VbL3HfS709y6Yu7LwrkCcy4Lr05PeBInPizErcXSokw==} '@types/inquirer@7.3.3': resolution: {integrity: sha512-HhxyLejTHMfohAuhRun4csWigAMjXTmRyiJTU1Y/I1xmggikFMkOUoMQRlFm+zQcPEGHSs3io/0FAmNZf8EymQ==} + '@types/inquirer@8.2.11': + resolution: {integrity: sha512-15UboTvxb9SOaPG7CcXZ9dkv8lNqfiAwuh/5WxJDLjmElBt9tbx1/FDsEnJddUBKvN4mlPKvr8FyO1rAmBanzg==} + '@types/js-yaml@4.0.9': resolution: {integrity: sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==} @@ -2593,6 +2596,9 @@ packages: resolution: {integrity: sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ==} engines: {npm: '>=2.0.0'} + rxjs@7.8.2: + resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} + sade@1.8.1: resolution: {integrity: sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==} engines: {node: '>=6'} @@ -3679,15 +3685,20 @@ snapshots: '@types/http-cache-semantics@4.0.4': {} - '@types/inquirer-autocomplete-prompt@3.0.3': + '@types/inquirer-autocomplete-prompt@1.3.5': dependencies: - '@types/inquirer': 7.3.3 + '@types/inquirer': 8.2.11 '@types/inquirer@7.3.3': dependencies: '@types/through': 0.0.33 rxjs: 6.6.7 + '@types/inquirer@8.2.11': + dependencies: + '@types/through': 0.0.33 + rxjs: 7.8.2 + '@types/js-yaml@4.0.9': {} '@types/json-schema@7.0.15': {} @@ -5773,6 +5784,10 @@ snapshots: dependencies: tslib: 1.14.1 + rxjs@7.8.2: + dependencies: + tslib: 2.8.1 + sade@1.8.1: dependencies: mri: 1.2.0 diff --git a/src/lib/@types/fuzzysearch.d.ts b/src/lib/@types/fuzzysearch.d.ts deleted file mode 100644 index 77b581e8..00000000 --- a/src/lib/@types/fuzzysearch.d.ts +++ /dev/null @@ -1,13 +0,0 @@ -declare module 'fuzzysearch' { - /** - * Fuzzy search - * - * @param needle - Needle to search - * @param haystack - Hay to search through - * @returns True if matching - */ - export default function fuzzysearch( - needle: string, - haystack: string, - ): boolean; -} diff --git a/src/lib/@types/inquirer-autocomplete-prompt.d.ts b/src/lib/@types/inquirer-autocomplete-prompt.d.ts deleted file mode 100644 index 0c0db0d3..00000000 --- a/src/lib/@types/inquirer-autocomplete-prompt.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -declare module 'inquirer-autocomplete-prompt' { - export default import('inquirer').PromptConstructor; -} diff --git a/tsconfig.json b/tsconfig.json index d241dbb0..929c7cca 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -24,7 +24,6 @@ "noFallthroughCasesInSwitch": true, "useUnknownInCatchVariables": false, - "typeRoots": ["src/lib/@types", "node_modules/@types"], "baseUrl": ".", "checkJs": true }, From 8e854a6dc4a04a081572c940dcf0f3258ed14e5f Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Thu, 17 Jul 2025 01:16:53 -0400 Subject: [PATCH 14/18] more --- .../impl.ts | 2 +- src/commands/inventory/scan-packages/impl.ts | 2 +- src/lib/ai/getGitFilesThatChanged.ts | 8 +++--- src/lib/api-keys/validateTranscendAuth.ts | 2 +- .../code-scanning/integrations/cocoaPods.ts | 2 +- .../integrations/composerJson.ts | 2 +- src/lib/code-scanning/integrations/gemfile.ts | 6 ++-- src/lib/code-scanning/integrations/gradle.ts | 2 +- .../integrations/javascriptPackageJson.ts | 2 +- src/lib/code-scanning/integrations/pubspec.ts | 2 +- .../integrations/pythonRequirementsTxt.ts | 12 ++++---- src/lib/code-scanning/integrations/swift.ts | 2 +- .../syncOneTrustAssessmentsFromFile.ts | 2 +- src/lib/readTranscendYaml.ts | 10 +++---- src/lib/requests/mapCsvRowsToRequestInputs.ts | 28 +++++++++++-------- 15 files changed, 45 insertions(+), 39 deletions(-) diff --git a/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts b/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts index 28ab8b64..d846f548 100644 --- a/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts +++ b/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts @@ -33,7 +33,7 @@ export function consentManagerServiceJsonToYml( // Read in each consent manager configuration const services = decodeCodec( t.array(ConsentManagerServiceMetadata), - readFileSync(file, 'utf-8'), + readFileSync(file, 'utf8'), ); // Create data flows and cookie configurations diff --git a/src/commands/inventory/scan-packages/impl.ts b/src/commands/inventory/scan-packages/impl.ts index 6b6abe12..d036821c 100644 --- a/src/commands/inventory/scan-packages/impl.ts +++ b/src/commands/inventory/scan-packages/impl.ts @@ -40,7 +40,7 @@ export async function scanPackages( `cd ${scanPath} && git config --get remote.origin.url`, ); // Trim and parse the URL - const url = name.toString('utf-8').trim(); + const url = name.toString('utf8').trim(); [gitRepositoryName] = url.includes('https:') ? url.split('/').slice(3).join('/').split('.') : (url.split(':').pop() || '').split('.'); diff --git a/src/lib/ai/getGitFilesThatChanged.ts b/src/lib/ai/getGitFilesThatChanged.ts index 7ccd2034..bd2a4f5d 100644 --- a/src/lib/ai/getGitFilesThatChanged.ts +++ b/src/lib/ai/getGitFilesThatChanged.ts @@ -42,12 +42,12 @@ export function getGitFilesThatChanged({ // Latest commit on base branch. If we are on the base branch, we take the prior commit const latestBasedCommit = execSync( `git ls-remote ${githubRepo} "refs/heads/${baseBranch}" | cut -f 1`, - { encoding: 'utf-8' }, + { encoding: 'utf8' }, ).split('\n')[0]; // This commit const latestThisCommit = execSync('git rev-parse HEAD', { - encoding: 'utf-8', + encoding: 'utf8', }).split('\n')[0]; // Ensure commits are present @@ -60,7 +60,7 @@ export function getGitFilesThatChanged({ `git fetch && git diff --name-only "${ baseBranch || latestBasedCommit }...${latestThisCommit}" -- ${rootDirectory}`, - { encoding: 'utf-8' }, + { encoding: 'utf8' }, ); // Filter out block list @@ -79,7 +79,7 @@ export function getGitFilesThatChanged({ const fileDiffs: Record = {}; for (const file of filteredChanges) { const contents = execSync(`git show ${latestThisCommit}:${file}`, { - encoding: 'utf-8', + encoding: 'utf8', }); fileDiffs[file] = contents; } diff --git a/src/lib/api-keys/validateTranscendAuth.ts b/src/lib/api-keys/validateTranscendAuth.ts index 7d69ba4a..9e7a240b 100644 --- a/src/lib/api-keys/validateTranscendAuth.ts +++ b/src/lib/api-keys/validateTranscendAuth.ts @@ -26,7 +26,7 @@ export function validateTranscendAuth(auth: string): string | StoredApiKey[] { // Read from disk if (existsSync(auth)) { // validate that file is a list of API keys - return decodeCodec(t.array(StoredApiKey), readFileSync(auth, 'utf-8')); + return decodeCodec(t.array(StoredApiKey), readFileSync(auth, 'utf8')); } // Return as single API key diff --git a/src/lib/code-scanning/integrations/cocoaPods.ts b/src/lib/code-scanning/integrations/cocoaPods.ts index 30dfebc5..0b6a9109 100644 --- a/src/lib/code-scanning/integrations/cocoaPods.ts +++ b/src/lib/code-scanning/integrations/cocoaPods.ts @@ -11,7 +11,7 @@ export const cocoaPods: CodeScanningConfig = { supportedFiles: ['Podfile'], ignoreDirs: ['Pods'], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, 'utf-8'); + const fileContents = readFileSync(filePath, 'utf8'); const targets = findAllWithRegex( { diff --git a/src/lib/code-scanning/integrations/composerJson.ts b/src/lib/code-scanning/integrations/composerJson.ts index 55a0b730..ff8ede44 100644 --- a/src/lib/code-scanning/integrations/composerJson.ts +++ b/src/lib/code-scanning/integrations/composerJson.ts @@ -7,7 +7,7 @@ export const composerJson: CodeScanningConfig = { supportedFiles: ['composer.json'], ignoreDirs: ['vendor', 'node_modules', 'cache', 'build', 'dist'], scanFunction: (filePath) => { - const file = readFileSync(filePath, 'utf-8'); + const file = readFileSync(filePath, 'utf8'); const directory = path.dirname(filePath); const asJson = JSON.parse(file); const { diff --git a/src/lib/code-scanning/integrations/gemfile.ts b/src/lib/code-scanning/integrations/gemfile.ts index 2f75dadb..fd171ae5 100644 --- a/src/lib/code-scanning/integrations/gemfile.ts +++ b/src/lib/code-scanning/integrations/gemfile.ts @@ -15,15 +15,13 @@ export const gemfile: CodeScanningConfig = { supportedFiles: ['Gemfile'], ignoreDirs: ['bin'], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, 'utf-8'); + const fileContents = readFileSync(filePath, 'utf8'); const directory = path.dirname(filePath); const filesInFolder = listFiles(directory); // parse gemspec file for name const gemspec = filesInFolder.find((file) => file === '.gemspec'); - const gemspecContents = gemspec - ? readFileSync(gemspec, 'utf-8') - : undefined; + const gemspecContents = gemspec ? readFileSync(gemspec, 'utf8') : undefined; const gemfileName = gemspecContents ? (GEMFILE_PACKAGE_NAME_REGEX.exec(gemspecContents) || [])[2] : undefined; diff --git a/src/lib/code-scanning/integrations/gradle.ts b/src/lib/code-scanning/integrations/gradle.ts index 113dda44..341dde17 100644 --- a/src/lib/code-scanning/integrations/gradle.ts +++ b/src/lib/code-scanning/integrations/gradle.ts @@ -28,7 +28,7 @@ export const gradle: CodeScanningConfig = { 'gradle-wrapper.properties', ], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, 'utf-8'); + const fileContents = readFileSync(filePath, 'utf8'); const directory = path.dirname(filePath); const targets = findAllWithRegex( diff --git a/src/lib/code-scanning/integrations/javascriptPackageJson.ts b/src/lib/code-scanning/integrations/javascriptPackageJson.ts index 8eb8aabc..e01b2a8a 100644 --- a/src/lib/code-scanning/integrations/javascriptPackageJson.ts +++ b/src/lib/code-scanning/integrations/javascriptPackageJson.ts @@ -7,7 +7,7 @@ export const javascriptPackageJson: CodeScanningConfig = { supportedFiles: ['package.json'], ignoreDirs: ['node_modules', 'serverless-build', 'lambda-build'], scanFunction: (filePath) => { - const file = readFileSync(filePath, 'utf-8'); + const file = readFileSync(filePath, 'utf8'); const directory = path.dirname(filePath); const asJson = JSON.parse(file); const { diff --git a/src/lib/code-scanning/integrations/pubspec.ts b/src/lib/code-scanning/integrations/pubspec.ts index 4310ba8a..ef4eaf89 100644 --- a/src/lib/code-scanning/integrations/pubspec.ts +++ b/src/lib/code-scanning/integrations/pubspec.ts @@ -34,7 +34,7 @@ export const pubspec: CodeScanningConfig = { ignoreDirs: ['build'], scanFunction: (filePath) => { const directory = path.dirname(filePath); - const fileContents = readFileSync(filePath, 'utf-8'); + const fileContents = readFileSync(filePath, 'utf8'); const { name, description, diff --git a/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts b/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts index 2d6177b5..8e01c59a 100644 --- a/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts +++ b/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts @@ -13,20 +13,20 @@ export const pythonRequirementsTxt: CodeScanningConfig = { supportedFiles: ['requirements.txt'], ignoreDirs: ['build', 'lib', 'lib64'], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, 'utf-8'); + const fileContents = readFileSync(filePath, 'utf8'); const directory = path.dirname(filePath); const filesInFolder = listFiles(directory); // parse setup file for name const setupFile = filesInFolder.find((file) => file === 'setup.py'); const setupFileContents = setupFile - ? readFileSync(path.join(directory, setupFile), 'utf-8') + ? readFileSync(path.join(directory, setupFile), 'utf8') : undefined; const packageName = setupFileContents - ? (PACKAGE_NAME.exec(setupFileContents) || [])[2] + ? (PACKAGE_NAME.exec(setupFileContents) ?? [])[2] : undefined; const packageDescription = setupFileContents - ? (PACKAGE_DESCRIPTION.exec(setupFileContents) || [])[2] + ? (PACKAGE_DESCRIPTION.exec(setupFileContents) ?? [])[2] : undefined; const targets = findAllWithRegex( @@ -39,7 +39,9 @@ export const pythonRequirementsTxt: CodeScanningConfig = { return [ { - name: packageName || directory.split('/').pop()!, + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + name: packageName || path.basename(directory), + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing description: packageDescription || undefined, type: CodePackageType.RequirementsTxt, softwareDevelopmentKits: targets.map((package_) => ({ diff --git a/src/lib/code-scanning/integrations/swift.ts b/src/lib/code-scanning/integrations/swift.ts index 740a9859..9379cd30 100644 --- a/src/lib/code-scanning/integrations/swift.ts +++ b/src/lib/code-scanning/integrations/swift.ts @@ -24,7 +24,7 @@ export const swift: CodeScanningConfig = { supportedFiles: ['Package.resolved'], ignoreDirs: [], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, 'utf-8'); + const fileContents = readFileSync(filePath, 'utf8'); const parsed = decodeCodec(SwiftPackage, fileContents); diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts index e2d3ae23..1b649fdb 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts @@ -26,7 +26,7 @@ export const syncOneTrustAssessmentsFromFile = ({ return new Promise((resolve, reject) => { // Create a readable stream from the file const fileStream = createReadStream(file, { - encoding: 'utf-8', + encoding: 'utf8', highWaterMark: 64 * 1024, // 64KB chunks }); diff --git a/src/lib/readTranscendYaml.ts b/src/lib/readTranscendYaml.ts index ea8e645e..d4141e24 100644 --- a/src/lib/readTranscendYaml.ts +++ b/src/lib/readTranscendYaml.ts @@ -1,5 +1,5 @@ import { readFileSync, writeFileSync } from 'node:fs'; -import { decodeCodec, ObjByString } from '@transcend-io/type-utils'; +import { decodeCodec } from '@transcend-io/type-utils'; import yaml from 'js-yaml'; import { TranscendInput } from '../codecs'; @@ -17,7 +17,7 @@ export const VARIABLE_PARAMETERS_NAME = 'parameters'; */ export function replaceVariablesInYaml( input: string, - variables: ObjByString, + variables: Record, extraErrorMessage = '', ): string { let contents = input; @@ -30,7 +30,7 @@ export function replaceVariablesInYaml( // Throw error if unfilled variables if (VARIABLE_PARAMETERS_REGEXP.test(contents)) { - const [, name] = VARIABLE_PARAMETERS_REGEXP.exec(contents) || []; + const [, name] = VARIABLE_PARAMETERS_REGEXP.exec(contents) ?? []; throw new Error( `Found variable that was not set: ${name}. Make sure you are passing all parameters through the --${VARIABLE_PARAMETERS_NAME}=${name}:value-for-param flag. @@ -51,10 +51,10 @@ ${extraErrorMessage}`, */ export function readTranscendYaml( filePath: string, - variables: ObjByString = {}, + variables: Record = {}, ): TranscendInput { // Read in contents - const fileContents = readFileSync(filePath, 'utf-8'); + const fileContents = readFileSync(filePath, 'utf8'); // Replace variables const replacedVariables = replaceVariablesInYaml( diff --git a/src/lib/requests/mapCsvRowsToRequestInputs.ts b/src/lib/requests/mapCsvRowsToRequestInputs.ts index da8ebefa..a9837dc6 100644 --- a/src/lib/requests/mapCsvRowsToRequestInputs.ts +++ b/src/lib/requests/mapCsvRowsToRequestInputs.ts @@ -8,7 +8,7 @@ import { NORMALIZE_PHONE_NUMBER, RequestAction, } from '@transcend-io/privacy-types'; -import { ObjByString, valuesOf } from '@transcend-io/type-utils'; +import { valuesOf } from '@transcend-io/type-utils'; import * as t from 'io-ts'; import { DateFromISOString } from 'io-ts-types'; import { AttributeKey } from '../graphql'; @@ -132,7 +132,7 @@ export function normalizeIdentifierValue( * @returns [raw input, request input] list */ export function mapCsvRowsToRequestInputs( - requestInputs: ObjByString[], + requestInputs: Record[], state: PersistedState, { columnNameMap, @@ -154,8 +154,16 @@ export function mapCsvRowsToRequestInputs( }, ): [Record, PrivacyRequestInput][] { // map the CSV to request input - const getMappedName = (attribute: ColumnName): string => - state.getValue('columnNames', attribute) || columnNameMap[attribute]!; + // Get mapped value + const getMappedName = (attribute: ColumnName): string => { + const value = + state.getValue('columnNames', attribute) ?? columnNameMap[attribute]; + if (value === undefined) { + throw new Error(`Column name ${attribute} is not mapped`); + } + return value; + }; + return requestInputs.map( (input): [Record, PrivacyRequestInput] => { // The extra identifiers to upload for this request @@ -166,9 +174,9 @@ export function mapCsvRowsToRequestInputs( // filter out skipped identifiers .filter(([, columnName]) => columnName !== NONE)) { // Determine the identifier type being specified - const identifierType = Object.values(IdentifierType).includes( - identifierName as any, // eslint-disable-line @typescript-eslint/no-explicit-any - ) + const identifierType = ( + Object.values(IdentifierType) as string[] + ).includes(identifierName) ? (identifierName as IdentifierType) : IdentifierType.Custom; @@ -182,9 +190,7 @@ export function mapCsvRowsToRequestInputs( ); if (normalized) { // Initialize - if (!attestedExtraIdentifiers[identifierType]) { - attestedExtraIdentifiers[identifierType] = []; - } + attestedExtraIdentifiers[identifierType] ??= []; // Add the identifier attestedExtraIdentifiers[identifierType].push({ @@ -211,7 +217,7 @@ export function mapCsvRowsToRequestInputs( attributes.push({ values: isMulti ? splitCsvToList(attributeValueString) - : attributeValueString, + : [attributeValueString], key: attributeName, }); } From c6aa082ae45af897da02c5d89ff154fd02425c08 Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Thu, 17 Jul 2025 01:34:09 -0400 Subject: [PATCH 15/18] more --- src/lib/consent-manager/uploadConsents.ts | 41 +++++++++++------ .../createTranscendConsentGotInstance.ts | 14 +++++- src/lib/requests/approvePrivacyRequests.ts | 21 +++++++-- src/lib/requests/bulkRestartRequests.ts | 46 ++++++++++--------- src/lib/requests/bulkRetryEnrichers.ts | 19 ++++---- src/lib/requests/cancelPrivacyRequests.ts | 8 ++-- .../requests/downloadPrivacyRequestFiles.ts | 10 ++-- src/lib/requests/extractClientError.ts | 4 +- .../getFileMetadataForPrivacyRequests.ts | 18 ++++++-- 9 files changed, 117 insertions(+), 64 deletions(-) diff --git a/src/lib/consent-manager/uploadConsents.ts b/src/lib/consent-manager/uploadConsents.ts index c682baca..4845103c 100644 --- a/src/lib/consent-manager/uploadConsents.ts +++ b/src/lib/consent-manager/uploadConsents.ts @@ -6,7 +6,10 @@ import * as t from 'io-ts'; import { DEFAULT_TRANSCEND_CONSENT_API } from '../../constants'; import { logger } from '../../logger'; import { map } from '../bluebird-replace'; -import { createTranscendConsentGotInstance } from '../graphql'; +import { + createTranscendConsentGotInstance, + isTranscendConsentError, +} from '../graphql'; import { createConsentToken } from './createConsentToken'; import type { ConsentPreferenceUpload } from './types'; @@ -100,7 +103,7 @@ export async function uploadConsents({ logger.info( colors.magenta( - `Uploading ${preferences.length} user preferences to partition ${partition}`, + `Uploading ${preferences.length.toLocaleString()} user preferences to partition ${partition}`, ), ); @@ -133,7 +136,7 @@ export async function uploadConsents({ // parse usp string const [, saleStatus] = consent.usp - ? USP_STRING_REGEX.exec(consent.usp) || [] + ? (USP_STRING_REGEX.exec(consent.usp) ?? []) : []; const input = { @@ -160,17 +163,27 @@ export async function uploadConsents({ }) .json(); } catch (error) { - try { - const parsed = JSON.parse(error?.response?.body || '{}'); - if (parsed.error) { - logger.error(colors.red(`Error: ${parsed.error}`)); + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + + if (isTranscendConsentError(error)) { + try { + const parsed = JSON.parse(error.response.body) as Record< + string, + unknown + >; + if ('error' in parsed && typeof parsed.error === 'string') { + logger.error(colors.red(`Error: ${parsed.error}`)); + } + } catch { + // continue } - } catch { - // continue } + throw new Error( `Received an error from server: ${ - error?.response?.body || error?.message + isTranscendConsentError(error) ? error.response.body : error.message }`, ); } @@ -187,11 +200,11 @@ export async function uploadConsents({ logger.info( colors.green( - `Successfully uploaded ${ - preferences.length - } user preferences to partition ${partition} in "${ + `Successfully uploaded ${preferences.length.toLocaleString()} user preferences to partition ${partition} in "${( totalTime / 1000 - }" seconds!`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds!`, ), ); } diff --git a/src/lib/graphql/createTranscendConsentGotInstance.ts b/src/lib/graphql/createTranscendConsentGotInstance.ts index 8fde4f07..8f27c262 100644 --- a/src/lib/graphql/createTranscendConsentGotInstance.ts +++ b/src/lib/graphql/createTranscendConsentGotInstance.ts @@ -1,4 +1,16 @@ -import got, { Got } from 'got'; +import got, { Got, RequestError, type Response } from 'got'; + +interface TranscendConsentError extends Omit { + response: Response; +} + +export function isTranscendConsentError( + error: unknown, +): error is TranscendConsentError { + return ( + error instanceof RequestError && typeof error.response?.body === 'string' + ); +} /** * Instantiate an instance of got that is capable of making requests diff --git a/src/lib/requests/approvePrivacyRequests.ts b/src/lib/requests/approvePrivacyRequests.ts index dd331f2e..3bc0904e 100644 --- a/src/lib/requests/approvePrivacyRequests.ts +++ b/src/lib/requests/approvePrivacyRequests.ts @@ -70,7 +70,11 @@ export async function approvePrivacyRequests({ }); // Notify Transcend - logger.info(colors.magenta(`Approving "${allRequests.length}" requests.`)); + logger.info( + colors.magenta( + `Approving "${allRequests.length.toLocaleString()}" requests.`, + ), + ); let total = 0; let skipped = 0; @@ -98,7 +102,10 @@ export async function approvePrivacyRequests({ input: { requestId: requestToApprove.id }, }); } catch (error) { - if (error.message.includes('Request must be in an approving state,')) { + if ( + error instanceof Error && + error.message.includes('Request must be in an approving state,') + ) { skipped += 1; } } @@ -113,13 +120,17 @@ export async function approvePrivacyRequests({ const t1 = Date.now(); const totalTime = t1 - t0; if (skipped > 0) { - logger.info(colors.yellow(`${skipped} requests were skipped.`)); + logger.info( + colors.yellow(`${skipped.toLocaleString()} requests were skipped.`), + ); } logger.info( colors.green( - `Successfully approved ${total} requests in "${ + `Successfully approved ${total.toLocaleString()} requests in "${( totalTime / 1000 - }" seconds!`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds!`, ), ); return allRequests.length; diff --git a/src/lib/requests/bulkRestartRequests.ts b/src/lib/requests/bulkRestartRequests.ts index 3590d271..fd2ac075 100644 --- a/src/lib/requests/bulkRestartRequests.ts +++ b/src/lib/requests/bulkRestartRequests.ts @@ -14,6 +14,7 @@ import { fetchAllRequestIdentifiers, fetchAllRequests, } from '../graphql'; +import { isSombraError } from '../graphql/createSombraGotInstance'; import { SuccessfulRequest } from './constants'; import { extractClientError } from './extractClientError'; import { restartPrivacyRequest } from './restartPrivacyRequest'; @@ -124,7 +125,7 @@ export async function bulkRestartRequests({ const requests = allRequests.filter( (request) => new Date(request.createdAt) < createdAt, ); - logger.info(`Found ${requests.length} requests to process`); + logger.info(`Found ${requests.length.toLocaleString()} requests to process`); if (copyIdentifiers) { logger.info('copyIdentifiers detected - All Identifiers will be copied.'); @@ -143,14 +144,11 @@ export async function bulkRestartRequests({ requests.map(({ id }) => id), ); if (missingRequests.length > 0) { - logger.error( - colors.red( - `Failed to find the following requests by ID: ${missingRequests.join( - ',', - )}.`, - ), + throw new Error( + `Failed to find the following requests by ID: ${missingRequests.join( + ',', + )}.`, ); - process.exit(1); } } @@ -199,11 +197,15 @@ export async function bulkRestartRequests({ }); await state.setValue(restartedRequests, 'restartedRequests'); } catch (error) { - const message = `${error.message} - ${JSON.stringify( - error.response?.body, - null, - 2, - )}`; + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + + const message = `${error.message} - ${ + isSombraError(error) + ? JSON.stringify(error.response.body, null, 2) + : '' + }`; const clientError = extractClientError(message); const failingRequests = state.getValue('failingRequests'); @@ -213,7 +215,7 @@ export async function bulkRestartRequests({ rowIndex: ind, coreIdentifier: request.coreIdentifier, attemptedAt: new Date().toISOString(), - error: clientError || message, + error: clientError ?? message, }); await state.setValue(failingRequests, 'failingRequests'); } @@ -230,18 +232,20 @@ export async function bulkRestartRequests({ // Log completion time logger.info( colors.green( - `Completed restarting of requests in "${totalTime / 1000}" seconds.`, + `Completed restarting of requests in "${(totalTime / 1000).toLocaleString( + undefined, + { + maximumFractionDigits: 2, + }, + )}" seconds.`, ), ); // Log errors if (state.getValue('failingRequests').length > 0) { - logger.error( - colors.red( - `Encountered "${state.getValue('failingRequests').length}" errors. ` + - `See "${cacheFile}" to review the error messages and inputs.`, - ), + throw new Error( + `Encountered "${state.getValue('failingRequests').length.toLocaleString()}" errors. ` + + `See "${cacheFile}" to review the error messages and inputs.`, ); - process.exit(1); } } diff --git a/src/lib/requests/bulkRetryEnrichers.ts b/src/lib/requests/bulkRetryEnrichers.ts index d854fb35..92bc34ba 100644 --- a/src/lib/requests/bulkRetryEnrichers.ts +++ b/src/lib/requests/bulkRetryEnrichers.ts @@ -81,14 +81,11 @@ export async function bulkRetryEnrichers({ requests.map(({ id }) => id), ); if (missingRequests.length > 0) { - logger.error( - colors.red( - `Failed to find the following requests by ID: ${missingRequests.join( - ',', - )}.`, - ), + throw new Error( + `Failed to find the following requests by ID: ${missingRequests.join( + ',', + )}.`, ); - process.exit(1); } } @@ -126,11 +123,11 @@ export async function bulkRetryEnrichers({ // Log completion time logger.info( colors.green( - `Completed restarting of ${ - requests.length - } requests and ${totalRestarted} enrichers in "${ + `Completed restarting of ${requests.length.toLocaleString()} requests and ${totalRestarted.toLocaleString()} enrichers in "${( totalTime / 1000 - }" seconds.`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds.`, ), ); } diff --git a/src/lib/requests/cancelPrivacyRequests.ts b/src/lib/requests/cancelPrivacyRequests.ts index 815ff5d5..3525196e 100644 --- a/src/lib/requests/cancelPrivacyRequests.ts +++ b/src/lib/requests/cancelPrivacyRequests.ts @@ -103,7 +103,7 @@ export async function cancelPrivacyRequests({ // Notify Transcend logger.info( colors.magenta( - `Canceling "${allRequests.length}" requests${ + `Canceling "${allRequests.length.toLocaleString()}" requests${ cancelationTemplate ? ` Using template: ${cancelationTemplate.title}` : '' @@ -155,9 +155,11 @@ export async function cancelPrivacyRequests({ logger.info( colors.green( - `Successfully canceled ${total} requests in "${ + `Successfully canceled ${total.toLocaleString()} requests in "${( totalTime / 1000 - }" seconds!`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds!`, ), ); return allRequests.length; diff --git a/src/lib/requests/downloadPrivacyRequestFiles.ts b/src/lib/requests/downloadPrivacyRequestFiles.ts index 9632c657..f9d4df9b 100644 --- a/src/lib/requests/downloadPrivacyRequestFiles.ts +++ b/src/lib/requests/downloadPrivacyRequestFiles.ts @@ -143,14 +143,18 @@ export async function downloadPrivacyRequestFiles({ logger.info( colors.green( - `Successfully downloaded ${total} requests in "${ + `Successfully downloaded ${total.toLocaleString()} requests in "${( totalTime / 1000 - }" seconds!`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds!`, ), ); if (totalApproved > 0) { logger.info( - colors.green(`Approved ${totalApproved} requests in Transcend.`), + colors.green( + `Approved ${totalApproved.toLocaleString()} requests in Transcend.`, + ), ); } return allRequests.length; diff --git a/src/lib/requests/extractClientError.ts b/src/lib/requests/extractClientError.ts index 41eb2e58..c1500730 100644 --- a/src/lib/requests/extractClientError.ts +++ b/src/lib/requests/extractClientError.ts @@ -7,5 +7,7 @@ const CLIENT_ERROR = /{\\"message\\":\\"(.+?)\\",/; * @returns Client error or null */ export function extractClientError(error: string): string | null { - return CLIENT_ERROR.test(error) ? CLIENT_ERROR.exec(error)![1] : null; + return CLIENT_ERROR.test(error) + ? (CLIENT_ERROR.exec(error)?.[1] ?? null) + : null; } diff --git a/src/lib/requests/getFileMetadataForPrivacyRequests.ts b/src/lib/requests/getFileMetadataForPrivacyRequests.ts index 0c856642..0fab1960 100644 --- a/src/lib/requests/getFileMetadataForPrivacyRequests.ts +++ b/src/lib/requests/getFileMetadataForPrivacyRequests.ts @@ -6,7 +6,7 @@ import type { Got } from 'got'; import * as t from 'io-ts'; import { logger } from '../../logger'; import { map } from '../bluebird-replace'; -import { PrivacyRequest } from '../graphql'; +import { isSombraError, PrivacyRequest } from '../graphql'; export const IntlMessage = t.type({ /** The message key */ @@ -108,7 +108,9 @@ export async function getFileMetadataForPrivacyRequests( }, ): Promise<[Pick, RequestFileMetadata[]][]> { logger.info( - colors.magenta(`Pulling file metadata for ${requests.length} requests`), + colors.magenta( + `Pulling file metadata for ${requests.length.toLocaleString()} requests`, + ), ); // Time duration @@ -160,9 +162,13 @@ export async function getFileMetadataForPrivacyRequests( shouldContinue = !!response._links.next && response.nodes.length === limit; } catch (error) { + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + throw new Error( `Received an error from server: ${ - error?.response?.body || error?.message + isSombraError(error) ? error.response.body : error.message }`, ); } @@ -181,9 +187,11 @@ export async function getFileMetadataForPrivacyRequests( logger.info( colors.green( - `Successfully downloaded file metadata ${requests.length} requests in "${ + `Successfully downloaded file metadata ${requests.length.toLocaleString()} requests in "${( totalTime / 1000 - }" seconds!`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds!`, ), ); From 707db2f739d27893305cbf3bae66066e0e9daffc Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Thu, 17 Jul 2025 01:36:30 -0400 Subject: [PATCH 16/18] fix --- .../requests/uploadPrivacyRequestsFromCsv.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/lib/requests/uploadPrivacyRequestsFromCsv.ts b/src/lib/requests/uploadPrivacyRequestsFromCsv.ts index ed403c09..a79094f8 100644 --- a/src/lib/requests/uploadPrivacyRequestsFromCsv.ts +++ b/src/lib/requests/uploadPrivacyRequestsFromCsv.ts @@ -11,6 +11,7 @@ import { buildTranscendGraphQLClient, createSombraGotInstance, fetchAllRequestAttributeKeys, + isSombraError, } from '../graphql'; import { CachedFileState, CachedRequestState } from './constants'; import { extractClientError } from './extractClientError'; @@ -257,11 +258,14 @@ export async function uploadPrivacyRequestsFromCsv({ }); await requestState.setValue(successfulRequests, 'successfulRequests'); } catch (error) { - const message = `${(error as { message: string }).message} - ${JSON.stringify( - (error as { response?: { body: string } }).response?.body, - null, - 2, - )}`; + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + const message = `${error.message} - ${ + isSombraError(error) + ? JSON.stringify(error.response.body, null, 2) + : '' + }`; const clientError = extractClientError(message); if ( @@ -320,7 +324,9 @@ export async function uploadPrivacyRequestsFromCsv({ // Log completion time logger.info( colors.green( - `Completed upload in "${(totalTime / 1000).toLocaleString()}" seconds.`, + `Completed upload in "${(totalTime / 1000).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds.`, ), ); From 10f4869586a5e7e853e5c652467852cfba6fe575 Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Thu, 17 Jul 2025 01:43:16 -0400 Subject: [PATCH 17/18] pref --- ...ferenceManagementPreferencesInteractive.ts | 85 ++++++++++--------- 1 file changed, 47 insertions(+), 38 deletions(-) diff --git a/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts b/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts index 0f592e5e..b7270194 100644 --- a/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts +++ b/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts @@ -12,6 +12,7 @@ import { createSombraGotInstance, fetchAllPreferenceTopics, fetchAllPurposes, + isSombraError, PreferenceTopic, Purpose, } from '../graphql'; @@ -87,12 +88,12 @@ export async function uploadPreferenceManagementPreferencesInteractive({ logger.info( colors.magenta( 'Restored cache, there are: \n' + - `${ - Object.values(failingRequests).length - } failing requests to be retried\n` + - `${ - Object.values(pendingRequests).length - } pending requests to be processed\n` + + `${Object.values( + failingRequests, + ).length.toLocaleString()} failing requests to be retried\n` + + `${Object.values( + pendingRequests, + ).length.toLocaleString()} pending requests to be processed\n` + `The following files are stored in cache and will be used:\n${Object.keys( fileMetadata, ) @@ -138,23 +139,23 @@ export async function uploadPreferenceManagementPreferencesInteractive({ logger.info( colors.magenta( - `Found ${ - Object.entries(metadata.pendingSafeUpdates).length - } safe updates in ${file}`, + `Found ${Object.entries( + metadata.pendingSafeUpdates, + ).length.toLocaleString()} safe updates in ${file}`, ), ); logger.info( colors.magenta( - `Found ${ - Object.entries(metadata.pendingConflictUpdates).length - } conflict updates in ${file}`, + `Found ${Object.entries( + metadata.pendingConflictUpdates, + ).length.toLocaleString()} conflict updates in ${file}`, ), ); logger.info( colors.magenta( - `Found ${ - Object.entries(metadata.skippedUpdates).length - } skipped updates in ${file}`, + `Found ${Object.entries( + metadata.skippedUpdates, + ).length.toLocaleString()} skipped updates in ${file}`, ), ); @@ -167,9 +168,10 @@ export async function uploadPreferenceManagementPreferencesInteractive({ })) { // Determine timestamp const timestamp = - metadata.timestampColum === NONE_PREFERENCE_MAP + metadata.timestampColum === NONE_PREFERENCE_MAP || + metadata.timestampColum === undefined ? new Date() - : new Date(update[metadata.timestampColum!]); + : new Date(update[metadata.timestampColum]); // Determine updates const updates = getPreferenceUpdatesFromRow({ @@ -200,9 +202,9 @@ export async function uploadPreferenceManagementPreferencesInteractive({ if (dryRun) { logger.info( colors.green( - `Dry run complete, exiting. ${ - Object.values(pendingUpdates).length - } pending updates. Check file: ${receiptFilepath}`, + `Dry run complete, exiting. ${Object.values( + pendingUpdates, + ).length.toLocaleString()} pending updates. Check file: ${receiptFilepath}`, ), ); return; @@ -210,9 +212,9 @@ export async function uploadPreferenceManagementPreferencesInteractive({ logger.info( colors.magenta( - `Uploading ${ - Object.values(pendingUpdates).length - } preferences to partition: ${partition}`, + `Uploading ${Object.values( + pendingUpdates, + ).length.toLocaleString()} preferences to partition: ${partition}`, ), ); @@ -245,20 +247,27 @@ export async function uploadPreferenceManagementPreferencesInteractive({ }) .json(); } catch (error) { - try { - const parsed = JSON.parse(error?.response?.body || '{}'); - if (parsed.error) { - logger.error(colors.red(`Error: ${parsed.error}`)); + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + + if (isSombraError(error)) { + try { + const parsed = JSON.parse(error.response.body) as Record< + string, + unknown + >; + if ('error' in parsed && typeof parsed.error === 'string') { + logger.error(colors.red(`Error: ${parsed.error}`)); + } + } catch { + // continue } - } catch { - // continue } logger.error( colors.red( - `Failed to upload ${ - currentChunk.length - } user preferences to partition ${partition}: ${ - error?.response?.body || error?.message + `Failed to upload ${currentChunk.length.toLocaleString()} user preferences to partition ${partition}: ${ + isSombraError(error) ? error.response.body : error.message }`, ), ); @@ -267,7 +276,7 @@ export async function uploadPreferenceManagementPreferencesInteractive({ failingUpdates[userId] = { uploadedAt: new Date().toISOString(), update, - error: error?.response?.body || error?.message || 'Unknown error', + error: isSombraError(error) ? error.response.body : error.message, }; } await preferenceState.setValue(failingUpdates, 'failingUpdates'); @@ -286,11 +295,11 @@ export async function uploadPreferenceManagementPreferencesInteractive({ const totalTime = t1 - t0; logger.info( colors.green( - `Successfully uploaded ${ - updatesToRun.length - } user preferences to partition ${partition} in "${ + `Successfully uploaded ${updatesToRun.length.toLocaleString()} user preferences to partition ${partition} in "${( totalTime / 1000 - }" seconds!`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds!`, ), ); } From dd72d6b91ab7bb02dfa0eb02f051c1b933f86e7d Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Thu, 17 Jul 2025 01:59:31 -0400 Subject: [PATCH 18/18] pref2 --- .../parsePreferenceAndPurposeValuesFromCsv.ts | 10 ++++-- .../parsePreferenceIdentifiersFromCsv.ts | 31 ++++++++++++------- .../parsePreferenceManagementCsv.ts | 26 +++++++++++----- .../parsePreferenceTimestampsFromCsv.ts | 12 ++++--- .../tests/getPreferenceUpdatesFromRow.test.ts | 8 ++--- 5 files changed, 57 insertions(+), 30 deletions(-) diff --git a/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts b/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts index c0845f58..eb9088a1 100644 --- a/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts +++ b/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts @@ -6,7 +6,7 @@ import { logger } from '../../logger'; import { mapSeries } from '../bluebird-replace'; import { PreferenceTopic } from '../graphql'; import { splitCsvToList } from '../requests'; -import { FileMetadataState } from './codecs'; +import { FileMetadataState, type PurposeRowMapping } from './codecs'; /** * Parse out the purpose.enabled and preference values from a CSV file @@ -59,7 +59,9 @@ export async function parsePreferenceAndPurposeValuesFromCsv( const uniqueValues = uniq(preferences.map((x) => x[col])); // Map the column to a purpose - let purposeMapping = currentState.columnToPurposeName[col]; + let purposeMapping = currentState.columnToPurposeName[col] as + | PurposeRowMapping + | undefined; if (purposeMapping) { logger.info( colors.magenta( @@ -164,7 +166,9 @@ export async function parsePreferenceAndPurposeValuesFromCsv( return; } - if (preferenceTopic.type === PreferenceTopicType.MultiSelect) { + if ( + (preferenceTopic.type as string) === PreferenceTopicType.MultiSelect + ) { const parsedValues = splitCsvToList(value); // need to do this serially await mapSeries(parsedValues, async (parsedValue) => { diff --git a/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts b/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts index df9357ce..581ec484 100644 --- a/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts +++ b/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts @@ -47,7 +47,7 @@ export async function parsePreferenceIdentifiersFromCsv( default: remainingColumnsForIdentifier.find((col) => col.toLowerCase().includes('email'), - ) || remainingColumnsForIdentifier[0], + ) ?? remainingColumnsForIdentifier[0], choices: remainingColumnsForIdentifier, }, ]); @@ -59,9 +59,18 @@ export async function parsePreferenceIdentifiersFromCsv( ), ); + if (!currentState.identifierColumn) { + throw new TypeError('No identifier column found'); + } + if (!currentState.timestampColum) { + throw new TypeError('No timestamp column found'); + } + + const { identifierColumn, timestampColum } = currentState; + // Validate that the identifier column is present for all rows and unique const identifierColumnsMissing = preferences - .map((pref, ind) => (pref[currentState.identifierColumn!] ? null : [ind])) + .map((pref, ind) => (pref[identifierColumn] ? null : [ind])) .filter((x): x is number[] => !!x) .flat(); if (identifierColumnsMissing.length > 0) { @@ -82,32 +91,30 @@ export async function parsePreferenceIdentifiersFromCsv( // Filter out rows missing an identifier const previous = preferences.length; - preferences = preferences.filter( - (pref) => pref[currentState.identifierColumn!], - ); + preferences = preferences.filter((pref) => pref[identifierColumn]); logger.info( colors.yellow( - `Skipped ${previous - preferences.length} rows missing an identifier`, + `Skipped ${(previous - preferences.length).toLocaleString()} rows missing an identifier`, ), ); } logger.info( colors.magenta( - `The identifier column "${currentState.identifierColumn}" is present for all rows`, + `The identifier column "${identifierColumn}" is present for all rows`, ), ); // Validate that all identifiers are unique - const rowsByUserId = groupBy(preferences, currentState.identifierColumn); + const rowsByUserId = groupBy(preferences, identifierColumn); const duplicateIdentifiers = Object.entries(rowsByUserId).filter( ([, rows]) => rows.length > 1, ); if (duplicateIdentifiers.length > 0) { const message = `The identifier column "${ - currentState.identifierColumn + identifierColumn }" has duplicate values for the following rows: ${duplicateIdentifiers .slice(0, 10) - .map(([userId, rows]) => `${userId} (${rows.length})`) + .map(([userId, rows]) => `${userId} (${rows.length.toLocaleString()})`) .join('\n')}`; logger.warn(colors.yellow(message)); @@ -123,8 +130,8 @@ export async function parsePreferenceIdentifiersFromCsv( .map(([, rows]) => { const sorted = rows.sort( (a, b) => - new Date(b[currentState.timestampColum!]).getTime() - - new Date(a[currentState.timestampColum!]).getTime(), + new Date(b[timestampColum]).getTime() - + new Date(a[timestampColum]).getTime(), ); return sorted[0]; }) diff --git a/src/lib/preference-management/parsePreferenceManagementCsv.ts b/src/lib/preference-management/parsePreferenceManagementCsv.ts index c0d629f5..e41e8b1f 100644 --- a/src/lib/preference-management/parsePreferenceManagementCsv.ts +++ b/src/lib/preference-management/parsePreferenceManagementCsv.ts @@ -1,4 +1,5 @@ import { PersistedState } from '@transcend-io/persisted-state'; +import type { PreferenceQueryResponseItem } from '@transcend-io/privacy-types'; import colors from 'colors'; import type { Got } from 'got'; import * as t from 'io-ts'; @@ -67,7 +68,7 @@ export async function parsePreferenceManagementCsvWithCache( pendingConflictUpdates: {}, skippedUpdates: {}, // Load in the last fetched time - ...((fileMetadata[file] || {}) as Partial), + ...((fileMetadata[file] ?? {}) as Partial), lastFetchedAt: new Date().toISOString(), }; @@ -103,10 +104,14 @@ export async function parsePreferenceManagementCsvWithCache( fileMetadata[file] = currentState; await cache.setValue(fileMetadata, 'fileMetadata'); + if (!currentState.identifierColumn) { + throw new TypeError('No identifier column found'); + } + + const { identifierColumn } = currentState; + // Grab existing preference store records - const identifiers = preferences.map( - (pref) => pref[currentState.identifierColumn!], - ); + const identifiers = preferences.map((pref) => pref[identifierColumn]); const existingConsentRecords = skipExistingRecordCheck ? [] : await getPreferencesForIdentifiers(sombra, { @@ -123,7 +128,7 @@ export async function parsePreferenceManagementCsvWithCache( // Process each row for (const pref of preferences) { // Grab unique Id for the user - const userId = pref[currentState.identifierColumn!]; + const userId = pref[identifierColumn]; // determine updates for user const pendingUpdates = getPreferenceUpdatesFromRow({ @@ -134,7 +139,9 @@ export async function parsePreferenceManagementCsvWithCache( }); // Grab current state of the update - const currentConsentRecord = consentRecordByIdentifier[userId]; + const currentConsentRecord = consentRecordByIdentifier[userId] as + | PreferenceQueryResponseItem + | undefined; if (forceTriggerWorkflows && !currentConsentRecord) { throw new Error( `No existing consent record found for user with id: ${userId}. @@ -183,7 +190,12 @@ export async function parsePreferenceManagementCsvWithCache( const t1 = Date.now(); logger.info( colors.green( - `Successfully pre-processed file: "${file}" in ${(t1 - t0) / 1000}s`, + `Successfully pre-processed file: "${file}" in ${( + (t1 - t0) / + 1000 + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}s`, ), ); } diff --git a/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts b/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts index 09b2fb48..6014dc75 100644 --- a/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts +++ b/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts @@ -45,10 +45,10 @@ export async function parsePreferenceTimestampsFromCsv( default: remainingColumnsForTimestamp.find((col) => col.toLowerCase().includes('date'), - ) || + ) ?? remainingColumnsForTimestamp.find((col) => col.toLowerCase().includes('time'), - ) || + ) ?? remainingColumnsForTimestamp[0], choices: [...remainingColumnsForTimestamp, NONE_PREFERENCE_MAP], }, @@ -60,9 +60,13 @@ export async function parsePreferenceTimestampsFromCsv( ); // Validate that all rows have valid timestamp - if (currentState.timestampColum !== NONE_PREFERENCE_MAP) { + if ( + currentState.timestampColum !== NONE_PREFERENCE_MAP && + currentState.timestampColum + ) { + const { timestampColum } = currentState; const timestampColumnsMissing = preferences - .map((pref, ind) => (pref[currentState.timestampColum!] ? null : [ind])) + .map((pref, ind) => (pref[timestampColum] ? null : [ind])) .filter((x): x is number[] => !!x) .flat(); if (timestampColumnsMissing.length > 0) { diff --git a/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts b/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts index 1c4a3c90..534fc029 100644 --- a/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts +++ b/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts @@ -526,7 +526,7 @@ describe('getPreferenceUpdatesFromRow', () => { }); expect.fail('Should have thrown'); } catch (error) { - expect(error.message).to.include('No mapping provided'); + expect((error as Error).message).to.include('No mapping provided'); } }); @@ -590,7 +590,7 @@ describe('getPreferenceUpdatesFromRow', () => { }); expect.fail('Should have thrown'); } catch (error) { - expect(error.message).to.equal( + expect((error as Error).message).to.equal( 'Invalid purpose slug: InvalidPurpose, expected: Marketing, Advertising', ); } @@ -652,7 +652,7 @@ describe('getPreferenceUpdatesFromRow', () => { }); expect.fail('Should have thrown'); } catch (error) { - expect(error.message).to.equal( + expect((error as Error).message).to.equal( 'Invalid value for select preference: SingleSelectPreference, expected string or null, got: true', ); } @@ -714,7 +714,7 @@ describe('getPreferenceUpdatesFromRow', () => { }); expect.fail('Should have thrown'); } catch (error) { - expect(error.message).to.equal( + expect((error as Error).message).to.equal( 'Invalid value for multi select preference: MultiSelectPreference, expected one of: Value1, Value2, got: true', ); }