From 1f489572e0ea5fbe2cfcb552b187927fa37813a2 Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Wed, 29 Mar 2023 18:35:21 +0300 Subject: [PATCH 01/24] basic sql store --- packages/mongodb-store/README.md | 2 +- packages/mysql-store/.npmignore | 6 ++ packages/mysql-store/README.md | 30 +++++++ packages/mysql-store/jest.config.json | 3 + packages/mysql-store/package.json | 43 ++++++++++ packages/mysql-store/src/index.ts | 1 + packages/mysql-store/src/store.ts | 10 +++ packages/mysql-store/tsconfig.json | 10 +++ packages/mysql-store/tsup.config.json | 6 ++ packages/sql-store/.npmignore | 6 ++ packages/sql-store/README.md | 32 ++++++++ packages/sql-store/jest.config.json | 3 + packages/sql-store/package.json | 43 ++++++++++ packages/sql-store/src/index.ts | 1 + packages/sql-store/src/mappers/collection.ts | 8 ++ packages/sql-store/src/mappers/field.ts | 42 ++++++++++ packages/sql-store/src/mappers/index.ts | 3 + .../sql-store/src/mappers/store-index.test.ts | 27 +++++++ packages/sql-store/src/mappers/store-index.ts | 76 +++++++++++++++++ packages/sql-store/src/queries/connection.ts | 3 + packages/sql-store/src/queries/index.ts | 4 + .../src/queries/list-table-columns.ts | 24 ++++++ .../src/queries/list-table-statistics.ts | 25 ++++++ packages/sql-store/src/queries/list-tables.ts | 15 ++++ packages/sql-store/src/store.ts | 81 +++++++++++++++++++ packages/sql-store/tsconfig.json | 10 +++ packages/sql-store/tsup.config.json | 6 ++ yarn.lock | 50 +++++++++++- 28 files changed, 568 insertions(+), 2 deletions(-) create mode 100644 packages/mysql-store/.npmignore create mode 100644 packages/mysql-store/README.md create mode 100644 packages/mysql-store/jest.config.json create mode 100644 packages/mysql-store/package.json create mode 100644 packages/mysql-store/src/index.ts create mode 100644 packages/mysql-store/src/store.ts create mode 100644 packages/mysql-store/tsconfig.json create mode 100644 packages/mysql-store/tsup.config.json create mode 100644 packages/sql-store/.npmignore create mode 100644 packages/sql-store/README.md create mode 100644 packages/sql-store/jest.config.json create mode 100644 packages/sql-store/package.json create mode 100644 packages/sql-store/src/index.ts create mode 100644 packages/sql-store/src/mappers/collection.ts create mode 100644 packages/sql-store/src/mappers/field.ts create mode 100644 packages/sql-store/src/mappers/index.ts create mode 100644 packages/sql-store/src/mappers/store-index.test.ts create mode 100644 packages/sql-store/src/mappers/store-index.ts create mode 100644 packages/sql-store/src/queries/connection.ts create mode 100644 packages/sql-store/src/queries/index.ts create mode 100644 packages/sql-store/src/queries/list-table-columns.ts create mode 100644 packages/sql-store/src/queries/list-table-statistics.ts create mode 100644 packages/sql-store/src/queries/list-tables.ts create mode 100644 packages/sql-store/src/store.ts create mode 100644 packages/sql-store/tsconfig.json create mode 100644 packages/sql-store/tsup.config.json diff --git a/packages/mongodb-store/README.md b/packages/mongodb-store/README.md index 1dc51c2..5e55ee3 100644 --- a/packages/mongodb-store/README.md +++ b/packages/mongodb-store/README.md @@ -14,7 +14,7 @@ npm install @neuledge/mongodb-store import { Engine } from '@neuledge/engine'; import { MongoDBStore } from '@neuledge/mongodb-store'; -const store = store: new MongoDBStore({ +const store = new MongoDBStore({ url: process.env.MONGODB_URL ?? 'mongodb://localhost:27017', name: process.env.MONGODB_DATABASE ?? 'my-database', }); diff --git a/packages/mysql-store/.npmignore b/packages/mysql-store/.npmignore new file mode 100644 index 0000000..d2ee3b5 --- /dev/null +++ b/packages/mysql-store/.npmignore @@ -0,0 +1,6 @@ +/* +!/dist/*.js +!/dist/*.js.map +!/dist/*.mjs +!/dist/*.mjs.map +!/dist/**/*.d.ts \ No newline at end of file diff --git a/packages/mysql-store/README.md b/packages/mysql-store/README.md new file mode 100644 index 0000000..ad5cf63 --- /dev/null +++ b/packages/mysql-store/README.md @@ -0,0 +1,30 @@ +# Neuledge MySQL Store + +A store for [Neuledge Engine](https://neuledge.com) that uses [MySQL](https://www.mysql.com) database as a backend. + +## 📦 Installation + +```bash +npm install @neuledge/mysql-store +``` + +## 🚀 Getting started + +```ts +import { MySQLStore } from '@neuledge/mysql-store'; + +const store = new MySQLStore({ + uri: process.env.MYSQL_URI ?? 'mysql://localhost:3306', + database: process.env.MYSQL_DATABASE ?? 'my-database', +}); + +const engine = new Engine({ + store, +}); +``` + +For more information, please refer to the [main repository](https://github.com/neuledge/engine-js). + +## 📄 License + +Neuledge is [Apache 2.0 licensed](https://github.com/neuledge/engine-js/blob/main/LICENSE). diff --git a/packages/mysql-store/jest.config.json b/packages/mysql-store/jest.config.json new file mode 100644 index 0000000..5901941 --- /dev/null +++ b/packages/mysql-store/jest.config.json @@ -0,0 +1,3 @@ +{ + "preset": "@neuledge/jest-ts-preset" +} diff --git a/packages/mysql-store/package.json b/packages/mysql-store/package.json new file mode 100644 index 0000000..be5cc6a --- /dev/null +++ b/packages/mysql-store/package.json @@ -0,0 +1,43 @@ +{ + "name": "@neuledge/mysql-store", + "version": "0.0.0", + "deascription": "MySQL store implementation for Neuledge Engine", + "keywords": [ + "neuledge", + "mysql", + "store", + "database" + ], + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.js", + "exports": { + ".": { + "require": "./dist/index.js", + "import": "./dist/index.mjs", + "types": "./dist/index.d.ts" + } + }, + "repository": { + "type": "git", + "url": "https://github.com/neuledge/engine-js.git" + }, + "license": "Apache-2.0", + "publishConfig": { + "access": "public" + }, + "engines": { + "node": ">= 16" + }, + "scripts": { + "types": "rimraf --glob dist/*.{d.ts,d.ts.map} dist/**/*.{d.ts,d.ts.map} && tsc --emitDeclarationOnly && tsc-alias", + "build": "rimraf --glob dist/*.{js,js.map,mjs,mjs.map} && tsup", + "test": "jest", + "lint": "eslint . --ext \"js,jsx,ts,tsx,mjs,cjs\"", + "lint:strict": "yarn lint --max-warnings 0" + }, + "dependencies": { + "@neuledge/sql-store": "^0.0.0", + "mysql2": "^3.2.0" + } +} diff --git a/packages/mysql-store/src/index.ts b/packages/mysql-store/src/index.ts new file mode 100644 index 0000000..d406816 --- /dev/null +++ b/packages/mysql-store/src/index.ts @@ -0,0 +1 @@ +export * from './store'; diff --git a/packages/mysql-store/src/store.ts b/packages/mysql-store/src/store.ts new file mode 100644 index 0000000..7fee6fd --- /dev/null +++ b/packages/mysql-store/src/store.ts @@ -0,0 +1,10 @@ +import { ConnectionOptions, PoolOptions, createPool } from 'mysql2/promise'; +import { SQLStore } from '@neuledge/sql-store'; + +export type MySQLStoreOptions = PoolOptions & ConnectionOptions; + +export class MySQLStore extends SQLStore { + constructor(options: MySQLStoreOptions) { + super(createPool(options)); + } +} diff --git a/packages/mysql-store/tsconfig.json b/packages/mysql-store/tsconfig.json new file mode 100644 index 0000000..c67724d --- /dev/null +++ b/packages/mysql-store/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "@neuledge/tsconfig/base.json", + "compilerOptions": { + "baseUrl": "src", + "rootDir": "src", + "outDir": "dist" + }, + "include": ["src"], + "exclude": ["node_modules", "**/__ignore__/**"] +} diff --git a/packages/mysql-store/tsup.config.json b/packages/mysql-store/tsup.config.json new file mode 100644 index 0000000..2f3a43d --- /dev/null +++ b/packages/mysql-store/tsup.config.json @@ -0,0 +1,6 @@ +{ + "entry": ["src/index.ts"], + "format": ["esm", "cjs"], + "sourcemap": true, + "shims": true +} diff --git a/packages/sql-store/.npmignore b/packages/sql-store/.npmignore new file mode 100644 index 0000000..d2ee3b5 --- /dev/null +++ b/packages/sql-store/.npmignore @@ -0,0 +1,6 @@ +/* +!/dist/*.js +!/dist/*.js.map +!/dist/*.mjs +!/dist/*.mjs.map +!/dist/**/*.d.ts \ No newline at end of file diff --git a/packages/sql-store/README.md b/packages/sql-store/README.md new file mode 100644 index 0000000..79302fe --- /dev/null +++ b/packages/sql-store/README.md @@ -0,0 +1,32 @@ +# Neuledge MySQL Store + +An store for [Neuledge Engine](https://neuledge.com) that uses [SQL](https://en.wikipedia.org/wiki/SQL) connection as a backend. + +This library is not intended to be used directly. It is a dependency of the SQL-based stores such as **MySQL** and **PostgreSQL**. For more information, please refer to the [main repository](https://github.com/neuledge/engine-js) + +## 📦 Installation + +```bash +npm install @neuledge/sql-store +``` + +## 🚀 Getting started + +```ts +import { SQLStore } from '@neuledge/mysql-store'; + +// create a connection to your SQL database somehow +const connection = createPool({ + // ... +}); + +const store = new SQLStore(connection); + +const engine = new Engine({ + store, +}); +``` + +## 📄 License + +Neuledge is [Apache 2.0 licensed](https://github.com/neuledge/engine-js/blob/main/LICENSE). diff --git a/packages/sql-store/jest.config.json b/packages/sql-store/jest.config.json new file mode 100644 index 0000000..5901941 --- /dev/null +++ b/packages/sql-store/jest.config.json @@ -0,0 +1,3 @@ +{ + "preset": "@neuledge/jest-ts-preset" +} diff --git a/packages/sql-store/package.json b/packages/sql-store/package.json new file mode 100644 index 0000000..00f552f --- /dev/null +++ b/packages/sql-store/package.json @@ -0,0 +1,43 @@ +{ + "name": "@neuledge/sql-store", + "version": "0.0.0", + "deascription": "Abstract SQL store implementation for Neuledge Engine", + "keywords": [ + "neuledge", + "abstract", + "sql", + "store", + "database" + ], + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.js", + "exports": { + ".": { + "require": "./dist/index.js", + "import": "./dist/index.mjs", + "types": "./dist/index.d.ts" + } + }, + "repository": { + "type": "git", + "url": "https://github.com/neuledge/engine-js.git" + }, + "license": "Apache-2.0", + "publishConfig": { + "access": "public" + }, + "engines": { + "node": ">= 16" + }, + "scripts": { + "types": "rimraf --glob dist/*.{d.ts,d.ts.map} dist/**/*.{d.ts,d.ts.map} && tsc --emitDeclarationOnly && tsc-alias", + "build": "rimraf --glob dist/*.{js,js.map,mjs,mjs.map} && tsup", + "test": "jest", + "lint": "eslint . --ext \"js,jsx,ts,tsx,mjs,cjs\"", + "lint:strict": "yarn lint --max-warnings 0" + }, + "dependencies": { + "@neuledge/store": "^0.2.0" + } +} diff --git a/packages/sql-store/src/index.ts b/packages/sql-store/src/index.ts new file mode 100644 index 0000000..d406816 --- /dev/null +++ b/packages/sql-store/src/index.ts @@ -0,0 +1 @@ +export * from './store'; diff --git a/packages/sql-store/src/mappers/collection.ts b/packages/sql-store/src/mappers/collection.ts new file mode 100644 index 0000000..dac9b35 --- /dev/null +++ b/packages/sql-store/src/mappers/collection.ts @@ -0,0 +1,8 @@ +import { StoreCollection_Slim } from '@neuledge/store'; +import { SQLTable } from '@/queries'; + +export const toStoreCollection_Slim = ( + table: SQLTable, +): StoreCollection_Slim => ({ + name: table.table_name, +}); diff --git a/packages/sql-store/src/mappers/field.ts b/packages/sql-store/src/mappers/field.ts new file mode 100644 index 0000000..31992f6 --- /dev/null +++ b/packages/sql-store/src/mappers/field.ts @@ -0,0 +1,42 @@ +import { StoreError, StoreField, StoreShapeType } from '@neuledge/store'; +import { SQLColumn } from '@/queries'; + +/** + * Map the SQL data types to the corresponding StoreShapeType + */ +const dataTypeMap: Record = { + varchar: 'string', + char: 'string', + text: 'string', + numeric: 'number', + decimal: 'number', + float: 'number', + double: 'number', + integer: 'number', + bigint: 'number', + boolean: 'boolean', + bytea: 'binary', + timestamp: 'date-time', + timestamptz: 'date-time', + json: 'json', + jsonb: 'json', +}; + +export const toStoreField = (column: SQLColumn): StoreField => { + const type = dataTypeMap[column.data_type]; + if (!type) { + throw new StoreError( + StoreError.Code.NOT_SUPPORTED, + `Unsupported data type "${column.data_type}" for column "${column.column_name}"`, + ); + } + + return { + name: column.column_name, + type, + nullable: column.is_nullable === 'YES', + size: column.character_maximum_length, + precision: column.numeric_precision, + scale: column.numeric_scale, + }; +}; diff --git a/packages/sql-store/src/mappers/index.ts b/packages/sql-store/src/mappers/index.ts new file mode 100644 index 0000000..93e2374 --- /dev/null +++ b/packages/sql-store/src/mappers/index.ts @@ -0,0 +1,3 @@ +export * from './collection'; +export * from './field'; +export * from './store-index'; diff --git a/packages/sql-store/src/mappers/store-index.test.ts b/packages/sql-store/src/mappers/store-index.test.ts new file mode 100644 index 0000000..1904adb --- /dev/null +++ b/packages/sql-store/src/mappers/store-index.test.ts @@ -0,0 +1,27 @@ +import { toStoreIndexes } from './store-index'; + +describe('mappers/store-index', () => { + describe('toStoreIndexes()', () => { + it('should convert a single primary index', () => { + expect( + toStoreIndexes([ + { + index_name: 'PRIMARY', + non_unique: 0, + column_name: 'field_name', + seq_in_index: 1, + collation: 'A', + }, + ]), + ).toEqual([ + { + name: 'PRIMARY', + unique: 'primary', + fields: { + field_name: { sort: 'asc' }, + }, + }, + ]); + }); + }); +}); diff --git a/packages/sql-store/src/mappers/store-index.ts b/packages/sql-store/src/mappers/store-index.ts new file mode 100644 index 0000000..7cad662 --- /dev/null +++ b/packages/sql-store/src/mappers/store-index.ts @@ -0,0 +1,76 @@ +import { SQLStatistic } from '@/queries'; +import { StoreError, StoreIndex, StorePrimaryKey } from '@neuledge/store'; + +export const toStoreIndexes = ( + statistics: SQLStatistic[], +): [primary: StorePrimaryKey, ...indexes: StoreIndex[]] => { + const indexStatistics = groupTableStatistics(statistics); + + let primaryKey: StorePrimaryKey | undefined; + const indexes: StoreIndex[] = []; + + for (const indexColumn of indexStatistics) { + const index = toStoreIndex(indexColumn); + + if (index.unique === 'primary') { + primaryKey = index as StorePrimaryKey; + } else { + indexes.push(index); + } + } + + if (!primaryKey) { + throw new StoreError( + StoreError.Code.INVALID_DATA, + `Primary key not found for collection "${name}"`, + ); + } + + return [primaryKey, ...indexes]; +}; + +const toStoreIndex = ( + indexStatistics: SQLStatistic[], +): StoreIndex | StorePrimaryKey => { + const { index_name, non_unique, column_extra } = indexStatistics[0]; + + const index: StoreIndex | StorePrimaryKey = { + name: index_name, + unique: !non_unique, + fields: {}, + }; + + if (index.unique && index_name === 'PRIMARY') { + index.unique = 'primary'; + + if (column_extra === 'auto_increment') { + (index as StorePrimaryKey).auto = 'increment'; + } + } + + indexStatistics.sort((a, b) => a.seq_in_index - b.seq_in_index); + + for (const statistic of indexStatistics) { + index.fields[statistic.column_name] = { + sort: statistic.collation === 'A' ? 'asc' : 'desc', + }; + } + + return index; +}; + +const groupTableStatistics = (statistics: SQLStatistic[]): SQLStatistic[][] => { + const groupMap: Record = {}; + + for (const statistic of statistics) { + let group = groupMap[statistic.index_name]; + if (!group) { + group = []; + groupMap[statistic.index_name] = group; + } + + group.push(statistic); + } + + return Object.values(groupMap); +}; diff --git a/packages/sql-store/src/queries/connection.ts b/packages/sql-store/src/queries/connection.ts new file mode 100644 index 0000000..0598c31 --- /dev/null +++ b/packages/sql-store/src/queries/connection.ts @@ -0,0 +1,3 @@ +export interface SQLConnection { + query(query: string, params?: unknown[]): Promise; +} diff --git a/packages/sql-store/src/queries/index.ts b/packages/sql-store/src/queries/index.ts new file mode 100644 index 0000000..45599ce --- /dev/null +++ b/packages/sql-store/src/queries/index.ts @@ -0,0 +1,4 @@ +export * from './connection'; +export * from './list-tables'; +export * from './list-table-columns'; +export * from './list-table-statistics'; diff --git a/packages/sql-store/src/queries/list-table-columns.ts b/packages/sql-store/src/queries/list-table-columns.ts new file mode 100644 index 0000000..a52d591 --- /dev/null +++ b/packages/sql-store/src/queries/list-table-columns.ts @@ -0,0 +1,24 @@ +import { SQLConnection } from './connection'; + +/** + * A table column from the information_schema.columns table. + */ +export interface SQLColumn { + column_name: string; + data_type: string; + character_maximum_length?: number | null; + numeric_precision?: number | null; + numeric_scale?: number | null; + is_nullable: 'YES' | 'NO'; +} + +export const listTableColumns = async ( + connection: SQLConnection, + tableName: string, +): Promise => + connection.query( + `SELECT column_name, data_type, character_maximum_length, numeric_precision, numeric_scale, is_nullable +FROM information_schema.columns +WHERE table_name = ? AND table_schema = DATABASE()`, + [tableName], + ); diff --git a/packages/sql-store/src/queries/list-table-statistics.ts b/packages/sql-store/src/queries/list-table-statistics.ts new file mode 100644 index 0000000..f47560c --- /dev/null +++ b/packages/sql-store/src/queries/list-table-statistics.ts @@ -0,0 +1,25 @@ +import { SQLConnection } from './connection'; + +/** + * A table statistic row from the information_schema.statistics table. + */ +export interface SQLStatistic { + index_name: string; + column_name: string; + seq_in_index: number; + collation: 'A' | 'D'; + non_unique: number; + column_extra?: string; +} + +export const listTableStatistics = async ( + connection: SQLConnection, + tableName: string, +): Promise => + connection.query( + `SELECT index_name, column_name, seq_in_index, collation, non_unique, extra AS column_extra + FROM information_schema.statistics INNER JOIN information_schema.columns USING (table_schema, table_name, column_name) + WHERE table_schema = DATABASE() AND table_name = ? + ORDER BY index_name, seq_in_index`, + [tableName], + ); diff --git a/packages/sql-store/src/queries/list-tables.ts b/packages/sql-store/src/queries/list-tables.ts new file mode 100644 index 0000000..e29a81e --- /dev/null +++ b/packages/sql-store/src/queries/list-tables.ts @@ -0,0 +1,15 @@ +import { SQLConnection } from './connection'; + +/** + * The tables in the database. This is a view of the `information_schema.tables` table. + */ +export interface SQLTable { + table_name: string; +} + +export const listTables = async ( + connection: SQLConnection, +): Promise => + connection.query( + `SELECT table_name AS name FROM information_schema.tables WHERE table_schema = DATABASE()`, + ); diff --git a/packages/sql-store/src/store.ts b/packages/sql-store/src/store.ts new file mode 100644 index 0000000..48b168d --- /dev/null +++ b/packages/sql-store/src/store.ts @@ -0,0 +1,81 @@ +import { + Store, + StoreCollection, + StoreCollection_Slim, + StoreDeleteOptions, + StoreDescribeCollectionOptions, + StoreDocument, + StoreDropCollectionOptions, + StoreEnsureCollectionOptions, + StoreFindOptions, + StoreInsertOptions, + StoreInsertionResponse, + StoreList, + StoreMutationResponse, + StoreUpdateOptions, +} from '@neuledge/store'; +import { + SQLConnection, + listTableColumns, + listTableStatistics, + listTables, +} from './queries'; +import { + toStoreCollection_Slim, + toStoreField, + toStoreIndexes, +} from './mappers'; + +export class SQLStore implements Store { + constructor(public readonly connection: SQLConnection) {} + + async listCollections(): Promise { + const tables = await listTables(this.connection); + return tables.map((table) => toStoreCollection_Slim(table)); + } + + async describeCollection( + options: StoreDescribeCollectionOptions, + ): Promise { + const { name } = options.collection; + + const [columns, statistics] = await Promise.all([ + listTableColumns(this.connection, name), + listTableStatistics(this.connection, name), + ]); + + const fields = columns.map((column) => toStoreField(column)); + const indexes = toStoreIndexes(statistics); + + return { + name, + primaryKey: indexes[0], + indexes: Object.fromEntries(indexes.map((index) => [index.name, index])), + fields: Object.fromEntries(fields.map((field) => [field.name, field])), + }; + } + + ensureCollection(options: StoreEnsureCollectionOptions): Promise { + throw new Error('Method not implemented.'); + } + + dropCollection(options: StoreDropCollectionOptions): Promise { + throw new Error('Method not implemented.'); + } + + find(options: StoreFindOptions): Promise> { + throw new Error('Method not implemented.'); + } + + insert(options: StoreInsertOptions): Promise { + throw new Error('Method not implemented.'); + } + + update(options: StoreUpdateOptions): Promise { + throw new Error('Method not implemented.'); + } + + delete(options: StoreDeleteOptions): Promise { + throw new Error('Method not implemented.'); + } +} diff --git a/packages/sql-store/tsconfig.json b/packages/sql-store/tsconfig.json new file mode 100644 index 0000000..c67724d --- /dev/null +++ b/packages/sql-store/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "@neuledge/tsconfig/base.json", + "compilerOptions": { + "baseUrl": "src", + "rootDir": "src", + "outDir": "dist" + }, + "include": ["src"], + "exclude": ["node_modules", "**/__ignore__/**"] +} diff --git a/packages/sql-store/tsup.config.json b/packages/sql-store/tsup.config.json new file mode 100644 index 0000000..2f3a43d --- /dev/null +++ b/packages/sql-store/tsup.config.json @@ -0,0 +1,6 @@ +{ + "entry": ["src/index.ts"], + "format": ["esm", "cjs"], + "sourcemap": true, + "shims": true +} diff --git a/yarn.lock b/yarn.lock index d4c0766..32b5f0e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3196,6 +3196,11 @@ delayed-stream@~1.0.0: resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== +denque@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1" + integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== + depd@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" @@ -4376,6 +4381,13 @@ iconv-lite@^0.4.24: dependencies: safer-buffer ">= 2.1.2 < 3" +iconv-lite@^0.6.3: + version "0.6.3" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501" + integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw== + dependencies: + safer-buffer ">= 2.1.2 < 3.0.0" + ieee754@^1.1.13, ieee754@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" @@ -5423,6 +5435,11 @@ log-update@^4.0.0: slice-ansi "^4.0.0" wrap-ansi "^6.2.0" +long@^5.2.1: + version "5.2.1" + resolved "https://registry.yarnpkg.com/long/-/long-5.2.1.tgz#e27595d0083d103d2fa2c20c7699f8e0c92b897f" + integrity sha512-GKSNGeNAtw8IryjjkhZxuKB3JzlcLTwjtiQCHKvqQet81I93kXslhDQruGI/QsddO83mcDToBVy7GqGS/zYf/A== + loose-envify@^1.0.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" @@ -5728,6 +5745,20 @@ mylas@^2.1.9: resolved "https://registry.yarnpkg.com/mylas/-/mylas-2.1.13.tgz#1e23b37d58fdcc76e15d8a5ed23f9ae9fc0cbdf4" integrity sha512-+MrqnJRtxdF+xngFfUUkIMQrUUL0KsxbADUkn23Z/4ibGg192Q+z+CQyiYwvWTsYjJygmMR8+w3ZDa98Zh6ESg== +mysql2@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/mysql2/-/mysql2-3.2.0.tgz#3613a8903bcb7ade0ae35b29945a0378eb67da89" + integrity sha512-0Vn6a9WSrq6fWwvPgrvIwnOCldiEcgbzapVRDAtDZ4cMTxN7pnGqCTx8EG32S/NYXl6AXkdO+9hV1tSIi/LigA== + dependencies: + denque "^2.1.0" + generate-function "^2.3.1" + iconv-lite "^0.6.3" + long "^5.2.1" + lru-cache "^7.14.1" + named-placeholders "^1.1.3" + seq-queue "^0.0.5" + sqlstring "^2.3.2" + mz@^2.7.0: version "2.7.0" resolved "https://registry.yarnpkg.com/mz/-/mz-2.7.0.tgz#95008057a56cafadc2bc63dde7f9ff6955948e32" @@ -5737,6 +5768,13 @@ mz@^2.7.0: object-assign "^4.0.1" thenify-all "^1.0.0" +named-placeholders@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/named-placeholders/-/named-placeholders-1.1.3.tgz#df595799a36654da55dda6152ba7a137ad1d9351" + integrity sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w== + dependencies: + lru-cache "^7.14.1" + napi-build-utils@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-1.0.2.tgz#b1fddc0b2c46e380a0b7a76f984dd47c41a13806" @@ -6629,7 +6667,7 @@ safe-stable-stringify@^2.0.0, safe-stable-stringify@^2.3.0, safe-stable-stringif resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz#138c84b6f6edb3db5f8ef3ef7115b8f55ccbf886" integrity sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g== -"safer-buffer@>= 2.1.2 < 3": +"safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0": version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== @@ -6682,6 +6720,11 @@ sentence-case@^3.0.4: tslib "^2.0.3" upper-case-first "^2.0.2" +seq-queue@^0.0.5: + version "0.0.5" + resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" + integrity sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q== + set-blocking@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" @@ -6917,6 +6960,11 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" integrity sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g== +sqlstring@^2.3.2: + version "2.3.3" + resolved "https://registry.yarnpkg.com/sqlstring/-/sqlstring-2.3.3.tgz#2ddc21f03bce2c387ed60680e739922c65751d0c" + integrity sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg== + stack-utils@^2.0.3: version "2.0.6" resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-2.0.6.tgz#aaf0748169c02fc33c8232abccf933f54a1cc34f" From 9cbc26c06b2d9b37d7b522bada142748e187a26c Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Wed, 29 Mar 2023 23:46:54 +0300 Subject: [PATCH 02/24] update main readme stores --- packages/engine/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/engine/README.md b/packages/engine/README.md index 80fb8e3..497079d 100644 --- a/packages/engine/README.md +++ b/packages/engine/README.md @@ -12,8 +12,9 @@

MongoDB ⇄ - MySQL (soon) ⇄ - PostgreSQL (soon) + MySQL ⇄ + PostgreSQL ⇄ + [Request more](https://github.com/neuledge/engine-js/issues/new)


From 4dd29d9114d0abf8132fa652258f799bf5fa38b3 Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Wed, 29 Mar 2023 23:48:46 +0300 Subject: [PATCH 03/24] update readme --- packages/engine/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/engine/README.md b/packages/engine/README.md index 497079d..fabd823 100644 --- a/packages/engine/README.md +++ b/packages/engine/README.md @@ -14,7 +14,7 @@ MongoDB ⇄ MySQL ⇄ PostgreSQL ⇄ - [Request more](https://github.com/neuledge/engine-js/issues/new) + Your DB (request)


From b5aee0665e79b2bb9a1ef83b5f41c9a0339a5cf7 Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Thu, 30 Mar 2023 00:05:40 +0300 Subject: [PATCH 04/24] use mysql --- packages/mysql-store/jest.config.json | 3 - packages/mysql-store/package.json | 4 +- packages/mysql-store/src/store.ts | 25 +++- packages/sql-store/src/queries/connection.ts | 2 +- yarn.lock | 115 +++++++++++-------- 5 files changed, 92 insertions(+), 57 deletions(-) delete mode 100644 packages/mysql-store/jest.config.json diff --git a/packages/mysql-store/jest.config.json b/packages/mysql-store/jest.config.json deleted file mode 100644 index 5901941..0000000 --- a/packages/mysql-store/jest.config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "preset": "@neuledge/jest-ts-preset" -} diff --git a/packages/mysql-store/package.json b/packages/mysql-store/package.json index be5cc6a..7571c1b 100644 --- a/packages/mysql-store/package.json +++ b/packages/mysql-store/package.json @@ -32,12 +32,12 @@ "scripts": { "types": "rimraf --glob dist/*.{d.ts,d.ts.map} dist/**/*.{d.ts,d.ts.map} && tsc --emitDeclarationOnly && tsc-alias", "build": "rimraf --glob dist/*.{js,js.map,mjs,mjs.map} && tsup", - "test": "jest", "lint": "eslint . --ext \"js,jsx,ts,tsx,mjs,cjs\"", "lint:strict": "yarn lint --max-warnings 0" }, "dependencies": { "@neuledge/sql-store": "^0.0.0", - "mysql2": "^3.2.0" + "@types/mysql": "^2.15.21", + "mysql": "^2.18.1" } } diff --git a/packages/mysql-store/src/store.ts b/packages/mysql-store/src/store.ts index 7fee6fd..11829c8 100644 --- a/packages/mysql-store/src/store.ts +++ b/packages/mysql-store/src/store.ts @@ -1,10 +1,29 @@ -import { ConnectionOptions, PoolOptions, createPool } from 'mysql2/promise'; +import { Pool, PoolConfig, createPool } from 'mysql'; import { SQLStore } from '@neuledge/sql-store'; -export type MySQLStoreOptions = PoolOptions & ConnectionOptions; +export type MySQLStoreOptions = PoolConfig; export class MySQLStore extends SQLStore { + private pool: Pool; + constructor(options: MySQLStoreOptions) { - super(createPool(options)); + const pool = createPool(options); + + super({ + query: (sql, values) => + new Promise((resolve, reject) => + pool.query(sql, values, (error, results) => + error ? reject(error) : resolve(results), + ), + ), + }); + + this.pool = pool; + } + + async close(): Promise { + await new Promise((resolve, reject) => + this.pool.end((error) => (error ? reject(error) : resolve())), + ); } } diff --git a/packages/sql-store/src/queries/connection.ts b/packages/sql-store/src/queries/connection.ts index 0598c31..5176269 100644 --- a/packages/sql-store/src/queries/connection.ts +++ b/packages/sql-store/src/queries/connection.ts @@ -1,3 +1,3 @@ export interface SQLConnection { - query(query: string, params?: unknown[]): Promise; + query(sql: string, params?: unknown[]): Promise; } diff --git a/yarn.lock b/yarn.lock index 32b5f0e..bcee074 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1980,6 +1980,13 @@ resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.2.tgz#ee771e2ba4b3dc5b372935d549fd9617bf345b8c" integrity sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ== +"@types/mysql@^2.15.21": + version "2.15.21" + resolved "https://registry.yarnpkg.com/@types/mysql/-/mysql-2.15.21.tgz#7516cba7f9d077f980100c85fd500c8210bd5e45" + integrity sha512-NPotx5CVful7yB+qZbWtXL2fA4e7aEHkihHLjklc6ID8aq7bhguHgeIoC1EmSNTAuCgI6ZXrjt2ZSaXnYX0EUg== + dependencies: + "@types/node" "*" + "@types/node@*", "@types/node@^18.14.2": version "18.15.5" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.15.5.tgz#3af577099a99c61479149b716183e70b5239324a" @@ -2556,6 +2563,11 @@ better-path-resolve@1.0.0: dependencies: is-windows "^1.0.0" +bignumber.js@9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-9.0.0.tgz#805880f84a329b5eac6e7cb6f8274b6d82bdf075" + integrity sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A== + binary-extensions@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" @@ -3020,6 +3032,11 @@ cookie@^0.5.0: resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b" integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw== +core-util-is@~1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" + integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== + cosmiconfig@8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-8.0.0.tgz#e9feae014eab580f858f8a0288f38997a7bebe97" @@ -3196,11 +3213,6 @@ delayed-stream@~1.0.0: resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== -denque@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1" - integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== - depd@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" @@ -4381,13 +4393,6 @@ iconv-lite@^0.4.24: dependencies: safer-buffer ">= 2.1.2 < 3" -iconv-lite@^0.6.3: - version "0.6.3" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501" - integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw== - dependencies: - safer-buffer ">= 2.1.2 < 3.0.0" - ieee754@^1.1.13, ieee754@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" @@ -4442,7 +4447,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.4, inherits@^2.0.3, inherits@^2.0.4: +inherits@2, inherits@2.0.4, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.3: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -4732,6 +4737,11 @@ is-windows@^1.0.0, is-windows@^1.0.1: resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d" integrity sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA== +isarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + integrity sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ== + isexe@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" @@ -5435,11 +5445,6 @@ log-update@^4.0.0: slice-ansi "^4.0.0" wrap-ansi "^6.2.0" -long@^5.2.1: - version "5.2.1" - resolved "https://registry.yarnpkg.com/long/-/long-5.2.1.tgz#e27595d0083d103d2fa2c20c7699f8e0c92b897f" - integrity sha512-GKSNGeNAtw8IryjjkhZxuKB3JzlcLTwjtiQCHKvqQet81I93kXslhDQruGI/QsddO83mcDToBVy7GqGS/zYf/A== - loose-envify@^1.0.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" @@ -5745,19 +5750,15 @@ mylas@^2.1.9: resolved "https://registry.yarnpkg.com/mylas/-/mylas-2.1.13.tgz#1e23b37d58fdcc76e15d8a5ed23f9ae9fc0cbdf4" integrity sha512-+MrqnJRtxdF+xngFfUUkIMQrUUL0KsxbADUkn23Z/4ibGg192Q+z+CQyiYwvWTsYjJygmMR8+w3ZDa98Zh6ESg== -mysql2@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/mysql2/-/mysql2-3.2.0.tgz#3613a8903bcb7ade0ae35b29945a0378eb67da89" - integrity sha512-0Vn6a9WSrq6fWwvPgrvIwnOCldiEcgbzapVRDAtDZ4cMTxN7pnGqCTx8EG32S/NYXl6AXkdO+9hV1tSIi/LigA== +mysql@^2.18.1: + version "2.18.1" + resolved "https://registry.yarnpkg.com/mysql/-/mysql-2.18.1.tgz#2254143855c5a8c73825e4522baf2ea021766717" + integrity sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig== dependencies: - denque "^2.1.0" - generate-function "^2.3.1" - iconv-lite "^0.6.3" - long "^5.2.1" - lru-cache "^7.14.1" - named-placeholders "^1.1.3" - seq-queue "^0.0.5" - sqlstring "^2.3.2" + bignumber.js "9.0.0" + readable-stream "2.3.7" + safe-buffer "5.1.2" + sqlstring "2.3.1" mz@^2.7.0: version "2.7.0" @@ -5768,13 +5769,6 @@ mz@^2.7.0: object-assign "^4.0.1" thenify-all "^1.0.0" -named-placeholders@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/named-placeholders/-/named-placeholders-1.1.3.tgz#df595799a36654da55dda6152ba7a137ad1d9351" - integrity sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w== - dependencies: - lru-cache "^7.14.1" - napi-build-utils@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-1.0.2.tgz#b1fddc0b2c46e380a0b7a76f984dd47c41a13806" @@ -6280,6 +6274,11 @@ pretty-format@^29.0.0, pretty-format@^29.5.0: ansi-styles "^5.0.0" react-is "^18.0.0" +process-nextick-args@~2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" + integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== + process-warning@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/process-warning/-/process-warning-2.1.0.tgz#1e60e3bfe8183033bbc1e702c2da74f099422d1a" @@ -6436,6 +6435,19 @@ read@^1.0.7: dependencies: mute-stream "~0.0.4" +readable-stream@2.3.7: + version "2.3.7" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" + integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + readable-stream@^3.1.1, readable-stream@^3.4.0: version "3.6.2" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.2.tgz#56a9b36ea965c00c5a93ef31eb111a0f11056967" @@ -6634,6 +6646,11 @@ rxjs@^7.5.5: dependencies: tslib "^2.1.0" +safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: + version "5.1.2" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== + safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" @@ -6667,7 +6684,7 @@ safe-stable-stringify@^2.0.0, safe-stable-stringify@^2.3.0, safe-stable-stringif resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz#138c84b6f6edb3db5f8ef3ef7115b8f55ccbf886" integrity sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g== -"safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0": +"safer-buffer@>= 2.1.2 < 3": version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== @@ -6720,11 +6737,6 @@ sentence-case@^3.0.4: tslib "^2.0.3" upper-case-first "^2.0.2" -seq-queue@^0.0.5: - version "0.0.5" - resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" - integrity sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q== - set-blocking@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" @@ -6960,10 +6972,10 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" integrity sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g== -sqlstring@^2.3.2: - version "2.3.3" - resolved "https://registry.yarnpkg.com/sqlstring/-/sqlstring-2.3.3.tgz#2ddc21f03bce2c387ed60680e739922c65751d0c" - integrity sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg== +sqlstring@2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/sqlstring/-/sqlstring-2.3.1.tgz#475393ff9e91479aea62dcaf0ca3d14983a7fb40" + integrity sha512-ooAzh/7dxIG5+uDik1z/Rd1vli0+38izZhGzSa34FwR7IbelPWCCKSNIl8jlL/F7ERvy8CB2jNeM1E9i9mXMAQ== stack-utils@^2.0.3: version "2.0.6" @@ -7050,6 +7062,13 @@ string_decoder@^1.1.1: dependencies: safe-buffer "~5.2.0" +string_decoder@~1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" + integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== + dependencies: + safe-buffer "~5.1.0" + strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" @@ -7596,7 +7615,7 @@ urlpattern-polyfill@^6.0.2: dependencies: braces "^3.0.2" -util-deprecate@^1.0.1: +util-deprecate@^1.0.1, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== From 919f2e9dc9f5e994e771060cb0fee0eba388a66c Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Sun, 2 Apr 2023 19:33:41 +0300 Subject: [PATCH 05/24] add postgresql --- package.json | 2 +- packages/mysql-store/package.json | 1 + .../src/queries/connection.ts | 0 .../src/queries/index.ts | 0 .../src/queries/list-table-columns.ts | 44 +++++ .../src/queries/list-table-statistics.ts | 25 +++ .../src/queries/list-tables.ts | 8 +- packages/mysql-store/src/store.ts | 82 +++++++- packages/postgresql-store/.npmignore | 6 + packages/postgresql-store/README.md | 30 +++ packages/postgresql-store/package.json | 45 +++++ packages/postgresql-store/src/index.ts | 1 + .../src/queries/connection.ts | 3 + .../postgresql-store/src/queries/index.ts | 4 + .../src/queries/list-table-columns.ts | 77 ++++++++ .../src/queries/list-table-statistics.ts | 40 ++++ .../src/queries/list-tables.ts | 15 ++ .../src/store.ts | 66 ++++--- packages/postgresql-store/tsconfig.json | 10 + packages/postgresql-store/tsup.config.json | 6 + packages/sql-store/README.md | 16 +- packages/sql-store/src/index.ts | 3 +- packages/sql-store/src/logic/collections.ts | 115 +++++++++++ packages/sql-store/src/logic/index.ts | 1 + packages/sql-store/src/mappers/collection.ts | 5 +- packages/sql-store/src/mappers/field.ts | 36 ++-- .../sql-store/src/mappers/store-index.test.ts | 29 +-- packages/sql-store/src/mappers/store-index.ts | 81 +++----- .../src/queries/list-table-columns.ts | 24 --- .../src/queries/list-table-statistics.ts | 25 --- yarn.lock | 179 ++++++++++++++---- 31 files changed, 740 insertions(+), 239 deletions(-) rename packages/{sql-store => mysql-store}/src/queries/connection.ts (100%) rename packages/{sql-store => mysql-store}/src/queries/index.ts (100%) create mode 100644 packages/mysql-store/src/queries/list-table-columns.ts create mode 100644 packages/mysql-store/src/queries/list-table-statistics.ts rename packages/{sql-store => mysql-store}/src/queries/list-tables.ts (56%) create mode 100644 packages/postgresql-store/.npmignore create mode 100644 packages/postgresql-store/README.md create mode 100644 packages/postgresql-store/package.json create mode 100644 packages/postgresql-store/src/index.ts create mode 100644 packages/postgresql-store/src/queries/connection.ts create mode 100644 packages/postgresql-store/src/queries/index.ts create mode 100644 packages/postgresql-store/src/queries/list-table-columns.ts create mode 100644 packages/postgresql-store/src/queries/list-table-statistics.ts create mode 100644 packages/postgresql-store/src/queries/list-tables.ts rename packages/{sql-store => postgresql-store}/src/store.ts (61%) create mode 100644 packages/postgresql-store/tsconfig.json create mode 100644 packages/postgresql-store/tsup.config.json create mode 100644 packages/sql-store/src/logic/collections.ts create mode 100644 packages/sql-store/src/logic/index.ts delete mode 100644 packages/sql-store/src/queries/list-table-columns.ts delete mode 100644 packages/sql-store/src/queries/list-table-statistics.ts diff --git a/package.json b/package.json index c3800ef..3da1245 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "ts-node": "^10.9.1", "tsc-alias": "^1.8.3", "tsup": "^6.6.3", - "turbo": "^1.8.3", + "turbo": "^1.8.8", "typescript": "^5.0.2" } } diff --git a/packages/mysql-store/package.json b/packages/mysql-store/package.json index 7571c1b..7a44483 100644 --- a/packages/mysql-store/package.json +++ b/packages/mysql-store/package.json @@ -36,6 +36,7 @@ "lint:strict": "yarn lint --max-warnings 0" }, "dependencies": { + "@neuledge/store": "^0.2.0", "@neuledge/sql-store": "^0.0.0", "@types/mysql": "^2.15.21", "mysql": "^2.18.1" diff --git a/packages/sql-store/src/queries/connection.ts b/packages/mysql-store/src/queries/connection.ts similarity index 100% rename from packages/sql-store/src/queries/connection.ts rename to packages/mysql-store/src/queries/connection.ts diff --git a/packages/sql-store/src/queries/index.ts b/packages/mysql-store/src/queries/index.ts similarity index 100% rename from packages/sql-store/src/queries/index.ts rename to packages/mysql-store/src/queries/index.ts diff --git a/packages/mysql-store/src/queries/list-table-columns.ts b/packages/mysql-store/src/queries/list-table-columns.ts new file mode 100644 index 0000000..6c3ade8 --- /dev/null +++ b/packages/mysql-store/src/queries/list-table-columns.ts @@ -0,0 +1,44 @@ +import { StoreShapeType } from '@neuledge/store'; +import { SQLConnection } from './connection'; + +/** + * A table column from the information_schema.columns table. + */ +export interface MySQLColumn { + column_name: string; + data_type: string; + character_maximum_length: number | null; + numeric_precision: number | null; + numeric_scale: number | null; + is_nullable: 1 | 0; + is_auto_increment: 1 | 0; +} + +export const listTableColumns = async ( + tableName: string, + connection: SQLConnection, +): Promise => + connection.query( + `SELECT column_name, data_type, character_maximum_length, numeric_precision, numeric_scale, (is_nullable = 'YES') AS is_nullable, extra LIKE '%auto_increment%' AS is_auto_increment +FROM information_schema.columns +WHERE table_schema = DATABASE() AND table_name = ?`, + [tableName], + ); + +export const dataTypeMap: Record = { + varchar: 'string', + char: 'string', + text: 'string', + numeric: 'number', + decimal: 'number', + float: 'number', + double: 'number', + integer: 'number', + bigint: 'number', + boolean: 'boolean', + bytea: 'binary', + timestamp: 'date-time', + timestamptz: 'date-time', + json: 'json', + jsonb: 'json', +}; diff --git a/packages/mysql-store/src/queries/list-table-statistics.ts b/packages/mysql-store/src/queries/list-table-statistics.ts new file mode 100644 index 0000000..5f3ad45 --- /dev/null +++ b/packages/mysql-store/src/queries/list-table-statistics.ts @@ -0,0 +1,25 @@ +import { SQLConnection } from './connection'; + +/** + * A table statistic row from the information_schema.statistics table. + */ +export interface MySQLIndexAttribute { + index_name: string; + column_name: string; + seq_in_index: number; + direction: 'ASC' | 'DESC'; + is_unique: 1 | 0; + is_primary: 1 | 0; +} + +export const listIndexAttributes = async ( + tableName: string, + connection: SQLConnection, +): Promise => + connection.query( + `SELECT index_name, column_name, seq_in_index, CASE collation WHEN 'A' THEN 'ASC' ELSE 'DESC' END AS direction, non_unique, (index_name == 'PRIMARY') as is_primary + FROM information_schema.statistics + WHERE table_schema = DATABASE() AND table_name = ? + ORDER BY index_name, seq_in_index`, + [tableName], + ); diff --git a/packages/sql-store/src/queries/list-tables.ts b/packages/mysql-store/src/queries/list-tables.ts similarity index 56% rename from packages/sql-store/src/queries/list-tables.ts rename to packages/mysql-store/src/queries/list-tables.ts index e29a81e..828fbbc 100644 --- a/packages/sql-store/src/queries/list-tables.ts +++ b/packages/mysql-store/src/queries/list-tables.ts @@ -3,13 +3,13 @@ import { SQLConnection } from './connection'; /** * The tables in the database. This is a view of the `information_schema.tables` table. */ -export interface SQLTable { +export interface MySQLTable { table_name: string; } export const listTables = async ( connection: SQLConnection, -): Promise => - connection.query( - `SELECT table_name AS name FROM information_schema.tables WHERE table_schema = DATABASE()`, +): Promise => + connection.query( + `SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE()`, ); diff --git a/packages/mysql-store/src/store.ts b/packages/mysql-store/src/store.ts index 11829c8..f51c672 100644 --- a/packages/mysql-store/src/store.ts +++ b/packages/mysql-store/src/store.ts @@ -1,29 +1,95 @@ import { Pool, PoolConfig, createPool } from 'mysql'; -import { SQLStore } from '@neuledge/sql-store'; +import { + Store, + StoreCollection, + StoreCollection_Slim, + StoreDeleteOptions, + StoreDescribeCollectionOptions, + StoreDocument, + StoreDropCollectionOptions, + StoreEnsureCollectionOptions, + StoreFindOptions, + StoreInsertOptions, + StoreInsertionResponse, + StoreList, + StoreMutationResponse, + StoreUpdateOptions, +} from '@neuledge/store'; +import { + SQLConnection, + dataTypeMap, + listTableColumns, + listIndexAttributes, + listTables, +} from './queries'; +import { getCollection, getStoreCollections } from '@neuledge/sql-store'; export type MySQLStoreOptions = PoolConfig; -export class MySQLStore extends SQLStore { +export class MySQLStore implements Store { private pool: Pool; + private connection: SQLConnection; constructor(options: MySQLStoreOptions) { - const pool = createPool(options); + this.pool = createPool(options); - super({ + this.connection = { query: (sql, values) => new Promise((resolve, reject) => - pool.query(sql, values, (error, results) => + this.pool.query(sql, values, (error, results) => error ? reject(error) : resolve(results), ), ), - }); - - this.pool = pool; + }; } + // connection methods + async close(): Promise { await new Promise((resolve, reject) => this.pool.end((error) => (error ? reject(error) : resolve())), ); } + + // store methods + + async listCollections(): Promise { + return getStoreCollections(listTables, this.connection); + } + + async describeCollection( + options: StoreDescribeCollectionOptions, + ): Promise { + return getCollection( + options, + listTableColumns, + listIndexAttributes, + dataTypeMap, + this.connection, + ); + } + + ensureCollection(options: StoreEnsureCollectionOptions): Promise { + throw new Error('Method not implemented.'); + } + + dropCollection(options: StoreDropCollectionOptions): Promise { + throw new Error('Method not implemented.'); + } + + find(options: StoreFindOptions): Promise> { + throw new Error('Method not implemented.'); + } + + insert(options: StoreInsertOptions): Promise { + throw new Error('Method not implemented.'); + } + + update(options: StoreUpdateOptions): Promise { + throw new Error('Method not implemented.'); + } + + delete(options: StoreDeleteOptions): Promise { + throw new Error('Method not implemented.'); + } } diff --git a/packages/postgresql-store/.npmignore b/packages/postgresql-store/.npmignore new file mode 100644 index 0000000..d2ee3b5 --- /dev/null +++ b/packages/postgresql-store/.npmignore @@ -0,0 +1,6 @@ +/* +!/dist/*.js +!/dist/*.js.map +!/dist/*.mjs +!/dist/*.mjs.map +!/dist/**/*.d.ts \ No newline at end of file diff --git a/packages/postgresql-store/README.md b/packages/postgresql-store/README.md new file mode 100644 index 0000000..f438fe2 --- /dev/null +++ b/packages/postgresql-store/README.md @@ -0,0 +1,30 @@ +# Neuledge PostgreSQL Store + +A store for [Neuledge Engine](https://neuledge.com) that uses [PostgreSQL](https://www.postgresql.org) database as a backend. + +## 📦 Installation + +```bash +npm install @neuledge/postgresql-store +``` + +## 🚀 Getting started + +```ts +import { PostgreSQLStore } from '@neuledge/postgresql-store'; + +const store = new PostgreSQLStore({ + uri: process.env.MYSQL_URI ?? 'mysql://localhost:3306', + database: process.env.MYSQL_DATABASE ?? 'my-database', +}); + +const engine = new Engine({ + store, +}); +``` + +For more information, please refer to the [main repository](https://github.com/neuledge/engine-js). + +## 📄 License + +Neuledge is [Apache 2.0 licensed](https://github.com/neuledge/engine-js/blob/main/LICENSE). diff --git a/packages/postgresql-store/package.json b/packages/postgresql-store/package.json new file mode 100644 index 0000000..9d64233 --- /dev/null +++ b/packages/postgresql-store/package.json @@ -0,0 +1,45 @@ +{ + "name": "@neuledge/postgresql-store", + "version": "0.0.0", + "deascription": "PostgreSQL store implementation for Neuledge Engine", + "keywords": [ + "neuledge", + "postgres", + "postgresql", + "store", + "database" + ], + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.js", + "exports": { + ".": { + "require": "./dist/index.js", + "import": "./dist/index.mjs", + "types": "./dist/index.d.ts" + } + }, + "repository": { + "type": "git", + "url": "https://github.com/neuledge/engine-js.git" + }, + "license": "Apache-2.0", + "publishConfig": { + "access": "public" + }, + "engines": { + "node": ">= 16" + }, + "scripts": { + "types": "rimraf --glob dist/*.{d.ts,d.ts.map} dist/**/*.{d.ts,d.ts.map} && tsc --emitDeclarationOnly && tsc-alias", + "build": "rimraf --glob dist/*.{js,js.map,mjs,mjs.map} && tsup", + "lint": "eslint . --ext \"js,jsx,ts,tsx,mjs,cjs\"", + "lint:strict": "yarn lint --max-warnings 0" + }, + "dependencies": { + "@neuledge/store": "^0.2.0", + "@neuledge/sql-store": "^0.0.0", + "@types/pg": "^8.6.6", + "pg": "^8.10.0" + } +} diff --git a/packages/postgresql-store/src/index.ts b/packages/postgresql-store/src/index.ts new file mode 100644 index 0000000..d406816 --- /dev/null +++ b/packages/postgresql-store/src/index.ts @@ -0,0 +1 @@ +export * from './store'; diff --git a/packages/postgresql-store/src/queries/connection.ts b/packages/postgresql-store/src/queries/connection.ts new file mode 100644 index 0000000..5176269 --- /dev/null +++ b/packages/postgresql-store/src/queries/connection.ts @@ -0,0 +1,3 @@ +export interface SQLConnection { + query(sql: string, params?: unknown[]): Promise; +} diff --git a/packages/postgresql-store/src/queries/index.ts b/packages/postgresql-store/src/queries/index.ts new file mode 100644 index 0000000..45599ce --- /dev/null +++ b/packages/postgresql-store/src/queries/index.ts @@ -0,0 +1,4 @@ +export * from './connection'; +export * from './list-tables'; +export * from './list-table-columns'; +export * from './list-table-statistics'; diff --git a/packages/postgresql-store/src/queries/list-table-columns.ts b/packages/postgresql-store/src/queries/list-table-columns.ts new file mode 100644 index 0000000..869e950 --- /dev/null +++ b/packages/postgresql-store/src/queries/list-table-columns.ts @@ -0,0 +1,77 @@ +import { StoreShapeType } from '@neuledge/store'; +import { SQLConnection } from './connection'; + +/** + * A table column from the information_schema.columns table. + */ +export interface PostgreSQLColumn { + column_name: string; + data_type: string; + character_maximum_length: number | null; + numeric_precision: number | null; + numeric_scale: number | null; + is_nullable: boolean; + is_auto_increment: boolean; +} + +export const listTableColumns = async ( + tableName: string, + connection: SQLConnection, +): Promise => + connection.query( + `SELECT column_name, data_type, character_maximum_length, numeric_precision, numeric_scale, (is_nullable = 'YES') as is_nullable, column_default LIKE 'nextval(%)' AS is_auto_increment + FROM information_schema.columns + WHERE table_catalog = current_database() AND table_schema = current_schema() AND table_name = ?`, + [tableName], + ); + +export const dataTypeMap: Record = { + character: 'string', + 'character varying': 'string', + 'double precision': 'number', + smallint: 'number', + real: 'number', + 'timestamp without time zone': 'date-time', + 'timestamp with time zone': 'date-time', + 'time without time zone': 'date-time', + 'time with time zone': 'date-time', + 'interval year to month': 'date-time', + 'interval day to second': 'date-time', + 'bit varying': 'binary', + bit: 'binary', + varbit: 'binary', + bytea: 'binary', + text: 'string', + json: 'json', + jsonb: 'json', + uuid: 'string', + xml: 'string', + cidr: 'string', + inet: 'string', + macaddr: 'string', + tsvector: 'string', + tsquery: 'string', + regconfig: 'string', + regdictionary: 'string', + regnamespace: 'string', + regoper: 'string', + regoperator: 'string', + regproc: 'string', + regprocedure: 'string', + regrole: 'string', + regtype: 'string', + int2: 'number', + int4: 'number', + int8: 'number', + float4: 'number', + float8: 'number', + bool: 'boolean', + date: 'date-time', + time: 'date-time', + timestamp: 'date-time', + timestamptz: 'date-time', + interval: 'date-time', + numeric: 'number', + decimal: 'number', + money: 'number', +}; diff --git a/packages/postgresql-store/src/queries/list-table-statistics.ts b/packages/postgresql-store/src/queries/list-table-statistics.ts new file mode 100644 index 0000000..150bb55 --- /dev/null +++ b/packages/postgresql-store/src/queries/list-table-statistics.ts @@ -0,0 +1,40 @@ +import { SQLConnection } from './connection'; + +/** + * A table statistic row from the information_schema.statistics table. + */ +export interface PostgreSQLIndexAttribute { + index_name: string; + column_name: string; + seq_in_index: number; + direction: 'ASC' | 'DESC'; + nulls: 'FIRST' | 'LAST'; + is_unique: boolean; + is_primary: boolean; +} + +export const listIndexAttributes = async ( + tableName: string, + connection: SQLConnection, +): Promise => + connection.query( + `SELECT + irel.relname AS index_name, + a.attname AS column_name, + c.ordinality as seq_in_index, + CASE o.option & 1 WHEN 1 THEN 'DESC' ELSE 'ASC' END AS direction, + CASE o.option & 2 WHEN 2 THEN 'FIRST' ELSE 'LAST' END AS nulls, + i.indisunique AS is_unique, + i.indisprimary AS is_primary + FROM pg_index AS i + JOIN pg_class AS trel ON trel.oid = i.indrelid + JOIN pg_namespace AS tnsp ON trel.relnamespace = tnsp.oid + JOIN pg_class AS irel ON irel.oid = i.indexrelid + CROSS JOIN LATERAL unnest (i.indkey) WITH ORDINALITY AS c (colnum, ordinality) + LEFT JOIN LATERAL unnest (i.indoption) WITH ORDINALITY AS o (option, ordinality) + ON c.ordinality = o.ordinality + JOIN pg_attribute AS a ON trel.oid = a.attrelid AND a.attnum = c.colnum + WHERE tnsp.nspname = current_schema() AND trel.relname = ? + ORDER BY index_name, seq_in_index`, + [tableName], + ); diff --git a/packages/postgresql-store/src/queries/list-tables.ts b/packages/postgresql-store/src/queries/list-tables.ts new file mode 100644 index 0000000..bb4e5c5 --- /dev/null +++ b/packages/postgresql-store/src/queries/list-tables.ts @@ -0,0 +1,15 @@ +import { SQLConnection } from './connection'; + +/** + * The tables in the database. This is a view of the `information_schema.tables` table. + */ +export interface PostgreSQLTable { + table_name: string; +} + +export const listTables = async ( + connection: SQLConnection, +): Promise => + connection.query( + `SELECT table_name FROM information_schema.tables WHERE table_catalog = current_database() AND table_schema = current_schema()`, + ); diff --git a/packages/sql-store/src/store.ts b/packages/postgresql-store/src/store.ts similarity index 61% rename from packages/sql-store/src/store.ts rename to packages/postgresql-store/src/store.ts index 48b168d..c51eb15 100644 --- a/packages/sql-store/src/store.ts +++ b/packages/postgresql-store/src/store.ts @@ -1,3 +1,11 @@ +import { Pool, PoolConfig } from 'pg'; +import { + SQLConnection, + dataTypeMap, + listTableColumns, + listIndexAttributes, + listTables, +} from './queries'; import { Store, StoreCollection, @@ -14,45 +22,41 @@ import { StoreMutationResponse, StoreUpdateOptions, } from '@neuledge/store'; -import { - SQLConnection, - listTableColumns, - listTableStatistics, - listTables, -} from './queries'; -import { - toStoreCollection_Slim, - toStoreField, - toStoreIndexes, -} from './mappers'; +import { getCollection, getStoreCollections } from '@neuledge/sql-store'; -export class SQLStore implements Store { - constructor(public readonly connection: SQLConnection) {} +export type PostgreSQLStoreOptions = PoolConfig; + +export class PostgreSQLStore implements Store { + private pool: Pool; + private connection: SQLConnection; + + constructor(options: PostgreSQLStoreOptions) { + this.pool = new Pool(options); + this.connection = this.pool; + } + + // connection methods + + async close(): Promise { + await this.pool.end(); + } + + // store methods async listCollections(): Promise { - const tables = await listTables(this.connection); - return tables.map((table) => toStoreCollection_Slim(table)); + return getStoreCollections(listTables, this.connection); } async describeCollection( options: StoreDescribeCollectionOptions, ): Promise { - const { name } = options.collection; - - const [columns, statistics] = await Promise.all([ - listTableColumns(this.connection, name), - listTableStatistics(this.connection, name), - ]); - - const fields = columns.map((column) => toStoreField(column)); - const indexes = toStoreIndexes(statistics); - - return { - name, - primaryKey: indexes[0], - indexes: Object.fromEntries(indexes.map((index) => [index.name, index])), - fields: Object.fromEntries(fields.map((field) => [field.name, field])), - }; + return getCollection( + options, + listTableColumns, + listIndexAttributes, + dataTypeMap, + this.connection, + ); } ensureCollection(options: StoreEnsureCollectionOptions): Promise { diff --git a/packages/postgresql-store/tsconfig.json b/packages/postgresql-store/tsconfig.json new file mode 100644 index 0000000..c67724d --- /dev/null +++ b/packages/postgresql-store/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "@neuledge/tsconfig/base.json", + "compilerOptions": { + "baseUrl": "src", + "rootDir": "src", + "outDir": "dist" + }, + "include": ["src"], + "exclude": ["node_modules", "**/__ignore__/**"] +} diff --git a/packages/postgresql-store/tsup.config.json b/packages/postgresql-store/tsup.config.json new file mode 100644 index 0000000..2f3a43d --- /dev/null +++ b/packages/postgresql-store/tsup.config.json @@ -0,0 +1,6 @@ +{ + "entry": ["src/index.ts"], + "format": ["esm", "cjs"], + "sourcemap": true, + "shims": true +} diff --git a/packages/sql-store/README.md b/packages/sql-store/README.md index 79302fe..932c199 100644 --- a/packages/sql-store/README.md +++ b/packages/sql-store/README.md @@ -12,19 +12,15 @@ npm install @neuledge/sql-store ## 🚀 Getting started +Import the util functions you need and use them to create your own store: + ```ts -import { SQLStore } from '@neuledge/mysql-store'; +import { Store } from '@neuledge/engine'; +import { ... } from '@neuledge/sql-store'; -// create a connection to your SQL database somehow -const connection = createPool({ +export class MyStore implements Store { // ... -}); - -const store = new SQLStore(connection); - -const engine = new Engine({ - store, -}); +} ``` ## 📄 License diff --git a/packages/sql-store/src/index.ts b/packages/sql-store/src/index.ts index d406816..ef390d6 100644 --- a/packages/sql-store/src/index.ts +++ b/packages/sql-store/src/index.ts @@ -1 +1,2 @@ -export * from './store'; +export * from './logic'; +export * from './mappers'; diff --git a/packages/sql-store/src/logic/collections.ts b/packages/sql-store/src/logic/collections.ts new file mode 100644 index 0000000..38abaec --- /dev/null +++ b/packages/sql-store/src/logic/collections.ts @@ -0,0 +1,115 @@ +import { + SQLColumn, + SQLIndexAttribute, + SQLIndexColumn, + SQLTable, + toStoreCollection_Slim, + toStoreField, + toStoreIndex, +} from '@/mappers'; +import { + StoreCollection, + StoreCollection_Slim, + StoreDescribeCollectionOptions, + StoreError, + StoreShapeType, +} from '@neuledge/store'; + +export const getStoreCollections = async < + A extends unknown[], + T extends SQLTable, +>( + listTables: (...args: A) => Promise, + ...params: A +): Promise => { + const tables = await listTables(...params); + return tables.map((table) => toStoreCollection_Slim(table)); +}; + +export const getCollection = async < + A extends unknown[], + C extends SQLColumn, + I extends SQLIndexAttribute & Omit, +>( + options: StoreDescribeCollectionOptions, + listTableColumns: (name: string, ...args: A) => Promise, + listIndexAttributes: (name: string, ...args: A) => Promise, + dataTypeMap: Record, + ...params: A +): Promise => { + const { name } = options.collection; + + const [columns, indexAttributes] = await Promise.all([ + listTableColumns(name, ...params), + listIndexAttributes(name, ...params), + ]); + + const columnMap = Object.fromEntries( + columns.map((column) => [column.column_name, column]), + ); + + const fields = Object.fromEntries( + columns.map((column) => [ + column.column_name, + toStoreField(dataTypeMap, column), + ]), + ); + + const indexColumns = groupIndexColumns(columnMap, indexAttributes); + + let primaryKey: string | undefined; + const indexes = Object.fromEntries( + indexColumns.map((columns) => { + const index = toStoreIndex(columns); + if (index.unique === 'primary') { + primaryKey = index.name; + } + + return [index.name, index]; + }), + ); + + if (!primaryKey) { + throw new StoreError( + StoreError.Code.INVALID_DATA, + `Primary key not found for collection "${name}"`, + ); + } + + return { + name, + primaryKey: indexes[primaryKey] as StoreCollection['primaryKey'], + indexes, + fields, + }; +}; + +const groupIndexColumns = < + C extends SQLColumn, + I extends SQLIndexAttribute & Omit, +>( + columnMap: Record, + indexAttributes: I[], +): SQLIndexColumn[][] => { + const groupMap: Record = {}; + + for (const statistic of indexAttributes) { + let group = groupMap[statistic.index_name]; + if (!group) { + group = []; + groupMap[statistic.index_name] = group; + } + + const column = columnMap[statistic.column_name]; + if (!column) { + throw new StoreError( + StoreError.Code.INVALID_DATA, + `Column "${statistic.column_name}" not found for index "${statistic.index_name}"`, + ); + } + + group.push({ ...column, ...statistic } as never); + } + + return Object.values(groupMap); +}; diff --git a/packages/sql-store/src/logic/index.ts b/packages/sql-store/src/logic/index.ts new file mode 100644 index 0000000..3eee1ab --- /dev/null +++ b/packages/sql-store/src/logic/index.ts @@ -0,0 +1 @@ +export * from './collections'; diff --git a/packages/sql-store/src/mappers/collection.ts b/packages/sql-store/src/mappers/collection.ts index dac9b35..1eae8f4 100644 --- a/packages/sql-store/src/mappers/collection.ts +++ b/packages/sql-store/src/mappers/collection.ts @@ -1,5 +1,8 @@ import { StoreCollection_Slim } from '@neuledge/store'; -import { SQLTable } from '@/queries'; + +export interface SQLTable { + table_name: string; +} export const toStoreCollection_Slim = ( table: SQLTable, diff --git a/packages/sql-store/src/mappers/field.ts b/packages/sql-store/src/mappers/field.ts index 31992f6..c28b8f5 100644 --- a/packages/sql-store/src/mappers/field.ts +++ b/packages/sql-store/src/mappers/field.ts @@ -1,28 +1,18 @@ import { StoreError, StoreField, StoreShapeType } from '@neuledge/store'; -import { SQLColumn } from '@/queries'; -/** - * Map the SQL data types to the corresponding StoreShapeType - */ -const dataTypeMap: Record = { - varchar: 'string', - char: 'string', - text: 'string', - numeric: 'number', - decimal: 'number', - float: 'number', - double: 'number', - integer: 'number', - bigint: 'number', - boolean: 'boolean', - bytea: 'binary', - timestamp: 'date-time', - timestamptz: 'date-time', - json: 'json', - jsonb: 'json', -}; +export interface SQLColumn { + column_name: string; + data_type: string; + character_maximum_length: number | null; + numeric_precision: number | null; + numeric_scale: number | null; + is_nullable: boolean | 1 | 0; +} -export const toStoreField = (column: SQLColumn): StoreField => { +export const toStoreField = ( + dataTypeMap: Record, + column: SQLColumn, +): StoreField => { const type = dataTypeMap[column.data_type]; if (!type) { throw new StoreError( @@ -34,7 +24,7 @@ export const toStoreField = (column: SQLColumn): StoreField => { return { name: column.column_name, type, - nullable: column.is_nullable === 'YES', + nullable: !!column.is_nullable, size: column.character_maximum_length, precision: column.numeric_precision, scale: column.numeric_scale, diff --git a/packages/sql-store/src/mappers/store-index.test.ts b/packages/sql-store/src/mappers/store-index.test.ts index 1904adb..0129e5d 100644 --- a/packages/sql-store/src/mappers/store-index.test.ts +++ b/packages/sql-store/src/mappers/store-index.test.ts @@ -1,27 +1,28 @@ -import { toStoreIndexes } from './store-index'; +import { toStoreIndex } from './store-index'; describe('mappers/store-index', () => { - describe('toStoreIndexes()', () => { + describe('toStoreIndex()', () => { it('should convert a single primary index', () => { expect( - toStoreIndexes([ + toStoreIndex([ { - index_name: 'PRIMARY', - non_unique: 0, + index_name: 'id_index', column_name: 'field_name', seq_in_index: 1, - collation: 'A', + direction: 'ASC', + is_unique: true, + is_primary: true, + is_auto_increment: true, }, ]), - ).toEqual([ - { - name: 'PRIMARY', - unique: 'primary', - fields: { - field_name: { sort: 'asc' }, - }, + ).toEqual({ + name: 'id_index', + unique: 'primary', + auto: 'increment', + fields: { + field_name: { sort: 'asc' }, }, - ]); + }); }); }); }); diff --git a/packages/sql-store/src/mappers/store-index.ts b/packages/sql-store/src/mappers/store-index.ts index 7cad662..d69b25f 100644 --- a/packages/sql-store/src/mappers/store-index.ts +++ b/packages/sql-store/src/mappers/store-index.ts @@ -1,76 +1,45 @@ -import { SQLStatistic } from '@/queries'; -import { StoreError, StoreIndex, StorePrimaryKey } from '@neuledge/store'; - -export const toStoreIndexes = ( - statistics: SQLStatistic[], -): [primary: StorePrimaryKey, ...indexes: StoreIndex[]] => { - const indexStatistics = groupTableStatistics(statistics); - - let primaryKey: StorePrimaryKey | undefined; - const indexes: StoreIndex[] = []; - - for (const indexColumn of indexStatistics) { - const index = toStoreIndex(indexColumn); - - if (index.unique === 'primary') { - primaryKey = index as StorePrimaryKey; - } else { - indexes.push(index); - } - } - - if (!primaryKey) { - throw new StoreError( - StoreError.Code.INVALID_DATA, - `Primary key not found for collection "${name}"`, - ); - } - - return [primaryKey, ...indexes]; -}; - -const toStoreIndex = ( - indexStatistics: SQLStatistic[], +import { StoreIndex, StorePrimaryKey } from '@neuledge/store'; + +export interface SQLIndexAttribute { + index_name: string; + column_name: string; + seq_in_index: number; + direction: 'ASC' | 'DESC'; + is_unique: boolean | 1 | 0; +} + +export interface SQLIndexColumn extends SQLIndexAttribute { + is_primary: boolean | 1 | 0; + is_auto_increment: boolean | 1 | 0; +} + +export const toStoreIndex = ( + indexColumns: SQLIndexColumn[], ): StoreIndex | StorePrimaryKey => { - const { index_name, non_unique, column_extra } = indexStatistics[0]; + const { index_name, is_unique, is_primary, is_auto_increment } = + indexColumns[0]; const index: StoreIndex | StorePrimaryKey = { name: index_name, - unique: !non_unique, + unique: !!is_unique, fields: {}, }; - if (index.unique && index_name === 'PRIMARY') { + if (index.unique && is_primary) { index.unique = 'primary'; - if (column_extra === 'auto_increment') { + if (is_auto_increment) { (index as StorePrimaryKey).auto = 'increment'; } } - indexStatistics.sort((a, b) => a.seq_in_index - b.seq_in_index); + indexColumns.sort((a, b) => a.seq_in_index - b.seq_in_index); - for (const statistic of indexStatistics) { + for (const statistic of indexColumns) { index.fields[statistic.column_name] = { - sort: statistic.collation === 'A' ? 'asc' : 'desc', + sort: statistic.direction === 'ASC' ? 'asc' : 'desc', }; } return index; }; - -const groupTableStatistics = (statistics: SQLStatistic[]): SQLStatistic[][] => { - const groupMap: Record = {}; - - for (const statistic of statistics) { - let group = groupMap[statistic.index_name]; - if (!group) { - group = []; - groupMap[statistic.index_name] = group; - } - - group.push(statistic); - } - - return Object.values(groupMap); -}; diff --git a/packages/sql-store/src/queries/list-table-columns.ts b/packages/sql-store/src/queries/list-table-columns.ts deleted file mode 100644 index a52d591..0000000 --- a/packages/sql-store/src/queries/list-table-columns.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { SQLConnection } from './connection'; - -/** - * A table column from the information_schema.columns table. - */ -export interface SQLColumn { - column_name: string; - data_type: string; - character_maximum_length?: number | null; - numeric_precision?: number | null; - numeric_scale?: number | null; - is_nullable: 'YES' | 'NO'; -} - -export const listTableColumns = async ( - connection: SQLConnection, - tableName: string, -): Promise => - connection.query( - `SELECT column_name, data_type, character_maximum_length, numeric_precision, numeric_scale, is_nullable -FROM information_schema.columns -WHERE table_name = ? AND table_schema = DATABASE()`, - [tableName], - ); diff --git a/packages/sql-store/src/queries/list-table-statistics.ts b/packages/sql-store/src/queries/list-table-statistics.ts deleted file mode 100644 index f47560c..0000000 --- a/packages/sql-store/src/queries/list-table-statistics.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { SQLConnection } from './connection'; - -/** - * A table statistic row from the information_schema.statistics table. - */ -export interface SQLStatistic { - index_name: string; - column_name: string; - seq_in_index: number; - collation: 'A' | 'D'; - non_unique: number; - column_extra?: string; -} - -export const listTableStatistics = async ( - connection: SQLConnection, - tableName: string, -): Promise => - connection.query( - `SELECT index_name, column_name, seq_in_index, collation, non_unique, extra AS column_extra - FROM information_schema.statistics INNER JOIN information_schema.columns USING (table_schema, table_name, column_name) - WHERE table_schema = DATABASE() AND table_name = ? - ORDER BY index_name, seq_in_index`, - [tableName], - ); diff --git a/yarn.lock b/yarn.lock index b0c2102..c83e1c3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2007,6 +2007,15 @@ resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA== +"@types/pg@^8.6.6": + version "8.6.6" + resolved "https://registry.yarnpkg.com/@types/pg/-/pg-8.6.6.tgz#21cdf873a3e345a6e78f394677e3b3b1b543cb80" + integrity sha512-O2xNmXebtwVekJDD+02udOncjVcMZQuTEQEMpKJ0ZRf5E7/9JJX3izhKUcUifBkyKpljyUM6BTgy2trmviKlpw== + dependencies: + "@types/node" "*" + pg-protocol "*" + pg-types "^2.2.0" + "@types/pluralize@^0.0.29": version "0.0.29" resolved "https://registry.yarnpkg.com/@types/pluralize/-/pluralize-0.0.29.tgz#6ffa33ed1fc8813c469b859681d09707eb40d03c" @@ -2655,6 +2664,11 @@ buffer-from@^1.0.0: resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== +buffer-writer@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-2.0.0.tgz#ce7eb81a38f7829db09c873f2fbb792c0c98ec04" + integrity sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw== + buffer@^5.5.0: version "5.7.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" @@ -6021,6 +6035,11 @@ p-try@^2.0.0: resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== +packet-reader@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-1.0.0.tgz#9238e5480dedabacfe1fe3f2771063f164157d74" + integrity sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ== + param-case@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/param-case/-/param-case-3.0.4.tgz#7d17fe4aa12bde34d4a77d91acfb6219caad01c5" @@ -6143,6 +6162,57 @@ pend@~1.2.0: resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" integrity sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg== +pg-connection-string@^2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34" + integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ== + +pg-int8@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" + integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw== + +pg-pool@^3.6.0: + version "3.6.0" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.0.tgz#3190df3e4747a0d23e5e9e8045bcd99bda0a712e" + integrity sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ== + +pg-protocol@*, pg-protocol@^1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.0.tgz#4c91613c0315349363af2084608db843502f8833" + integrity sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q== + +pg-types@^2.1.0, pg-types@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3" + integrity sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA== + dependencies: + pg-int8 "1.0.1" + postgres-array "~2.0.0" + postgres-bytea "~1.0.0" + postgres-date "~1.0.4" + postgres-interval "^1.1.0" + +pg@^8.10.0: + version "8.10.0" + resolved "https://registry.yarnpkg.com/pg/-/pg-8.10.0.tgz#5b8379c9b4a36451d110fc8cd98fc325fe62ad24" + integrity sha512-ke7o7qSTMb47iwzOSaZMfeR7xToFdkE71ifIipOAAaLIM0DYzfOAXlgFFmYUIE2BcJtvnVlGCID84ZzCegE8CQ== + dependencies: + buffer-writer "2.0.0" + packet-reader "1.0.0" + pg-connection-string "^2.5.0" + pg-pool "^3.6.0" + pg-protocol "^1.6.0" + pg-types "^2.1.0" + pgpass "1.x" + +pgpass@1.x: + version "1.0.5" + resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d" + integrity sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug== + dependencies: + split2 "^4.1.0" + picocolors@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" @@ -6220,6 +6290,28 @@ postcss-load-config@^3.0.1: lilconfig "^2.0.5" yaml "^1.10.2" +postgres-array@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-2.0.0.tgz#48f8fce054fbc69671999329b8834b772652d82e" + integrity sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA== + +postgres-bytea@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/postgres-bytea/-/postgres-bytea-1.0.0.tgz#027b533c0aa890e26d172d47cf9ccecc521acd35" + integrity sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w== + +postgres-date@~1.0.4: + version "1.0.7" + resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.7.tgz#51bc086006005e5061c591cee727f2531bf641a8" + integrity sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q== + +postgres-interval@^1.1.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.2.0.tgz#b460c82cb1587507788819a06aa0fffdb3544695" + integrity sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ== + dependencies: + xtend "^4.0.0" + prebuild-install@^7.0.1: version "7.1.1" resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-7.1.1.tgz#de97d5b34a70a0c81334fd24641f2a1702352e45" @@ -6960,6 +7052,11 @@ split2@^4.0.0: resolved "https://registry.yarnpkg.com/split2/-/split2-4.1.0.tgz#101907a24370f85bb782f08adaabe4e281ecf809" integrity sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ== +split2@^4.1.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/split2/-/split2-4.2.0.tgz#c9c5920904d148bab0b9f67145f245a86aadbfa4" + integrity sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg== + sponge-case@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/sponge-case/-/sponge-case-1.0.1.tgz#260833b86453883d974f84854cdb63aecc5aef4c" @@ -7423,47 +7520,47 @@ tunnel@0.0.6: resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c" integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg== -turbo-darwin-64@1.8.5: - version "1.8.5" - resolved "https://registry.yarnpkg.com/turbo-darwin-64/-/turbo-darwin-64-1.8.5.tgz#6fdd2e9e2b8afead04e17380fc222863794e6007" - integrity sha512-CAYh56bzeHfnh7jTm03r29bh8p5a/EjQo1Id5yLUH7hS7msTau/+YpxJWPodLbN0UQsUYivUqHQkglJ+eMJ7xA== - -turbo-darwin-arm64@1.8.5: - version "1.8.5" - resolved "https://registry.yarnpkg.com/turbo-darwin-arm64/-/turbo-darwin-arm64-1.8.5.tgz#8d72d67a87b6d247565de79477b0c7aed6a83c2b" - integrity sha512-R3jCPOv+lu3dcvMhj8b/Defv6dyUwX6W+tbX7d6YUCA46Plf/bGCQ8+MSbxmr/4E1GyGOVFsn1wRfiYk0us/Dg== - -turbo-linux-64@1.8.5: - version "1.8.5" - resolved "https://registry.yarnpkg.com/turbo-linux-64/-/turbo-linux-64-1.8.5.tgz#a080015aa1c725604637a743a5b878d100aaf88b" - integrity sha512-YRc/KNRZeUVvth11UO4SDQZR2IqGgl9MSsbzqoHuFz4B4Q5QXH7onHogv9aXWE/BZBBbcrSBTlwBSG0Gg+J8hg== - -turbo-linux-arm64@1.8.5: - version "1.8.5" - resolved "https://registry.yarnpkg.com/turbo-linux-arm64/-/turbo-linux-arm64-1.8.5.tgz#96fa915f10e81a16eccf74e122e96fcba4e132e0" - integrity sha512-8exVZb7XBl/V3gHSweuUyG2D9IzfWqwLvlXoeLWlVYSj61Ajgdv+WU7lvUmx+H2s+sSKqmIFmewA5Lw6YY37sg== - -turbo-windows-64@1.8.5: - version "1.8.5" - resolved "https://registry.yarnpkg.com/turbo-windows-64/-/turbo-windows-64-1.8.5.tgz#14ca1a577e982c34fd606fa6499eaf4fea0eca36" - integrity sha512-fA8PU5ZNoFnQkapG06WiEqfsVQ5wbIPkIqTwUsd/M2Lp+KgxE79SQbuEI+2vQ9SmwM5qoMi515IPjgvXAJXgCw== - -turbo-windows-arm64@1.8.5: - version "1.8.5" - resolved "https://registry.yarnpkg.com/turbo-windows-arm64/-/turbo-windows-arm64-1.8.5.tgz#286c7aacd8c9e2a8a0f5ab767268aadc8f6c7955" - integrity sha512-SW/NvIdhckLsAWjU/iqBbCB0S8kXupKscUK3kEW1DZIr3MYcP/yIuaE/IdPuqcoF3VP0I3TLD4VTYCCKAo3tKA== - -turbo@^1.8.3: - version "1.8.5" - resolved "https://registry.yarnpkg.com/turbo/-/turbo-1.8.5.tgz#933413257783ede75471b8ebf2435ebc7be50ad7" - integrity sha512-UBnH2wIFb5g6OQCk8f34Ud15ZXV4xEMmugeDJTU5Ur2LpVRsNEny0isSCYdb3Iu3howoNyyXmtpaxWsAwNYkkg== +turbo-darwin-64@1.8.8: + version "1.8.8" + resolved "https://registry.yarnpkg.com/turbo-darwin-64/-/turbo-darwin-64-1.8.8.tgz#f72b1b6275415b17238f450032c8ef5e5fc71777" + integrity sha512-18cSeIm7aeEvIxGyq7PVoFyEnPpWDM/0CpZvXKHpQ6qMTkfNt517qVqUTAwsIYqNS8xazcKAqkNbvU1V49n65Q== + +turbo-darwin-arm64@1.8.8: + version "1.8.8" + resolved "https://registry.yarnpkg.com/turbo-darwin-arm64/-/turbo-darwin-arm64-1.8.8.tgz#8ec78848e0d5978fd732b3588a1b406fdb978839" + integrity sha512-ruGRI9nHxojIGLQv1TPgN7ud4HO4V8mFBwSgO6oDoZTNuk5ybWybItGR+yu6fni5vJoyMHXOYA2srnxvOc7hjQ== + +turbo-linux-64@1.8.8: + version "1.8.8" + resolved "https://registry.yarnpkg.com/turbo-linux-64/-/turbo-linux-64-1.8.8.tgz#b1f707b23bc6e22b2894dd8063fc2fa4dbb6ffb9" + integrity sha512-N/GkHTHeIQogXB1/6ZWfxHx+ubYeb8Jlq3b/3jnU4zLucpZzTQ8XkXIAfJG/TL3Q7ON7xQ8yGOyGLhHL7MpFRg== + +turbo-linux-arm64@1.8.8: + version "1.8.8" + resolved "https://registry.yarnpkg.com/turbo-linux-arm64/-/turbo-linux-arm64-1.8.8.tgz#34575bdffd2af8c835d9ba3dd9e3a83e0d31dac9" + integrity sha512-hKqLbBHgUkYf2Ww8uBL9UYdBFQ5677a7QXdsFhONXoACbDUPvpK4BKlz3NN7G4NZ+g9dGju+OJJjQP0VXRHb5w== + +turbo-windows-64@1.8.8: + version "1.8.8" + resolved "https://registry.yarnpkg.com/turbo-windows-64/-/turbo-windows-64-1.8.8.tgz#73f67969d54269c95cbf7f082e22c20368aedddc" + integrity sha512-2ndjDJyzkNslXxLt+PQuU21AHJWc8f6MnLypXy3KsN4EyX/uKKGZS0QJWz27PeHg0JS75PVvhfFV+L9t9i+Yyg== + +turbo-windows-arm64@1.8.8: + version "1.8.8" + resolved "https://registry.yarnpkg.com/turbo-windows-arm64/-/turbo-windows-arm64-1.8.8.tgz#c80b9a170adf6ee028e9dcae45b07755af83f3f2" + integrity sha512-xCA3oxgmW9OMqpI34AAmKfOVsfDljhD5YBwgs0ZDsn5h3kCHhC4x9W5dDk1oyQ4F5EXSH3xVym5/xl1J6WRpUg== + +turbo@^1.8.8: + version "1.8.8" + resolved "https://registry.yarnpkg.com/turbo/-/turbo-1.8.8.tgz#8bb331e3f0bd9656b20321339e91e899ad499012" + integrity sha512-qYJ5NjoTX+591/x09KgsDOPVDUJfU9GoS+6jszQQlLp1AHrf1wRFA3Yps8U+/HTG03q0M4qouOfOLtRQP4QypA== optionalDependencies: - turbo-darwin-64 "1.8.5" - turbo-darwin-arm64 "1.8.5" - turbo-linux-64 "1.8.5" - turbo-linux-arm64 "1.8.5" - turbo-windows-64 "1.8.5" - turbo-windows-arm64 "1.8.5" + turbo-darwin-64 "1.8.8" + turbo-darwin-arm64 "1.8.8" + turbo-linux-64 "1.8.8" + turbo-linux-arm64 "1.8.8" + turbo-windows-64 "1.8.8" + turbo-windows-arm64 "1.8.8" type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" @@ -7831,7 +7928,7 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== -xtend@^4.0.2: +xtend@^4.0.0, xtend@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== From d88f010b78e5a7f0e3c8a56dd188db280ca7983d Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Sun, 2 Apr 2023 19:53:34 +0300 Subject: [PATCH 06/24] supports store dropCollection --- packages/mysql-store/src/queries/index.ts | 1 - .../src/queries/list-table-columns.ts | 2 +- .../src/queries/list-table-statistics.ts | 2 +- .../mysql-store/src/queries/list-tables.ts | 2 +- packages/mysql-store/src/store.ts | 22 +++++++++++-------- .../src/queries/connection.ts | 3 --- .../postgresql-store/src/queries/index.ts | 1 - .../src/queries/list-table-columns.ts | 2 +- .../src/queries/list-table-statistics.ts | 2 +- .../src/queries/list-tables.ts | 2 +- packages/postgresql-store/src/store.ts | 22 +++++++++++-------- packages/sql-store/src/index.ts | 1 + packages/sql-store/src/logic/collections.ts | 9 ++++++++ .../src/queries/connection.ts | 0 packages/sql-store/src/queries/drop-table.ts | 6 +++++ packages/sql-store/src/queries/index.ts | 2 ++ 16 files changed, 50 insertions(+), 29 deletions(-) delete mode 100644 packages/postgresql-store/src/queries/connection.ts rename packages/{mysql-store => sql-store}/src/queries/connection.ts (100%) create mode 100644 packages/sql-store/src/queries/drop-table.ts create mode 100644 packages/sql-store/src/queries/index.ts diff --git a/packages/mysql-store/src/queries/index.ts b/packages/mysql-store/src/queries/index.ts index 45599ce..78749e0 100644 --- a/packages/mysql-store/src/queries/index.ts +++ b/packages/mysql-store/src/queries/index.ts @@ -1,4 +1,3 @@ -export * from './connection'; export * from './list-tables'; export * from './list-table-columns'; export * from './list-table-statistics'; diff --git a/packages/mysql-store/src/queries/list-table-columns.ts b/packages/mysql-store/src/queries/list-table-columns.ts index 6c3ade8..3e97bba 100644 --- a/packages/mysql-store/src/queries/list-table-columns.ts +++ b/packages/mysql-store/src/queries/list-table-columns.ts @@ -1,5 +1,5 @@ import { StoreShapeType } from '@neuledge/store'; -import { SQLConnection } from './connection'; +import { SQLConnection } from '@neuledge/sql-store'; /** * A table column from the information_schema.columns table. diff --git a/packages/mysql-store/src/queries/list-table-statistics.ts b/packages/mysql-store/src/queries/list-table-statistics.ts index 5f3ad45..fb1509e 100644 --- a/packages/mysql-store/src/queries/list-table-statistics.ts +++ b/packages/mysql-store/src/queries/list-table-statistics.ts @@ -1,4 +1,4 @@ -import { SQLConnection } from './connection'; +import { SQLConnection } from '@neuledge/sql-store'; /** * A table statistic row from the information_schema.statistics table. diff --git a/packages/mysql-store/src/queries/list-tables.ts b/packages/mysql-store/src/queries/list-tables.ts index 828fbbc..23fdb8f 100644 --- a/packages/mysql-store/src/queries/list-tables.ts +++ b/packages/mysql-store/src/queries/list-tables.ts @@ -1,4 +1,4 @@ -import { SQLConnection } from './connection'; +import { SQLConnection } from '@neuledge/sql-store'; /** * The tables in the database. This is a view of the `information_schema.tables` table. diff --git a/packages/mysql-store/src/store.ts b/packages/mysql-store/src/store.ts index f51c672..77ee271 100644 --- a/packages/mysql-store/src/store.ts +++ b/packages/mysql-store/src/store.ts @@ -16,13 +16,17 @@ import { StoreUpdateOptions, } from '@neuledge/store'; import { - SQLConnection, dataTypeMap, listTableColumns, listIndexAttributes, listTables, } from './queries'; -import { getCollection, getStoreCollections } from '@neuledge/sql-store'; +import { + SQLConnection, + dropCollection, + getCollection, + getStoreCollections, +} from '@neuledge/sql-store'; export type MySQLStoreOptions = PoolConfig; @@ -69,27 +73,27 @@ export class MySQLStore implements Store { ); } - ensureCollection(options: StoreEnsureCollectionOptions): Promise { + async ensureCollection(options: StoreEnsureCollectionOptions): Promise { throw new Error('Method not implemented.'); } - dropCollection(options: StoreDropCollectionOptions): Promise { - throw new Error('Method not implemented.'); + async dropCollection(options: StoreDropCollectionOptions): Promise { + return dropCollection(options, this.connection); } - find(options: StoreFindOptions): Promise> { + async find(options: StoreFindOptions): Promise> { throw new Error('Method not implemented.'); } - insert(options: StoreInsertOptions): Promise { + async insert(options: StoreInsertOptions): Promise { throw new Error('Method not implemented.'); } - update(options: StoreUpdateOptions): Promise { + async update(options: StoreUpdateOptions): Promise { throw new Error('Method not implemented.'); } - delete(options: StoreDeleteOptions): Promise { + async delete(options: StoreDeleteOptions): Promise { throw new Error('Method not implemented.'); } } diff --git a/packages/postgresql-store/src/queries/connection.ts b/packages/postgresql-store/src/queries/connection.ts deleted file mode 100644 index 5176269..0000000 --- a/packages/postgresql-store/src/queries/connection.ts +++ /dev/null @@ -1,3 +0,0 @@ -export interface SQLConnection { - query(sql: string, params?: unknown[]): Promise; -} diff --git a/packages/postgresql-store/src/queries/index.ts b/packages/postgresql-store/src/queries/index.ts index 45599ce..78749e0 100644 --- a/packages/postgresql-store/src/queries/index.ts +++ b/packages/postgresql-store/src/queries/index.ts @@ -1,4 +1,3 @@ -export * from './connection'; export * from './list-tables'; export * from './list-table-columns'; export * from './list-table-statistics'; diff --git a/packages/postgresql-store/src/queries/list-table-columns.ts b/packages/postgresql-store/src/queries/list-table-columns.ts index 869e950..7453fea 100644 --- a/packages/postgresql-store/src/queries/list-table-columns.ts +++ b/packages/postgresql-store/src/queries/list-table-columns.ts @@ -1,5 +1,5 @@ import { StoreShapeType } from '@neuledge/store'; -import { SQLConnection } from './connection'; +import { SQLConnection } from '@neuledge/sql-store'; /** * A table column from the information_schema.columns table. diff --git a/packages/postgresql-store/src/queries/list-table-statistics.ts b/packages/postgresql-store/src/queries/list-table-statistics.ts index 150bb55..325f8b3 100644 --- a/packages/postgresql-store/src/queries/list-table-statistics.ts +++ b/packages/postgresql-store/src/queries/list-table-statistics.ts @@ -1,4 +1,4 @@ -import { SQLConnection } from './connection'; +import { SQLConnection } from '@neuledge/sql-store'; /** * A table statistic row from the information_schema.statistics table. diff --git a/packages/postgresql-store/src/queries/list-tables.ts b/packages/postgresql-store/src/queries/list-tables.ts index bb4e5c5..933bc0e 100644 --- a/packages/postgresql-store/src/queries/list-tables.ts +++ b/packages/postgresql-store/src/queries/list-tables.ts @@ -1,4 +1,4 @@ -import { SQLConnection } from './connection'; +import { SQLConnection } from '@neuledge/sql-store'; /** * The tables in the database. This is a view of the `information_schema.tables` table. diff --git a/packages/postgresql-store/src/store.ts b/packages/postgresql-store/src/store.ts index c51eb15..17649da 100644 --- a/packages/postgresql-store/src/store.ts +++ b/packages/postgresql-store/src/store.ts @@ -1,6 +1,5 @@ import { Pool, PoolConfig } from 'pg'; import { - SQLConnection, dataTypeMap, listTableColumns, listIndexAttributes, @@ -22,7 +21,12 @@ import { StoreMutationResponse, StoreUpdateOptions, } from '@neuledge/store'; -import { getCollection, getStoreCollections } from '@neuledge/sql-store'; +import { + SQLConnection, + dropCollection, + getCollection, + getStoreCollections, +} from '@neuledge/sql-store'; export type PostgreSQLStoreOptions = PoolConfig; @@ -59,27 +63,27 @@ export class PostgreSQLStore implements Store { ); } - ensureCollection(options: StoreEnsureCollectionOptions): Promise { + async ensureCollection(options: StoreEnsureCollectionOptions): Promise { throw new Error('Method not implemented.'); } - dropCollection(options: StoreDropCollectionOptions): Promise { - throw new Error('Method not implemented.'); + async dropCollection(options: StoreDropCollectionOptions): Promise { + return dropCollection(options, this.connection); } - find(options: StoreFindOptions): Promise> { + async find(options: StoreFindOptions): Promise> { throw new Error('Method not implemented.'); } - insert(options: StoreInsertOptions): Promise { + async insert(options: StoreInsertOptions): Promise { throw new Error('Method not implemented.'); } - update(options: StoreUpdateOptions): Promise { + async update(options: StoreUpdateOptions): Promise { throw new Error('Method not implemented.'); } - delete(options: StoreDeleteOptions): Promise { + async delete(options: StoreDeleteOptions): Promise { throw new Error('Method not implemented.'); } } diff --git a/packages/sql-store/src/index.ts b/packages/sql-store/src/index.ts index ef390d6..d941abe 100644 --- a/packages/sql-store/src/index.ts +++ b/packages/sql-store/src/index.ts @@ -1,2 +1,3 @@ export * from './logic'; export * from './mappers'; +export * from './queries'; diff --git a/packages/sql-store/src/logic/collections.ts b/packages/sql-store/src/logic/collections.ts index 38abaec..87abb6f 100644 --- a/packages/sql-store/src/logic/collections.ts +++ b/packages/sql-store/src/logic/collections.ts @@ -7,10 +7,12 @@ import { toStoreField, toStoreIndex, } from '@/mappers'; +import { SQLConnection, dropTableIfExists } from '@/queries'; import { StoreCollection, StoreCollection_Slim, StoreDescribeCollectionOptions, + StoreDropCollectionOptions, StoreError, StoreShapeType, } from '@neuledge/store'; @@ -113,3 +115,10 @@ const groupIndexColumns = < return Object.values(groupMap); }; + +export const dropCollection = async ( + options: StoreDropCollectionOptions, + connection: SQLConnection, +): Promise => { + await dropTableIfExists(connection, options.collection.name); +}; diff --git a/packages/mysql-store/src/queries/connection.ts b/packages/sql-store/src/queries/connection.ts similarity index 100% rename from packages/mysql-store/src/queries/connection.ts rename to packages/sql-store/src/queries/connection.ts diff --git a/packages/sql-store/src/queries/drop-table.ts b/packages/sql-store/src/queries/drop-table.ts new file mode 100644 index 0000000..a6a0689 --- /dev/null +++ b/packages/sql-store/src/queries/drop-table.ts @@ -0,0 +1,6 @@ +import { SQLConnection } from './connection'; + +export const dropTableIfExists = async ( + connection: SQLConnection, + tableName: string, +): Promise => connection.query(`DROP TABLE IF EXISTS ?`, [tableName]); diff --git a/packages/sql-store/src/queries/index.ts b/packages/sql-store/src/queries/index.ts new file mode 100644 index 0000000..b2a91d3 --- /dev/null +++ b/packages/sql-store/src/queries/index.ts @@ -0,0 +1,2 @@ +export * from './connection'; +export * from './drop-table'; From 7d5b2b91da155bd7acb1513ec9918b85fe516df2 Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Thu, 13 Apr 2023 16:10:50 +0300 Subject: [PATCH 07/24] ensure table --- package.json | 2 +- .../mysql-store/src/queries/add-column.ts | 8 ++ packages/mysql-store/src/queries/add-index.ts | 17 ++++ .../mysql-store/src/queries/create-table.ts | 16 +++ packages/mysql-store/src/queries/index.ts | 3 + packages/mysql-store/src/store.ts | 25 +++-- .../src/queries/add-column.ts | 8 ++ .../postgresql-store/src/queries/add-index.ts | 15 +++ .../src/queries/create-table.ts | 15 +++ .../src/queries/drop-index.ts | 11 +++ .../postgresql-store/src/queries/index.ts | 4 + packages/postgresql-store/src/store.test.ts | 31 ++++++ packages/postgresql-store/src/store.ts | 41 +++++--- packages/sql-store/package.json | 3 +- .../describe.ts} | 53 +++++----- .../sql-store/src/logic/collections/drop.ts | 9 ++ .../sql-store/src/logic/collections/ensure.ts | 98 +++++++++++++++++++ .../sql-store/src/logic/collections/index.ts | 4 + .../sql-store/src/logic/collections/list.ts | 15 +++ packages/sql-store/src/queries/drop-column.ts | 9 ++ packages/sql-store/src/queries/drop-index.ts | 9 ++ .../sql-store/src/queries/index-columns.ts | 6 ++ packages/sql-store/src/queries/index.ts | 3 + yarn.lock | 80 +++++++-------- 24 files changed, 393 insertions(+), 92 deletions(-) create mode 100644 packages/mysql-store/src/queries/add-column.ts create mode 100644 packages/mysql-store/src/queries/add-index.ts create mode 100644 packages/mysql-store/src/queries/create-table.ts create mode 100644 packages/postgresql-store/src/queries/add-column.ts create mode 100644 packages/postgresql-store/src/queries/add-index.ts create mode 100644 packages/postgresql-store/src/queries/create-table.ts create mode 100644 packages/postgresql-store/src/queries/drop-index.ts create mode 100644 packages/postgresql-store/src/store.test.ts rename packages/sql-store/src/logic/{collections.ts => collections/describe.ts} (69%) create mode 100644 packages/sql-store/src/logic/collections/drop.ts create mode 100644 packages/sql-store/src/logic/collections/ensure.ts create mode 100644 packages/sql-store/src/logic/collections/index.ts create mode 100644 packages/sql-store/src/logic/collections/list.ts create mode 100644 packages/sql-store/src/queries/drop-column.ts create mode 100644 packages/sql-store/src/queries/drop-index.ts create mode 100644 packages/sql-store/src/queries/index-columns.ts diff --git a/package.json b/package.json index 58ebd5f..9ff0097 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "ts-node": "^10.9.1", "tsc-alias": "^1.8.5", "tsup": "^6.6.3", - "turbo": "^1.8.8", + "turbo": "^1.9.1", "typescript": "^5.0.2" } } diff --git a/packages/mysql-store/src/queries/add-column.ts b/packages/mysql-store/src/queries/add-column.ts new file mode 100644 index 0000000..0526579 --- /dev/null +++ b/packages/mysql-store/src/queries/add-column.ts @@ -0,0 +1,8 @@ +import { SQLConnection } from '@neuledge/sql-store'; +import { StoreField } from '@neuledge/store'; + +export const addColumn = async ( + tableName: string, + field: StoreField, + connection: SQLConnection, +): Promise => {}; diff --git a/packages/mysql-store/src/queries/add-index.ts b/packages/mysql-store/src/queries/add-index.ts new file mode 100644 index 0000000..aec306f --- /dev/null +++ b/packages/mysql-store/src/queries/add-index.ts @@ -0,0 +1,17 @@ +import { SQLConnection, indexColumns } from '@neuledge/sql-store'; +import { StoreIndex } from '@neuledge/store'; + +// FIXME handle if not exists on mysql + +export const addIndex = async ( + tableName: string, + index: StoreIndex, + connection: SQLConnection, +): Promise => { + await connection.query( + `CREATE ${ + index.unique ? 'UNIQUE INDEX' : 'INDEX' + } IF NOT EXISTS ? ON ? (${indexColumns(index)})`, + [index.name, tableName], + ); +}; diff --git a/packages/mysql-store/src/queries/create-table.ts b/packages/mysql-store/src/queries/create-table.ts new file mode 100644 index 0000000..af8195e --- /dev/null +++ b/packages/mysql-store/src/queries/create-table.ts @@ -0,0 +1,16 @@ +import { SQLConnection } from '@neuledge/sql-store'; +import { StoreCollection } from '@neuledge/store'; + +// FIXME handle if not exists on mysql + +export const createTableIfNotExists = async ( + collection: StoreCollection, + connection: SQLConnection, +): Promise => { + await connection.query( + `CREATE TABLE IF NOT EXISTS ? ( + ${/* FIXME add columns */ ''} + )`, + [collection.name], + ); +}; diff --git a/packages/mysql-store/src/queries/index.ts b/packages/mysql-store/src/queries/index.ts index 78749e0..45b41a4 100644 --- a/packages/mysql-store/src/queries/index.ts +++ b/packages/mysql-store/src/queries/index.ts @@ -1,3 +1,6 @@ +export * from './add-column'; +export * from './add-index'; +export * from './create-table'; export * from './list-tables'; export * from './list-table-columns'; export * from './list-table-statistics'; diff --git a/packages/mysql-store/src/store.ts b/packages/mysql-store/src/store.ts index 77ee271..922fceb 100644 --- a/packages/mysql-store/src/store.ts +++ b/packages/mysql-store/src/store.ts @@ -20,12 +20,16 @@ import { listTableColumns, listIndexAttributes, listTables, + createTableIfNotExists, + addIndex, + addColumn, } from './queries'; import { SQLConnection, + describeCollection, dropCollection, - getCollection, - getStoreCollections, + ensureCollection, + listCollections, } from '@neuledge/sql-store'; export type MySQLStoreOptions = PoolConfig; @@ -58,23 +62,28 @@ export class MySQLStore implements Store { // store methods async listCollections(): Promise { - return getStoreCollections(listTables, this.connection); + return listCollections(this.connection, { listTables }); } async describeCollection( options: StoreDescribeCollectionOptions, ): Promise { - return getCollection( - options, + return describeCollection(options, this.connection, { listTableColumns, listIndexAttributes, dataTypeMap, - this.connection, - ); + }); } async ensureCollection(options: StoreEnsureCollectionOptions): Promise { - throw new Error('Method not implemented.'); + return ensureCollection(options, this.connection, { + createTableIfNotExists, + addIndex, + addColumn, + listTableColumns, + listIndexAttributes, + dataTypeMap, + }); } async dropCollection(options: StoreDropCollectionOptions): Promise { diff --git a/packages/postgresql-store/src/queries/add-column.ts b/packages/postgresql-store/src/queries/add-column.ts new file mode 100644 index 0000000..0526579 --- /dev/null +++ b/packages/postgresql-store/src/queries/add-column.ts @@ -0,0 +1,8 @@ +import { SQLConnection } from '@neuledge/sql-store'; +import { StoreField } from '@neuledge/store'; + +export const addColumn = async ( + tableName: string, + field: StoreField, + connection: SQLConnection, +): Promise => {}; diff --git a/packages/postgresql-store/src/queries/add-index.ts b/packages/postgresql-store/src/queries/add-index.ts new file mode 100644 index 0000000..75dd409 --- /dev/null +++ b/packages/postgresql-store/src/queries/add-index.ts @@ -0,0 +1,15 @@ +import { SQLConnection, indexColumns } from '@neuledge/sql-store'; +import { StoreIndex } from '@neuledge/store'; + +export const addIndex = async ( + tableName: string, + index: StoreIndex, + connection: SQLConnection, +): Promise => { + await connection.query( + `CREATE ${ + index.unique ? 'UNIQUE INDEX' : 'INDEX' + } IF NOT EXISTS ? (${indexColumns(index)})`, + [`${tableName}_${index.name}_idx`], + ); +}; diff --git a/packages/postgresql-store/src/queries/create-table.ts b/packages/postgresql-store/src/queries/create-table.ts new file mode 100644 index 0000000..bb63a12 --- /dev/null +++ b/packages/postgresql-store/src/queries/create-table.ts @@ -0,0 +1,15 @@ +import { SQLConnection, indexColumns } from '@neuledge/sql-store'; +import { StoreCollection } from '@neuledge/store'; + +export const createTableIfNotExists = async ( + collection: StoreCollection, + connection: SQLConnection, +): Promise => { + await connection.query( + `CREATE TABLE IF NOT EXISTS ? ( + ${/* FIXME add columns */ ''} + CONSTRAINT ? PRIMARY KEY (${indexColumns(collection.primaryKey)}) + )`, + [collection.name], + ); +}; diff --git a/packages/postgresql-store/src/queries/drop-index.ts b/packages/postgresql-store/src/queries/drop-index.ts new file mode 100644 index 0000000..595fa2d --- /dev/null +++ b/packages/postgresql-store/src/queries/drop-index.ts @@ -0,0 +1,11 @@ +import { SQLConnection } from '@neuledge/sql-store'; + +export const dropIndex = async ( + tableName: string, + index: string, + connection: SQLConnection, +): Promise => { + await connection.query(`DROP INDEX IF EXISTS ?`, [ + `${tableName}_${index}_idx`, + ]); +}; diff --git a/packages/postgresql-store/src/queries/index.ts b/packages/postgresql-store/src/queries/index.ts index 78749e0..66fa894 100644 --- a/packages/postgresql-store/src/queries/index.ts +++ b/packages/postgresql-store/src/queries/index.ts @@ -1,3 +1,7 @@ +export * from './add-column'; +export * from './add-index'; +export * from './drop-index'; +export * from './create-table'; export * from './list-tables'; export * from './list-table-columns'; export * from './list-table-statistics'; diff --git a/packages/postgresql-store/src/store.test.ts b/packages/postgresql-store/src/store.test.ts new file mode 100644 index 0000000..944909c --- /dev/null +++ b/packages/postgresql-store/src/store.test.ts @@ -0,0 +1,31 @@ +import { PostgreSQLStore } from './store'; + +describe('store', () => { + describe('PostgreSQLStore()', () => { + describe('.constructor()', () => { + it('should be able to create a new store', () => { + const store = new PostgreSQLStore({ + pool: {} as never, + }); + + expect(store).toBeInstanceOf(PostgreSQLStore); + }); + }); + + describe('.close()', () => { + it('should be able to close the store', async () => { + const end = jest.fn(); + + const store = new PostgreSQLStore({ + pool: { end } as never, + }); + + expect(store).toBeInstanceOf(PostgreSQLStore); + expect(end).toHaveBeenCalledTimes(0); + + await store.close(); + expect(end).toHaveBeenCalledTimes(1); + }); + }); + }); +}); diff --git a/packages/postgresql-store/src/store.ts b/packages/postgresql-store/src/store.ts index 17649da..b917ada 100644 --- a/packages/postgresql-store/src/store.ts +++ b/packages/postgresql-store/src/store.ts @@ -4,6 +4,10 @@ import { listTableColumns, listIndexAttributes, listTables, + dropIndex, + addIndex, + createTableIfNotExists, + addColumn, } from './queries'; import { Store, @@ -24,18 +28,27 @@ import { import { SQLConnection, dropCollection, - getCollection, - getStoreCollections, + describeCollection, + listCollections, + ensureCollection, } from '@neuledge/sql-store'; -export type PostgreSQLStoreOptions = PoolConfig; +export type PostgreSQLStorePool = + | (SQLConnection & { end: () => unknown }) + | Pool; + +export type PostgreSQLStoreOptions = + | PoolConfig + | { + pool: PostgreSQLStorePool; + }; export class PostgreSQLStore implements Store { - private pool: Pool; + private pool: PostgreSQLStorePool; private connection: SQLConnection; constructor(options: PostgreSQLStoreOptions) { - this.pool = new Pool(options); + this.pool = 'pool' in options ? options.pool : new Pool(options); this.connection = this.pool; } @@ -48,23 +61,29 @@ export class PostgreSQLStore implements Store { // store methods async listCollections(): Promise { - return getStoreCollections(listTables, this.connection); + return listCollections(this.connection, { listTables }); } async describeCollection( options: StoreDescribeCollectionOptions, ): Promise { - return getCollection( - options, + return describeCollection(options, this.connection, { listTableColumns, listIndexAttributes, dataTypeMap, - this.connection, - ); + }); } async ensureCollection(options: StoreEnsureCollectionOptions): Promise { - throw new Error('Method not implemented.'); + return ensureCollection(options, this.connection, { + createTableIfNotExists, + addIndex, + addColumn, + dropIndex, + listTableColumns, + listIndexAttributes, + dataTypeMap, + }); } async dropCollection(options: StoreDropCollectionOptions): Promise { diff --git a/packages/sql-store/package.json b/packages/sql-store/package.json index 00f552f..8fd4baa 100644 --- a/packages/sql-store/package.json +++ b/packages/sql-store/package.json @@ -38,6 +38,7 @@ "lint:strict": "yarn lint --max-warnings 0" }, "dependencies": { - "@neuledge/store": "^0.2.0" + "@neuledge/store": "^0.2.0", + "p-limit": "^3.1.0" } } diff --git a/packages/sql-store/src/logic/collections.ts b/packages/sql-store/src/logic/collections/describe.ts similarity index 69% rename from packages/sql-store/src/logic/collections.ts rename to packages/sql-store/src/logic/collections/describe.ts index 87abb6f..e01ff38 100644 --- a/packages/sql-store/src/logic/collections.ts +++ b/packages/sql-store/src/logic/collections/describe.ts @@ -2,48 +2,46 @@ import { SQLColumn, SQLIndexAttribute, SQLIndexColumn, - SQLTable, - toStoreCollection_Slim, toStoreField, toStoreIndex, } from '@/mappers'; -import { SQLConnection, dropTableIfExists } from '@/queries'; +import { SQLConnection } from '@/queries'; import { StoreCollection, - StoreCollection_Slim, StoreDescribeCollectionOptions, - StoreDropCollectionOptions, StoreError, StoreShapeType, } from '@neuledge/store'; -export const getStoreCollections = async < - A extends unknown[], - T extends SQLTable, ->( - listTables: (...args: A) => Promise, - ...params: A -): Promise => { - const tables = await listTables(...params); - return tables.map((table) => toStoreCollection_Slim(table)); -}; - -export const getCollection = async < - A extends unknown[], +export interface DescribeCollectionQueries< + C extends SQLColumn, + I extends SQLIndexAttribute & Omit, +> { + listTableColumns: (name: string, connection: SQLConnection) => Promise; + listIndexAttributes: ( + name: string, + connection: SQLConnection, + ) => Promise; + dataTypeMap: Record; +} + +export const describeCollection = async < C extends SQLColumn, I extends SQLIndexAttribute & Omit, >( options: StoreDescribeCollectionOptions, - listTableColumns: (name: string, ...args: A) => Promise, - listIndexAttributes: (name: string, ...args: A) => Promise, - dataTypeMap: Record, - ...params: A + connection: SQLConnection, + { + listTableColumns, + listIndexAttributes, + dataTypeMap, + }: DescribeCollectionQueries, ): Promise => { const { name } = options.collection; const [columns, indexAttributes] = await Promise.all([ - listTableColumns(name, ...params), - listIndexAttributes(name, ...params), + listTableColumns(name, connection), + listIndexAttributes(name, connection), ]); const columnMap = Object.fromEntries( @@ -115,10 +113,3 @@ const groupIndexColumns = < return Object.values(groupMap); }; - -export const dropCollection = async ( - options: StoreDropCollectionOptions, - connection: SQLConnection, -): Promise => { - await dropTableIfExists(connection, options.collection.name); -}; diff --git a/packages/sql-store/src/logic/collections/drop.ts b/packages/sql-store/src/logic/collections/drop.ts new file mode 100644 index 0000000..f7e9d1b --- /dev/null +++ b/packages/sql-store/src/logic/collections/drop.ts @@ -0,0 +1,9 @@ +import { SQLConnection, dropTableIfExists } from '@/queries'; +import { StoreDropCollectionOptions } from '@neuledge/store'; + +export const dropCollection = async ( + options: StoreDropCollectionOptions, + connection: SQLConnection, +): Promise => { + await dropTableIfExists(connection, options.collection.name); +}; diff --git a/packages/sql-store/src/logic/collections/ensure.ts b/packages/sql-store/src/logic/collections/ensure.ts new file mode 100644 index 0000000..42ff986 --- /dev/null +++ b/packages/sql-store/src/logic/collections/ensure.ts @@ -0,0 +1,98 @@ +import { + SQLConnection, + dropColumn as dropColumnDefault, + dropIndex as dropIndexDefault, +} from '@/queries'; +import pLimit from 'p-limit'; +import { + StoreCollection, + StoreEnsureCollectionOptions, + StoreField, + StoreIndex, +} from '@neuledge/store'; +import { SQLColumn, SQLIndexAttribute, SQLIndexColumn } from '@/mappers'; +import { DescribeCollectionQueries, describeCollection } from './describe'; + +export interface EnsureCollectionQueries { + createTableIfNotExists: ( + collection: StoreCollection, + connection: SQLConnection, + ) => Promise; + addIndex: ( + tableName: string, + index: StoreIndex, + connection: SQLConnection, + ) => Promise; + addColumn: ( + tableName: string, + field: StoreField, + connection: SQLConnection, + ) => Promise; + dropIndex?: ( + tableName: string, + index: string, + connection: SQLConnection, + ) => Promise; + dropColumn?: ( + tableName: string, + field: string, + connection: SQLConnection, + ) => Promise; +} + +export const ensureCollection = async < + C extends SQLColumn, + I extends SQLIndexAttribute & Omit, +>( + options: StoreEnsureCollectionOptions, + connection: SQLConnection, + { + createTableIfNotExists, + addIndex, + addColumn, + dropIndex = dropIndexDefault, + dropColumn = dropColumnDefault, + ...describeCollectionQueries + }: EnsureCollectionQueries & DescribeCollectionQueries, +): Promise => { + await createTableIfNotExists(options.collection, connection); + + const tableName = options.collection.name; + const asyncLimit = pLimit(4); + + await Promise.all( + options.dropIndexes?.map((index) => + asyncLimit(() => dropIndex(tableName, index, connection)), + ) || [], + ); + + await Promise.all( + options.dropFields?.map((field) => + asyncLimit(() => dropColumn(tableName, field, connection)), + ) || [], + ); + + const collection = await describeCollection( + options, + connection, + describeCollectionQueries, + ); + + // although we support adding columns with non-nullables types, it will be + // rejected by the database and for a good reason. It's the responsibility of + // the engine to ensure that new columns are nullable if inserted after the + // collection has been created and this is the current implementation. + + await Promise.all([ + ...(options.indexes + ?.filter((index) => !collection.indexes[index.name]) + .map((index) => + asyncLimit(() => addIndex(tableName, index, connection)), + ) || []), + ...(options.fields + ?.filter((field) => !collection.fields[field.name]) + .map((field) => + asyncLimit(() => addColumn(tableName, field, connection)), + ) || []), + ]); +}; diff --git a/packages/sql-store/src/logic/collections/index.ts b/packages/sql-store/src/logic/collections/index.ts new file mode 100644 index 0000000..cef6614 --- /dev/null +++ b/packages/sql-store/src/logic/collections/index.ts @@ -0,0 +1,4 @@ +export * from './describe'; +export * from './drop'; +export * from './ensure'; +export * from './list'; diff --git a/packages/sql-store/src/logic/collections/list.ts b/packages/sql-store/src/logic/collections/list.ts new file mode 100644 index 0000000..bcddebb --- /dev/null +++ b/packages/sql-store/src/logic/collections/list.ts @@ -0,0 +1,15 @@ +import { SQLTable, toStoreCollection_Slim } from '@/mappers'; +import { SQLConnection } from '@/queries'; +import { StoreCollection_Slim } from '@neuledge/store'; + +export interface ListCollectionsQueries { + listTables: (connection: SQLConnection) => Promise; +} + +export const listCollections = async ( + connection: SQLConnection, + { listTables }: ListCollectionsQueries, +): Promise => { + const tables = await listTables(connection); + return tables.map((table) => toStoreCollection_Slim(table)); +}; diff --git a/packages/sql-store/src/queries/drop-column.ts b/packages/sql-store/src/queries/drop-column.ts new file mode 100644 index 0000000..6d775a8 --- /dev/null +++ b/packages/sql-store/src/queries/drop-column.ts @@ -0,0 +1,9 @@ +import { SQLConnection } from './connection'; + +export const dropColumn = async ( + tableName: string, + field: string, + connection: SQLConnection, +): Promise => { + await connection.query(`ALTER TABLE ? DROP COLUMN ?`, [tableName, field]); +}; diff --git a/packages/sql-store/src/queries/drop-index.ts b/packages/sql-store/src/queries/drop-index.ts new file mode 100644 index 0000000..642fb80 --- /dev/null +++ b/packages/sql-store/src/queries/drop-index.ts @@ -0,0 +1,9 @@ +import { SQLConnection } from './connection'; + +export const dropIndex = async ( + tableName: string, + index: string, + connection: SQLConnection, +): Promise => { + await connection.query(`DROP INDEX ? ON ?`, [index, tableName]); +}; diff --git a/packages/sql-store/src/queries/index-columns.ts b/packages/sql-store/src/queries/index-columns.ts new file mode 100644 index 0000000..9273dbe --- /dev/null +++ b/packages/sql-store/src/queries/index-columns.ts @@ -0,0 +1,6 @@ +import { StoreIndex } from '@neuledge/store'; + +export const indexColumns = (index: StoreIndex): string => + Object.entries(index.fields) + .map(([key, val]) => `${key} ${val.sort === 'desc' ? 'DESC' : 'ASC'}`) + .join(', '); diff --git a/packages/sql-store/src/queries/index.ts b/packages/sql-store/src/queries/index.ts index b2a91d3..85a79c9 100644 --- a/packages/sql-store/src/queries/index.ts +++ b/packages/sql-store/src/queries/index.ts @@ -1,2 +1,5 @@ export * from './connection'; +export * from './drop-column'; +export * from './drop-index'; export * from './drop-table'; +export * from './index-columns'; diff --git a/yarn.lock b/yarn.lock index 4e4479c..c300ed2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7520,47 +7520,47 @@ tunnel@0.0.6: resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c" integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg== -turbo-darwin-64@1.8.8: - version "1.8.8" - resolved "https://registry.yarnpkg.com/turbo-darwin-64/-/turbo-darwin-64-1.8.8.tgz#f72b1b6275415b17238f450032c8ef5e5fc71777" - integrity sha512-18cSeIm7aeEvIxGyq7PVoFyEnPpWDM/0CpZvXKHpQ6qMTkfNt517qVqUTAwsIYqNS8xazcKAqkNbvU1V49n65Q== - -turbo-darwin-arm64@1.8.8: - version "1.8.8" - resolved "https://registry.yarnpkg.com/turbo-darwin-arm64/-/turbo-darwin-arm64-1.8.8.tgz#8ec78848e0d5978fd732b3588a1b406fdb978839" - integrity sha512-ruGRI9nHxojIGLQv1TPgN7ud4HO4V8mFBwSgO6oDoZTNuk5ybWybItGR+yu6fni5vJoyMHXOYA2srnxvOc7hjQ== - -turbo-linux-64@1.8.8: - version "1.8.8" - resolved "https://registry.yarnpkg.com/turbo-linux-64/-/turbo-linux-64-1.8.8.tgz#b1f707b23bc6e22b2894dd8063fc2fa4dbb6ffb9" - integrity sha512-N/GkHTHeIQogXB1/6ZWfxHx+ubYeb8Jlq3b/3jnU4zLucpZzTQ8XkXIAfJG/TL3Q7ON7xQ8yGOyGLhHL7MpFRg== - -turbo-linux-arm64@1.8.8: - version "1.8.8" - resolved "https://registry.yarnpkg.com/turbo-linux-arm64/-/turbo-linux-arm64-1.8.8.tgz#34575bdffd2af8c835d9ba3dd9e3a83e0d31dac9" - integrity sha512-hKqLbBHgUkYf2Ww8uBL9UYdBFQ5677a7QXdsFhONXoACbDUPvpK4BKlz3NN7G4NZ+g9dGju+OJJjQP0VXRHb5w== - -turbo-windows-64@1.8.8: - version "1.8.8" - resolved "https://registry.yarnpkg.com/turbo-windows-64/-/turbo-windows-64-1.8.8.tgz#73f67969d54269c95cbf7f082e22c20368aedddc" - integrity sha512-2ndjDJyzkNslXxLt+PQuU21AHJWc8f6MnLypXy3KsN4EyX/uKKGZS0QJWz27PeHg0JS75PVvhfFV+L9t9i+Yyg== - -turbo-windows-arm64@1.8.8: - version "1.8.8" - resolved "https://registry.yarnpkg.com/turbo-windows-arm64/-/turbo-windows-arm64-1.8.8.tgz#c80b9a170adf6ee028e9dcae45b07755af83f3f2" - integrity sha512-xCA3oxgmW9OMqpI34AAmKfOVsfDljhD5YBwgs0ZDsn5h3kCHhC4x9W5dDk1oyQ4F5EXSH3xVym5/xl1J6WRpUg== - -turbo@^1.8.8: - version "1.8.8" - resolved "https://registry.yarnpkg.com/turbo/-/turbo-1.8.8.tgz#8bb331e3f0bd9656b20321339e91e899ad499012" - integrity sha512-qYJ5NjoTX+591/x09KgsDOPVDUJfU9GoS+6jszQQlLp1AHrf1wRFA3Yps8U+/HTG03q0M4qouOfOLtRQP4QypA== +turbo-darwin-64@1.9.1: + version "1.9.1" + resolved "https://registry.yarnpkg.com/turbo-darwin-64/-/turbo-darwin-64-1.9.1.tgz#1f04e716ad6cf071822f0c1a499f4fcd7bd40f4b" + integrity sha512-IX/Ph4CO80lFKd9pPx3BWpN2dynt6mcUFifyuHUNVkOP1Usza/G9YuZnKQFG6wUwKJbx40morFLjk1TTeLe04w== + +turbo-darwin-arm64@1.9.1: + version "1.9.1" + resolved "https://registry.yarnpkg.com/turbo-darwin-arm64/-/turbo-darwin-arm64-1.9.1.tgz#19ec161858fb26dfbd529e0ec9d6c9b4484e91b0" + integrity sha512-6tCbmIboy9dTbhIZ/x9KIpje73nvxbiyVnHbr9xKnsxLJavD0xqjHZzbL5U2tHp8chqmYf0E4WYOXd+XCNg+OQ== + +turbo-linux-64@1.9.1: + version "1.9.1" + resolved "https://registry.yarnpkg.com/turbo-linux-64/-/turbo-linux-64-1.9.1.tgz#5b47e0f0912f709a9a2e325feaa7e260aa76cf49" + integrity sha512-ti8XofnJFO1XaadL92lYJXgxb0VBl03Yu9VfhxkOTywFe7USTLBkJcdvQ4EpFk/KZwLiTdCmT2NQVxsG4AxBiQ== + +turbo-linux-arm64@1.9.1: + version "1.9.1" + resolved "https://registry.yarnpkg.com/turbo-linux-arm64/-/turbo-linux-arm64-1.9.1.tgz#db035061760e8512a408a64cc2d7d568f5102ab7" + integrity sha512-XYvIbeiCCCr+ENujd2Jtck/lJPTKWb8T2MSL/AEBx21Zy3Sa7HgrQX6LX0a0pNHjaleHz00XXt1D0W5hLeP+tA== + +turbo-windows-64@1.9.1: + version "1.9.1" + resolved "https://registry.yarnpkg.com/turbo-windows-64/-/turbo-windows-64-1.9.1.tgz#56ad98e4701b4523f118397d98d64ebef5dab88f" + integrity sha512-x7lWAspe4/v3XQ0gaFRWDX/X9uyWdhwFBPEfb8BA0YKtnsrPOHkV0mRHCRrXzvzjA7pcDCl2agGzb7o863O+Jg== + +turbo-windows-arm64@1.9.1: + version "1.9.1" + resolved "https://registry.yarnpkg.com/turbo-windows-arm64/-/turbo-windows-arm64-1.9.1.tgz#f0c780cc906dedff85eef20f063f59a9b2a865fc" + integrity sha512-QSLNz8dRBLDqXOUv/KnoesBomSbIz2Huef/a3l2+Pat5wkQVgMfzFxDOnkK5VWujPYXz+/prYz+/7cdaC78/kw== + +turbo@^1.9.1: + version "1.9.1" + resolved "https://registry.yarnpkg.com/turbo/-/turbo-1.9.1.tgz#7ff6252cb7271142f82cff36cada918eaae67025" + integrity sha512-Rqe8SP96e53y4Pk29kk2aZbA8EF11UtHJ3vzXJseadrc1T3V6UhzvAWwiKJL//x/jojyOoX1axnoxmX3UHbZ0g== optionalDependencies: - turbo-darwin-64 "1.8.8" - turbo-darwin-arm64 "1.8.8" - turbo-linux-64 "1.8.8" - turbo-linux-arm64 "1.8.8" - turbo-windows-64 "1.8.8" - turbo-windows-arm64 "1.8.8" + turbo-darwin-64 "1.9.1" + turbo-darwin-arm64 "1.9.1" + turbo-linux-64 "1.9.1" + turbo-linux-arm64 "1.9.1" + turbo-windows-64 "1.9.1" + turbo-windows-arm64 "1.9.1" type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" From db3563e64524aceae9067f2772a1e81b9c153734 Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Fri, 14 Apr 2023 18:33:03 +0300 Subject: [PATCH 08/24] test describeCollection on pg store --- packages/mysql-store/src/store.ts | 14 +- packages/postgresql-store/jest.config.json | 3 + packages/postgresql-store/package.json | 1 + .../src/queries/list-table-columns.ts | 76 +++--- .../src/queries/list-tables.ts | 4 +- packages/postgresql-store/src/store.test.ts | 254 +++++++++++++++++- packages/postgresql-store/src/store.ts | 22 +- 7 files changed, 315 insertions(+), 59 deletions(-) create mode 100644 packages/postgresql-store/jest.config.json diff --git a/packages/mysql-store/src/store.ts b/packages/mysql-store/src/store.ts index 922fceb..522f486 100644 --- a/packages/mysql-store/src/store.ts +++ b/packages/mysql-store/src/store.ts @@ -1,4 +1,4 @@ -import { Pool, PoolConfig, createPool } from 'mysql'; +import { Connection, Pool, PoolConfig, createPool } from 'mysql'; import { Store, StoreCollection, @@ -32,19 +32,21 @@ import { listCollections, } from '@neuledge/sql-store'; -export type MySQLStoreOptions = PoolConfig; +export type MySQLStoreClient = Pool | Connection; + +export type MySQLStoreOptions = PoolConfig | { client: MySQLStoreClient }; export class MySQLStore implements Store { - private pool: Pool; + private client: MySQLStoreClient; private connection: SQLConnection; constructor(options: MySQLStoreOptions) { - this.pool = createPool(options); + this.client = 'client' in options ? options.client : createPool(options); this.connection = { query: (sql, values) => new Promise((resolve, reject) => - this.pool.query(sql, values, (error, results) => + this.client.query(sql, values, (error, results) => error ? reject(error) : resolve(results), ), ), @@ -55,7 +57,7 @@ export class MySQLStore implements Store { async close(): Promise { await new Promise((resolve, reject) => - this.pool.end((error) => (error ? reject(error) : resolve())), + this.client.end((error) => (error ? reject(error) : resolve())), ); } diff --git a/packages/postgresql-store/jest.config.json b/packages/postgresql-store/jest.config.json new file mode 100644 index 0000000..5901941 --- /dev/null +++ b/packages/postgresql-store/jest.config.json @@ -0,0 +1,3 @@ +{ + "preset": "@neuledge/jest-ts-preset" +} diff --git a/packages/postgresql-store/package.json b/packages/postgresql-store/package.json index 9d64233..4d1091f 100644 --- a/packages/postgresql-store/package.json +++ b/packages/postgresql-store/package.json @@ -33,6 +33,7 @@ "scripts": { "types": "rimraf --glob dist/*.{d.ts,d.ts.map} dist/**/*.{d.ts,d.ts.map} && tsc --emitDeclarationOnly && tsc-alias", "build": "rimraf --glob dist/*.{js,js.map,mjs,mjs.map} && tsup", + "test": "jest", "lint": "eslint . --ext \"js,jsx,ts,tsx,mjs,cjs\"", "lint:strict": "yarn lint --max-warnings 0" }, diff --git a/packages/postgresql-store/src/queries/list-table-columns.ts b/packages/postgresql-store/src/queries/list-table-columns.ts index 7453fea..25948a1 100644 --- a/packages/postgresql-store/src/queries/list-table-columns.ts +++ b/packages/postgresql-store/src/queries/list-table-columns.ts @@ -25,53 +25,49 @@ export const listTableColumns = async ( [tableName], ); +// https://www.postgresql.org/docs/current/datatype.html export const dataTypeMap: Record = { + bigint: 'number', + bigserial: 'number', + bit: 'string', + 'bit varying': 'string', + boolean: 'boolean', + box: 'string', + bytea: 'string', character: 'string', 'character varying': 'string', + cidr: 'string', + circle: 'string', + date: 'string', 'double precision': 'number', - smallint: 'number', - real: 'number', - 'timestamp without time zone': 'date-time', - 'timestamp with time zone': 'date-time', - 'time without time zone': 'date-time', - 'time with time zone': 'date-time', - 'interval year to month': 'date-time', - 'interval day to second': 'date-time', - 'bit varying': 'binary', - bit: 'binary', - varbit: 'binary', - bytea: 'binary', - text: 'string', + inet: 'string', + integer: 'number', + interval: 'string', json: 'json', jsonb: 'json', - uuid: 'string', - xml: 'string', - cidr: 'string', - inet: 'string', + line: 'string', + lseg: 'string', macaddr: 'string', - tsvector: 'string', - tsquery: 'string', - regconfig: 'string', - regdictionary: 'string', - regnamespace: 'string', - regoper: 'string', - regoperator: 'string', - regproc: 'string', - regprocedure: 'string', - regrole: 'string', - regtype: 'string', - int2: 'number', - int4: 'number', - int8: 'number', - float4: 'number', - float8: 'number', - bool: 'boolean', - date: 'date-time', + money: 'string', + numeric: 'number', + path: 'string', + pg_lsn: 'string', + point: 'string', + polygon: 'string', + real: 'number', + smallint: 'number', + smallserial: 'number', + serial: 'number', + text: 'string', time: 'date-time', + 'time with time zone': 'date-time', + 'time without time zone': 'date-time', timestamp: 'date-time', - timestamptz: 'date-time', - interval: 'date-time', - numeric: 'number', - decimal: 'number', - money: 'number', + 'timestamp with time zone': 'date-time', + 'timestamp without time zone': 'date-time', + tsquery: 'string', + tsvector: 'string', + txid_snapshot: 'string', + uuid: 'string', + xml: 'string', }; diff --git a/packages/postgresql-store/src/queries/list-tables.ts b/packages/postgresql-store/src/queries/list-tables.ts index 933bc0e..a8eee46 100644 --- a/packages/postgresql-store/src/queries/list-tables.ts +++ b/packages/postgresql-store/src/queries/list-tables.ts @@ -11,5 +11,7 @@ export const listTables = async ( connection: SQLConnection, ): Promise => connection.query( - `SELECT table_name FROM information_schema.tables WHERE table_catalog = current_database() AND table_schema = current_schema()`, + `SELECT table_name + FROM information_schema.tables + WHERE table_catalog = current_database() AND table_schema = current_schema() AND table_type = 'BASE TABLE'`, ); diff --git a/packages/postgresql-store/src/store.test.ts b/packages/postgresql-store/src/store.test.ts index 944909c..39b22e3 100644 --- a/packages/postgresql-store/src/store.test.ts +++ b/packages/postgresql-store/src/store.test.ts @@ -1,11 +1,13 @@ import { PostgreSQLStore } from './store'; +/* eslint-disable max-lines-per-function */ + describe('store', () => { describe('PostgreSQLStore()', () => { describe('.constructor()', () => { it('should be able to create a new store', () => { const store = new PostgreSQLStore({ - pool: {} as never, + client: {} as never, }); expect(store).toBeInstanceOf(PostgreSQLStore); @@ -17,7 +19,7 @@ describe('store', () => { const end = jest.fn(); const store = new PostgreSQLStore({ - pool: { end } as never, + client: { end } as never, }); expect(store).toBeInstanceOf(PostgreSQLStore); @@ -27,5 +29,253 @@ describe('store', () => { expect(end).toHaveBeenCalledTimes(1); }); }); + + let store: PostgreSQLStore; + let query: jest.Mock; + + beforeEach(() => { + query = jest.fn(); + + store = new PostgreSQLStore({ + client: { query } as never, + }); + }); + + describe('.listCollections()', () => { + it('should be able to list collections', async () => { + query.mockResolvedValueOnce({ + rows: [{ table_name: 'foo' }, { table_name: 'bar' }], + }); + + const collections = await store.listCollections(); + + expect(query).toHaveBeenCalledTimes(1); + expect(query).toHaveBeenCalledWith( + `SELECT table_name + FROM information_schema.tables + WHERE table_catalog = current_database() AND table_schema = current_schema() AND table_type = 'BASE TABLE'`, + undefined, + ); + + expect(collections).toEqual([{ name: 'foo' }, { name: 'bar' }]); + }); + }); + + describe('.describeCollection()', () => { + it('should be able to describe a collection', async () => { + query.mockResolvedValueOnce({ + rows: [ + { + column_name: 'id', + data_type: 'integer', + character_maximum_length: null, + numeric_precision: 32, + numeric_scale: 0, + is_nullable: false, + is_auto_increment: true, + }, + { + column_name: 'name', + data_type: 'character varying', + character_maximum_length: 50, + numeric_precision: null, + numeric_scale: null, + is_nullable: true, + is_auto_increment: null, + }, + { + column_name: 'email', + data_type: 'character varying', + character_maximum_length: 100, + numeric_precision: null, + numeric_scale: null, + is_nullable: false, + is_auto_increment: null, + }, + { + column_name: 'phone', + data_type: 'character varying', + character_maximum_length: 20, + numeric_precision: null, + numeric_scale: null, + is_nullable: true, + is_auto_increment: null, + }, + { + column_name: 'created_at', + data_type: 'timestamp without time zone', + character_maximum_length: null, + numeric_precision: null, + numeric_scale: null, + is_nullable: false, + is_auto_increment: null, + }, + { + column_name: 'updated_at', + data_type: 'timestamp without time zone', + character_maximum_length: null, + numeric_precision: null, + numeric_scale: null, + is_nullable: false, + is_auto_increment: null, + }, + ], + }); + + query.mockResolvedValueOnce({ + rows: [ + { + index_name: 'idx_email', + column_name: 'email', + seq_in_index: 1, + direction: 'ASC', + nulls: 'LAST', + is_unique: true, + is_primary: false, + }, + { + index_name: 'idx_phone_email', + column_name: 'phone', + seq_in_index: 1, + direction: 'DESC', + nulls: 'FIRST', + is_unique: false, + is_primary: false, + }, + { + index_name: 'idx_phone_email', + column_name: 'email', + seq_in_index: 2, + direction: 'ASC', + nulls: 'LAST', + is_unique: false, + is_primary: false, + }, + { + index_name: 'foo_pkey', + column_name: 'id', + seq_in_index: 1, + direction: 'ASC', + nulls: 'LAST', + is_unique: true, + is_primary: true, + }, + ], + }); + + const collection = await store.describeCollection({ + collection: { name: 'foo' }, + }); + + expect(query).toHaveBeenCalledTimes(2); + expect(query).toHaveBeenNthCalledWith( + 1, + `SELECT column_name, data_type, character_maximum_length, numeric_precision, numeric_scale, (is_nullable = 'YES') as is_nullable, column_default LIKE 'nextval(%)' AS is_auto_increment + FROM information_schema.columns + WHERE table_catalog = current_database() AND table_schema = current_schema() AND table_name = ?`, + ['foo'], + ); + expect(query).toHaveBeenNthCalledWith( + 2, + `SELECT + irel.relname AS index_name, + a.attname AS column_name, + c.ordinality as seq_in_index, + CASE o.option & 1 WHEN 1 THEN 'DESC' ELSE 'ASC' END AS direction, + CASE o.option & 2 WHEN 2 THEN 'FIRST' ELSE 'LAST' END AS nulls, + i.indisunique AS is_unique, + i.indisprimary AS is_primary + FROM pg_index AS i + JOIN pg_class AS trel ON trel.oid = i.indrelid + JOIN pg_namespace AS tnsp ON trel.relnamespace = tnsp.oid + JOIN pg_class AS irel ON irel.oid = i.indexrelid + CROSS JOIN LATERAL unnest (i.indkey) WITH ORDINALITY AS c (colnum, ordinality) + LEFT JOIN LATERAL unnest (i.indoption) WITH ORDINALITY AS o (option, ordinality) + ON c.ordinality = o.ordinality + JOIN pg_attribute AS a ON trel.oid = a.attrelid AND a.attnum = c.colnum + WHERE tnsp.nspname = current_schema() AND trel.relname = ? + ORDER BY index_name, seq_in_index`, + ['foo'], + ); + + expect(collection).toEqual({ + name: 'foo', + fields: { + id: { + name: 'id', + type: 'number', + nullable: false, + size: null, + precision: 32, + scale: 0, + }, + name: { + name: 'name', + type: 'string', + nullable: true, + size: 50, + precision: null, + scale: null, + }, + email: { + name: 'email', + type: 'string', + nullable: false, + size: 100, + precision: null, + scale: null, + }, + phone: { + name: 'phone', + type: 'string', + nullable: true, + size: 20, + precision: null, + scale: null, + }, + created_at: { + name: 'created_at', + type: 'date-time', + nullable: false, + size: null, + precision: null, + scale: null, + }, + updated_at: { + name: 'updated_at', + type: 'date-time', + nullable: false, + size: null, + precision: null, + scale: null, + }, + }, + primaryKey: { + name: 'foo_pkey', + fields: { id: { sort: 'asc' } }, + unique: 'primary', + auto: 'increment', + }, + indexes: { + foo_pkey: { + name: 'foo_pkey', + fields: { id: { sort: 'asc' } }, + unique: 'primary', + auto: 'increment', + }, + idx_email: { + name: 'idx_email', + fields: { email: { sort: 'asc' } }, + unique: true, + }, + idx_phone_email: { + name: 'idx_phone_email', + fields: { phone: { sort: 'desc' }, email: { sort: 'asc' } }, + unique: false, + }, + }, + }); + }); + }); }); }); diff --git a/packages/postgresql-store/src/store.ts b/packages/postgresql-store/src/store.ts index b917ada..26ac942 100644 --- a/packages/postgresql-store/src/store.ts +++ b/packages/postgresql-store/src/store.ts @@ -1,4 +1,4 @@ -import { Pool, PoolConfig } from 'pg'; +import { Client, Pool, PoolConfig } from 'pg'; import { dataTypeMap, listTableColumns, @@ -26,36 +26,38 @@ import { StoreUpdateOptions, } from '@neuledge/store'; import { - SQLConnection, dropCollection, describeCollection, listCollections, ensureCollection, + SQLConnection, } from '@neuledge/sql-store'; -export type PostgreSQLStorePool = - | (SQLConnection & { end: () => unknown }) - | Pool; +export type PostgreSQLStoreClient = Client | Pool; export type PostgreSQLStoreOptions = | PoolConfig | { - pool: PostgreSQLStorePool; + client: PostgreSQLStoreClient; }; export class PostgreSQLStore implements Store { - private pool: PostgreSQLStorePool; + private client: PostgreSQLStoreClient; private connection: SQLConnection; constructor(options: PostgreSQLStoreOptions) { - this.pool = 'pool' in options ? options.pool : new Pool(options); - this.connection = this.pool; + this.client = 'client' in options ? options.client : new Pool(options); + + this.connection = { + query: (sql, values) => + this.client.query(sql, values).then((result) => result.rows as never), + }; } // connection methods async close(): Promise { - await this.pool.end(); + await this.client.end(); } // store methods From 321de110c3abe31cc78112b8e5db4088d22c6537 Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Sat, 15 Apr 2023 00:28:36 +0300 Subject: [PATCH 09/24] test pg createTableIfNotExists --- .../src/queries/__fixtures__/users-table.ts | 196 +++++++++++++ .../src/queries/add-column.ts | 88 +++++- .../src/queries/create-table.ts | 19 +- .../src/queries/list-table-columns.ts | 13 +- .../src/queries/list-table-statistics.ts | 43 +-- .../src/queries/list-tables.ts | 10 +- packages/postgresql-store/src/store.test.ts | 264 ++++-------------- packages/postgresql-store/src/store.ts | 4 +- packages/store/src/collection.ts | 2 +- 9 files changed, 382 insertions(+), 257 deletions(-) create mode 100644 packages/postgresql-store/src/queries/__fixtures__/users-table.ts diff --git a/packages/postgresql-store/src/queries/__fixtures__/users-table.ts b/packages/postgresql-store/src/queries/__fixtures__/users-table.ts new file mode 100644 index 0000000..baf5ff4 --- /dev/null +++ b/packages/postgresql-store/src/queries/__fixtures__/users-table.ts @@ -0,0 +1,196 @@ +import { StoreCollection, StoreCollection_Slim } from '@neuledge/store'; +import { PostgreSQLTable } from '../list-tables'; +import { PostgreSQLColumn } from '../list-table-columns'; +import { PostgreSQLIndexAttribute } from '..'; + +export const usersTableName = 'users'; + +export const usersTable_createSql = `CREATE TABLE IF NOT EXISTS ${usersTableName} ( + id BIGSERIAL NOT NULL, + name VARCHAR(50), + email VARCHAR(100) NOT NULL, + phone VARCHAR(20), + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + CONSTRAINT ${usersTableName}_pkey PRIMARY KEY (id) +)`; + +export const usersTable: PostgreSQLTable = { table_name: usersTableName }; + +export const usersCollection_slim: StoreCollection_Slim = { + name: usersTableName, +}; + +export const usersTableColumns: PostgreSQLColumn[] = [ + { + column_name: 'id', + data_type: 'integer', + character_maximum_length: null, + numeric_precision: 32, + numeric_scale: 0, + is_nullable: false, + is_auto_increment: true, + }, + { + column_name: 'name', + data_type: 'character varying', + character_maximum_length: 50, + numeric_precision: null, + numeric_scale: null, + is_nullable: true, + is_auto_increment: null, + }, + { + column_name: 'email', + data_type: 'character varying', + character_maximum_length: 100, + numeric_precision: null, + numeric_scale: null, + is_nullable: false, + is_auto_increment: null, + }, + { + column_name: 'phone', + data_type: 'character varying', + character_maximum_length: 20, + numeric_precision: null, + numeric_scale: null, + is_nullable: true, + is_auto_increment: null, + }, + { + column_name: 'created_at', + data_type: 'timestamp without time zone', + character_maximum_length: null, + numeric_precision: null, + numeric_scale: null, + is_nullable: false, + is_auto_increment: null, + }, + { + column_name: 'updated_at', + data_type: 'timestamp without time zone', + character_maximum_length: null, + numeric_precision: null, + numeric_scale: null, + is_nullable: false, + is_auto_increment: null, + }, +]; + +export const usersTableIndexes: PostgreSQLIndexAttribute[] = [ + { + index_name: 'idx_email', + column_name: 'email', + seq_in_index: 1, + direction: 'ASC', + nulls: 'LAST', + is_unique: true, + is_primary: false, + }, + { + index_name: 'idx_phone_email', + column_name: 'phone', + seq_in_index: 1, + direction: 'DESC', + nulls: 'FIRST', + is_unique: false, + is_primary: false, + }, + { + index_name: 'idx_phone_email', + column_name: 'email', + seq_in_index: 2, + direction: 'ASC', + nulls: 'LAST', + is_unique: false, + is_primary: false, + }, + { + index_name: 'users_pkey', + column_name: 'id', + seq_in_index: 1, + direction: 'ASC', + nulls: 'LAST', + is_unique: true, + is_primary: true, + }, +]; + +export const usersCollection: StoreCollection = { + name: usersTableName, + fields: { + id: { + name: 'id', + type: 'number', + nullable: false, + size: null, + precision: 32, + scale: 0, + }, + name: { + name: 'name', + type: 'string', + nullable: true, + size: 50, + precision: null, + scale: null, + }, + email: { + name: 'email', + type: 'string', + nullable: false, + size: 100, + precision: null, + scale: null, + }, + phone: { + name: 'phone', + type: 'string', + nullable: true, + size: 20, + precision: null, + scale: null, + }, + created_at: { + name: 'created_at', + type: 'date-time', + nullable: false, + size: null, + precision: null, + scale: null, + }, + updated_at: { + name: 'updated_at', + type: 'date-time', + nullable: false, + size: null, + precision: null, + scale: null, + }, + }, + primaryKey: { + name: `${usersTableName}_pkey`, + fields: { id: { sort: 'asc' } }, + unique: 'primary', + auto: 'increment', + }, + indexes: { + [`${usersTableName}_pkey`]: { + name: `${usersTableName}_pkey`, + fields: { id: { sort: 'asc' } }, + unique: 'primary', + auto: 'increment', + }, + idx_email: { + name: 'idx_email', + fields: { email: { sort: 'asc' } }, + unique: true, + }, + idx_phone_email: { + name: 'idx_phone_email', + fields: { phone: { sort: 'desc' }, email: { sort: 'asc' } }, + unique: false, + }, + }, +}; diff --git a/packages/postgresql-store/src/queries/add-column.ts b/packages/postgresql-store/src/queries/add-column.ts index 0526579..6b1800b 100644 --- a/packages/postgresql-store/src/queries/add-column.ts +++ b/packages/postgresql-store/src/queries/add-column.ts @@ -1,8 +1,94 @@ import { SQLConnection } from '@neuledge/sql-store'; -import { StoreField } from '@neuledge/store'; +import { StoreCollection, StoreField } from '@neuledge/store'; export const addColumn = async ( tableName: string, field: StoreField, connection: SQLConnection, ): Promise => {}; + +export const getColumnDefinition = ( + field: StoreField, + collection: StoreCollection, +): string => + `${field.name} ${getColumnDataType(field, collection)}${ + field.nullable ? '' : ' NOT NULL' + }`; + +const getColumnDataType = ( + field: StoreField, + collection: StoreCollection, +): string => { + switch (field.type) { + case 'string': { + if (field.size) { + return `VARCHAR(${field.size})`; + } + + return 'TEXT'; + } + + case 'number': { + return getNumberDateType(field, collection); + } + + case 'date-time': { + return 'TIMESTAMP'; + } + + case 'boolean': { + return 'BOOLEAN'; + } + + case 'json': { + return 'JSONB'; + } + + case 'enum': { + return `ENUM(${field.values?.map((value) => `'${value}'`).join(', ')})`; + } + + case 'binary': { + return 'BYTEA'; + } + + default: { + throw new Error(`Unsupported field type: ${field.type}`); + } + } +}; + +// https://www.postgresql.org/docs/current/datatype-numeric.html +const getNumberDateType = ( + field: StoreField, + collection: StoreCollection, +): string => { + if (field.scale === 0) { + if ( + collection.primaryKey.auto === 'increment' && + collection.primaryKey.fields[field.name] + ) { + if (!field.precision || field.precision >= 10) { + return 'BIGSERIAL'; + } + + if (field.precision < 5) { + return 'SMALLSERIAL'; + } + + return 'SERIAL'; + } + + if (field.precision) { + return `NUMERIC(${field.precision})`; + } + + return 'BIGINT'; + } + + if (field.precision && field.scale) { + return `NUMERIC(${field.precision}, ${field.scale})`; + } + + return 'DOUBLE PRECISION'; +}; diff --git a/packages/postgresql-store/src/queries/create-table.ts b/packages/postgresql-store/src/queries/create-table.ts index bb63a12..6a0a9e1 100644 --- a/packages/postgresql-store/src/queries/create-table.ts +++ b/packages/postgresql-store/src/queries/create-table.ts @@ -1,15 +1,22 @@ -import { SQLConnection, indexColumns } from '@neuledge/sql-store'; +import { SQLConnection } from '@neuledge/sql-store'; import { StoreCollection } from '@neuledge/store'; +import { getColumnDefinition } from './add-column'; export const createTableIfNotExists = async ( collection: StoreCollection, connection: SQLConnection, ): Promise => { await connection.query( - `CREATE TABLE IF NOT EXISTS ? ( - ${/* FIXME add columns */ ''} - CONSTRAINT ? PRIMARY KEY (${indexColumns(collection.primaryKey)}) - )`, - [collection.name], + `CREATE TABLE IF NOT EXISTS ${collection.name} ( + ${Object.values(collection.fields) + .map((field) => getColumnDefinition(field, collection)) + .join(',\n ')}, + CONSTRAINT ${collection.name}_pkey PRIMARY KEY (${Object.keys( + collection.primaryKey.fields, + ).join(', ')}) +)`, ); + + // FIXME add unique constraints if primary key has descending fields + // https://stackoverflow.com/a/45604459/518153 }; diff --git a/packages/postgresql-store/src/queries/list-table-columns.ts b/packages/postgresql-store/src/queries/list-table-columns.ts index 25948a1..029775b 100644 --- a/packages/postgresql-store/src/queries/list-table-columns.ts +++ b/packages/postgresql-store/src/queries/list-table-columns.ts @@ -11,19 +11,18 @@ export interface PostgreSQLColumn { numeric_precision: number | null; numeric_scale: number | null; is_nullable: boolean; - is_auto_increment: boolean; + is_auto_increment: boolean | null; } export const listTableColumns = async ( tableName: string, connection: SQLConnection, ): Promise => - connection.query( - `SELECT column_name, data_type, character_maximum_length, numeric_precision, numeric_scale, (is_nullable = 'YES') as is_nullable, column_default LIKE 'nextval(%)' AS is_auto_increment - FROM information_schema.columns - WHERE table_catalog = current_database() AND table_schema = current_schema() AND table_name = ?`, - [tableName], - ); + connection.query(listTableColumns_sql, [tableName]); + +export const listTableColumns_sql = `SELECT column_name, data_type, character_maximum_length, numeric_precision, numeric_scale, (is_nullable = 'YES') as is_nullable, column_default LIKE 'nextval(%)' AS is_auto_increment +FROM information_schema.columns +WHERE table_catalog = current_database() AND table_schema = current_schema() AND table_name = ?`; // https://www.postgresql.org/docs/current/datatype.html export const dataTypeMap: Record = { diff --git a/packages/postgresql-store/src/queries/list-table-statistics.ts b/packages/postgresql-store/src/queries/list-table-statistics.ts index 325f8b3..e6a2006 100644 --- a/packages/postgresql-store/src/queries/list-table-statistics.ts +++ b/packages/postgresql-store/src/queries/list-table-statistics.ts @@ -17,24 +17,25 @@ export const listIndexAttributes = async ( tableName: string, connection: SQLConnection, ): Promise => - connection.query( - `SELECT - irel.relname AS index_name, - a.attname AS column_name, - c.ordinality as seq_in_index, - CASE o.option & 1 WHEN 1 THEN 'DESC' ELSE 'ASC' END AS direction, - CASE o.option & 2 WHEN 2 THEN 'FIRST' ELSE 'LAST' END AS nulls, - i.indisunique AS is_unique, - i.indisprimary AS is_primary - FROM pg_index AS i - JOIN pg_class AS trel ON trel.oid = i.indrelid - JOIN pg_namespace AS tnsp ON trel.relnamespace = tnsp.oid - JOIN pg_class AS irel ON irel.oid = i.indexrelid - CROSS JOIN LATERAL unnest (i.indkey) WITH ORDINALITY AS c (colnum, ordinality) - LEFT JOIN LATERAL unnest (i.indoption) WITH ORDINALITY AS o (option, ordinality) - ON c.ordinality = o.ordinality - JOIN pg_attribute AS a ON trel.oid = a.attrelid AND a.attnum = c.colnum - WHERE tnsp.nspname = current_schema() AND trel.relname = ? - ORDER BY index_name, seq_in_index`, - [tableName], - ); + connection.query(listIndexAttributes_sql, [ + tableName, + ]); + +export const listIndexAttributes_sql = `SELECT +irel.relname AS index_name, +a.attname AS column_name, +c.ordinality as seq_in_index, +CASE o.option & 1 WHEN 1 THEN 'DESC' ELSE 'ASC' END AS direction, +CASE o.option & 2 WHEN 2 THEN 'FIRST' ELSE 'LAST' END AS nulls, +i.indisunique AS is_unique, +i.indisprimary AS is_primary +FROM pg_index AS i +JOIN pg_class AS trel ON trel.oid = i.indrelid +JOIN pg_namespace AS tnsp ON trel.relnamespace = tnsp.oid +JOIN pg_class AS irel ON irel.oid = i.indexrelid +CROSS JOIN LATERAL unnest (i.indkey) WITH ORDINALITY AS c (colnum, ordinality) +LEFT JOIN LATERAL unnest (i.indoption) WITH ORDINALITY AS o (option, ordinality) +ON c.ordinality = o.ordinality +JOIN pg_attribute AS a ON trel.oid = a.attrelid AND a.attnum = c.colnum +WHERE tnsp.nspname = current_schema() AND trel.relname = ? +ORDER BY index_name, seq_in_index`; diff --git a/packages/postgresql-store/src/queries/list-tables.ts b/packages/postgresql-store/src/queries/list-tables.ts index a8eee46..e4aad48 100644 --- a/packages/postgresql-store/src/queries/list-tables.ts +++ b/packages/postgresql-store/src/queries/list-tables.ts @@ -10,8 +10,8 @@ export interface PostgreSQLTable { export const listTables = async ( connection: SQLConnection, ): Promise => - connection.query( - `SELECT table_name - FROM information_schema.tables - WHERE table_catalog = current_database() AND table_schema = current_schema() AND table_type = 'BASE TABLE'`, - ); + connection.query(listTables_sql); + +export const listTables_sql = `SELECT table_name +FROM information_schema.tables +WHERE table_catalog = current_database() AND table_schema = current_schema() AND table_type = 'BASE TABLE'`; diff --git a/packages/postgresql-store/src/store.test.ts b/packages/postgresql-store/src/store.test.ts index 39b22e3..95d2b85 100644 --- a/packages/postgresql-store/src/store.test.ts +++ b/packages/postgresql-store/src/store.test.ts @@ -1,3 +1,17 @@ +import { + listIndexAttributes_sql, + listTableColumns_sql, + listTables_sql, +} from './queries'; +import { + usersCollection, + usersCollection_slim, + usersTable, + usersTableColumns, + usersTableIndexes, + usersTableName, + usersTable_createSql, +} from './queries/__fixtures__/users-table'; import { PostgreSQLStore } from './store'; /* eslint-disable max-lines-per-function */ @@ -43,238 +57,58 @@ describe('store', () => { describe('.listCollections()', () => { it('should be able to list collections', async () => { - query.mockResolvedValueOnce({ - rows: [{ table_name: 'foo' }, { table_name: 'bar' }], - }); + query.mockResolvedValueOnce({ rows: [usersTable] }); const collections = await store.listCollections(); expect(query).toHaveBeenCalledTimes(1); - expect(query).toHaveBeenCalledWith( - `SELECT table_name - FROM information_schema.tables - WHERE table_catalog = current_database() AND table_schema = current_schema() AND table_type = 'BASE TABLE'`, - undefined, - ); + expect(query).toHaveBeenCalledWith(listTables_sql, []); - expect(collections).toEqual([{ name: 'foo' }, { name: 'bar' }]); + expect(collections).toEqual([usersCollection_slim]); }); }); describe('.describeCollection()', () => { it('should be able to describe a collection', async () => { - query.mockResolvedValueOnce({ - rows: [ - { - column_name: 'id', - data_type: 'integer', - character_maximum_length: null, - numeric_precision: 32, - numeric_scale: 0, - is_nullable: false, - is_auto_increment: true, - }, - { - column_name: 'name', - data_type: 'character varying', - character_maximum_length: 50, - numeric_precision: null, - numeric_scale: null, - is_nullable: true, - is_auto_increment: null, - }, - { - column_name: 'email', - data_type: 'character varying', - character_maximum_length: 100, - numeric_precision: null, - numeric_scale: null, - is_nullable: false, - is_auto_increment: null, - }, - { - column_name: 'phone', - data_type: 'character varying', - character_maximum_length: 20, - numeric_precision: null, - numeric_scale: null, - is_nullable: true, - is_auto_increment: null, - }, - { - column_name: 'created_at', - data_type: 'timestamp without time zone', - character_maximum_length: null, - numeric_precision: null, - numeric_scale: null, - is_nullable: false, - is_auto_increment: null, - }, - { - column_name: 'updated_at', - data_type: 'timestamp without time zone', - character_maximum_length: null, - numeric_precision: null, - numeric_scale: null, - is_nullable: false, - is_auto_increment: null, - }, - ], - }); - - query.mockResolvedValueOnce({ - rows: [ - { - index_name: 'idx_email', - column_name: 'email', - seq_in_index: 1, - direction: 'ASC', - nulls: 'LAST', - is_unique: true, - is_primary: false, - }, - { - index_name: 'idx_phone_email', - column_name: 'phone', - seq_in_index: 1, - direction: 'DESC', - nulls: 'FIRST', - is_unique: false, - is_primary: false, - }, - { - index_name: 'idx_phone_email', - column_name: 'email', - seq_in_index: 2, - direction: 'ASC', - nulls: 'LAST', - is_unique: false, - is_primary: false, - }, - { - index_name: 'foo_pkey', - column_name: 'id', - seq_in_index: 1, - direction: 'ASC', - nulls: 'LAST', - is_unique: true, - is_primary: true, - }, - ], - }); + query.mockResolvedValueOnce({ rows: usersTableColumns }); + query.mockResolvedValueOnce({ rows: usersTableIndexes }); const collection = await store.describeCollection({ - collection: { name: 'foo' }, + collection: usersCollection_slim, }); expect(query).toHaveBeenCalledTimes(2); - expect(query).toHaveBeenNthCalledWith( - 1, - `SELECT column_name, data_type, character_maximum_length, numeric_precision, numeric_scale, (is_nullable = 'YES') as is_nullable, column_default LIKE 'nextval(%)' AS is_auto_increment - FROM information_schema.columns - WHERE table_catalog = current_database() AND table_schema = current_schema() AND table_name = ?`, - ['foo'], - ); - expect(query).toHaveBeenNthCalledWith( - 2, - `SELECT - irel.relname AS index_name, - a.attname AS column_name, - c.ordinality as seq_in_index, - CASE o.option & 1 WHEN 1 THEN 'DESC' ELSE 'ASC' END AS direction, - CASE o.option & 2 WHEN 2 THEN 'FIRST' ELSE 'LAST' END AS nulls, - i.indisunique AS is_unique, - i.indisprimary AS is_primary - FROM pg_index AS i - JOIN pg_class AS trel ON trel.oid = i.indrelid - JOIN pg_namespace AS tnsp ON trel.relnamespace = tnsp.oid - JOIN pg_class AS irel ON irel.oid = i.indexrelid - CROSS JOIN LATERAL unnest (i.indkey) WITH ORDINALITY AS c (colnum, ordinality) - LEFT JOIN LATERAL unnest (i.indoption) WITH ORDINALITY AS o (option, ordinality) - ON c.ordinality = o.ordinality - JOIN pg_attribute AS a ON trel.oid = a.attrelid AND a.attnum = c.colnum - WHERE tnsp.nspname = current_schema() AND trel.relname = ? - ORDER BY index_name, seq_in_index`, - ['foo'], - ); + expect(query).toHaveBeenNthCalledWith(1, listTableColumns_sql, [ + usersTableName, + ]); + expect(query).toHaveBeenNthCalledWith(2, listIndexAttributes_sql, [ + usersTableName, + ]); + + expect(collection).toEqual(usersCollection); + }); + }); - expect(collection).toEqual({ - name: 'foo', - fields: { - id: { - name: 'id', - type: 'number', - nullable: false, - size: null, - precision: 32, - scale: 0, - }, - name: { - name: 'name', - type: 'string', - nullable: true, - size: 50, - precision: null, - scale: null, - }, - email: { - name: 'email', - type: 'string', - nullable: false, - size: 100, - precision: null, - scale: null, - }, - phone: { - name: 'phone', - type: 'string', - nullable: true, - size: 20, - precision: null, - scale: null, - }, - created_at: { - name: 'created_at', - type: 'date-time', - nullable: false, - size: null, - precision: null, - scale: null, - }, - updated_at: { - name: 'updated_at', - type: 'date-time', - nullable: false, - size: null, - precision: null, - scale: null, - }, - }, - primaryKey: { - name: 'foo_pkey', - fields: { id: { sort: 'asc' } }, - unique: 'primary', - auto: 'increment', - }, - indexes: { - foo_pkey: { - name: 'foo_pkey', - fields: { id: { sort: 'asc' } }, - unique: 'primary', - auto: 'increment', - }, - idx_email: { - name: 'idx_email', - fields: { email: { sort: 'asc' } }, - unique: true, - }, - idx_phone_email: { - name: 'idx_phone_email', - fields: { phone: { sort: 'desc' }, email: { sort: 'asc' } }, - unique: false, - }, - }, + describe('.ensureCollection()', () => { + it('should skip create an existing table', async () => { + query.mockResolvedValueOnce({ rows: [] }); + query.mockResolvedValueOnce({ rows: usersTableColumns }); + query.mockResolvedValueOnce({ rows: usersTableIndexes }); + + await store.ensureCollection({ + collection: usersCollection, + fields: Object.values(usersCollection.fields), + indexes: Object.values(usersCollection.indexes), }); + + expect(query).toHaveBeenCalledTimes(3); + expect(query).toHaveBeenNthCalledWith(1, usersTable_createSql, []); + expect(query).toHaveBeenNthCalledWith(2, listTableColumns_sql, [ + usersTableName, + ]); + expect(query).toHaveBeenNthCalledWith(3, listIndexAttributes_sql, [ + usersTableName, + ]); }); }); }); diff --git a/packages/postgresql-store/src/store.ts b/packages/postgresql-store/src/store.ts index 26ac942..9956eb2 100644 --- a/packages/postgresql-store/src/store.ts +++ b/packages/postgresql-store/src/store.ts @@ -50,7 +50,9 @@ export class PostgreSQLStore implements Store { this.connection = { query: (sql, values) => - this.client.query(sql, values).then((result) => result.rows as never), + this.client + .query(sql, values ?? []) + .then((result) => result.rows as never), }; } diff --git a/packages/store/src/collection.ts b/packages/store/src/collection.ts index ea40065..3838d53 100644 --- a/packages/store/src/collection.ts +++ b/packages/store/src/collection.ts @@ -4,7 +4,7 @@ import { StoreSortDirection } from './sort'; export interface StoreCollection { name: string; primaryKey: StorePrimaryKey; - indexes: Record; + indexes: Record; fields: Record; } From 795dff4a2fd2057557221da6deadf9cafd5d11d1 Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Sat, 15 Apr 2023 01:09:35 +0300 Subject: [PATCH 10/24] ensureCollection pg tests --- .../mysql-store/src/queries/add-column.ts | 4 +- packages/mysql-store/src/queries/add-index.ts | 6 +- .../src/queries/__fixtures__/users-table.ts | 48 +++++++----- .../src/queries/add-column.ts | 11 ++- .../postgresql-store/src/queries/add-index.ts | 11 ++- .../src/queries/drop-index.ts | 5 +- .../src/queries/list-table-statistics.ts | 20 ++++- packages/postgresql-store/src/store.test.ts | 74 ++++++++++++++++++- .../sql-store/src/logic/collections/ensure.ts | 36 ++++----- packages/sql-store/src/queries/drop-column.ts | 8 +- packages/sql-store/src/queries/drop-index.ts | 5 +- 11 files changed, 168 insertions(+), 60 deletions(-) diff --git a/packages/mysql-store/src/queries/add-column.ts b/packages/mysql-store/src/queries/add-column.ts index 0526579..4b0716a 100644 --- a/packages/mysql-store/src/queries/add-column.ts +++ b/packages/mysql-store/src/queries/add-column.ts @@ -1,8 +1,8 @@ import { SQLConnection } from '@neuledge/sql-store'; -import { StoreField } from '@neuledge/store'; +import { StoreCollection, StoreField } from '@neuledge/store'; export const addColumn = async ( - tableName: string, + collection: StoreCollection, field: StoreField, connection: SQLConnection, ): Promise => {}; diff --git a/packages/mysql-store/src/queries/add-index.ts b/packages/mysql-store/src/queries/add-index.ts index aec306f..699b8d7 100644 --- a/packages/mysql-store/src/queries/add-index.ts +++ b/packages/mysql-store/src/queries/add-index.ts @@ -1,10 +1,10 @@ import { SQLConnection, indexColumns } from '@neuledge/sql-store'; -import { StoreIndex } from '@neuledge/store'; +import { StoreCollection, StoreIndex } from '@neuledge/store'; // FIXME handle if not exists on mysql export const addIndex = async ( - tableName: string, + collection: StoreCollection, index: StoreIndex, connection: SQLConnection, ): Promise => { @@ -12,6 +12,6 @@ export const addIndex = async ( `CREATE ${ index.unique ? 'UNIQUE INDEX' : 'INDEX' } IF NOT EXISTS ? ON ? (${indexColumns(index)})`, - [index.name, tableName], + [index.name, collection.name], ); }; diff --git a/packages/postgresql-store/src/queries/__fixtures__/users-table.ts b/packages/postgresql-store/src/queries/__fixtures__/users-table.ts index baf5ff4..ad3fc41 100644 --- a/packages/postgresql-store/src/queries/__fixtures__/users-table.ts +++ b/packages/postgresql-store/src/queries/__fixtures__/users-table.ts @@ -15,6 +15,12 @@ export const usersTable_createSql = `CREATE TABLE IF NOT EXISTS ${usersTableName CONSTRAINT ${usersTableName}_pkey PRIMARY KEY (id) )`; +export const usersTable_phoneAddSql = `ALTER TABLE ${usersTableName} ADD COLUMN phone VARCHAR(20)`; + +export const usersTable_emailIndexCreateSql = `CREATE UNIQUE INDEX IF NOT EXISTS ${usersTableName}_email_idx ON ${usersTableName} (email ASC)`; + +export const usersTable_phoneEmailIndexCreateSql = `CREATE INDEX IF NOT EXISTS ${usersTableName}_phone_email_idx ON ${usersTableName} (phone DESC, email ASC)`; + export const usersTable: PostgreSQLTable = { table_name: usersTableName }; export const usersCollection_slim: StoreCollection_Slim = { @@ -78,9 +84,21 @@ export const usersTableColumns: PostgreSQLColumn[] = [ }, ]; +export const usersTablePrimaryIndexes: PostgreSQLIndexAttribute[] = [ + { + index_name: `${usersTableName}_id_idx`, + column_name: 'id', + seq_in_index: 1, + direction: 'ASC', + nulls: 'LAST', + is_unique: true, + is_primary: true, + }, +]; + export const usersTableIndexes: PostgreSQLIndexAttribute[] = [ { - index_name: 'idx_email', + index_name: `${usersTableName}_email_idx`, column_name: 'email', seq_in_index: 1, direction: 'ASC', @@ -89,7 +107,7 @@ export const usersTableIndexes: PostgreSQLIndexAttribute[] = [ is_primary: false, }, { - index_name: 'idx_phone_email', + index_name: `${usersTableName}_phone_email_idx`, column_name: 'phone', seq_in_index: 1, direction: 'DESC', @@ -98,7 +116,7 @@ export const usersTableIndexes: PostgreSQLIndexAttribute[] = [ is_primary: false, }, { - index_name: 'idx_phone_email', + index_name: `${usersTableName}_phone_email_idx`, column_name: 'email', seq_in_index: 2, direction: 'ASC', @@ -106,15 +124,7 @@ export const usersTableIndexes: PostgreSQLIndexAttribute[] = [ is_unique: false, is_primary: false, }, - { - index_name: 'users_pkey', - column_name: 'id', - seq_in_index: 1, - direction: 'ASC', - nulls: 'LAST', - is_unique: true, - is_primary: true, - }, + ...usersTablePrimaryIndexes, ]; export const usersCollection: StoreCollection = { @@ -170,25 +180,25 @@ export const usersCollection: StoreCollection = { }, }, primaryKey: { - name: `${usersTableName}_pkey`, + name: 'id', fields: { id: { sort: 'asc' } }, unique: 'primary', auto: 'increment', }, indexes: { - [`${usersTableName}_pkey`]: { - name: `${usersTableName}_pkey`, + id: { + name: 'id', fields: { id: { sort: 'asc' } }, unique: 'primary', auto: 'increment', }, - idx_email: { - name: 'idx_email', + email: { + name: 'email', fields: { email: { sort: 'asc' } }, unique: true, }, - idx_phone_email: { - name: 'idx_phone_email', + phone_email: { + name: 'phone_email', fields: { phone: { sort: 'desc' }, email: { sort: 'asc' } }, unique: false, }, diff --git a/packages/postgresql-store/src/queries/add-column.ts b/packages/postgresql-store/src/queries/add-column.ts index 6b1800b..0e973d3 100644 --- a/packages/postgresql-store/src/queries/add-column.ts +++ b/packages/postgresql-store/src/queries/add-column.ts @@ -2,10 +2,17 @@ import { SQLConnection } from '@neuledge/sql-store'; import { StoreCollection, StoreField } from '@neuledge/store'; export const addColumn = async ( - tableName: string, + collection: StoreCollection, field: StoreField, connection: SQLConnection, -): Promise => {}; +): Promise => { + await connection.query( + `ALTER TABLE ${collection.name} ADD COLUMN ${getColumnDefinition( + field, + collection, + )}`, + ); +}; export const getColumnDefinition = ( field: StoreField, diff --git a/packages/postgresql-store/src/queries/add-index.ts b/packages/postgresql-store/src/queries/add-index.ts index 75dd409..b05b8e6 100644 --- a/packages/postgresql-store/src/queries/add-index.ts +++ b/packages/postgresql-store/src/queries/add-index.ts @@ -1,15 +1,14 @@ import { SQLConnection, indexColumns } from '@neuledge/sql-store'; -import { StoreIndex } from '@neuledge/store'; +import { StoreCollection, StoreIndex } from '@neuledge/store'; export const addIndex = async ( - tableName: string, + collection: StoreCollection, index: StoreIndex, connection: SQLConnection, ): Promise => { await connection.query( - `CREATE ${ - index.unique ? 'UNIQUE INDEX' : 'INDEX' - } IF NOT EXISTS ? (${indexColumns(index)})`, - [`${tableName}_${index.name}_idx`], + `CREATE ${index.unique ? 'UNIQUE INDEX' : 'INDEX'} IF NOT EXISTS ${ + collection.name + }_${index.name}_idx ON ${collection.name} (${indexColumns(index)})`, ); }; diff --git a/packages/postgresql-store/src/queries/drop-index.ts b/packages/postgresql-store/src/queries/drop-index.ts index 595fa2d..13ff480 100644 --- a/packages/postgresql-store/src/queries/drop-index.ts +++ b/packages/postgresql-store/src/queries/drop-index.ts @@ -1,11 +1,12 @@ +import { StoreCollection } from '@neuledge/store'; import { SQLConnection } from '@neuledge/sql-store'; export const dropIndex = async ( - tableName: string, + collection: StoreCollection, index: string, connection: SQLConnection, ): Promise => { await connection.query(`DROP INDEX IF EXISTS ?`, [ - `${tableName}_${index}_idx`, + `${collection.name}_${index}_idx`, ]); }; diff --git a/packages/postgresql-store/src/queries/list-table-statistics.ts b/packages/postgresql-store/src/queries/list-table-statistics.ts index e6a2006..d020ef7 100644 --- a/packages/postgresql-store/src/queries/list-table-statistics.ts +++ b/packages/postgresql-store/src/queries/list-table-statistics.ts @@ -16,10 +16,22 @@ export interface PostgreSQLIndexAttribute { export const listIndexAttributes = async ( tableName: string, connection: SQLConnection, -): Promise => - connection.query(listIndexAttributes_sql, [ - tableName, - ]); +): Promise => { + const res = await connection.query( + listIndexAttributes_sql, + [tableName], + ); + + for (const row of res) { + if (!row.index_name.startsWith(`${tableName}_`)) continue; + + row.index_name = row.index_name + .slice(tableName.length + 1) + .replace(/_idx$/, ''); + } + + return res; +}; export const listIndexAttributes_sql = `SELECT irel.relname AS index_name, diff --git a/packages/postgresql-store/src/store.test.ts b/packages/postgresql-store/src/store.test.ts index 95d2b85..dec57f1 100644 --- a/packages/postgresql-store/src/store.test.ts +++ b/packages/postgresql-store/src/store.test.ts @@ -10,7 +10,11 @@ import { usersTableColumns, usersTableIndexes, usersTableName, + usersTablePrimaryIndexes, usersTable_createSql, + usersTable_emailIndexCreateSql, + usersTable_phoneAddSql, + usersTable_phoneEmailIndexCreateSql, } from './queries/__fixtures__/users-table'; import { PostgreSQLStore } from './store'; @@ -48,7 +52,7 @@ describe('store', () => { let query: jest.Mock; beforeEach(() => { - query = jest.fn(); + query = jest.fn().mockRejectedValue(new Error('unexpected query call')); store = new PostgreSQLStore({ client: { query } as never, @@ -110,6 +114,74 @@ describe('store', () => { usersTableName, ]); }); + + it('should create a new table', async () => { + query.mockResolvedValueOnce({ rows: [] }); + query.mockResolvedValueOnce({ rows: usersTableColumns }); + query.mockResolvedValueOnce({ rows: usersTablePrimaryIndexes }); + query.mockResolvedValueOnce({ rows: [] }); + query.mockResolvedValueOnce({ rows: [] }); + + await store.ensureCollection({ + collection: usersCollection, + fields: Object.values(usersCollection.fields), + indexes: Object.values(usersCollection.indexes), + }); + + expect(query).toHaveBeenCalledTimes(5); + expect(query).toHaveBeenNthCalledWith(1, usersTable_createSql, []); + expect(query).toHaveBeenNthCalledWith(2, listTableColumns_sql, [ + usersTableName, + ]); + expect(query).toHaveBeenNthCalledWith(3, listIndexAttributes_sql, [ + usersTableName, + ]); + expect(query).toHaveBeenNthCalledWith( + 4, + usersTable_emailIndexCreateSql, + [], + ); + expect(query).toHaveBeenNthCalledWith( + 5, + usersTable_phoneEmailIndexCreateSql, + [], + ); + }); + + it('should create fill missing fields and indexes', async () => { + query.mockResolvedValueOnce({ rows: [] }); + query.mockResolvedValueOnce({ + rows: usersTableColumns.filter((c) => c.column_name !== 'phone'), + }); + query.mockResolvedValueOnce({ + rows: usersTableIndexes.filter( + (i) => !i.index_name.includes('phone'), + ), + }); + query.mockResolvedValueOnce({ rows: [] }); + query.mockResolvedValueOnce({ rows: [] }); + + await store.ensureCollection({ + collection: usersCollection, + fields: Object.values(usersCollection.fields), + indexes: Object.values(usersCollection.indexes), + }); + + expect(query).toHaveBeenCalledTimes(5); + expect(query).toHaveBeenNthCalledWith(1, usersTable_createSql, []); + expect(query).toHaveBeenNthCalledWith(2, listTableColumns_sql, [ + usersTableName, + ]); + expect(query).toHaveBeenNthCalledWith(3, listIndexAttributes_sql, [ + usersTableName, + ]); + expect(query).toHaveBeenNthCalledWith(4, usersTable_phoneAddSql, []); + expect(query).toHaveBeenNthCalledWith( + 5, + usersTable_phoneEmailIndexCreateSql, + [], + ); + }); }); }); }); diff --git a/packages/sql-store/src/logic/collections/ensure.ts b/packages/sql-store/src/logic/collections/ensure.ts index 42ff986..abe72ac 100644 --- a/packages/sql-store/src/logic/collections/ensure.ts +++ b/packages/sql-store/src/logic/collections/ensure.ts @@ -19,22 +19,22 @@ export interface EnsureCollectionQueries { connection: SQLConnection, ) => Promise; addIndex: ( - tableName: string, + collection: StoreCollection, index: StoreIndex, connection: SQLConnection, ) => Promise; addColumn: ( - tableName: string, + collection: StoreCollection, field: StoreField, connection: SQLConnection, ) => Promise; dropIndex?: ( - tableName: string, + collection: StoreCollection, index: string, connection: SQLConnection, ) => Promise; dropColumn?: ( - tableName: string, + collection: StoreCollection, field: string, connection: SQLConnection, ) => Promise; @@ -57,18 +57,17 @@ export const ensureCollection = async < ): Promise => { await createTableIfNotExists(options.collection, connection); - const tableName = options.collection.name; const asyncLimit = pLimit(4); await Promise.all( options.dropIndexes?.map((index) => - asyncLimit(() => dropIndex(tableName, index, connection)), + asyncLimit(() => dropIndex(options.collection, index, connection)), ) || [], ); await Promise.all( options.dropFields?.map((field) => - asyncLimit(() => dropColumn(tableName, field, connection)), + asyncLimit(() => dropColumn(options.collection, field, connection)), ) || [], ); @@ -83,16 +82,19 @@ export const ensureCollection = async < // the engine to ensure that new columns are nullable if inserted after the // collection has been created and this is the current implementation. - await Promise.all([ - ...(options.indexes - ?.filter((index) => !collection.indexes[index.name]) - .map((index) => - asyncLimit(() => addIndex(tableName, index, connection)), - ) || []), - ...(options.fields + await Promise.all( + options.fields ?.filter((field) => !collection.fields[field.name]) .map((field) => - asyncLimit(() => addColumn(tableName, field, connection)), - ) || []), - ]); + asyncLimit(() => addColumn(collection, field, connection)), + ) || [], + ); + + await Promise.all( + options.indexes + ?.filter((index) => !collection.indexes[index.name]) + .map((index) => + asyncLimit(() => addIndex(collection, index, connection)), + ) || [], + ); }; diff --git a/packages/sql-store/src/queries/drop-column.ts b/packages/sql-store/src/queries/drop-column.ts index 6d775a8..54dd8ec 100644 --- a/packages/sql-store/src/queries/drop-column.ts +++ b/packages/sql-store/src/queries/drop-column.ts @@ -1,9 +1,13 @@ +import { StoreCollection } from '@neuledge/store'; import { SQLConnection } from './connection'; export const dropColumn = async ( - tableName: string, + collection: StoreCollection, field: string, connection: SQLConnection, ): Promise => { - await connection.query(`ALTER TABLE ? DROP COLUMN ?`, [tableName, field]); + await connection.query(`ALTER TABLE ? DROP COLUMN ?`, [ + collection.name, + field, + ]); }; diff --git a/packages/sql-store/src/queries/drop-index.ts b/packages/sql-store/src/queries/drop-index.ts index 642fb80..7c46308 100644 --- a/packages/sql-store/src/queries/drop-index.ts +++ b/packages/sql-store/src/queries/drop-index.ts @@ -1,9 +1,10 @@ +import { StoreCollection } from '@neuledge/store'; import { SQLConnection } from './connection'; export const dropIndex = async ( - tableName: string, + collection: StoreCollection, index: string, connection: SQLConnection, ): Promise => { - await connection.query(`DROP INDEX ? ON ?`, [index, tableName]); + await connection.query(`DROP INDEX ? ON ?`, [index, collection.name]); }; From f7788d96ba3a4532272afdc09ed1c62c3eea7d7b Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Sat, 15 Apr 2023 01:12:51 +0300 Subject: [PATCH 11/24] fix pg pk with desc columns --- packages/postgresql-store/src/queries/create-table.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/postgresql-store/src/queries/create-table.ts b/packages/postgresql-store/src/queries/create-table.ts index 6a0a9e1..184048d 100644 --- a/packages/postgresql-store/src/queries/create-table.ts +++ b/packages/postgresql-store/src/queries/create-table.ts @@ -1,6 +1,7 @@ import { SQLConnection } from '@neuledge/sql-store'; import { StoreCollection } from '@neuledge/store'; import { getColumnDefinition } from './add-column'; +import { addIndex } from './add-index'; export const createTableIfNotExists = async ( collection: StoreCollection, @@ -17,6 +18,13 @@ export const createTableIfNotExists = async ( )`, ); - // FIXME add unique constraints if primary key has descending fields + // add unique constraints if primary key has descending fields // https://stackoverflow.com/a/45604459/518153 + if ( + Object.values(collection.primaryKey.fields).some( + (field) => field.sort === 'desc', + ) + ) { + await addIndex(collection, collection.primaryKey, connection); + } }; From d1e1fb1ba02c24d3a1866851c2b8edd41c8d9826 Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Sat, 15 Apr 2023 01:17:42 +0300 Subject: [PATCH 12/24] test dropCollection pg --- .../src/queries/__fixtures__/users-table.ts | 2 +- packages/postgresql-store/src/store.test.ts | 14 ++++++++++++++ packages/sql-store/src/queries/drop-table.ts | 4 +++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/postgresql-store/src/queries/__fixtures__/users-table.ts b/packages/postgresql-store/src/queries/__fixtures__/users-table.ts index ad3fc41..d55ca00 100644 --- a/packages/postgresql-store/src/queries/__fixtures__/users-table.ts +++ b/packages/postgresql-store/src/queries/__fixtures__/users-table.ts @@ -1,7 +1,7 @@ import { StoreCollection, StoreCollection_Slim } from '@neuledge/store'; import { PostgreSQLTable } from '../list-tables'; import { PostgreSQLColumn } from '../list-table-columns'; -import { PostgreSQLIndexAttribute } from '..'; +import { PostgreSQLIndexAttribute } from '../list-table-statistics'; export const usersTableName = 'users'; diff --git a/packages/postgresql-store/src/store.test.ts b/packages/postgresql-store/src/store.test.ts index dec57f1..1a8e824 100644 --- a/packages/postgresql-store/src/store.test.ts +++ b/packages/postgresql-store/src/store.test.ts @@ -1,3 +1,4 @@ +import { dropTableIfExists_sql } from '@neuledge/sql-store'; import { listIndexAttributes_sql, listTableColumns_sql, @@ -183,5 +184,18 @@ describe('store', () => { ); }); }); + + describe('.dropCollection()', () => { + it('should be able to drop a collection', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await store.dropCollection({ collection: usersCollection }); + + expect(query).toHaveBeenCalledTimes(1); + expect(query).toHaveBeenCalledWith(dropTableIfExists_sql, [ + usersTableName, + ]); + }); + }); }); }); diff --git a/packages/sql-store/src/queries/drop-table.ts b/packages/sql-store/src/queries/drop-table.ts index a6a0689..874609c 100644 --- a/packages/sql-store/src/queries/drop-table.ts +++ b/packages/sql-store/src/queries/drop-table.ts @@ -3,4 +3,6 @@ import { SQLConnection } from './connection'; export const dropTableIfExists = async ( connection: SQLConnection, tableName: string, -): Promise => connection.query(`DROP TABLE IF EXISTS ?`, [tableName]); +): Promise => connection.query(dropTableIfExists_sql, [tableName]); + +export const dropTableIfExists_sql = `DROP TABLE IF EXISTS ?`; From e466c71fe4323cbb2eacb3b1b82701b1b623a4ff Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Sun, 16 Apr 2023 19:42:23 +0300 Subject: [PATCH 13/24] pg literal escapes --- .../mysql-store/src/queries/add-column.ts | 4 +- packages/mysql-store/src/queries/add-index.ts | 15 +-- .../mysql-store/src/queries/connection.ts | 3 + .../mysql-store/src/queries/create-table.ts | 15 +-- .../mysql-store/src/queries/drop-column.ts | 8 ++ .../mysql-store/src/queries/drop-index.ts | 8 ++ .../mysql-store/src/queries/drop-table.ts | 6 ++ packages/mysql-store/src/queries/index.ts | 5 +- .../src/queries/list-table-columns.ts | 13 ++- .../src/queries/list-table-statistics.ts | 13 ++- .../mysql-store/src/queries/list-tables.ts | 11 ++- packages/mysql-store/src/store.ts | 25 ++--- packages/postgresql-store/package.json | 8 +- .../src/queries/__fixtures__/users-table.ts | 24 ++--- .../src/queries/add-column.ts | 14 +-- .../postgresql-store/src/queries/add-index.ts | 18 +++- .../src/queries/connection.ts | 3 + .../src/queries/create-table.ts | 17 ++-- .../src/queries/drop-column.ts | 15 +++ .../src/queries/drop-index.ts | 11 ++- .../src/queries/drop-table.ts | 9 ++ .../postgresql-store/src/queries/index.ts | 2 + .../src/queries/list-table-columns.ts | 10 +- .../src/queries/list-table-statistics.ts | 12 +-- .../src/queries/list-tables.ts | 8 +- packages/postgresql-store/src/store.test.ts | 19 ++-- packages/postgresql-store/src/store.ts | 20 ++-- packages/sql-store/src/index.ts | 1 - .../src/logic/collections/describe.ts | 93 +++++++++++-------- .../sql-store/src/logic/collections/drop.ts | 10 +- .../sql-store/src/logic/collections/ensure.ts | 61 ++++++------ .../sql-store/src/logic/collections/list.ts | 11 +-- packages/sql-store/src/queries/connection.ts | 3 - packages/sql-store/src/queries/drop-column.ts | 13 --- packages/sql-store/src/queries/drop-index.ts | 10 -- packages/sql-store/src/queries/drop-table.ts | 8 -- .../sql-store/src/queries/index-columns.ts | 6 -- packages/sql-store/src/queries/index.ts | 5 - yarn.lock | 10 ++ 39 files changed, 292 insertions(+), 255 deletions(-) create mode 100644 packages/mysql-store/src/queries/connection.ts create mode 100644 packages/mysql-store/src/queries/drop-column.ts create mode 100644 packages/mysql-store/src/queries/drop-index.ts create mode 100644 packages/mysql-store/src/queries/drop-table.ts create mode 100644 packages/postgresql-store/src/queries/connection.ts create mode 100644 packages/postgresql-store/src/queries/drop-column.ts create mode 100644 packages/postgresql-store/src/queries/drop-table.ts delete mode 100644 packages/sql-store/src/queries/connection.ts delete mode 100644 packages/sql-store/src/queries/drop-column.ts delete mode 100644 packages/sql-store/src/queries/drop-index.ts delete mode 100644 packages/sql-store/src/queries/drop-table.ts delete mode 100644 packages/sql-store/src/queries/index-columns.ts delete mode 100644 packages/sql-store/src/queries/index.ts diff --git a/packages/mysql-store/src/queries/add-column.ts b/packages/mysql-store/src/queries/add-column.ts index 4b0716a..166da74 100644 --- a/packages/mysql-store/src/queries/add-column.ts +++ b/packages/mysql-store/src/queries/add-column.ts @@ -1,8 +1,8 @@ -import { SQLConnection } from '@neuledge/sql-store'; import { StoreCollection, StoreField } from '@neuledge/store'; +import { MySQLConnection } from './connection'; export const addColumn = async ( + connection: MySQLConnection, collection: StoreCollection, field: StoreField, - connection: SQLConnection, ): Promise => {}; diff --git a/packages/mysql-store/src/queries/add-index.ts b/packages/mysql-store/src/queries/add-index.ts index 699b8d7..4cb7f21 100644 --- a/packages/mysql-store/src/queries/add-index.ts +++ b/packages/mysql-store/src/queries/add-index.ts @@ -1,17 +1,8 @@ -import { SQLConnection, indexColumns } from '@neuledge/sql-store'; import { StoreCollection, StoreIndex } from '@neuledge/store'; - -// FIXME handle if not exists on mysql +import { MySQLConnection } from './connection'; export const addIndex = async ( + connection: MySQLConnection, collection: StoreCollection, index: StoreIndex, - connection: SQLConnection, -): Promise => { - await connection.query( - `CREATE ${ - index.unique ? 'UNIQUE INDEX' : 'INDEX' - } IF NOT EXISTS ? ON ? (${indexColumns(index)})`, - [index.name, collection.name], - ); -}; +): Promise => {}; diff --git a/packages/mysql-store/src/queries/connection.ts b/packages/mysql-store/src/queries/connection.ts new file mode 100644 index 0000000..d65c071 --- /dev/null +++ b/packages/mysql-store/src/queries/connection.ts @@ -0,0 +1,3 @@ +import { Connection, Pool } from 'mysql'; + +export type MySQLConnection = Pick; diff --git a/packages/mysql-store/src/queries/create-table.ts b/packages/mysql-store/src/queries/create-table.ts index af8195e..1deeaf3 100644 --- a/packages/mysql-store/src/queries/create-table.ts +++ b/packages/mysql-store/src/queries/create-table.ts @@ -1,16 +1,7 @@ -import { SQLConnection } from '@neuledge/sql-store'; import { StoreCollection } from '@neuledge/store'; - -// FIXME handle if not exists on mysql +import { MySQLConnection } from './connection'; export const createTableIfNotExists = async ( + connection: MySQLConnection, collection: StoreCollection, - connection: SQLConnection, -): Promise => { - await connection.query( - `CREATE TABLE IF NOT EXISTS ? ( - ${/* FIXME add columns */ ''} - )`, - [collection.name], - ); -}; +): Promise => {}; diff --git a/packages/mysql-store/src/queries/drop-column.ts b/packages/mysql-store/src/queries/drop-column.ts new file mode 100644 index 0000000..594fdad --- /dev/null +++ b/packages/mysql-store/src/queries/drop-column.ts @@ -0,0 +1,8 @@ +import { StoreCollection } from '@neuledge/store'; +import { MySQLConnection } from './connection'; + +export const dropColumn = async ( + connection: MySQLConnection, + collection: StoreCollection, + field: string, +): Promise => {}; diff --git a/packages/mysql-store/src/queries/drop-index.ts b/packages/mysql-store/src/queries/drop-index.ts new file mode 100644 index 0000000..0d05fb1 --- /dev/null +++ b/packages/mysql-store/src/queries/drop-index.ts @@ -0,0 +1,8 @@ +import { StoreCollection } from '@neuledge/store'; +import { MySQLConnection } from './connection'; + +export const dropIndex = async ( + connection: MySQLConnection, + collection: StoreCollection, + index: string, +): Promise => {}; diff --git a/packages/mysql-store/src/queries/drop-table.ts b/packages/mysql-store/src/queries/drop-table.ts new file mode 100644 index 0000000..14716fb --- /dev/null +++ b/packages/mysql-store/src/queries/drop-table.ts @@ -0,0 +1,6 @@ +import { MySQLConnection } from './connection'; + +export const dropTableIfExists = async ( + connection: MySQLConnection, + tableName: string, +): Promise => {}; diff --git a/packages/mysql-store/src/queries/index.ts b/packages/mysql-store/src/queries/index.ts index 45b41a4..fe9dba7 100644 --- a/packages/mysql-store/src/queries/index.ts +++ b/packages/mysql-store/src/queries/index.ts @@ -1,6 +1,9 @@ export * from './add-column'; export * from './add-index'; export * from './create-table'; -export * from './list-tables'; +export * from './drop-column'; +export * from './drop-index'; +export * from './drop-table'; export * from './list-table-columns'; export * from './list-table-statistics'; +export * from './list-tables'; diff --git a/packages/mysql-store/src/queries/list-table-columns.ts b/packages/mysql-store/src/queries/list-table-columns.ts index 3e97bba..bfb3018 100644 --- a/packages/mysql-store/src/queries/list-table-columns.ts +++ b/packages/mysql-store/src/queries/list-table-columns.ts @@ -1,5 +1,5 @@ import { StoreShapeType } from '@neuledge/store'; -import { SQLConnection } from '@neuledge/sql-store'; +import { MySQLConnection } from './connection'; /** * A table column from the information_schema.columns table. @@ -15,14 +15,17 @@ export interface MySQLColumn { } export const listTableColumns = async ( + connection: MySQLConnection, tableName: string, - connection: SQLConnection, ): Promise => - connection.query( - `SELECT column_name, data_type, character_maximum_length, numeric_precision, numeric_scale, (is_nullable = 'YES') AS is_nullable, extra LIKE '%auto_increment%' AS is_auto_increment + new Promise((resolve, reject) => + connection.query( + `SELECT column_name, data_type, character_maximum_length, numeric_precision, numeric_scale, (is_nullable = 'YES') AS is_nullable, extra LIKE '%auto_increment%' AS is_auto_increment FROM information_schema.columns WHERE table_schema = DATABASE() AND table_name = ?`, - [tableName], + [tableName], + (error, results) => (error ? reject(error) : resolve(results)), + ), ); export const dataTypeMap: Record = { diff --git a/packages/mysql-store/src/queries/list-table-statistics.ts b/packages/mysql-store/src/queries/list-table-statistics.ts index fb1509e..f6816f0 100644 --- a/packages/mysql-store/src/queries/list-table-statistics.ts +++ b/packages/mysql-store/src/queries/list-table-statistics.ts @@ -1,4 +1,4 @@ -import { SQLConnection } from '@neuledge/sql-store'; +import { MySQLConnection } from './connection'; /** * A table statistic row from the information_schema.statistics table. @@ -13,13 +13,16 @@ export interface MySQLIndexAttribute { } export const listIndexAttributes = async ( + connection: MySQLConnection, tableName: string, - connection: SQLConnection, ): Promise => - connection.query( - `SELECT index_name, column_name, seq_in_index, CASE collation WHEN 'A' THEN 'ASC' ELSE 'DESC' END AS direction, non_unique, (index_name == 'PRIMARY') as is_primary + new Promise((resolve, reject) => + connection.query( + `SELECT index_name, column_name, seq_in_index, CASE collation WHEN 'A' THEN 'ASC' ELSE 'DESC' END AS direction, non_unique, (index_name == 'PRIMARY') as is_primary FROM information_schema.statistics WHERE table_schema = DATABASE() AND table_name = ? ORDER BY index_name, seq_in_index`, - [tableName], + [tableName], + (error, results) => (error ? reject(error) : resolve(results)), + ), ); diff --git a/packages/mysql-store/src/queries/list-tables.ts b/packages/mysql-store/src/queries/list-tables.ts index 23fdb8f..ee6a9ee 100644 --- a/packages/mysql-store/src/queries/list-tables.ts +++ b/packages/mysql-store/src/queries/list-tables.ts @@ -1,4 +1,4 @@ -import { SQLConnection } from '@neuledge/sql-store'; +import { MySQLConnection } from './connection'; /** * The tables in the database. This is a view of the `information_schema.tables` table. @@ -8,8 +8,11 @@ export interface MySQLTable { } export const listTables = async ( - connection: SQLConnection, + connection: MySQLConnection, ): Promise => - connection.query( - `SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE()`, + new Promise((resolve, reject) => + connection.query( + `SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE()`, + (error, results) => (error ? reject(error) : resolve(results)), + ), ); diff --git a/packages/mysql-store/src/store.ts b/packages/mysql-store/src/store.ts index 522f486..741b69f 100644 --- a/packages/mysql-store/src/store.ts +++ b/packages/mysql-store/src/store.ts @@ -23,9 +23,11 @@ import { createTableIfNotExists, addIndex, addColumn, + dropColumn, + dropIndex, + dropTableIfExists, } from './queries'; import { - SQLConnection, describeCollection, dropCollection, ensureCollection, @@ -37,27 +39,18 @@ export type MySQLStoreClient = Pool | Connection; export type MySQLStoreOptions = PoolConfig | { client: MySQLStoreClient }; export class MySQLStore implements Store { - private client: MySQLStoreClient; - private connection: SQLConnection; + private connection: MySQLStoreClient; constructor(options: MySQLStoreOptions) { - this.client = 'client' in options ? options.client : createPool(options); - - this.connection = { - query: (sql, values) => - new Promise((resolve, reject) => - this.client.query(sql, values, (error, results) => - error ? reject(error) : resolve(results), - ), - ), - }; + this.connection = + 'client' in options ? options.client : createPool(options); } // connection methods async close(): Promise { await new Promise((resolve, reject) => - this.client.end((error) => (error ? reject(error) : resolve())), + this.connection.end((error) => (error ? reject(error) : resolve())), ); } @@ -82,6 +75,8 @@ export class MySQLStore implements Store { createTableIfNotExists, addIndex, addColumn, + dropColumn, + dropIndex, listTableColumns, listIndexAttributes, dataTypeMap, @@ -89,7 +84,7 @@ export class MySQLStore implements Store { } async dropCollection(options: StoreDropCollectionOptions): Promise { - return dropCollection(options, this.connection); + return dropCollection(options, this.connection, { dropTableIfExists }); } async find(options: StoreFindOptions): Promise> { diff --git a/packages/postgresql-store/package.json b/packages/postgresql-store/package.json index 4d1091f..cf2d864 100644 --- a/packages/postgresql-store/package.json +++ b/packages/postgresql-store/package.json @@ -38,9 +38,13 @@ "lint:strict": "yarn lint --max-warnings 0" }, "dependencies": { - "@neuledge/store": "^0.2.0", "@neuledge/sql-store": "^0.0.0", + "@neuledge/store": "^0.2.0", "@types/pg": "^8.6.6", - "pg": "^8.10.0" + "pg": "^8.10.0", + "pg-format": "^1.0.4" + }, + "devDependencies": { + "@types/pg-format": "^1.0.2" } } diff --git a/packages/postgresql-store/src/queries/__fixtures__/users-table.ts b/packages/postgresql-store/src/queries/__fixtures__/users-table.ts index d55ca00..9d21cb5 100644 --- a/packages/postgresql-store/src/queries/__fixtures__/users-table.ts +++ b/packages/postgresql-store/src/queries/__fixtures__/users-table.ts @@ -5,21 +5,23 @@ import { PostgreSQLIndexAttribute } from '../list-table-statistics'; export const usersTableName = 'users'; -export const usersTable_createSql = `CREATE TABLE IF NOT EXISTS ${usersTableName} ( - id BIGSERIAL NOT NULL, - name VARCHAR(50), - email VARCHAR(100) NOT NULL, - phone VARCHAR(20), - created_at TIMESTAMP NOT NULL, - updated_at TIMESTAMP NOT NULL, - CONSTRAINT ${usersTableName}_pkey PRIMARY KEY (id) +export const usersTable_dropSql = `DROP TABLE IF EXISTS '${usersTableName}'`; + +export const usersTable_createSql = `CREATE TABLE IF NOT EXISTS '${usersTableName}' ( + 'id' BIGSERIAL NOT NULL, + 'name' VARCHAR(50), + 'email' VARCHAR(100) NOT NULL, + 'phone' VARCHAR(20), + 'created_at' TIMESTAMP NOT NULL, + 'updated_at' TIMESTAMP NOT NULL, + CONSTRAINT '${usersTableName}_pkey' PRIMARY KEY ('id') )`; -export const usersTable_phoneAddSql = `ALTER TABLE ${usersTableName} ADD COLUMN phone VARCHAR(20)`; +export const usersTable_phoneAddSql = `ALTER TABLE '${usersTableName}' ADD COLUMN 'phone' VARCHAR(20)`; -export const usersTable_emailIndexCreateSql = `CREATE UNIQUE INDEX IF NOT EXISTS ${usersTableName}_email_idx ON ${usersTableName} (email ASC)`; +export const usersTable_emailIndexCreateSql = `CREATE UNIQUE INDEX IF NOT EXISTS '${usersTableName}_email_idx' ON '${usersTableName}' ('email' ASC)`; -export const usersTable_phoneEmailIndexCreateSql = `CREATE INDEX IF NOT EXISTS ${usersTableName}_phone_email_idx ON ${usersTableName} (phone DESC, email ASC)`; +export const usersTable_phoneEmailIndexCreateSql = `CREATE INDEX IF NOT EXISTS '${usersTableName}_phone_email_idx' ON '${usersTableName}' ('phone' DESC, 'email' ASC)`; export const usersTable: PostgreSQLTable = { table_name: usersTableName }; diff --git a/packages/postgresql-store/src/queries/add-column.ts b/packages/postgresql-store/src/queries/add-column.ts index 0e973d3..31226e8 100644 --- a/packages/postgresql-store/src/queries/add-column.ts +++ b/packages/postgresql-store/src/queries/add-column.ts @@ -1,16 +1,16 @@ -import { SQLConnection } from '@neuledge/sql-store'; import { StoreCollection, StoreField } from '@neuledge/store'; +import { PostgreSQLConnection } from './connection'; +import format from 'pg-format'; export const addColumn = async ( + connection: PostgreSQLConnection, collection: StoreCollection, field: StoreField, - connection: SQLConnection, ): Promise => { await connection.query( - `ALTER TABLE ${collection.name} ADD COLUMN ${getColumnDefinition( - field, - collection, - )}`, + `ALTER TABLE ${format.literal( + collection.name, + )} ADD COLUMN ${getColumnDefinition(field, collection)}`, ); }; @@ -18,7 +18,7 @@ export const getColumnDefinition = ( field: StoreField, collection: StoreCollection, ): string => - `${field.name} ${getColumnDataType(field, collection)}${ + `${format.literal(field.name)} ${getColumnDataType(field, collection)}${ field.nullable ? '' : ' NOT NULL' }`; diff --git a/packages/postgresql-store/src/queries/add-index.ts b/packages/postgresql-store/src/queries/add-index.ts index b05b8e6..0e83e11 100644 --- a/packages/postgresql-store/src/queries/add-index.ts +++ b/packages/postgresql-store/src/queries/add-index.ts @@ -1,14 +1,22 @@ -import { SQLConnection, indexColumns } from '@neuledge/sql-store'; import { StoreCollection, StoreIndex } from '@neuledge/store'; +import { PostgreSQLConnection } from './connection'; +import format from 'pg-format'; export const addIndex = async ( + connection: PostgreSQLConnection, collection: StoreCollection, index: StoreIndex, - connection: SQLConnection, ): Promise => { await connection.query( - `CREATE ${index.unique ? 'UNIQUE INDEX' : 'INDEX'} IF NOT EXISTS ${ - collection.name - }_${index.name}_idx ON ${collection.name} (${indexColumns(index)})`, + `CREATE ${ + index.unique ? 'UNIQUE INDEX' : 'INDEX' + } IF NOT EXISTS ${format.literal( + `${collection.name}_${index.name}_idx`, + )} ON ${format.literal(collection.name)} (${Object.entries(index.fields) + .map( + ([field, { sort }]) => + `${format.literal(field)} ${sort === 'desc' ? 'DESC' : 'ASC'}`, + ) + .join(', ')})`, ); }; diff --git a/packages/postgresql-store/src/queries/connection.ts b/packages/postgresql-store/src/queries/connection.ts new file mode 100644 index 0000000..2c4239d --- /dev/null +++ b/packages/postgresql-store/src/queries/connection.ts @@ -0,0 +1,3 @@ +import { Client, Pool } from 'pg'; + +export type PostgreSQLConnection = Pick; diff --git a/packages/postgresql-store/src/queries/create-table.ts b/packages/postgresql-store/src/queries/create-table.ts index 184048d..95d6e5d 100644 --- a/packages/postgresql-store/src/queries/create-table.ts +++ b/packages/postgresql-store/src/queries/create-table.ts @@ -1,20 +1,23 @@ -import { SQLConnection } from '@neuledge/sql-store'; import { StoreCollection } from '@neuledge/store'; import { getColumnDefinition } from './add-column'; import { addIndex } from './add-index'; +import { PostgreSQLConnection } from './connection'; +import format from 'pg-format'; export const createTableIfNotExists = async ( + connection: PostgreSQLConnection, collection: StoreCollection, - connection: SQLConnection, ): Promise => { await connection.query( - `CREATE TABLE IF NOT EXISTS ${collection.name} ( + `CREATE TABLE IF NOT EXISTS ${format.literal(collection.name)} ( ${Object.values(collection.fields) .map((field) => getColumnDefinition(field, collection)) .join(',\n ')}, - CONSTRAINT ${collection.name}_pkey PRIMARY KEY (${Object.keys( - collection.primaryKey.fields, - ).join(', ')}) + CONSTRAINT ${format.literal( + `${collection.name}_pkey`, + )} PRIMARY KEY (${Object.keys(collection.primaryKey.fields) + .map((val) => format.literal(val)) + .join(', ')}) )`, ); @@ -25,6 +28,6 @@ export const createTableIfNotExists = async ( (field) => field.sort === 'desc', ) ) { - await addIndex(collection, collection.primaryKey, connection); + await addIndex(connection, collection, collection.primaryKey); } }; diff --git a/packages/postgresql-store/src/queries/drop-column.ts b/packages/postgresql-store/src/queries/drop-column.ts new file mode 100644 index 0000000..663d5e4 --- /dev/null +++ b/packages/postgresql-store/src/queries/drop-column.ts @@ -0,0 +1,15 @@ +import { StoreCollection } from '@neuledge/store'; +import { PostgreSQLConnection } from './connection'; +import format from 'pg-format'; + +export const dropColumn = async ( + connection: PostgreSQLConnection, + collection: StoreCollection, + field: string, +): Promise => { + await connection.query( + `ALTER TABLE ${format.literal( + collection.name, + )} DROP COLUMN ${format.literal(field)}`, + ); +}; diff --git a/packages/postgresql-store/src/queries/drop-index.ts b/packages/postgresql-store/src/queries/drop-index.ts index 13ff480..63d48de 100644 --- a/packages/postgresql-store/src/queries/drop-index.ts +++ b/packages/postgresql-store/src/queries/drop-index.ts @@ -1,12 +1,13 @@ import { StoreCollection } from '@neuledge/store'; -import { SQLConnection } from '@neuledge/sql-store'; +import { PostgreSQLConnection } from './connection'; +import format from 'pg-format'; export const dropIndex = async ( + connection: PostgreSQLConnection, collection: StoreCollection, index: string, - connection: SQLConnection, ): Promise => { - await connection.query(`DROP INDEX IF EXISTS ?`, [ - `${collection.name}_${index}_idx`, - ]); + await connection.query( + `DROP INDEX IF EXISTS ${format.literal(`${collection.name}_${index}_idx`)}`, + ); }; diff --git a/packages/postgresql-store/src/queries/drop-table.ts b/packages/postgresql-store/src/queries/drop-table.ts new file mode 100644 index 0000000..3eee5e9 --- /dev/null +++ b/packages/postgresql-store/src/queries/drop-table.ts @@ -0,0 +1,9 @@ +import format from 'pg-format'; +import { PostgreSQLConnection } from './connection'; + +export const dropTableIfExists = async ( + connection: PostgreSQLConnection, + tableName: string, +): Promise => { + await connection.query(`DROP TABLE IF EXISTS ${format.literal(tableName)}`); +}; diff --git a/packages/postgresql-store/src/queries/index.ts b/packages/postgresql-store/src/queries/index.ts index 66fa894..671584a 100644 --- a/packages/postgresql-store/src/queries/index.ts +++ b/packages/postgresql-store/src/queries/index.ts @@ -1,6 +1,8 @@ export * from './add-column'; export * from './add-index'; +export * from './drop-column'; export * from './drop-index'; +export * from './drop-table'; export * from './create-table'; export * from './list-tables'; export * from './list-table-columns'; diff --git a/packages/postgresql-store/src/queries/list-table-columns.ts b/packages/postgresql-store/src/queries/list-table-columns.ts index 029775b..625e032 100644 --- a/packages/postgresql-store/src/queries/list-table-columns.ts +++ b/packages/postgresql-store/src/queries/list-table-columns.ts @@ -1,5 +1,5 @@ import { StoreShapeType } from '@neuledge/store'; -import { SQLConnection } from '@neuledge/sql-store'; +import { PostgreSQLConnection } from './connection'; /** * A table column from the information_schema.columns table. @@ -15,14 +15,16 @@ export interface PostgreSQLColumn { } export const listTableColumns = async ( + connection: PostgreSQLConnection, tableName: string, - connection: SQLConnection, ): Promise => - connection.query(listTableColumns_sql, [tableName]); + connection + .query(listTableColumns_sql, [tableName]) + .then((result) => result.rows); export const listTableColumns_sql = `SELECT column_name, data_type, character_maximum_length, numeric_precision, numeric_scale, (is_nullable = 'YES') as is_nullable, column_default LIKE 'nextval(%)' AS is_auto_increment FROM information_schema.columns -WHERE table_catalog = current_database() AND table_schema = current_schema() AND table_name = ?`; +WHERE table_catalog = current_database() AND table_schema = current_schema() AND table_name = $1`; // https://www.postgresql.org/docs/current/datatype.html export const dataTypeMap: Record = { diff --git a/packages/postgresql-store/src/queries/list-table-statistics.ts b/packages/postgresql-store/src/queries/list-table-statistics.ts index d020ef7..6a23fde 100644 --- a/packages/postgresql-store/src/queries/list-table-statistics.ts +++ b/packages/postgresql-store/src/queries/list-table-statistics.ts @@ -1,4 +1,4 @@ -import { SQLConnection } from '@neuledge/sql-store'; +import { PostgreSQLConnection } from './connection'; /** * A table statistic row from the information_schema.statistics table. @@ -14,15 +14,15 @@ export interface PostgreSQLIndexAttribute { } export const listIndexAttributes = async ( + connection: PostgreSQLConnection, tableName: string, - connection: SQLConnection, ): Promise => { - const res = await connection.query( + const { rows } = await connection.query( listIndexAttributes_sql, [tableName], ); - for (const row of res) { + for (const row of rows) { if (!row.index_name.startsWith(`${tableName}_`)) continue; row.index_name = row.index_name @@ -30,7 +30,7 @@ export const listIndexAttributes = async ( .replace(/_idx$/, ''); } - return res; + return rows; }; export const listIndexAttributes_sql = `SELECT @@ -49,5 +49,5 @@ CROSS JOIN LATERAL unnest (i.indkey) WITH ORDINALITY AS c (colnum, ordinality) LEFT JOIN LATERAL unnest (i.indoption) WITH ORDINALITY AS o (option, ordinality) ON c.ordinality = o.ordinality JOIN pg_attribute AS a ON trel.oid = a.attrelid AND a.attnum = c.colnum -WHERE tnsp.nspname = current_schema() AND trel.relname = ? +WHERE tnsp.nspname = current_schema() AND trel.relname = $1 ORDER BY index_name, seq_in_index`; diff --git a/packages/postgresql-store/src/queries/list-tables.ts b/packages/postgresql-store/src/queries/list-tables.ts index e4aad48..b5c29ee 100644 --- a/packages/postgresql-store/src/queries/list-tables.ts +++ b/packages/postgresql-store/src/queries/list-tables.ts @@ -1,4 +1,4 @@ -import { SQLConnection } from '@neuledge/sql-store'; +import { PostgreSQLConnection } from './connection'; /** * The tables in the database. This is a view of the `information_schema.tables` table. @@ -8,9 +8,11 @@ export interface PostgreSQLTable { } export const listTables = async ( - connection: SQLConnection, + connection: PostgreSQLConnection, ): Promise => - connection.query(listTables_sql); + connection + .query(listTables_sql) + .then((result) => result.rows); export const listTables_sql = `SELECT table_name FROM information_schema.tables diff --git a/packages/postgresql-store/src/store.test.ts b/packages/postgresql-store/src/store.test.ts index 1a8e824..426c464 100644 --- a/packages/postgresql-store/src/store.test.ts +++ b/packages/postgresql-store/src/store.test.ts @@ -1,4 +1,3 @@ -import { dropTableIfExists_sql } from '@neuledge/sql-store'; import { listIndexAttributes_sql, listTableColumns_sql, @@ -13,6 +12,7 @@ import { usersTableName, usersTablePrimaryIndexes, usersTable_createSql, + usersTable_dropSql, usersTable_emailIndexCreateSql, usersTable_phoneAddSql, usersTable_phoneEmailIndexCreateSql, @@ -67,7 +67,7 @@ describe('store', () => { const collections = await store.listCollections(); expect(query).toHaveBeenCalledTimes(1); - expect(query).toHaveBeenCalledWith(listTables_sql, []); + expect(query).toHaveBeenCalledWith(listTables_sql); expect(collections).toEqual([usersCollection_slim]); }); @@ -107,7 +107,7 @@ describe('store', () => { }); expect(query).toHaveBeenCalledTimes(3); - expect(query).toHaveBeenNthCalledWith(1, usersTable_createSql, []); + expect(query).toHaveBeenNthCalledWith(1, usersTable_createSql); expect(query).toHaveBeenNthCalledWith(2, listTableColumns_sql, [ usersTableName, ]); @@ -130,7 +130,7 @@ describe('store', () => { }); expect(query).toHaveBeenCalledTimes(5); - expect(query).toHaveBeenNthCalledWith(1, usersTable_createSql, []); + expect(query).toHaveBeenNthCalledWith(1, usersTable_createSql); expect(query).toHaveBeenNthCalledWith(2, listTableColumns_sql, [ usersTableName, ]); @@ -140,12 +140,10 @@ describe('store', () => { expect(query).toHaveBeenNthCalledWith( 4, usersTable_emailIndexCreateSql, - [], ); expect(query).toHaveBeenNthCalledWith( 5, usersTable_phoneEmailIndexCreateSql, - [], ); }); @@ -169,18 +167,17 @@ describe('store', () => { }); expect(query).toHaveBeenCalledTimes(5); - expect(query).toHaveBeenNthCalledWith(1, usersTable_createSql, []); + expect(query).toHaveBeenNthCalledWith(1, usersTable_createSql); expect(query).toHaveBeenNthCalledWith(2, listTableColumns_sql, [ usersTableName, ]); expect(query).toHaveBeenNthCalledWith(3, listIndexAttributes_sql, [ usersTableName, ]); - expect(query).toHaveBeenNthCalledWith(4, usersTable_phoneAddSql, []); + expect(query).toHaveBeenNthCalledWith(4, usersTable_phoneAddSql); expect(query).toHaveBeenNthCalledWith( 5, usersTable_phoneEmailIndexCreateSql, - [], ); }); }); @@ -192,9 +189,7 @@ describe('store', () => { await store.dropCollection({ collection: usersCollection }); expect(query).toHaveBeenCalledTimes(1); - expect(query).toHaveBeenCalledWith(dropTableIfExists_sql, [ - usersTableName, - ]); + expect(query).toHaveBeenCalledWith(usersTable_dropSql); }); }); }); diff --git a/packages/postgresql-store/src/store.ts b/packages/postgresql-store/src/store.ts index 9956eb2..3e6e4e2 100644 --- a/packages/postgresql-store/src/store.ts +++ b/packages/postgresql-store/src/store.ts @@ -8,6 +8,8 @@ import { addIndex, createTableIfNotExists, addColumn, + dropColumn, + dropTableIfExists, } from './queries'; import { Store, @@ -30,7 +32,6 @@ import { describeCollection, listCollections, ensureCollection, - SQLConnection, } from '@neuledge/sql-store'; export type PostgreSQLStoreClient = Client | Pool; @@ -42,24 +43,16 @@ export type PostgreSQLStoreOptions = }; export class PostgreSQLStore implements Store { - private client: PostgreSQLStoreClient; - private connection: SQLConnection; + private connection: PostgreSQLStoreClient; constructor(options: PostgreSQLStoreOptions) { - this.client = 'client' in options ? options.client : new Pool(options); - - this.connection = { - query: (sql, values) => - this.client - .query(sql, values ?? []) - .then((result) => result.rows as never), - }; + this.connection = 'client' in options ? options.client : new Pool(options); } // connection methods async close(): Promise { - await this.client.end(); + await this.connection.end(); } // store methods @@ -84,6 +77,7 @@ export class PostgreSQLStore implements Store { addIndex, addColumn, dropIndex, + dropColumn, listTableColumns, listIndexAttributes, dataTypeMap, @@ -91,7 +85,7 @@ export class PostgreSQLStore implements Store { } async dropCollection(options: StoreDropCollectionOptions): Promise { - return dropCollection(options, this.connection); + return dropCollection(options, this.connection, { dropTableIfExists }); } async find(options: StoreFindOptions): Promise> { diff --git a/packages/sql-store/src/index.ts b/packages/sql-store/src/index.ts index d941abe..ef390d6 100644 --- a/packages/sql-store/src/index.ts +++ b/packages/sql-store/src/index.ts @@ -1,3 +1,2 @@ export * from './logic'; export * from './mappers'; -export * from './queries'; diff --git a/packages/sql-store/src/logic/collections/describe.ts b/packages/sql-store/src/logic/collections/describe.ts index e01ff38..9a26217 100644 --- a/packages/sql-store/src/logic/collections/describe.ts +++ b/packages/sql-store/src/logic/collections/describe.ts @@ -5,7 +5,6 @@ import { toStoreField, toStoreIndex, } from '@/mappers'; -import { SQLConnection } from '@/queries'; import { StoreCollection, StoreDescribeCollectionOptions, @@ -14,49 +13,33 @@ import { } from '@neuledge/store'; export interface DescribeCollectionQueries< - C extends SQLColumn, - I extends SQLIndexAttribute & Omit, + Connection, + Column extends SQLColumn, + IndexAttribute extends SQLIndexAttribute & Omit, > { - listTableColumns: (name: string, connection: SQLConnection) => Promise; - listIndexAttributes: ( + listTableColumns(connection: Connection, name: string): Promise; + listIndexAttributes( + connection: Connection, name: string, - connection: SQLConnection, - ) => Promise; + ): Promise; dataTypeMap: Record; } export const describeCollection = async < - C extends SQLColumn, - I extends SQLIndexAttribute & Omit, + Connection, + Column extends SQLColumn, + IndexAttribute extends SQLIndexAttribute & Omit, >( options: StoreDescribeCollectionOptions, - connection: SQLConnection, - { - listTableColumns, - listIndexAttributes, - dataTypeMap, - }: DescribeCollectionQueries, + connection: Connection, + queries: DescribeCollectionQueries, ): Promise => { - const { name } = options.collection; - - const [columns, indexAttributes] = await Promise.all([ - listTableColumns(name, connection), - listIndexAttributes(name, connection), - ]); - - const columnMap = Object.fromEntries( - columns.map((column) => [column.column_name, column]), + const { name, fields, indexColumns } = await getCollectionDetails( + options, + connection, + queries, ); - const fields = Object.fromEntries( - columns.map((column) => [ - column.column_name, - toStoreField(dataTypeMap, column), - ]), - ); - - const indexColumns = groupIndexColumns(columnMap, indexAttributes); - let primaryKey: string | undefined; const indexes = Object.fromEntries( indexColumns.map((columns) => { @@ -84,12 +67,48 @@ export const describeCollection = async < }; }; +const getCollectionDetails = async < + Connection, + Column extends SQLColumn, + IndexAttribute extends SQLIndexAttribute & Omit, +>( + options: StoreDescribeCollectionOptions, + connection: Connection, + { + listTableColumns, + listIndexAttributes, + dataTypeMap, + }: DescribeCollectionQueries, +) => { + const { name } = options.collection; + + const [columns, indexAttributes] = await Promise.all([ + listTableColumns(connection, name), + listIndexAttributes(connection, name), + ]); + + const columnMap = Object.fromEntries( + columns.map((column) => [column.column_name, column]), + ); + + const fields = Object.fromEntries( + columns.map((column) => [ + column.column_name, + toStoreField(dataTypeMap, column), + ]), + ); + + const indexColumns = groupIndexColumns(columnMap, indexAttributes); + + return { name, fields, indexColumns }; +}; + const groupIndexColumns = < - C extends SQLColumn, - I extends SQLIndexAttribute & Omit, + Column extends SQLColumn, + IndexAttribute extends SQLIndexAttribute & Omit, >( - columnMap: Record, - indexAttributes: I[], + columnMap: Record, + indexAttributes: IndexAttribute[], ): SQLIndexColumn[][] => { const groupMap: Record = {}; diff --git a/packages/sql-store/src/logic/collections/drop.ts b/packages/sql-store/src/logic/collections/drop.ts index f7e9d1b..ba3241c 100644 --- a/packages/sql-store/src/logic/collections/drop.ts +++ b/packages/sql-store/src/logic/collections/drop.ts @@ -1,9 +1,13 @@ -import { SQLConnection, dropTableIfExists } from '@/queries'; import { StoreDropCollectionOptions } from '@neuledge/store'; -export const dropCollection = async ( +export interface DropCollectionQueries { + dropTableIfExists(connection: Connection, name: string): Promise; +} + +export const dropCollection = async ( options: StoreDropCollectionOptions, - connection: SQLConnection, + connection: Connection, + { dropTableIfExists }: DropCollectionQueries, ): Promise => { await dropTableIfExists(connection, options.collection.name); }; diff --git a/packages/sql-store/src/logic/collections/ensure.ts b/packages/sql-store/src/logic/collections/ensure.ts index abe72ac..38f0827 100644 --- a/packages/sql-store/src/logic/collections/ensure.ts +++ b/packages/sql-store/src/logic/collections/ensure.ts @@ -1,8 +1,3 @@ -import { - SQLConnection, - dropColumn as dropColumnDefault, - dropIndex as dropIndexDefault, -} from '@/queries'; import pLimit from 'p-limit'; import { StoreCollection, @@ -13,61 +8,63 @@ import { import { SQLColumn, SQLIndexAttribute, SQLIndexColumn } from '@/mappers'; import { DescribeCollectionQueries, describeCollection } from './describe'; -export interface EnsureCollectionQueries { - createTableIfNotExists: ( +export interface EnsureCollectionQueries { + createTableIfNotExists( + connection: Connection, collection: StoreCollection, - connection: SQLConnection, - ) => Promise; - addIndex: ( + ): Promise; + addIndex( + connection: Connection, collection: StoreCollection, index: StoreIndex, - connection: SQLConnection, - ) => Promise; - addColumn: ( + ): Promise; + addColumn( + connection: Connection, collection: StoreCollection, field: StoreField, - connection: SQLConnection, - ) => Promise; - dropIndex?: ( + ): Promise; + dropIndex( + connection: Connection, collection: StoreCollection, index: string, - connection: SQLConnection, - ) => Promise; - dropColumn?: ( + ): Promise; + dropColumn( + connection: Connection, collection: StoreCollection, field: string, - connection: SQLConnection, - ) => Promise; + ): Promise; } export const ensureCollection = async < - C extends SQLColumn, - I extends SQLIndexAttribute & Omit, + Connection, + Column extends SQLColumn, + IndexAttribute extends SQLIndexAttribute & Omit, >( options: StoreEnsureCollectionOptions, - connection: SQLConnection, + connection: Connection, { createTableIfNotExists, addIndex, addColumn, - dropIndex = dropIndexDefault, - dropColumn = dropColumnDefault, + dropIndex, + dropColumn, ...describeCollectionQueries - }: EnsureCollectionQueries & DescribeCollectionQueries, + }: EnsureCollectionQueries & + DescribeCollectionQueries, ): Promise => { - await createTableIfNotExists(options.collection, connection); + await createTableIfNotExists(connection, options.collection); const asyncLimit = pLimit(4); await Promise.all( options.dropIndexes?.map((index) => - asyncLimit(() => dropIndex(options.collection, index, connection)), + asyncLimit(() => dropIndex(connection, options.collection, index)), ) || [], ); await Promise.all( options.dropFields?.map((field) => - asyncLimit(() => dropColumn(options.collection, field, connection)), + asyncLimit(() => dropColumn(connection, options.collection, field)), ) || [], ); @@ -86,7 +83,7 @@ export const ensureCollection = async < options.fields ?.filter((field) => !collection.fields[field.name]) .map((field) => - asyncLimit(() => addColumn(collection, field, connection)), + asyncLimit(() => addColumn(connection, collection, field)), ) || [], ); @@ -94,7 +91,7 @@ export const ensureCollection = async < options.indexes ?.filter((index) => !collection.indexes[index.name]) .map((index) => - asyncLimit(() => addIndex(collection, index, connection)), + asyncLimit(() => addIndex(connection, collection, index)), ) || [], ); }; diff --git a/packages/sql-store/src/logic/collections/list.ts b/packages/sql-store/src/logic/collections/list.ts index bcddebb..2268bcd 100644 --- a/packages/sql-store/src/logic/collections/list.ts +++ b/packages/sql-store/src/logic/collections/list.ts @@ -1,14 +1,13 @@ import { SQLTable, toStoreCollection_Slim } from '@/mappers'; -import { SQLConnection } from '@/queries'; import { StoreCollection_Slim } from '@neuledge/store'; -export interface ListCollectionsQueries { - listTables: (connection: SQLConnection) => Promise; +export interface ListCollectionsQueries { + listTables(connection: Connection): Promise; } -export const listCollections = async ( - connection: SQLConnection, - { listTables }: ListCollectionsQueries, +export const listCollections = async ( + connection: Connection, + { listTables }: ListCollectionsQueries, ): Promise => { const tables = await listTables(connection); return tables.map((table) => toStoreCollection_Slim(table)); diff --git a/packages/sql-store/src/queries/connection.ts b/packages/sql-store/src/queries/connection.ts deleted file mode 100644 index 5176269..0000000 --- a/packages/sql-store/src/queries/connection.ts +++ /dev/null @@ -1,3 +0,0 @@ -export interface SQLConnection { - query(sql: string, params?: unknown[]): Promise; -} diff --git a/packages/sql-store/src/queries/drop-column.ts b/packages/sql-store/src/queries/drop-column.ts deleted file mode 100644 index 54dd8ec..0000000 --- a/packages/sql-store/src/queries/drop-column.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { StoreCollection } from '@neuledge/store'; -import { SQLConnection } from './connection'; - -export const dropColumn = async ( - collection: StoreCollection, - field: string, - connection: SQLConnection, -): Promise => { - await connection.query(`ALTER TABLE ? DROP COLUMN ?`, [ - collection.name, - field, - ]); -}; diff --git a/packages/sql-store/src/queries/drop-index.ts b/packages/sql-store/src/queries/drop-index.ts deleted file mode 100644 index 7c46308..0000000 --- a/packages/sql-store/src/queries/drop-index.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { StoreCollection } from '@neuledge/store'; -import { SQLConnection } from './connection'; - -export const dropIndex = async ( - collection: StoreCollection, - index: string, - connection: SQLConnection, -): Promise => { - await connection.query(`DROP INDEX ? ON ?`, [index, collection.name]); -}; diff --git a/packages/sql-store/src/queries/drop-table.ts b/packages/sql-store/src/queries/drop-table.ts deleted file mode 100644 index 874609c..0000000 --- a/packages/sql-store/src/queries/drop-table.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { SQLConnection } from './connection'; - -export const dropTableIfExists = async ( - connection: SQLConnection, - tableName: string, -): Promise => connection.query(dropTableIfExists_sql, [tableName]); - -export const dropTableIfExists_sql = `DROP TABLE IF EXISTS ?`; diff --git a/packages/sql-store/src/queries/index-columns.ts b/packages/sql-store/src/queries/index-columns.ts deleted file mode 100644 index 9273dbe..0000000 --- a/packages/sql-store/src/queries/index-columns.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { StoreIndex } from '@neuledge/store'; - -export const indexColumns = (index: StoreIndex): string => - Object.entries(index.fields) - .map(([key, val]) => `${key} ${val.sort === 'desc' ? 'DESC' : 'ASC'}`) - .join(', '); diff --git a/packages/sql-store/src/queries/index.ts b/packages/sql-store/src/queries/index.ts deleted file mode 100644 index 85a79c9..0000000 --- a/packages/sql-store/src/queries/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export * from './connection'; -export * from './drop-column'; -export * from './drop-index'; -export * from './drop-table'; -export * from './index-columns'; diff --git a/yarn.lock b/yarn.lock index c300ed2..1bd1482 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2007,6 +2007,11 @@ resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA== +"@types/pg-format@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@types/pg-format/-/pg-format-1.0.2.tgz#3c63fcb3723a6888c8fad740866b1061634d037e" + integrity sha512-D3MEO6u3BObw3G4Xewjdx05MF5v/fiog78CedtrXe8BhONM8GvUz2dPfLWtI0BPRBoRd6anPHXe+sbrPReZouQ== + "@types/pg@^8.6.6": version "8.6.6" resolved "https://registry.yarnpkg.com/@types/pg/-/pg-8.6.6.tgz#21cdf873a3e345a6e78f394677e3b3b1b543cb80" @@ -6167,6 +6172,11 @@ pg-connection-string@^2.5.0: resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34" integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ== +pg-format@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/pg-format/-/pg-format-1.0.4.tgz#27734236c2ad3f4e5064915a59334e20040a828e" + integrity sha512-YyKEF78pEA6wwTAqOUaHIN/rWpfzzIuMh9KdAhc3rSLQ/7zkRFcCgYBAEGatDstLyZw4g0s9SNICmaTGnBVeyw== + pg-int8@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" From e185daf2d105a0fa588f34cd79932b0ab8ea691e Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Sun, 16 Apr 2023 20:33:18 +0300 Subject: [PATCH 14/24] pg insert --- packages/mysql-store/src/store.ts | 2 +- .../src/queries/__fixtures__/users-table.ts | 32 +++++----- .../src/queries/add-column.ts | 4 +- .../postgresql-store/src/queries/add-index.ts | 6 +- .../src/queries/create-table.ts | 6 +- .../src/queries/drop-column.ts | 6 +- .../src/queries/drop-index.ts | 2 +- .../src/queries/drop-table.ts | 2 +- .../postgresql-store/src/queries/index.ts | 3 +- .../src/queries/insert-into.ts | 30 +++++++++ packages/postgresql-store/src/store.test.ts | 61 +++++++++++++++++++ packages/postgresql-store/src/store.ts | 6 +- packages/sql-store/src/logic/index.ts | 1 + packages/sql-store/src/logic/insert.ts | 43 +++++++++++++ 14 files changed, 171 insertions(+), 33 deletions(-) create mode 100644 packages/postgresql-store/src/queries/insert-into.ts create mode 100644 packages/sql-store/src/logic/insert.ts diff --git a/packages/mysql-store/src/store.ts b/packages/mysql-store/src/store.ts index 741b69f..7e52de8 100644 --- a/packages/mysql-store/src/store.ts +++ b/packages/mysql-store/src/store.ts @@ -34,7 +34,7 @@ import { listCollections, } from '@neuledge/sql-store'; -export type MySQLStoreClient = Pool | Connection; +export type MySQLStoreClient = Pick; export type MySQLStoreOptions = PoolConfig | { client: MySQLStoreClient }; diff --git a/packages/postgresql-store/src/queries/__fixtures__/users-table.ts b/packages/postgresql-store/src/queries/__fixtures__/users-table.ts index 9d21cb5..739d234 100644 --- a/packages/postgresql-store/src/queries/__fixtures__/users-table.ts +++ b/packages/postgresql-store/src/queries/__fixtures__/users-table.ts @@ -5,23 +5,23 @@ import { PostgreSQLIndexAttribute } from '../list-table-statistics'; export const usersTableName = 'users'; -export const usersTable_dropSql = `DROP TABLE IF EXISTS '${usersTableName}'`; +export const usersTable_dropSql = `DROP TABLE IF EXISTS users`; -export const usersTable_createSql = `CREATE TABLE IF NOT EXISTS '${usersTableName}' ( - 'id' BIGSERIAL NOT NULL, - 'name' VARCHAR(50), - 'email' VARCHAR(100) NOT NULL, - 'phone' VARCHAR(20), - 'created_at' TIMESTAMP NOT NULL, - 'updated_at' TIMESTAMP NOT NULL, - CONSTRAINT '${usersTableName}_pkey' PRIMARY KEY ('id') +export const usersTable_createSql = `CREATE TABLE IF NOT EXISTS users ( + id BIGSERIAL NOT NULL, + name VARCHAR(50), + email VARCHAR(100) NOT NULL, + phone VARCHAR(20), + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + CONSTRAINT users_pkey PRIMARY KEY (id) )`; -export const usersTable_phoneAddSql = `ALTER TABLE '${usersTableName}' ADD COLUMN 'phone' VARCHAR(20)`; +export const usersTable_phoneAddSql = `ALTER TABLE users ADD COLUMN phone VARCHAR(20)`; -export const usersTable_emailIndexCreateSql = `CREATE UNIQUE INDEX IF NOT EXISTS '${usersTableName}_email_idx' ON '${usersTableName}' ('email' ASC)`; +export const usersTable_emailIndexCreateSql = `CREATE UNIQUE INDEX IF NOT EXISTS users_email_idx ON users (email ASC)`; -export const usersTable_phoneEmailIndexCreateSql = `CREATE INDEX IF NOT EXISTS '${usersTableName}_phone_email_idx' ON '${usersTableName}' ('phone' DESC, 'email' ASC)`; +export const usersTable_phoneEmailIndexCreateSql = `CREATE INDEX IF NOT EXISTS users_phone_email_idx ON users (phone DESC, email ASC)`; export const usersTable: PostgreSQLTable = { table_name: usersTableName }; @@ -88,7 +88,7 @@ export const usersTableColumns: PostgreSQLColumn[] = [ export const usersTablePrimaryIndexes: PostgreSQLIndexAttribute[] = [ { - index_name: `${usersTableName}_id_idx`, + index_name: `users_id_idx`, column_name: 'id', seq_in_index: 1, direction: 'ASC', @@ -100,7 +100,7 @@ export const usersTablePrimaryIndexes: PostgreSQLIndexAttribute[] = [ export const usersTableIndexes: PostgreSQLIndexAttribute[] = [ { - index_name: `${usersTableName}_email_idx`, + index_name: `users_email_idx`, column_name: 'email', seq_in_index: 1, direction: 'ASC', @@ -109,7 +109,7 @@ export const usersTableIndexes: PostgreSQLIndexAttribute[] = [ is_primary: false, }, { - index_name: `${usersTableName}_phone_email_idx`, + index_name: `users_phone_email_idx`, column_name: 'phone', seq_in_index: 1, direction: 'DESC', @@ -118,7 +118,7 @@ export const usersTableIndexes: PostgreSQLIndexAttribute[] = [ is_primary: false, }, { - index_name: `${usersTableName}_phone_email_idx`, + index_name: `users_phone_email_idx`, column_name: 'email', seq_in_index: 2, direction: 'ASC', diff --git a/packages/postgresql-store/src/queries/add-column.ts b/packages/postgresql-store/src/queries/add-column.ts index 31226e8..f168ce7 100644 --- a/packages/postgresql-store/src/queries/add-column.ts +++ b/packages/postgresql-store/src/queries/add-column.ts @@ -8,7 +8,7 @@ export const addColumn = async ( field: StoreField, ): Promise => { await connection.query( - `ALTER TABLE ${format.literal( + `ALTER TABLE ${format.ident( collection.name, )} ADD COLUMN ${getColumnDefinition(field, collection)}`, ); @@ -18,7 +18,7 @@ export const getColumnDefinition = ( field: StoreField, collection: StoreCollection, ): string => - `${format.literal(field.name)} ${getColumnDataType(field, collection)}${ + `${format.ident(field.name)} ${getColumnDataType(field, collection)}${ field.nullable ? '' : ' NOT NULL' }`; diff --git a/packages/postgresql-store/src/queries/add-index.ts b/packages/postgresql-store/src/queries/add-index.ts index 0e83e11..e924fa3 100644 --- a/packages/postgresql-store/src/queries/add-index.ts +++ b/packages/postgresql-store/src/queries/add-index.ts @@ -10,12 +10,12 @@ export const addIndex = async ( await connection.query( `CREATE ${ index.unique ? 'UNIQUE INDEX' : 'INDEX' - } IF NOT EXISTS ${format.literal( + } IF NOT EXISTS ${format.ident( `${collection.name}_${index.name}_idx`, - )} ON ${format.literal(collection.name)} (${Object.entries(index.fields) + )} ON ${format.ident(collection.name)} (${Object.entries(index.fields) .map( ([field, { sort }]) => - `${format.literal(field)} ${sort === 'desc' ? 'DESC' : 'ASC'}`, + `${format.ident(field)} ${sort === 'desc' ? 'DESC' : 'ASC'}`, ) .join(', ')})`, ); diff --git a/packages/postgresql-store/src/queries/create-table.ts b/packages/postgresql-store/src/queries/create-table.ts index 95d6e5d..6d120c6 100644 --- a/packages/postgresql-store/src/queries/create-table.ts +++ b/packages/postgresql-store/src/queries/create-table.ts @@ -9,14 +9,14 @@ export const createTableIfNotExists = async ( collection: StoreCollection, ): Promise => { await connection.query( - `CREATE TABLE IF NOT EXISTS ${format.literal(collection.name)} ( + `CREATE TABLE IF NOT EXISTS ${format.ident(collection.name)} ( ${Object.values(collection.fields) .map((field) => getColumnDefinition(field, collection)) .join(',\n ')}, - CONSTRAINT ${format.literal( + CONSTRAINT ${format.ident( `${collection.name}_pkey`, )} PRIMARY KEY (${Object.keys(collection.primaryKey.fields) - .map((val) => format.literal(val)) + .map((val) => format.ident(val)) .join(', ')}) )`, ); diff --git a/packages/postgresql-store/src/queries/drop-column.ts b/packages/postgresql-store/src/queries/drop-column.ts index 663d5e4..71a197a 100644 --- a/packages/postgresql-store/src/queries/drop-column.ts +++ b/packages/postgresql-store/src/queries/drop-column.ts @@ -8,8 +8,8 @@ export const dropColumn = async ( field: string, ): Promise => { await connection.query( - `ALTER TABLE ${format.literal( - collection.name, - )} DROP COLUMN ${format.literal(field)}`, + `ALTER TABLE ${format.ident(collection.name)} DROP COLUMN ${format.ident( + field, + )}`, ); }; diff --git a/packages/postgresql-store/src/queries/drop-index.ts b/packages/postgresql-store/src/queries/drop-index.ts index 63d48de..1fd989b 100644 --- a/packages/postgresql-store/src/queries/drop-index.ts +++ b/packages/postgresql-store/src/queries/drop-index.ts @@ -8,6 +8,6 @@ export const dropIndex = async ( index: string, ): Promise => { await connection.query( - `DROP INDEX IF EXISTS ${format.literal(`${collection.name}_${index}_idx`)}`, + `DROP INDEX IF EXISTS ${format.ident(`${collection.name}_${index}_idx`)}`, ); }; diff --git a/packages/postgresql-store/src/queries/drop-table.ts b/packages/postgresql-store/src/queries/drop-table.ts index 3eee5e9..b462ba0 100644 --- a/packages/postgresql-store/src/queries/drop-table.ts +++ b/packages/postgresql-store/src/queries/drop-table.ts @@ -5,5 +5,5 @@ export const dropTableIfExists = async ( connection: PostgreSQLConnection, tableName: string, ): Promise => { - await connection.query(`DROP TABLE IF EXISTS ${format.literal(tableName)}`); + await connection.query(`DROP TABLE IF EXISTS ${format.ident(tableName)}`); }; diff --git a/packages/postgresql-store/src/queries/index.ts b/packages/postgresql-store/src/queries/index.ts index 671584a..5cbc256 100644 --- a/packages/postgresql-store/src/queries/index.ts +++ b/packages/postgresql-store/src/queries/index.ts @@ -1,9 +1,10 @@ export * from './add-column'; export * from './add-index'; +export * from './create-table'; export * from './drop-column'; export * from './drop-index'; export * from './drop-table'; -export * from './create-table'; +export * from './insert-into'; export * from './list-tables'; export * from './list-table-columns'; export * from './list-table-statistics'; diff --git a/packages/postgresql-store/src/queries/insert-into.ts b/packages/postgresql-store/src/queries/insert-into.ts new file mode 100644 index 0000000..7a53774 --- /dev/null +++ b/packages/postgresql-store/src/queries/insert-into.ts @@ -0,0 +1,30 @@ +import { StoreDocument, StoreScalarValue } from '@neuledge/store'; +import { PostgreSQLConnection } from './connection'; +import format from 'pg-format'; + +// FIXME support format.literal for any scalar value + +export const insertInto = async ( + connection: PostgreSQLConnection, + name: string, + columns: string[], + values: (StoreScalarValue | undefined)[][], + returns: string[], +): Promise => + connection + .query( + `INSERT INTO ${format.ident(name)} (${columns + .map((column) => format.ident(column)) + .join(', ')}) +VALUES (${values + .map((arr) => + arr + .map((v) => + v === undefined ? 'DEFAULT' : format.literal(v as never), + ) + .join(', '), + ) + .join('), (')}) +RETURNING ${returns.map((column) => format.ident(column)).join(', ')}`, + ) + .then((res) => res.rows); diff --git a/packages/postgresql-store/src/store.test.ts b/packages/postgresql-store/src/store.test.ts index 426c464..24ac3e1 100644 --- a/packages/postgresql-store/src/store.test.ts +++ b/packages/postgresql-store/src/store.test.ts @@ -192,5 +192,66 @@ describe('store', () => { expect(query).toHaveBeenCalledWith(usersTable_dropSql); }); }); + + describe('.insert()', () => { + it('should be able to insert a document with auto increment', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 1234 }] }); + + const res = await store.insert({ + collection: usersCollection, + documents: [ + { + name: 'John Doe', + email: 'john@example.com', + created_at: new Date('2020-01-01T00:00:00.000Z'), + updated_at: new Date('2020-01-01T00:00:00.000Z'), + }, + ], + }); + + expect(query).toHaveBeenCalledTimes(1); + + expect(query).toHaveBeenCalledWith( + `INSERT INTO ${usersTableName} (id, name, email, phone, created_at, updated_at) +VALUES (DEFAULT, 'John Doe', 'john@example.com', NULL, '2020-01-01 00:00:00.000+00', '2020-01-01 00:00:00.000+00') +RETURNING id`, + ); + + expect(res).toEqual({ + affectedCount: 1, + insertedIds: [{ id: 1234 }], + }); + }); + + it('should be able to insert a document with custom id', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 789 }] }); + + const res = await store.insert({ + collection: usersCollection, + documents: [ + { + id: 789, + name: 'John Doe', + email: 'john@example.com', + created_at: new Date('2020-01-01T00:00:00.000Z'), + updated_at: new Date('2020-01-01T00:00:00.000Z'), + }, + ], + }); + + expect(query).toHaveBeenCalledTimes(1); + + expect(query).toHaveBeenCalledWith( + `INSERT INTO ${usersTableName} (id, name, email, phone, created_at, updated_at) +VALUES ('789', 'John Doe', 'john@example.com', NULL, '2020-01-01 00:00:00.000+00', '2020-01-01 00:00:00.000+00') +RETURNING id`, + ); + + expect(res).toEqual({ + affectedCount: 1, + insertedIds: [{ id: 789 }], + }); + }); + }); }); }); diff --git a/packages/postgresql-store/src/store.ts b/packages/postgresql-store/src/store.ts index 3e6e4e2..71b32b5 100644 --- a/packages/postgresql-store/src/store.ts +++ b/packages/postgresql-store/src/store.ts @@ -10,6 +10,7 @@ import { addColumn, dropColumn, dropTableIfExists, + insertInto, } from './queries'; import { Store, @@ -32,9 +33,10 @@ import { describeCollection, listCollections, ensureCollection, + insert, } from '@neuledge/sql-store'; -export type PostgreSQLStoreClient = Client | Pool; +export type PostgreSQLStoreClient = Pick; export type PostgreSQLStoreOptions = | PoolConfig @@ -93,7 +95,7 @@ export class PostgreSQLStore implements Store { } async insert(options: StoreInsertOptions): Promise { - throw new Error('Method not implemented.'); + return insert(options, this.connection, { insertInto }); } async update(options: StoreUpdateOptions): Promise { diff --git a/packages/sql-store/src/logic/index.ts b/packages/sql-store/src/logic/index.ts index 3eee1ab..b71af69 100644 --- a/packages/sql-store/src/logic/index.ts +++ b/packages/sql-store/src/logic/index.ts @@ -1 +1,2 @@ export * from './collections'; +export * from './insert'; diff --git a/packages/sql-store/src/logic/insert.ts b/packages/sql-store/src/logic/insert.ts new file mode 100644 index 0000000..4bca571 --- /dev/null +++ b/packages/sql-store/src/logic/insert.ts @@ -0,0 +1,43 @@ +import { + StoreDocument, + StoreInsertOptions, + StoreInsertionResponse, + StoreScalarValue, +} from '@neuledge/store'; + +export interface InsertQueries { + insertInto( + connection: Connection, + name: string, + columns: string[], + values: (StoreScalarValue | undefined)[][], + returns: string[], + ): Promise; +} + +export const insert = async ( + options: StoreInsertOptions, + connection: Connection, + { insertInto }: InsertQueries, +): Promise => { + const { collection, documents } = options; + const { name, fields, primaryKey } = collection; + + const columns = Object.keys(fields); + + const values = documents.map((document) => + columns.map( + (column) => + document[column] ?? (primaryKey.fields[column] ? undefined : null), + ), + ); + + const returns = Object.keys(primaryKey.fields); + + const res = await insertInto(connection, name, columns, values, returns); + + return { + affectedCount: res.length, + insertedIds: res, + }; +}; From 125827d7374b9c65707c2d0a9efa06d4ce755ca5 Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Thu, 20 Apr 2023 15:39:35 +0300 Subject: [PATCH 15/24] store delete pg --- .../src/queries/add-column.ts | 7 +- .../postgresql-store/src/queries/add-index.ts | 9 +- .../src/queries/connection.ts | 14 ++ .../src/queries/create-table.ts | 9 +- .../src/queries/delete-from.ts | 20 +++ .../src/queries/drop-column.ts | 9 +- .../src/queries/drop-index.ts | 7 +- .../postgresql-store/src/queries/index.ts | 1 + .../src/queries/insert-into.ts | 19 ++- packages/postgresql-store/src/store.test.ts | 25 ++++ packages/postgresql-store/src/store.ts | 5 +- packages/sql-store/src/helpers/index.ts | 2 + packages/sql-store/src/helpers/query.ts | 6 + .../sql-store/src/helpers/where-clause.ts | 140 ++++++++++++++++++ packages/sql-store/src/index.ts | 1 + .../sql-store/src/logic/collections/ensure.ts | 12 +- packages/sql-store/src/logic/delete.ts | 32 ++++ packages/sql-store/src/logic/index.ts | 1 + 18 files changed, 283 insertions(+), 36 deletions(-) create mode 100644 packages/postgresql-store/src/queries/delete-from.ts create mode 100644 packages/sql-store/src/helpers/index.ts create mode 100644 packages/sql-store/src/helpers/query.ts create mode 100644 packages/sql-store/src/helpers/where-clause.ts create mode 100644 packages/sql-store/src/logic/delete.ts diff --git a/packages/postgresql-store/src/queries/add-column.ts b/packages/postgresql-store/src/queries/add-column.ts index f168ce7..dbe1a8d 100644 --- a/packages/postgresql-store/src/queries/add-column.ts +++ b/packages/postgresql-store/src/queries/add-column.ts @@ -1,6 +1,5 @@ import { StoreCollection, StoreField } from '@neuledge/store'; -import { PostgreSQLConnection } from './connection'; -import format from 'pg-format'; +import { PostgreSQLConnection, encodeIdentifier } from './connection'; export const addColumn = async ( connection: PostgreSQLConnection, @@ -8,7 +7,7 @@ export const addColumn = async ( field: StoreField, ): Promise => { await connection.query( - `ALTER TABLE ${format.ident( + `ALTER TABLE ${encodeIdentifier( collection.name, )} ADD COLUMN ${getColumnDefinition(field, collection)}`, ); @@ -18,7 +17,7 @@ export const getColumnDefinition = ( field: StoreField, collection: StoreCollection, ): string => - `${format.ident(field.name)} ${getColumnDataType(field, collection)}${ + `${encodeIdentifier(field.name)} ${getColumnDataType(field, collection)}${ field.nullable ? '' : ' NOT NULL' }`; diff --git a/packages/postgresql-store/src/queries/add-index.ts b/packages/postgresql-store/src/queries/add-index.ts index e924fa3..a4c8dd8 100644 --- a/packages/postgresql-store/src/queries/add-index.ts +++ b/packages/postgresql-store/src/queries/add-index.ts @@ -1,6 +1,5 @@ import { StoreCollection, StoreIndex } from '@neuledge/store'; -import { PostgreSQLConnection } from './connection'; -import format from 'pg-format'; +import { PostgreSQLConnection, encodeIdentifier } from './connection'; export const addIndex = async ( connection: PostgreSQLConnection, @@ -10,12 +9,12 @@ export const addIndex = async ( await connection.query( `CREATE ${ index.unique ? 'UNIQUE INDEX' : 'INDEX' - } IF NOT EXISTS ${format.ident( + } IF NOT EXISTS ${encodeIdentifier( `${collection.name}_${index.name}_idx`, - )} ON ${format.ident(collection.name)} (${Object.entries(index.fields) + )} ON ${encodeIdentifier(collection.name)} (${Object.entries(index.fields) .map( ([field, { sort }]) => - `${format.ident(field)} ${sort === 'desc' ? 'DESC' : 'ASC'}`, + `${encodeIdentifier(field)} ${sort === 'desc' ? 'DESC' : 'ASC'}`, ) .join(', ')})`, ); diff --git a/packages/postgresql-store/src/queries/connection.ts b/packages/postgresql-store/src/queries/connection.ts index 2c4239d..0d37e09 100644 --- a/packages/postgresql-store/src/queries/connection.ts +++ b/packages/postgresql-store/src/queries/connection.ts @@ -1,3 +1,17 @@ +import { QueryHelpers } from '@neuledge/sql-store'; +import { StoreScalarValue } from '@neuledge/store'; import { Client, Pool } from 'pg'; +import format from 'pg-format'; export type PostgreSQLConnection = Pick; + +export const encodeIdentifier = format.ident; + +export const encodeLiteral = (val: StoreScalarValue) => + // format.literal will convert everything else to string which will work fine for bigint + format.literal(val as Exclude); + +export const queryHelpers: QueryHelpers = { + encodeIdentifier, + encodeLiteral, +}; diff --git a/packages/postgresql-store/src/queries/create-table.ts b/packages/postgresql-store/src/queries/create-table.ts index 6d120c6..4b3ee4d 100644 --- a/packages/postgresql-store/src/queries/create-table.ts +++ b/packages/postgresql-store/src/queries/create-table.ts @@ -1,22 +1,21 @@ import { StoreCollection } from '@neuledge/store'; import { getColumnDefinition } from './add-column'; import { addIndex } from './add-index'; -import { PostgreSQLConnection } from './connection'; -import format from 'pg-format'; +import { PostgreSQLConnection, encodeIdentifier } from './connection'; export const createTableIfNotExists = async ( connection: PostgreSQLConnection, collection: StoreCollection, ): Promise => { await connection.query( - `CREATE TABLE IF NOT EXISTS ${format.ident(collection.name)} ( + `CREATE TABLE IF NOT EXISTS ${encodeIdentifier(collection.name)} ( ${Object.values(collection.fields) .map((field) => getColumnDefinition(field, collection)) .join(',\n ')}, - CONSTRAINT ${format.ident( + CONSTRAINT ${encodeIdentifier( `${collection.name}_pkey`, )} PRIMARY KEY (${Object.keys(collection.primaryKey.fields) - .map((val) => format.ident(val)) + .map((val) => encodeIdentifier(val)) .join(', ')}) )`, ); diff --git a/packages/postgresql-store/src/queries/delete-from.ts b/packages/postgresql-store/src/queries/delete-from.ts new file mode 100644 index 0000000..e2a18f3 --- /dev/null +++ b/packages/postgresql-store/src/queries/delete-from.ts @@ -0,0 +1,20 @@ +import { PostgreSQLConnection, encodeIdentifier } from './connection'; + +export const deleteFrom = async ( + connection: PostgreSQLConnection, + name: string, + where: string | null, + limit: number, +): Promise => { + const res = await connection.query( + `DELETE FROM ${encodeIdentifier(name)}${ + where + ? ` +WHERE ${where}` + : '' + } +LIMIT ${Number(limit)}`, + ); + + return res.rowCount; +}; diff --git a/packages/postgresql-store/src/queries/drop-column.ts b/packages/postgresql-store/src/queries/drop-column.ts index 71a197a..810fe3c 100644 --- a/packages/postgresql-store/src/queries/drop-column.ts +++ b/packages/postgresql-store/src/queries/drop-column.ts @@ -1,6 +1,5 @@ import { StoreCollection } from '@neuledge/store'; -import { PostgreSQLConnection } from './connection'; -import format from 'pg-format'; +import { PostgreSQLConnection, encodeIdentifier } from './connection'; export const dropColumn = async ( connection: PostgreSQLConnection, @@ -8,8 +7,8 @@ export const dropColumn = async ( field: string, ): Promise => { await connection.query( - `ALTER TABLE ${format.ident(collection.name)} DROP COLUMN ${format.ident( - field, - )}`, + `ALTER TABLE ${encodeIdentifier( + collection.name, + )} DROP COLUMN ${encodeIdentifier(field)}`, ); }; diff --git a/packages/postgresql-store/src/queries/drop-index.ts b/packages/postgresql-store/src/queries/drop-index.ts index 1fd989b..f942c2c 100644 --- a/packages/postgresql-store/src/queries/drop-index.ts +++ b/packages/postgresql-store/src/queries/drop-index.ts @@ -1,6 +1,5 @@ import { StoreCollection } from '@neuledge/store'; -import { PostgreSQLConnection } from './connection'; -import format from 'pg-format'; +import { PostgreSQLConnection, encodeIdentifier } from './connection'; export const dropIndex = async ( connection: PostgreSQLConnection, @@ -8,6 +7,8 @@ export const dropIndex = async ( index: string, ): Promise => { await connection.query( - `DROP INDEX IF EXISTS ${format.ident(`${collection.name}_${index}_idx`)}`, + `DROP INDEX IF EXISTS ${encodeIdentifier( + `${collection.name}_${index}_idx`, + )}`, ); }; diff --git a/packages/postgresql-store/src/queries/index.ts b/packages/postgresql-store/src/queries/index.ts index 5cbc256..e5660f7 100644 --- a/packages/postgresql-store/src/queries/index.ts +++ b/packages/postgresql-store/src/queries/index.ts @@ -1,6 +1,7 @@ export * from './add-column'; export * from './add-index'; export * from './create-table'; +export * from './delete-from'; export * from './drop-column'; export * from './drop-index'; export * from './drop-table'; diff --git a/packages/postgresql-store/src/queries/insert-into.ts b/packages/postgresql-store/src/queries/insert-into.ts index 7a53774..78ea348 100644 --- a/packages/postgresql-store/src/queries/insert-into.ts +++ b/packages/postgresql-store/src/queries/insert-into.ts @@ -1,8 +1,9 @@ import { StoreDocument, StoreScalarValue } from '@neuledge/store'; -import { PostgreSQLConnection } from './connection'; -import format from 'pg-format'; - -// FIXME support format.literal for any scalar value +import { + PostgreSQLConnection, + encodeIdentifier, + encodeLiteral, +} from './connection'; export const insertInto = async ( connection: PostgreSQLConnection, @@ -13,18 +14,16 @@ export const insertInto = async ( ): Promise => connection .query( - `INSERT INTO ${format.ident(name)} (${columns - .map((column) => format.ident(column)) + `INSERT INTO ${encodeIdentifier(name)} (${columns + .map((column) => encodeIdentifier(column)) .join(', ')}) VALUES (${values .map((arr) => arr - .map((v) => - v === undefined ? 'DEFAULT' : format.literal(v as never), - ) + .map((v) => (v === undefined ? 'DEFAULT' : encodeLiteral(v))) .join(', '), ) .join('), (')}) -RETURNING ${returns.map((column) => format.ident(column)).join(', ')}`, +RETURNING ${returns.map((column) => encodeIdentifier(column)).join(', ')}`, ) .then((res) => res.rows); diff --git a/packages/postgresql-store/src/store.test.ts b/packages/postgresql-store/src/store.test.ts index 24ac3e1..937b598 100644 --- a/packages/postgresql-store/src/store.test.ts +++ b/packages/postgresql-store/src/store.test.ts @@ -253,5 +253,30 @@ RETURNING id`, }); }); }); + + // describe '.update()' + + describe('.delete()', () => { + it('should be able to delete a document', async () => { + query.mockResolvedValueOnce({ rowCount: 1 }); + + const res = await store.delete({ + collection: usersCollection, + where: { id: { $eq: 123 } }, + limit: 1, + }); + + expect(query).toHaveBeenCalledTimes(1); + expect(query).toHaveBeenCalledWith( + `DELETE FROM ${usersTableName} +WHERE id = '123' +LIMIT 1`, + ); + + expect(res).toEqual({ + affectedCount: 1, + }); + }); + }); }); }); diff --git a/packages/postgresql-store/src/store.ts b/packages/postgresql-store/src/store.ts index 71b32b5..557531a 100644 --- a/packages/postgresql-store/src/store.ts +++ b/packages/postgresql-store/src/store.ts @@ -11,6 +11,7 @@ import { dropColumn, dropTableIfExists, insertInto, + deleteFrom, } from './queries'; import { Store, @@ -35,6 +36,8 @@ import { ensureCollection, insert, } from '@neuledge/sql-store'; +import { deletes } from '@neuledge/sql-store'; +import { queryHelpers } from './queries/connection'; export type PostgreSQLStoreClient = Pick; @@ -103,6 +106,6 @@ export class PostgreSQLStore implements Store { } async delete(options: StoreDeleteOptions): Promise { - throw new Error('Method not implemented.'); + return deletes(options, this.connection, { deleteFrom, queryHelpers }); } } diff --git a/packages/sql-store/src/helpers/index.ts b/packages/sql-store/src/helpers/index.ts new file mode 100644 index 0000000..65adbee --- /dev/null +++ b/packages/sql-store/src/helpers/index.ts @@ -0,0 +1,2 @@ +export * from './query'; +export * from './where-clause'; diff --git a/packages/sql-store/src/helpers/query.ts b/packages/sql-store/src/helpers/query.ts new file mode 100644 index 0000000..19ab3e3 --- /dev/null +++ b/packages/sql-store/src/helpers/query.ts @@ -0,0 +1,6 @@ +import { StoreScalarValue } from '@neuledge/store'; + +export interface QueryHelpers { + encodeIdentifier(identifier: string): string; + encodeLiteral(literal: StoreScalarValue): string; +} diff --git a/packages/sql-store/src/helpers/where-clause.ts b/packages/sql-store/src/helpers/where-clause.ts new file mode 100644 index 0000000..6a5c105 --- /dev/null +++ b/packages/sql-store/src/helpers/where-clause.ts @@ -0,0 +1,140 @@ +import { StoreWhere, StoreWhereRecord, StoreWhereTerm } from '@neuledge/store'; +import { QueryHelpers } from './query'; + +export const whereClause = ( + helpers: QueryHelpers, + where: StoreWhere, +): string | null => { + const { $or } = where; + + if (!Array.isArray($or)) { + return whereRecord(helpers, where as StoreWhereRecord) || null; + } + + const sql = $or.map((record) => whereRecord(helpers, record)).filter(Boolean); + + if (sql.length === 0) { + return null; + } + + return `(${sql.join(') OR (')})`; +}; + +const whereRecord = (helpers: QueryHelpers, record: StoreWhereRecord): string => + Object.entries(record) + .map(([columnName, term]) => whereTerm(helpers, columnName, term)) + .filter(Boolean) + .join(' AND '); + +const whereTerm = ( + helpers: QueryHelpers, + columnName: string, + term: StoreWhereTerm, +): string => + [ + ...whereComparisonTerm(helpers, columnName, term), + ...whereLikeTerm(helpers, columnName, term), + ...whereInTerm(helpers, columnName, term), + ].join(' AND '); + +const whereComparisonTerm = ( + { encodeIdentifier, encodeLiteral }: QueryHelpers, + columnName: string, + term: StoreWhereTerm, +): string[] => { + const sql: string[] = []; + + if ('$eq' in term) { + sql.push(`${encodeIdentifier(columnName)} = ${encodeLiteral(term.$eq)}`); + } + + if ('$ne' in term) { + sql.push(`${encodeIdentifier(columnName)} != ${encodeLiteral(term.$ne)}`); + } + + if ('$gt' in term) { + sql.push(`${encodeIdentifier(columnName)} > ${encodeLiteral(term.$gt)}`); + } + + if ('$gte' in term) { + sql.push(`${encodeIdentifier(columnName)} >= ${encodeLiteral(term.$gte)}`); + } + + if ('$lt' in term) { + sql.push(`${encodeIdentifier(columnName)} < ${encodeLiteral(term.$lt)}`); + } + + if ('$lte' in term) { + sql.push(`${encodeIdentifier(columnName)} <= ${encodeLiteral(term.$lte)}`); + } + + return sql; +}; + +const whereLikeTerm = ( + { encodeIdentifier, encodeLiteral }: QueryHelpers, + columnName: string, + term: StoreWhereTerm, +): string[] => { + const sql: string[] = []; + + if ('$contains' in term) { + sql.push( + `${encodeIdentifier(columnName)} LIKE ${encodeLiteral( + `%${term.$contains}%`, + )}`, + ); + } + + if ('$startsWith' in term) { + sql.push( + `${encodeIdentifier(columnName)} LIKE ${encodeLiteral( + `${term.$startsWith}%`, + )}`, + ); + } + + if ('$endsWith' in term) { + sql.push( + `${encodeIdentifier(columnName)} LIKE ${encodeLiteral( + `%${term.$endsWith}`, + )}`, + ); + } + + if ('$in' in term) { + if (term.$in.length === 0) { + sql.push('FALSE'); + } else { + sql.push( + `${encodeIdentifier(columnName)} IN (${term.$in + .map((v) => encodeLiteral(v)) + .join(', ')})`, + ); + } + } + + return sql; +}; + +const whereInTerm = ( + { encodeIdentifier, encodeLiteral }: QueryHelpers, + columnName: string, + term: StoreWhereTerm, +): string[] => { + const sql: string[] = []; + + if ('$in' in term) { + if (term.$in.length === 0) { + sql.push('FALSE'); + } else { + sql.push( + `${encodeIdentifier(columnName)} IN (${term.$in + .map((v) => encodeLiteral(v)) + .join(', ')})`, + ); + } + } + + return sql; +}; diff --git a/packages/sql-store/src/index.ts b/packages/sql-store/src/index.ts index ef390d6..b4f856d 100644 --- a/packages/sql-store/src/index.ts +++ b/packages/sql-store/src/index.ts @@ -1,2 +1,3 @@ +export type { QueryHelpers } from './helpers'; export * from './logic'; export * from './mappers'; diff --git a/packages/sql-store/src/logic/collections/ensure.ts b/packages/sql-store/src/logic/collections/ensure.ts index 38f0827..0b409dd 100644 --- a/packages/sql-store/src/logic/collections/ensure.ts +++ b/packages/sql-store/src/logic/collections/ensure.ts @@ -8,7 +8,14 @@ import { import { SQLColumn, SQLIndexAttribute, SQLIndexColumn } from '@/mappers'; import { DescribeCollectionQueries, describeCollection } from './describe'; -export interface EnsureCollectionQueries { +export interface EnsureCollectionQueries< + Connection, + Column extends SQLColumn, + IndexAttribute extends SQLIndexAttribute & Omit, +> extends EnsureCollectionQueriesOnly, + DescribeCollectionQueries {} + +export interface EnsureCollectionQueriesOnly { createTableIfNotExists( connection: Connection, collection: StoreCollection, @@ -49,8 +56,7 @@ export const ensureCollection = async < dropIndex, dropColumn, ...describeCollectionQueries - }: EnsureCollectionQueries & - DescribeCollectionQueries, + }: EnsureCollectionQueries, ): Promise => { await createTableIfNotExists(connection, options.collection); diff --git a/packages/sql-store/src/logic/delete.ts b/packages/sql-store/src/logic/delete.ts new file mode 100644 index 0000000..03196ef --- /dev/null +++ b/packages/sql-store/src/logic/delete.ts @@ -0,0 +1,32 @@ +import { QueryHelpers, whereClause } from '@/helpers'; +import { StoreDeleteOptions, StoreMutationResponse } from '@neuledge/store'; + +export interface DeleteQueries { + deleteFrom( + connection: Connection, + name: string, + where: string | null, + limit: number, + ): Promise; + queryHelpers: QueryHelpers; +} + +export const deletes = async ( + options: StoreDeleteOptions, + connection: Connection, + { deleteFrom, queryHelpers }: DeleteQueries, +): Promise => { + const { collection, where, limit } = options; + const { name } = collection; + + const affectedCount = await deleteFrom( + connection, + name, + where ? whereClause(queryHelpers, where) : null, + limit, + ); + + return { + affectedCount, + }; +}; diff --git a/packages/sql-store/src/logic/index.ts b/packages/sql-store/src/logic/index.ts index b71af69..5760d8d 100644 --- a/packages/sql-store/src/logic/index.ts +++ b/packages/sql-store/src/logic/index.ts @@ -1,2 +1,3 @@ export * from './collections'; +export * from './delete'; export * from './insert'; From afc37fb152d30ca946a1232eadb6daeb08ca76f3 Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Thu, 20 Apr 2023 16:17:40 +0300 Subject: [PATCH 16/24] support pg update --- packages/engine/src/engine/exec/alter.test.ts | 10 --- packages/engine/src/engine/metadata/store.ts | 1 - packages/engine/src/engine/mutations/store.ts | 3 - packages/mongodb-store/src/store.ts | 51 ++----------- .../src/queries/delete-from.ts | 11 +-- .../postgresql-store/src/queries/index.ts | 1 + .../src/queries/update-set.ts | 26 +++++++ packages/postgresql-store/src/store.test.ts | 71 +++++++++++++++++-- packages/postgresql-store/src/store.ts | 4 +- packages/sql-store/src/logic/delete.ts | 4 +- packages/sql-store/src/logic/index.ts | 1 + packages/sql-store/src/logic/update.ts | 36 ++++++++++ packages/store/src/store.ts | 2 - 13 files changed, 142 insertions(+), 79 deletions(-) create mode 100644 packages/postgresql-store/src/queries/update-set.ts create mode 100644 packages/sql-store/src/logic/update.ts diff --git a/packages/engine/src/engine/exec/alter.test.ts b/packages/engine/src/engine/exec/alter.test.ts index 1d7d152..5bf70c7 100644 --- a/packages/engine/src/engine/exec/alter.test.ts +++ b/packages/engine/src/engine/exec/alter.test.ts @@ -63,13 +63,11 @@ describe('engine/exec/alter', () => { collection: metadata['collections']['categories'], where: { id: { $eq: 1 }, __h: { $eq: hash }, __v: { $eq: 0 } }, set: { name: 'foo', description: 'bar', __v: 1 }, - limit: 1, }); expect(update).toHaveBeenNthCalledWith(2, { collection: metadata['collections']['categories'], where: { id: { $eq: 2 }, __h: { $eq: hash }, __v: { $eq: 0 } }, set: { name: 'foo', description: 'bar', __v: 1 }, - limit: 1, }); }); @@ -129,13 +127,11 @@ describe('engine/exec/alter', () => { collection: metadata['collections']['categories'], where: { id: { $eq: 1 }, __h: { $eq: hash }, __v: { $eq: 0 } }, set: { name: 'foo', description: 'bar', __v: 1 }, - limit: 1, }); expect(update).toHaveBeenNthCalledWith(2, { collection: metadata['collections']['categories'], where: { id: { $eq: 2 }, __h: { $eq: hash }, __v: { $eq: 0 } }, set: { name: 'foo', description: 'bar', __v: 1 }, - limit: 1, }); }); @@ -182,13 +178,11 @@ describe('engine/exec/alter', () => { collection: metadata['collections']['categories'], where: { id: { $eq: 1 }, __h: { $eq: hash }, __v: { $eq: 0 } }, set: { name: 'foo', description: 'bar', __v: 1 }, - limit: 1, }); expect(update).toHaveBeenNthCalledWith(2, { collection: metadata['collections']['categories'], where: { id: { $eq: 2 }, __h: { $eq: hash }, __v: { $eq: 0 } }, set: { name: 'foo', description: 'bar', __v: 1 }, - limit: 1, }); }); @@ -264,13 +258,11 @@ describe('engine/exec/alter', () => { collection: metadata['collections']['posts'], where: { id: { $eq: 1 }, __h: { $eq: hash }, __v: { $eq: 0 } }, set: { title: 'foo', content: 'bar', category_id: 1, __v: 1 }, - limit: 1, }); expect(update).toHaveBeenNthCalledWith(2, { collection: metadata['collections']['posts'], where: { id: { $eq: 2 }, __h: { $eq: hash }, __v: { $eq: 0 } }, set: { title: 'foo', content: 'bar', category_id: 1, __v: 1 }, - limit: 1, }); }); }); @@ -376,7 +368,6 @@ describe('engine/exec/alter', () => { collection: metadata['collections']['categories'], where: { id: { $eq: 1 }, __h: { $eq: hash }, __v: { $eq: 0 } }, set: { name: 'foo', description: 'bar', __v: 1 }, - limit: 1, }); }); @@ -454,7 +445,6 @@ describe('engine/exec/alter', () => { collection: metadata['collections']['categories'], where: { id: { $eq: 1 }, __h: { $eq: hash }, __v: { $eq: 0 } }, set: { name: 'foo', description: 'bar', __v: 1 }, - limit: 1, }); }); diff --git a/packages/engine/src/engine/metadata/store.ts b/packages/engine/src/engine/metadata/store.ts index ab9dcea..a32a7c5 100644 --- a/packages/engine/src/engine/metadata/store.ts +++ b/packages/engine/src/engine/metadata/store.ts @@ -140,7 +140,6 @@ export const syncStoreMetadata = async ( collection: metadataCollection, where: { hash: { $eq: hash } }, set: set as never, - limit: 1, }), ), ), diff --git a/packages/engine/src/engine/mutations/store.ts b/packages/engine/src/engine/mutations/store.ts index 1faf2e2..91e202b 100644 --- a/packages/engine/src/engine/mutations/store.ts +++ b/packages/engine/src/engine/mutations/store.ts @@ -58,7 +58,6 @@ const updateStoreDocument = async ( collection, where: getWhereRecordByPrimaryKeys(collection, document), set: Object.fromEntries(setEntries), - limit: 1, }); return !!res.affectedCount; @@ -74,7 +73,6 @@ const deleteStoreDocuments = async ( await store.delete({ collection, where: getWhereByPrimaryKeys(collection, documents), - limit: documents.length, }); }; @@ -86,7 +84,6 @@ const deleteStoreDocuments = async ( // await store.delete({ // collectionName: collection.name, // where: getWhereRecord(collection.primaryKeys, document), -// limit: 1, // }); // }; diff --git a/packages/mongodb-store/src/store.ts b/packages/mongodb-store/src/store.ts index 4693a06..7afa1e0 100644 --- a/packages/mongodb-store/src/store.ts +++ b/packages/mongodb-store/src/store.ts @@ -271,33 +271,13 @@ export class MongoDBStore implements Store { const filter = options.where ? findFilter(options.collection.primaryKey, options.where) : {}; + const update = updateFilter( options.collection.primaryKey, options.set as Document, ); - let res; - - if (options.limit === 1) { - res = await collection.updateOne(filter, update); - } else { - const ids = await collection - // unicon issue: https://github.com/sindresorhus/eslint-plugin-unicorn/issues/1947 - // eslint-disable-next-line unicorn/no-array-callback-reference, unicorn/no-array-method-this-argument - .find(filter, { - limit: options.limit, - projection: { _id: 1 }, - }) - .toArray(); - - if (!ids.length) { - return { affectedCount: 0 }; - } - - res = await collection.updateMany( - { ...filter, _id: { $in: ids.map((id) => id._id) } }, - update, - ); - } + + const res = await collection.updateMany(filter, update); return { affectedCount: res.modifiedCount, @@ -310,29 +290,8 @@ export class MongoDBStore implements Store { const filter = options.where ? findFilter(options.collection.primaryKey, options.where) : {}; - let res; - - if (options.limit === 1) { - res = await collection.deleteOne(filter); - } else { - const ids = await collection - // unicon issue: https://github.com/sindresorhus/eslint-plugin-unicorn/issues/1947 - // eslint-disable-next-line unicorn/no-array-callback-reference, unicorn/no-array-method-this-argument - .find(filter, { - limit: options.limit, - projection: { _id: 1 }, - }) - .toArray(); - - if (!ids.length) { - return { affectedCount: 0 }; - } - - res = await collection.deleteMany({ - ...filter, - _id: { $in: ids.map((id) => id._id) }, - }); - } + + const res = await collection.deleteMany(filter); return { affectedCount: res.deletedCount }; } diff --git a/packages/postgresql-store/src/queries/delete-from.ts b/packages/postgresql-store/src/queries/delete-from.ts index e2a18f3..7ddf5df 100644 --- a/packages/postgresql-store/src/queries/delete-from.ts +++ b/packages/postgresql-store/src/queries/delete-from.ts @@ -4,16 +4,11 @@ export const deleteFrom = async ( connection: PostgreSQLConnection, name: string, where: string | null, - limit: number, ): Promise => { const res = await connection.query( - `DELETE FROM ${encodeIdentifier(name)}${ - where - ? ` -WHERE ${where}` - : '' - } -LIMIT ${Number(limit)}`, + where + ? `DELETE FROM ${encodeIdentifier(name)}\nWHERE ${where}` + : `TRUNCATE TABLE ${encodeIdentifier(name)}`, ); return res.rowCount; diff --git a/packages/postgresql-store/src/queries/index.ts b/packages/postgresql-store/src/queries/index.ts index e5660f7..50fac3c 100644 --- a/packages/postgresql-store/src/queries/index.ts +++ b/packages/postgresql-store/src/queries/index.ts @@ -9,3 +9,4 @@ export * from './insert-into'; export * from './list-tables'; export * from './list-table-columns'; export * from './list-table-statistics'; +export * from './update-set'; diff --git a/packages/postgresql-store/src/queries/update-set.ts b/packages/postgresql-store/src/queries/update-set.ts new file mode 100644 index 0000000..d2e7780 --- /dev/null +++ b/packages/postgresql-store/src/queries/update-set.ts @@ -0,0 +1,26 @@ +import { StoreDocument } from '@neuledge/store'; +import { + PostgreSQLConnection, + encodeIdentifier, + encodeLiteral, +} from './connection'; + +export const updateSet = async ( + connection: PostgreSQLConnection, + name: string, + set: StoreDocument, + where: string | null, +): Promise => { + const setClauses = Object.entries(set).map( + ([key, value]) => + `${encodeIdentifier(key)} = ${encodeLiteral(value ?? null)}`, + ); + + const res = await connection.query( + `UPDATE ${encodeIdentifier(name)}\nSET ${setClauses.join(', ')}${ + where ? `\nWHERE ${where}` : '' + }`, + ); + + return res.rowCount; +}; diff --git a/packages/postgresql-store/src/store.test.ts b/packages/postgresql-store/src/store.test.ts index 937b598..01c19f4 100644 --- a/packages/postgresql-store/src/store.test.ts +++ b/packages/postgresql-store/src/store.test.ts @@ -254,7 +254,56 @@ RETURNING id`, }); }); - // describe '.update()' + describe('.update()', () => { + it('should be able to update a document', async () => { + query.mockResolvedValueOnce({ rowCount: 1 }); + + const res = await store.update({ + collection: usersCollection, + where: { id: { $eq: 123 } }, + set: { + name: 'John Doe', + email: 'john@example.com', + phone: undefined, + updated_at: new Date('2020-01-01T00:00:00.000Z'), + }, + }); + + expect(query).toHaveBeenCalledTimes(1); + expect(query).toHaveBeenCalledWith( + `UPDATE ${usersTableName} +SET name = 'John Doe', email = 'john@example.com', phone = NULL, updated_at = '2020-01-01 00:00:00.000+00' +WHERE id = '123'`, + ); + + expect(res).toEqual({ + affectedCount: 1, + }); + }); + + it('should be able to update multiple arbitrary documents', async () => { + query.mockResolvedValueOnce({ rowCount: 2 }); + + const res = await store.update({ + collection: usersCollection, + set: { + name: 'John Doe', + email: 'john@example.com', + updated_at: new Date('2020-01-01T00:00:00.000Z'), + }, + }); + + expect(query).toHaveBeenCalledTimes(1); + expect(query).toHaveBeenCalledWith( + `UPDATE ${usersTableName} +SET name = 'John Doe', email = 'john@example.com', updated_at = '2020-01-01 00:00:00.000+00'`, + ); + + expect(res).toEqual({ + affectedCount: 2, + }); + }); + }); describe('.delete()', () => { it('should be able to delete a document', async () => { @@ -263,20 +312,32 @@ RETURNING id`, const res = await store.delete({ collection: usersCollection, where: { id: { $eq: 123 } }, - limit: 1, }); expect(query).toHaveBeenCalledTimes(1); expect(query).toHaveBeenCalledWith( - `DELETE FROM ${usersTableName} -WHERE id = '123' -LIMIT 1`, + `DELETE FROM ${usersTableName}\nWHERE id = '123'`, ); expect(res).toEqual({ affectedCount: 1, }); }); + + it('should be able to delete multiple arbitrary documents', async () => { + query.mockResolvedValueOnce({ rowCount: 2 }); + + const res = await store.delete({ + collection: usersCollection, + }); + + expect(query).toHaveBeenCalledTimes(1); + expect(query).toHaveBeenCalledWith(`TRUNCATE TABLE ${usersTableName}`); + + expect(res).toEqual({ + affectedCount: 2, + }); + }); }); }); }); diff --git a/packages/postgresql-store/src/store.ts b/packages/postgresql-store/src/store.ts index 557531a..0e6e16d 100644 --- a/packages/postgresql-store/src/store.ts +++ b/packages/postgresql-store/src/store.ts @@ -12,6 +12,7 @@ import { dropTableIfExists, insertInto, deleteFrom, + updateSet, } from './queries'; import { Store, @@ -35,6 +36,7 @@ import { listCollections, ensureCollection, insert, + update, } from '@neuledge/sql-store'; import { deletes } from '@neuledge/sql-store'; import { queryHelpers } from './queries/connection'; @@ -102,7 +104,7 @@ export class PostgreSQLStore implements Store { } async update(options: StoreUpdateOptions): Promise { - throw new Error('Method not implemented.'); + return update(options, this.connection, { updateSet, queryHelpers }); } async delete(options: StoreDeleteOptions): Promise { diff --git a/packages/sql-store/src/logic/delete.ts b/packages/sql-store/src/logic/delete.ts index 03196ef..465c7cf 100644 --- a/packages/sql-store/src/logic/delete.ts +++ b/packages/sql-store/src/logic/delete.ts @@ -6,7 +6,6 @@ export interface DeleteQueries { connection: Connection, name: string, where: string | null, - limit: number, ): Promise; queryHelpers: QueryHelpers; } @@ -16,14 +15,13 @@ export const deletes = async ( connection: Connection, { deleteFrom, queryHelpers }: DeleteQueries, ): Promise => { - const { collection, where, limit } = options; + const { collection, where } = options; const { name } = collection; const affectedCount = await deleteFrom( connection, name, where ? whereClause(queryHelpers, where) : null, - limit, ); return { diff --git a/packages/sql-store/src/logic/index.ts b/packages/sql-store/src/logic/index.ts index 5760d8d..b6adaf4 100644 --- a/packages/sql-store/src/logic/index.ts +++ b/packages/sql-store/src/logic/index.ts @@ -1,3 +1,4 @@ export * from './collections'; export * from './delete'; export * from './insert'; +export * from './update'; diff --git a/packages/sql-store/src/logic/update.ts b/packages/sql-store/src/logic/update.ts new file mode 100644 index 0000000..e0776e3 --- /dev/null +++ b/packages/sql-store/src/logic/update.ts @@ -0,0 +1,36 @@ +import { QueryHelpers, whereClause } from '@/helpers'; +import { + StoreDocument, + StoreMutationResponse, + StoreUpdateOptions, +} from '@neuledge/store'; + +export interface UpdateQueries { + updateSet( + connection: Connection, + name: string, + set: StoreDocument, + where: string | null, + ): Promise; + queryHelpers: QueryHelpers; +} + +export const update = async ( + options: StoreUpdateOptions, + connection: Connection, + { updateSet, queryHelpers }: UpdateQueries, +): Promise => { + const { collection, set, where } = options; + const { name } = collection; + + const affectedCount = await updateSet( + connection, + name, + set, + where ? whereClause(queryHelpers, where) : null, + ); + + return { + affectedCount, + }; +}; diff --git a/packages/store/src/store.ts b/packages/store/src/store.ts index 564bbc2..c529110 100644 --- a/packages/store/src/store.ts +++ b/packages/store/src/store.ts @@ -70,13 +70,11 @@ export interface StoreUpdateOptions { collection: StoreCollection; where?: StoreWhere | null; set: StoreDocument; - limit: number; } export interface StoreDeleteOptions { collection: StoreCollection; where?: StoreWhere | null; - limit: number; } export interface StoreInsertionResponse extends StoreMutationResponse { From e72664dda4b2329bc114e282925547cb817237c9 Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Fri, 21 Apr 2023 13:09:21 +0200 Subject: [PATCH 17/24] pg basic sql find --- packages/mysql-store/src/store.ts | 3 +- .../src/queries/__fixtures__/users-table.ts | 15 +++- .../postgresql-store/src/queries/index.ts | 1 + .../src/queries/select-from.ts | 28 +++++++ packages/postgresql-store/src/store.test.ts | 73 +++++++++++++++++++ packages/postgresql-store/src/store.ts | 7 +- packages/sql-store/src/logic/find.ts | 55 ++++++++++++++ packages/sql-store/src/logic/index.ts | 1 + packages/store/src/store.ts | 12 +++ 9 files changed, 189 insertions(+), 6 deletions(-) create mode 100644 packages/postgresql-store/src/queries/select-from.ts create mode 100644 packages/sql-store/src/logic/find.ts diff --git a/packages/mysql-store/src/store.ts b/packages/mysql-store/src/store.ts index 7e52de8..fde3725 100644 --- a/packages/mysql-store/src/store.ts +++ b/packages/mysql-store/src/store.ts @@ -5,7 +5,6 @@ import { StoreCollection_Slim, StoreDeleteOptions, StoreDescribeCollectionOptions, - StoreDocument, StoreDropCollectionOptions, StoreEnsureCollectionOptions, StoreFindOptions, @@ -87,7 +86,7 @@ export class MySQLStore implements Store { return dropCollection(options, this.connection, { dropTableIfExists }); } - async find(options: StoreFindOptions): Promise> { + async find(options: StoreFindOptions): Promise { throw new Error('Method not implemented.'); } diff --git a/packages/postgresql-store/src/queries/__fixtures__/users-table.ts b/packages/postgresql-store/src/queries/__fixtures__/users-table.ts index 739d234..dda1a62 100644 --- a/packages/postgresql-store/src/queries/__fixtures__/users-table.ts +++ b/packages/postgresql-store/src/queries/__fixtures__/users-table.ts @@ -1,4 +1,8 @@ -import { StoreCollection, StoreCollection_Slim } from '@neuledge/store'; +import { + StoreCollection, + StoreCollection_Slim, + StoreDocument, +} from '@neuledge/store'; import { PostgreSQLTable } from '../list-tables'; import { PostgreSQLColumn } from '../list-table-columns'; import { PostgreSQLIndexAttribute } from '../list-table-statistics'; @@ -206,3 +210,12 @@ export const usersCollection: StoreCollection = { }, }, }; + +export const usersTableRow1: StoreDocument = { + id: 1, + name: 'John Doe', + email: 'john@example.com', + phone: '+1 555 555 5555', + created_at: new Date('2020-01-01T00:00:00.000Z'), + updated_at: new Date('2020-01-01T00:02:00.000Z'), +}; diff --git a/packages/postgresql-store/src/queries/index.ts b/packages/postgresql-store/src/queries/index.ts index 50fac3c..7c03111 100644 --- a/packages/postgresql-store/src/queries/index.ts +++ b/packages/postgresql-store/src/queries/index.ts @@ -10,3 +10,4 @@ export * from './list-tables'; export * from './list-table-columns'; export * from './list-table-statistics'; export * from './update-set'; +export * from './select-from'; diff --git a/packages/postgresql-store/src/queries/select-from.ts b/packages/postgresql-store/src/queries/select-from.ts new file mode 100644 index 0000000..575f9da --- /dev/null +++ b/packages/postgresql-store/src/queries/select-from.ts @@ -0,0 +1,28 @@ +import { StoreDocument } from '@neuledge/store'; +import { PostgreSQLConnection, encodeIdentifier } from './connection'; + +export const selectFrom = async ( + connection: PostgreSQLConnection, + name: string, + select: string[] | true, + where: string | null, + limit: number, + offset: number, +): Promise => { + const { rows } = await connection.query( + `SELECT ${ + select === true + ? '*' + : select.map((column) => encodeIdentifier(column)).join(', ') + } +FROM ${encodeIdentifier(name)}${ + where + ? ` +WHERE ${where}` + : '' + } +LIMIT ${Number(limit)} OFFSET ${Number(offset)}`, + ); + + return rows; +}; diff --git a/packages/postgresql-store/src/store.test.ts b/packages/postgresql-store/src/store.test.ts index 01c19f4..3e54e80 100644 --- a/packages/postgresql-store/src/store.test.ts +++ b/packages/postgresql-store/src/store.test.ts @@ -11,6 +11,7 @@ import { usersTableIndexes, usersTableName, usersTablePrimaryIndexes, + usersTableRow1, usersTable_createSql, usersTable_dropSql, usersTable_emailIndexCreateSql, @@ -193,6 +194,78 @@ describe('store', () => { }); }); + describe('.find()', () => { + it('should be able to find documents', async () => { + query.mockResolvedValueOnce({ rows: [usersTableRow1] }); + + const res = await store.find({ + collection: usersCollection, + where: { + email: { $eq: 'john@example.com' }, + }, + limit: 1, + }); + + expect(query).toHaveBeenCalledTimes(1); + + expect(query).toHaveBeenCalledWith( + `SELECT * +FROM ${usersTableName} +WHERE email = 'john@example.com' +LIMIT 1 OFFSET 0`, + ); + + expect(res).toEqual(Object.assign([usersTableRow1], { nextOffset: 1 })); + }); + + it('should be able to find documents with offset', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + const res = await store.find({ + collection: usersCollection, + where: { + email: { $eq: 'john@example.com' }, + }, + limit: 1, + offset: 1, + }); + expect(query).toHaveBeenCalledTimes(1); + + expect(query).toHaveBeenCalledWith( + `SELECT * +FROM ${usersTableName} +WHERE email = 'john@example.com' +LIMIT 1 OFFSET 1`, + ); + + expect(res).toEqual(Object.assign([], { nextOffset: null })); + }); + + it('should be able to select columns', async () => { + query.mockResolvedValueOnce({ rows: [usersTableRow1] }); + + const res = await store.find({ + collection: usersCollection, + select: { + id: true, + name: true, + phone: false, + }, + limit: 1, + }); + + expect(query).toHaveBeenCalledTimes(1); + + expect(query).toHaveBeenCalledWith( + `SELECT id, name +FROM ${usersTableName} +LIMIT 1 OFFSET 0`, + ); + + expect(res).toEqual(Object.assign([usersTableRow1], { nextOffset: 1 })); + }); + }); + describe('.insert()', () => { it('should be able to insert a document with auto increment', async () => { query.mockResolvedValueOnce({ rows: [{ id: 1234 }] }); diff --git a/packages/postgresql-store/src/store.ts b/packages/postgresql-store/src/store.ts index 0e6e16d..5156daf 100644 --- a/packages/postgresql-store/src/store.ts +++ b/packages/postgresql-store/src/store.ts @@ -13,6 +13,7 @@ import { insertInto, deleteFrom, updateSet, + selectFrom, } from './queries'; import { Store, @@ -20,7 +21,6 @@ import { StoreCollection_Slim, StoreDeleteOptions, StoreDescribeCollectionOptions, - StoreDocument, StoreDropCollectionOptions, StoreEnsureCollectionOptions, StoreFindOptions, @@ -40,6 +40,7 @@ import { } from '@neuledge/sql-store'; import { deletes } from '@neuledge/sql-store'; import { queryHelpers } from './queries/connection'; +import { find } from '@neuledge/sql-store'; export type PostgreSQLStoreClient = Pick; @@ -95,8 +96,8 @@ export class PostgreSQLStore implements Store { return dropCollection(options, this.connection, { dropTableIfExists }); } - async find(options: StoreFindOptions): Promise> { - throw new Error('Method not implemented.'); + async find(options: StoreFindOptions): Promise { + return find(options, this.connection, { selectFrom, queryHelpers }); } async insert(options: StoreInsertOptions): Promise { diff --git a/packages/sql-store/src/logic/find.ts b/packages/sql-store/src/logic/find.ts new file mode 100644 index 0000000..7326968 --- /dev/null +++ b/packages/sql-store/src/logic/find.ts @@ -0,0 +1,55 @@ +import { QueryHelpers, whereClause } from '@/helpers'; +import { StoreDocument, StoreFindOptions, StoreList } from '@neuledge/store'; + +export interface FindQueries { + selectFrom( + connection: Connection, + name: string, + select: string[] | true, + where: string | null, + limit: number, + offset: number, + ): Promise; + queryHelpers: QueryHelpers; +} + +export const find = async ( + options: StoreFindOptions, + connection: Connection, + { selectFrom, queryHelpers }: FindQueries, +): Promise => { + const { + collection, + select: selectMap, + where, + innerJoin, + leftJoin, + limit, + offset: storeOffset, + sort, + } = options; + const { name } = collection; + + if (innerJoin || leftJoin || sort) { + // FIXME implement joins and sorting on postgresql + throw new Error('Joins and sorting are not supported yet'); + } + + const select = selectMap + ? Object.keys(selectMap).filter((key) => selectMap[key]) + : true; + const offset = storeOffset ? Number(storeOffset) : 0; + + const rows = await selectFrom( + connection, + name, + select, + where ? whereClause(queryHelpers, where) : null, + limit, + offset, + ); + + const nextOffset = rows.length < limit ? null : offset + limit; + + return Object.assign(rows, { nextOffset }); +}; diff --git a/packages/sql-store/src/logic/index.ts b/packages/sql-store/src/logic/index.ts index b6adaf4..4750cb4 100644 --- a/packages/sql-store/src/logic/index.ts +++ b/packages/sql-store/src/logic/index.ts @@ -1,4 +1,5 @@ export * from './collections'; export * from './delete'; export * from './insert'; +export * from './find'; export * from './update'; diff --git a/packages/store/src/store.ts b/packages/store/src/store.ts index c529110..54e1bc8 100644 --- a/packages/store/src/store.ts +++ b/packages/store/src/store.ts @@ -52,7 +52,13 @@ export interface StoreDropCollectionOptions { export interface StoreFindOptions { collection: StoreCollection; + + /** + * Select only the specified fields to be returned. + * If not specified, all fields will be returned. + */ select?: StoreSelect | null; + where?: StoreWhere | null; innerJoin?: StoreJoin | null; leftJoin?: StoreLeftJoin | null; @@ -69,6 +75,12 @@ export interface StoreInsertOptions { export interface StoreUpdateOptions { collection: StoreCollection; where?: StoreWhere | null; + + /** + * Set is a document that contains the fields to be updated and their new values. + * The fields that are not present in the set document will not be updated. + * `undefined` values will be converted to `null` values. + */ set: StoreDocument; } From e8b469da599e947a0ff149fdb0ec0f7dde191970 Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Thu, 27 Apr 2023 21:49:33 +0300 Subject: [PATCH 18/24] basic sql joins --- .../src/queries/__fixtures__/users-table.ts | 10 +- .../src/queries/create-table.ts | 17 +- .../src/queries/delete-from.ts | 2 +- .../src/queries/insert-into.ts | 8 +- .../src/queries/list-table-columns.ts | 4 +- .../src/queries/list-table-statistics.ts | 36 +-- .../src/queries/list-tables.ts | 4 +- .../src/queries/select-from.ts | 22 +- .../src/queries/update-set.ts | 4 +- packages/postgresql-store/src/store.test.ts | 62 +++-- packages/sql-store/src/helpers/index.ts | 5 +- .../sql-store/src/helpers/join/documents.ts | 65 +++++ .../sql-store/src/helpers/join/from.test.ts | 197 +++++++++++++++ packages/sql-store/src/helpers/join/from.ts | 224 ++++++++++++++++++ packages/sql-store/src/helpers/join/index.ts | 2 + packages/sql-store/src/helpers/order.ts | 15 ++ packages/sql-store/src/helpers/select.ts | 40 ++++ .../src/helpers/{where-clause.ts => where.ts} | 2 +- packages/sql-store/src/logic/delete.ts | 4 +- packages/sql-store/src/logic/find.ts | 83 ++++--- packages/sql-store/src/logic/update.ts | 4 +- 21 files changed, 688 insertions(+), 122 deletions(-) create mode 100644 packages/sql-store/src/helpers/join/documents.ts create mode 100644 packages/sql-store/src/helpers/join/from.test.ts create mode 100644 packages/sql-store/src/helpers/join/from.ts create mode 100644 packages/sql-store/src/helpers/join/index.ts create mode 100644 packages/sql-store/src/helpers/order.ts create mode 100644 packages/sql-store/src/helpers/select.ts rename packages/sql-store/src/helpers/{where-clause.ts => where.ts} (99%) diff --git a/packages/postgresql-store/src/queries/__fixtures__/users-table.ts b/packages/postgresql-store/src/queries/__fixtures__/users-table.ts index dda1a62..ffbf2d6 100644 --- a/packages/postgresql-store/src/queries/__fixtures__/users-table.ts +++ b/packages/postgresql-store/src/queries/__fixtures__/users-table.ts @@ -11,15 +11,7 @@ export const usersTableName = 'users'; export const usersTable_dropSql = `DROP TABLE IF EXISTS users`; -export const usersTable_createSql = `CREATE TABLE IF NOT EXISTS users ( - id BIGSERIAL NOT NULL, - name VARCHAR(50), - email VARCHAR(100) NOT NULL, - phone VARCHAR(20), - created_at TIMESTAMP NOT NULL, - updated_at TIMESTAMP NOT NULL, - CONSTRAINT users_pkey PRIMARY KEY (id) -)`; +export const usersTable_createSql = `CREATE TABLE IF NOT EXISTS users (id BIGSERIAL NOT NULL, name VARCHAR(50), email VARCHAR(100) NOT NULL, phone VARCHAR(20), created_at TIMESTAMP NOT NULL, updated_at TIMESTAMP NOT NULL, CONSTRAINT users_pkey PRIMARY KEY (id))`; export const usersTable_phoneAddSql = `ALTER TABLE users ADD COLUMN phone VARCHAR(20)`; diff --git a/packages/postgresql-store/src/queries/create-table.ts b/packages/postgresql-store/src/queries/create-table.ts index 4b3ee4d..e04523e 100644 --- a/packages/postgresql-store/src/queries/create-table.ts +++ b/packages/postgresql-store/src/queries/create-table.ts @@ -8,16 +8,15 @@ export const createTableIfNotExists = async ( collection: StoreCollection, ): Promise => { await connection.query( - `CREATE TABLE IF NOT EXISTS ${encodeIdentifier(collection.name)} ( - ${Object.values(collection.fields) - .map((field) => getColumnDefinition(field, collection)) - .join(',\n ')}, - CONSTRAINT ${encodeIdentifier( - `${collection.name}_pkey`, - )} PRIMARY KEY (${Object.keys(collection.primaryKey.fields) + `CREATE TABLE IF NOT EXISTS ${encodeIdentifier( + collection.name, + )} (${Object.values(collection.fields) + .map((field) => getColumnDefinition(field, collection)) + .join(', ')}, CONSTRAINT ${encodeIdentifier( + `${collection.name}_pkey`, + )} PRIMARY KEY (${Object.keys(collection.primaryKey.fields) .map((val) => encodeIdentifier(val)) - .join(', ')}) -)`, + .join(', ')}))`, ); // add unique constraints if primary key has descending fields diff --git a/packages/postgresql-store/src/queries/delete-from.ts b/packages/postgresql-store/src/queries/delete-from.ts index 7ddf5df..eb47fa6 100644 --- a/packages/postgresql-store/src/queries/delete-from.ts +++ b/packages/postgresql-store/src/queries/delete-from.ts @@ -7,7 +7,7 @@ export const deleteFrom = async ( ): Promise => { const res = await connection.query( where - ? `DELETE FROM ${encodeIdentifier(name)}\nWHERE ${where}` + ? `DELETE FROM ${encodeIdentifier(name)} WHERE ${where}` : `TRUNCATE TABLE ${encodeIdentifier(name)}`, ); diff --git a/packages/postgresql-store/src/queries/insert-into.ts b/packages/postgresql-store/src/queries/insert-into.ts index 78ea348..285c664 100644 --- a/packages/postgresql-store/src/queries/insert-into.ts +++ b/packages/postgresql-store/src/queries/insert-into.ts @@ -16,14 +16,14 @@ export const insertInto = async ( .query( `INSERT INTO ${encodeIdentifier(name)} (${columns .map((column) => encodeIdentifier(column)) - .join(', ')}) -VALUES (${values + .join(', ')}) VALUES (${values .map((arr) => arr .map((v) => (v === undefined ? 'DEFAULT' : encodeLiteral(v))) .join(', '), ) - .join('), (')}) -RETURNING ${returns.map((column) => encodeIdentifier(column)).join(', ')}`, + .join('), (')}) RETURNING ${returns + .map((column) => encodeIdentifier(column)) + .join(', ')}`, ) .then((res) => res.rows); diff --git a/packages/postgresql-store/src/queries/list-table-columns.ts b/packages/postgresql-store/src/queries/list-table-columns.ts index 625e032..e313bbd 100644 --- a/packages/postgresql-store/src/queries/list-table-columns.ts +++ b/packages/postgresql-store/src/queries/list-table-columns.ts @@ -22,9 +22,7 @@ export const listTableColumns = async ( .query(listTableColumns_sql, [tableName]) .then((result) => result.rows); -export const listTableColumns_sql = `SELECT column_name, data_type, character_maximum_length, numeric_precision, numeric_scale, (is_nullable = 'YES') as is_nullable, column_default LIKE 'nextval(%)' AS is_auto_increment -FROM information_schema.columns -WHERE table_catalog = current_database() AND table_schema = current_schema() AND table_name = $1`; +export const listTableColumns_sql = `SELECT column_name, data_type, character_maximum_length, numeric_precision, numeric_scale, (is_nullable = 'YES') as is_nullable, column_default LIKE 'nextval(%)' AS is_auto_increment FROM information_schema.columns WHERE table_catalog = current_database() AND table_schema = current_schema() AND table_name = $1`; // https://www.postgresql.org/docs/current/datatype.html export const dataTypeMap: Record = { diff --git a/packages/postgresql-store/src/queries/list-table-statistics.ts b/packages/postgresql-store/src/queries/list-table-statistics.ts index 6a23fde..4f231cc 100644 --- a/packages/postgresql-store/src/queries/list-table-statistics.ts +++ b/packages/postgresql-store/src/queries/list-table-statistics.ts @@ -33,21 +33,21 @@ export const listIndexAttributes = async ( return rows; }; -export const listIndexAttributes_sql = `SELECT -irel.relname AS index_name, -a.attname AS column_name, -c.ordinality as seq_in_index, -CASE o.option & 1 WHEN 1 THEN 'DESC' ELSE 'ASC' END AS direction, -CASE o.option & 2 WHEN 2 THEN 'FIRST' ELSE 'LAST' END AS nulls, -i.indisunique AS is_unique, -i.indisprimary AS is_primary -FROM pg_index AS i -JOIN pg_class AS trel ON trel.oid = i.indrelid -JOIN pg_namespace AS tnsp ON trel.relnamespace = tnsp.oid -JOIN pg_class AS irel ON irel.oid = i.indexrelid -CROSS JOIN LATERAL unnest (i.indkey) WITH ORDINALITY AS c (colnum, ordinality) -LEFT JOIN LATERAL unnest (i.indoption) WITH ORDINALITY AS o (option, ordinality) -ON c.ordinality = o.ordinality -JOIN pg_attribute AS a ON trel.oid = a.attrelid AND a.attnum = c.colnum -WHERE tnsp.nspname = current_schema() AND trel.relname = $1 -ORDER BY index_name, seq_in_index`; +export const listIndexAttributes_sql = + `SELECT` + + ` irel.relname AS index_name,` + + ` a.attname AS column_name,` + + ` c.ordinality as seq_in_index,` + + ` CASE o.option & 1 WHEN 1 THEN 'DESC' ELSE 'ASC' END AS direction,` + + ` CASE o.option & 2 WHEN 2 THEN 'FIRST' ELSE 'LAST' END AS nulls,` + + ` i.indisunique AS is_unique,` + + ` i.indisprimary AS is_primary` + + ` FROM pg_index AS i` + + ` JOIN pg_class AS trel ON trel.oid = i.indrelid` + + ` JOIN pg_namespace AS tnsp ON trel.relnamespace = tnsp.oid` + + ` JOIN pg_class AS irel ON irel.oid = i.indexrelid` + + ` CROSS JOIN LATERAL unnest (i.indkey) WITH ORDINALITY AS c (colnum, ordinality)` + + ` LEFT JOIN LATERAL unnest (i.indoption) WITH ORDINALITY AS o (option, ordinality) ON c.ordinality = o.ordinality` + + ` JOIN pg_attribute AS a ON trel.oid = a.attrelid AND a.attnum = c.colnum` + + ` WHERE tnsp.nspname = current_schema() AND trel.relname = $1` + + ` ORDER BY index_name, seq_in_index`; diff --git a/packages/postgresql-store/src/queries/list-tables.ts b/packages/postgresql-store/src/queries/list-tables.ts index b5c29ee..8e0e7f8 100644 --- a/packages/postgresql-store/src/queries/list-tables.ts +++ b/packages/postgresql-store/src/queries/list-tables.ts @@ -14,6 +14,4 @@ export const listTables = async ( .query(listTables_sql) .then((result) => result.rows); -export const listTables_sql = `SELECT table_name -FROM information_schema.tables -WHERE table_catalog = current_database() AND table_schema = current_schema() AND table_type = 'BASE TABLE'`; +export const listTables_sql = `SELECT table_name FROM information_schema.tables WHERE table_catalog = current_database() AND table_schema = current_schema() AND table_type = 'BASE TABLE'`; diff --git a/packages/postgresql-store/src/queries/select-from.ts b/packages/postgresql-store/src/queries/select-from.ts index 575f9da..ff1f0ae 100644 --- a/packages/postgresql-store/src/queries/select-from.ts +++ b/packages/postgresql-store/src/queries/select-from.ts @@ -1,27 +1,19 @@ import { StoreDocument } from '@neuledge/store'; -import { PostgreSQLConnection, encodeIdentifier } from './connection'; +import { PostgreSQLConnection } from './connection'; export const selectFrom = async ( connection: PostgreSQLConnection, - name: string, - select: string[] | true, + select: string, + from: string, where: string | null, + orderBy: string | null, limit: number, offset: number, ): Promise => { const { rows } = await connection.query( - `SELECT ${ - select === true - ? '*' - : select.map((column) => encodeIdentifier(column)).join(', ') - } -FROM ${encodeIdentifier(name)}${ - where - ? ` -WHERE ${where}` - : '' - } -LIMIT ${Number(limit)} OFFSET ${Number(offset)}`, + `SELECT ${select} FROM ${from}${where ? ` WHERE ${where}` : ''}${ + orderBy ? ` ORDER BY ${orderBy}` : '' + } LIMIT ${Number(limit)} OFFSET ${Number(offset)}`, ); return rows; diff --git a/packages/postgresql-store/src/queries/update-set.ts b/packages/postgresql-store/src/queries/update-set.ts index d2e7780..019a8f3 100644 --- a/packages/postgresql-store/src/queries/update-set.ts +++ b/packages/postgresql-store/src/queries/update-set.ts @@ -17,8 +17,8 @@ export const updateSet = async ( ); const res = await connection.query( - `UPDATE ${encodeIdentifier(name)}\nSET ${setClauses.join(', ')}${ - where ? `\nWHERE ${where}` : '' + `UPDATE ${encodeIdentifier(name)} SET ${setClauses.join(', ')}${ + where ? ` WHERE ${where}` : '' }`, ); diff --git a/packages/postgresql-store/src/store.test.ts b/packages/postgresql-store/src/store.test.ts index 3e54e80..5470a9f 100644 --- a/packages/postgresql-store/src/store.test.ts +++ b/packages/postgresql-store/src/store.test.ts @@ -209,10 +209,7 @@ describe('store', () => { expect(query).toHaveBeenCalledTimes(1); expect(query).toHaveBeenCalledWith( - `SELECT * -FROM ${usersTableName} -WHERE email = 'john@example.com' -LIMIT 1 OFFSET 0`, + `SELECT * FROM ${usersTableName} WHERE email = 'john@example.com' LIMIT 1 OFFSET 0`, ); expect(res).toEqual(Object.assign([usersTableRow1], { nextOffset: 1 })); @@ -232,10 +229,7 @@ LIMIT 1 OFFSET 0`, expect(query).toHaveBeenCalledTimes(1); expect(query).toHaveBeenCalledWith( - `SELECT * -FROM ${usersTableName} -WHERE email = 'john@example.com' -LIMIT 1 OFFSET 1`, + `SELECT * FROM ${usersTableName} WHERE email = 'john@example.com' LIMIT 1 OFFSET 1`, ); expect(res).toEqual(Object.assign([], { nextOffset: null })); @@ -257,9 +251,38 @@ LIMIT 1 OFFSET 1`, expect(query).toHaveBeenCalledTimes(1); expect(query).toHaveBeenCalledWith( - `SELECT id, name -FROM ${usersTableName} -LIMIT 1 OFFSET 0`, + `SELECT id, name FROM ${usersTableName} LIMIT 1 OFFSET 0`, + ); + + expect(res).toEqual( + Object.assign( + [ + { + id: usersTableRow1.id, + name: usersTableRow1.name, + }, + ], + { nextOffset: 1 }, + ), + ); + }); + + it('should be able to sort documents', async () => { + query.mockResolvedValueOnce({ rows: [usersTableRow1] }); + + const res = await store.find({ + collection: usersCollection, + sort: { + name: 'desc', + email: 'asc', + }, + limit: 1, + }); + + expect(query).toHaveBeenCalledTimes(1); + + expect(query).toHaveBeenCalledWith( + `SELECT * FROM ${usersTableName} ORDER BY name DESC, email ASC LIMIT 1 OFFSET 0`, ); expect(res).toEqual(Object.assign([usersTableRow1], { nextOffset: 1 })); @@ -285,9 +308,7 @@ LIMIT 1 OFFSET 0`, expect(query).toHaveBeenCalledTimes(1); expect(query).toHaveBeenCalledWith( - `INSERT INTO ${usersTableName} (id, name, email, phone, created_at, updated_at) -VALUES (DEFAULT, 'John Doe', 'john@example.com', NULL, '2020-01-01 00:00:00.000+00', '2020-01-01 00:00:00.000+00') -RETURNING id`, + `INSERT INTO ${usersTableName} (id, name, email, phone, created_at, updated_at) VALUES (DEFAULT, 'John Doe', 'john@example.com', NULL, '2020-01-01 00:00:00.000+00', '2020-01-01 00:00:00.000+00') RETURNING id`, ); expect(res).toEqual({ @@ -315,9 +336,7 @@ RETURNING id`, expect(query).toHaveBeenCalledTimes(1); expect(query).toHaveBeenCalledWith( - `INSERT INTO ${usersTableName} (id, name, email, phone, created_at, updated_at) -VALUES ('789', 'John Doe', 'john@example.com', NULL, '2020-01-01 00:00:00.000+00', '2020-01-01 00:00:00.000+00') -RETURNING id`, + `INSERT INTO ${usersTableName} (id, name, email, phone, created_at, updated_at) VALUES ('789', 'John Doe', 'john@example.com', NULL, '2020-01-01 00:00:00.000+00', '2020-01-01 00:00:00.000+00') RETURNING id`, ); expect(res).toEqual({ @@ -344,9 +363,7 @@ RETURNING id`, expect(query).toHaveBeenCalledTimes(1); expect(query).toHaveBeenCalledWith( - `UPDATE ${usersTableName} -SET name = 'John Doe', email = 'john@example.com', phone = NULL, updated_at = '2020-01-01 00:00:00.000+00' -WHERE id = '123'`, + `UPDATE ${usersTableName} SET name = 'John Doe', email = 'john@example.com', phone = NULL, updated_at = '2020-01-01 00:00:00.000+00' WHERE id = '123'`, ); expect(res).toEqual({ @@ -368,8 +385,7 @@ WHERE id = '123'`, expect(query).toHaveBeenCalledTimes(1); expect(query).toHaveBeenCalledWith( - `UPDATE ${usersTableName} -SET name = 'John Doe', email = 'john@example.com', updated_at = '2020-01-01 00:00:00.000+00'`, + `UPDATE ${usersTableName} SET name = 'John Doe', email = 'john@example.com', updated_at = '2020-01-01 00:00:00.000+00'`, ); expect(res).toEqual({ @@ -389,7 +405,7 @@ SET name = 'John Doe', email = 'john@example.com', updated_at = '2020-01-01 00:0 expect(query).toHaveBeenCalledTimes(1); expect(query).toHaveBeenCalledWith( - `DELETE FROM ${usersTableName}\nWHERE id = '123'`, + `DELETE FROM ${usersTableName} WHERE id = '123'`, ); expect(res).toEqual({ diff --git a/packages/sql-store/src/helpers/index.ts b/packages/sql-store/src/helpers/index.ts index 65adbee..a9bf3b7 100644 --- a/packages/sql-store/src/helpers/index.ts +++ b/packages/sql-store/src/helpers/index.ts @@ -1,2 +1,5 @@ +export * from './join'; +export * from './order'; export * from './query'; -export * from './where-clause'; +export * from './select'; +export * from './where'; diff --git a/packages/sql-store/src/helpers/join/documents.ts b/packages/sql-store/src/helpers/join/documents.ts new file mode 100644 index 0000000..c16ce71 --- /dev/null +++ b/packages/sql-store/src/helpers/join/documents.ts @@ -0,0 +1,65 @@ +import { StoreDocument, StoreJoinChoice } from '@neuledge/store'; + +export const fillJoinedDocuments = ( + { innerJoin, leftJoin }: Pick, + rawDoc: StoreDocument, + doc: StoreDocument, +): StoreDocument | null => { + if (innerJoin) { + for (const [key, choices] of Object.entries(innerJoin)) { + const joinedDoc = findJoinedDocument(choices, rawDoc); + if (!joinedDoc) return null; + + if (joinedDoc !== true) { + doc[key] = joinedDoc; + } + } + } + + if (leftJoin) { + for (const [key, choices] of Object.entries(leftJoin)) { + const joinedDoc = findJoinedDocument(choices, rawDoc); + if (!joinedDoc) continue; + + if (joinedDoc !== true) { + doc[key] = joinedDoc; + } + } + } + + return doc; +}; + +// local helpers + +const findJoinedDocument = ( + choices: StoreJoinChoice[], + rawDoc: StoreDocument, +): StoreDocument | null | true => { + for (const choice of choices) { + const { collection, by, select } = choice; + + const found = Object.entries(by).every(([key, term]) => { + const value = rawDoc[`${collection.name}.${key}`]; + if (value === undefined) return false; + + return term.field ? value === rawDoc[term.field] : value === term.value; + }); + if (!found) continue; + + if (!select) { + return true; + } + + const keys = + select === true + ? Object.keys(collection.fields) + : Object.keys(select).filter((key) => select[key]); + + return Object.fromEntries( + keys.map((key) => [key, rawDoc[`${collection.name}.${key}`]]), + ); + } + + return null; +}; diff --git a/packages/sql-store/src/helpers/join/from.test.ts b/packages/sql-store/src/helpers/join/from.test.ts new file mode 100644 index 0000000..d5736a6 --- /dev/null +++ b/packages/sql-store/src/helpers/join/from.test.ts @@ -0,0 +1,197 @@ +import { StoreCollection } from '@neuledge/store'; +import { QueryHelpers } from '../query'; +import { getFromJoins } from './from'; + +/* eslint-disable max-lines-per-function */ + +describe('helpers/join/from', () => { + describe('getFromJoins()', () => { + const helpers: QueryHelpers = { + encodeIdentifier: (name) => `\`${name.replace(/([\\`])/g, '\\$1')}\``, + encodeLiteral: (value) => JSON.stringify(value), + }; + + const collection: StoreCollection = { + name: 'collection', + primaryKey: { fields: { id: true } }, + fields: { id: true, name: true, foo: true, bar: true }, + } as never; + + const otherCollection: StoreCollection = { + name: 'otherCollection', + primaryKey: { fields: { id: true, subId: true } }, + fields: { id: true, subId: true, title: true, description: true }, + } as never; + + const otherCollection2: StoreCollection = { + name: 'otherCollection2', + primaryKey: { fields: { id: true } }, + fields: { id: true, image: true, url: true }, + } as never; + + it('should return null if no joins', () => { + expect( + getFromJoins(helpers, { + collection, + }), + ).toBeNull(); + }); + + it('should handle simple single inner join', () => { + expect( + getFromJoins(helpers, { + collection, + innerJoin: { + foo: [ + { + collection: otherCollection, + select: true, + by: { id: { field: 'foo' } }, + }, + ], + }, + }), + ).toEqual({ + selectColumns: [ + '`foo$0`.`id` AS `foo$0.id`', + '`foo$0`.`subId` AS `foo$0.subId`', + '`foo$0`.`title` AS `foo$0.title`', + '`foo$0`.`description` AS `foo$0.description`', + ], + fromAlias: '$', + fromJoins: [ + 'INNER JOIN `otherCollection` `foo$0` ON (`foo$0`.`id` = `$`.`foo`)', + ], + whereClauses: [], + }); + }); + + it('should handle simple single left join', () => { + expect( + getFromJoins(helpers, { + collection, + leftJoin: { + foo: [ + { + collection: otherCollection, + select: { title: true }, + by: { id: { field: 'foo' }, subId: { field: 'bar' } }, + }, + ], + }, + }), + ).toEqual({ + selectColumns: ['`foo$0`.`title` AS `foo$0.title`'], + fromAlias: '$', + fromJoins: [ + 'LEFT JOIN `otherCollection` `foo$0` ON (`foo$0`.`id` = `$`.`foo` AND `foo$0`.`subId` = `$`.`bar`)', + ], + whereClauses: [], + }); + }); + + it('should handle multiple joins', () => { + expect( + getFromJoins(helpers, { + collection, + innerJoin: { + foo: [ + { + collection: otherCollection, + select: { title: true }, + by: { id: { field: 'foo' } }, + }, + ], + }, + leftJoin: { + bar: [ + { + collection: otherCollection, + select: { description: true }, + by: { subId: { field: 'bar' } }, + }, + ], + }, + }), + ).toEqual({ + selectColumns: [ + '`foo$0`.`title` AS `foo$0.title`', + '`bar$0`.`description` AS `bar$0.description`', + ], + fromAlias: '$', + fromJoins: [ + 'INNER JOIN `otherCollection` `foo$0` ON (`foo$0`.`id` = `$`.`foo`)', + 'LEFT JOIN `otherCollection` `bar$0` ON (`bar$0`.`subId` = `$`.`bar`)', + ], + whereClauses: [], + }); + }); + + it('should handle multiple inner join choices on same collection', () => { + expect( + getFromJoins(helpers, { + collection, + innerJoin: { + foo: [ + { + collection: otherCollection, + select: { title: true }, + by: { id: { field: 'foo' } }, + }, + { + collection: otherCollection, + select: { title: true }, + by: { subId: { field: 'bar' } }, + }, + ], + }, + }), + ).toEqual({ + selectColumns: [ + '`foo$0`.`title` AS `foo$0.title`', + '`foo$0`.`title` AS `foo$1.title`', + ], + fromAlias: '$', + fromJoins: [ + 'INNER JOIN `otherCollection` `foo$0` ON (`foo$0`.`id` = `$`.`foo`) OR (`foo$0`.`subId` = `$`.`bar`)', + ], + whereClauses: [], + }); + }); + + it('should handle multiple inner join choices on different collections', () => { + expect( + getFromJoins(helpers, { + collection, + innerJoin: { + foo: [ + { + collection: otherCollection, + select: { title: true }, + by: { subId: { field: 'foo' } }, + }, + { + collection: otherCollection2, + select: { url: true }, + by: { id: { field: 'bar' } }, + }, + ], + }, + }), + ).toEqual({ + selectColumns: [ + '`foo$0`.`title` AS `foo$0.title`', + '`foo$1`.`url` AS `foo$1.url`', + ], + fromAlias: '$', + fromJoins: [ + 'LEFT JOIN `otherCollection` `foo$0` ON (`foo$0`.`subId` = `$`.`foo`)', + 'LEFT JOIN `otherCollection2` `foo$1` ON (`foo$1`.`id` = `$`.`bar`)', + ], + whereClauses: [ + '(`foo$0`.`id` IS NOT NULL) OR (`foo$1`.`id` IS NOT NULL)', + ], + }); + }); + }); +}); diff --git a/packages/sql-store/src/helpers/join/from.ts b/packages/sql-store/src/helpers/join/from.ts new file mode 100644 index 0000000..4f8a868 --- /dev/null +++ b/packages/sql-store/src/helpers/join/from.ts @@ -0,0 +1,224 @@ +import { + StoreCollection, + StoreError, + StoreJoin, + StoreJoinChoice, +} from '@neuledge/store'; +import { QueryHelpers } from '../query'; +import { getSelectColumn } from '../select'; + +export const getFromJoins = ( + helpers: QueryHelpers, + options: Pick, +): { + selectColumns: string[]; + fromAlias: string; + fromJoins: string[]; + whereClauses: string[]; +} | null => { + const fromAlias = '$'; + + const joins = handleStoreOptions(fromAlias, options); + if (!joins.length) return null; + + const selectColumns: string[] = []; + const fromJoins: string[] = []; + let whereClauses: string[] = []; + + for (const join of joins) { + const { select, fromJoin, where } = getFromJoin(helpers, join); + + selectColumns.push(...select); + fromJoins.push(fromJoin); + whereClauses.push(...where); + } + + // remove where duplicates + whereClauses = [...new Set(whereClauses)]; + + return { selectColumns, fromAlias, fromJoins, whereClauses }; +}; + +// local helpers + +const handleStoreOptions = ( + fromAlias: string, + { + innerJoin, + leftJoin, + }: Pick, +): Join[] => { + const joins: Join[] = []; + + if (innerJoin) { + joins.push(...handleStoreJoin(fromAlias, innerJoin, true)); + } + + if (leftJoin) { + joins.push(...handleStoreJoin(fromAlias, leftJoin)); + } + + return joins; +}; + +const handleStoreJoin = ( + fromAlias: string, + join: StoreJoin, + required?: boolean, +): Join[] => + Object.entries(join).flatMap(([key, choices]) => + handleStoreJoinChoices(fromAlias, key, choices, required), + ); + +const handleStoreJoinChoices = ( + fromAlias: string, + key: string, + choices: StoreJoinChoice[], + required?: boolean, +): Join[] => { + const joinsFrom: Record = {}; + const childJoins: Join[] = []; + + for (const [i, choice] of choices.entries()) { + const { collection, by, select, where } = choice; + + // join with where are tricky. if it's done with inner join, it will + // remove the parent document if the join is not found. If it's done + // with left join, it will keep the parent document even if the join + // is not found. This is problematic when we have multiple join choices + // for the same collection but with different where clauses or from + // different join choices set. + + let join = joinsFrom[collection.name]; + if (!join) { + join = { + collection, + alias: `${key}$${i}`, + select: {}, + ons: [], + }; + joinsFrom[collection.name] = join; + } + + join.ons.push({ fromAlias: fromAlias, by }); + + if (select) { + for (const name of Object.keys( + select === true ? collection.fields : select, + )) { + join.select[`${key}$${i}.${name}`] = name; + } + } + + childJoins.push(...handleStoreOptions(join.alias, choice)); + } + + const joins = Object.values(joinsFrom); + if (required) { + for (const join of joins) { + join.required = joins; + } + } + + return [...joins, ...childJoins]; +}; + +interface Join { + collection: StoreCollection; + alias: string; + select: Record; + ons: { fromAlias: string; by: StoreJoinChoice['by'] }[]; + required?: Join[]; +} + +const getFromJoin = ( + helpers: QueryHelpers, + join: Join, +): { select: string[]; fromJoin: string; where: string[] } => { + const { collection, alias: joinAlias, select, ons, required } = join; + + const selectColumns = Object.entries(select).map(([alias, name]) => + getSelectColumn(helpers, joinAlias, name, alias), + ); + + let joinType: 'INNER' | 'LEFT'; + const where: string[] = []; + + if (required?.length === 1) { + joinType = 'INNER'; + } else { + joinType = 'LEFT'; + + if (required) { + where.push(getJoinRequiredWhere(helpers, required)); + } + } + + const fromJoin = `${joinType} JOIN ${helpers.encodeIdentifier( + collection.name, + )} ${helpers.encodeIdentifier(joinAlias)} ON (${ons + .map(({ fromAlias, by }) => getJoinOn(helpers, fromAlias, joinAlias, by)) + .join(') OR (')})`; + + return { select: selectColumns, fromJoin, where }; +}; + +const getJoinOn = ( + helpers: QueryHelpers, + fromAlias: string, + joinAlias: string, + by: StoreJoinChoice['by'], +): string => + Object.entries(by) + .map(([key, term]) => { + const field = `${helpers.encodeIdentifier( + joinAlias, + )}.${helpers.encodeIdentifier(key)}`; + + if (term.field) { + return `${field} = ${helpers.encodeIdentifier( + fromAlias, + )}.${helpers.encodeIdentifier(term.field)}`; + } + + if (term.value != null) { + return `${field} = ${helpers.encodeLiteral(term.value)}`; + } + + return `${field} IS NULL`; + }) + .join(' AND '); + +/** + * Check that at least one of the required joins is not null. + */ +const getJoinRequiredWhere = ( + helpers: QueryHelpers, + required: Join[], +): string => + `(${required + .map((join) => { + const { collection, alias } = join; + + let field = Object.keys(collection.primaryKey.fields).find( + (name) => !collection.fields[name].nullable, + ); + + if (!field) { + field = Object.keys(collection.fields).find( + (name) => !collection.fields[name].nullable, + ); + + if (!field) { + throw new StoreError( + StoreError.Code.INVALID_DATA, + `Cannot find a non-nullable field in collection ${collection.name}`, + ); + } + } + + return `${helpers.encodeIdentifier(alias)}.${helpers.encodeIdentifier( + field, + )} IS NOT NULL`; + }) + .join(') OR (')})`; diff --git a/packages/sql-store/src/helpers/join/index.ts b/packages/sql-store/src/helpers/join/index.ts new file mode 100644 index 0000000..fb57193 --- /dev/null +++ b/packages/sql-store/src/helpers/join/index.ts @@ -0,0 +1,2 @@ +export * from './documents'; +export * from './from'; diff --git a/packages/sql-store/src/helpers/order.ts b/packages/sql-store/src/helpers/order.ts new file mode 100644 index 0000000..ef523fd --- /dev/null +++ b/packages/sql-store/src/helpers/order.ts @@ -0,0 +1,15 @@ +import { StoreSort } from '@neuledge/store'; +import { QueryHelpers } from './query'; + +export const getOrderBy = ( + helpers: QueryHelpers, + sort: StoreSort, +): string | null => + Object.entries(sort) + .map( + ([fieldName, direction]) => + `${helpers.encodeIdentifier(fieldName)} ${ + direction === 'asc' ? 'ASC' : 'DESC' + }`, + ) + .join(', ') || null; diff --git a/packages/sql-store/src/helpers/select.ts b/packages/sql-store/src/helpers/select.ts new file mode 100644 index 0000000..6b3a010 --- /dev/null +++ b/packages/sql-store/src/helpers/select.ts @@ -0,0 +1,40 @@ +import { StoreCollection, StoreDocument, StoreSelect } from '@neuledge/store'; +import { QueryHelpers } from './query'; + +export const getSelectColumns = ( + helpers: QueryHelpers, + from: string | null, + select: StoreSelect | StoreCollection['fields'], +): string[] => + Object.keys(select) + .filter((key) => select[key]) + .map((name) => getSelectColumn(helpers, from, name)); + +export const getSelectColumn = ( + helpers: QueryHelpers, + from: string | null, + name: string, + alias?: string | null, +): string => + `${ + from ? `${helpers.encodeIdentifier(from)}.` : '' + }${helpers.encodeIdentifier(name)}${ + alias ? ` AS ${helpers.encodeIdentifier(alias)}` : '' + }`; + +export const getSelectAny = ( + helpers: QueryHelpers, + from: string | null, +): string => `${from ? `${helpers.encodeIdentifier(from)}.` : ''}*`; + +export const getSelectedDocument = ( + collection: StoreCollection, + select: StoreSelect | null | undefined, + rawDoc: StoreDocument, +): StoreDocument => { + const keyFilter = select || collection.fields; + + return Object.fromEntries( + Object.entries(rawDoc).filter(([key]) => keyFilter[key]), + ); +}; diff --git a/packages/sql-store/src/helpers/where-clause.ts b/packages/sql-store/src/helpers/where.ts similarity index 99% rename from packages/sql-store/src/helpers/where-clause.ts rename to packages/sql-store/src/helpers/where.ts index 6a5c105..390a17d 100644 --- a/packages/sql-store/src/helpers/where-clause.ts +++ b/packages/sql-store/src/helpers/where.ts @@ -1,7 +1,7 @@ import { StoreWhere, StoreWhereRecord, StoreWhereTerm } from '@neuledge/store'; import { QueryHelpers } from './query'; -export const whereClause = ( +export const getWhere = ( helpers: QueryHelpers, where: StoreWhere, ): string | null => { diff --git a/packages/sql-store/src/logic/delete.ts b/packages/sql-store/src/logic/delete.ts index 465c7cf..30043d5 100644 --- a/packages/sql-store/src/logic/delete.ts +++ b/packages/sql-store/src/logic/delete.ts @@ -1,4 +1,4 @@ -import { QueryHelpers, whereClause } from '@/helpers'; +import { QueryHelpers, getWhere } from '@/helpers'; import { StoreDeleteOptions, StoreMutationResponse } from '@neuledge/store'; export interface DeleteQueries { @@ -21,7 +21,7 @@ export const deletes = async ( const affectedCount = await deleteFrom( connection, name, - where ? whereClause(queryHelpers, where) : null, + where ? getWhere(queryHelpers, where) : null, ); return { diff --git a/packages/sql-store/src/logic/find.ts b/packages/sql-store/src/logic/find.ts index 7326968..b8dfc6e 100644 --- a/packages/sql-store/src/logic/find.ts +++ b/packages/sql-store/src/logic/find.ts @@ -1,12 +1,22 @@ -import { QueryHelpers, whereClause } from '@/helpers'; +import { + QueryHelpers, + fillJoinedDocuments, + getFromJoins, + getOrderBy, + getSelectAny, + getSelectColumns, + getSelectedDocument, + getWhere, +} from '@/helpers'; import { StoreDocument, StoreFindOptions, StoreList } from '@neuledge/store'; export interface FindQueries { selectFrom( connection: Connection, - name: string, - select: string[] | true, + select: string, + from: string, where: string | null, + orderBy: string | null, limit: number, offset: number, ): Promise; @@ -18,38 +28,53 @@ export const find = async ( connection: Connection, { selectFrom, queryHelpers }: FindQueries, ): Promise => { - const { - collection, - select: selectMap, - where, - innerJoin, - leftJoin, - limit, - offset: storeOffset, - sort, - } = options; - const { name } = collection; - - if (innerJoin || leftJoin || sort) { - // FIXME implement joins and sorting on postgresql - throw new Error('Joins and sorting are not supported yet'); + const { collection, select, where, limit, offset, sort } = options; + + let from = queryHelpers.encodeIdentifier(collection.name); + const join = getFromJoins(queryHelpers, options); + + let selectColumns; + const whereClauses = where ? [getWhere(queryHelpers, where)] : []; + + if (join) { + from += `${queryHelpers.encodeIdentifier( + join.fromAlias, + )} ${join.fromJoins.join(' ')}`; + + selectColumns = select + ? getSelectColumns(queryHelpers, join.fromAlias, select) + : [getSelectAny(queryHelpers, join.fromAlias)]; + + selectColumns.push(...join.selectColumns); + whereClauses.push(...join.whereClauses); + } else { + selectColumns = select + ? getSelectColumns(queryHelpers, null, select) + : ['*']; } - const select = selectMap - ? Object.keys(selectMap).filter((key) => selectMap[key]) - : true; - const offset = storeOffset ? Number(storeOffset) : 0; + const offsetNumber = offset ? Number(offset) : 0; - const rows = await selectFrom( + const rawDocs = await selectFrom( connection, - name, - select, - where ? whereClause(queryHelpers, where) : null, + selectColumns.join(', '), + from, + whereClauses.join(' AND ') || null, + sort ? getOrderBy(queryHelpers, sort) : null, limit, - offset, + offsetNumber, ); + const nextOffset = rawDocs.length < limit ? null : offsetNumber + limit; - const nextOffset = rows.length < limit ? null : offset + limit; + const docs = rawDocs + .map((rawDoc) => + fillJoinedDocuments( + options, + rawDoc, + getSelectedDocument(collection, select, rawDoc), + ), + ) + .filter((doc): doc is StoreDocument => doc != null); - return Object.assign(rows, { nextOffset }); + return Object.assign(docs, { nextOffset }); }; diff --git a/packages/sql-store/src/logic/update.ts b/packages/sql-store/src/logic/update.ts index e0776e3..38ff73f 100644 --- a/packages/sql-store/src/logic/update.ts +++ b/packages/sql-store/src/logic/update.ts @@ -1,4 +1,4 @@ -import { QueryHelpers, whereClause } from '@/helpers'; +import { QueryHelpers, getWhere } from '@/helpers'; import { StoreDocument, StoreMutationResponse, @@ -27,7 +27,7 @@ export const update = async ( connection, name, set, - where ? whereClause(queryHelpers, where) : null, + where ? getWhere(queryHelpers, where) : null, ); return { From 70b678789e406f4e71a02cc92c189f748c7ae5d9 Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Thu, 27 Apr 2023 22:26:58 +0300 Subject: [PATCH 19/24] fix convertRawDocument --- packages/postgresql-store/src/store.test.ts | 14 +-- .../sql-store/src/helpers/documents.test.ts | 97 +++++++++++++++++++ packages/sql-store/src/helpers/documents.ts | 41 ++++++++ packages/sql-store/src/helpers/index.ts | 1 + .../{join/from.test.ts => join.test.ts} | 10 +- .../src/helpers/{join/from.ts => join.ts} | 4 +- .../sql-store/src/helpers/join/documents.ts | 65 ------------- packages/sql-store/src/helpers/join/index.ts | 2 - packages/sql-store/src/helpers/select.ts | 14 +-- packages/sql-store/src/logic/find.ts | 15 +-- 10 files changed, 155 insertions(+), 108 deletions(-) create mode 100644 packages/sql-store/src/helpers/documents.test.ts create mode 100644 packages/sql-store/src/helpers/documents.ts rename packages/sql-store/src/helpers/{join/from.test.ts => join.test.ts} (96%) rename packages/sql-store/src/helpers/{join/from.ts => join.ts} (98%) delete mode 100644 packages/sql-store/src/helpers/join/documents.ts delete mode 100644 packages/sql-store/src/helpers/join/index.ts diff --git a/packages/postgresql-store/src/store.test.ts b/packages/postgresql-store/src/store.test.ts index 5470a9f..96fc3c0 100644 --- a/packages/postgresql-store/src/store.test.ts +++ b/packages/postgresql-store/src/store.test.ts @@ -254,19 +254,11 @@ describe('store', () => { `SELECT id, name FROM ${usersTableName} LIMIT 1 OFFSET 0`, ); - expect(res).toEqual( - Object.assign( - [ - { - id: usersTableRow1.id, - name: usersTableRow1.name, - }, - ], - { nextOffset: 1 }, - ), - ); + expect(res).toEqual(Object.assign([usersTableRow1], { nextOffset: 1 })); }); + // FIXME test joins + it('should be able to sort documents', async () => { query.mockResolvedValueOnce({ rows: [usersTableRow1] }); diff --git a/packages/sql-store/src/helpers/documents.test.ts b/packages/sql-store/src/helpers/documents.test.ts new file mode 100644 index 0000000..f8cbb46 --- /dev/null +++ b/packages/sql-store/src/helpers/documents.test.ts @@ -0,0 +1,97 @@ +import { convertRawDocument } from './documents'; + +/* eslint-disable max-lines-per-function */ + +describe('helpers/documents', () => { + describe('convertRawDocument()', () => { + it('should convert raw document to nested document', () => { + expect( + convertRawDocument({ + id: 123, + url: null, + 'foo$0.bar': 'baz', + 'foo$0.qux': 'quux', + 'foo$0.quux': 'corge', + 'foo$0.baz.qux': 'grault', + 'foo$0.baz.quux.corge.grault': 'fred', + }), + ).toEqual({ + id: 123, + url: null, + foo: { + bar: 'baz', + qux: 'quux', + quux: 'corge', + baz: { + qux: 'grault', + quux: { + corge: { + grault: 'fred', + }, + }, + }, + }, + }); + }); + + it('should prefer lower choices over higher choices (props asc)', () => { + expect( + convertRawDocument({ + id: 123, + url: null, + 'foo$0.bar': 'baz', + 'foo$1.bar': 'qux', + 'foo$1.baz': 123, + }), + ).toEqual({ + id: 123, + url: null, + foo: { + bar: 'baz', + }, + }); + }); + + it('should prefer lower choices over higher choices (props desc)', () => { + expect( + convertRawDocument({ + id: 123, + url: null, + 'foo$1.bar': 'qux', + 'foo$1.baz': 123, + 'foo$0.bar': 'baz', + }), + ).toEqual({ + id: 123, + url: null, + foo: { + bar: 'baz', + }, + }); + }); + + it('should prefer lower choices over higher choices (nested)', () => { + expect( + convertRawDocument({ + id: 123, + url: null, + 'foo$0.id': 123, + 'foo$0.bar$0.id': 1, + 'foo$0.bar$1.id': 2, + 'foo$1.id': 456, + 'foo$1.bar$0.id': 3, + 'foo$1.bar$1.id': 4, + }), + ).toEqual({ + id: 123, + url: null, + foo: { + id: 123, + bar: { + id: 1, + }, + }, + }); + }); + }); +}); diff --git a/packages/sql-store/src/helpers/documents.ts b/packages/sql-store/src/helpers/documents.ts new file mode 100644 index 0000000..4c51718 --- /dev/null +++ b/packages/sql-store/src/helpers/documents.ts @@ -0,0 +1,41 @@ +import { StoreDocument } from '@neuledge/store'; + +export const convertRawDocument = (rawDoc: StoreDocument): StoreDocument => { + const doc: StoreDocument = {}; + + // split by dot notation + for (const [key, value] of Object.entries(rawDoc)) { + const path = key.split('.'); + + let current = doc; + for (let i = 0; i < path.length; i++) { + const name = path[i]; + + if (i === path.length - 1) { + current[name] = value; + } else { + current = current[name] = (current[name] || {}) as StoreDocument; + } + } + } + + preferLowerChoices(doc); + + return doc; +}; + +const preferLowerChoices = (doc: StoreDocument): void => { + for (const [key, value] of Object.entries(doc).sort()) { + const choice = key.match(/^(.+)\$(\d+)$/); + if (!choice) continue; + + delete doc[key]; + + const name = choice[1]; + if (name in doc) continue; + + doc[name] = value; + + preferLowerChoices(value as StoreDocument); + } +}; diff --git a/packages/sql-store/src/helpers/index.ts b/packages/sql-store/src/helpers/index.ts index a9bf3b7..76316bd 100644 --- a/packages/sql-store/src/helpers/index.ts +++ b/packages/sql-store/src/helpers/index.ts @@ -1,3 +1,4 @@ +export * from './documents'; export * from './join'; export * from './order'; export * from './query'; diff --git a/packages/sql-store/src/helpers/join/from.test.ts b/packages/sql-store/src/helpers/join.test.ts similarity index 96% rename from packages/sql-store/src/helpers/join/from.test.ts rename to packages/sql-store/src/helpers/join.test.ts index d5736a6..4e4dec5 100644 --- a/packages/sql-store/src/helpers/join/from.test.ts +++ b/packages/sql-store/src/helpers/join.test.ts @@ -1,10 +1,10 @@ import { StoreCollection } from '@neuledge/store'; -import { QueryHelpers } from '../query'; -import { getFromJoins } from './from'; +import { QueryHelpers } from './query'; +import { getFromJoins } from './join'; /* eslint-disable max-lines-per-function */ -describe('helpers/join/from', () => { +describe('helpers/join', () => { describe('getFromJoins()', () => { const helpers: QueryHelpers = { encodeIdentifier: (name) => `\`${name.replace(/([\\`])/g, '\\$1')}\``, @@ -193,5 +193,9 @@ describe('helpers/join/from', () => { ], }); }); + + // FIXME test where clauses + + // FIXME test recursive joins }); }); diff --git a/packages/sql-store/src/helpers/join/from.ts b/packages/sql-store/src/helpers/join.ts similarity index 98% rename from packages/sql-store/src/helpers/join/from.ts rename to packages/sql-store/src/helpers/join.ts index 4f8a868..18b4ba9 100644 --- a/packages/sql-store/src/helpers/join/from.ts +++ b/packages/sql-store/src/helpers/join.ts @@ -4,8 +4,8 @@ import { StoreJoin, StoreJoinChoice, } from '@neuledge/store'; -import { QueryHelpers } from '../query'; -import { getSelectColumn } from '../select'; +import { QueryHelpers } from './query'; +import { getSelectColumn } from './select'; export const getFromJoins = ( helpers: QueryHelpers, diff --git a/packages/sql-store/src/helpers/join/documents.ts b/packages/sql-store/src/helpers/join/documents.ts deleted file mode 100644 index c16ce71..0000000 --- a/packages/sql-store/src/helpers/join/documents.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { StoreDocument, StoreJoinChoice } from '@neuledge/store'; - -export const fillJoinedDocuments = ( - { innerJoin, leftJoin }: Pick, - rawDoc: StoreDocument, - doc: StoreDocument, -): StoreDocument | null => { - if (innerJoin) { - for (const [key, choices] of Object.entries(innerJoin)) { - const joinedDoc = findJoinedDocument(choices, rawDoc); - if (!joinedDoc) return null; - - if (joinedDoc !== true) { - doc[key] = joinedDoc; - } - } - } - - if (leftJoin) { - for (const [key, choices] of Object.entries(leftJoin)) { - const joinedDoc = findJoinedDocument(choices, rawDoc); - if (!joinedDoc) continue; - - if (joinedDoc !== true) { - doc[key] = joinedDoc; - } - } - } - - return doc; -}; - -// local helpers - -const findJoinedDocument = ( - choices: StoreJoinChoice[], - rawDoc: StoreDocument, -): StoreDocument | null | true => { - for (const choice of choices) { - const { collection, by, select } = choice; - - const found = Object.entries(by).every(([key, term]) => { - const value = rawDoc[`${collection.name}.${key}`]; - if (value === undefined) return false; - - return term.field ? value === rawDoc[term.field] : value === term.value; - }); - if (!found) continue; - - if (!select) { - return true; - } - - const keys = - select === true - ? Object.keys(collection.fields) - : Object.keys(select).filter((key) => select[key]); - - return Object.fromEntries( - keys.map((key) => [key, rawDoc[`${collection.name}.${key}`]]), - ); - } - - return null; -}; diff --git a/packages/sql-store/src/helpers/join/index.ts b/packages/sql-store/src/helpers/join/index.ts deleted file mode 100644 index fb57193..0000000 --- a/packages/sql-store/src/helpers/join/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './documents'; -export * from './from'; diff --git a/packages/sql-store/src/helpers/select.ts b/packages/sql-store/src/helpers/select.ts index 6b3a010..7d1cdda 100644 --- a/packages/sql-store/src/helpers/select.ts +++ b/packages/sql-store/src/helpers/select.ts @@ -1,4 +1,4 @@ -import { StoreCollection, StoreDocument, StoreSelect } from '@neuledge/store'; +import { StoreCollection, StoreSelect } from '@neuledge/store'; import { QueryHelpers } from './query'; export const getSelectColumns = ( @@ -26,15 +26,3 @@ export const getSelectAny = ( helpers: QueryHelpers, from: string | null, ): string => `${from ? `${helpers.encodeIdentifier(from)}.` : ''}*`; - -export const getSelectedDocument = ( - collection: StoreCollection, - select: StoreSelect | null | undefined, - rawDoc: StoreDocument, -): StoreDocument => { - const keyFilter = select || collection.fields; - - return Object.fromEntries( - Object.entries(rawDoc).filter(([key]) => keyFilter[key]), - ); -}; diff --git a/packages/sql-store/src/logic/find.ts b/packages/sql-store/src/logic/find.ts index b8dfc6e..d8f4f77 100644 --- a/packages/sql-store/src/logic/find.ts +++ b/packages/sql-store/src/logic/find.ts @@ -1,11 +1,10 @@ import { QueryHelpers, - fillJoinedDocuments, + convertRawDocument, getFromJoins, getOrderBy, getSelectAny, getSelectColumns, - getSelectedDocument, getWhere, } from '@/helpers'; import { StoreDocument, StoreFindOptions, StoreList } from '@neuledge/store'; @@ -64,17 +63,9 @@ export const find = async ( limit, offsetNumber, ); - const nextOffset = rawDocs.length < limit ? null : offsetNumber + limit; - const docs = rawDocs - .map((rawDoc) => - fillJoinedDocuments( - options, - rawDoc, - getSelectedDocument(collection, select, rawDoc), - ), - ) - .filter((doc): doc is StoreDocument => doc != null); + const docs = rawDocs.map((rawDoc) => convertRawDocument(rawDoc)); + const nextOffset = rawDocs.length < limit ? null : offsetNumber + limit; return Object.assign(docs, { nextOffset }); }; From d34ec691e653f0b9f877573629cedefd99842ef7 Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Fri, 28 Apr 2023 01:16:14 +0300 Subject: [PATCH 20/24] handle joins where --- packages/sql-store/src/helpers/join.test.ts | 59 ++++++++++++++- packages/sql-store/src/helpers/join.ts | 30 ++++---- packages/sql-store/src/helpers/where.ts | 84 ++++++++++----------- 3 files changed, 116 insertions(+), 57 deletions(-) diff --git a/packages/sql-store/src/helpers/join.test.ts b/packages/sql-store/src/helpers/join.test.ts index 4e4dec5..ca9700b 100644 --- a/packages/sql-store/src/helpers/join.test.ts +++ b/packages/sql-store/src/helpers/join.test.ts @@ -194,7 +194,64 @@ describe('helpers/join', () => { }); }); - // FIXME test where clauses + it('should handle inner join with where', () => { + expect( + getFromJoins(helpers, { + collection, + innerJoin: { + foo: [ + { + collection: otherCollection, + select: { title: true }, + by: { id: { field: 'foo' } }, + where: { subId: { $eq: 123 } }, + }, + ], + }, + }), + ).toEqual({ + selectColumns: ['`foo$0`.`title` AS `foo$0.title`'], + fromAlias: '$', + fromJoins: [ + 'INNER JOIN `otherCollection` `foo$0` ON (`foo$0`.`id` = `$`.`foo` AND `foo$0`.`subId` = 123)', + ], + whereClauses: [], + }); + }); + + it('should handle left joins with where', () => { + expect( + getFromJoins(helpers, { + collection, + leftJoin: { + foo: [ + { + collection: otherCollection, + select: { title: true }, + by: { id: { field: 'foo' } }, + where: { subId: { $lt: 123 } }, + }, + { + collection: otherCollection, + select: { description: true }, + by: { subId: { field: 'bar' } }, + where: { title: { $eq: 'hello' } }, + }, + ], + }, + }), + ).toEqual({ + selectColumns: [ + '`foo$0`.`title` AS `foo$0.title`', + '`foo$0`.`description` AS `foo$1.description`', + ], + fromAlias: '$', + fromJoins: [ + 'LEFT JOIN `otherCollection` `foo$0` ON (`foo$0`.`id` = `$`.`foo` AND `foo$0`.`subId` < 123) OR (`foo$0`.`subId` = `$`.`bar` AND `foo$0`.`title` = "hello")', + ], + whereClauses: [], + }); + }); // FIXME test recursive joins }); diff --git a/packages/sql-store/src/helpers/join.ts b/packages/sql-store/src/helpers/join.ts index 18b4ba9..d182d6f 100644 --- a/packages/sql-store/src/helpers/join.ts +++ b/packages/sql-store/src/helpers/join.ts @@ -6,6 +6,7 @@ import { } from '@neuledge/store'; import { QueryHelpers } from './query'; import { getSelectColumn } from './select'; +import { getWhere } from './where'; export const getFromJoins = ( helpers: QueryHelpers, @@ -82,13 +83,6 @@ const handleStoreJoinChoices = ( for (const [i, choice] of choices.entries()) { const { collection, by, select, where } = choice; - // join with where are tricky. if it's done with inner join, it will - // remove the parent document if the join is not found. If it's done - // with left join, it will keep the parent document even if the join - // is not found. This is problematic when we have multiple join choices - // for the same collection but with different where clauses or from - // different join choices set. - let join = joinsFrom[collection.name]; if (!join) { join = { @@ -100,7 +94,7 @@ const handleStoreJoinChoices = ( joinsFrom[collection.name] = join; } - join.ons.push({ fromAlias: fromAlias, by }); + join.ons.push({ fromAlias: fromAlias, by, where }); if (select) { for (const name of Object.keys( @@ -127,7 +121,11 @@ interface Join { collection: StoreCollection; alias: string; select: Record; - ons: { fromAlias: string; by: StoreJoinChoice['by'] }[]; + ons: { + fromAlias: string; + by: StoreJoinChoice['by']; + where: StoreJoinChoice['where']; + }[]; required?: Join[]; } @@ -157,7 +155,9 @@ const getFromJoin = ( const fromJoin = `${joinType} JOIN ${helpers.encodeIdentifier( collection.name, )} ${helpers.encodeIdentifier(joinAlias)} ON (${ons - .map(({ fromAlias, by }) => getJoinOn(helpers, fromAlias, joinAlias, by)) + .map(({ fromAlias, by, where }) => + getJoinOn(helpers, fromAlias, joinAlias, by, where), + ) .join(') OR (')})`; return { select: selectColumns, fromJoin, where }; @@ -168,9 +168,10 @@ const getJoinOn = ( fromAlias: string, joinAlias: string, by: StoreJoinChoice['by'], + where: StoreJoinChoice['where'], ): string => - Object.entries(by) - .map(([key, term]) => { + [ + ...Object.entries(by).map(([key, term]) => { const field = `${helpers.encodeIdentifier( joinAlias, )}.${helpers.encodeIdentifier(key)}`; @@ -186,8 +187,9 @@ const getJoinOn = ( } return `${field} IS NULL`; - }) - .join(' AND '); + }), + ...(where ? [getWhere(helpers, where, joinAlias)] : []), + ].join(' AND '); /** * Check that at least one of the required joins is not null. diff --git a/packages/sql-store/src/helpers/where.ts b/packages/sql-store/src/helpers/where.ts index 390a17d..64ec230 100644 --- a/packages/sql-store/src/helpers/where.ts +++ b/packages/sql-store/src/helpers/where.ts @@ -4,14 +4,17 @@ import { QueryHelpers } from './query'; export const getWhere = ( helpers: QueryHelpers, where: StoreWhere, + from?: string | null, ): string | null => { const { $or } = where; if (!Array.isArray($or)) { - return whereRecord(helpers, where as StoreWhereRecord) || null; + return whereRecord(helpers, where as StoreWhereRecord, from) || null; } - const sql = $or.map((record) => whereRecord(helpers, record)).filter(Boolean); + const sql = $or + .map((record) => whereRecord(helpers, record, from)) + .filter(Boolean); if (sql.length === 0) { return null; @@ -20,86 +23,87 @@ export const getWhere = ( return `(${sql.join(') OR (')})`; }; -const whereRecord = (helpers: QueryHelpers, record: StoreWhereRecord): string => - Object.entries(record) - .map(([columnName, term]) => whereTerm(helpers, columnName, term)) +const whereRecord = ( + helpers: QueryHelpers, + record: StoreWhereRecord, + from?: string | null, +): string => { + const fromEntry = from ? `${helpers.encodeIdentifier(from)}.` : ''; + + return Object.entries(record) + .map(([columnName, term]) => + whereTerm( + helpers, + `${fromEntry}${helpers.encodeIdentifier(columnName)}`, + term, + ), + ) .filter(Boolean) .join(' AND '); +}; const whereTerm = ( helpers: QueryHelpers, - columnName: string, + entry: string, term: StoreWhereTerm, ): string => [ - ...whereComparisonTerm(helpers, columnName, term), - ...whereLikeTerm(helpers, columnName, term), - ...whereInTerm(helpers, columnName, term), + ...whereComparisonTerm(helpers, entry, term), + ...whereLikeTerm(helpers, entry, term), + ...whereInTerm(helpers, entry, term), ].join(' AND '); const whereComparisonTerm = ( - { encodeIdentifier, encodeLiteral }: QueryHelpers, - columnName: string, + { encodeLiteral }: QueryHelpers, + entry: string, term: StoreWhereTerm, ): string[] => { const sql: string[] = []; if ('$eq' in term) { - sql.push(`${encodeIdentifier(columnName)} = ${encodeLiteral(term.$eq)}`); + sql.push(`${entry} = ${encodeLiteral(term.$eq)}`); } if ('$ne' in term) { - sql.push(`${encodeIdentifier(columnName)} != ${encodeLiteral(term.$ne)}`); + sql.push(`${entry} != ${encodeLiteral(term.$ne)}`); } if ('$gt' in term) { - sql.push(`${encodeIdentifier(columnName)} > ${encodeLiteral(term.$gt)}`); + sql.push(`${entry} > ${encodeLiteral(term.$gt)}`); } if ('$gte' in term) { - sql.push(`${encodeIdentifier(columnName)} >= ${encodeLiteral(term.$gte)}`); + sql.push(`${entry} >= ${encodeLiteral(term.$gte)}`); } if ('$lt' in term) { - sql.push(`${encodeIdentifier(columnName)} < ${encodeLiteral(term.$lt)}`); + sql.push(`${entry} < ${encodeLiteral(term.$lt)}`); } if ('$lte' in term) { - sql.push(`${encodeIdentifier(columnName)} <= ${encodeLiteral(term.$lte)}`); + sql.push(`${entry} <= ${encodeLiteral(term.$lte)}`); } return sql; }; const whereLikeTerm = ( - { encodeIdentifier, encodeLiteral }: QueryHelpers, - columnName: string, + { encodeLiteral }: QueryHelpers, + entry: string, term: StoreWhereTerm, ): string[] => { const sql: string[] = []; if ('$contains' in term) { - sql.push( - `${encodeIdentifier(columnName)} LIKE ${encodeLiteral( - `%${term.$contains}%`, - )}`, - ); + sql.push(`${entry} LIKE ${encodeLiteral(`%${term.$contains}%`)}`); } if ('$startsWith' in term) { - sql.push( - `${encodeIdentifier(columnName)} LIKE ${encodeLiteral( - `${term.$startsWith}%`, - )}`, - ); + sql.push(`${entry} LIKE ${encodeLiteral(`${term.$startsWith}%`)}`); } if ('$endsWith' in term) { - sql.push( - `${encodeIdentifier(columnName)} LIKE ${encodeLiteral( - `%${term.$endsWith}`, - )}`, - ); + sql.push(`${entry} LIKE ${encodeLiteral(`%${term.$endsWith}`)}`); } if ('$in' in term) { @@ -107,9 +111,7 @@ const whereLikeTerm = ( sql.push('FALSE'); } else { sql.push( - `${encodeIdentifier(columnName)} IN (${term.$in - .map((v) => encodeLiteral(v)) - .join(', ')})`, + `${entry} IN (${term.$in.map((v) => encodeLiteral(v)).join(', ')})`, ); } } @@ -118,8 +120,8 @@ const whereLikeTerm = ( }; const whereInTerm = ( - { encodeIdentifier, encodeLiteral }: QueryHelpers, - columnName: string, + { encodeLiteral }: QueryHelpers, + entry: string, term: StoreWhereTerm, ): string[] => { const sql: string[] = []; @@ -129,9 +131,7 @@ const whereInTerm = ( sql.push('FALSE'); } else { sql.push( - `${encodeIdentifier(columnName)} IN (${term.$in - .map((v) => encodeLiteral(v)) - .join(', ')})`, + `${entry} IN (${term.$in.map((v) => encodeLiteral(v)).join(', ')})`, ); } } From 6aabafb4f81a7f8e9be80225607b5349b2f4efbc Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Sat, 29 Apr 2023 11:44:24 +0300 Subject: [PATCH 21/24] handle recursive sql joins --- packages/sql-store/src/helpers/join.test.ts | 74 ++++++++++++++++++++- packages/sql-store/src/helpers/join.ts | 17 +++-- 2 files changed, 85 insertions(+), 6 deletions(-) diff --git a/packages/sql-store/src/helpers/join.test.ts b/packages/sql-store/src/helpers/join.test.ts index ca9700b..dbf5800 100644 --- a/packages/sql-store/src/helpers/join.test.ts +++ b/packages/sql-store/src/helpers/join.test.ts @@ -253,6 +253,78 @@ describe('helpers/join', () => { }); }); - // FIXME test recursive joins + it('should handle inner join within left join', () => { + expect( + getFromJoins(helpers, { + collection, + leftJoin: { + foo: [ + { + collection: otherCollection, + select: { title: true }, + by: { id: { field: 'foo' } }, + innerJoin: { + bar: [ + { + collection: otherCollection2, + select: { url: true }, + by: { id: { field: 'bar' } }, + }, + ], + }, + }, + ], + }, + }), + ).toEqual({ + selectColumns: [ + '`foo$0`.`title` AS `foo$0.title`', + '`foo$0.bar$0`.`url` AS `foo$0.bar$0.url`', + ], + fromAlias: '$', + fromJoins: [ + 'LEFT JOIN `otherCollection` `foo$0` ON (`foo$0`.`id` = `$`.`foo`)', + 'INNER JOIN `otherCollection2` `foo$0.bar$0` ON (`foo$0.bar$0`.`id` = `foo$0`.`bar`)', + ], + whereClauses: [], + }); + }); + + it('should handle overlapping inner join within left join', () => { + expect( + getFromJoins(helpers, { + collection, + leftJoin: { + test: [ + { + collection: otherCollection, + select: { title: true }, + by: { id: { field: 'foo' } }, + innerJoin: { + test: [ + { + collection: otherCollection2, + select: { url: true }, + by: { id: { field: 'bar' } }, + }, + ], + }, + }, + ], + }, + }), + ).toEqual({ + selectColumns: [ + '`test$0`.`title` AS `test$0.title`', + '`test$0.test$0`.`url` AS `test$0.test$0.url`', + ], + fromAlias: '$', + fromJoins: [ + 'LEFT JOIN `otherCollection` `test$0` ON (`test$0`.`id` = `$`.`foo`)', + 'INNER JOIN `otherCollection2` `test$0.test$0` ON (`test$0.test$0`.`id` = `test$0`.`bar`)', + ], + whereClauses: [], + }); + }); }); }); diff --git a/packages/sql-store/src/helpers/join.ts b/packages/sql-store/src/helpers/join.ts index d182d6f..60e08b7 100644 --- a/packages/sql-store/src/helpers/join.ts +++ b/packages/sql-store/src/helpers/join.ts @@ -19,7 +19,7 @@ export const getFromJoins = ( } | null => { const fromAlias = '$'; - const joins = handleStoreOptions(fromAlias, options); + const joins = handleStoreOptions(fromAlias, '', options); if (!joins.length) return null; const selectColumns: string[] = []; @@ -44,6 +44,7 @@ export const getFromJoins = ( const handleStoreOptions = ( fromAlias: string, + path: string, { innerJoin, leftJoin, @@ -52,11 +53,11 @@ const handleStoreOptions = ( const joins: Join[] = []; if (innerJoin) { - joins.push(...handleStoreJoin(fromAlias, innerJoin, true)); + joins.push(...handleStoreJoin(fromAlias, path, innerJoin, true)); } if (leftJoin) { - joins.push(...handleStoreJoin(fromAlias, leftJoin)); + joins.push(...handleStoreJoin(fromAlias, path, leftJoin)); } return joins; @@ -64,11 +65,17 @@ const handleStoreOptions = ( const handleStoreJoin = ( fromAlias: string, + path: string, join: StoreJoin, required?: boolean, ): Join[] => Object.entries(join).flatMap(([key, choices]) => - handleStoreJoinChoices(fromAlias, key, choices, required), + handleStoreJoinChoices( + fromAlias, + path ? `${path}.${key}` : key, + choices, + required, + ), ); const handleStoreJoinChoices = ( @@ -104,7 +111,7 @@ const handleStoreJoinChoices = ( } } - childJoins.push(...handleStoreOptions(join.alias, choice)); + childJoins.push(...handleStoreOptions(join.alias, join.alias, choice)); } const joins = Object.values(joinsFrom); From 9c0f6ac5fa9fc40b5fa535fa8eb31a69e76a897c Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Sat, 29 Apr 2023 12:01:01 +0300 Subject: [PATCH 22/24] test joins on pg --- .../src/queries/__fixtures__/posts-table.ts | 85 +++++++++++++++++++ packages/postgresql-store/src/store.test.ts | 37 +++++++- packages/sql-store/src/logic/find.ts | 2 +- 3 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 packages/postgresql-store/src/queries/__fixtures__/posts-table.ts diff --git a/packages/postgresql-store/src/queries/__fixtures__/posts-table.ts b/packages/postgresql-store/src/queries/__fixtures__/posts-table.ts new file mode 100644 index 0000000..921e258 --- /dev/null +++ b/packages/postgresql-store/src/queries/__fixtures__/posts-table.ts @@ -0,0 +1,85 @@ +import { StoreCollection, StoreDocument } from '@neuledge/store'; + +export const postsTableName = 'posts'; + +export const postsCollection: StoreCollection = { + name: postsTableName, + fields: { + id: { + name: 'id', + type: 'number', + nullable: false, + size: null, + precision: 32, + scale: 0, + }, + author_id: { + name: 'author_id', + type: 'number', + nullable: false, + size: null, + precision: 32, + scale: 0, + }, + title: { + name: 'title', + type: 'string', + nullable: false, + size: 100, + precision: null, + scale: null, + }, + body: { + name: 'body', + type: 'string', + nullable: false, + size: 1000, + precision: null, + scale: null, + }, + created_at: { + name: 'created_at', + type: 'date-time', + nullable: false, + size: null, + precision: null, + scale: null, + }, + updated_at: { + name: 'updated_at', + type: 'date-time', + nullable: false, + size: null, + precision: null, + scale: null, + }, + }, + primaryKey: { + name: 'id', + fields: { id: { sort: 'asc' } }, + unique: 'primary', + auto: 'increment', + }, + indexes: { + id: { + name: 'id', + fields: { id: { sort: 'asc' } }, + unique: 'primary', + auto: 'increment', + }, + posts_author_id_index: { + name: 'posts_author_id_index', + fields: { author_id: { sort: 'asc' } }, + unique: false, + }, + }, +}; + +export const postsTableRow1: StoreDocument = { + id: 1, + author_id: 1, + title: 'Post 1', + body: 'Post 1 body', + created_at: new Date('2020-01-01T00:00:00.000Z'), + updated_at: new Date('2020-01-01T00:00:00.000Z'), +}; diff --git a/packages/postgresql-store/src/store.test.ts b/packages/postgresql-store/src/store.test.ts index 96fc3c0..72fc2da 100644 --- a/packages/postgresql-store/src/store.test.ts +++ b/packages/postgresql-store/src/store.test.ts @@ -3,6 +3,11 @@ import { listTableColumns_sql, listTables_sql, } from './queries'; +import { + postsCollection, + postsTableName, + postsTableRow1, +} from './queries/__fixtures__/posts-table'; import { usersCollection, usersCollection_slim, @@ -257,7 +262,37 @@ describe('store', () => { expect(res).toEqual(Object.assign([usersTableRow1], { nextOffset: 1 })); }); - // FIXME test joins + it('should be able to join tables', async () => { + query.mockResolvedValueOnce({ + rows: [{ ...postsTableRow1, author$0: usersTableRow1 }], + }); + + const res = await store.find({ + collection: postsCollection, + innerJoin: { + author: [ + { + collection: usersCollection, + select: true, + by: { author_id: { field: 'id' } }, + }, + ], + }, + limit: 1, + }); + + expect(query).toHaveBeenCalledTimes(1); + + expect(query).toHaveBeenCalledWith( + `SELECT "$".*, author$0.id AS "author$0.id", author$0.name AS "author$0.name", author$0.email AS "author$0.email", author$0.phone AS "author$0.phone", author$0.created_at AS "author$0.created_at", author$0.updated_at AS "author$0.updated_at" FROM ${postsTableName} "$" INNER JOIN ${usersTableName} author$0 ON (author$0.author_id = "$".id) LIMIT 1 OFFSET 0`, + ); + + expect(res).toEqual( + Object.assign([{ ...postsTableRow1, author: usersTableRow1 }], { + nextOffset: 1, + }), + ); + }); it('should be able to sort documents', async () => { query.mockResolvedValueOnce({ rows: [usersTableRow1] }); diff --git a/packages/sql-store/src/logic/find.ts b/packages/sql-store/src/logic/find.ts index d8f4f77..2166502 100644 --- a/packages/sql-store/src/logic/find.ts +++ b/packages/sql-store/src/logic/find.ts @@ -36,7 +36,7 @@ export const find = async ( const whereClauses = where ? [getWhere(queryHelpers, where)] : []; if (join) { - from += `${queryHelpers.encodeIdentifier( + from += ` ${queryHelpers.encodeIdentifier( join.fromAlias, )} ${join.fromJoins.join(' ')}`; From 70c71a15e81a60c653d96b150506d0264e54784c Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Sun, 30 Apr 2023 14:02:34 +0300 Subject: [PATCH 23/24] many fixes to sql describe --- packages/engine/src/engine/metadata/load.ts | 2 +- packages/engine/src/engine/metadata/state.ts | 38 +++++++++++- packages/engine/src/engine/metadata/store.ts | 39 +----------- packages/mongodb-store/src/indexes.ts | 26 +++++--- packages/mongodb-store/src/inserted-ids.ts | 26 ++++---- packages/mongodb-store/src/store.ts | 51 +++++++++------ packages/mysql-store/src/store.ts | 13 +++- packages/postgresql-store/README.md | 8 ++- .../src/queries/__fixtures__/posts-table.ts | 6 ++ .../src/queries/__fixtures__/users-table.ts | 12 ++++ .../src/queries/add-column.ts | 4 +- .../src/queries/list-table-columns.ts | 7 ++- packages/postgresql-store/src/store.test.ts | 2 +- packages/postgresql-store/src/store.ts | 11 +++- .../src/logic/collections/describe.ts | 3 +- .../sql-store/src/logic/collections/drop.ts | 6 +- .../sql-store/src/logic/collections/ensure.ts | 62 ++++++++++++------- .../sql-store/src/logic/collections/list.ts | 4 +- packages/sql-store/src/logic/delete.ts | 8 ++- packages/sql-store/src/logic/find.ts | 9 ++- packages/sql-store/src/logic/insert.ts | 9 ++- packages/sql-store/src/logic/update.ts | 3 +- packages/sql-store/src/mappers/field.ts | 2 + packages/store/src/error.ts | 26 +++++++- 24 files changed, 254 insertions(+), 123 deletions(-) diff --git a/packages/engine/src/engine/metadata/load.ts b/packages/engine/src/engine/metadata/load.ts index 274c7f4..d440653 100644 --- a/packages/engine/src/engine/metadata/load.ts +++ b/packages/engine/src/engine/metadata/load.ts @@ -4,10 +4,10 @@ import { Store } from '@neuledge/store'; import { ensureStoreCollections } from './collections'; import { ensureMetadataCollection, - getMetadataCollection, getStoreMetadataSnapshot, syncStoreMetadata, } from './store'; +import { getMetadataCollection } from './state'; const DEFAULT_METADATA_COLLECTION_NAME = '__neuledge_metadata'; diff --git a/packages/engine/src/engine/metadata/state.ts b/packages/engine/src/engine/metadata/state.ts index 7458771..8374479 100644 --- a/packages/engine/src/engine/metadata/state.ts +++ b/packages/engine/src/engine/metadata/state.ts @@ -1,12 +1,14 @@ +import { StoreCollection, StoreField, StorePrimaryKey } from '@neuledge/store'; import { NeuledgeError } from '@/error'; import { StateSnapshot, StateFieldSnapshot, StateRelationSnapshot, + METADATA_HASH_BYTES, } from '@/metadata'; export interface StoreMetadataState { - collectionName: string; + collection_name: string; name: string; hash: Buffer; fields: StoreMetadataStateField[]; @@ -32,6 +34,36 @@ interface StoreMetadataStateRelation { index: number; } +export const getMetadataCollection = ( + metadataCollectionName: string, +): StoreCollection => { + const hash: StoreField = { + name: 'hash', + type: 'binary', + size: METADATA_HASH_BYTES, + }; + + const primaryKey: StorePrimaryKey = { + name: 'hash', + fields: { [hash.name]: { sort: 'asc' } }, + unique: 'primary', + }; + + return { + name: metadataCollectionName, + primaryKey, + indexes: { [primaryKey.name]: primaryKey }, + fields: { + [hash.name]: hash, + collection_name: { name: 'collection_name', type: 'string' }, + name: { name: 'name', type: 'string' }, + fields: { name: 'fields', type: 'json', list: true }, + relations: { name: 'relations', type: 'json', list: true }, + v: { name: 'v', type: 'number', unsigned: true, scale: 0, precision: 4 }, + }, + }; +}; + export const fromStoreMetadataState = ( getState: (hash: Buffer) => StateSnapshot, getType: (key: string) => StateFieldSnapshot['type'], @@ -45,7 +77,7 @@ export const fromStoreMetadataState = ( } return getState(doc.hash).assign({ - collectionName: doc.collectionName, + collectionName: doc.collection_name, name: doc.name, hash: doc.hash, fields: doc.fields.map((field) => @@ -60,7 +92,7 @@ export const fromStoreMetadataState = ( export const toStoreMetadataState = ( state: StateSnapshot, ): StoreMetadataState => ({ - collectionName: state.collectionName, + collection_name: state.collectionName, name: state.name, hash: state.hash, fields: state.fields.map((field) => toStoreMetadataStateField(field)), diff --git a/packages/engine/src/engine/metadata/store.ts b/packages/engine/src/engine/metadata/store.ts index a32a7c5..23a9a1b 100644 --- a/packages/engine/src/engine/metadata/store.ts +++ b/packages/engine/src/engine/metadata/store.ts @@ -1,13 +1,7 @@ import { NeuledgeError } from '@/error'; -import { MetadataChange, StateSnapshot, METADATA_HASH_BYTES } from '@/metadata'; +import { MetadataChange, StateSnapshot } from '@/metadata'; import { MetadataSnapshot } from '@/metadata/snapshot'; -import { - Store, - StoreCollection, - StoreField, - StoreList, - StorePrimaryKey, -} from '@neuledge/store'; +import { Store, StoreCollection, StoreList } from '@neuledge/store'; import pLimit from 'p-limit'; import { fromStoreMetadataState, @@ -16,34 +10,7 @@ import { } from './state'; const HASH_ENCODING = 'base64url'; -const COLLECTION_FIND_LIMIT = 1000; - -export const getMetadataCollection = ( - metadataCollectionName: string, -): StoreCollection => { - const hash: StoreField = { - name: 'hash', - type: 'binary', - size: METADATA_HASH_BYTES, - }; - - const primaryKey: StorePrimaryKey = { - name: 'hash', - fields: { [hash.name]: { sort: 'asc' } }, - unique: 'primary', - }; - - return { - name: metadataCollectionName, - primaryKey, - indexes: { [primaryKey.name]: primaryKey }, - fields: { - [hash.name]: hash, - key: { name: 'key', type: 'string' }, - payload: { name: 'payload', type: 'json' }, - }, - }; -}; +const COLLECTION_FIND_LIMIT = 100; export const ensureMetadataCollection = async ( store: Store, diff --git a/packages/mongodb-store/src/indexes.ts b/packages/mongodb-store/src/indexes.ts index 4c3d31d..ae48a60 100644 --- a/packages/mongodb-store/src/indexes.ts +++ b/packages/mongodb-store/src/indexes.ts @@ -1,4 +1,4 @@ -import { StoreIndex, StorePrimaryKey } from '@neuledge/store'; +import { StoreIndex, StorePrimaryKey, throwStoreError } from '@neuledge/store'; import { Collection } from 'mongodb'; import { escapeFieldName } from './fields'; @@ -6,7 +6,9 @@ export const dropIndexes = async ( collection: Collection, indexes: string[], ): Promise => { - await Promise.all(indexes.map((index) => collection.dropIndex(index))); + await Promise.all(indexes.map((index) => collection.dropIndex(index))).catch( + throwStoreError, + ); }; export const ensureIndexes = async ( @@ -14,7 +16,11 @@ export const ensureIndexes = async ( collection: Collection, indexes: StoreIndex[], ): Promise => { - const exists = await collection.listIndexes().toArray(); + const exists = await collection + .listIndexes() + .toArray() + .catch(throwStoreError); + const existMap = new Map(exists.map((item) => [item.name, item])); for (const index of indexes) { @@ -39,11 +45,13 @@ export const ensureIndexes = async ( // documents that don't have the indexed fields. This maintains the same // behavior with relational databases where NULL values are not indexed. - await collection.createIndex(indexSpec, { - name: index.name, - unique: !!index.unique, - sparse: true, - background: true, - }); + await collection + .createIndex(indexSpec, { + name: index.name, + unique: !!index.unique, + sparse: true, + background: true, + }) + .catch(throwStoreError); } }; diff --git a/packages/mongodb-store/src/inserted-ids.ts b/packages/mongodb-store/src/inserted-ids.ts index 7d4ae86..70d2f66 100644 --- a/packages/mongodb-store/src/inserted-ids.ts +++ b/packages/mongodb-store/src/inserted-ids.ts @@ -1,4 +1,4 @@ -import { StoreCollection, StoreError } from '@neuledge/store'; +import { StoreCollection, StoreError, throwStoreError } from '@neuledge/store'; import { Collection } from 'mongodb'; export interface AutoIncrementDocument { @@ -56,17 +56,19 @@ const autoIncrementPrimaryKey = async ( autoIncrement: Collection, collectionName: string, ): Promise => { - const { value: doc } = await autoIncrement.findOneAndUpdate( - { - _id: collectionName, - }, - { - $inc: { value: 1 }, - }, - { - upsert: true, - }, - ); + const { value: doc } = await autoIncrement + .findOneAndUpdate( + { + _id: collectionName, + }, + { + $inc: { value: 1 }, + }, + { + upsert: true, + }, + ) + .catch(throwStoreError); return (doc?.value ?? 0) + 1; }; diff --git a/packages/mongodb-store/src/store.ts b/packages/mongodb-store/src/store.ts index 7afa1e0..3bf1046 100644 --- a/packages/mongodb-store/src/store.ts +++ b/packages/mongodb-store/src/store.ts @@ -17,6 +17,7 @@ import { StoreIndex, StoreIndexField, StorePrimaryKey, + throwStoreError, } from '@neuledge/store'; import { Db, @@ -95,7 +96,10 @@ export class MongoDBStore implements Store { typeof name === 'string' ? this.client .connect() - .then((client) => client.db(name, db as DbOptions | undefined)) + .then( + (client) => client.db(name, db as DbOptions | undefined), + throwStoreError, + ) : Promise.resolve(db as Db); this.collections = {}; @@ -112,12 +116,15 @@ export class MongoDBStore implements Store { } async close(): Promise { - await this.client.close(); + await this.client.close().catch(throwStoreError); } async listCollections(): Promise { const db = await this.db; - const res = await db.listCollections({}, { nameOnly: true }).toArray(); + const res = await db + .listCollections({}, { nameOnly: true }) + .toArray() + .catch(throwStoreError); return res.map((item): StoreCollection_Slim => ({ name: item.name })); } @@ -126,7 +133,10 @@ export class MongoDBStore implements Store { options: StoreDescribeCollectionOptions, ): Promise { const collection = await this.collection(options.collection.name); - const indexes = await collection.listIndexes().toArray(); + const indexes = await collection + .listIndexes() + .toArray() + .catch(throwStoreError); const storeIndexes = indexes.map( (index): StoreIndex => ({ @@ -189,7 +199,7 @@ export class MongoDBStore implements Store { async dropCollection(options: StoreDropCollectionOptions): Promise { const db = await this.db; - await db.dropCollection(options.collection.name); + await db.dropCollection(options.collection.name).catch(throwStoreError); } async find(options: StoreFindOptions): Promise { @@ -221,7 +231,7 @@ export class MongoDBStore implements Store { ); } - const rawDocs = await query.toArray(); + const rawDocs = await query.toArray().catch(throwStoreError); let docs = rawDocs.map((doc) => unescapeDocument(options.collection, doc)); const asyncLimit = pLimit(this.readConcurrency); @@ -250,14 +260,16 @@ export class MongoDBStore implements Store { ), ); - const res = await collection.insertMany( - insertedIds.map((insertedId, i) => - escapeDocument(options.collection, { - ...options.documents[i], - ...insertedId, - }), - ), - ); + const res = await collection + .insertMany( + insertedIds.map((insertedId, i) => + escapeDocument(options.collection, { + ...options.documents[i], + ...insertedId, + }), + ), + ) + .catch(throwStoreError); return { insertedIds: insertedIds, @@ -277,7 +289,9 @@ export class MongoDBStore implements Store { options.set as Document, ); - const res = await collection.updateMany(filter, update); + const res = await collection + .updateMany(filter, update) + .catch(throwStoreError); return { affectedCount: res.modifiedCount, @@ -291,7 +305,7 @@ export class MongoDBStore implements Store { ? findFilter(options.collection.primaryKey, options.where) : {}; - const res = await collection.deleteMany(filter); + const res = await collection.deleteMany(filter).catch(throwStoreError); return { affectedCount: res.deletedCount }; } @@ -306,7 +320,8 @@ export class MongoDBStore implements Store { async (db) => { const [exists] = await db .listCollections({ name: collectionName }, { nameOnly: true }) - .toArray(); + .toArray() + .catch(throwStoreError); if (!exists) { // allow retry on next call @@ -348,7 +363,7 @@ export class MongoDBStore implements Store { query = query.project(join.project ?? { _id: 1 }); } - const rawDocs = await query.toArray(); + const rawDocs = await query.toArray().catch(throwStoreError); return rawDocs.map((doc) => unescapeDocument(join.collection, doc)); } diff --git a/packages/mysql-store/src/store.ts b/packages/mysql-store/src/store.ts index fde3725..cb8372f 100644 --- a/packages/mysql-store/src/store.ts +++ b/packages/mysql-store/src/store.ts @@ -7,6 +7,7 @@ import { StoreDescribeCollectionOptions, StoreDropCollectionOptions, StoreEnsureCollectionOptions, + StoreError, StoreFindOptions, StoreInsertOptions, StoreInsertionResponse, @@ -49,7 +50,17 @@ export class MySQLStore implements Store { async close(): Promise { await new Promise((resolve, reject) => - this.connection.end((error) => (error ? reject(error) : resolve())), + this.connection.end((error) => + error + ? reject( + new StoreError( + StoreError.Code.INTERNAL_ERROR, + error.message, + error, + ), + ) + : resolve(), + ), ); } diff --git a/packages/postgresql-store/README.md b/packages/postgresql-store/README.md index f438fe2..977c75b 100644 --- a/packages/postgresql-store/README.md +++ b/packages/postgresql-store/README.md @@ -14,8 +14,12 @@ npm install @neuledge/postgresql-store import { PostgreSQLStore } from '@neuledge/postgresql-store'; const store = new PostgreSQLStore({ - uri: process.env.MYSQL_URI ?? 'mysql://localhost:3306', - database: process.env.MYSQL_DATABASE ?? 'my-database', + host: process.env.POSTGRESQL_HOST ?? 'localhost', + port: Number(process.env.POSTGRESQL_PORT) ?? 5432, + user: process.env.POSTGRESQL_USER ?? 'postgres', + password: process.env.POSTGRESQL_PASSWORD, + ssl: process.env.POSTGRESQL_SSL === 'true', + database: process.env.POSTGRESQL_DATABASE ?? 'my-database', }); const engine = new Engine({ diff --git a/packages/postgresql-store/src/queries/__fixtures__/posts-table.ts b/packages/postgresql-store/src/queries/__fixtures__/posts-table.ts index 921e258..0f05ad5 100644 --- a/packages/postgresql-store/src/queries/__fixtures__/posts-table.ts +++ b/packages/postgresql-store/src/queries/__fixtures__/posts-table.ts @@ -8,6 +8,7 @@ export const postsCollection: StoreCollection = { id: { name: 'id', type: 'number', + list: false, nullable: false, size: null, precision: 32, @@ -16,6 +17,7 @@ export const postsCollection: StoreCollection = { author_id: { name: 'author_id', type: 'number', + list: false, nullable: false, size: null, precision: 32, @@ -24,6 +26,7 @@ export const postsCollection: StoreCollection = { title: { name: 'title', type: 'string', + list: false, nullable: false, size: 100, precision: null, @@ -32,6 +35,7 @@ export const postsCollection: StoreCollection = { body: { name: 'body', type: 'string', + list: false, nullable: false, size: 1000, precision: null, @@ -40,6 +44,7 @@ export const postsCollection: StoreCollection = { created_at: { name: 'created_at', type: 'date-time', + list: false, nullable: false, size: null, precision: null, @@ -48,6 +53,7 @@ export const postsCollection: StoreCollection = { updated_at: { name: 'updated_at', type: 'date-time', + list: false, nullable: false, size: null, precision: null, diff --git a/packages/postgresql-store/src/queries/__fixtures__/users-table.ts b/packages/postgresql-store/src/queries/__fixtures__/users-table.ts index ffbf2d6..ba8f4f5 100644 --- a/packages/postgresql-store/src/queries/__fixtures__/users-table.ts +++ b/packages/postgresql-store/src/queries/__fixtures__/users-table.ts @@ -29,6 +29,7 @@ export const usersTableColumns: PostgreSQLColumn[] = [ { column_name: 'id', data_type: 'integer', + list: false, character_maximum_length: null, numeric_precision: 32, numeric_scale: 0, @@ -38,6 +39,7 @@ export const usersTableColumns: PostgreSQLColumn[] = [ { column_name: 'name', data_type: 'character varying', + list: false, character_maximum_length: 50, numeric_precision: null, numeric_scale: null, @@ -47,6 +49,7 @@ export const usersTableColumns: PostgreSQLColumn[] = [ { column_name: 'email', data_type: 'character varying', + list: false, character_maximum_length: 100, numeric_precision: null, numeric_scale: null, @@ -56,6 +59,7 @@ export const usersTableColumns: PostgreSQLColumn[] = [ { column_name: 'phone', data_type: 'character varying', + list: false, character_maximum_length: 20, numeric_precision: null, numeric_scale: null, @@ -65,6 +69,7 @@ export const usersTableColumns: PostgreSQLColumn[] = [ { column_name: 'created_at', data_type: 'timestamp without time zone', + list: false, character_maximum_length: null, numeric_precision: null, numeric_scale: null, @@ -74,6 +79,7 @@ export const usersTableColumns: PostgreSQLColumn[] = [ { column_name: 'updated_at', data_type: 'timestamp without time zone', + list: false, character_maximum_length: null, numeric_precision: null, numeric_scale: null, @@ -131,6 +137,7 @@ export const usersCollection: StoreCollection = { id: { name: 'id', type: 'number', + list: false, nullable: false, size: null, precision: 32, @@ -139,6 +146,7 @@ export const usersCollection: StoreCollection = { name: { name: 'name', type: 'string', + list: false, nullable: true, size: 50, precision: null, @@ -147,6 +155,7 @@ export const usersCollection: StoreCollection = { email: { name: 'email', type: 'string', + list: false, nullable: false, size: 100, precision: null, @@ -155,6 +164,7 @@ export const usersCollection: StoreCollection = { phone: { name: 'phone', type: 'string', + list: false, nullable: true, size: 20, precision: null, @@ -163,6 +173,7 @@ export const usersCollection: StoreCollection = { created_at: { name: 'created_at', type: 'date-time', + list: false, nullable: false, size: null, precision: null, @@ -171,6 +182,7 @@ export const usersCollection: StoreCollection = { updated_at: { name: 'updated_at', type: 'date-time', + list: false, nullable: false, size: null, precision: null, diff --git a/packages/postgresql-store/src/queries/add-column.ts b/packages/postgresql-store/src/queries/add-column.ts index dbe1a8d..994854b 100644 --- a/packages/postgresql-store/src/queries/add-column.ts +++ b/packages/postgresql-store/src/queries/add-column.ts @@ -18,8 +18,8 @@ export const getColumnDefinition = ( collection: StoreCollection, ): string => `${encodeIdentifier(field.name)} ${getColumnDataType(field, collection)}${ - field.nullable ? '' : ' NOT NULL' - }`; + field.list ? '[]' : '' + }${field.nullable ? '' : ' NOT NULL'}`; const getColumnDataType = ( field: StoreField, diff --git a/packages/postgresql-store/src/queries/list-table-columns.ts b/packages/postgresql-store/src/queries/list-table-columns.ts index e313bbd..7a7b6d6 100644 --- a/packages/postgresql-store/src/queries/list-table-columns.ts +++ b/packages/postgresql-store/src/queries/list-table-columns.ts @@ -7,6 +7,7 @@ import { PostgreSQLConnection } from './connection'; export interface PostgreSQLColumn { column_name: string; data_type: string; + list: boolean; character_maximum_length: number | null; numeric_precision: number | null; numeric_scale: number | null; @@ -22,9 +23,11 @@ export const listTableColumns = async ( .query(listTableColumns_sql, [tableName]) .then((result) => result.rows); -export const listTableColumns_sql = `SELECT column_name, data_type, character_maximum_length, numeric_precision, numeric_scale, (is_nullable = 'YES') as is_nullable, column_default LIKE 'nextval(%)' AS is_auto_increment FROM information_schema.columns WHERE table_catalog = current_database() AND table_schema = current_schema() AND table_name = $1`; +export const listTableColumns_sql = `SELECT c.column_name, COALESCE(o.data_type, c.data_type) as data_type,(c.data_type = 'ARRAY') as list, c.character_maximum_length, c.numeric_precision, c.numeric_scale, (c.is_nullable = 'YES') as is_nullable, c.column_default LIKE 'nextval(%)' AS is_auto_increment FROM information_schema.columns c LEFT JOIN information_schema.element_types o ON o.object_catalog = c.table_catalog AND o.object_schema = c.table_schema AND o.object_name = c.table_name AND o.object_type = 'TABLE' AND o.collection_type_identifier = c.dtd_identifier WHERE c.table_catalog = current_database() AND c.table_schema = current_schema() AND c.table_name = $1`; + +// will prduce typnames instead of data_type: +// export const listTableColumns_sql = `SELECT s.column_name, COALESCE(e.typname, t.typname) as data_type, a.attndims as dimensions,s.character_maximum_length, s.numeric_precision, s.numeric_scale, (s.is_nullable = 'YES') as is_nullable, s.column_default LIKE 'nextval(%)' AS is_auto_increment FROM information_schema.columns AS s JOIN pg_namespace AS n ON n.nspname = s.table_schema JOIN pg_class AS c ON c.relnamespace = n.oid AND c.relname = s.table_name JOIN pg_attribute AS a ON a.attrelid = c.oid AND a.attname = s.column_name JOIN pg_type t ON t.oid = a.atttypid LEFT JOIN pg_type e ON e.oid = t.typelem WHERE table_catalog = current_database() AND table_schema = current_schema() AND table_name = $1`; -// https://www.postgresql.org/docs/current/datatype.html export const dataTypeMap: Record = { bigint: 'number', bigserial: 'number', diff --git a/packages/postgresql-store/src/store.test.ts b/packages/postgresql-store/src/store.test.ts index 72fc2da..aee1df4 100644 --- a/packages/postgresql-store/src/store.test.ts +++ b/packages/postgresql-store/src/store.test.ts @@ -41,7 +41,7 @@ describe('store', () => { describe('.close()', () => { it('should be able to close the store', async () => { - const end = jest.fn(); + const end = jest.fn().mockResolvedValue(void 0); const store = new PostgreSQLStore({ client: { end } as never, diff --git a/packages/postgresql-store/src/store.ts b/packages/postgresql-store/src/store.ts index 5156daf..5631ba8 100644 --- a/packages/postgresql-store/src/store.ts +++ b/packages/postgresql-store/src/store.ts @@ -29,6 +29,7 @@ import { StoreList, StoreMutationResponse, StoreUpdateOptions, + throwStoreError, } from '@neuledge/store'; import { dropCollection, @@ -54,13 +55,19 @@ export class PostgreSQLStore implements Store { private connection: PostgreSQLStoreClient; constructor(options: PostgreSQLStoreOptions) { - this.connection = 'client' in options ? options.client : new Pool(options); + this.connection = + 'client' in options + ? options.client + : new Pool({ + connectionTimeoutMillis: 5000, + ...options, + }); } // connection methods async close(): Promise { - await this.connection.end(); + await this.connection.end().catch(throwStoreError); } // store methods diff --git a/packages/sql-store/src/logic/collections/describe.ts b/packages/sql-store/src/logic/collections/describe.ts index 9a26217..f7fd454 100644 --- a/packages/sql-store/src/logic/collections/describe.ts +++ b/packages/sql-store/src/logic/collections/describe.ts @@ -10,6 +10,7 @@ import { StoreDescribeCollectionOptions, StoreError, StoreShapeType, + throwStoreError, } from '@neuledge/store'; export interface DescribeCollectionQueries< @@ -85,7 +86,7 @@ const getCollectionDetails = async < const [columns, indexAttributes] = await Promise.all([ listTableColumns(connection, name), listIndexAttributes(connection, name), - ]); + ]).catch(throwStoreError); const columnMap = Object.fromEntries( columns.map((column) => [column.column_name, column]), diff --git a/packages/sql-store/src/logic/collections/drop.ts b/packages/sql-store/src/logic/collections/drop.ts index ba3241c..4eb1621 100644 --- a/packages/sql-store/src/logic/collections/drop.ts +++ b/packages/sql-store/src/logic/collections/drop.ts @@ -1,4 +1,4 @@ -import { StoreDropCollectionOptions } from '@neuledge/store'; +import { StoreDropCollectionOptions, throwStoreError } from '@neuledge/store'; export interface DropCollectionQueries { dropTableIfExists(connection: Connection, name: string): Promise; @@ -9,5 +9,7 @@ export const dropCollection = async ( connection: Connection, { dropTableIfExists }: DropCollectionQueries, ): Promise => { - await dropTableIfExists(connection, options.collection.name); + await dropTableIfExists(connection, options.collection.name).catch( + throwStoreError, + ); }; diff --git a/packages/sql-store/src/logic/collections/ensure.ts b/packages/sql-store/src/logic/collections/ensure.ts index 0b409dd..469b39c 100644 --- a/packages/sql-store/src/logic/collections/ensure.ts +++ b/packages/sql-store/src/logic/collections/ensure.ts @@ -4,6 +4,7 @@ import { StoreEnsureCollectionOptions, StoreField, StoreIndex, + throwStoreError, } from '@neuledge/store'; import { SQLColumn, SQLIndexAttribute, SQLIndexColumn } from '@/mappers'; import { DescribeCollectionQueries, describeCollection } from './describe'; @@ -49,36 +50,50 @@ export const ensureCollection = async < >( options: StoreEnsureCollectionOptions, connection: Connection, - { - createTableIfNotExists, - addIndex, - addColumn, - dropIndex, - dropColumn, - ...describeCollectionQueries - }: EnsureCollectionQueries, + queries: EnsureCollectionQueries, ): Promise => { - await createTableIfNotExists(connection, options.collection); + await queries + .createTableIfNotExists(connection, options.collection) + .catch(throwStoreError); + await dropProperties(options, connection, queries); + + const existsCollection = await describeCollection( + options, + connection, + queries, + ); + + await addProperties(options, connection, existsCollection, queries); +}; + +const dropProperties = async ( + options: StoreEnsureCollectionOptions, + connection: Connection, + { dropIndex, dropColumn }: EnsureCollectionQueriesOnly, +) => { const asyncLimit = pLimit(4); await Promise.all( options.dropIndexes?.map((index) => asyncLimit(() => dropIndex(connection, options.collection, index)), ) || [], - ); + ).catch(throwStoreError); await Promise.all( options.dropFields?.map((field) => asyncLimit(() => dropColumn(connection, options.collection, field)), ) || [], - ); + ).catch(throwStoreError); +}; - const collection = await describeCollection( - options, - connection, - describeCollectionQueries, - ); +const addProperties = async ( + options: StoreEnsureCollectionOptions, + connection: Connection, + existsCollection: StoreCollection, + { addIndex, addColumn }: EnsureCollectionQueriesOnly, +) => { + const asyncLimit = pLimit(4); // although we support adding columns with non-nullables types, it will be // rejected by the database and for a good reason. It's the responsibility of @@ -87,17 +102,20 @@ export const ensureCollection = async < await Promise.all( options.fields - ?.filter((field) => !collection.fields[field.name]) + ?.filter((field) => !existsCollection.fields[field.name]) .map((field) => - asyncLimit(() => addColumn(connection, collection, field)), + asyncLimit(() => addColumn(connection, existsCollection, field)), ) || [], - ); + ).catch(throwStoreError); await Promise.all( options.indexes - ?.filter((index) => !collection.indexes[index.name]) + ?.filter( + (index) => + !existsCollection.indexes[index.name] && index.unique !== 'primary', + ) .map((index) => - asyncLimit(() => addIndex(connection, collection, index)), + asyncLimit(() => addIndex(connection, existsCollection, index)), ) || [], - ); + ).catch(throwStoreError); }; diff --git a/packages/sql-store/src/logic/collections/list.ts b/packages/sql-store/src/logic/collections/list.ts index 2268bcd..f4eeb7c 100644 --- a/packages/sql-store/src/logic/collections/list.ts +++ b/packages/sql-store/src/logic/collections/list.ts @@ -1,5 +1,5 @@ import { SQLTable, toStoreCollection_Slim } from '@/mappers'; -import { StoreCollection_Slim } from '@neuledge/store'; +import { StoreCollection_Slim, throwStoreError } from '@neuledge/store'; export interface ListCollectionsQueries { listTables(connection: Connection): Promise; @@ -9,6 +9,6 @@ export const listCollections = async ( connection: Connection, { listTables }: ListCollectionsQueries, ): Promise => { - const tables = await listTables(connection); + const tables = await listTables(connection).catch(throwStoreError); return tables.map((table) => toStoreCollection_Slim(table)); }; diff --git a/packages/sql-store/src/logic/delete.ts b/packages/sql-store/src/logic/delete.ts index 30043d5..67fe7fb 100644 --- a/packages/sql-store/src/logic/delete.ts +++ b/packages/sql-store/src/logic/delete.ts @@ -1,5 +1,9 @@ import { QueryHelpers, getWhere } from '@/helpers'; -import { StoreDeleteOptions, StoreMutationResponse } from '@neuledge/store'; +import { + StoreDeleteOptions, + StoreMutationResponse, + throwStoreError, +} from '@neuledge/store'; export interface DeleteQueries { deleteFrom( @@ -22,7 +26,7 @@ export const deletes = async ( connection, name, where ? getWhere(queryHelpers, where) : null, - ); + ).catch(throwStoreError); return { affectedCount, diff --git a/packages/sql-store/src/logic/find.ts b/packages/sql-store/src/logic/find.ts index 2166502..f47394b 100644 --- a/packages/sql-store/src/logic/find.ts +++ b/packages/sql-store/src/logic/find.ts @@ -7,7 +7,12 @@ import { getSelectColumns, getWhere, } from '@/helpers'; -import { StoreDocument, StoreFindOptions, StoreList } from '@neuledge/store'; +import { + StoreDocument, + StoreFindOptions, + StoreList, + throwStoreError, +} from '@neuledge/store'; export interface FindQueries { selectFrom( @@ -62,7 +67,7 @@ export const find = async ( sort ? getOrderBy(queryHelpers, sort) : null, limit, offsetNumber, - ); + ).catch(throwStoreError); const docs = rawDocs.map((rawDoc) => convertRawDocument(rawDoc)); const nextOffset = rawDocs.length < limit ? null : offsetNumber + limit; diff --git a/packages/sql-store/src/logic/insert.ts b/packages/sql-store/src/logic/insert.ts index 4bca571..73c7881 100644 --- a/packages/sql-store/src/logic/insert.ts +++ b/packages/sql-store/src/logic/insert.ts @@ -3,6 +3,7 @@ import { StoreInsertOptions, StoreInsertionResponse, StoreScalarValue, + throwStoreError, } from '@neuledge/store'; export interface InsertQueries { @@ -34,7 +35,13 @@ export const insert = async ( const returns = Object.keys(primaryKey.fields); - const res = await insertInto(connection, name, columns, values, returns); + const res = await insertInto( + connection, + name, + columns, + values, + returns, + ).catch(throwStoreError); return { affectedCount: res.length, diff --git a/packages/sql-store/src/logic/update.ts b/packages/sql-store/src/logic/update.ts index 38ff73f..9d547dd 100644 --- a/packages/sql-store/src/logic/update.ts +++ b/packages/sql-store/src/logic/update.ts @@ -3,6 +3,7 @@ import { StoreDocument, StoreMutationResponse, StoreUpdateOptions, + throwStoreError, } from '@neuledge/store'; export interface UpdateQueries { @@ -28,7 +29,7 @@ export const update = async ( name, set, where ? getWhere(queryHelpers, where) : null, - ); + ).catch(throwStoreError); return { affectedCount, diff --git a/packages/sql-store/src/mappers/field.ts b/packages/sql-store/src/mappers/field.ts index c28b8f5..e77c966 100644 --- a/packages/sql-store/src/mappers/field.ts +++ b/packages/sql-store/src/mappers/field.ts @@ -3,6 +3,7 @@ import { StoreError, StoreField, StoreShapeType } from '@neuledge/store'; export interface SQLColumn { column_name: string; data_type: string; + list?: boolean | 1 | 0 | null; character_maximum_length: number | null; numeric_precision: number | null; numeric_scale: number | null; @@ -24,6 +25,7 @@ export const toStoreField = ( return { name: column.column_name, type, + list: !!column.list, nullable: !!column.is_nullable, size: column.character_maximum_length, precision: column.numeric_precision, diff --git a/packages/store/src/error.ts b/packages/store/src/error.ts index bce2e44..307acc7 100644 --- a/packages/store/src/error.ts +++ b/packages/store/src/error.ts @@ -24,9 +24,33 @@ export namespace StoreError { export class StoreError extends Error { static Code = StoreErrorCode; + public readonly originalError?: Error; - constructor(public readonly code: StoreErrorCode, message: string) { + constructor( + public readonly code: StoreErrorCode, + message: string, + originalError?: Error | unknown, + ) { super(message); this.name = 'StoreError'; + + if (originalError) { + this.originalError = + originalError instanceof Error + ? originalError + : new Error(String(originalError)); + } } } + +export const throwStoreError = (error: unknown): never => { + if (error instanceof StoreError) { + throw error; + } + + throw new StoreError( + StoreError.Code.INTERNAL_ERROR, + String((error as Error)?.message || error), + error, + ); +}; From 052def36aa4a8bcc8556639443af8a635c386051 Mon Sep 17 00:00:00 2001 From: Moshe Simantov Date: Wed, 16 Aug 2023 09:25:41 +0300 Subject: [PATCH 24/24] wip --- examples/graphql/package.json | 1 + .../engine/src/engine/metadata/collections.ts | 2 + packages/engine/src/engine/metadata/state.ts | 2 + .../src/queries/add-column.ts | 8 +-- .../src/queries/connection.ts | 36 +++++++++++-- .../src/queries/insert-into.ts | 10 ++-- .../src/queries/update-set.ts | 10 ++-- packages/sql-store/src/helpers/documents.ts | 25 ++++++++- packages/sql-store/src/helpers/join.test.ts | 36 +++++++++++++ packages/sql-store/src/helpers/join.ts | 36 +++++++++---- packages/sql-store/src/helpers/query.ts | 4 +- packages/sql-store/src/helpers/where.ts | 53 +++++++++++++------ packages/sql-store/src/logic/delete.ts | 2 +- packages/sql-store/src/logic/find.ts | 13 +++-- packages/sql-store/src/logic/insert.ts | 11 ++-- packages/sql-store/src/logic/update.ts | 18 +++++-- 16 files changed, 207 insertions(+), 60 deletions(-) diff --git a/examples/graphql/package.json b/examples/graphql/package.json index e41fb22..02fa937 100644 --- a/examples/graphql/package.json +++ b/examples/graphql/package.json @@ -14,6 +14,7 @@ }, "dependencies": { "@neuledge/engine": "^0.2.1", + "@neuledge/postgresql-store": "*", "@neuledge/mongodb-store": "^0.2.0", "dotenv": "^16.0.3", "fastify": "^4.14.1", diff --git a/packages/engine/src/engine/metadata/collections.ts b/packages/engine/src/engine/metadata/collections.ts index 8b53ff2..ea496c4 100644 --- a/packages/engine/src/engine/metadata/collections.ts +++ b/packages/engine/src/engine/metadata/collections.ts @@ -19,6 +19,8 @@ const ensureStoreCollection = async ( store: Store, collection: MetadataCollection, ): Promise => { + // FIXME how we handle fields or indexes changes? (e.g. a field changed to be nullable) + await store.ensureCollection({ collection, indexes: Object.values(collection.indexes), diff --git a/packages/engine/src/engine/metadata/state.ts b/packages/engine/src/engine/metadata/state.ts index 8374479..8e9e963 100644 --- a/packages/engine/src/engine/metadata/state.ts +++ b/packages/engine/src/engine/metadata/state.ts @@ -34,6 +34,8 @@ interface StoreMetadataStateRelation { index: number; } +// FIXME we can't save buffers on json fields. We need to encode them somehow or use relations for `fields` and `relations` fields. + export const getMetadataCollection = ( metadataCollectionName: string, ): StoreCollection => { diff --git a/packages/postgresql-store/src/queries/add-column.ts b/packages/postgresql-store/src/queries/add-column.ts index 994854b..8d34c92 100644 --- a/packages/postgresql-store/src/queries/add-column.ts +++ b/packages/postgresql-store/src/queries/add-column.ts @@ -21,9 +21,9 @@ export const getColumnDefinition = ( field.list ? '[]' : '' }${field.nullable ? '' : ' NOT NULL'}`; -const getColumnDataType = ( +export const getColumnDataType = ( field: StoreField, - collection: StoreCollection, + collection?: StoreCollection, ): string => { switch (field.type) { case 'string': { @@ -67,11 +67,11 @@ const getColumnDataType = ( // https://www.postgresql.org/docs/current/datatype-numeric.html const getNumberDateType = ( field: StoreField, - collection: StoreCollection, + collection?: StoreCollection, ): string => { if (field.scale === 0) { if ( - collection.primaryKey.auto === 'increment' && + collection?.primaryKey.auto === 'increment' && collection.primaryKey.fields[field.name] ) { if (!field.precision || field.precision >= 10) { diff --git a/packages/postgresql-store/src/queries/connection.ts b/packages/postgresql-store/src/queries/connection.ts index 0d37e09..c46434b 100644 --- a/packages/postgresql-store/src/queries/connection.ts +++ b/packages/postgresql-store/src/queries/connection.ts @@ -1,15 +1,45 @@ import { QueryHelpers } from '@neuledge/sql-store'; -import { StoreScalarValue } from '@neuledge/store'; +import { + StoreError, + StoreField, + StoreScalarValue, + StoreShape, +} from '@neuledge/store'; import { Client, Pool } from 'pg'; import format from 'pg-format'; +import { getColumnDataType } from './add-column'; export type PostgreSQLConnection = Pick; export const encodeIdentifier = format.ident; -export const encodeLiteral = (val: StoreScalarValue) => +export const encodeLiteral = (val: StoreScalarValue, field: StoreField) => { + if (field.list) { + if (!Array.isArray(val)) { + throw new StoreError( + StoreError.Code.INVALID_INPUT, + `Expected array value for field ${field.name} but got ${val}`, + ); + } + + if (!val.length) { + return `ARRAY[]::${getColumnDataType(field)}[]`; + } + + return `ARRAY[${val.map((v) => encodeScalar(v, field)).join(', ')}]`; + } + + return encodeScalar(val, field); +}; + +const encodeScalar = (val: StoreScalarValue, shape: StoreShape) => { + if (shape.type === 'json') { + return `${format.literal(JSON.stringify(val))}::JSONB`; + } + // format.literal will convert everything else to string which will work fine for bigint - format.literal(val as Exclude); + return format.literal(val as Exclude); +}; export const queryHelpers: QueryHelpers = { encodeIdentifier, diff --git a/packages/postgresql-store/src/queries/insert-into.ts b/packages/postgresql-store/src/queries/insert-into.ts index 285c664..4eefd77 100644 --- a/packages/postgresql-store/src/queries/insert-into.ts +++ b/packages/postgresql-store/src/queries/insert-into.ts @@ -1,4 +1,4 @@ -import { StoreDocument, StoreScalarValue } from '@neuledge/store'; +import { StoreDocument, StoreField, StoreScalarValue } from '@neuledge/store'; import { PostgreSQLConnection, encodeIdentifier, @@ -8,18 +8,20 @@ import { export const insertInto = async ( connection: PostgreSQLConnection, name: string, - columns: string[], + columns: StoreField[], values: (StoreScalarValue | undefined)[][], returns: string[], ): Promise => connection .query( `INSERT INTO ${encodeIdentifier(name)} (${columns - .map((column) => encodeIdentifier(column)) + .map((column) => encodeIdentifier(column.name)) .join(', ')}) VALUES (${values .map((arr) => arr - .map((v) => (v === undefined ? 'DEFAULT' : encodeLiteral(v))) + .map((v, i) => + v === undefined ? 'DEFAULT' : encodeLiteral(v, columns[i]), + ) .join(', '), ) .join('), (')}) RETURNING ${returns diff --git a/packages/postgresql-store/src/queries/update-set.ts b/packages/postgresql-store/src/queries/update-set.ts index 019a8f3..165d3a2 100644 --- a/packages/postgresql-store/src/queries/update-set.ts +++ b/packages/postgresql-store/src/queries/update-set.ts @@ -1,4 +1,4 @@ -import { StoreDocument } from '@neuledge/store'; +import { StoreField, StoreScalarValue } from '@neuledge/store'; import { PostgreSQLConnection, encodeIdentifier, @@ -8,12 +8,12 @@ import { export const updateSet = async ( connection: PostgreSQLConnection, name: string, - set: StoreDocument, + setValues: [field: StoreField, value: StoreScalarValue][], where: string | null, ): Promise => { - const setClauses = Object.entries(set).map( - ([key, value]) => - `${encodeIdentifier(key)} = ${encodeLiteral(value ?? null)}`, + const setClauses = setValues.map( + ([field, value]) => + `${encodeIdentifier(field.name)} = ${encodeLiteral(value, field)}`, ); const res = await connection.query( diff --git a/packages/sql-store/src/helpers/documents.ts b/packages/sql-store/src/helpers/documents.ts index 4c51718..2c1bc32 100644 --- a/packages/sql-store/src/helpers/documents.ts +++ b/packages/sql-store/src/helpers/documents.ts @@ -1,4 +1,4 @@ -import { StoreDocument } from '@neuledge/store'; +import { StoreCollection, StoreDocument, StoreField } from '@neuledge/store'; export const convertRawDocument = (rawDoc: StoreDocument): StoreDocument => { const doc: StoreDocument = {}; @@ -39,3 +39,26 @@ const preferLowerChoices = (doc: StoreDocument): void => { preferLowerChoices(value as StoreDocument); } }; + +// not sure if we need this method, probably best to pass the responsibility +// to the underlying database driver and make sure it's consistent with +// javascript's values. +export const parseRawDocument = ( + fields: Record, + rawDoc: StoreDocument, +): StoreDocument => { + for (const key in rawDoc) { + const field = fields[key]; + if (field?.type !== 'number') continue; + + const value = rawDoc[key]; + if (typeof value !== 'string') continue; + + rawDoc[key] = + field.scale === 0 && (!field.precision || field.precision > 15) + ? BigInt(value) + : Number(value); + } + + return rawDoc; +}; diff --git a/packages/sql-store/src/helpers/join.test.ts b/packages/sql-store/src/helpers/join.test.ts index dbf5800..c08879f 100644 --- a/packages/sql-store/src/helpers/join.test.ts +++ b/packages/sql-store/src/helpers/join.test.ts @@ -58,6 +58,12 @@ describe('helpers/join', () => { '`foo$0`.`title` AS `foo$0.title`', '`foo$0`.`description` AS `foo$0.description`', ], + joinFields: { + 'foo$0.id': otherCollection.fields.id, + 'foo$0.subId': otherCollection.fields.subId, + 'foo$0.title': otherCollection.fields.title, + 'foo$0.description': otherCollection.fields.description, + }, fromAlias: '$', fromJoins: [ 'INNER JOIN `otherCollection` `foo$0` ON (`foo$0`.`id` = `$`.`foo`)', @@ -82,6 +88,9 @@ describe('helpers/join', () => { }), ).toEqual({ selectColumns: ['`foo$0`.`title` AS `foo$0.title`'], + joinFields: { + 'foo$0.title': otherCollection.fields.title, + }, fromAlias: '$', fromJoins: [ 'LEFT JOIN `otherCollection` `foo$0` ON (`foo$0`.`id` = `$`.`foo` AND `foo$0`.`subId` = `$`.`bar`)', @@ -118,6 +127,10 @@ describe('helpers/join', () => { '`foo$0`.`title` AS `foo$0.title`', '`bar$0`.`description` AS `bar$0.description`', ], + joinFields: { + 'foo$0.title': otherCollection.fields.title, + 'bar$0.description': otherCollection.fields.description, + }, fromAlias: '$', fromJoins: [ 'INNER JOIN `otherCollection` `foo$0` ON (`foo$0`.`id` = `$`.`foo`)', @@ -151,6 +164,10 @@ describe('helpers/join', () => { '`foo$0`.`title` AS `foo$0.title`', '`foo$0`.`title` AS `foo$1.title`', ], + joinFields: { + 'foo$0.title': otherCollection.fields.title, + 'foo$1.title': otherCollection.fields.title, + }, fromAlias: '$', fromJoins: [ 'INNER JOIN `otherCollection` `foo$0` ON (`foo$0`.`id` = `$`.`foo`) OR (`foo$0`.`subId` = `$`.`bar`)', @@ -183,6 +200,10 @@ describe('helpers/join', () => { '`foo$0`.`title` AS `foo$0.title`', '`foo$1`.`url` AS `foo$1.url`', ], + joinFields: { + 'foo$0.title': otherCollection.fields.title, + 'foo$1.url': otherCollection2.fields.url, + }, fromAlias: '$', fromJoins: [ 'LEFT JOIN `otherCollection` `foo$0` ON (`foo$0`.`subId` = `$`.`foo`)', @@ -211,6 +232,9 @@ describe('helpers/join', () => { }), ).toEqual({ selectColumns: ['`foo$0`.`title` AS `foo$0.title`'], + joinFields: { + 'foo$0.title': otherCollection.fields.title, + }, fromAlias: '$', fromJoins: [ 'INNER JOIN `otherCollection` `foo$0` ON (`foo$0`.`id` = `$`.`foo` AND `foo$0`.`subId` = 123)', @@ -245,6 +269,10 @@ describe('helpers/join', () => { '`foo$0`.`title` AS `foo$0.title`', '`foo$0`.`description` AS `foo$1.description`', ], + joinFields: { + 'foo$0.title': otherCollection.fields.title, + 'foo$1.description': otherCollection.fields.description, + }, fromAlias: '$', fromJoins: [ 'LEFT JOIN `otherCollection` `foo$0` ON (`foo$0`.`id` = `$`.`foo` AND `foo$0`.`subId` < 123) OR (`foo$0`.`subId` = `$`.`bar` AND `foo$0`.`title` = "hello")', @@ -281,6 +309,10 @@ describe('helpers/join', () => { '`foo$0`.`title` AS `foo$0.title`', '`foo$0.bar$0`.`url` AS `foo$0.bar$0.url`', ], + joinFields: { + 'foo$0.title': otherCollection.fields.title, + 'foo$0.bar$0.url': otherCollection2.fields.url, + }, fromAlias: '$', fromJoins: [ 'LEFT JOIN `otherCollection` `foo$0` ON (`foo$0`.`id` = `$`.`foo`)', @@ -318,6 +350,10 @@ describe('helpers/join', () => { '`test$0`.`title` AS `test$0.title`', '`test$0.test$0`.`url` AS `test$0.test$0.url`', ], + joinFields: { + 'test$0.title': otherCollection.fields.title, + 'test$0.test$0.url': otherCollection2.fields.url, + }, fromAlias: '$', fromJoins: [ 'LEFT JOIN `otherCollection` `test$0` ON (`test$0`.`id` = `$`.`foo`)', diff --git a/packages/sql-store/src/helpers/join.ts b/packages/sql-store/src/helpers/join.ts index 60e08b7..06b05b5 100644 --- a/packages/sql-store/src/helpers/join.ts +++ b/packages/sql-store/src/helpers/join.ts @@ -1,6 +1,7 @@ import { StoreCollection, StoreError, + StoreField, StoreJoin, StoreJoinChoice, } from '@neuledge/store'; @@ -13,6 +14,7 @@ export const getFromJoins = ( options: Pick, ): { selectColumns: string[]; + joinFields: Record; fromAlias: string; fromJoins: string[]; whereClauses: string[]; @@ -23,21 +25,23 @@ export const getFromJoins = ( if (!joins.length) return null; const selectColumns: string[] = []; + const joinFields: Record = {}; const fromJoins: string[] = []; let whereClauses: string[] = []; for (const join of joins) { - const { select, fromJoin, where } = getFromJoin(helpers, join); + const { select, fields, fromJoin, where } = getFromJoin(helpers, join); selectColumns.push(...select); fromJoins.push(fromJoin); whereClauses.push(...where); + Object.assign(joinFields, fields); } // remove where duplicates whereClauses = [...new Set(whereClauses)]; - return { selectColumns, fromAlias, fromJoins, whereClauses }; + return { selectColumns, joinFields, fromAlias, fromJoins, whereClauses }; }; // local helpers @@ -139,12 +143,20 @@ interface Join { const getFromJoin = ( helpers: QueryHelpers, join: Join, -): { select: string[]; fromJoin: string; where: string[] } => { +): { + select: string[]; + fields: Record; + fromJoin: string; + where: string[]; +} => { const { collection, alias: joinAlias, select, ons, required } = join; - const selectColumns = Object.entries(select).map(([alias, name]) => - getSelectColumn(helpers, joinAlias, name, alias), - ); + const joinFields: Record = {}; + + const selectColumns = Object.entries(select).map(([alias, name]) => { + joinFields[alias] = collection.fields[name]; + return getSelectColumn(helpers, joinAlias, name, alias); + }); let joinType: 'INNER' | 'LEFT'; const where: string[] = []; @@ -163,15 +175,16 @@ const getFromJoin = ( collection.name, )} ${helpers.encodeIdentifier(joinAlias)} ON (${ons .map(({ fromAlias, by, where }) => - getJoinOn(helpers, fromAlias, joinAlias, by, where), + getJoinOn(helpers, collection, fromAlias, joinAlias, by, where), ) .join(') OR (')})`; - return { select: selectColumns, fromJoin, where }; + return { select: selectColumns, fields: joinFields, fromJoin, where }; }; const getJoinOn = ( helpers: QueryHelpers, + collection: StoreCollection, fromAlias: string, joinAlias: string, by: StoreJoinChoice['by'], @@ -190,12 +203,15 @@ const getJoinOn = ( } if (term.value != null) { - return `${field} = ${helpers.encodeLiteral(term.value)}`; + return `${field} = ${helpers.encodeLiteral( + term.value, + collection.fields[key], + )}`; } return `${field} IS NULL`; }), - ...(where ? [getWhere(helpers, where, joinAlias)] : []), + ...(where ? [getWhere(helpers, collection, where, joinAlias)] : []), ].join(' AND '); /** diff --git a/packages/sql-store/src/helpers/query.ts b/packages/sql-store/src/helpers/query.ts index 19ab3e3..0e8d379 100644 --- a/packages/sql-store/src/helpers/query.ts +++ b/packages/sql-store/src/helpers/query.ts @@ -1,6 +1,6 @@ -import { StoreScalarValue } from '@neuledge/store'; +import { StoreField, StoreScalarValue } from '@neuledge/store'; export interface QueryHelpers { encodeIdentifier(identifier: string): string; - encodeLiteral(literal: StoreScalarValue): string; + encodeLiteral(literal: StoreScalarValue, field: StoreField): string; } diff --git a/packages/sql-store/src/helpers/where.ts b/packages/sql-store/src/helpers/where.ts index 64ec230..9f6a46c 100644 --- a/packages/sql-store/src/helpers/where.ts +++ b/packages/sql-store/src/helpers/where.ts @@ -1,19 +1,28 @@ -import { StoreWhere, StoreWhereRecord, StoreWhereTerm } from '@neuledge/store'; +import { + StoreCollection, + StoreField, + StoreWhere, + StoreWhereRecord, + StoreWhereTerm, +} from '@neuledge/store'; import { QueryHelpers } from './query'; export const getWhere = ( helpers: QueryHelpers, + collection: StoreCollection, where: StoreWhere, from?: string | null, ): string | null => { const { $or } = where; if (!Array.isArray($or)) { - return whereRecord(helpers, where as StoreWhereRecord, from) || null; + return ( + whereRecord(helpers, collection, where as StoreWhereRecord, from) || null + ); } const sql = $or - .map((record) => whereRecord(helpers, record, from)) + .map((record) => whereRecord(helpers, collection, record, from)) .filter(Boolean); if (sql.length === 0) { @@ -25,6 +34,7 @@ export const getWhere = ( const whereRecord = ( helpers: QueryHelpers, + collection: StoreCollection, record: StoreWhereRecord, from?: string | null, ): string => { @@ -35,6 +45,7 @@ const whereRecord = ( whereTerm( helpers, `${fromEntry}${helpers.encodeIdentifier(columnName)}`, + collection.fields[columnName], term, ), ) @@ -45,43 +56,45 @@ const whereRecord = ( const whereTerm = ( helpers: QueryHelpers, entry: string, + field: StoreField, term: StoreWhereTerm, ): string => [ - ...whereComparisonTerm(helpers, entry, term), - ...whereLikeTerm(helpers, entry, term), - ...whereInTerm(helpers, entry, term), + ...whereComparisonTerm(helpers, entry, field, term), + ...whereLikeTerm(helpers, entry, field, term), + ...whereInTerm(helpers, entry, field, term), ].join(' AND '); const whereComparisonTerm = ( { encodeLiteral }: QueryHelpers, entry: string, + field: StoreField, term: StoreWhereTerm, ): string[] => { const sql: string[] = []; if ('$eq' in term) { - sql.push(`${entry} = ${encodeLiteral(term.$eq)}`); + sql.push(`${entry} = ${encodeLiteral(term.$eq, field)}`); } if ('$ne' in term) { - sql.push(`${entry} != ${encodeLiteral(term.$ne)}`); + sql.push(`${entry} != ${encodeLiteral(term.$ne, field)}`); } if ('$gt' in term) { - sql.push(`${entry} > ${encodeLiteral(term.$gt)}`); + sql.push(`${entry} > ${encodeLiteral(term.$gt, field)}`); } if ('$gte' in term) { - sql.push(`${entry} >= ${encodeLiteral(term.$gte)}`); + sql.push(`${entry} >= ${encodeLiteral(term.$gte, field)}`); } if ('$lt' in term) { - sql.push(`${entry} < ${encodeLiteral(term.$lt)}`); + sql.push(`${entry} < ${encodeLiteral(term.$lt, field)}`); } if ('$lte' in term) { - sql.push(`${entry} <= ${encodeLiteral(term.$lte)}`); + sql.push(`${entry} <= ${encodeLiteral(term.$lte, field)}`); } return sql; @@ -90,20 +103,21 @@ const whereComparisonTerm = ( const whereLikeTerm = ( { encodeLiteral }: QueryHelpers, entry: string, + field: StoreField, term: StoreWhereTerm, ): string[] => { const sql: string[] = []; if ('$contains' in term) { - sql.push(`${entry} LIKE ${encodeLiteral(`%${term.$contains}%`)}`); + sql.push(`${entry} LIKE ${encodeLiteral(`%${term.$contains}%`, field)}`); } if ('$startsWith' in term) { - sql.push(`${entry} LIKE ${encodeLiteral(`${term.$startsWith}%`)}`); + sql.push(`${entry} LIKE ${encodeLiteral(`${term.$startsWith}%`, field)}`); } if ('$endsWith' in term) { - sql.push(`${entry} LIKE ${encodeLiteral(`%${term.$endsWith}`)}`); + sql.push(`${entry} LIKE ${encodeLiteral(`%${term.$endsWith}`, field)}`); } if ('$in' in term) { @@ -111,7 +125,9 @@ const whereLikeTerm = ( sql.push('FALSE'); } else { sql.push( - `${entry} IN (${term.$in.map((v) => encodeLiteral(v)).join(', ')})`, + `${entry} IN (${term.$in + .map((v) => encodeLiteral(v, field)) + .join(', ')})`, ); } } @@ -122,6 +138,7 @@ const whereLikeTerm = ( const whereInTerm = ( { encodeLiteral }: QueryHelpers, entry: string, + field: StoreField, term: StoreWhereTerm, ): string[] => { const sql: string[] = []; @@ -131,7 +148,9 @@ const whereInTerm = ( sql.push('FALSE'); } else { sql.push( - `${entry} IN (${term.$in.map((v) => encodeLiteral(v)).join(', ')})`, + `${entry} IN (${term.$in + .map((v) => encodeLiteral(v, field)) + .join(', ')})`, ); } } diff --git a/packages/sql-store/src/logic/delete.ts b/packages/sql-store/src/logic/delete.ts index 67fe7fb..a28bd63 100644 --- a/packages/sql-store/src/logic/delete.ts +++ b/packages/sql-store/src/logic/delete.ts @@ -25,7 +25,7 @@ export const deletes = async ( const affectedCount = await deleteFrom( connection, name, - where ? getWhere(queryHelpers, where) : null, + where ? getWhere(queryHelpers, collection, where) : null, ).catch(throwStoreError); return { diff --git a/packages/sql-store/src/logic/find.ts b/packages/sql-store/src/logic/find.ts index f47394b..a868bb2 100644 --- a/packages/sql-store/src/logic/find.ts +++ b/packages/sql-store/src/logic/find.ts @@ -6,6 +6,7 @@ import { getSelectAny, getSelectColumns, getWhere, + parseRawDocument, } from '@/helpers'; import { StoreDocument, @@ -34,11 +35,12 @@ export const find = async ( ): Promise => { const { collection, select, where, limit, offset, sort } = options; + let selectColumns; let from = queryHelpers.encodeIdentifier(collection.name); - const join = getFromJoins(queryHelpers, options); + let { fields } = collection; - let selectColumns; - const whereClauses = where ? [getWhere(queryHelpers, where)] : []; + const join = getFromJoins(queryHelpers, options); + const whereClauses = where ? [getWhere(queryHelpers, collection, where)] : []; if (join) { from += ` ${queryHelpers.encodeIdentifier( @@ -51,6 +53,7 @@ export const find = async ( selectColumns.push(...join.selectColumns); whereClauses.push(...join.whereClauses); + fields = { ...fields, ...join.joinFields }; } else { selectColumns = select ? getSelectColumns(queryHelpers, null, select) @@ -69,7 +72,9 @@ export const find = async ( offsetNumber, ).catch(throwStoreError); - const docs = rawDocs.map((rawDoc) => convertRawDocument(rawDoc)); + const docs = rawDocs.map((rawDoc) => + convertRawDocument(parseRawDocument(fields, rawDoc)), + ); const nextOffset = rawDocs.length < limit ? null : offsetNumber + limit; return Object.assign(docs, { nextOffset }); diff --git a/packages/sql-store/src/logic/insert.ts b/packages/sql-store/src/logic/insert.ts index 73c7881..660dc21 100644 --- a/packages/sql-store/src/logic/insert.ts +++ b/packages/sql-store/src/logic/insert.ts @@ -1,5 +1,7 @@ +import { parseRawDocument } from '@/helpers'; import { StoreDocument, + StoreField, StoreInsertOptions, StoreInsertionResponse, StoreScalarValue, @@ -10,7 +12,7 @@ export interface InsertQueries { insertInto( connection: Connection, name: string, - columns: string[], + columns: StoreField[], values: (StoreScalarValue | undefined)[][], returns: string[], ): Promise; @@ -24,12 +26,13 @@ export const insert = async ( const { collection, documents } = options; const { name, fields, primaryKey } = collection; - const columns = Object.keys(fields); + const columns = Object.values(fields); const values = documents.map((document) => columns.map( (column) => - document[column] ?? (primaryKey.fields[column] ? undefined : null), + document[column.name] ?? + (primaryKey.fields[column.name] ? undefined : null), ), ); @@ -45,6 +48,6 @@ export const insert = async ( return { affectedCount: res.length, - insertedIds: res, + insertedIds: res.map((rawDoc) => parseRawDocument(fields, rawDoc)), }; }; diff --git a/packages/sql-store/src/logic/update.ts b/packages/sql-store/src/logic/update.ts index 9d547dd..c30e2eb 100644 --- a/packages/sql-store/src/logic/update.ts +++ b/packages/sql-store/src/logic/update.ts @@ -1,7 +1,8 @@ import { QueryHelpers, getWhere } from '@/helpers'; import { - StoreDocument, + StoreField, StoreMutationResponse, + StoreScalarValue, StoreUpdateOptions, throwStoreError, } from '@neuledge/store'; @@ -10,7 +11,7 @@ export interface UpdateQueries { updateSet( connection: Connection, name: string, - set: StoreDocument, + setValues: [field: StoreField, value: StoreScalarValue][], where: string | null, ): Promise; queryHelpers: QueryHelpers; @@ -22,13 +23,20 @@ export const update = async ( { updateSet, queryHelpers }: UpdateQueries, ): Promise => { const { collection, set, where } = options; - const { name } = collection; + const { name, fields } = collection; + + const setValues = Object.entries(set).map( + ([key, value]): [field: StoreField, value: StoreScalarValue] => [ + fields[key], + value ?? null, + ], + ); const affectedCount = await updateSet( connection, name, - set, - where ? getWhere(queryHelpers, where) : null, + setValues, + where ? getWhere(queryHelpers, collection, where) : null, ).catch(throwStoreError); return {