diff --git a/cmd/sling/resource/llm_CONNECTION_DATABASE.md b/cmd/sling/resource/llm_CONNECTION_DATABASE.md index d34cdd127..a4fe38d7b 100644 --- a/cmd/sling/resource/llm_CONNECTION_DATABASE.md +++ b/cmd/sling/resource/llm_CONNECTION_DATABASE.md @@ -222,6 +222,7 @@ Run read-only SQL queries on database connections: "input": { "connection": "MY_POSTGRES", "query": "SELECT * FROM public.users LIMIT 10", + "description": "Preview first 10 user records to understand table structure and data format", "limit": 100, "transient": false } @@ -231,6 +232,7 @@ Run read-only SQL queries on database connections: **Parameters:** - `connection` (required) - Database connection name - `query` (required) - SQL query to execute +- `description` (optional but strongly recommended) - A brief description of the intent and expected result of this query. Explain *why* you are running this query and what the result should tell you. This is logged for observability so that query activity can be understood in context. **Always provide this when executing a query.** - `limit` (optional) - Maximum rows to return (default: 100) - `transient` (optional) - Use transient connection (default: false) @@ -279,6 +281,7 @@ If a destructive operation is deemed necessary: "input": { "connection": "MY_POSTGRES", "query": "SELECT * FROM public.users LIMIT 5", + "description": "Preview first 5 user records to understand table structure and sample data", "limit": 5 } } @@ -291,6 +294,7 @@ If a destructive operation is deemed necessary: "input": { "connection": "MY_POSTGRES", "query": "SELECT status, COUNT(*) as count FROM orders WHERE created_date >= '2024-01-01' GROUP BY status ORDER BY count DESC LIMIT 20", + "description": "Get order count breakdown by status for 2024 to identify distribution of order states", "limit": 20 } } @@ -303,6 +307,7 @@ If a destructive operation is deemed necessary: "input": { "connection": "MY_POSTGRES", "query": "SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = 'public' LIMIT 50", + "description": "Retrieve column metadata for all public schema tables to map data types", "limit": 50 } } @@ -315,6 +320,7 @@ If a destructive operation is deemed necessary: "input": { "connection": "MY_POSTGRES", "query": "SELECT version()", + "description": "Check PostgreSQL server version for compatibility verification", "transient": true } } @@ -365,7 +371,8 @@ If a destructive operation is deemed necessary: "action": "query", "input": { "connection": "MY_DB", - "query": "SELECT * FROM production.users LIMIT 3" + "query": "SELECT * FROM production.users LIMIT 3", + "description": "Sample 3 rows from production.users to inspect data values and format" } } ``` @@ -378,7 +385,8 @@ If a destructive operation is deemed necessary: "action": "query", "input": { "connection": "MY_DB", - "query": "SELECT COUNT(*) as total_records FROM sales.orders" + "query": "SELECT COUNT(*) as total_records FROM sales.orders", + "description": "Get total row count of sales.orders to understand data volume" } } ``` @@ -389,7 +397,8 @@ If a destructive operation is deemed necessary: "action": "query", "input": { "connection": "MY_DB", - "query": "SELECT status, COUNT(*) as count FROM sales.orders GROUP BY status ORDER BY count DESC LIMIT 10" + "query": "SELECT status, COUNT(*) as count FROM sales.orders GROUP BY status ORDER BY count DESC LIMIT 10", + "description": "Analyze order status distribution to understand the breakdown of order states" } } ``` @@ -400,7 +409,8 @@ If a destructive operation is deemed necessary: "action": "query", "input": { "connection": "MY_DB", - "query": "SELECT COUNT(*) as nulls FROM sales.orders WHERE customer_id IS NULL" + "query": "SELECT COUNT(*) as nulls FROM sales.orders WHERE customer_id IS NULL", + "description": "Check for null customer_id values in orders to assess data quality" } } ``` @@ -434,7 +444,8 @@ When working with multiple databases: "action": "query", "input": { "connection": "SOURCE_DB", - "query": "SELECT COUNT(*), MIN(created_at), MAX(created_at) FROM public.users" + "query": "SELECT COUNT(*), MIN(created_at), MAX(created_at) FROM public.users", + "description": "Get row count and date range of source users table for cross-database comparison" } } ``` @@ -451,7 +462,8 @@ When working with multiple databases: "action": "query", "input": { "connection": "MY_POSTGRES", - "query": "SELECT schemaname, tablename, pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) as size FROM pg_tables WHERE schemaname = 'public' ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC LIMIT 10" + "query": "SELECT schemaname, tablename, pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) as size FROM pg_tables WHERE schemaname = 'public' ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC LIMIT 10", + "description": "Get top 10 largest tables in public schema by total size for capacity planning" } } @@ -460,7 +472,8 @@ When working with multiple databases: "action": "query", "input": { "connection": "MY_POSTGRES", - "query": "SELECT indexname, indexdef FROM pg_indexes WHERE tablename = 'users' LIMIT 20" + "query": "SELECT indexname, indexdef FROM pg_indexes WHERE tablename = 'users' LIMIT 20", + "description": "List indexes on users table to review indexing strategy" } } ``` @@ -473,7 +486,8 @@ When working with multiple databases: "action": "query", "input": { "connection": "MY_MYSQL", - "query": "SELECT table_schema, ROUND(SUM(data_length + index_length) / 1024 / 1024, 2) AS 'DB Size in MB' FROM information_schema.tables GROUP BY table_schema" + "query": "SELECT table_schema, ROUND(SUM(data_length + index_length) / 1024 / 1024, 2) AS 'DB Size in MB' FROM information_schema.tables GROUP BY table_schema", + "description": "Get size of each database schema in MB for storage overview" } } ``` @@ -486,7 +500,8 @@ When working with multiple databases: "action": "query", "input": { "connection": "MY_SNOWFLAKE", - "query": "SHOW WAREHOUSES" + "query": "SHOW WAREHOUSES", + "description": "List available Snowflake warehouses and their current status" } } @@ -495,7 +510,8 @@ When working with multiple databases: "action": "query", "input": { "connection": "MY_SNOWFLAKE", - "query": "SELECT * FROM information_schema.tables WHERE table_schema = 'PUBLIC' LIMIT 10" + "query": "SELECT * FROM information_schema.tables WHERE table_schema = 'PUBLIC' LIMIT 10", + "description": "List tables in PUBLIC schema to review table metadata and clustering info" } } ``` @@ -508,7 +524,8 @@ When working with multiple databases: "action": "query", "input": { "connection": "MY_BIGQUERY", - "query": "SELECT schema_name, catalog_name FROM INFORMATION_SCHEMA.SCHEMATA LIMIT 20" + "query": "SELECT schema_name, catalog_name FROM INFORMATION_SCHEMA.SCHEMATA LIMIT 20", + "description": "List available BigQuery datasets and their parent projects" } } @@ -517,7 +534,8 @@ When working with multiple databases: "action": "query", "input": { "connection": "MY_BIGQUERY", - "query": "SELECT table_name, partition_id, total_rows FROM `project.dataset.INFORMATION_SCHEMA.PARTITIONS` WHERE table_name = 'my_table' LIMIT 10" + "query": "SELECT table_name, partition_id, total_rows FROM `project.dataset.INFORMATION_SCHEMA.PARTITIONS` WHERE table_name = 'my_table' LIMIT 10", + "description": "Check partition layout and row counts for my_table to understand data distribution" } } ``` @@ -530,7 +548,8 @@ When working with multiple databases: "action": "query", "input": { "connection": "MY_CLICKHOUSE", - "query": "SELECT * FROM system.databases LIMIT 20" + "query": "SELECT * FROM system.databases LIMIT 20", + "description": "List ClickHouse databases to discover available data stores" } } @@ -539,7 +558,8 @@ When working with multiple databases: "action": "query", "input": { "connection": "MY_CLICKHOUSE", - "query": "SELECT database, name, engine, total_rows FROM system.tables WHERE database = 'default' LIMIT 20" + "query": "SELECT database, name, engine, total_rows FROM system.tables WHERE database = 'default' LIMIT 20", + "description": "List tables in default database with engine types and row counts for overview" } } ``` diff --git a/cmd/sling/resource/mcp.yaml b/cmd/sling/resource/mcp.yaml index fe294e4be..698c99522 100644 --- a/cmd/sling/resource/mcp.yaml +++ b/cmd/sling/resource/mcp.yaml @@ -100,9 +100,10 @@ tools: **For `action: "query"`**: - `connection` (string, required): The name of the database connection to execute the query on - `query` (string, required): The SQL query to execute + - `description` (string, required): A brief description of the intent and expected result of this query. Explain *why* you are running this query and what the result should tell you. Always provide this when executing a query. - `limit` (number, optional): The limit of rows to return (defaults to 100) - `transient` (boolean, optional): Whether to use a transient connection (default: false) - + This action executes a SQL query on a database connection and return the results. **WARNING: Only use this tool for SELECT queries and other read-only operations. Never execute destructive queries such as DELETE, DROP, TRUNCATE, ALTER, UPDATE, INSERT, or any other data modification operations.** * If a destructive operation (e.g., dropping an object, deleting significant data, altering table structures) is deemed necessary, **DO NOT execute it directly**. Instead, formulate the required SQL query/statement and **return it to the USER for manual review and execution**. @@ -126,7 +127,8 @@ tools: "action": "query", "input": { "connection": "MY_PG", - "query": "SELECT * FROM users LIMIT 10" + "query": "SELECT * FROM users LIMIT 10", + "description": "Preview first 10 user records to understand table structure and data format" } } @@ -183,16 +185,18 @@ tools: "input": { "connection": "MY_PG", "query": "SELECT status, COUNT(*) as count FROM orders WHERE created_date >= '2024-01-01' GROUP BY status ORDER BY count DESC LIMIT 50", + "description": "Get order count breakdown by status for 2024 to identify distribution of order states", "limit": 50 } } - + # Use transient connection for one-off query { "action": "query", "input": { "connection": "MY_PG", "query": "SELECT version()", + "description": "Check the PostgreSQL server version for compatibility verification", "transient": true } } diff --git a/cmd/sling/tests/pipelines/p.27.adjust_column_type_expand.yaml b/cmd/sling/tests/pipelines/p.27.adjust_column_type_expand.yaml new file mode 100644 index 000000000..0f147c314 --- /dev/null +++ b/cmd/sling/tests/pipelines/p.27.adjust_column_type_expand.yaml @@ -0,0 +1,114 @@ +# Test that adjust_column_type expands varchar and decimal columns when +# the source schema widens. Source starts with VARCHAR(50)/NUMERIC(10,2), +# then grows to VARCHAR(200)/NUMERIC(18,6). The target should expand +# column types to match, avoiding truncation or precision loss. + +steps: + # 1. Create source table with narrow types + - id: setup_source + connection: POSTGRES + query: | + DROP TABLE IF EXISTS public.adjust_type_src CASCADE; + CREATE TABLE public.adjust_type_src ( + id INT PRIMARY KEY, + name VARCHAR(50), + amount NUMERIC(10,2), + description VARCHAR(100) + ); + INSERT INTO public.adjust_type_src VALUES + (1, 'short', 123.45, 'short desc'), + (2, 'test', 678.90, 'another desc'); + + # 2. Create target table with same narrow types (simulating existing target) + - id: setup_target + connection: POSTGRES + query: | + DROP TABLE IF EXISTS public.adjust_type_tgt CASCADE; + CREATE TABLE public.adjust_type_tgt ( + id INT, + name VARCHAR(50), + amount NUMERIC(10,2), + description VARCHAR(100) + ); + INSERT INTO public.adjust_type_tgt VALUES + (1, 'short', 123.45, 'short desc'), + (2, 'test', 678.90, 'another desc'); + + # 3. Widen source columns and insert data that needs wider columns + - id: widen_source + connection: POSTGRES + query: | + ALTER TABLE public.adjust_type_src ALTER COLUMN name TYPE VARCHAR(200); + ALTER TABLE public.adjust_type_src ALTER COLUMN amount TYPE NUMERIC(18,6); + ALTER TABLE public.adjust_type_src ALTER COLUMN description TYPE VARCHAR(500); + INSERT INTO public.adjust_type_src VALUES + (3, 'this is a much longer name that exceeds fifty characters easily and needs expansion', 123456789.123456, 'this description is long enough to exceed one hundred characters and requires the target column to be expanded to avoid any truncation errors in the database'); + + - log: "Source setup complete: narrow types widened, wide data inserted" + + # 4. Run replication with adjust_column_type enabled + - id: replicate + replication: + source: POSTGRES + target: POSTGRES + streams: + public.adjust_type_src: + object: public.adjust_type_tgt + mode: full-refresh + target_options: + adjust_column_type: true + on_failure: abort + + - log: "Replication completed" + + # 5. Verify row count + - connection: POSTGRES + query: SELECT COUNT(*) as count FROM public.adjust_type_tgt + into: target_count + + - check: int_parse(store.target_count[0].count) == 3 + failure_message: "Expected 3 rows, got {store.target_count[0].count}" + + # 6. Verify the wide data was inserted without truncation + - connection: POSTGRES + query: SELECT id, name, amount, description FROM public.adjust_type_tgt WHERE id = 3 + into: wide_row + + - log: | + Wide row data: + name length: {length(store.wide_row[0].name)} + amount: {store.wide_row[0].amount} + description length: {length(store.wide_row[0].description)} + + # Verify string was not truncated (original is 85 chars, wider than VARCHAR(50)) + - check: length(store.wide_row[0].name) > 50 + failure_message: "Name was truncated: length={length(store.wide_row[0].name)}, expected > 50" + + # Verify decimal precision was preserved + - check: float_parse(store.wide_row[0].amount) == 123456789.123456 + failure_message: "Amount precision lost: got {store.wide_row[0].amount}, expected 123456789.123456" + + # Verify description was not truncated (original is > 100 chars) + - check: length(store.wide_row[0].description) > 100 + failure_message: "Description was truncated: length={length(store.wide_row[0].description)}, expected > 100" + + # 7. Verify target column types were expanded + - connection: POSTGRES + query: | + SELECT column_name, data_type, character_maximum_length, numeric_precision, numeric_scale + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'adjust_type_tgt' + ORDER BY ordinal_position + into: target_cols + + - log: | + Target column types after replication: + {pretty_table(store.target_cols)} + + - log: "SUCCESS: adjust_column_type correctly expanded varchar and decimal columns" + + # 8. Clean up + - connection: POSTGRES + query: | + DROP TABLE IF EXISTS public.adjust_type_src CASCADE; + DROP TABLE IF EXISTS public.adjust_type_tgt CASCADE; diff --git a/cmd/sling/tests/suite.cli.yaml b/cmd/sling/tests/suite.cli.yaml index 722295893..aa4ccef92 100644 --- a/cmd/sling/tests/suite.cli.yaml +++ b/cmd/sling/tests/suite.cli.yaml @@ -2193,4 +2193,13 @@ run: 'sling run -d -p cmd/sling/tests/pipelines/p.26.duckdb_arrow_ipc_output.yaml' output_contains: - 'SUCCESS: DuckDB Arrow IPC output produced correct row count' - - 'DuckDB Arrow IPC output test complete' \ No newline at end of file + - 'DuckDB Arrow IPC output test complete' + +# adjust_column_type: expand varchar and decimal columns when source types widen +# Source starts with VARCHAR(50)/NUMERIC(10,2), then grows to VARCHAR(200)/NUMERIC(18,6). +# The target should expand column types to match, avoiding truncation or precision loss. +- id: 229 + name: 'adjust_column_type expands varchar and decimal columns' + run: 'sling run -d -p cmd/sling/tests/pipelines/p.27.adjust_column_type_expand.yaml' + output_contains: + - 'SUCCESS: adjust_column_type correctly expanded varchar and decimal columns' \ No newline at end of file diff --git a/core/dbio/database/database.go b/core/dbio/database/database.go index cb60de3d7..8572d5e6d 100755 --- a/core/dbio/database/database.go +++ b/core/dbio/database/database.go @@ -3295,57 +3295,76 @@ func GetOptimizeTableStatements(conn Connection, table *Table, newColumns iop.Co newCol, ok := newColumnsMap[strings.ToLower(col.Name)] if !ok { continue - } else if col.Type == newCol.Type { - continue } - msg := g.F("optimizing existing '%s' (%s) vs new '%s' (%s) => ", col.Name, col.Type, newCol.Name, newCol.Type) - switch { - case col.Type.IsDecimal() && newCol.Type.IsDecimal(): - continue - case col.Type.IsDatetime() && newCol.Type.IsDatetime(): - newCol.Type = iop.TimestampType - case col.Type.IsDatetime() && newCol.Type.IsDate(): - newCol.Type = iop.TimestampType - case col.Type.IsInteger() && newCol.Type.IsDecimal(): - newCol.Type = iop.DecimalType - case col.Type.IsInteger() && newCol.Type.IsFloat(): - newCol.Type = iop.FloatType - case col.Type.IsDecimal() && newCol.Type.IsInteger(): - newCol.Type = iop.DecimalType - case col.Type.IsInteger() && newCol.Type == iop.BigIntType: - newCol.Type = iop.BigIntType - case col.Type == iop.BigIntType && newCol.Type.IsInteger(): - newCol.Type = iop.BigIntType - case col.Type == iop.SmallIntType && newCol.Type == iop.IntegerType: - newCol.Type = iop.IntegerType - case col.Type == iop.IntegerType && newCol.Type == iop.SmallIntType: - newCol.Type = iop.IntegerType - case col.Type.IsInteger() && newCol.Type.IsBool(): - // integer and bool are compatible when bool_as is integer - // check if the database stores bools as integers - if conn.GetTemplateValue("variable.bool_as") == "integer" { - continue // no change needed, keep integer type + + needTypeExpansion := false + if col.Type == newCol.Type { + // Same general type — check if precision/length needs expanding + switch { + case col.IsString() && col.Sourced && newCol.Sourced && col.DbPrecision > 0 && newCol.DbPrecision > 0: + if newCol.DbPrecision > col.DbPrecision { + needTypeExpansion = true // source varchar grew, need to expand target + } + case col.IsDecimal() && col.Sourced && newCol.Sourced: + if newCol.DbPrecision > col.DbPrecision || newCol.DbScale > col.DbScale { + // use max of both for safety + newCol.DbPrecision = max(col.DbPrecision, newCol.DbPrecision) + newCol.DbScale = max(col.DbScale, newCol.DbScale) + needTypeExpansion = true + } } - newCol.Type = iop.StringType // otherwise convert to string - case col.Type.IsBool() && newCol.Type.IsInteger(): - // bool and integer are compatible when bool_as is integer - if conn.GetTemplateValue("variable.bool_as") == "integer" { - continue // no change needed, keep bool type + if !needTypeExpansion { + continue } - newCol.Type = iop.StringType // otherwise convert to string - case isTemp && col.IsString() && newCol.HasNulls() && (newCol.IsDatetime() || newCol.IsDate() || newCol.IsNumber() || newCol.IsBool()): - // use new type - case col.Type == iop.TextType || newCol.Type == iop.TextType: - newCol.Type = iop.TextType - default: - newCol.Type = iop.StringType } - if col.Type == newCol.Type { - continue - } + if !needTypeExpansion { + switch { + case col.Type.IsDecimal() && newCol.Type.IsDecimal(): + continue + case col.Type.IsDatetime() && newCol.Type.IsDatetime(): + newCol.Type = iop.TimestampType + case col.Type.IsDatetime() && newCol.Type.IsDate(): + newCol.Type = iop.TimestampType + case col.Type.IsInteger() && newCol.Type.IsDecimal(): + newCol.Type = iop.DecimalType + case col.Type.IsInteger() && newCol.Type.IsFloat(): + newCol.Type = iop.FloatType + case col.Type.IsDecimal() && newCol.Type.IsInteger(): + newCol.Type = iop.DecimalType + case col.Type.IsInteger() && newCol.Type == iop.BigIntType: + newCol.Type = iop.BigIntType + case col.Type == iop.BigIntType && newCol.Type.IsInteger(): + newCol.Type = iop.BigIntType + case col.Type == iop.SmallIntType && newCol.Type == iop.IntegerType: + newCol.Type = iop.IntegerType + case col.Type == iop.IntegerType && newCol.Type == iop.SmallIntType: + newCol.Type = iop.IntegerType + case col.Type.IsInteger() && newCol.Type.IsBool(): + // integer and bool are compatible when bool_as is integer + // check if the database stores bools as integers + if conn.GetTemplateValue("variable.bool_as") == "integer" { + continue // no change needed, keep integer type + } + newCol.Type = iop.StringType // otherwise convert to string + case col.Type.IsBool() && newCol.Type.IsInteger(): + // bool and integer are compatible when bool_as is integer + if conn.GetTemplateValue("variable.bool_as") == "integer" { + continue // no change needed, keep bool type + } + newCol.Type = iop.StringType // otherwise convert to string + case isTemp && col.IsString() && newCol.HasNulls() && (newCol.IsDatetime() || newCol.IsDate() || newCol.IsNumber() || newCol.IsBool()): + // use new type + case col.Type == iop.TextType || newCol.Type == iop.TextType: + newCol.Type = iop.TextType + default: + newCol.Type = iop.StringType + } - g.Debug(msg + string(newCol.Type)) + if col.Type == newCol.Type { + continue + } + } oldNativeType, err := conn.GetNativeType(col) if err != nil { @@ -3361,7 +3380,11 @@ func GetOptimizeTableStatements(conn Connection, table *Table, newColumns iop.Co continue } + g.Debug("optimizing existing '%s' (%s) vs new '%s' (%s) => %s", col.Name, col.Type, newCol.Name, newCol.Type, newNativeType) + table.Columns[i].Type = newCol.Type + table.Columns[i].DbPrecision = newCol.DbPrecision + table.Columns[i].DbScale = newCol.DbScale table.Columns[i].DbType = newNativeType colsChanging = append(colsChanging, table.Columns[i]) oldCols = append(oldCols, col) diff --git a/core/dbio/database/database_oracle.go b/core/dbio/database/database_oracle.go index 9661f7a32..d0757d376 100755 --- a/core/dbio/database/database_oracle.go +++ b/core/dbio/database/database_oracle.go @@ -128,7 +128,17 @@ func (conn *OracleConn) ConnString() string { // infinite timeout by default options := map[string]string{"TIMEOUT": "0"} + // When service_name is explicitly provided, do not forward `sid` as an + // option — go-ora's ConnectionConfig prefers SID over ServiceName when + // both are set, which causes ORA-12505 against PDBs/services like + // FREEPDB1. The `sid` prop is often auto-populated from the URL path + // (see connection.setURL), so dropping it here is the safe choice when + // the caller asked for service-name semantics. + serviceName := conn.GetProp("service_name") for key, new_key := range propMapping { + if key == "sid" && serviceName != "" { + continue + } if val := conn.GetProp(key); val != "" { options[new_key] = val } diff --git a/core/dbio/database/optimize_table_test.go b/core/dbio/database/optimize_table_test.go new file mode 100644 index 000000000..faabc5f4e --- /dev/null +++ b/core/dbio/database/optimize_table_test.go @@ -0,0 +1,249 @@ +package database + +import ( + "testing" + + "github.com/slingdata-io/sling-cli/core/dbio/iop" + "github.com/stretchr/testify/assert" +) + +// getOptimizeTestConn returns a Postgres connection for testing. +func getOptimizeTestConn(t *testing.T) Connection { + t.Helper() + + db := DBs["postgres"] + conn, err := connect(db) + if err != nil { + t.Skip("POSTGRES connection not available: " + err.Error()) + return nil + } + return conn +} + +// TestGetOptimizeTableStatements_WithinTypeExpansion tests that +// GetOptimizeTableStatements detects and handles within-type expansion +// (e.g., VARCHAR(100) -> VARCHAR(500), DECIMAL(10,2) -> DECIMAL(18,6)) +func TestGetOptimizeTableStatements_WithinTypeExpansion(t *testing.T) { + + conn := getOptimizeTestConn(t) + if conn == nil { + return + } + defer conn.Close() + + tests := []struct { + name string + tableCols iop.Columns // existing table columns (target) + newCols iop.Columns // incoming columns (source) + expectAlter bool // whether ALTER should be generated + desc string + }{ + { + name: "string_expansion", + tableCols: iop.Columns{ + {Name: "name", Type: iop.StringType, DbPrecision: 100, Sourced: true}, + }, + newCols: iop.Columns{ + {Name: "name", Type: iop.StringType, DbPrecision: 500, Sourced: true}, + }, + expectAlter: true, + desc: "VARCHAR(100) -> VARCHAR(500) should expand", + }, + { + name: "string_no_shrink", + tableCols: iop.Columns{ + {Name: "name", Type: iop.StringType, DbPrecision: 500, Sourced: true}, + }, + newCols: iop.Columns{ + {Name: "name", Type: iop.StringType, DbPrecision: 100, Sourced: true}, + }, + expectAlter: false, + desc: "VARCHAR(500) -> VARCHAR(100) should NOT shrink", + }, + { + name: "string_equal", + tableCols: iop.Columns{ + {Name: "name", Type: iop.StringType, DbPrecision: 255, Sourced: true}, + }, + newCols: iop.Columns{ + {Name: "name", Type: iop.StringType, DbPrecision: 255, Sourced: true}, + }, + expectAlter: false, + desc: "VARCHAR(255) -> VARCHAR(255) should be no-op", + }, + { + name: "string_not_sourced", + tableCols: iop.Columns{ + {Name: "name", Type: iop.StringType, DbPrecision: 100, Sourced: true}, + }, + newCols: iop.Columns{ + {Name: "name", Type: iop.StringType, DbPrecision: 500, Sourced: false}, + }, + expectAlter: false, + desc: "Non-sourced new column should not trigger expansion", + }, + { + name: "decimal_precision_expansion", + tableCols: iop.Columns{ + {Name: "amount", Type: iop.DecimalType, DbPrecision: 10, DbScale: 2, Sourced: true}, + }, + newCols: iop.Columns{ + {Name: "amount", Type: iop.DecimalType, DbPrecision: 18, DbScale: 2, Sourced: true}, + }, + expectAlter: true, + desc: "DECIMAL(10,2) -> DECIMAL(18,2) should expand precision", + }, + { + name: "decimal_scale_expansion", + tableCols: iop.Columns{ + {Name: "amount", Type: iop.DecimalType, DbPrecision: 10, DbScale: 2, Sourced: true}, + }, + newCols: iop.Columns{ + {Name: "amount", Type: iop.DecimalType, DbPrecision: 10, DbScale: 6, Sourced: true}, + }, + expectAlter: true, + desc: "DECIMAL(10,2) -> DECIMAL(10,6) should expand scale", + }, + { + name: "decimal_both_expansion", + tableCols: iop.Columns{ + {Name: "amount", Type: iop.DecimalType, DbPrecision: 10, DbScale: 2, Sourced: true}, + }, + newCols: iop.Columns{ + {Name: "amount", Type: iop.DecimalType, DbPrecision: 18, DbScale: 6, Sourced: true}, + }, + expectAlter: true, + desc: "DECIMAL(10,2) -> DECIMAL(18,6) should expand both", + }, + { + name: "decimal_no_shrink", + tableCols: iop.Columns{ + {Name: "amount", Type: iop.DecimalType, DbPrecision: 18, DbScale: 6, Sourced: true}, + }, + newCols: iop.Columns{ + {Name: "amount", Type: iop.DecimalType, DbPrecision: 10, DbScale: 2, Sourced: true}, + }, + expectAlter: false, + desc: "DECIMAL(18,6) -> DECIMAL(10,2) should NOT shrink", + }, + { + name: "decimal_equal", + tableCols: iop.Columns{ + {Name: "amount", Type: iop.DecimalType, DbPrecision: 10, DbScale: 2, Sourced: true}, + }, + newCols: iop.Columns{ + {Name: "amount", Type: iop.DecimalType, DbPrecision: 10, DbScale: 2, Sourced: true}, + }, + expectAlter: false, + desc: "DECIMAL(10,2) -> DECIMAL(10,2) should be no-op", + }, + { + name: "decimal_not_sourced", + tableCols: iop.Columns{ + {Name: "amount", Type: iop.DecimalType, DbPrecision: 10, DbScale: 2, Sourced: true}, + }, + newCols: iop.Columns{ + {Name: "amount", Type: iop.DecimalType, DbPrecision: 18, DbScale: 6, Sourced: false}, + }, + expectAlter: false, + desc: "Non-sourced new column should not trigger expansion", + }, + { + name: "decimal_mixed_expansion", + tableCols: iop.Columns{ + {Name: "amount", Type: iop.DecimalType, DbPrecision: 18, DbScale: 2, Sourced: true}, + }, + newCols: iop.Columns{ + {Name: "amount", Type: iop.DecimalType, DbPrecision: 10, DbScale: 6, Sourced: true}, + }, + expectAlter: true, + desc: "DECIMAL(18,2) -> DECIMAL(10,6): scale grew, should expand to max(18,10),max(2,6)", + }, + { + name: "cross_type_int_to_bigint", + tableCols: iop.Columns{ + {Name: "id", Type: iop.IntegerType, Sourced: true}, + }, + newCols: iop.Columns{ + {Name: "id", Type: iop.BigIntType, Sourced: true}, + }, + expectAlter: true, + desc: "INT -> BIGINT cross-type promotion should still work", + }, + { + name: "cross_type_int_to_decimal", + tableCols: iop.Columns{ + {Name: "val", Type: iop.IntegerType, Sourced: true}, + }, + newCols: iop.Columns{ + {Name: "val", Type: iop.DecimalType, DbPrecision: 10, DbScale: 2, Sourced: true}, + }, + expectAlter: true, + desc: "INT -> DECIMAL cross-type promotion should still work", + }, + { + name: "multi_column_mixed", + tableCols: iop.Columns{ + {Name: "name", Type: iop.StringType, DbPrecision: 100, Sourced: true}, + {Name: "amount", Type: iop.DecimalType, DbPrecision: 10, DbScale: 2, Sourced: true}, + {Name: "id", Type: iop.IntegerType, Sourced: true}, + }, + newCols: iop.Columns{ + {Name: "name", Type: iop.StringType, DbPrecision: 500, Sourced: true}, + {Name: "amount", Type: iop.DecimalType, DbPrecision: 18, DbScale: 6, Sourced: true}, + {Name: "id", Type: iop.IntegerType, Sourced: true}, + }, + expectAlter: true, + desc: "Multiple columns: string expansion + decimal expansion + no-change integer", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + table := Table{ + Columns: tt.tableCols, + } + + ok, ddlParts, err := GetOptimizeTableStatements(conn, &table, tt.newCols, false) + assert.NoError(t, err, tt.desc) + + if tt.expectAlter { + assert.True(t, ok, "%s: expected ALTER to be generated", tt.desc) + assert.NotEmpty(t, ddlParts, "%s: expected DDL parts", tt.desc) + } else { + assert.False(t, ok, "%s: expected no ALTER", tt.desc) + assert.Empty(t, ddlParts, "%s: expected no DDL parts", tt.desc) + } + }) + } +} + +// TestGetOptimizeTableStatements_DecimalMaxPrecision verifies that when +// decimal expansion happens, the resulting column uses max(old, new) for +// both precision and scale. +func TestGetOptimizeTableStatements_DecimalMaxPrecision(t *testing.T) { + + conn := getOptimizeTestConn(t) + if conn == nil { + return + } + defer conn.Close() + + // DECIMAL(18,2) -> DECIMAL(10,6): precision stays 18, scale expands to 6 + table := Table{ + Columns: iop.Columns{ + {Name: "amount", Type: iop.DecimalType, DbPrecision: 18, DbScale: 2, Sourced: true}, + }, + } + newCols := iop.Columns{ + {Name: "amount", Type: iop.DecimalType, DbPrecision: 10, DbScale: 6, Sourced: true}, + } + + ok, _, err := GetOptimizeTableStatements(conn, &table, newCols, false) + assert.NoError(t, err) + assert.True(t, ok, "expected ALTER for scale expansion") + + // Verify the table column was updated with max values + assert.Equal(t, 18, table.Columns[0].DbPrecision, "precision should be max(18,10) = 18") + assert.Equal(t, 6, table.Columns[0].DbScale, "scale should be max(2,6) = 6") +} diff --git a/core/dbio/dbio_types.go b/core/dbio/dbio_types.go index 024962134..df6aedc57 100644 --- a/core/dbio/dbio_types.go +++ b/core/dbio/dbio_types.go @@ -256,6 +256,11 @@ func (t Type) IsMySQLLike() bool { return g.In(t, TypeDbMySQL, TypeDbMariaDB, TypeDbStarRocks) } +// IsPostgresLike returns true if postgres flavor +func (t Type) IsPostgresLike() bool { + return g.In(t, TypeDbPostgres) +} + // IsSQLServer returns true is sql server flavor func (t Type) IsSQLServer() bool { return g.In(t, TypeDbSQLServer, TypeDbAzure, TypeDbAzureDWH, TypeDbFabric) diff --git a/core/dbio/iop/datatype.go b/core/dbio/iop/datatype.go index 9b552019b..b258fd24c 100755 --- a/core/dbio/iop/datatype.go +++ b/core/dbio/iop/datatype.go @@ -1534,14 +1534,14 @@ remap: } } - nativeType = strings.ReplaceAll( - nativeType, - "(,)", - fmt.Sprintf("(%d,%d)", precision, scale), - ) - - // BigQuery: use BIGNUMERIC if scale > 9 or precision > 38 - if t == dbio.TypeDbBigQuery && strings.EqualFold(nativeType, "numeric") && + if strings.Contains(nativeType, "(,)") { + nativeType = strings.ReplaceAll( + nativeType, + "(,)", + fmt.Sprintf("(%d,%d)", precision, scale), + ) + } else if t == dbio.TypeDbBigQuery && strings.EqualFold(nativeType, "numeric") && + // BigQuery: use BIGNUMERIC if scale > 9 or precision > 38 (scale > 9 || precision > 38) { nativeType = "bignumeric" } diff --git a/core/sling/task.go b/core/sling/task.go index 129dce839..04db8d4cf 100644 --- a/core/sling/task.go +++ b/core/sling/task.go @@ -130,6 +130,9 @@ func (t *TaskExecution) SetProgress(text string, args ...interface{}) { progressText := g.F(text, args...) t.ProgressHist = append(t.ProgressHist, progressText) t.Progress = progressText + if t.Replication == nil { + return + } if !t.PBar.started || t.PBar.finished { if strings.Contains(text, "execution failed") { text = env.RedString(text) diff --git a/go.mod b/go.mod index d82731a0d..da0496917 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/slingdata-io/sling-cli -go 1.25.0 +go 1.25.5 require ( cloud.google.com/go v0.121.6 @@ -44,7 +44,7 @@ require ( github.com/integrii/flaggy v1.5.2 github.com/itchyny/gojq v0.12.18 github.com/itchyny/timefmt-go v0.1.7 - github.com/jackc/pgx/v5 v5.7.6 + github.com/jackc/pgx/v5 v5.9.1 github.com/jaswdr/faker v1.19.1 github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/jinzhu/copier v0.4.0 @@ -63,6 +63,7 @@ require ( github.com/mattn/go-isatty v0.0.20 github.com/mattn/go-sqlite3 v1.14.28 github.com/microsoft/go-mssqldb v1.9.3 + github.com/nikolalohinski/gonja/v2 v2.5.2 github.com/nqd/flat v0.1.1 github.com/orcaman/concurrent-map/v2 v2.0.1 github.com/parquet-go/parquet-go v0.23.0 @@ -78,6 +79,7 @@ require ( github.com/shopspring/decimal v1.4.0 github.com/sijms/go-ora/v2 v2.8.24 github.com/slingdata-io/godbc v0.0.5 + github.com/slingdata-io/golyglot v1.0.1 github.com/slingdata-io/sling v0.0.0-20240426022644-3c31b1eb088e github.com/snowflakedb/gosnowflake v1.17.1 github.com/spf13/cast v1.7.1 @@ -94,11 +96,11 @@ require ( go.opentelemetry.io/otel/log v0.14.0 go.opentelemetry.io/otel/sdk v1.39.0 go.opentelemetry.io/otel/sdk/log v0.14.0 - golang.org/x/crypto v0.46.0 - golang.org/x/net v0.48.0 + golang.org/x/crypto v0.48.0 + golang.org/x/net v0.50.0 golang.org/x/oauth2 v0.34.0 - golang.org/x/term v0.38.0 - golang.org/x/text v0.32.0 + golang.org/x/term v0.40.0 + golang.org/x/text v0.35.0 google.golang.org/api v0.258.0 gopkg.in/cheggaaa/pb.v2 v2.0.7 gopkg.in/yaml.v2 v2.4.0 @@ -163,6 +165,7 @@ require ( github.com/clipperhouse/uax29/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect github.com/cockroachdb/apd/v3 v3.2.1 // indirect + github.com/coder/websocket v1.8.14 // indirect github.com/containerd/console v1.0.5 // indirect github.com/coreos/go-oidc/v3 v3.5.0 // indirect github.com/creack/pty v1.1.18 // indirect @@ -226,6 +229,8 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/imdario/mergo v0.3.16 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect @@ -345,12 +350,12 @@ require ( gocloud.dev v0.41.0 // indirect golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect golang.org/x/image v0.25.0 // indirect - golang.org/x/mod v0.31.0 // indirect - golang.org/x/sync v0.19.0 // indirect - golang.org/x/sys v0.39.0 // indirect - golang.org/x/telemetry v0.0.0-20251203150158-8fff8a5912fc // indirect + golang.org/x/mod v0.33.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 // indirect golang.org/x/time v0.14.0 // indirect - golang.org/x/tools v0.40.0 // indirect + golang.org/x/tools v0.42.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/genproto v0.0.0-20250603155806-513f23925822 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect @@ -376,6 +381,8 @@ replace github.com/flarco/g => ../g replace github.com/slingdata-io/sling => ../sling +// replace github.com/slingdata-io/golyglot => ../golyglot + replace github.com/apache/iceberg-go => github.com/flarco/iceberg-go v0.0.0-20260105175128-f16b74585ee2 // replace github.com/apache/iceberg-go => ../iceberg-go diff --git a/go.sum b/go.sum index 8d1a38cfc..436e7d06c 100644 --- a/go.sum +++ b/go.sum @@ -107,6 +107,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0/go.mod h1:jUZ5LYlw40WMd07qxcQJD5M40aUxrfwqQX1g7zxYnrQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo= +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= +github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs= github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8= github.com/MarvinJWendt/testza v0.2.8/go.mod h1:nwIcjmr0Zz+Rcwfh3/4UhBp7ePKVhuBExvZqnKYWlII= @@ -234,6 +236,8 @@ github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4= github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg= github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc= +github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g= +github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/compose-spec/compose-go/v2 v2.6.0 h1:/+oBD2ixSENOeN/TlJqWZmUak0xM8A7J08w/z661Wd4= github.com/compose-spec/compose-go/v2 v2.6.0/go.mod h1:vPlkN0i+0LjLf9rv52lodNMUTJF5YHVfHVGLLIP67NA= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= @@ -414,6 +418,8 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= @@ -497,8 +503,8 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= -github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= @@ -577,6 +583,8 @@ github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510 h1:+PJCokZ2BhyDKlncScmiNzBwqOx+yH1i8xRlWN/wn6A= +github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510/go.mod h1:UzTJ5Jjuf4O9hYWW+HYVwVldYz9J7CaePW0iuNJkrPQ= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= @@ -587,8 +595,8 @@ github.com/jackc/pgtype v1.14.4 h1:fKuNiCumbKTAIxQwXfB/nsrnkEI6bPJrrSiMKgbJ2j8= github.com/jackc/pgtype v1.14.4/go.mod h1:aKeozOde08iifGosdJpz9MBZonJOUJxqNpPBcMJTlVA= github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA= github.com/jackc/pgx/v4 v4.18.3/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= -github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= -github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jaswdr/faker v1.19.1 h1:xBoz8/O6r0QAR8eEvKJZMdofxiRH+F0M/7MU9eNKhsM= @@ -798,12 +806,18 @@ github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJm github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nikolalohinski/gonja/v2 v2.5.2 h1:ZiQxf5ImXYse2oL8U5wROiIYjf3kCbkrAv55tN4NRmM= +github.com/nikolalohinski/gonja/v2 v2.5.2/go.mod h1:UIzXPVuOsr5h7dZ5DUbqk3/Z7oFA/NLGQGMjqT4L2aU= github.com/nqd/flat v0.1.1 h1:sKa3CZipbb7WYD9tORSJD6Ylm/00f6D9Wse7+UkSa+4= github.com/nqd/flat v0.1.1/go.mod h1:FOuslZmNY082wVfVUUb7qAGWKl8z8Nor9FMg+Xj2Nss= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= +github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= +github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= +github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= @@ -960,6 +974,8 @@ github.com/slingdata-io/arrow-adbc/go/adbc v0.0.0-20260225105818-efcf366e7dd4 h1 github.com/slingdata-io/arrow-adbc/go/adbc v0.0.0-20260225105818-efcf366e7dd4/go.mod h1:ikEb6zQgczMp7Alx/RYGaBEn/Mwwz1kfXbirnkbgWUo= github.com/slingdata-io/godbc v0.0.5 h1:DsKO/esOWddXPsdCWpf2reIHT5gIB+01almqr437WWk= github.com/slingdata-io/godbc v0.0.5/go.mod h1:oBLg0zDZSK1BhLpcNhZjsvCLdfvB4OiTReAreXCrb3M= +github.com/slingdata-io/golyglot v1.0.1 h1:8ADzP1DRZ1zsT4g2R5WNNmmCm26jqevmj+61zlImbII= +github.com/slingdata-io/golyglot v1.0.1/go.mod h1:Gw0MF6BgUt9XUWAxYahbTsfWzaTdQQoSccUXbOxlGUs= github.com/slingdata-io/pocketbase v0.22.136 h1:RtAvPvYdK0qm9EB1r8GzNeEfSiqDK+tV8jyxwbpKKBA= github.com/slingdata-io/pocketbase v0.22.136/go.mod h1:RYAdoMZtW+3OIgKqg+YhgWGIiwjtcBHGxRcVF2+1klA= github.com/snowflakedb/gosnowflake v1.17.1 h1:sBYExPDRv6hHF7fCqeXMT745L326Byw/cROxvCiEJzo= @@ -1168,6 +1184,8 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= +go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= @@ -1199,8 +1217,8 @@ golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1m golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= -golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= @@ -1220,8 +1238,8 @@ golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= -golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1253,8 +1271,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= +golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1278,8 +1296,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1325,11 +1343,11 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= -golang.org/x/telemetry v0.0.0-20251203150158-8fff8a5912fc h1:bH6xUXay0AIFMElXG2rQ4uiE+7ncwtiOdPfYK1NK2XA= -golang.org/x/telemetry v0.0.0-20251203150158-8fff8a5912fc/go.mod h1:hKdjCMrbv9skySur+Nek8Hd0uJ0GuxJIoIX2payrIdQ= +golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 h1:bTLqdHv7xrGlFbvf5/TXNxy/iUwwdkjhqQTJDjW7aj0= +golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4/go.mod h1:g5NllXBEermZrmR51cJDQxmJUHUOfRAaNyWBM+R+548= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1343,8 +1361,8 @@ golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= -golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= +golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= +golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -1360,8 +1378,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= @@ -1385,8 +1403,8 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= -golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=