Merge pull request #713 from slingdata-io/v1.5.10

V1.5.10

Author: flarco

README.md

@@ -195,14 +195,6 @@ If you're using the python wrapper, decompress the downloaded binary and put the
 
 We welcome contributions to improve Sling! Here are some guidelines to help you get started.
 
-### Branch Naming Convention
-
-When creating a new branch for your contribution, please use the following naming convention:
-
-- `feature/your-feature-name` for new features
-- `bugfix/issue-description` for bug fixes
-- `docs/update-description` for documentation updates
-
 ### Testing Guidelines
 
 Sling has three main test suites: Database, File and CLI. When contributing, please ensure that your changes pass the relevant tests.

cmd/sling/resource/llm_CONNECTION.md

@@ -316,6 +316,7 @@ Validate that a connection works properly:
   "action": "test",
   "input": {
     "connection": "MY_POSTGRES",
+    "working_dir": "/optional/work/dir",
     "debug": true,
     "trace": false
   }
@@ -618,6 +619,7 @@ streams:
   "action": "test",
   "input": {
     "connection": "SALESFORCE_API",
+    "working_dir": "/optional/work/dir",
     "endpoints": ["accounts", "contacts"],
     "limit": 10,
     "debug": true
@@ -665,6 +667,7 @@ streams:
   "action": "test",
   "input": {
     "connection": "MY_DB",
+    "working_dir": "/optional/work/dir",
     "debug": true,
     "trace": true
   }

cmd/sling/resource/llm_PIPELINE.md

@@ -68,15 +68,17 @@ The Sling MCP tool provides these pipeline commands:
 {
   "action": "parse",
   "input": {
-    "file_path": "/path/to/pipeline.yaml"
+    "file_path": "/path/to/pipeline.yaml",
+    "working_dir": "/optional/work/dir"
   }
 }
 
 // Run a pipeline
 {
   "action": "run",
   "input": {
-    "file_path": "/path/to/pipeline.yaml"
+    "file_path": "/path/to/pipeline.yaml",
+    "working_dir": "/optional/work/dir"
   }
 }
 ```
@@ -603,7 +605,8 @@ steps:
 {
   "action": "parse",
   "input": {
-    "file_path": "/path/to/pipeline.yaml"
+    "file_path": "/path/to/pipeline.yaml",
+    "working_dir": "/optional/work/dir"
   }
 }
 ```
@@ -614,6 +617,7 @@ steps:
   "action": "run",
   "input": {
     "file_path": "/path/to/pipeline.yaml",
+    "working_dir": "/optional/work/dir",
     "env": {
       "CUSTOM_VAR": "value"
     }

cmd/sling/resource/mcp.yaml

@@ -314,12 +314,14 @@ tools:
     
     **For `action: "parse"`**:
     - `file_path` (string, required): The file path of the API specification.
+    - `working_dir` (string, optional): Working directory to change to before running
     
     **For `action: "docs"`**:
     - No parameters required (empty object)
     
     **For `action: "test"`**:
     - `connection` (string, required): The name of the connection to test
+    - `working_dir` (string, optional): Working directory to change to before running
     - `debug` (boolean, optional): Enable debug logging during the test (default: false)
     - `trace` (boolean, optional): Enable trace logging during the test (default: false)
     - `endpoints` (array, optional): Specify the endpoints to test (only for API connections)
@@ -342,6 +344,7 @@ tools:
       "action": "test",
       "input": {
         "connection": "MY_API",
+        "working_dir": "/path/to/dir",
         "debug": true,
         "limit": 15
       }
@@ -351,7 +354,8 @@ tools:
     {
       "action": "parse",
       "input": {
-        "file_path": "path/to/github_api.yaml"
+        "file_path": "path/to/github_api.yaml",
+        "working_dir": "/path/to/dir"
       }
     }
     ```
@@ -446,9 +450,11 @@ tools:
 
     **For `action: "parse"`**:
     - `file_path` (string, required): Path to the pipeline configuration file to read.
+    - `working_dir` (string, optional): Working directory to change to before reading
     
     **For `action: "run"`**:
     - `file_path` (string, required): Path to the pipeline configuration file to execute.
+    - `working_dir` (string, optional): Working directory to change to before running
     - `env` (object, optional): Environment variables to set for the pipeline run.
     
     **Output**: The output format depends on the action and audience:

cmd/sling/sling_cli.go

@@ -117,6 +117,11 @@ var cliRunFlags = []g.Flag{
 		Type:        "string",
 		Description: "An object/map to specify the type that a column should be cast as (JSON or YAML).",
 	},
+	{
+		Name:        "cdc-options",
+		Type:        "string",
+		Description: "in-line CDC options to configure change-capture mode (JSON or YAML).\n",
+	},
 	{
 		Name:        "streams",
 		ShortName:   "",

cmd/sling/sling_run.go

@@ -45,6 +45,8 @@ func processRun(c *g.CliSC) (ok bool, err error) {
 	cfg := &sling.Config{
 		Source: sling.Source{Options: &sling.SourceOptions{}},
 		Target: sling.Target{Options: &sling.TargetOptions{}},
+		// for CDC options
+		ReplicationStream: &sling.ReplicationStreamConfig{},
 	}
 
 	var replicationCfgPath, pipelineCfgPath, directoryPath string
@@ -208,6 +210,17 @@ func processRun(c *g.CliSC) (ok bool, err error) {
 			selectStreams = strings.Split(cast.ToString(v), ",")
 		case "examples":
 			showExamples = cast.ToBool(v)
+		case "cdc-options":
+			payload := cast.ToString(v)
+			options, err := parsePayload(payload, true)
+			if err != nil {
+				return ok, g.Error(err, "invalid cdc options -> %s", payload)
+			}
+
+			err = g.JSONConvert(options, &cfg.ReplicationStream.CDCOptions)
+			if err != nil {
+				return ok, g.Error(err, "invalid cdc options -> %s", payload)
+			}
 		}
 	}
 

cmd/sling/sling_update.go

@@ -221,25 +221,27 @@ func checkUpdate() {
 	)
 	respMap := map[string]string{}
 	g.JSONUnmarshal(respB, &respMap)
-	if time.Now().Second()%6 == 0 && len(respMap) > 0 {
+	if len(respMap) > 0 {
 		updateVersion = respMap["version_latest"]
-		if isDevChannel {
-			if core.VersionSlash() != updateVersion && updateVersion != "" {
-				updateMessage = env.GreenString(g.F("FYI there is a new sling dev build released => %s. You can run `sling update` to download it.", updateVersion))
-			}
-		} else {
-			isNew, err := g.CompareVersions(core.Version, updateVersion)
-			if err != nil {
-				g.Debug("Error comparing versions: %s", err.Error())
-			} else if isNew {
-				updateMessage = env.GreenString(g.F("FYI there is a new sling version released (%s). %s", updateVersion, instruction))
-			}
-		}
-		if message := respMap["media"]; message != "" {
-			if updateMessage != "" {
-				updateMessage = updateMessage + "\n" + env.BlueString(message)
+		if time.Now().Second()%6 == 0 {
+			if isDevChannel {
+				if core.VersionSlash() != updateVersion && updateVersion != "" {
+					updateMessage = env.GreenString(g.F("FYI there is a new sling dev build released => %s. You can run `sling update` to download it.", updateVersion))
+				}
 			} else {
-				updateMessage = env.BlueString(message)
+				isNew, err := g.CompareVersions(core.Version, updateVersion)
+				if err != nil {
+					g.Debug("Error comparing versions: %s", err.Error())
+				} else if isNew {
+					updateMessage = env.GreenString(g.F("FYI there is a new sling version released (%s). %s", updateVersion, instruction))
+				}
+			}
+			if message := respMap["media"]; message != "" {
+				if updateMessage != "" {
+					updateMessage = updateMessage + "\n" + env.BlueString(message)
+				} else {
+					updateMessage = env.BlueString(message)
+				}
 			}
 		}
 	}

cmd/sling/tests/pipelines/adbc/Dockerfile.amd64

@@ -0,0 +1,31 @@
+FROM --platform=linux/amd64 ubuntu:jammy
+
+RUN groupadd -r sling && useradd -r -g sling sling
+
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y curl ca-certificates unzip zstd && \
+    cd /tmp && \
+    curl -sSL -o dm.conda https://conda.anaconda.org/conda-forge/linux-64/libadbc-driver-manager-1.10.0-hb700be7_0.conda && \
+    unzip -o dm.conda && \
+    tar --use-compress-program=unzstd -xf pkg-libadbc-driver-manager-1.10.0-hb700be7_0.tar.zst && \
+    cp lib/libadbc_driver_manager.so* /usr/local/lib/ && \
+    ldconfig && \
+    rm -rf /tmp/* && \
+    apt-get clean && rm -rf /var/lib/apt/lists /var/cache/apt
+
+# Copy sling binary (built externally by run_docker_test.sh)
+COPY --chown=sling:sling sling /usr/local/bin/sling
+RUN chmod 755 /usr/local/bin/sling
+
+# Setup sling user home
+RUN mkdir -p /home/sling && chmod 755 /home/sling && chown sling:sling /home/sling
+USER sling
+
+# Install dbc CLI + DuckDB ADBC driver
+RUN curl -LsSf https://dbc.columnar.tech/install.sh | sh && \
+    /home/sling/.local/bin/dbc install duckdb
+
+ENV PATH="${PATH}:/home/sling/.local/bin"
+ENV SLING_PACKAGE="DOCKER"
+
+ENTRYPOINT ["sling"]

cmd/sling/tests/pipelines/adbc/Dockerfile.arm64

@@ -0,0 +1,31 @@
+FROM --platform=linux/arm64 ubuntu:jammy
+
+RUN groupadd -r sling && useradd -r -g sling sling
+
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y curl ca-certificates unzip zstd && \
+    cd /tmp && \
+    curl -sSL -o dm.conda https://conda.anaconda.org/conda-forge/linux-aarch64/libadbc-driver-manager-1.10.0-hfefdfc9_0.conda && \
+    unzip -o dm.conda && \
+    tar --use-compress-program=unzstd -xf pkg-libadbc-driver-manager-1.10.0-hfefdfc9_0.tar.zst && \
+    cp lib/libadbc_driver_manager.so* /usr/local/lib/ && \
+    ldconfig && \
+    rm -rf /tmp/* && \
+    apt-get clean && rm -rf /var/lib/apt/lists /var/cache/apt
+
+# Copy sling binary (built externally by run_docker_test.sh)
+COPY --chown=sling:sling sling /usr/local/bin/sling
+RUN chmod 755 /usr/local/bin/sling
+
+# Setup sling user home
+RUN mkdir -p /home/sling && chmod 755 /home/sling && chown sling:sling /home/sling
+USER sling
+
+# Install dbc CLI + DuckDB ADBC driver
+RUN curl -LsSf https://dbc.columnar.tech/install.sh | sh && \
+    /home/sling/.local/bin/dbc install duckdb
+
+ENV PATH="${PATH}:/home/sling/.local/bin"
+ENV SLING_PACKAGE="DOCKER"
+
+ENTRYPOINT ["sling"]

cmd/sling/tests/pipelines/adbc/p.40.adbc_duckdb.yaml

@@ -0,0 +1,43 @@
+# ADBC DuckDB regression test
+# Validates: purego struct return fix (Linux), duckdb_adbc_init entrypoint auto-detection
+# Requires: dbc CLI installed, ADBC driver manager, DUCKDB_ADBC and POSTGRES connections
+
+steps:
+  # Write test data into DuckDB via ADBC using inline replication from Postgres
+  - replication:
+      source: postgres
+      target: DUCKDB_ADBC
+      defaults:
+        mode: full-refresh
+      streams:
+        "select 1 as id, 'hello' as name, 42.5 as value":
+          sql: "select 1 as id, 'hello' as name, 42.5 as value"
+          object: main.adbc_test
+
+  - log: "ADBC write completed"
+
+  # Read back from DuckDB via ADBC and verify
+  - type: query
+    connection: DUCKDB_ADBC
+    query: select id, name, value from main.adbc_test order by id
+    into: result
+
+  - log: "ADBC read result: {store.result}"
+
+  - type: check
+    check: int_parse(store.result[0].id) == 1
+    message: "Expected id=1, got {store.result[0].id}"
+
+  - type: check
+    check: store.result[0].name == "hello"
+    message: "Expected name='hello', got {store.result[0].name}"
+
+  - type: check
+    check: float_parse(store.result[0].value) == 42.5
+    message: "Expected value=42.5, got {store.result[0].value}"
+
+  - log: "ADBC DuckDB test PASSED"
+
+  # Cleanup
+  - connection: DUCKDB_ADBC
+    query: DROP TABLE IF EXISTS main.adbc_test