Add testcontainer-based JDBC tests

.github/workflows/pxf-ci.yml

@@ -301,10 +301,142 @@ jobs:
           exit 1
         fi
 
+  # Stage 2b: Testcontainers-based tests
+  build-pxf-testcontainer-image:
+    name: Build PXF Testcontainer Image
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout PXF source
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+
+    - name: Build pxf-cbdb testcontainer image
+      run: |
+        docker build \
+          -t pxf/cbdb-testcontainer:1 \
+          automation/src/main/resources/testcontainers/pxf-cbdb
+        docker save pxf/cbdb-testcontainer:1 > /tmp/pxf-cbdb-testcontainer-image.tar
+
+    - name: Upload pxf-cbdb testcontainer image
+      uses: actions/upload-artifact@v4
+      with:
+        name: pxf-cbdb-testcontainer-image
+        path: /tmp/pxf-cbdb-testcontainer-image.tar
+        retention-days: 1
+
+  pxf-testcontainer-test:
+    name: "TC Test - ${{ matrix.tc_group }} (${{ matrix.test_mode }})"
+    needs: [build-pxf-testcontainer-image]
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - tc_group: jdbc-tc
+            test_mode: external-table
+            use_fdw: "false"
+          - tc_group: jdbc-tc
+            test_mode: fdw
+            use_fdw: "true"
+    steps:
+
+    - name: Checkout PXF source
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+        submodules: true
+
+    - name: Set up JDK ${{ env.JAVA_VERSION }}
+      uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: ${{ env.JAVA_VERSION }}
+
+    - name: Download pxf-cbdb testcontainer image
+      uses: actions/download-artifact@v4
+      with:
+        name: pxf-cbdb-testcontainer-image
+        path: /tmp
+
+    - name: Load pxf-cbdb testcontainer image
+      run: |
+        docker load < /tmp/pxf-cbdb-testcontainer-image.tar
+
+    - name: Build PXF stage artifacts (no unit tests)
+      run: |
+        make -C server stage-notest
+
+    - name: Run Testcontainers tests - ${{ matrix.tc_group }} (${{ matrix.test_mode }})
+      id: run_test
+      continue-on-error: true
+      timeout-minutes: 120
+      working-directory: automation
+      env:
+        PXF_HOME: ${{ github.workspace }}/server/build/stage
+      run: |
+        make test-tc TC_GROUP=${{ matrix.tc_group }} USE_FDW=${{ matrix.use_fdw }}
+
+    - name: Collect artifacts and generate stats
+      if: always()
+      id: collect_artifacts
+      run: |
+        mkdir -p artifacts/logs
+        TC_GROUP="${{ matrix.tc_group }}"
+        TEST_MODE="${{ matrix.test_mode }}"
+        TEST_RESULT="${{ steps.run_test.outcome }}"
+
+        TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0
+        for xml in automation/target/surefire-reports/TEST-*.xml; do
+          if [ -f "$xml" ]; then
+            tests=$(grep -oP 'tests="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0")
+            failures=$(grep -oP 'failures="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0")
+            errors=$(grep -oP 'errors="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0")
+            skipped=$(grep -oP 'skipped="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0")
+            TOTAL=$((TOTAL + tests))
+            FAILED=$((FAILED + failures + errors))
+            SKIPPED=$((SKIPPED + skipped))
+          fi
+        done
+        PASSED=$((TOTAL - FAILED - SKIPPED))
+
+        cat > artifacts/test_stats.json <<EOF
+        {
+          "group": "tc:$TC_GROUP:$TEST_MODE",
+          "result": "$TEST_RESULT",
+          "total": $TOTAL,
+          "passed": $PASSED,
+          "failed": $FAILED,
+          "skipped": $SKIPPED
+        }
+        EOF
+
+        echo "failed_count=$FAILED" >> $GITHUB_OUTPUT
+        echo "skipped_count=$SKIPPED" >> $GITHUB_OUTPUT
+        echo "Test stats for tc:$TC_GROUP ($TEST_MODE): total=$TOTAL, passed=$PASSED, failed=$FAILED, skipped=$SKIPPED"
+
+    - name: Upload test artifacts
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: test-results-tc-${{ matrix.tc_group }}-${{ matrix.test_mode }}
+        path: artifacts/**
+        if-no-files-found: ignore
+        retention-days: 7
+
+    - name: Check test result
+      if: always()
+      run: |
+        FAILED_COUNT="${{ steps.collect_artifacts.outputs.failed_count || 0 }}"
+        if [ "${{ steps.run_test.outcome }}" == "failure" ] || [ "$FAILED_COUNT" -gt 0 ]; then
+          echo "Testcontainer test group ${{ matrix.tc_group }} (${{ matrix.test_mode }}) failed (Failures: $FAILED_COUNT)"
+          exit 1
+        fi
+
   # Stage 3: Summary job
   test-summary:
     name: Test Summary
-    needs: [pxf-test]
+    needs: [pxf-test, pxf-testcontainer-test]
     if: always()
     runs-on: ubuntu-latest
     steps:

automation/Makefile

@@ -22,6 +22,11 @@ ifneq "$(GROUP)" ""
 	MAVEN_TEST_OPTS+= -Dgroups=$(GROUP)
 endif
 
+EXCLUDED_GROUPS ?= testcontainers
+ifneq "$(EXCLUDED_GROUPS)" ""
+	MAVEN_TEST_OPTS+= -DexcludedGroups=$(EXCLUDED_GROUPS)
+endif
+
 MAVEN_TEST_OPTS+= -Djava.awt.headless=true -DuseFDW=$(USE_FDW) -Duser.timezone=UTC
 
 ifneq "$(OFFLINE)" "true"
@@ -94,7 +99,11 @@ MVN=mvn
 all: test
 
 check-env:
-	@if [ -z "$(PXF_HOME)" ]; then echo 'ERROR: PXF_HOME must be set'; exit 1; fi
+	@if [ -z "$(PXF_HOME)" ]; then \
+		echo 'ERROR: PXF_HOME must be set'; \
+		echo 'Example: export PXF_HOME="$(abspath ../server/build/stage)"'; \
+		exit 1; \
+	fi
 
 symlink_pxf_jars: check-env
 	@if [ -d "$(PXF_HOME)/application" ]; then \
@@ -245,6 +254,18 @@ else
 	@ls src/test/java/org/apache/cloudberry/pxf/automation/features/*/*Test.java | sed 's/.*\///g' | sed 's/\.java//g' | awk '{print "* ", $$1}'
 endif
 
+# Run Testcontainers-based tests.
+# Usage:
+#   make test-tc                          => run all testcontainers tests
+#   make test-tc TC_GROUP=jdbc-tc         => run only jdbc-tc group
+.PHONY: test-tc
+test-tc: check-env symlink_pxf_jars pxf_regress
+	$(MVN) -B -e -Djava.awt.headless=true -Duser.timezone=UTC \
+		-DuseFDW=$(USE_FDW) \
+		-Dgroups=$(or $(TC_GROUP),testcontainers) \
+		-DexcludedGroups= \
+		test
+
 .PHONY: pxf_regress
 pxf_regress:
 	$(MAKE) -C pxf_regress

automation/pom.xml

@@ -76,6 +76,12 @@
         </plugins>
 
         <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <includes>
+                    <include>**/*</include>
+                </includes>
+            </resource>
             <resource>
                 <directory>src/test/resources</directory>
                 <includes>
@@ -177,6 +183,30 @@
             <version>4.2.0</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>testcontainers</artifactId>
+            <version>2.0.3</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.17.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <version>2.17.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+            <version>1.26.2</version>
+        </dependency>
+
         <dependency>
             <groupId>org.jsystemtest</groupId>
             <artifactId>jsystemCore</artifactId>
@@ -221,6 +251,13 @@
             <version>42.7.2</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.clickhouse</groupId>
+            <artifactId>clickhouse-jdbc</artifactId>
+            <version>0.9.6</version>
+            <classifier>all</classifier>
+        </dependency>
+
         <dependency>
             <groupId>org.springframework</groupId>
             <artifactId>spring</artifactId>
@@ -262,19 +299,19 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-core</artifactId>
-            <version>2.14.3</version>
+            <version>2.20.2</version>
         </dependency>
 
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.14.3</version>
+            <version>2.20.2</version>
         </dependency>
 
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-annotations</artifactId>
-            <version>2.14.3</version>
+            <version>2.20</version>
         </dependency>
 
         <dependency>

automation/sqlrepo/features/jdbc/clickhouse/read_types/expected/query01.ans

@@ -0,0 +1,47 @@
+-- start_ignore
+-- end_ignore
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- @description ClickHouse JDBC read: expect one row of primitive types from PXF (values compared by regress output)
+SET timezone='utc';
+SET
+
+SET bytea_output='hex';
+SET
+
+SELECT
+    i_int,
+    s_small,
+    b_big,
+    f_float32,
+    d_float64,
+    b_bool,
+    dec,
+    t_text,
+    bin,
+    d_date,
+    d_ts,
+    d_tstz,
+    d_uuid
+FROM pxf_ch_clickhouse_read_types
+LIMIT 1;
+ i_int | s_small | b_big | f_float32 | d_float64 | b_bool |        dec        | t_text |    bin     |   d_date   |         d_ts          |        d_tstz         |                d_uuid                
+-------+---------+-------+-----------+-----------+--------+-------------------+--------+------------+------------+-----------------------+-----------------------+--------------------------------------
+     1 |       2 |     3 |      1.25 | 3.1415926 | t      | 12345.6789012345 | hello  | \x5b36352c36362c36372c36385d | 2020-01-02 | 2020-01-02 03:04:05.006 | 2020-01-02 03:04:05.006+00 | 550e8400-e29b-41d4-a716-446655440000
+(1 row)
+

automation/sqlrepo/features/jdbc/clickhouse/read_types/sql/query01.sql

@@ -0,0 +1,37 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- @description ClickHouse JDBC read: expect one row of primitive types from PXF (values compared by regress output)
+SET timezone='utc';
+SET bytea_output='hex';
+
+SELECT
+    i_int,
+    s_small,
+    b_big,
+    f_float32,
+    d_float64,
+    b_bool,
+    dec,
+    t_text,
+    bin,
+    d_date,
+    d_ts,
+    d_tstz,
+    d_uuid
+FROM pxf_ch_clickhouse_read_types
+LIMIT 1;