From b6863a620b9534832d106dc4fc2245f845d8e26b Mon Sep 17 00:00:00 2001 From: fucang Date: Mon, 16 Mar 2026 16:11:20 +0800 Subject: [PATCH 1/7] feat: provide engine image builder --- examples/tutorial/docker/build.sh | 237 ++++++++++++++++++ .../tutorial/docker/scql-ubuntu.Dockerfile | 14 ++ examples/tutorial/setup.sh | 7 + 3 files changed, 258 insertions(+) create mode 100644 examples/tutorial/docker/build.sh create mode 100644 examples/tutorial/docker/scql-ubuntu.Dockerfile diff --git a/examples/tutorial/docker/build.sh b/examples/tutorial/docker/build.sh new file mode 100644 index 00000000..8bcf2e53 --- /dev/null +++ b/examples/tutorial/docker/build.sh @@ -0,0 +1,237 @@ +#!/bin/bash +# +# Copyright 2026 Ant Group Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -eu + +# Global variables +SCQL_IMAGE=secretflow/scql +IMAGE_TAG=latest +ENABLE_CACHE=false +TARGET_PLATFORM="" +PROTECT_DEV_ENV=false +CONTAINER_ID="" +OVERLAY_VOLUME="" +WORKSPACE_MOUNT_ARGS="" +CACHE_MOUNT_ARGS="" +TMP_PATH="" + +# Display usage information +usage() { + echo "Usage: $0 [-n Name] [-t Tag] [-p Platform] [-c] [-d]" + echo "" + echo "Options:" + echo " -n name, image name, default is \"scql\"" + echo " -t tag, image tag, default is \"latest\"" + echo " -p target platform, default is \"linux/amd64\", support \"linux/arm64\" and \"linux/amd64\"." + echo " -c, enable host disk bazel cache to speedup build process" + echo " -d, protect development environment (use isolated build)" +} + +# Parse command line arguments +parse_arguments() { + while getopts "n:t:p:cd" options; do + case "${options}" in + n) + SCQL_IMAGE=${OPTARG} + ;; + t) + IMAGE_TAG=${OPTARG} + ;; + c) + ENABLE_CACHE=true + ;; + d) + PROTECT_DEV_ENV=true + ;; + p) + TARGET_PLATFORM=${OPTARG} + ;; + *) + usage + exit 1 + ;; + esac + done +} + +# Setup environment variables and directories +setup_environment() { + set -x + + # get work dir + SCRIPT_DIR=$( + cd "$(dirname "$0")" + pwd + ) + + WORK_DIR=$( + cd $SCRIPT_DIR/../../.. + pwd + ) + + echo "build image $SCQL_IMAGE:$IMAGE_TAG" +} + +# Configure Docker mount options and platform settings +configure_docker_settings() { + # Configure cache mount options + CACHE_MOUNT_ARGS="" + if ${ENABLE_CACHE}; then + local cache_vol_name="scql-ubuntu-buildcache" + CACHE_MOUNT_ARGS="--mount type=volume,source=${cache_vol_name},target=/root/.cache" + fi + + # Configure workspace mount strategy based on protection mode + if [ "$PROTECT_DEV_ENV" = "true" ]; then + # Use overlay volume for complete isolation with direct mount + local overlay_volume="scql-build-overlay-$(date +%s)" + local changes_dir="$WORK_DIR/.build_changes" + echo "Creating overlay volume with direct mount: $overlay_volume" + + # Create overlay directories in workspace + mkdir -p "$changes_dir/upper" "$changes_dir/work" + + # Create overlay volume with workspace-relative paths + docker volume create "${overlay_volume}" \ + --driver local \ + --opt type=overlay \ + --opt device=overlay \ + --opt "o=lowerdir=${WORK_DIR},upperdir=${changes_dir}/upper,workdir=${changes_dir}/work" \ + >/dev/null + + OVERLAY_VOLUME="$overlay_volume" + + WORKSPACE_MOUNT_ARGS="--mount type=volume,source=${overlay_volume},target=/home/admin/dev/" + else + # Standard bind mount for non-protected builds + WORKSPACE_MOUNT_ARGS="--mount type=bind,source=${WORK_DIR},target=/home/admin/dev/" + fi + + # Configure platform settings + HOST_PLATFORM="" + case "$(arch)" in + x86_64) HOST_PLATFORM="linux/amd64" ;; + aarch64 | arm64) HOST_PLATFORM="linux/arm64" ;; + *) + echo "Unsupported host architecture: $(arch)" >&2 + exit 1 + ;; + esac + + # If TargetPlatform is not set, set to host platform + TARGET_PLATFORM=${TARGET_PLATFORM:-$HOST_PLATFORM} + + BUILDER=secretflow/scql-ci:latest +} + +# Create Docker container and build engine binary +create_container_and_build() { + echo "Creating Docker container and building engine binary..." + + # Create and start the container + CONTAINER_ID=$(docker run -it --rm --detach \ + ${WORKSPACE_MOUNT_ARGS} \ + -w /home/admin/dev ${CACHE_MOUNT_ARGS} \ + $BUILDER tail -f /dev/null) + + # Setup cleanup trap for both container and build artifacts + trap 'cleanup_on_exit' EXIT + + # Prepare git configuration + docker exec -it ${CONTAINER_ID} bash -c "git config --global --add safe.directory /home/admin/dev" + + # Build engine binary only + docker exec -it ${CONTAINER_ID} bash -c "cd /home/admin/dev && bazelisk --host_jvm_args=-Xmx8g build //engine/exe:scqlengine -c opt --jobs=32" +} + +# Prepare temporary directory and copy build files +prepare_build_files() { + # prepare temporary path for file copies + TMP_PATH=$WORK_DIR/.buildtmp/$IMAGE_TAG + rm -rf $TMP_PATH + mkdir -p $TMP_PATH + mkdir -p $TMP_PATH/$TARGET_PLATFORM + echo "copy files to dir: $TMP_PATH" + + # Copy scqlengine binary + docker cp ${CONTAINER_ID}:/home/admin/dev/bazel-bin/engine/exe/scqlengine $TMP_PATH/$TARGET_PLATFORM + + # copy dockerfile + cp ${SCRIPT_DIR}/scql-ubuntu.Dockerfile $TMP_PATH/Dockerfile +} + +# Build Docker image +build_docker_image() { + cd $TMP_PATH + echo "start to build scql image in $(pwd)" + + # If target == host, no need to use buildx + if [ "$HOST_PLATFORM" == "$TARGET_PLATFORM" ]; then + docker build --build-arg="TARGETPLATFORM=${TARGET_PLATFORM}" -f Dockerfile -t $SCQL_IMAGE:$IMAGE_TAG . + else + docker buildx build --platform $TARGET_PLATFORM -f Dockerfile -t $SCQL_IMAGE:$IMAGE_TAG . + fi +} + +# Cleanup function that runs on exit (success or failure) +cleanup_on_exit() { + echo "Starting cleanup process..." + + # Clean up root-owned directories + if [ "$PROTECT_DEV_ENV" = "true" ] && [ -n "${OVERLAY_VOLUME:-}" ]; then + local changes_dir="$WORK_DIR/.build_changes" + if [ -d "$changes_dir" ]; then + # Use a separate container to clean up root-owned files by mounting the changes directory + docker run --rm --mount type=bind,source="$changes_dir",target=/cleanup alpine sh -c "rm -rf /cleanup/*" 2>/dev/null || true + fi + fi + + # Stop Docker container if it exists + if [ -n "${CONTAINER_ID}" ]; then + echo "Stopping Docker container: ${CONTAINER_ID}" + docker stop ${CONTAINER_ID} 2>/dev/null || true + fi + + # Clean up overlay volume if it was created + if [ "$PROTECT_DEV_ENV" = "true" ] && [ -n "${OVERLAY_VOLUME:-}" ]; then + docker volume rm "${OVERLAY_VOLUME}" 2>/dev/null || true + + local changes_dir="$WORK_DIR/.build_changes" + if [ -d "$changes_dir" ]; then + rm -rf "$changes_dir" 2>/dev/null || true + fi + fi + + # Clean up .buildtmp directory + local buildtmp_dir="$WORK_DIR/.buildtmp" + if [ -d "$buildtmp_dir" ]; then + rm -rf "$buildtmp_dir" + echo "Successfully cleaned up .buildtmp directory" + fi +} + +main() { + parse_arguments "$@" + setup_environment + configure_docker_settings + create_container_and_build + prepare_build_files + build_docker_image + echo "Build completed successfully!" +} + +main "$@" diff --git a/examples/tutorial/docker/scql-ubuntu.Dockerfile b/examples/tutorial/docker/scql-ubuntu.Dockerfile new file mode 100644 index 00000000..6df27ebe --- /dev/null +++ b/examples/tutorial/docker/scql-ubuntu.Dockerfile @@ -0,0 +1,14 @@ +FROM ubuntu:jammy + +ARG TARGETPLATFORM +RUN echo "I'm building for $TARGETPLATFORM" + +ENV TZ=Asia/Shanghai + +RUN apt update \ + && apt upgrade -y \ + && apt install -y libgomp1 \ + && apt install -y curl iputils-ping \ + && apt clean + +COPY ./$TARGETPLATFORM/scqlengine /home/admin/bin/scqlengine diff --git a/examples/tutorial/setup.sh b/examples/tutorial/setup.sh index 60c6f27e..c1791586 100755 --- a/examples/tutorial/setup.sh +++ b/examples/tutorial/setup.sh @@ -32,3 +32,10 @@ PROJECT_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd) cd "${PROJECT_ROOT}" go build -o "${SCRIPT_DIR}/opencore-demo" ./opencore-demo/main.go + +# check if secretflow/scql:latest image exists +if ! docker image inspect secretflow/scql:latest >/dev/null 2>&1; then + echo "WARNING: Docker image secretflow/scql:latest not found." + echo "Please build the image first by running:" + echo " bash ${SCRIPT_DIR}/docker/build.sh" +fi From 25700911a69be82ace229a32b27ca9722f19540e Mon Sep 17 00:00:00 2001 From: fucangfy Date: Wed, 18 Mar 2026 10:15:13 +0800 Subject: [PATCH 2/7] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- examples/tutorial/setup.sh | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/examples/tutorial/setup.sh b/examples/tutorial/setup.sh index c1791586..1c7de5fd 100755 --- a/examples/tutorial/setup.sh +++ b/examples/tutorial/setup.sh @@ -31,11 +31,22 @@ SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) PROJECT_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd) cd "${PROJECT_ROOT}" + +# load environment variables from .env if present (e.g., SCQL_IMAGE) +if [ -f .env ]; then + set -a + . .env + set +a +fi + go build -o "${SCRIPT_DIR}/opencore-demo" ./opencore-demo/main.go -# check if secretflow/scql:latest image exists -if ! docker image inspect secretflow/scql:latest >/dev/null 2>&1; then - echo "WARNING: Docker image secretflow/scql:latest not found." +# determine SCQL image name (default to secretflow/scql:latest if not set) +IMAGE_NAME="${SCQL_IMAGE:-secretflow/scql:latest}" + +# check if the configured SCQL image exists +if ! docker image inspect "${IMAGE_NAME}" >/dev/null 2>&1; then + echo "WARNING: Docker image ${IMAGE_NAME} not found." echo "Please build the image first by running:" echo " bash ${SCRIPT_DIR}/docker/build.sh" fi From 229113250a2c9bacf491f64fa83321185f48b6b5 Mon Sep 17 00:00:00 2001 From: fucangfy Date: Wed, 18 Mar 2026 10:16:06 +0800 Subject: [PATCH 3/7] Update examples/tutorial/docker/build.sh Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- examples/tutorial/docker/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tutorial/docker/build.sh b/examples/tutorial/docker/build.sh index 8bcf2e53..6ca8a138 100644 --- a/examples/tutorial/docker/build.sh +++ b/examples/tutorial/docker/build.sh @@ -143,7 +143,7 @@ create_container_and_build() { echo "Creating Docker container and building engine binary..." # Create and start the container - CONTAINER_ID=$(docker run -it --rm --detach \ + CONTAINER_ID=$(docker run -d --rm \ ${WORKSPACE_MOUNT_ARGS} \ -w /home/admin/dev ${CACHE_MOUNT_ARGS} \ $BUILDER tail -f /dev/null) From 5dd75fda63eefcdf9ebec060fba9051c90142226 Mon Sep 17 00:00:00 2001 From: fucangfy Date: Wed, 18 Mar 2026 10:17:05 +0800 Subject: [PATCH 4/7] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- examples/tutorial/docker/scql-ubuntu.Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/tutorial/docker/scql-ubuntu.Dockerfile b/examples/tutorial/docker/scql-ubuntu.Dockerfile index 6df27ebe..20ef3f09 100644 --- a/examples/tutorial/docker/scql-ubuntu.Dockerfile +++ b/examples/tutorial/docker/scql-ubuntu.Dockerfile @@ -6,9 +6,9 @@ RUN echo "I'm building for $TARGETPLATFORM" ENV TZ=Asia/Shanghai RUN apt update \ - && apt upgrade -y \ - && apt install -y libgomp1 \ - && apt install -y curl iputils-ping \ - && apt clean + && apt install -y --no-install-recommends libgomp1 curl iputils-ping \ + && apt clean \ + && rm -rf /var/lib/apt/lists/* + COPY ./$TARGETPLATFORM/scqlengine /home/admin/bin/scqlengine From 3dde6d8d20748a93dd2db9b0bf9167610b4db94c Mon Sep 17 00:00:00 2001 From: fucangfy Date: Wed, 18 Mar 2026 10:17:42 +0800 Subject: [PATCH 5/7] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- examples/tutorial/docker/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/tutorial/docker/build.sh b/examples/tutorial/docker/build.sh index 6ca8a138..22385c52 100644 --- a/examples/tutorial/docker/build.sh +++ b/examples/tutorial/docker/build.sh @@ -34,9 +34,9 @@ usage() { echo "Usage: $0 [-n Name] [-t Tag] [-p Platform] [-c] [-d]" echo "" echo "Options:" - echo " -n name, image name, default is \"scql\"" + echo " -n name, image name, default is \"secretflow/scql\"" echo " -t tag, image tag, default is \"latest\"" - echo " -p target platform, default is \"linux/amd64\", support \"linux/arm64\" and \"linux/amd64\"." + echo " -p target platform, default is host architecture, support \"linux/arm64\" and \"linux/amd64\"." echo " -c, enable host disk bazel cache to speedup build process" echo " -d, protect development environment (use isolated build)" } From b6347b5d509fcc6f6aad3a524c76445b0afeed7e Mon Sep 17 00:00:00 2001 From: fucangfy Date: Wed, 18 Mar 2026 10:17:49 +0800 Subject: [PATCH 6/7] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- examples/tutorial/docker/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tutorial/docker/build.sh b/examples/tutorial/docker/build.sh index 22385c52..4425fb18 100644 --- a/examples/tutorial/docker/build.sh +++ b/examples/tutorial/docker/build.sh @@ -183,7 +183,7 @@ build_docker_image() { if [ "$HOST_PLATFORM" == "$TARGET_PLATFORM" ]; then docker build --build-arg="TARGETPLATFORM=${TARGET_PLATFORM}" -f Dockerfile -t $SCQL_IMAGE:$IMAGE_TAG . else - docker buildx build --platform $TARGET_PLATFORM -f Dockerfile -t $SCQL_IMAGE:$IMAGE_TAG . + docker buildx build --platform $TARGET_PLATFORM --load -f Dockerfile -t $SCQL_IMAGE:$IMAGE_TAG . fi } From 0e0e312910c1263fcfe68aadf9727de0171c00ed Mon Sep 17 00:00:00 2001 From: fucang Date: Wed, 18 Mar 2026 10:33:23 +0800 Subject: [PATCH 7/7] fix --- examples/tutorial/docker/build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/tutorial/docker/build.sh b/examples/tutorial/docker/build.sh index 4425fb18..c85af6fc 100644 --- a/examples/tutorial/docker/build.sh +++ b/examples/tutorial/docker/build.sh @@ -146,16 +146,17 @@ create_container_and_build() { CONTAINER_ID=$(docker run -d --rm \ ${WORKSPACE_MOUNT_ARGS} \ -w /home/admin/dev ${CACHE_MOUNT_ARGS} \ + --entrypoint "" \ $BUILDER tail -f /dev/null) # Setup cleanup trap for both container and build artifacts trap 'cleanup_on_exit' EXIT # Prepare git configuration - docker exec -it ${CONTAINER_ID} bash -c "git config --global --add safe.directory /home/admin/dev" + docker exec ${CONTAINER_ID} bash -c "git config --global --add safe.directory /home/admin/dev" # Build engine binary only - docker exec -it ${CONTAINER_ID} bash -c "cd /home/admin/dev && bazelisk --host_jvm_args=-Xmx8g build //engine/exe:scqlengine -c opt --jobs=32" + docker exec ${CONTAINER_ID} bash -c "cd /home/admin/dev && bazelisk --host_jvm_args=-Xmx8g build //engine/exe:scqlengine -c opt --jobs=32" } # Prepare temporary directory and copy build files