From 972096e8497ce2f1c0dcdb22bc66db915cd8c84b Mon Sep 17 00:00:00 2001 From: Sameeul Samee Date: Thu, 19 Feb 2026 06:46:20 -0500 Subject: [PATCH 1/5] feat: two-stage CI build with cached prereqs and TensorStore FetchContent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a prebuild job (one per OS/arch) that runs before the parallel wheel-build jobs. The prebuild populates two caches: - prereq_cache/: filepattern + pybind11 (keyed on install scripts) - ts_fc_cache/: TensorStore FetchContent source + build state (keyed on root CMakeLists.txt + tensorstore_prebuild/CMakeLists.txt) Wheel-build jobs restore both caches and skip the heavy dependency compilation on cache hits. FETCHCONTENT_BASE_DIR is wired into CMAKE_ARGS for all platforms so cmake reuses the pre-fetched TensorStore source tree. New files: ci-utils/tensorstore_prebuild/ — minimal cmake project for TS-only prebuild ci-utils/prebuild_linux.sh — runs inside manylinux container ci-utils/install_sys_deps_linux.sh — builds zlib/jpeg/png to /usr/local ci-utils/before_all_{linux,macos}.sh / before_all_win.bat — replace inline CIBW_BEFORE_ALL commands Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/publish_pypi.yml | 172 ++++++++++++++---- .github/workflows/wheel_build.yml | 168 ++++++++++++++--- .gitignore | 6 +- ci-utils/before_all_linux.sh | 30 +++ ci-utils/before_all_macos.sh | 22 +++ ci-utils/before_all_win.bat | 19 ++ ci-utils/install_prereq_linux.sh | 20 +- ci-utils/install_sys_deps_linux.sh | 48 +++++ ci-utils/prebuild_linux.sh | 35 ++++ ci-utils/tensorstore_prebuild/CMakeLists.txt | 25 +++ .../ts_prebuild_marker.cc | 6 + 11 files changed, 486 insertions(+), 65 deletions(-) create mode 100755 ci-utils/before_all_linux.sh create mode 100755 ci-utils/before_all_macos.sh create mode 100644 ci-utils/before_all_win.bat create mode 100755 ci-utils/install_sys_deps_linux.sh create mode 100755 ci-utils/prebuild_linux.sh create mode 100644 ci-utils/tensorstore_prebuild/CMakeLists.txt create mode 100644 ci-utils/tensorstore_prebuild/ts_prebuild_marker.cc diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 7c8b2fb..561a105 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -11,9 +11,108 @@ concurrency: cancel-in-progress: true jobs: + # ────────────────────────────────────────────────────────────────────────── + # Stage 1: Prebuild (one job per OS/arch) + # - Builds filepattern + pybind11 → prereq_cache/ + # - Builds TensorStore deps via FetchContent → ts_fc_cache/ + # ────────────────────────────────────────────────────────────────────────── + prebuild: + name: Prebuild on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-22.04 + cibw_archs: auto64 + - os: macos-15-intel + cibw_archs: auto64 + - os: windows-latest + cibw_archs: auto64 + - os: macos-14 + cibw_archs: arm64 + + steps: + - uses: actions/checkout@v3 + name: Check out + + - name: Restore prereq cache + id: prereq-cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + id: ts-cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + # ── Linux: run prebuild inside the same manylinux container cibuildwheel uses ── + - name: Run Linux prebuild in manylinux container + if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && matrix.os == 'ubuntu-22.04' + run: | + docker run --rm \ + -v ${{ github.workspace }}:/project \ + -w /project \ + quay.io/pypa/manylinux_2_28_x86_64 \ + bash ci-utils/prebuild_linux.sh + + # ── macOS: brew + install prereqs + cmake TensorStore prebuild ── + - name: Run macOS prebuild + if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && startsWith(matrix.os, 'macos') + run: | + brew install nasm + if [[ "${{ matrix.os }}" == "macos-14" ]]; then + brew uninstall --ignore-dependencies jpeg-turbo || true + else + brew uninstall --ignore-dependencies openssl || true + sudo rm -rf /usr/local/include/openssl + fi + # Build filepattern + pybind11 (and sys deps zlib/jpeg/png) → prereq_cache/local_install + bash ci-utils/install_prereq_linux.sh ./prereq_cache/local_install + # Build TensorStore using the minimal prebuild cmake project (no Python/pybind11 needed) + CMAKE_FLAGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" + if [[ "${{ matrix.os }}" == "macos-15-intel" ]]; then + CMAKE_FLAGS="${CMAKE_FLAGS} -DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON" + fi + cmake -S ci-utils/tensorstore_prebuild -B /tmp/ts_cmake_prebuild \ + -DCMAKE_BUILD_TYPE=Release ${CMAKE_FLAGS} + cmake --build /tmp/ts_cmake_prebuild -j$(sysctl -n hw.ncpu) + + # ── Windows: MSVC env + NASM + Ninja + install prereqs + cmake TensorStore prebuild ── + - name: Set up MSVC environment (Windows) + if: matrix.os == 'windows-latest' + uses: ilammy/msvc-dev-cmd@v1 + + - name: Add NASM (Windows) + if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && matrix.os == 'windows-latest' + uses: ilammy/setup-nasm@v1 + + - name: Add Ninja (Windows) + if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && matrix.os == 'windows-latest' + uses: seanmiddleditch/gha-setup-ninja@master + + - name: Run Windows prebuild + if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && matrix.os == 'windows-latest' + shell: cmd + run: | + ci-utils\install_prereq_win.bat + if %errorlevel% neq 0 exit /b %errorlevel% + xcopy /E /I /y local_install prereq_cache\local_install + cmake -S ci-utils\tensorstore_prebuild -B C:\ts_cmake_prebuild -DCMAKE_GENERATOR=Ninja -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache -DCMAKE_BUILD_TYPE=Release + if %errorlevel% neq 0 exit /b %errorlevel% + cmake --build C:\ts_cmake_prebuild + if %errorlevel% neq 0 exit /b %errorlevel% + + # ────────────────────────────────────────────────────────────────────────── + # Stage 2: Build wheels (4 parallel jobs per Python version, per OS) + # ────────────────────────────────────────────────────────────────────────── build_wheels: name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} + needs: [prebuild] continue-on-error: true env: MACOSX_DEPLOYMENT_TARGET: "10.15" @@ -27,6 +126,18 @@ jobs: - uses: actions/checkout@v3 name: Check out + - name: Restore prereq cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + - uses: ilammy/msvc-dev-cmd@v1 name: Add MSVS Path @@ -47,7 +158,7 @@ jobs: run: | python -m pip install cibuildwheel delvewheel wheel - - name: Building wheels + - name: Building wheels run: | python -m cibuildwheel --output-dir dist env: @@ -56,36 +167,24 @@ jobs: CIBW_SKIP: "*musllinux*" CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 CIBW_BEFORE_ALL_MACOS: brew install nasm && - bash ci-utils/install_prereq_linux.sh && - mkdir -p /tmp/argolid_bld && brew uninstall --ignore-dependencies openssl || true && sudo rm -rf /usr/local/include/openssl && - cp -r local_install /tmp/argolid_bld - CIBW_BEFORE_ALL_LINUX: curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 -o nasm-2.15.05.tar.bz2 && - tar -xjf nasm-2.15.05.tar.bz2 && - cd nasm-2.15.05 && - ./configure && - make && - make install && - cd .. && - bash ci-utils/install_prereq_linux.sh && - mkdir -p /tmp/argolid_bld && - cp -r local_install /tmp/argolid_bld - CIBW_BEFORE_ALL_WINDOWS: ci-utils\install_prereq_win.bat && - xcopy /E /I /y local_install C:\TEMP\argolid_bld\local_install + bash ci-utils/before_all_macos.sh + CIBW_BEFORE_ALL_LINUX: bash /project/ci-utils/before_all_linux.sh + CIBW_BEFORE_ALL_WINDOWS: ci-utils\before_all_win.bat CIBW_ENVIRONMENT_MACOS: >- MACOSX_DEPLOYMENT_TARGET=11.0 REPAIR_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" - CMAKE_ARGS="-DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw -DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON" + CMAKE_ARGS="-DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw -DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" CIBW_ENVIRONMENT_LINUX: >- LD_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64:$LD_LIBRARY_PATH" - ON_GITHUB="TRUE" + ON_GITHUB="TRUE" ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" - CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON" + CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON -DFETCHCONTENT_BASE_DIR=/project/ts_fc_cache" CIBW_REPAIR_WHEEL_COMMAND_MACOS: DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} && DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} - CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\argolid\\bin;$PATH" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="C:\\TEMP\\argolid_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja" + CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\argolid\\bin;$PATH" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="C:\\TEMP\\argolid_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}" CIBW_ARCHS: ${{ matrix.cibw_archs }} CIBW_BEFORE_TEST_LINUX: dnf -y install maven java @@ -94,7 +193,7 @@ jobs: - name: Install Dependencies run: python -m pip install --upgrade twine requests - + - name: Publish to PyPi run: python -m twine upload dist/*.whl env: @@ -105,6 +204,7 @@ jobs: build_wheels_apple_arm64: name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} + needs: [prebuild] continue-on-error: true env: MACOSX_DEPLOYMENT_TARGET: "11.0" @@ -118,33 +218,43 @@ jobs: - uses: actions/checkout@v3 name: Check out + - name: Restore prereq cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + - uses: actions/setup-python@v4 name: Install Python with: python-version: '3.11' - + - name: Install cibuildwheel run: | python -m pip install cibuildwheel delocate wheel - - name: Building wheels + - name: Building wheels run: | python -m cibuildwheel --output-dir dist env: CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_BUILD_VERBOSITY: 3 CIBW_ARCHS_MACOS: arm64 - CIBW_BEFORE_ALL_MACOS: brew install nasm && - brew uninstall --ignore-dependencies jpeg-turbo && - bash ci-utils/install_prereq_linux.sh && - mkdir -p /tmp/argolid_bld && - cp -r local_install /tmp/argolid_bld + CIBW_BEFORE_ALL_MACOS: brew install nasm && + brew uninstall --ignore-dependencies jpeg-turbo && + bash ci-utils/before_all_macos.sh CIBW_ENVIRONMENT_MACOS: >- MACOSX_DEPLOYMENT_TARGET=11.0 REPAIR_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" - CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON" + CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" CIBW_REPAIR_WHEEL_COMMAND_MACOS: | DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} && MACOSX_DEPLOYMENT_TARGET=11.0 DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} -e libc++ -e libunwind @@ -154,10 +264,10 @@ jobs: - name: Install Dependencies run: python -m pip install --upgrade twine requests - + - name: Publish to PyPi run: python -m twine upload dist/*.whl env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - TWINE_REPOSITORY: pypi \ No newline at end of file + TWINE_REPOSITORY: pypi diff --git a/.github/workflows/wheel_build.yml b/.github/workflows/wheel_build.yml index 02c3138..7addd5e 100644 --- a/.github/workflows/wheel_build.yml +++ b/.github/workflows/wheel_build.yml @@ -9,9 +9,110 @@ concurrency: cancel-in-progress: true jobs: + # ────────────────────────────────────────────────────────────────────────── + # Stage 1: Prebuild (one job per OS/arch) + # - Builds filepattern + pybind11 → prereq_cache/ + # - Builds TensorStore deps via FetchContent → ts_fc_cache/ + # - Both caches are restored by wheel-build jobs below. + # ────────────────────────────────────────────────────────────────────────── + prebuild: + name: Prebuild on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-22.04 + cibw_archs: auto64 + - os: macos-15-intel + cibw_archs: auto64 + - os: windows-latest + cibw_archs: auto64 + - os: macos-14 + cibw_archs: arm64 + + steps: + - uses: actions/checkout@v3 + name: Check out + + - name: Restore prereq cache + id: prereq-cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + id: ts-cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + # ── Linux: run prebuild inside the same manylinux container cibuildwheel uses ── + - name: Run Linux prebuild in manylinux container + if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && matrix.os == 'ubuntu-22.04' + run: | + docker run --rm \ + -v ${{ github.workspace }}:/project \ + -w /project \ + quay.io/pypa/manylinux_2_28_x86_64 \ + bash ci-utils/prebuild_linux.sh + + # ── macOS: brew + install prereqs + cmake TensorStore prebuild ── + - name: Run macOS prebuild + if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && startsWith(matrix.os, 'macos') + run: | + brew install nasm + if [[ "${{ matrix.os }}" == "macos-14" ]]; then + brew uninstall --ignore-dependencies jpeg-turbo || true + else + brew uninstall --ignore-dependencies openssl || true + sudo rm -rf /usr/local/include/openssl + fi + # Build filepattern + pybind11 (and sys deps zlib/jpeg/png) → prereq_cache/local_install + bash ci-utils/install_prereq_linux.sh ./prereq_cache/local_install + # Build TensorStore using the minimal prebuild cmake project (no Python/pybind11 needed) + CMAKE_FLAGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" + if [[ "${{ matrix.os }}" == "macos-15-intel" ]]; then + CMAKE_FLAGS="${CMAKE_FLAGS} -DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON" + fi + cmake -S ci-utils/tensorstore_prebuild -B /tmp/ts_cmake_prebuild \ + -DCMAKE_BUILD_TYPE=Release ${CMAKE_FLAGS} + cmake --build /tmp/ts_cmake_prebuild -j$(sysctl -n hw.ncpu) + + # ── Windows: MSVC env + NASM + Ninja + install prereqs + cmake TensorStore prebuild ── + - name: Set up MSVC environment (Windows) + if: matrix.os == 'windows-latest' + uses: ilammy/msvc-dev-cmd@v1 + + - name: Add NASM (Windows) + if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && matrix.os == 'windows-latest' + uses: ilammy/setup-nasm@v1 + + - name: Add Ninja (Windows) + if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && matrix.os == 'windows-latest' + uses: seanmiddleditch/gha-setup-ninja@master + + - name: Run Windows prebuild + if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && matrix.os == 'windows-latest' + shell: cmd + run: | + ci-utils\install_prereq_win.bat + if %errorlevel% neq 0 exit /b %errorlevel% + xcopy /E /I /y local_install prereq_cache\local_install + cmake -S ci-utils\tensorstore_prebuild -B C:\ts_cmake_prebuild -DCMAKE_GENERATOR=Ninja -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache -DCMAKE_BUILD_TYPE=Release + if %errorlevel% neq 0 exit /b %errorlevel% + cmake --build C:\ts_cmake_prebuild + if %errorlevel% neq 0 exit /b %errorlevel% + + # ────────────────────────────────────────────────────────────────────────── + # Stage 2: Build wheels (4 parallel jobs per Python version, per OS) + # Each job restores both caches from the prebuild stage above. + # ────────────────────────────────────────────────────────────────────────── build_wheels: name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} + needs: [prebuild] continue-on-error: true env: @@ -26,6 +127,18 @@ jobs: - uses: actions/checkout@v3 name: Check out + - name: Restore prereq cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + - uses: ilammy/msvc-dev-cmd@v1 name: Add MSVS Path @@ -46,7 +159,7 @@ jobs: run: | python -m pip install cibuildwheel delvewheel wheel - - name: Building wheels + - name: Building wheels run: | python -m cibuildwheel --output-dir dist env: @@ -55,36 +168,24 @@ jobs: CIBW_SKIP: "*musllinux*" CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 CIBW_BEFORE_ALL_MACOS: brew install nasm && - bash ci-utils/install_prereq_linux.sh && - mkdir -p /tmp/argolid_bld && brew uninstall --ignore-dependencies openssl || true && sudo rm -rf /usr/local/include/openssl && - cp -r local_install /tmp/argolid_bld - CIBW_BEFORE_ALL_LINUX: curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 -o nasm-2.15.05.tar.bz2 && - tar -xjf nasm-2.15.05.tar.bz2 && - cd nasm-2.15.05 && - ./configure && - make && - make install && - cd .. && - bash ci-utils/install_prereq_linux.sh && - mkdir -p /tmp/argolid_bld && - cp -r local_install /tmp/argolid_bld - CIBW_BEFORE_ALL_WINDOWS: ci-utils\install_prereq_win.bat && - xcopy /E /I /y local_install C:\TEMP\argolid_bld\local_install + bash ci-utils/before_all_macos.sh + CIBW_BEFORE_ALL_LINUX: bash /project/ci-utils/before_all_linux.sh + CIBW_BEFORE_ALL_WINDOWS: ci-utils\before_all_win.bat CIBW_ENVIRONMENT_MACOS: >- MACOSX_DEPLOYMENT_TARGET=11.0 REPAIR_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" - CMAKE_ARGS="-DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw -DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON" + CMAKE_ARGS="-DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw -DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" CIBW_ENVIRONMENT_LINUX: >- LD_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64:$LD_LIBRARY_PATH" - ON_GITHUB="TRUE" + ON_GITHUB="TRUE" ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" - CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON" + CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON -DFETCHCONTENT_BASE_DIR=/project/ts_fc_cache" CIBW_REPAIR_WHEEL_COMMAND_MACOS: DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} && DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} - CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\argolid\\bin;$PATH" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="C:\\TEMP\\argolid_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja" + CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\argolid\\bin;$PATH" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="C:\\TEMP\\argolid_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}" CIBW_ARCHS: ${{ matrix.cibw_archs }} CIBW_BEFORE_TEST_LINUX: dnf -y install maven java @@ -101,6 +202,7 @@ jobs: build_wheels_apple_arm64: name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} + needs: [prebuild] continue-on-error: true env: MACOSX_DEPLOYMENT_TARGET: "11.0" @@ -114,33 +216,43 @@ jobs: - uses: actions/checkout@v3 name: Check out + - name: Restore prereq cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + - uses: actions/setup-python@v4 name: Install Python with: python-version: '3.11' - + - name: Install cibuildwheel run: | python -m pip install cibuildwheel delocate wheel - - name: Building wheels + - name: Building wheels run: | python -m cibuildwheel --output-dir dist env: CIBW_BUILD: ${{ matrix.cibw_build }}-* CIBW_BUILD_VERBOSITY: 3 CIBW_ARCHS_MACOS: arm64 - CIBW_BEFORE_ALL_MACOS: brew install nasm && + CIBW_BEFORE_ALL_MACOS: brew install nasm && brew uninstall --ignore-dependencies jpeg-turbo && - bash ci-utils/install_prereq_linux.sh && - mkdir -p /tmp/argolid_bld && - cp -r local_install /tmp/argolid_bld + bash ci-utils/before_all_macos.sh CIBW_ENVIRONMENT_MACOS: >- MACOSX_DEPLOYMENT_TARGET=11.0 REPAIR_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" - CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON" + CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" CIBW_REPAIR_WHEEL_COMMAND_MACOS: | DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} && MACOSX_DEPLOYMENT_TARGET=11.0 DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} -e libc++ -e libunwind @@ -153,4 +265,4 @@ jobs: with: name: argolid-wheels-apple-arm64-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ matrix.cibw_build }} path: dist/*.whl - retention-days: 1 \ No newline at end of file + retention-days: 1 diff --git a/.gitignore b/.gitignore index cce0baa..4ad6e9d 100644 --- a/.gitignore +++ b/.gitignore @@ -562,9 +562,13 @@ dmypy.json # Cython debug symbols cython_debug/ -# scratch +# scratch scratch/ +# CI build caches (populated by prebuild job, restored by wheel-build jobs) +prereq_cache/ +ts_fc_cache/ + build/ build_man/ venv/ diff --git a/ci-utils/before_all_linux.sh b/ci-utils/before_all_linux.sh new file mode 100755 index 0000000..859de49 --- /dev/null +++ b/ci-utils/before_all_linux.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# before_all_linux.sh +# Replaces the inline CIBW_BEFORE_ALL_LINUX command. +# Runs inside the manylinux container during the wheel-build job. +# Expects: +# /project/prereq_cache/local_install — populated by prebuild job (may be absent on cache miss) +set -e + +cd /project + +# Always install NASM (required by TensorStore's aom/dav1d deps) +curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 \ + -o nasm.tar.bz2 +tar -xjf nasm.tar.bz2 +cd nasm-2.15.05 && ./configure && make -j"$(nproc)" && make install +cd /project + +mkdir -p /tmp/argolid_bld + +if [ -d "/project/prereq_cache/local_install" ]; then + echo "==> Prereq cache hit: restoring filepattern + pybind11" + cp -r /project/prereq_cache/local_install /tmp/argolid_bld/local_install + # zlib/jpeg/png cannot be cached from /usr/local (container-internal path), + # so rebuild them quickly from source (~1 min total). + bash ci-utils/install_sys_deps_linux.sh +else + echo "==> Prereq cache miss: building filepattern + pybind11 + sys deps from scratch" + bash ci-utils/install_prereq_linux.sh + cp -r /project/local_install /tmp/argolid_bld/local_install +fi diff --git a/ci-utils/before_all_macos.sh b/ci-utils/before_all_macos.sh new file mode 100755 index 0000000..bef1520 --- /dev/null +++ b/ci-utils/before_all_macos.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# before_all_macos.sh +# Replaces the inline CIBW_BEFORE_ALL_MACOS commands for wheel-build jobs. +# brew uninstalls (openssl for intel, jpeg-turbo for arm64) should be done +# by the caller (CIBW_BEFORE_ALL inline) BEFORE invoking this script. +# Expects: +# prereq_cache/local_install — populated by prebuild job (may be absent on cache miss) +set -e + +mkdir -p /tmp/argolid_bld + +if [ -d "prereq_cache/local_install" ]; then + echo "==> Prereq cache hit: restoring filepattern + pybind11" + cp -r prereq_cache/local_install /tmp/argolid_bld/local_install + # Rebuild sys deps (zlib/jpeg/png) to /usr/local — these are fast and + # cannot be cached across jobs since they live outside the workspace. + bash ci-utils/install_sys_deps_linux.sh +else + echo "==> Prereq cache miss: building filepattern + pybind11 + sys deps from scratch" + bash ci-utils/install_prereq_linux.sh + cp -r local_install /tmp/argolid_bld/local_install +fi diff --git a/ci-utils/before_all_win.bat b/ci-utils/before_all_win.bat new file mode 100644 index 0000000..1e9223d --- /dev/null +++ b/ci-utils/before_all_win.bat @@ -0,0 +1,19 @@ +@echo off +REM before_all_win.bat +REM Replaces the inline CIBW_BEFORE_ALL_WINDOWS command for wheel-build jobs. +REM Expects: +REM prereq_cache\local_install — populated by prebuild job (may be absent on cache miss) + +if exist prereq_cache\local_install ( + echo =^> Prereq cache hit: restoring filepattern + pybind11 + xcopy /E /I /y prereq_cache\local_install C:\TEMP\argolid_bld\local_install + REM Replicate the DLL copy that install_prereq_win.bat does for PATH + if exist prereq_cache\local_install\bin ( + xcopy /E /I /y prereq_cache\local_install\bin %TEMP%\argolid\bin + ) +) else ( + echo =^> Prereq cache miss: building filepattern + pybind11 from scratch + ci-utils\install_prereq_win.bat + if errorlevel 1 exit 1 + xcopy /E /I /y local_install C:\TEMP\argolid_bld\local_install +) diff --git a/ci-utils/install_prereq_linux.sh b/ci-utils/install_prereq_linux.sh index 4b233bd..86ef95e 100755 --- a/ci-utils/install_prereq_linux.sh +++ b/ci-utils/install_prereq_linux.sh @@ -1,8 +1,16 @@ #!/bin/bash -# Usage: $bash install_prereq_linux.sh $INSTALL_DIR +# Usage: bash install_prereq_linux.sh [$INSTALL_DIR] [--skip-sys-deps] # Default $INSTALL_DIR = ./local_install +# --skip-sys-deps: skip building zlib, libjpeg-turbo, libpng (use when already installed) # -if [ -z "$1" ] +SKIP_SYS_DEPS=false +for arg in "$@"; do + case "$arg" in + --skip-sys-deps) SKIP_SYS_DEPS=true ;; + esac +done + +if [ -z "$1" ] || [ "$1" = "--skip-sys-deps" ] then echo "No path to the Argolid source location provided" echo "Creating local_install directory" @@ -34,14 +42,15 @@ make install -j4 cd ../../ +if [ "$SKIP_SYS_DEPS" != "true" ]; then curl -L https://github.com/madler/zlib/releases/download/v1.3.1/zlib131.zip -o zlib131.zip unzip zlib131.zip cd zlib-1.3.1 mkdir build_man cd build_man -cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local .. -cmake --build . -cmake --build . --target install +cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local .. +cmake --build . +cmake --build . --target install cd ../../ curl -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/refs/tags/3.1.0.zip -o 3.1.0.zip @@ -67,3 +76,4 @@ cd build_man cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local .. make install -j4 cd ../../ +fi diff --git a/ci-utils/install_sys_deps_linux.sh b/ci-utils/install_sys_deps_linux.sh new file mode 100755 index 0000000..24a74fb --- /dev/null +++ b/ci-utils/install_sys_deps_linux.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# install_sys_deps_linux.sh +# Builds and installs zlib, libjpeg-turbo, and libpng to /usr/local. +# Works on Linux (manylinux) and macOS. +# These are required by TensorStore when built with TENSORSTORE_USE_SYSTEM_* flags. +set -e + +JOBS=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) +BUILD_TMPDIR="$(mktemp -d)" + +# zlib +curl -L https://github.com/madler/zlib/releases/download/v1.3.1/zlib131.zip -o "${BUILD_TMPDIR}/zlib131.zip" +unzip -q "${BUILD_TMPDIR}/zlib131.zip" -d "${BUILD_TMPDIR}" +mkdir -p "${BUILD_TMPDIR}/zlib-1.3.1/build_man" +cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local \ + -S "${BUILD_TMPDIR}/zlib-1.3.1" -B "${BUILD_TMPDIR}/zlib-1.3.1/build_man" +cmake --build "${BUILD_TMPDIR}/zlib-1.3.1/build_man" -j"${JOBS}" +cmake --build "${BUILD_TMPDIR}/zlib-1.3.1/build_man" --target install + +# libjpeg-turbo +curl -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/refs/tags/3.1.0.zip \ + -o "${BUILD_TMPDIR}/3.1.0.zip" +unzip -q "${BUILD_TMPDIR}/3.1.0.zip" -d "${BUILD_TMPDIR}" +mkdir -p "${BUILD_TMPDIR}/libjpeg-turbo-3.1.0/build_man" +cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_STATIC=FALSE \ + -DCMAKE_BUILD_TYPE=Release \ + -S "${BUILD_TMPDIR}/libjpeg-turbo-3.1.0" \ + -B "${BUILD_TMPDIR}/libjpeg-turbo-3.1.0/build_man" +if [[ "$OSTYPE" == "darwin"* ]]; then + sudo cmake --build "${BUILD_TMPDIR}/libjpeg-turbo-3.1.0/build_man" \ + --target install -j"${JOBS}" +else + cmake --build "${BUILD_TMPDIR}/libjpeg-turbo-3.1.0/build_man" \ + --target install -j"${JOBS}" +fi + +# libpng +curl -L https://github.com/glennrp/libpng/archive/refs/tags/v1.6.53.zip \ + -o "${BUILD_TMPDIR}/v1.6.53.zip" +unzip -q "${BUILD_TMPDIR}/v1.6.53.zip" -d "${BUILD_TMPDIR}" +mkdir -p "${BUILD_TMPDIR}/libpng-1.6.53/build_man" +cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local \ + -S "${BUILD_TMPDIR}/libpng-1.6.53" \ + -B "${BUILD_TMPDIR}/libpng-1.6.53/build_man" +cmake --build "${BUILD_TMPDIR}/libpng-1.6.53/build_man" \ + --target install -j"${JOBS}" + +rm -rf "${BUILD_TMPDIR}" diff --git a/ci-utils/prebuild_linux.sh b/ci-utils/prebuild_linux.sh new file mode 100755 index 0000000..372300c --- /dev/null +++ b/ci-utils/prebuild_linux.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# prebuild_linux.sh +# Runs inside the manylinux_2_28_x86_64 Docker container during the prebuild job. +# Populates: +# /project/prereq_cache/local_install — filepattern + pybind11 headers/libs +# /project/ts_fc_cache/ — TensorStore FetchContent source + build +set -e + +cd /project + +# Install NASM (required by aom/dav1d inside TensorStore) +curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 \ + -o nasm.tar.bz2 +tar -xjf nasm.tar.bz2 +cd nasm-2.15.05 && ./configure && make -j"$(nproc)" && make install +cd /project + +# Build zlib, libjpeg-turbo, libpng to /usr/local (needed for TENSORSTORE_USE_SYSTEM_* flags) +bash ci-utils/install_sys_deps_linux.sh + +# Build filepattern + pybind11 → prereq_cache/local_install +# (--skip-sys-deps: sys deps already installed above) +bash ci-utils/install_prereq_linux.sh ./prereq_cache/local_install --skip-sys-deps + +# Pre-build TensorStore using the minimal cmake project (no Python, no pybind11 needed). +# FETCHCONTENT_BASE_DIR stores TensorStore source + build state in ts_fc_cache/ +# so wheel-build jobs can skip the expensive TensorStore download and reuse cmake state. +cmake -S /project/ci-utils/tensorstore_prebuild \ + -B /tmp/ts_cmake_prebuild \ + -DFETCHCONTENT_BASE_DIR=/project/ts_fc_cache \ + -DTENSORSTORE_USE_SYSTEM_JPEG=ON \ + -DTENSORSTORE_USE_SYSTEM_ZLIB=ON \ + -DTENSORSTORE_USE_SYSTEM_PNG=ON \ + -DCMAKE_BUILD_TYPE=Release +cmake --build /tmp/ts_cmake_prebuild -j"$(nproc)" diff --git a/ci-utils/tensorstore_prebuild/CMakeLists.txt b/ci-utils/tensorstore_prebuild/CMakeLists.txt new file mode 100644 index 0000000..972a200 --- /dev/null +++ b/ci-utils/tensorstore_prebuild/CMakeLists.txt @@ -0,0 +1,25 @@ +cmake_minimum_required(VERSION 3.24) +project(TensorStorePrebuild) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +include(FetchContent) + +# Keep URL + SHA256 in sync with root CMakeLists.txt +# Cache key hashes both this file and root CMakeLists.txt +FetchContent_Declare( + tensorstore + URL "https://github.com/google/tensorstore/archive/refs/tags/v0.1.80.tar.gz" + URL_HASH SHA256=94866de34b6139d77d30e828a50f9e8df98e7dd68e848393470879aeb50ea7bf +) + +FetchContent_MakeAvailable(tensorstore) + +# Executable target forces TensorStore and all its transitive deps to be compiled. +# No Python, no pybind11, no filepattern needed. +add_executable(ts_prebuild_marker ts_prebuild_marker.cc) +target_link_libraries(ts_prebuild_marker PRIVATE + tensorstore::tensorstore + tensorstore::all_drivers +) diff --git a/ci-utils/tensorstore_prebuild/ts_prebuild_marker.cc b/ci-utils/tensorstore_prebuild/ts_prebuild_marker.cc new file mode 100644 index 0000000..9ac9b7d --- /dev/null +++ b/ci-utils/tensorstore_prebuild/ts_prebuild_marker.cc @@ -0,0 +1,6 @@ +#include "tensorstore/open.h" + +// Minimal stub that forces the linker to pull in TensorStore symbols. +// This file exists solely to make ts_prebuild_marker a non-trivial executable +// so cmake compiles all TensorStore transitive dependencies during the prebuild. +int main() { return 0; } From 6fb24e8f73642999825309cbd2a8056a424f7249 Mon Sep 17 00:00:00 2001 From: Sameeul Samee Date: Thu, 19 Feb 2026 07:33:15 -0500 Subject: [PATCH 2/5] fix: use CALL for bat file invocations and add --parallel to Windows cmake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without CALL, invoking a .bat file from within another batch context transfers execution permanently — the caller never resumes. This caused the xcopy and cmake TensorStore steps to be silently skipped after install_prereq_win.bat completed. Also adds --parallel to cmake --build on Windows to match the -j$(nproc) flag used on Linux/macOS, and explicitly copies DLLs to %TEMP%\argolid\bin in before_all_win.bat's cache-miss path rather than relying on the conditional copy inside install_prereq_win.bat. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/publish_pypi.yml | 4 ++-- .github/workflows/wheel_build.yml | 4 ++-- ci-utils/before_all_win.bat | 7 ++++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 561a105..e008def 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -98,12 +98,12 @@ jobs: if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && matrix.os == 'windows-latest' shell: cmd run: | - ci-utils\install_prereq_win.bat + call ci-utils\install_prereq_win.bat if %errorlevel% neq 0 exit /b %errorlevel% xcopy /E /I /y local_install prereq_cache\local_install cmake -S ci-utils\tensorstore_prebuild -B C:\ts_cmake_prebuild -DCMAKE_GENERATOR=Ninja -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache -DCMAKE_BUILD_TYPE=Release if %errorlevel% neq 0 exit /b %errorlevel% - cmake --build C:\ts_cmake_prebuild + cmake --build C:\ts_cmake_prebuild --parallel if %errorlevel% neq 0 exit /b %errorlevel% # ────────────────────────────────────────────────────────────────────────── diff --git a/.github/workflows/wheel_build.yml b/.github/workflows/wheel_build.yml index 7addd5e..e269068 100644 --- a/.github/workflows/wheel_build.yml +++ b/.github/workflows/wheel_build.yml @@ -97,12 +97,12 @@ jobs: if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && matrix.os == 'windows-latest' shell: cmd run: | - ci-utils\install_prereq_win.bat + call ci-utils\install_prereq_win.bat if %errorlevel% neq 0 exit /b %errorlevel% xcopy /E /I /y local_install prereq_cache\local_install cmake -S ci-utils\tensorstore_prebuild -B C:\ts_cmake_prebuild -DCMAKE_GENERATOR=Ninja -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache -DCMAKE_BUILD_TYPE=Release if %errorlevel% neq 0 exit /b %errorlevel% - cmake --build C:\ts_cmake_prebuild + cmake --build C:\ts_cmake_prebuild --parallel if %errorlevel% neq 0 exit /b %errorlevel% # ────────────────────────────────────────────────────────────────────────── diff --git a/ci-utils/before_all_win.bat b/ci-utils/before_all_win.bat index 1e9223d..4c3deab 100644 --- a/ci-utils/before_all_win.bat +++ b/ci-utils/before_all_win.bat @@ -13,7 +13,12 @@ if exist prereq_cache\local_install ( ) ) else ( echo =^> Prereq cache miss: building filepattern + pybind11 from scratch - ci-utils\install_prereq_win.bat + call ci-utils\install_prereq_win.bat if errorlevel 1 exit 1 xcopy /E /I /y local_install C:\TEMP\argolid_bld\local_install + REM Explicitly copy DLLs to PATH location (don't rely on install_prereq_win.bat's + REM conditional copy which requires ON_GITHUB to be already set) + if exist local_install\bin ( + xcopy /E /I /y local_install\bin %TEMP%\argolid\bin + ) ) From 21cb0724a0f0f81a667d01389748741603bbad37 Mon Sep 17 00:00:00 2001 From: Sameeul Samee Date: Thu, 19 Feb 2026 07:40:49 -0500 Subject: [PATCH 3/5] refactor: decouple prebuild and wheel-build jobs per OS Previously a single prebuild matrix job blocked all wheel-build jobs until every OS finished. Now each OS has its own named prebuild job (prebuild_linux, prebuild_macos_intel, prebuild_windows, prebuild_macos_arm64) and its own wheel-build job that depends only on its matching prebuild. Linux wheels start building as soon as the Linux prebuild is done, independently of macOS/Windows. The build_wheels matrix is also split into per-OS jobs (build_wheels_linux, build_wheels_macos_intel, build_wheels_windows, build_wheels_apple_arm64), each retaining its cibw_build matrix for the 4 Python versions. Cache keys are hardcoded per job rather than derived from matrix variables. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/publish_pypi.yml | 333 +++++++++++++++++++--------- .github/workflows/wheel_build.yml | 340 ++++++++++++++++++++--------- 2 files changed, 468 insertions(+), 205 deletions(-) diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index e008def..5bc95a4 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -12,46 +12,31 @@ concurrency: jobs: # ────────────────────────────────────────────────────────────────────────── - # Stage 1: Prebuild (one job per OS/arch) - # - Builds filepattern + pybind11 → prereq_cache/ - # - Builds TensorStore deps via FetchContent → ts_fc_cache/ + # Stage 1 — Prebuild jobs (one per OS/arch, run in parallel) # ────────────────────────────────────────────────────────────────────────── - prebuild: - name: Prebuild on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - include: - - os: ubuntu-22.04 - cibw_archs: auto64 - - os: macos-15-intel - cibw_archs: auto64 - - os: windows-latest - cibw_archs: auto64 - - os: macos-14 - cibw_archs: arm64 + prebuild_linux: + name: Prebuild (Linux) + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - name: Check out - name: Restore prereq cache id: prereq-cache uses: actions/cache@v4 with: path: prereq_cache - key: prereq-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + key: prereq-ubuntu-22.04-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} - name: Restore TensorStore FetchContent cache id: ts-cache uses: actions/cache@v4 with: path: ts_fc_cache - key: ts-fc-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + key: ts-fc-ubuntu-22.04-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} - # ── Linux: run prebuild inside the same manylinux container cibuildwheel uses ── - name: Run Linux prebuild in manylinux container - if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && matrix.os == 'ubuntu-22.04' + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' run: | docker run --rm \ -v ${{ github.workspace }}:/project \ @@ -59,43 +44,75 @@ jobs: quay.io/pypa/manylinux_2_28_x86_64 \ bash ci-utils/prebuild_linux.sh - # ── macOS: brew + install prereqs + cmake TensorStore prebuild ── - - name: Run macOS prebuild - if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && startsWith(matrix.os, 'macos') + prebuild_macos_intel: + name: Prebuild (macOS intel) + runs-on: macos-15-intel + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + id: prereq-cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-macos-15-intel-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + id: ts-cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-macos-15-intel-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - name: Run macOS intel prebuild + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' run: | brew install nasm - if [[ "${{ matrix.os }}" == "macos-14" ]]; then - brew uninstall --ignore-dependencies jpeg-turbo || true - else - brew uninstall --ignore-dependencies openssl || true - sudo rm -rf /usr/local/include/openssl - fi - # Build filepattern + pybind11 (and sys deps zlib/jpeg/png) → prereq_cache/local_install + brew uninstall --ignore-dependencies openssl || true + sudo rm -rf /usr/local/include/openssl bash ci-utils/install_prereq_linux.sh ./prereq_cache/local_install - # Build TensorStore using the minimal prebuild cmake project (no Python/pybind11 needed) - CMAKE_FLAGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" - if [[ "${{ matrix.os }}" == "macos-15-intel" ]]; then - CMAKE_FLAGS="${CMAKE_FLAGS} -DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON" - fi cmake -S ci-utils/tensorstore_prebuild -B /tmp/ts_cmake_prebuild \ - -DCMAKE_BUILD_TYPE=Release ${CMAKE_FLAGS} + -DCMAKE_BUILD_TYPE=Release \ + -DTENSORSTORE_USE_SYSTEM_JPEG=ON \ + -DTENSORSTORE_USE_SYSTEM_ZLIB=ON \ + -DTENSORSTORE_USE_SYSTEM_PNG=ON \ + -DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw \ + -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache cmake --build /tmp/ts_cmake_prebuild -j$(sysctl -n hw.ncpu) - # ── Windows: MSVC env + NASM + Ninja + install prereqs + cmake TensorStore prebuild ── - - name: Set up MSVC environment (Windows) - if: matrix.os == 'windows-latest' + prebuild_windows: + name: Prebuild (Windows) + runs-on: windows-latest + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + id: prereq-cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-windows-latest-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + id: ts-cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-windows-latest-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - name: Set up MSVC environment uses: ilammy/msvc-dev-cmd@v1 - - name: Add NASM (Windows) - if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && matrix.os == 'windows-latest' + - name: Add NASM + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' uses: ilammy/setup-nasm@v1 - - name: Add Ninja (Windows) - if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && matrix.os == 'windows-latest' + - name: Add Ninja + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' uses: seanmiddleditch/gha-setup-ninja@master - name: Run Windows prebuild - if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && matrix.os == 'windows-latest' + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' shell: cmd run: | call ci-utils\install_prereq_win.bat @@ -106,90 +123,212 @@ jobs: cmake --build C:\ts_cmake_prebuild --parallel if %errorlevel% neq 0 exit /b %errorlevel% + prebuild_macos_arm64: + name: Prebuild (macOS arm64) + runs-on: macos-14 + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + id: prereq-cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-macos-14-arm64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + id: ts-cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-macos-14-arm64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - name: Run macOS arm64 prebuild + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' + run: | + brew install nasm + brew uninstall --ignore-dependencies jpeg-turbo || true + bash ci-utils/install_prereq_linux.sh ./prereq_cache/local_install + cmake -S ci-utils/tensorstore_prebuild -B /tmp/ts_cmake_prebuild \ + -DCMAKE_BUILD_TYPE=Release \ + -DTENSORSTORE_USE_SYSTEM_JPEG=ON \ + -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache + cmake --build /tmp/ts_cmake_prebuild -j$(sysctl -n hw.ncpu) + # ────────────────────────────────────────────────────────────────────────── - # Stage 2: Build wheels (4 parallel jobs per Python version, per OS) + # Stage 2 — Wheel-build + publish jobs (4 Python versions per OS, in parallel) + # Each job depends only on its own OS's prebuild. # ────────────────────────────────────────────────────────────────────────── - build_wheels: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - needs: [prebuild] + + build_wheels_linux: + name: Build wheels (Linux, ${{ matrix.cibw_build }}) + runs-on: ubuntu-22.04 + needs: [prebuild_linux] continue-on-error: true - env: - MACOSX_DEPLOYMENT_TARGET: "10.15" strategy: matrix: - os: [ubuntu-22.04, macos-15-intel, windows-latest] - cibw_archs: ["auto64"] cibw_build: ["cp310-*", "cp311-*", "cp312-*", "cp313-*"] steps: - uses: actions/checkout@v3 - name: Check out - name: Restore prereq cache uses: actions/cache@v4 with: path: prereq_cache - key: prereq-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + key: prereq-ubuntu-22.04-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} - name: Restore TensorStore FetchContent cache uses: actions/cache@v4 with: path: ts_fc_cache - key: ts-fc-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + key: ts-fc-ubuntu-22.04-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} - - uses: ilammy/msvc-dev-cmd@v1 - name: Add MSVS Path + - uses: actions/setup-python@v4 + with: + python-version: '3.11' - - name: Add NASM - if: matrix.os == 'windows-latest' - uses: ilammy/setup-nasm@v1 + - name: Install cibuildwheel + run: python -m pip install cibuildwheel wheel - - name: Add Ninja - if: matrix.os == 'windows-latest' - uses: seanmiddleditch/gha-setup-ninja@master + - name: Build wheels + run: python -m cibuildwheel --output-dir dist + env: + CIBW_BUILD: ${{ matrix.cibw_build }} + CIBW_BUILD_VERBOSITY: 3 + CIBW_SKIP: "*musllinux*" + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + CIBW_BEFORE_ALL_LINUX: bash /project/ci-utils/before_all_linux.sh + CIBW_ENVIRONMENT_LINUX: >- + LD_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64:$LD_LIBRARY_PATH" + ON_GITHUB="TRUE" + ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" + CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON -DFETCHCONTENT_BASE_DIR=/project/ts_fc_cache" + CIBW_ARCHS: auto64 + CIBW_BEFORE_TEST_LINUX: dnf -y install maven java + CIBW_TEST_REQUIRES: bfio>=2.4.0 tensorstore numpy pytest + CIBW_TEST_COMMAND: python -W default -m pytest -vv -s {project}/tests/python + + - name: Install Dependencies + run: python -m pip install --upgrade twine requests + + - name: Publish to PyPi + run: python -m twine upload dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + TWINE_REPOSITORY: pypi + + build_wheels_macos_intel: + name: Build wheels (macOS intel, ${{ matrix.cibw_build }}) + runs-on: macos-15-intel + needs: [prebuild_macos_intel] + continue-on-error: true + env: + MACOSX_DEPLOYMENT_TARGET: "10.15" + strategy: + matrix: + cibw_build: ["cp310-*", "cp311-*", "cp312-*", "cp313-*"] + + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-macos-15-intel-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-macos-15-intel-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} - uses: actions/setup-python@v4 - name: Install Python with: python-version: '3.11' - name: Install cibuildwheel - run: | - python -m pip install cibuildwheel delvewheel wheel + run: python -m pip install cibuildwheel wheel - - name: Building wheels - run: | - python -m cibuildwheel --output-dir dist + - name: Build wheels + run: python -m cibuildwheel --output-dir dist env: CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_BUILD_VERBOSITY: 3 - CIBW_SKIP: "*musllinux*" - CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 CIBW_BEFORE_ALL_MACOS: brew install nasm && brew uninstall --ignore-dependencies openssl || true && sudo rm -rf /usr/local/include/openssl && bash ci-utils/before_all_macos.sh - CIBW_BEFORE_ALL_LINUX: bash /project/ci-utils/before_all_linux.sh - CIBW_BEFORE_ALL_WINDOWS: ci-utils\before_all_win.bat CIBW_ENVIRONMENT_MACOS: >- MACOSX_DEPLOYMENT_TARGET=11.0 REPAIR_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" CMAKE_ARGS="-DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw -DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" - CIBW_ENVIRONMENT_LINUX: >- - LD_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64:$LD_LIBRARY_PATH" - ON_GITHUB="TRUE" - ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" - CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON -DFETCHCONTENT_BASE_DIR=/project/ts_fc_cache" CIBW_REPAIR_WHEEL_COMMAND_MACOS: DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} && DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} + CIBW_ARCHS: auto64 + + - name: Install Dependencies + run: python -m pip install --upgrade twine requests + + - name: Publish to PyPi + run: python -m twine upload dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + TWINE_REPOSITORY: pypi + + build_wheels_windows: + name: Build wheels (Windows, ${{ matrix.cibw_build }}) + runs-on: windows-latest + needs: [prebuild_windows] + continue-on-error: true + strategy: + matrix: + cibw_build: ["cp310-*", "cp311-*", "cp312-*", "cp313-*"] + + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-windows-latest-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-windows-latest-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - uses: ilammy/msvc-dev-cmd@v1 + name: Add MSVS Path + + - name: Add NASM + uses: ilammy/setup-nasm@v1 + + - name: Add Ninja + uses: seanmiddleditch/gha-setup-ninja@master + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel delvewheel wheel + + - name: Build wheels + run: python -m cibuildwheel --output-dir dist + env: + CIBW_BUILD: ${{ matrix.cibw_build }} + CIBW_BUILD_VERBOSITY: 3 + CIBW_BEFORE_ALL_WINDOWS: ci-utils\before_all_win.bat CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\argolid\\bin;$PATH" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="C:\\TEMP\\argolid_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}" - CIBW_ARCHS: ${{ matrix.cibw_archs }} - CIBW_BEFORE_TEST_LINUX: dnf -y install maven java - CIBW_TEST_REQUIRES: bfio>=2.4.0 tensorstore numpy pytest - CIBW_TEST_COMMAND: python -W default -m pytest -vv -s {project}/tests/python + CIBW_ARCHS: auto64 - name: Install Dependencies run: python -m pip install --upgrade twine requests @@ -202,46 +341,40 @@ jobs: TWINE_REPOSITORY: pypi build_wheels_apple_arm64: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - needs: [prebuild] + name: Build wheels (macOS arm64, ${{ matrix.cibw_build }}) + runs-on: macos-14 + needs: [prebuild_macos_arm64] continue-on-error: true env: MACOSX_DEPLOYMENT_TARGET: "11.0" strategy: matrix: - os: [macos-14] - cibw_archs: ["arm64"] cibw_build: ["cp310-*", "cp311-*", "cp312-*", "cp313-*"] steps: - uses: actions/checkout@v3 - name: Check out - name: Restore prereq cache uses: actions/cache@v4 with: path: prereq_cache - key: prereq-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + key: prereq-macos-14-arm64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} - name: Restore TensorStore FetchContent cache uses: actions/cache@v4 with: path: ts_fc_cache - key: ts-fc-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + key: ts-fc-macos-14-arm64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} - uses: actions/setup-python@v4 - name: Install Python with: python-version: '3.11' - name: Install cibuildwheel - run: | - python -m pip install cibuildwheel delocate wheel + run: python -m pip install cibuildwheel delocate wheel - - name: Building wheels - run: | - python -m cibuildwheel --output-dir dist + - name: Build wheels + run: python -m cibuildwheel --output-dir dist env: CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_BUILD_VERBOSITY: 3 @@ -258,7 +391,7 @@ jobs: CIBW_REPAIR_WHEEL_COMMAND_MACOS: | DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} && MACOSX_DEPLOYMENT_TARGET=11.0 DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} -e libc++ -e libunwind - CIBW_ARCHS: ${{ matrix.cibw_archs }} + CIBW_ARCHS: arm64 CIBW_TEST_REQUIRES: bfio>=2.4.0 tensorstore numpy pytest CIBW_TEST_COMMAND: python -W default -m pytest -vv -s {project}/tests/python diff --git a/.github/workflows/wheel_build.yml b/.github/workflows/wheel_build.yml index e269068..2bd0f63 100644 --- a/.github/workflows/wheel_build.yml +++ b/.github/workflows/wheel_build.yml @@ -10,47 +10,36 @@ concurrency: jobs: # ────────────────────────────────────────────────────────────────────────── - # Stage 1: Prebuild (one job per OS/arch) - # - Builds filepattern + pybind11 → prereq_cache/ - # - Builds TensorStore deps via FetchContent → ts_fc_cache/ - # - Both caches are restored by wheel-build jobs below. + # Stage 1 — Prebuild jobs (one per OS/arch, run in parallel) + # Each populates two caches: + # prereq_cache/ — filepattern + pybind11 + # ts_fc_cache/ — TensorStore FetchContent source tree + # Wheel-build jobs below depend only on their matching prebuild job, + # so each OS starts building wheels as soon as its own prebuild is done. # ────────────────────────────────────────────────────────────────────────── - prebuild: - name: Prebuild on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - include: - - os: ubuntu-22.04 - cibw_archs: auto64 - - os: macos-15-intel - cibw_archs: auto64 - - os: windows-latest - cibw_archs: auto64 - - os: macos-14 - cibw_archs: arm64 + prebuild_linux: + name: Prebuild (Linux) + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - name: Check out - name: Restore prereq cache id: prereq-cache uses: actions/cache@v4 with: path: prereq_cache - key: prereq-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + key: prereq-ubuntu-22.04-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} - name: Restore TensorStore FetchContent cache id: ts-cache uses: actions/cache@v4 with: path: ts_fc_cache - key: ts-fc-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + key: ts-fc-ubuntu-22.04-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} - # ── Linux: run prebuild inside the same manylinux container cibuildwheel uses ── - name: Run Linux prebuild in manylinux container - if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && matrix.os == 'ubuntu-22.04' + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' run: | docker run --rm \ -v ${{ github.workspace }}:/project \ @@ -58,43 +47,75 @@ jobs: quay.io/pypa/manylinux_2_28_x86_64 \ bash ci-utils/prebuild_linux.sh - # ── macOS: brew + install prereqs + cmake TensorStore prebuild ── - - name: Run macOS prebuild - if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && startsWith(matrix.os, 'macos') + prebuild_macos_intel: + name: Prebuild (macOS intel) + runs-on: macos-15-intel + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + id: prereq-cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-macos-15-intel-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + id: ts-cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-macos-15-intel-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - name: Run macOS intel prebuild + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' run: | brew install nasm - if [[ "${{ matrix.os }}" == "macos-14" ]]; then - brew uninstall --ignore-dependencies jpeg-turbo || true - else - brew uninstall --ignore-dependencies openssl || true - sudo rm -rf /usr/local/include/openssl - fi - # Build filepattern + pybind11 (and sys deps zlib/jpeg/png) → prereq_cache/local_install + brew uninstall --ignore-dependencies openssl || true + sudo rm -rf /usr/local/include/openssl bash ci-utils/install_prereq_linux.sh ./prereq_cache/local_install - # Build TensorStore using the minimal prebuild cmake project (no Python/pybind11 needed) - CMAKE_FLAGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" - if [[ "${{ matrix.os }}" == "macos-15-intel" ]]; then - CMAKE_FLAGS="${CMAKE_FLAGS} -DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON" - fi cmake -S ci-utils/tensorstore_prebuild -B /tmp/ts_cmake_prebuild \ - -DCMAKE_BUILD_TYPE=Release ${CMAKE_FLAGS} + -DCMAKE_BUILD_TYPE=Release \ + -DTENSORSTORE_USE_SYSTEM_JPEG=ON \ + -DTENSORSTORE_USE_SYSTEM_ZLIB=ON \ + -DTENSORSTORE_USE_SYSTEM_PNG=ON \ + -DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw \ + -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache cmake --build /tmp/ts_cmake_prebuild -j$(sysctl -n hw.ncpu) - # ── Windows: MSVC env + NASM + Ninja + install prereqs + cmake TensorStore prebuild ── - - name: Set up MSVC environment (Windows) - if: matrix.os == 'windows-latest' + prebuild_windows: + name: Prebuild (Windows) + runs-on: windows-latest + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + id: prereq-cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-windows-latest-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + id: ts-cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-windows-latest-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - name: Set up MSVC environment uses: ilammy/msvc-dev-cmd@v1 - - name: Add NASM (Windows) - if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && matrix.os == 'windows-latest' + - name: Add NASM + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' uses: ilammy/setup-nasm@v1 - - name: Add Ninja (Windows) - if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && matrix.os == 'windows-latest' + - name: Add Ninja + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' uses: seanmiddleditch/gha-setup-ninja@master - name: Run Windows prebuild - if: (steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true') && matrix.os == 'windows-latest' + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' shell: cmd run: | call ci-utils\install_prereq_win.bat @@ -105,141 +126,250 @@ jobs: cmake --build C:\ts_cmake_prebuild --parallel if %errorlevel% neq 0 exit /b %errorlevel% + prebuild_macos_arm64: + name: Prebuild (macOS arm64) + runs-on: macos-14 + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + id: prereq-cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-macos-14-arm64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + id: ts-cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-macos-14-arm64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - name: Run macOS arm64 prebuild + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' + run: | + brew install nasm + brew uninstall --ignore-dependencies jpeg-turbo || true + bash ci-utils/install_prereq_linux.sh ./prereq_cache/local_install + cmake -S ci-utils/tensorstore_prebuild -B /tmp/ts_cmake_prebuild \ + -DCMAKE_BUILD_TYPE=Release \ + -DTENSORSTORE_USE_SYSTEM_JPEG=ON \ + -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache + cmake --build /tmp/ts_cmake_prebuild -j$(sysctl -n hw.ncpu) + # ────────────────────────────────────────────────────────────────────────── - # Stage 2: Build wheels (4 parallel jobs per Python version, per OS) - # Each job restores both caches from the prebuild stage above. + # Stage 2 — Wheel-build jobs (4 Python versions in parallel per OS) + # Each job needs only its own OS's prebuild, so it starts as soon as + # that prebuild completes — independent of the other OSes. # ────────────────────────────────────────────────────────────────────────── - build_wheels: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - needs: [prebuild] - continue-on-error: true - env: - MACOSX_DEPLOYMENT_TARGET: "10.15" + build_wheels_linux: + name: Build wheels (Linux, ${{ matrix.cibw_build }}) + runs-on: ubuntu-22.04 + needs: [prebuild_linux] + continue-on-error: true strategy: matrix: - os: [ubuntu-22.04, macos-15-intel, windows-latest] - cibw_archs: ["auto64"] cibw_build: ["cp310", "cp311", "cp312", "cp313"] steps: - uses: actions/checkout@v3 - name: Check out - name: Restore prereq cache uses: actions/cache@v4 with: path: prereq_cache - key: prereq-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + key: prereq-ubuntu-22.04-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} - name: Restore TensorStore FetchContent cache uses: actions/cache@v4 with: path: ts_fc_cache - key: ts-fc-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + key: ts-fc-ubuntu-22.04-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} - - uses: ilammy/msvc-dev-cmd@v1 - name: Add MSVS Path + - uses: actions/setup-python@v4 + with: + python-version: '3.11' - - name: Add NASM - if: matrix.os == 'windows-latest' - uses: ilammy/setup-nasm@v1 + - name: Install cibuildwheel + run: python -m pip install cibuildwheel wheel - - name: Add Ninja - if: matrix.os == 'windows-latest' - uses: seanmiddleditch/gha-setup-ninja@master + - name: Build wheels + run: python -m cibuildwheel --output-dir dist + env: + CIBW_BUILD: ${{ matrix.cibw_build }}-* + CIBW_BUILD_VERBOSITY: 3 + CIBW_SKIP: "*musllinux*" + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + CIBW_BEFORE_ALL_LINUX: bash /project/ci-utils/before_all_linux.sh + CIBW_ENVIRONMENT_LINUX: >- + LD_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64:$LD_LIBRARY_PATH" + ON_GITHUB="TRUE" + ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" + CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON -DFETCHCONTENT_BASE_DIR=/project/ts_fc_cache" + CIBW_ARCHS: auto64 + CIBW_BEFORE_TEST_LINUX: dnf -y install maven java + CIBW_TEST_REQUIRES: bfio>=2.4.0 tensorstore numpy pytest + CIBW_TEST_COMMAND: python -W default -m pytest -vv -s {project}/tests/python + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: argolid-wheels-linux-auto64-${{ matrix.cibw_build }} + path: dist/*.whl + retention-days: 1 + + build_wheels_macos_intel: + name: Build wheels (macOS intel, ${{ matrix.cibw_build }}) + runs-on: macos-15-intel + needs: [prebuild_macos_intel] + continue-on-error: true + env: + MACOSX_DEPLOYMENT_TARGET: "10.15" + strategy: + matrix: + cibw_build: ["cp310", "cp311", "cp312", "cp313"] + + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-macos-15-intel-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-macos-15-intel-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} - uses: actions/setup-python@v4 - name: Install Python with: python-version: '3.11' - name: Install cibuildwheel - run: | - python -m pip install cibuildwheel delvewheel wheel + run: python -m pip install cibuildwheel wheel - - name: Building wheels - run: | - python -m cibuildwheel --output-dir dist + - name: Build wheels + run: python -m cibuildwheel --output-dir dist env: CIBW_BUILD: ${{ matrix.cibw_build }}-* CIBW_BUILD_VERBOSITY: 3 - CIBW_SKIP: "*musllinux*" - CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 CIBW_BEFORE_ALL_MACOS: brew install nasm && brew uninstall --ignore-dependencies openssl || true && sudo rm -rf /usr/local/include/openssl && bash ci-utils/before_all_macos.sh - CIBW_BEFORE_ALL_LINUX: bash /project/ci-utils/before_all_linux.sh - CIBW_BEFORE_ALL_WINDOWS: ci-utils\before_all_win.bat CIBW_ENVIRONMENT_MACOS: >- MACOSX_DEPLOYMENT_TARGET=11.0 REPAIR_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" CMAKE_ARGS="-DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw -DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" - CIBW_ENVIRONMENT_LINUX: >- - LD_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64:$LD_LIBRARY_PATH" - ON_GITHUB="TRUE" - ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" - CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON -DFETCHCONTENT_BASE_DIR=/project/ts_fc_cache" CIBW_REPAIR_WHEEL_COMMAND_MACOS: DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} && DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} + CIBW_ARCHS: auto64 + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: argolid-wheels-macos-15-intel-auto64-${{ matrix.cibw_build }} + path: dist/*.whl + retention-days: 1 + + build_wheels_windows: + name: Build wheels (Windows, ${{ matrix.cibw_build }}) + runs-on: windows-latest + needs: [prebuild_windows] + continue-on-error: true + strategy: + matrix: + cibw_build: ["cp310", "cp311", "cp312", "cp313"] + + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-windows-latest-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-windows-latest-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - uses: ilammy/msvc-dev-cmd@v1 + name: Add MSVS Path + + - name: Add NASM + uses: ilammy/setup-nasm@v1 + + - name: Add Ninja + uses: seanmiddleditch/gha-setup-ninja@master + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel delvewheel wheel + + - name: Build wheels + run: python -m cibuildwheel --output-dir dist + env: + CIBW_BUILD: ${{ matrix.cibw_build }}-* + CIBW_BUILD_VERBOSITY: 3 + CIBW_BEFORE_ALL_WINDOWS: ci-utils\before_all_win.bat CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\argolid\\bin;$PATH" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="C:\\TEMP\\argolid_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}" - CIBW_ARCHS: ${{ matrix.cibw_archs }} - CIBW_BEFORE_TEST_LINUX: dnf -y install maven java - CIBW_TEST_REQUIRES: bfio>=2.4.0 tensorstore numpy pytest - CIBW_TEST_COMMAND: python -W default -m pytest -vv -s {project}/tests/python + CIBW_ARCHS: auto64 - name: Upload Artifact uses: actions/upload-artifact@v4 with: - name: argolid-wheels-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ matrix.cibw_build }} + name: argolid-wheels-windows-auto64-${{ matrix.cibw_build }} path: dist/*.whl retention-days: 1 build_wheels_apple_arm64: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - needs: [prebuild] + name: Build wheels (macOS arm64, ${{ matrix.cibw_build }}) + runs-on: macos-14 + needs: [prebuild_macos_arm64] continue-on-error: true env: MACOSX_DEPLOYMENT_TARGET: "11.0" strategy: matrix: - os: [macos-14] - cibw_archs: ["arm64"] cibw_build: ["cp310", "cp311", "cp312", "cp313"] steps: - uses: actions/checkout@v3 - name: Check out - name: Restore prereq cache uses: actions/cache@v4 with: path: prereq_cache - key: prereq-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + key: prereq-macos-14-arm64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} - name: Restore TensorStore FetchContent cache uses: actions/cache@v4 with: path: ts_fc_cache - key: ts-fc-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + key: ts-fc-macos-14-arm64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} - uses: actions/setup-python@v4 - name: Install Python with: python-version: '3.11' - name: Install cibuildwheel - run: | - python -m pip install cibuildwheel delocate wheel + run: python -m pip install cibuildwheel delocate wheel - - name: Building wheels - run: | - python -m cibuildwheel --output-dir dist + - name: Build wheels + run: python -m cibuildwheel --output-dir dist env: CIBW_BUILD: ${{ matrix.cibw_build }}-* CIBW_BUILD_VERBOSITY: 3 @@ -256,13 +386,13 @@ jobs: CIBW_REPAIR_WHEEL_COMMAND_MACOS: | DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} && MACOSX_DEPLOYMENT_TARGET=11.0 DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} -e libc++ -e libunwind - CIBW_ARCHS: ${{ matrix.cibw_archs }} + CIBW_ARCHS: arm64 CIBW_TEST_REQUIRES: bfio>=2.4.0 tensorstore numpy pytest CIBW_TEST_COMMAND: python -W default -m pytest -vv -s {project}/tests/python - name: Upload Artifact uses: actions/upload-artifact@v4 with: - name: argolid-wheels-apple-arm64-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ matrix.cibw_build }} + name: argolid-wheels-apple-arm64-${{ matrix.cibw_build }} path: dist/*.whl retention-days: 1 From 646ffa488a8e4f0e748de19f08e8fe92c5d31286 Mon Sep 17 00:00:00 2001 From: Sameeul Samee Date: Thu, 19 Feb 2026 09:35:00 -0500 Subject: [PATCH 4/5] feat: add ccache support for Linux CI to avoid TensorStore recompilation - Add ccache detection to CMakeLists.txt and tensorstore_prebuild/CMakeLists.txt via CMAKE_CXX/C_COMPILER_LAUNCHER so all compilation units go through ccache - Install ccache in prebuild_linux.sh and before_all_linux.sh via dnf - Set CCACHE_DIR=/project/ccache_dir in prebuild (env var) and wheel-build (CIBW_ENVIRONMENT_LINUX) so the cache is in the host-accessible workspace - Add ccache_dir GitHub Actions cache to prebuild_linux (saves after prebuild) and build_wheels_linux (restores from prebuild's key, saves per Python version) - Prebuild now also triggers Docker run on ccache miss, not just prereq/ts misses - Add ccache_dir/ to .gitignore Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/publish_pypi.yml | 22 +++++++++++++++++++- .github/workflows/wheel_build.yml | 22 +++++++++++++++++++- .gitignore | 1 + CMakeLists.txt | 9 ++++++++ ci-utils/before_all_linux.sh | 3 +++ ci-utils/prebuild_linux.sh | 6 ++++++ ci-utils/tensorstore_prebuild/CMakeLists.txt | 8 +++++++ 7 files changed, 69 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 5bc95a4..3a30da8 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -35,8 +35,17 @@ jobs: path: ts_fc_cache key: ts-fc-ubuntu-22.04-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + - name: Restore ccache + id: ccache + uses: actions/cache@v4 + with: + path: ccache_dir + key: ccache-linux-auto64-${{ github.sha }} + restore-keys: | + ccache-linux-auto64- + - name: Run Linux prebuild in manylinux container - if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' || steps.ccache.outputs.cache-hit != 'true' run: | docker run --rm \ -v ${{ github.workspace }}:/project \ @@ -184,6 +193,15 @@ jobs: path: ts_fc_cache key: ts-fc-ubuntu-22.04-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + - name: Restore ccache + uses: actions/cache@v4 + with: + path: ccache_dir + key: ccache-linux-auto64-${{ github.sha }}-${{ matrix.cibw_build }} + restore-keys: | + ccache-linux-auto64-${{ github.sha }} + ccache-linux-auto64- + - uses: actions/setup-python@v4 with: python-version: '3.11' @@ -203,6 +221,8 @@ jobs: LD_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64:$LD_LIBRARY_PATH" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" + CCACHE_DIR=/project/ccache_dir + CCACHE_MAXSIZE=5G CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON -DFETCHCONTENT_BASE_DIR=/project/ts_fc_cache" CIBW_ARCHS: auto64 CIBW_BEFORE_TEST_LINUX: dnf -y install maven java diff --git a/.github/workflows/wheel_build.yml b/.github/workflows/wheel_build.yml index 2bd0f63..feeb021 100644 --- a/.github/workflows/wheel_build.yml +++ b/.github/workflows/wheel_build.yml @@ -38,8 +38,17 @@ jobs: path: ts_fc_cache key: ts-fc-ubuntu-22.04-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + - name: Restore ccache + id: ccache + uses: actions/cache@v4 + with: + path: ccache_dir + key: ccache-linux-auto64-${{ github.sha }} + restore-keys: | + ccache-linux-auto64- + - name: Run Linux prebuild in manylinux container - if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' || steps.ccache.outputs.cache-hit != 'true' run: | docker run --rm \ -v ${{ github.workspace }}:/project \ @@ -188,6 +197,15 @@ jobs: path: ts_fc_cache key: ts-fc-ubuntu-22.04-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + - name: Restore ccache + uses: actions/cache@v4 + with: + path: ccache_dir + key: ccache-linux-auto64-${{ github.sha }}-${{ matrix.cibw_build }} + restore-keys: | + ccache-linux-auto64-${{ github.sha }} + ccache-linux-auto64- + - uses: actions/setup-python@v4 with: python-version: '3.11' @@ -207,6 +225,8 @@ jobs: LD_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64:$LD_LIBRARY_PATH" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" + CCACHE_DIR=/project/ccache_dir + CCACHE_MAXSIZE=5G CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON -DFETCHCONTENT_BASE_DIR=/project/ts_fc_cache" CIBW_ARCHS: auto64 CIBW_BEFORE_TEST_LINUX: dnf -y install maven java diff --git a/.gitignore b/.gitignore index 4ad6e9d..2b8b807 100644 --- a/.gitignore +++ b/.gitignore @@ -568,6 +568,7 @@ scratch/ # CI build caches (populated by prebuild job, restored by wheel-build jobs) prereq_cache/ ts_fc_cache/ +ccache_dir/ build/ build_man/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 34554cc..4f18f12 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,12 +36,21 @@ set(SOURCE src/cpp/utilities/utilities.cpp ) +# Use ccache if available to speed up repeated TensorStore compilation in CI +find_program(CCACHE_EXECUTABLE ccache) +if(CCACHE_EXECUTABLE) + message(STATUS "ccache found: ${CCACHE_EXECUTABLE}") + set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_EXECUTABLE}" CACHE STRING "" FORCE) + set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE_EXECUTABLE}" CACHE STRING "" FORCE) +endif() + include(FetchContent) FetchContent_Declare( tensorstore URL "https://github.com/google/tensorstore/archive/refs/tags/v0.1.80.tar.gz" URL_HASH SHA256=94866de34b6139d77d30e828a50f9e8df98e7dd68e848393470879aeb50ea7bf + FIND_PACKAGE_ARGS ) # Additional FetchContent_Declare calls as needed... diff --git a/ci-utils/before_all_linux.sh b/ci-utils/before_all_linux.sh index 859de49..3dfef2d 100755 --- a/ci-utils/before_all_linux.sh +++ b/ci-utils/before_all_linux.sh @@ -15,6 +15,9 @@ tar -xjf nasm.tar.bz2 cd nasm-2.15.05 && ./configure && make -j"$(nproc)" && make install cd /project +# Install ccache (CCACHE_DIR and CCACHE_MAXSIZE are set via CIBW_ENVIRONMENT_LINUX) +dnf install -y ccache + mkdir -p /tmp/argolid_bld if [ -d "/project/prereq_cache/local_install" ]; then diff --git a/ci-utils/prebuild_linux.sh b/ci-utils/prebuild_linux.sh index 372300c..09ef2cb 100755 --- a/ci-utils/prebuild_linux.sh +++ b/ci-utils/prebuild_linux.sh @@ -15,6 +15,12 @@ tar -xjf nasm.tar.bz2 cd nasm-2.15.05 && ./configure && make -j"$(nproc)" && make install cd /project +# Install ccache and point it at the workspace-mounted cache dir +dnf install -y ccache +mkdir -p /project/ccache_dir +export CCACHE_DIR=/project/ccache_dir +export CCACHE_MAXSIZE=5G + # Build zlib, libjpeg-turbo, libpng to /usr/local (needed for TENSORSTORE_USE_SYSTEM_* flags) bash ci-utils/install_sys_deps_linux.sh diff --git a/ci-utils/tensorstore_prebuild/CMakeLists.txt b/ci-utils/tensorstore_prebuild/CMakeLists.txt index 972a200..5a30369 100644 --- a/ci-utils/tensorstore_prebuild/CMakeLists.txt +++ b/ci-utils/tensorstore_prebuild/CMakeLists.txt @@ -4,6 +4,14 @@ project(TensorStorePrebuild) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) +# Use ccache if available to speed up repeated TensorStore compilation in CI +find_program(CCACHE_EXECUTABLE ccache) +if(CCACHE_EXECUTABLE) + message(STATUS "ccache found: ${CCACHE_EXECUTABLE}") + set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_EXECUTABLE}" CACHE STRING "" FORCE) + set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE_EXECUTABLE}" CACHE STRING "" FORCE) +endif() + include(FetchContent) # Keep URL + SHA256 in sync with root CMakeLists.txt From 19e5b194a94866fca3d01321896fea14c8f8ad96 Mon Sep 17 00:00:00 2001 From: Sameeul Samee Date: Thu, 19 Feb 2026 09:53:14 -0500 Subject: [PATCH 5/5] fix: uninstall openssl after curl downloads on macOS intel brew's curl dynamically links against openssl. Uninstalling openssl before before_all_macos.sh (which uses curl to download zlib/jpeg/png) caused curl to abort with a missing libssl dylib error. Reordered CIBW_BEFORE_ALL_MACOS so before_all_macos.sh (curl downloads) runs first, then openssl is removed before cmake runs. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/publish_pypi.yml | 4 ++-- .github/workflows/wheel_build.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 3a30da8..cddc952 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -278,9 +278,9 @@ jobs: CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_BUILD_VERBOSITY: 3 CIBW_BEFORE_ALL_MACOS: brew install nasm && + bash ci-utils/before_all_macos.sh && brew uninstall --ignore-dependencies openssl || true && - sudo rm -rf /usr/local/include/openssl && - bash ci-utils/before_all_macos.sh + sudo rm -rf /usr/local/include/openssl CIBW_ENVIRONMENT_MACOS: >- MACOSX_DEPLOYMENT_TARGET=11.0 REPAIR_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64" diff --git a/.github/workflows/wheel_build.yml b/.github/workflows/wheel_build.yml index feeb021..8be4dd4 100644 --- a/.github/workflows/wheel_build.yml +++ b/.github/workflows/wheel_build.yml @@ -279,9 +279,9 @@ jobs: CIBW_BUILD: ${{ matrix.cibw_build }}-* CIBW_BUILD_VERBOSITY: 3 CIBW_BEFORE_ALL_MACOS: brew install nasm && + bash ci-utils/before_all_macos.sh && brew uninstall --ignore-dependencies openssl || true && - sudo rm -rf /usr/local/include/openssl && - bash ci-utils/before_all_macos.sh + sudo rm -rf /usr/local/include/openssl CIBW_ENVIRONMENT_MACOS: >- MACOSX_DEPLOYMENT_TARGET=11.0 REPAIR_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64"