diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 7c8b2fb..cddc952 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -11,90 +11,348 @@ concurrency: cancel-in-progress: true jobs: - build_wheels: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - continue-on-error: true - env: - MACOSX_DEPLOYMENT_TARGET: "10.15" - strategy: - matrix: - os: [ubuntu-22.04, macos-15-intel, windows-latest] - cibw_archs: ["auto64"] - cibw_build: ["cp310-*", "cp311-*", "cp312-*", "cp313-*"] + # ────────────────────────────────────────────────────────────────────────── + # Stage 1 — Prebuild jobs (one per OS/arch, run in parallel) + # ────────────────────────────────────────────────────────────────────────── + prebuild_linux: + name: Prebuild (Linux) + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - name: Check out - - uses: ilammy/msvc-dev-cmd@v1 - name: Add MSVS Path + - name: Restore prereq cache + id: prereq-cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-ubuntu-22.04-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + id: ts-cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-ubuntu-22.04-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - name: Restore ccache + id: ccache + uses: actions/cache@v4 + with: + path: ccache_dir + key: ccache-linux-auto64-${{ github.sha }} + restore-keys: | + ccache-linux-auto64- + + - name: Run Linux prebuild in manylinux container + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' || steps.ccache.outputs.cache-hit != 'true' + run: | + docker run --rm \ + -v ${{ github.workspace }}:/project \ + -w /project \ + quay.io/pypa/manylinux_2_28_x86_64 \ + bash ci-utils/prebuild_linux.sh + + prebuild_macos_intel: + name: Prebuild (macOS intel) + runs-on: macos-15-intel + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + id: prereq-cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-macos-15-intel-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + id: ts-cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-macos-15-intel-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - name: Run macOS intel prebuild + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' + run: | + brew install nasm + brew uninstall --ignore-dependencies openssl || true + sudo rm -rf /usr/local/include/openssl + bash ci-utils/install_prereq_linux.sh ./prereq_cache/local_install + cmake -S ci-utils/tensorstore_prebuild -B /tmp/ts_cmake_prebuild \ + -DCMAKE_BUILD_TYPE=Release \ + -DTENSORSTORE_USE_SYSTEM_JPEG=ON \ + -DTENSORSTORE_USE_SYSTEM_ZLIB=ON \ + -DTENSORSTORE_USE_SYSTEM_PNG=ON \ + -DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw \ + -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache + cmake --build /tmp/ts_cmake_prebuild -j$(sysctl -n hw.ncpu) + + prebuild_windows: + name: Prebuild (Windows) + runs-on: windows-latest + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + id: prereq-cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-windows-latest-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + id: ts-cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-windows-latest-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - name: Set up MSVC environment + uses: ilammy/msvc-dev-cmd@v1 - name: Add NASM - if: matrix.os == 'windows-latest' + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' uses: ilammy/setup-nasm@v1 - name: Add Ninja - if: matrix.os == 'windows-latest' + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' uses: seanmiddleditch/gha-setup-ninja@master + - name: Run Windows prebuild + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' + shell: cmd + run: | + call ci-utils\install_prereq_win.bat + if %errorlevel% neq 0 exit /b %errorlevel% + xcopy /E /I /y local_install prereq_cache\local_install + cmake -S ci-utils\tensorstore_prebuild -B C:\ts_cmake_prebuild -DCMAKE_GENERATOR=Ninja -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache -DCMAKE_BUILD_TYPE=Release + if %errorlevel% neq 0 exit /b %errorlevel% + cmake --build C:\ts_cmake_prebuild --parallel + if %errorlevel% neq 0 exit /b %errorlevel% + + prebuild_macos_arm64: + name: Prebuild (macOS arm64) + runs-on: macos-14 + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + id: prereq-cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-macos-14-arm64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + id: ts-cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-macos-14-arm64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - name: Run macOS arm64 prebuild + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' + run: | + brew install nasm + brew uninstall --ignore-dependencies jpeg-turbo || true + bash ci-utils/install_prereq_linux.sh ./prereq_cache/local_install + cmake -S ci-utils/tensorstore_prebuild -B /tmp/ts_cmake_prebuild \ + -DCMAKE_BUILD_TYPE=Release \ + -DTENSORSTORE_USE_SYSTEM_JPEG=ON \ + -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache + cmake --build /tmp/ts_cmake_prebuild -j$(sysctl -n hw.ncpu) + + # ────────────────────────────────────────────────────────────────────────── + # Stage 2 — Wheel-build + publish jobs (4 Python versions per OS, in parallel) + # Each job depends only on its own OS's prebuild. + # ────────────────────────────────────────────────────────────────────────── + + build_wheels_linux: + name: Build wheels (Linux, ${{ matrix.cibw_build }}) + runs-on: ubuntu-22.04 + needs: [prebuild_linux] + continue-on-error: true + strategy: + matrix: + cibw_build: ["cp310-*", "cp311-*", "cp312-*", "cp313-*"] + + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-ubuntu-22.04-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-ubuntu-22.04-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - name: Restore ccache + uses: actions/cache@v4 + with: + path: ccache_dir + key: ccache-linux-auto64-${{ github.sha }}-${{ matrix.cibw_build }} + restore-keys: | + ccache-linux-auto64-${{ github.sha }} + ccache-linux-auto64- + - uses: actions/setup-python@v4 - name: Install Python with: python-version: '3.11' - name: Install cibuildwheel - run: | - python -m pip install cibuildwheel delvewheel wheel + run: python -m pip install cibuildwheel wheel - - name: Building wheels - run: | - python -m cibuildwheel --output-dir dist + - name: Build wheels + run: python -m cibuildwheel --output-dir dist env: CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_BUILD_VERBOSITY: 3 CIBW_SKIP: "*musllinux*" CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + CIBW_BEFORE_ALL_LINUX: bash /project/ci-utils/before_all_linux.sh + CIBW_ENVIRONMENT_LINUX: >- + LD_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64:$LD_LIBRARY_PATH" + ON_GITHUB="TRUE" + ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" + CCACHE_DIR=/project/ccache_dir + CCACHE_MAXSIZE=5G + CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON -DFETCHCONTENT_BASE_DIR=/project/ts_fc_cache" + CIBW_ARCHS: auto64 + CIBW_BEFORE_TEST_LINUX: dnf -y install maven java + CIBW_TEST_REQUIRES: bfio>=2.4.0 tensorstore numpy pytest + CIBW_TEST_COMMAND: python -W default -m pytest -vv -s {project}/tests/python + + - name: Install Dependencies + run: python -m pip install --upgrade twine requests + + - name: Publish to PyPi + run: python -m twine upload dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + TWINE_REPOSITORY: pypi + + build_wheels_macos_intel: + name: Build wheels (macOS intel, ${{ matrix.cibw_build }}) + runs-on: macos-15-intel + needs: [prebuild_macos_intel] + continue-on-error: true + env: + MACOSX_DEPLOYMENT_TARGET: "10.15" + strategy: + matrix: + cibw_build: ["cp310-*", "cp311-*", "cp312-*", "cp313-*"] + + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-macos-15-intel-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-macos-15-intel-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel wheel + + - name: Build wheels + run: python -m cibuildwheel --output-dir dist + env: + CIBW_BUILD: ${{ matrix.cibw_build }} + CIBW_BUILD_VERBOSITY: 3 CIBW_BEFORE_ALL_MACOS: brew install nasm && - bash ci-utils/install_prereq_linux.sh && - mkdir -p /tmp/argolid_bld && + bash ci-utils/before_all_macos.sh && brew uninstall --ignore-dependencies openssl || true && - sudo rm -rf /usr/local/include/openssl && - cp -r local_install /tmp/argolid_bld - CIBW_BEFORE_ALL_LINUX: curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 -o nasm-2.15.05.tar.bz2 && - tar -xjf nasm-2.15.05.tar.bz2 && - cd nasm-2.15.05 && - ./configure && - make && - make install && - cd .. && - bash ci-utils/install_prereq_linux.sh && - mkdir -p /tmp/argolid_bld && - cp -r local_install /tmp/argolid_bld - CIBW_BEFORE_ALL_WINDOWS: ci-utils\install_prereq_win.bat && - xcopy /E /I /y local_install C:\TEMP\argolid_bld\local_install + sudo rm -rf /usr/local/include/openssl CIBW_ENVIRONMENT_MACOS: >- MACOSX_DEPLOYMENT_TARGET=11.0 REPAIR_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" - CMAKE_ARGS="-DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw -DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON" - CIBW_ENVIRONMENT_LINUX: >- - LD_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64:$LD_LIBRARY_PATH" - ON_GITHUB="TRUE" - ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" - CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON" + CMAKE_ARGS="-DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw -DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" CIBW_REPAIR_WHEEL_COMMAND_MACOS: DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} && DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} - CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\argolid\\bin;$PATH" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="C:\\TEMP\\argolid_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja" + CIBW_ARCHS: auto64 + + - name: Install Dependencies + run: python -m pip install --upgrade twine requests + + - name: Publish to PyPi + run: python -m twine upload dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + TWINE_REPOSITORY: pypi + + build_wheels_windows: + name: Build wheels (Windows, ${{ matrix.cibw_build }}) + runs-on: windows-latest + needs: [prebuild_windows] + continue-on-error: true + strategy: + matrix: + cibw_build: ["cp310-*", "cp311-*", "cp312-*", "cp313-*"] + + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-windows-latest-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-windows-latest-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - uses: ilammy/msvc-dev-cmd@v1 + name: Add MSVS Path + + - name: Add NASM + uses: ilammy/setup-nasm@v1 + + - name: Add Ninja + uses: seanmiddleditch/gha-setup-ninja@master + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel delvewheel wheel + + - name: Build wheels + run: python -m cibuildwheel --output-dir dist + env: + CIBW_BUILD: ${{ matrix.cibw_build }} + CIBW_BUILD_VERBOSITY: 3 + CIBW_BEFORE_ALL_WINDOWS: ci-utils\before_all_win.bat + CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\argolid\\bin;$PATH" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="C:\\TEMP\\argolid_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}" - CIBW_ARCHS: ${{ matrix.cibw_archs }} - CIBW_BEFORE_TEST_LINUX: dnf -y install maven java - CIBW_TEST_REQUIRES: bfio>=2.4.0 tensorstore numpy pytest - CIBW_TEST_COMMAND: python -W default -m pytest -vv -s {project}/tests/python + CIBW_ARCHS: auto64 - name: Install Dependencies run: python -m pip install --upgrade twine requests - + - name: Publish to PyPi run: python -m twine upload dist/*.whl env: @@ -103,61 +361,66 @@ jobs: TWINE_REPOSITORY: pypi build_wheels_apple_arm64: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} + name: Build wheels (macOS arm64, ${{ matrix.cibw_build }}) + runs-on: macos-14 + needs: [prebuild_macos_arm64] continue-on-error: true env: MACOSX_DEPLOYMENT_TARGET: "11.0" strategy: matrix: - os: [macos-14] - cibw_archs: ["arm64"] cibw_build: ["cp310-*", "cp311-*", "cp312-*", "cp313-*"] steps: - uses: actions/checkout@v3 - name: Check out + + - name: Restore prereq cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-macos-14-arm64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-macos-14-arm64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} - uses: actions/setup-python@v4 - name: Install Python with: python-version: '3.11' - + - name: Install cibuildwheel - run: | - python -m pip install cibuildwheel delocate wheel + run: python -m pip install cibuildwheel delocate wheel - - name: Building wheels - run: | - python -m cibuildwheel --output-dir dist + - name: Build wheels + run: python -m cibuildwheel --output-dir dist env: CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_BUILD_VERBOSITY: 3 CIBW_ARCHS_MACOS: arm64 - CIBW_BEFORE_ALL_MACOS: brew install nasm && - brew uninstall --ignore-dependencies jpeg-turbo && - bash ci-utils/install_prereq_linux.sh && - mkdir -p /tmp/argolid_bld && - cp -r local_install /tmp/argolid_bld + CIBW_BEFORE_ALL_MACOS: brew install nasm && + brew uninstall --ignore-dependencies jpeg-turbo && + bash ci-utils/before_all_macos.sh CIBW_ENVIRONMENT_MACOS: >- MACOSX_DEPLOYMENT_TARGET=11.0 REPAIR_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" - CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON" + CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" CIBW_REPAIR_WHEEL_COMMAND_MACOS: | DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} && MACOSX_DEPLOYMENT_TARGET=11.0 DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} -e libc++ -e libunwind - CIBW_ARCHS: ${{ matrix.cibw_archs }} + CIBW_ARCHS: arm64 CIBW_TEST_REQUIRES: bfio>=2.4.0 tensorstore numpy pytest CIBW_TEST_COMMAND: python -W default -m pytest -vv -s {project}/tests/python - name: Install Dependencies run: python -m pip install --upgrade twine requests - + - name: Publish to PyPi run: python -m twine upload dist/*.whl env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - TWINE_REPOSITORY: pypi \ No newline at end of file + TWINE_REPOSITORY: pypi diff --git a/.github/workflows/wheel_build.yml b/.github/workflows/wheel_build.yml index 02c3138..8be4dd4 100644 --- a/.github/workflows/wheel_build.yml +++ b/.github/workflows/wheel_build.yml @@ -9,148 +9,410 @@ concurrency: cancel-in-progress: true jobs: - build_wheels: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - continue-on-error: true + # ────────────────────────────────────────────────────────────────────────── + # Stage 1 — Prebuild jobs (one per OS/arch, run in parallel) + # Each populates two caches: + # prereq_cache/ — filepattern + pybind11 + # ts_fc_cache/ — TensorStore FetchContent source tree + # Wheel-build jobs below depend only on their matching prebuild job, + # so each OS starts building wheels as soon as its own prebuild is done. + # ────────────────────────────────────────────────────────────────────────── - env: - MACOSX_DEPLOYMENT_TARGET: "10.15" - strategy: - matrix: - os: [ubuntu-22.04, macos-15-intel, windows-latest] - cibw_archs: ["auto64"] - cibw_build: ["cp310", "cp311", "cp312", "cp313"] + prebuild_linux: + name: Prebuild (Linux) + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + id: prereq-cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-ubuntu-22.04-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + - name: Restore TensorStore FetchContent cache + id: ts-cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-ubuntu-22.04-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - name: Restore ccache + id: ccache + uses: actions/cache@v4 + with: + path: ccache_dir + key: ccache-linux-auto64-${{ github.sha }} + restore-keys: | + ccache-linux-auto64- + + - name: Run Linux prebuild in manylinux container + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' || steps.ccache.outputs.cache-hit != 'true' + run: | + docker run --rm \ + -v ${{ github.workspace }}:/project \ + -w /project \ + quay.io/pypa/manylinux_2_28_x86_64 \ + bash ci-utils/prebuild_linux.sh + + prebuild_macos_intel: + name: Prebuild (macOS intel) + runs-on: macos-15-intel steps: - uses: actions/checkout@v3 - name: Check out - - uses: ilammy/msvc-dev-cmd@v1 - name: Add MSVS Path + - name: Restore prereq cache + id: prereq-cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-macos-15-intel-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + id: ts-cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-macos-15-intel-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - name: Run macOS intel prebuild + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' + run: | + brew install nasm + brew uninstall --ignore-dependencies openssl || true + sudo rm -rf /usr/local/include/openssl + bash ci-utils/install_prereq_linux.sh ./prereq_cache/local_install + cmake -S ci-utils/tensorstore_prebuild -B /tmp/ts_cmake_prebuild \ + -DCMAKE_BUILD_TYPE=Release \ + -DTENSORSTORE_USE_SYSTEM_JPEG=ON \ + -DTENSORSTORE_USE_SYSTEM_ZLIB=ON \ + -DTENSORSTORE_USE_SYSTEM_PNG=ON \ + -DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw \ + -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache + cmake --build /tmp/ts_cmake_prebuild -j$(sysctl -n hw.ncpu) + + prebuild_windows: + name: Prebuild (Windows) + runs-on: windows-latest + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + id: prereq-cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-windows-latest-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + id: ts-cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-windows-latest-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - name: Set up MSVC environment + uses: ilammy/msvc-dev-cmd@v1 - name: Add NASM - if: matrix.os == 'windows-latest' + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' uses: ilammy/setup-nasm@v1 - name: Add Ninja - if: matrix.os == 'windows-latest' + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' uses: seanmiddleditch/gha-setup-ninja@master + - name: Run Windows prebuild + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' + shell: cmd + run: | + call ci-utils\install_prereq_win.bat + if %errorlevel% neq 0 exit /b %errorlevel% + xcopy /E /I /y local_install prereq_cache\local_install + cmake -S ci-utils\tensorstore_prebuild -B C:\ts_cmake_prebuild -DCMAKE_GENERATOR=Ninja -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache -DCMAKE_BUILD_TYPE=Release + if %errorlevel% neq 0 exit /b %errorlevel% + cmake --build C:\ts_cmake_prebuild --parallel + if %errorlevel% neq 0 exit /b %errorlevel% + + prebuild_macos_arm64: + name: Prebuild (macOS arm64) + runs-on: macos-14 + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + id: prereq-cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-macos-14-arm64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + id: ts-cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-macos-14-arm64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - name: Run macOS arm64 prebuild + if: steps.prereq-cache.outputs.cache-hit != 'true' || steps.ts-cache.outputs.cache-hit != 'true' + run: | + brew install nasm + brew uninstall --ignore-dependencies jpeg-turbo || true + bash ci-utils/install_prereq_linux.sh ./prereq_cache/local_install + cmake -S ci-utils/tensorstore_prebuild -B /tmp/ts_cmake_prebuild \ + -DCMAKE_BUILD_TYPE=Release \ + -DTENSORSTORE_USE_SYSTEM_JPEG=ON \ + -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache + cmake --build /tmp/ts_cmake_prebuild -j$(sysctl -n hw.ncpu) + + # ────────────────────────────────────────────────────────────────────────── + # Stage 2 — Wheel-build jobs (4 Python versions in parallel per OS) + # Each job needs only its own OS's prebuild, so it starts as soon as + # that prebuild completes — independent of the other OSes. + # ────────────────────────────────────────────────────────────────────────── + + build_wheels_linux: + name: Build wheels (Linux, ${{ matrix.cibw_build }}) + runs-on: ubuntu-22.04 + needs: [prebuild_linux] + continue-on-error: true + strategy: + matrix: + cibw_build: ["cp310", "cp311", "cp312", "cp313"] + + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-ubuntu-22.04-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-ubuntu-22.04-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - name: Restore ccache + uses: actions/cache@v4 + with: + path: ccache_dir + key: ccache-linux-auto64-${{ github.sha }}-${{ matrix.cibw_build }} + restore-keys: | + ccache-linux-auto64-${{ github.sha }} + ccache-linux-auto64- + - uses: actions/setup-python@v4 - name: Install Python with: python-version: '3.11' - name: Install cibuildwheel - run: | - python -m pip install cibuildwheel delvewheel wheel + run: python -m pip install cibuildwheel wheel - - name: Building wheels - run: | - python -m cibuildwheel --output-dir dist + - name: Build wheels + run: python -m cibuildwheel --output-dir dist env: CIBW_BUILD: ${{ matrix.cibw_build }}-* CIBW_BUILD_VERBOSITY: 3 CIBW_SKIP: "*musllinux*" CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + CIBW_BEFORE_ALL_LINUX: bash /project/ci-utils/before_all_linux.sh + CIBW_ENVIRONMENT_LINUX: >- + LD_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64:$LD_LIBRARY_PATH" + ON_GITHUB="TRUE" + ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" + CCACHE_DIR=/project/ccache_dir + CCACHE_MAXSIZE=5G + CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON -DFETCHCONTENT_BASE_DIR=/project/ts_fc_cache" + CIBW_ARCHS: auto64 + CIBW_BEFORE_TEST_LINUX: dnf -y install maven java + CIBW_TEST_REQUIRES: bfio>=2.4.0 tensorstore numpy pytest + CIBW_TEST_COMMAND: python -W default -m pytest -vv -s {project}/tests/python + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: argolid-wheels-linux-auto64-${{ matrix.cibw_build }} + path: dist/*.whl + retention-days: 1 + + build_wheels_macos_intel: + name: Build wheels (macOS intel, ${{ matrix.cibw_build }}) + runs-on: macos-15-intel + needs: [prebuild_macos_intel] + continue-on-error: true + env: + MACOSX_DEPLOYMENT_TARGET: "10.15" + strategy: + matrix: + cibw_build: ["cp310", "cp311", "cp312", "cp313"] + + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-macos-15-intel-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-macos-15-intel-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel wheel + + - name: Build wheels + run: python -m cibuildwheel --output-dir dist + env: + CIBW_BUILD: ${{ matrix.cibw_build }}-* + CIBW_BUILD_VERBOSITY: 3 CIBW_BEFORE_ALL_MACOS: brew install nasm && - bash ci-utils/install_prereq_linux.sh && - mkdir -p /tmp/argolid_bld && + bash ci-utils/before_all_macos.sh && brew uninstall --ignore-dependencies openssl || true && - sudo rm -rf /usr/local/include/openssl && - cp -r local_install /tmp/argolid_bld - CIBW_BEFORE_ALL_LINUX: curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 -o nasm-2.15.05.tar.bz2 && - tar -xjf nasm-2.15.05.tar.bz2 && - cd nasm-2.15.05 && - ./configure && - make && - make install && - cd .. && - bash ci-utils/install_prereq_linux.sh && - mkdir -p /tmp/argolid_bld && - cp -r local_install /tmp/argolid_bld - CIBW_BEFORE_ALL_WINDOWS: ci-utils\install_prereq_win.bat && - xcopy /E /I /y local_install C:\TEMP\argolid_bld\local_install + sudo rm -rf /usr/local/include/openssl CIBW_ENVIRONMENT_MACOS: >- MACOSX_DEPLOYMENT_TARGET=11.0 REPAIR_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" - CMAKE_ARGS="-DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw -DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON" - CIBW_ENVIRONMENT_LINUX: >- - LD_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64:$LD_LIBRARY_PATH" - ON_GITHUB="TRUE" - ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" - CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON" + CMAKE_ARGS="-DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw -DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" CIBW_REPAIR_WHEEL_COMMAND_MACOS: DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} && DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} - CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\argolid\\bin;$PATH" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="C:\\TEMP\\argolid_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja" + CIBW_ARCHS: auto64 + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: argolid-wheels-macos-15-intel-auto64-${{ matrix.cibw_build }} + path: dist/*.whl + retention-days: 1 + + build_wheels_windows: + name: Build wheels (Windows, ${{ matrix.cibw_build }}) + runs-on: windows-latest + needs: [prebuild_windows] + continue-on-error: true + strategy: + matrix: + cibw_build: ["cp310", "cp311", "cp312", "cp313"] + + steps: + - uses: actions/checkout@v3 + + - name: Restore prereq cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-windows-latest-auto64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-windows-latest-auto64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} + + - uses: ilammy/msvc-dev-cmd@v1 + name: Add MSVS Path + + - name: Add NASM + uses: ilammy/setup-nasm@v1 + + - name: Add Ninja + uses: seanmiddleditch/gha-setup-ninja@master + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel delvewheel wheel + + - name: Build wheels + run: python -m cibuildwheel --output-dir dist + env: + CIBW_BUILD: ${{ matrix.cibw_build }}-* + CIBW_BUILD_VERBOSITY: 3 + CIBW_BEFORE_ALL_WINDOWS: ci-utils\before_all_win.bat + CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\argolid\\bin;$PATH" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="C:\\TEMP\\argolid_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}" - CIBW_ARCHS: ${{ matrix.cibw_archs }} - CIBW_BEFORE_TEST_LINUX: dnf -y install maven java - CIBW_TEST_REQUIRES: bfio>=2.4.0 tensorstore numpy pytest - CIBW_TEST_COMMAND: python -W default -m pytest -vv -s {project}/tests/python + CIBW_ARCHS: auto64 - name: Upload Artifact uses: actions/upload-artifact@v4 with: - name: argolid-wheels-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ matrix.cibw_build }} + name: argolid-wheels-windows-auto64-${{ matrix.cibw_build }} path: dist/*.whl retention-days: 1 build_wheels_apple_arm64: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} + name: Build wheels (macOS arm64, ${{ matrix.cibw_build }}) + runs-on: macos-14 + needs: [prebuild_macos_arm64] continue-on-error: true env: MACOSX_DEPLOYMENT_TARGET: "11.0" strategy: matrix: - os: [macos-14] - cibw_archs: ["arm64"] cibw_build: ["cp310", "cp311", "cp312", "cp313"] steps: - uses: actions/checkout@v3 - name: Check out + + - name: Restore prereq cache + uses: actions/cache@v4 + with: + path: prereq_cache + key: prereq-macos-14-arm64-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat') }} + + - name: Restore TensorStore FetchContent cache + uses: actions/cache@v4 + with: + path: ts_fc_cache + key: ts-fc-macos-14-arm64-${{ hashFiles('CMakeLists.txt', 'ci-utils/tensorstore_prebuild/CMakeLists.txt') }} - uses: actions/setup-python@v4 - name: Install Python with: python-version: '3.11' - + - name: Install cibuildwheel - run: | - python -m pip install cibuildwheel delocate wheel + run: python -m pip install cibuildwheel delocate wheel - - name: Building wheels - run: | - python -m cibuildwheel --output-dir dist + - name: Build wheels + run: python -m cibuildwheel --output-dir dist env: CIBW_BUILD: ${{ matrix.cibw_build }}-* CIBW_BUILD_VERBOSITY: 3 CIBW_ARCHS_MACOS: arm64 - CIBW_BEFORE_ALL_MACOS: brew install nasm && + CIBW_BEFORE_ALL_MACOS: brew install nasm && brew uninstall --ignore-dependencies jpeg-turbo && - bash ci-utils/install_prereq_linux.sh && - mkdir -p /tmp/argolid_bld && - cp -r local_install /tmp/argolid_bld + bash ci-utils/before_all_macos.sh CIBW_ENVIRONMENT_MACOS: >- MACOSX_DEPLOYMENT_TARGET=11.0 REPAIR_LIBRARY_PATH="/tmp/argolid_bld/local_install/lib:/tmp/argolid_bld/local_install/lib64" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="/tmp/argolid_bld/local_install" - CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON" + CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DFETCHCONTENT_BASE_DIR=${{ github.workspace }}/ts_fc_cache" CIBW_REPAIR_WHEEL_COMMAND_MACOS: | DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} && MACOSX_DEPLOYMENT_TARGET=11.0 DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} -e libc++ -e libunwind - CIBW_ARCHS: ${{ matrix.cibw_archs }} + CIBW_ARCHS: arm64 CIBW_TEST_REQUIRES: bfio>=2.4.0 tensorstore numpy pytest CIBW_TEST_COMMAND: python -W default -m pytest -vv -s {project}/tests/python - name: Upload Artifact uses: actions/upload-artifact@v4 with: - name: argolid-wheels-apple-arm64-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ matrix.cibw_build }} + name: argolid-wheels-apple-arm64-${{ matrix.cibw_build }} path: dist/*.whl - retention-days: 1 \ No newline at end of file + retention-days: 1 diff --git a/.gitignore b/.gitignore index cce0baa..2b8b807 100644 --- a/.gitignore +++ b/.gitignore @@ -562,9 +562,14 @@ dmypy.json # Cython debug symbols cython_debug/ -# scratch +# scratch scratch/ +# CI build caches (populated by prebuild job, restored by wheel-build jobs) +prereq_cache/ +ts_fc_cache/ +ccache_dir/ + build/ build_man/ venv/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 34554cc..4f18f12 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,12 +36,21 @@ set(SOURCE src/cpp/utilities/utilities.cpp ) +# Use ccache if available to speed up repeated TensorStore compilation in CI +find_program(CCACHE_EXECUTABLE ccache) +if(CCACHE_EXECUTABLE) + message(STATUS "ccache found: ${CCACHE_EXECUTABLE}") + set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_EXECUTABLE}" CACHE STRING "" FORCE) + set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE_EXECUTABLE}" CACHE STRING "" FORCE) +endif() + include(FetchContent) FetchContent_Declare( tensorstore URL "https://github.com/google/tensorstore/archive/refs/tags/v0.1.80.tar.gz" URL_HASH SHA256=94866de34b6139d77d30e828a50f9e8df98e7dd68e848393470879aeb50ea7bf + FIND_PACKAGE_ARGS ) # Additional FetchContent_Declare calls as needed... diff --git a/ci-utils/before_all_linux.sh b/ci-utils/before_all_linux.sh new file mode 100755 index 0000000..3dfef2d --- /dev/null +++ b/ci-utils/before_all_linux.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# before_all_linux.sh +# Replaces the inline CIBW_BEFORE_ALL_LINUX command. +# Runs inside the manylinux container during the wheel-build job. +# Expects: +# /project/prereq_cache/local_install — populated by prebuild job (may be absent on cache miss) +set -e + +cd /project + +# Always install NASM (required by TensorStore's aom/dav1d deps) +curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 \ + -o nasm.tar.bz2 +tar -xjf nasm.tar.bz2 +cd nasm-2.15.05 && ./configure && make -j"$(nproc)" && make install +cd /project + +# Install ccache (CCACHE_DIR and CCACHE_MAXSIZE are set via CIBW_ENVIRONMENT_LINUX) +dnf install -y ccache + +mkdir -p /tmp/argolid_bld + +if [ -d "/project/prereq_cache/local_install" ]; then + echo "==> Prereq cache hit: restoring filepattern + pybind11" + cp -r /project/prereq_cache/local_install /tmp/argolid_bld/local_install + # zlib/jpeg/png cannot be cached from /usr/local (container-internal path), + # so rebuild them quickly from source (~1 min total). + bash ci-utils/install_sys_deps_linux.sh +else + echo "==> Prereq cache miss: building filepattern + pybind11 + sys deps from scratch" + bash ci-utils/install_prereq_linux.sh + cp -r /project/local_install /tmp/argolid_bld/local_install +fi diff --git a/ci-utils/before_all_macos.sh b/ci-utils/before_all_macos.sh new file mode 100755 index 0000000..bef1520 --- /dev/null +++ b/ci-utils/before_all_macos.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# before_all_macos.sh +# Replaces the inline CIBW_BEFORE_ALL_MACOS commands for wheel-build jobs. +# brew uninstalls (openssl for intel, jpeg-turbo for arm64) should be done +# by the caller (CIBW_BEFORE_ALL inline) BEFORE invoking this script. +# Expects: +# prereq_cache/local_install — populated by prebuild job (may be absent on cache miss) +set -e + +mkdir -p /tmp/argolid_bld + +if [ -d "prereq_cache/local_install" ]; then + echo "==> Prereq cache hit: restoring filepattern + pybind11" + cp -r prereq_cache/local_install /tmp/argolid_bld/local_install + # Rebuild sys deps (zlib/jpeg/png) to /usr/local — these are fast and + # cannot be cached across jobs since they live outside the workspace. + bash ci-utils/install_sys_deps_linux.sh +else + echo "==> Prereq cache miss: building filepattern + pybind11 + sys deps from scratch" + bash ci-utils/install_prereq_linux.sh + cp -r local_install /tmp/argolid_bld/local_install +fi diff --git a/ci-utils/before_all_win.bat b/ci-utils/before_all_win.bat new file mode 100644 index 0000000..4c3deab --- /dev/null +++ b/ci-utils/before_all_win.bat @@ -0,0 +1,24 @@ +@echo off +REM before_all_win.bat +REM Replaces the inline CIBW_BEFORE_ALL_WINDOWS command for wheel-build jobs. +REM Expects: +REM prereq_cache\local_install — populated by prebuild job (may be absent on cache miss) + +if exist prereq_cache\local_install ( + echo =^> Prereq cache hit: restoring filepattern + pybind11 + xcopy /E /I /y prereq_cache\local_install C:\TEMP\argolid_bld\local_install + REM Replicate the DLL copy that install_prereq_win.bat does for PATH + if exist prereq_cache\local_install\bin ( + xcopy /E /I /y prereq_cache\local_install\bin %TEMP%\argolid\bin + ) +) else ( + echo =^> Prereq cache miss: building filepattern + pybind11 from scratch + call ci-utils\install_prereq_win.bat + if errorlevel 1 exit 1 + xcopy /E /I /y local_install C:\TEMP\argolid_bld\local_install + REM Explicitly copy DLLs to PATH location (don't rely on install_prereq_win.bat's + REM conditional copy which requires ON_GITHUB to be already set) + if exist local_install\bin ( + xcopy /E /I /y local_install\bin %TEMP%\argolid\bin + ) +) diff --git a/ci-utils/install_prereq_linux.sh b/ci-utils/install_prereq_linux.sh index 4b233bd..86ef95e 100755 --- a/ci-utils/install_prereq_linux.sh +++ b/ci-utils/install_prereq_linux.sh @@ -1,8 +1,16 @@ #!/bin/bash -# Usage: $bash install_prereq_linux.sh $INSTALL_DIR +# Usage: bash install_prereq_linux.sh [$INSTALL_DIR] [--skip-sys-deps] # Default $INSTALL_DIR = ./local_install +# --skip-sys-deps: skip building zlib, libjpeg-turbo, libpng (use when already installed) # -if [ -z "$1" ] +SKIP_SYS_DEPS=false +for arg in "$@"; do + case "$arg" in + --skip-sys-deps) SKIP_SYS_DEPS=true ;; + esac +done + +if [ -z "$1" ] || [ "$1" = "--skip-sys-deps" ] then echo "No path to the Argolid source location provided" echo "Creating local_install directory" @@ -34,14 +42,15 @@ make install -j4 cd ../../ +if [ "$SKIP_SYS_DEPS" != "true" ]; then curl -L https://github.com/madler/zlib/releases/download/v1.3.1/zlib131.zip -o zlib131.zip unzip zlib131.zip cd zlib-1.3.1 mkdir build_man cd build_man -cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local .. -cmake --build . -cmake --build . --target install +cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local .. +cmake --build . +cmake --build . --target install cd ../../ curl -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/refs/tags/3.1.0.zip -o 3.1.0.zip @@ -67,3 +76,4 @@ cd build_man cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local .. make install -j4 cd ../../ +fi diff --git a/ci-utils/install_sys_deps_linux.sh b/ci-utils/install_sys_deps_linux.sh new file mode 100755 index 0000000..24a74fb --- /dev/null +++ b/ci-utils/install_sys_deps_linux.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# install_sys_deps_linux.sh +# Builds and installs zlib, libjpeg-turbo, and libpng to /usr/local. +# Works on Linux (manylinux) and macOS. +# These are required by TensorStore when built with TENSORSTORE_USE_SYSTEM_* flags. +set -e + +JOBS=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) +BUILD_TMPDIR="$(mktemp -d)" + +# zlib +curl -L https://github.com/madler/zlib/releases/download/v1.3.1/zlib131.zip -o "${BUILD_TMPDIR}/zlib131.zip" +unzip -q "${BUILD_TMPDIR}/zlib131.zip" -d "${BUILD_TMPDIR}" +mkdir -p "${BUILD_TMPDIR}/zlib-1.3.1/build_man" +cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local \ + -S "${BUILD_TMPDIR}/zlib-1.3.1" -B "${BUILD_TMPDIR}/zlib-1.3.1/build_man" +cmake --build "${BUILD_TMPDIR}/zlib-1.3.1/build_man" -j"${JOBS}" +cmake --build "${BUILD_TMPDIR}/zlib-1.3.1/build_man" --target install + +# libjpeg-turbo +curl -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/refs/tags/3.1.0.zip \ + -o "${BUILD_TMPDIR}/3.1.0.zip" +unzip -q "${BUILD_TMPDIR}/3.1.0.zip" -d "${BUILD_TMPDIR}" +mkdir -p "${BUILD_TMPDIR}/libjpeg-turbo-3.1.0/build_man" +cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_STATIC=FALSE \ + -DCMAKE_BUILD_TYPE=Release \ + -S "${BUILD_TMPDIR}/libjpeg-turbo-3.1.0" \ + -B "${BUILD_TMPDIR}/libjpeg-turbo-3.1.0/build_man" +if [[ "$OSTYPE" == "darwin"* ]]; then + sudo cmake --build "${BUILD_TMPDIR}/libjpeg-turbo-3.1.0/build_man" \ + --target install -j"${JOBS}" +else + cmake --build "${BUILD_TMPDIR}/libjpeg-turbo-3.1.0/build_man" \ + --target install -j"${JOBS}" +fi + +# libpng +curl -L https://github.com/glennrp/libpng/archive/refs/tags/v1.6.53.zip \ + -o "${BUILD_TMPDIR}/v1.6.53.zip" +unzip -q "${BUILD_TMPDIR}/v1.6.53.zip" -d "${BUILD_TMPDIR}" +mkdir -p "${BUILD_TMPDIR}/libpng-1.6.53/build_man" +cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local \ + -S "${BUILD_TMPDIR}/libpng-1.6.53" \ + -B "${BUILD_TMPDIR}/libpng-1.6.53/build_man" +cmake --build "${BUILD_TMPDIR}/libpng-1.6.53/build_man" \ + --target install -j"${JOBS}" + +rm -rf "${BUILD_TMPDIR}" diff --git a/ci-utils/prebuild_linux.sh b/ci-utils/prebuild_linux.sh new file mode 100755 index 0000000..09ef2cb --- /dev/null +++ b/ci-utils/prebuild_linux.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# prebuild_linux.sh +# Runs inside the manylinux_2_28_x86_64 Docker container during the prebuild job. +# Populates: +# /project/prereq_cache/local_install — filepattern + pybind11 headers/libs +# /project/ts_fc_cache/ — TensorStore FetchContent source + build +set -e + +cd /project + +# Install NASM (required by aom/dav1d inside TensorStore) +curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 \ + -o nasm.tar.bz2 +tar -xjf nasm.tar.bz2 +cd nasm-2.15.05 && ./configure && make -j"$(nproc)" && make install +cd /project + +# Install ccache and point it at the workspace-mounted cache dir +dnf install -y ccache +mkdir -p /project/ccache_dir +export CCACHE_DIR=/project/ccache_dir +export CCACHE_MAXSIZE=5G + +# Build zlib, libjpeg-turbo, libpng to /usr/local (needed for TENSORSTORE_USE_SYSTEM_* flags) +bash ci-utils/install_sys_deps_linux.sh + +# Build filepattern + pybind11 → prereq_cache/local_install +# (--skip-sys-deps: sys deps already installed above) +bash ci-utils/install_prereq_linux.sh ./prereq_cache/local_install --skip-sys-deps + +# Pre-build TensorStore using the minimal cmake project (no Python, no pybind11 needed). +# FETCHCONTENT_BASE_DIR stores TensorStore source + build state in ts_fc_cache/ +# so wheel-build jobs can skip the expensive TensorStore download and reuse cmake state. +cmake -S /project/ci-utils/tensorstore_prebuild \ + -B /tmp/ts_cmake_prebuild \ + -DFETCHCONTENT_BASE_DIR=/project/ts_fc_cache \ + -DTENSORSTORE_USE_SYSTEM_JPEG=ON \ + -DTENSORSTORE_USE_SYSTEM_ZLIB=ON \ + -DTENSORSTORE_USE_SYSTEM_PNG=ON \ + -DCMAKE_BUILD_TYPE=Release +cmake --build /tmp/ts_cmake_prebuild -j"$(nproc)" diff --git a/ci-utils/tensorstore_prebuild/CMakeLists.txt b/ci-utils/tensorstore_prebuild/CMakeLists.txt new file mode 100644 index 0000000..5a30369 --- /dev/null +++ b/ci-utils/tensorstore_prebuild/CMakeLists.txt @@ -0,0 +1,33 @@ +cmake_minimum_required(VERSION 3.24) +project(TensorStorePrebuild) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# Use ccache if available to speed up repeated TensorStore compilation in CI +find_program(CCACHE_EXECUTABLE ccache) +if(CCACHE_EXECUTABLE) + message(STATUS "ccache found: ${CCACHE_EXECUTABLE}") + set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_EXECUTABLE}" CACHE STRING "" FORCE) + set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE_EXECUTABLE}" CACHE STRING "" FORCE) +endif() + +include(FetchContent) + +# Keep URL + SHA256 in sync with root CMakeLists.txt +# Cache key hashes both this file and root CMakeLists.txt +FetchContent_Declare( + tensorstore + URL "https://github.com/google/tensorstore/archive/refs/tags/v0.1.80.tar.gz" + URL_HASH SHA256=94866de34b6139d77d30e828a50f9e8df98e7dd68e848393470879aeb50ea7bf +) + +FetchContent_MakeAvailable(tensorstore) + +# Executable target forces TensorStore and all its transitive deps to be compiled. +# No Python, no pybind11, no filepattern needed. +add_executable(ts_prebuild_marker ts_prebuild_marker.cc) +target_link_libraries(ts_prebuild_marker PRIVATE + tensorstore::tensorstore + tensorstore::all_drivers +) diff --git a/ci-utils/tensorstore_prebuild/ts_prebuild_marker.cc b/ci-utils/tensorstore_prebuild/ts_prebuild_marker.cc new file mode 100644 index 0000000..9ac9b7d --- /dev/null +++ b/ci-utils/tensorstore_prebuild/ts_prebuild_marker.cc @@ -0,0 +1,6 @@ +#include "tensorstore/open.h" + +// Minimal stub that forces the linker to pull in TensorStore symbols. +// This file exists solely to make ts_prebuild_marker a non-trivial executable +// so cmake compiles all TensorStore transitive dependencies during the prebuild. +int main() { return 0; }