diff --git a/.github/workflows/build-cpp-runtime-bindings.yml b/.github/workflows/build-cpp-runtime-bindings.yml index 81ba47ec0..2066166c2 100644 --- a/.github/workflows/build-cpp-runtime-bindings.yml +++ b/.github/workflows/build-cpp-runtime-bindings.yml @@ -67,6 +67,13 @@ jobs: path: svs-cpp-runtime-bindings${{ matrix.suffix }}.tar.gz retention-days: 7 # Reduce retention due to size + - name: Upload conda package artifacts + uses: actions/upload-artifact@v4 + with: + name: libsvs-runtime-conda${{ matrix.suffix }} + path: conda-bld/linux-64/libsvs-runtime-*.conda + retention-days: 7 + # Run unit tests that were built as part of this job - name: Run unit tests in Docker container run: | @@ -97,21 +104,21 @@ jobs: docker build -t svs-manylinux228:latest -f docker/x86_64/manylinux228/Dockerfile . # Need to download for a new job - - name: Download shared libraries + - name: Download conda package uses: actions/download-artifact@v7 with: - name: svs-cpp-runtime-bindings${{ matrix.suffix }} - path: runtime_lib + name: libsvs-runtime-conda${{ matrix.suffix }} + path: runtime_conda - name: List available artifacts run: | - ls -la runtime_lib/ + ls -la runtime_conda/ - name: Test in Docker container run: | docker run --rm \ -v ${{ github.workspace }}:/workspace \ - -v ${{ github.workspace }}/runtime_lib:/runtime_lib \ + -v ${{ github.workspace }}/runtime_conda:/runtime_conda \ -w /workspace \ -e SUFFIX=${{ matrix.suffix }} \ svs-manylinux228:latest \ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 244476136..2601df03a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,7 @@ repos: hooks: - id: check-yaml files: \.ya?ml$ + exclude: conda-recipe/meta\.yaml$ - id: check-added-large-files - id: check-illegal-windows-names - id: check-json diff --git a/bindings/cpp/conda-recipe/build.sh b/bindings/cpp/conda-recipe/build.sh new file mode 100644 index 000000000..8b4347019 --- /dev/null +++ b/bindings/cpp/conda-recipe/build.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Copyright 2025 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# Source gcc-toolset-11 to ensure we use the system's GCC 11 +# This is required because we are not using Conda compilers +if [ -f "/opt/rh/gcc-toolset-11/enable" ]; then + source /opt/rh/gcc-toolset-11/enable +elif command -v scl_source >/dev/null 2>&1; then + source scl_source enable gcc-toolset-11 || true +else + echo "WARNING: gcc-toolset-11 not found, proceeding without sourcing it" +fi + +# build runtime tests flag? +CMAKE_ARGS=( + "-DCMAKE_INSTALL_PREFIX=${PREFIX}" + "-DSVS_BUILD_RUNTIME_TESTS=OFF" + "-DSVS_RUNTIME_ENABLE_LVQ_LEANVEC=${ENABLE_LVQ_LEANVEC:-ON}" +) + +# Add SVS_URL if specified (for fetching static library) +if [ -n "$SVS_URL" ]; then + CMAKE_ARGS+=("-DSVS_URL=$SVS_URL") +fi + +cmake -B build "${CMAKE_ARGS[@]}" -S bindings/cpp + +cmake --build build -j +cmake --install build + +# Create lib64 symlink if needed (for compatibility) +cd "${PREFIX}" +if [ ! -e lib64 ] && [ -d lib ]; then + ln -s lib lib64 +fi diff --git a/bindings/cpp/conda-recipe/meta.yaml b/bindings/cpp/conda-recipe/meta.yaml new file mode 100644 index 000000000..2f75ac699 --- /dev/null +++ b/bindings/cpp/conda-recipe/meta.yaml @@ -0,0 +1,61 @@ +# Copyright 2025 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +{% set version = '0.1.0' %} +{% set buildnumber = 0 %} +{% set variant_suffix = environ.get('SUFFIX', '').replace('-', '_') %} + +package: + name: libsvs-runtime + version: {{ version }} + +source: + path: ../../.. + +build: + number: {{ buildnumber }} + string: {{ GIT_DESCRIBE_HASH }}_{{ buildnumber }}{{ variant_suffix }} + skip: true # [not linux] + include_recipe: False + +requirements: + build: + # required for MAP_HUGE_* defines in allocator.h + - kernel-headers_linux-64 ==4.18.0 # [linux] + - python 3.10.* # Build tool only + run: + - python >=3.10 + +# Simple test validation handled in GitHub Actions + +about: + home: https://intel.github.io/ScalableVectorSearch/index.html + license: Intel Simplified Software License (Version October 2022) + license_file: + - LICENSE + - THIRD-PARTY-PROGRAMS + summary: SVS C++ shared libraries for CMake integration + description: | + LEGAL NOTICE: Use of this software package is subject to the + software license agreement (as set forth above, in the license section of + the installed Conda package and/or the README file) and all notices, + disclaimers or license terms for third party or open source software + included in or with the software. +

+ EULA: Intel Simplified Software License (Version October 2022) +

+ This package provides pre-compiled SVS shared and static libraries with CMake configuration files for C++ development. + dev_url: https://github.com/intel/ScalableVectorSearch + doc_url: https://intel.github.io/ScalableVectorSearch/index.html diff --git a/docker/x86_64/build-cpp-runtime-bindings.sh b/docker/x86_64/build-cpp-runtime-bindings.sh index ab8897482..1597382f1 100644 --- a/docker/x86_64/build-cpp-runtime-bindings.sh +++ b/docker/x86_64/build-cpp-runtime-bindings.sh @@ -42,6 +42,11 @@ CC=gcc CXX=g++ cmake .. "${CMAKE_ARGS[@]}" cmake --build . -j cmake --install . +# Build conda package for cpp runtime bindings +source /opt/conda/etc/profile.d/conda.sh +cd /workspace +ENABLE_LVQ_LEANVEC=${ENABLE_LVQ_LEANVEC:-ON} SVS_URL="${SVS_URL}" SUFFIX="${SUFFIX}" conda build bindings/cpp/conda-recipe --output-folder /workspace/conda-bld + # Create tarball with symlink for compatibility cd /workspace/install_cpp_bindings && \ ln -s lib lib64 && \ diff --git a/docker/x86_64/manylinux228/Dockerfile b/docker/x86_64/manylinux228/Dockerfile index e8582b554..627327156 100644 --- a/docker/x86_64/manylinux228/Dockerfile +++ b/docker/x86_64/manylinux228/Dockerfile @@ -32,5 +32,7 @@ RUN echo '# Configure gcc-11' > /etc/profile.d/01-gcc.sh && \ # Download and install Miniforge RUN wget -nv https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh -O /tmp/miniforge.sh && \ bash /tmp/miniforge.sh -b -p /opt/conda && \ - rm /tmp/miniforge.sh + rm /tmp/miniforge.sh && \ + /opt/conda/bin/conda install -y conda-build && \ + /opt/conda/bin/conda clean -afy ENV PATH="/opt/conda/bin:$PATH" diff --git a/docker/x86_64/test-cpp-runtime-bindings.sh b/docker/x86_64/test-cpp-runtime-bindings.sh index f0251294c..143da1f47 100644 --- a/docker/x86_64/test-cpp-runtime-bindings.sh +++ b/docker/x86_64/test-cpp-runtime-bindings.sh @@ -28,8 +28,11 @@ conda install -y -c conda-forge cmake=3.30.4 make=4.2 swig=4.0 "numpy>=2.0,<3.0" conda install -y -c conda-forge gxx_linux-64=14.2 sysroot_linux-64=2.17 conda install -y mkl=2022.2.1 mkl-devel=2022.2.1 +# Install libsvs-runtime from local conda package +conda install -y /runtime_conda/libsvs-runtime-*.conda + # Validate python and C++ tests against FAISS CI -git clone https://github.com/facebookresearch/faiss.git +git clone -b eglaser/svs-runtime-conda-enabling https://github.com/ethanglaser/faiss.git cd faiss echo "===============================================" @@ -37,7 +40,7 @@ echo " Running validation of library against FAISS CI" echo "-----------------------------------------------" echo " FAISS Build: " mkdir build && cd build -cmake -DBUILD_TESTING=ON -DFAISS_ENABLE_SVS=ON -DFAISS_ENABLE_GPU=OFF -DSVS_URL="file:///runtime_lib/svs-cpp-runtime-bindings${SUFFIX}.tar.gz" .. +cmake -DBUILD_TESTING=ON -DFAISS_ENABLE_SVS=ON -DFAISS_ENABLE_GPU=OFF .. make -j$(nproc) swigfaiss faiss_test echo "-----------------------------------------------" echo " FAISS C++ tests: "