diff --git a/scripts/build_pybind_from_subdir.sh b/scripts/build_pybind_from_subdir.sh index b26788f..711d7ff 100755 --- a/scripts/build_pybind_from_subdir.sh +++ b/scripts/build_pybind_from_subdir.sh @@ -31,24 +31,93 @@ export CCACHE_TEMPDIR mkdir -p "${CCACHE_TEMPDIR}" -rm -rf "${BUILD_DIR}" +# --------------------------------------------------------------------------- +# Generator selection +# --------------------------------------------------------------------------- +# Honour GEN env var (e.g. GEN=Unix\ Makefiles), otherwise default to Ninja. +if [[ -n "${GEN:-}" ]]; then + GENERATOR="${GEN}" +else + GENERATOR="Ninja" +fi +echo "[pybind] Generator: ${GENERATOR}" + +# If the build directory exists with a different generator, wipe it so +# CMake doesn't error out with a generator mismatch. +if [[ -f "${BUILD_DIR}/CMakeCache.txt" ]]; then + cached_gen="$(grep -m1 'CMAKE_GENERATOR:INTERNAL=' "${BUILD_DIR}/CMakeCache.txt" 2>/dev/null | cut -d= -f2)" + if [[ -n "${cached_gen}" && "${cached_gen}" != "${GENERATOR}" ]]; then + echo "[pybind] Generator changed from '${cached_gen}' to '${GENERATOR}' — wiping stale build directory." + rm -rf "${BUILD_DIR}" + fi +fi + +# --------------------------------------------------------------------------- +# CMake configuration (always re-runs if CMakeLists.txt changed) +# --------------------------------------------------------------------------- +# Only wipe the build tree when BUILD_CLEAN=1 is set explicitly. +if [[ "${BUILD_CLEAN:-0}" == "1" ]]; then + echo "[pybind] BUILD_CLEAN=1 — removing ${BUILD_DIR}" + rm -rf "${BUILD_DIR}" +fi + +cmake_args=( + -S "${ROOT_DIR}" + -B "${BUILD_DIR}" + -G "${GENERATOR}" + -DCMAKE_BUILD_TYPE=Release + -DLBUG_SOURCE_DIR="${LBUG_DIR}" + -DPYTHON_EXECUTABLE="${PYTHON_BIN}" + -DPython_EXECUTABLE="${PYTHON_BIN}" + -DPython3_EXECUTABLE="${PYTHON_BIN}" + -DPYBIND11_PYTHON_VERSION="${PYTHON_VERSION}" +) + +# Wire in ccache if it is available (vastly speeds up rebuilds). +if command -v ccache &>/dev/null; then + cmake_args+=( + -DCMAKE_C_COMPILER_LAUNCHER=ccache + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + ) +fi + +cmake "${cmake_args[@]}" + +# --------------------------------------------------------------------------- +# Build +# --------------------------------------------------------------------------- +# Parallelism: honour CMAKE_BUILD_PARALLEL_LEVEL or PARALLEL first, +# then fall back to the number of logical CPUs. +if [[ -n "${CMAKE_BUILD_PARALLEL_LEVEL:-}" ]]; then + NPROC="${CMAKE_BUILD_PARALLEL_LEVEL}" +elif [[ -n "${PARALLEL:-}" ]]; then + NPROC="${PARALLEL}" +else + NPROC="$(nproc 2>/dev/null || echo 4)" +fi -cmake \ - -S "${ROOT_DIR}" \ - -B "${BUILD_DIR}" \ - -DCMAKE_BUILD_TYPE=Release \ - -DLBUG_SOURCE_DIR="${LBUG_DIR}" \ - -DPYTHON_EXECUTABLE="${PYTHON_BIN}" \ - -DPython_EXECUTABLE="${PYTHON_BIN}" \ - -DPython3_EXECUTABLE="${PYTHON_BIN}" \ - -DPYBIND11_PYTHON_VERSION="${PYTHON_VERSION}" +build_args=( + --build "${BUILD_DIR}" + --config Release + --target _lbug + --parallel "${NPROC}" +) -cmake --build "${BUILD_DIR}" --config Release --target _lbug +echo "[pybind] Starting build with ${NPROC} parallel job(s) ..." +cmake "${build_args[@]}" -if compgen -G "${ROOT_DIR}/build/ladybug/_lbug*" > /dev/null; then +# --------------------------------------------------------------------------- +# Post-build check +# --------------------------------------------------------------------------- +if compgen -G "${ROOT_DIR}/build/ladybug/_lbug*" &>/dev/null; then echo "[pybind] Built extension into ${ROOT_DIR}/build/ladybug" +elif compgen -G "${BUILD_DIR}/ladybug/_lbug*" &>/dev/null; then + echo "[pybind] Built extension into ${BUILD_DIR}/ladybug" + # Copy to expected location so Makefile's test target finds it. + mkdir -p "${ROOT_DIR}/build/ladybug" + cp "${BUILD_DIR}"/ladybug/_lbug* "${ROOT_DIR}/build/ladybug/" else echo "[pybind] Build finished, but no _lbug extension artifact was found." >&2 - echo "Checked: ${ROOT_DIR}/build/ladybug" >&2 + echo "Checked: ${ROOT_DIR}/build/ladybug and ${BUILD_DIR}/ladybug" >&2 exit 1 fi diff --git a/src_cpp/pandas/pandas_bind.cpp b/src_cpp/pandas/pandas_bind.cpp index 62672af..de36918 100644 --- a/src_cpp/pandas/pandas_bind.cpp +++ b/src_cpp/pandas/pandas_bind.cpp @@ -7,6 +7,18 @@ namespace lbug { using namespace lbug::common; +// Ensure a numpy array is C-contiguous so that the scan code can safely use +// memcpy on the raw data pointer. Non-contiguous arrays can arise from +// DataFrame columns backed by views into larger 2D arrays (e.g. after +// transpose, concat, or slicing). +static py::array ensureContiguous(py::object obj) { + auto arr = py::array(obj); + if (!arr.attr("flags").attr("c_contiguous").cast()) { + arr = py::module_::import("numpy").attr("ascontiguousarray")(arr); + } + return arr; +} + struct PandasBindColumn { public: PandasBindColumn(py::handle name, py::handle type, py::object column) @@ -54,9 +66,8 @@ static common::LogicalType bindColumn(PandasBindColumn& bindColumn, } if (bindData->npType.type == NumpyNullableType::FLOAT_16) { - auto pandasArray = column.attr("array"); bindData->pandasCol = - std::make_unique(py::array(column.attr("to_numpy")("float32"))); + std::make_unique(ensureContiguous(column.attr("to_numpy")("float32"))); bindData->npType.type = NumpyNullableType::FLOAT_32; columnType = NumpyTypeUtils::numpyToLogicalType(bindData->npType); } else { @@ -64,15 +75,15 @@ static common::LogicalType bindColumn(PandasBindColumn& bindColumn, if (py::hasattr(pandasArray, "_data")) { // This means we can access the numpy array directly. bindData->pandasCol = - std::make_unique(column.attr("array").attr("_data")); + std::make_unique(ensureContiguous(column.attr("array").attr("_data"))); } else if (py::hasattr(pandasArray, "asi8")) { // This is a datetime object, has the option to get the array as int64_t's. bindData->pandasCol = - std::make_unique(py::array(pandasArray.attr("asi8"))); + std::make_unique(ensureContiguous(pandasArray.attr("asi8"))); } else { // Otherwise we have to get it through 'to_numpy()'. bindData->pandasCol = - std::make_unique(py::array(column.attr("to_numpy")())); + std::make_unique(ensureContiguous(column.attr("to_numpy")())); } columnType = NumpyTypeUtils::numpyToLogicalType(bindData->npType); }