From 3caf74b4f827fb6d569a7e300794be081c998eb1 Mon Sep 17 00:00:00 2001 From: Jonathan Shimwell Date: Sun, 21 Apr 2024 22:34:43 +0100 Subject: [PATCH 1/3] moved apt get to optional ci parts --- .github/workflows/ci.yml | 15 ++++++++++++--- tools/ci/gha-install-vectfit.sh | 2 -- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5accc5e0f3..a1de02616f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,16 +111,25 @@ jobs: run: | sudo apt -y update sudo apt install -y libpng-dev \ - libmpich-dev \ libnetcdf-dev \ libpnetcdf-dev \ libhdf5-serial-dev \ - libhdf5-mpich-dev \ - libeigen3-dev + libeigen3-dev \ + + - name: Optional apt dependencies for mpi + shell: bash + run: | + sudo apt install -y libhdf5-mpich-dev \ + libmpich-dev sudo update-alternatives --set mpi /usr/bin/mpicc.mpich sudo update-alternatives --set mpirun /usr/bin/mpirun.mpich sudo update-alternatives --set mpi-x86_64-linux-gnu /usr/include/x86_64-linux-gnu/mpich + - name: Optional apt dependencies for vectfit + shell: bash + if: ${{ matrix.vectfit == 'y' }} + run: sudo apt-get install -y libblas-dev liblapack-dev + - name: install shell: bash run: | diff --git a/tools/ci/gha-install-vectfit.sh b/tools/ci/gha-install-vectfit.sh index 8444c3036b3..bd38e1ea8cd 100755 --- a/tools/ci/gha-install-vectfit.sh +++ b/tools/ci/gha-install-vectfit.sh @@ -16,8 +16,6 @@ XTENSOR_PYTHON_REPO='https://github.com/xtensor-stack/xtensor-python' XTENSOR_BLAS_BRANCH='0.17.1' XTENSOR_BLAS_REPO='https://github.com/xtensor-stack/xtensor-blas' -sudo apt-get install -y libblas-dev liblapack-dev - cd $HOME git clone -b $PYBIND_BRANCH $PYBIND_REPO cd pybind11 && mkdir build && cd build && cmake .. && sudo make install From 2705b9c92ac1f87c0834d6d6b206846a76d9e01b Mon Sep 17 00:00:00 2001 From: Jonathan Shimwell Date: Sun, 21 Apr 2024 23:58:22 +0100 Subject: [PATCH 2/3] move many scripts to ci --- .github/workflows/ci.yml | 179 ++++++++++++++++++++++++++++++- tools/ci/gha-install-dagmc.sh | 35 ------ tools/ci/gha-install-libmesh.sh | 22 ---- tools/ci/gha-install-mcpl.sh | 7 -- tools/ci/gha-install-ncrystal.sh | 46 -------- tools/ci/gha-install-njoy.sh | 7 -- tools/ci/gha-install-vectfit.sh | 46 -------- 7 files changed, 174 insertions(+), 168 deletions(-) delete mode 100755 tools/ci/gha-install-dagmc.sh delete mode 100755 tools/ci/gha-install-libmesh.sh delete mode 100755 tools/ci/gha-install-mcpl.sh delete mode 100755 tools/ci/gha-install-ncrystal.sh delete mode 100755 tools/ci/gha-install-njoy.sh delete mode 100755 tools/ci/gha-install-vectfit.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1de02616f8..b31952a1179 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,6 +118,7 @@ jobs: - name: Optional apt dependencies for mpi shell: bash + if: ${{ matrix.mpi == 'y' }} run: | sudo apt install -y libhdf5-mpich-dev \ libmpich-dev @@ -125,16 +126,182 @@ jobs: sudo update-alternatives --set mpirun /usr/bin/mpirun.mpich sudo update-alternatives --set mpi-x86_64-linux-gnu /usr/include/x86_64-linux-gnu/mpich + # Upgrade pip, pytest, numpy before doing anything else. + - name: update pip, pytest, numpy and install NJOY + shell: bash + run: | + pip install --upgrade pip + pip install --upgrade pytest + pip install --upgrade numpy + echo "$HOME/NJOY2016/build" >> $GITHUB_PATH + set -ex + cd $HOME + git clone https://github.com/njoy/NJOY2016 + cd NJOY2016 + mkdir build && cd build + cmake -Dstatic=on .. && make 2>/dev/null && sudo make install + + - name: Optional install DAGMC + shell: bash + if: ${{ matrix.dagmc == 'y' }} + run: | + set -ex + MOAB_BRANCH='Version5.1.0' + MOAB_REPO='https://bitbucket.org/fathomteam/moab/' + MOAB_INSTALL_DIR=$HOME/MOAB/ + DAGMC_BRANCH='develop' + DAGMC_REPO='https://github.com/svalinn/dagmc' + DAGMC_INSTALL_DIR=$HOME/DAGMC/ + CURRENT_DIR=$(pwd) + cd $HOME + mkdir MOAB && cd MOAB + git clone -b $MOAB_BRANCH $MOAB_REPO + mkdir build && cd build + cmake ../moab -DENABLE_HDF5=ON -DENABLE_NETCDF=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=$MOAB_INSTALL_DIR -DENABLE_BLASLAPACK=OFF + make -j && make -j install + rm -rf $HOME/MOAB/moab $HOME/MOAB/build + cd $HOME + mkdir DAGMC && cd DAGMC + git clone -b $DAGMC_BRANCH $DAGMC_REPO + mkdir build && cd build + cmake ../dagmc -DBUILD_TALLY=ON -DCMAKE_INSTALL_PREFIX=$DAGMC_INSTALL_DIR -DBUILD_STATIC_LIBS=OFF -DMOAB_DIR=$MOAB_INSTALL_DIR + make -j install + rm -rf $HOME/DAGMC/dagmc $HOME/DAGMC/build + cd $CURRENT_DIR + $GITHUB_WORKSPACE/tools/ci/gha-install.sh + + - name: Optional install NCRYSTAL + shell: bash + if: ${{ matrix.ncrystal == 'y' }} + run: | + set -ex + cd $HOME + #Use the NCrystal develop branch (in the near future we can move this to master): + git clone https://github.com/mctools/ncrystal --branch develop --single-branch --depth 1 ncrystal_src + SRC_DIR="$PWD/ncrystal_src" + BLD_DIR="$PWD/ncrystal_bld" + INST_DIR="$PWD/ncrystal_inst" + PYTHON=$(which python3) + CPU_COUNT=1 + mkdir "$BLD_DIR" + cd ncrystal_bld + cmake \ + "${SRC_DIR}" \ + -DBUILD_SHARED_LIBS=ON \ + -DNCRYSTAL_NOTOUCH_CMAKE_BUILD_TYPE=ON \ + -DNCRYSTAL_MODIFY_RPATH=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DNCRYSTAL_ENABLE_EXAMPLES=OFF \ + -DNCRYSTAL_ENABLE_SETUPSH=OFF \ + -DNCRYSTAL_ENABLE_DATA=EMBED \ + -DCMAKE_INSTALL_PREFIX="${INST_DIR}" \ + -DPython3_EXECUTABLE="$PYTHON" + make -j${CPU_COUNT:-1} + make install + #Note: There is no "make test" or "make ctest" functionality for NCrystal + # yet. If it appears in the future, we should add it here. + # Output the configuration to the log + "${INST_DIR}/bin/ncrystal-config" --setup + # Change environmental variables + eval $( "${INST_DIR}/bin/ncrystal-config" --setup ) + # Check installation worked + nctool --test + - name: Optional apt dependencies for vectfit shell: bash if: ${{ matrix.vectfit == 'y' }} - run: sudo apt-get install -y libblas-dev liblapack-dev + run: | + sudo apt-get install -y libblas-dev liblapack-dev + set -ex + PYBIND_BRANCH='master' + PYBIND_REPO='https://github.com/pybind/pybind11' + XTL_BRANCH='0.6.13' + XTL_REPO='https://github.com/xtensor-stack/xtl' + XTENSOR_BRANCH='0.21.3' + XTENSOR_REPO='https://github.com/xtensor-stack/xtensor' + XTENSOR_PYTHON_BRANCH='0.24.1' + XTENSOR_PYTHON_REPO='https://github.com/xtensor-stack/xtensor-python' + XTENSOR_BLAS_BRANCH='0.17.1' + XTENSOR_BLAS_REPO='https://github.com/xtensor-stack/xtensor-blas' + cd $HOME + git clone -b $PYBIND_BRANCH $PYBIND_REPO + cd pybind11 && mkdir build && cd build && cmake .. && sudo make install + pip install $HOME/pybind11 + cd $HOME + git clone -b $XTL_BRANCH $XTL_REPO + cd xtl && mkdir build && cd build && cmake .. && sudo make install + cd $HOME + git clone -b $XTENSOR_BRANCH $XTENSOR_REPO + cd xtensor && mkdir build && cd build && cmake .. && sudo make install + cd $HOME + git clone -b $XTENSOR_PYTHON_BRANCH $XTENSOR_PYTHON_REPO + cd xtensor-python && mkdir build && cd build && cmake .. && sudo make install + cd $HOME + git clone -b $XTENSOR_BLAS_BRANCH $XTENSOR_BLAS_REPO + cd xtensor-blas && mkdir build && cd build && cmake .. && sudo make install + # Install wheel (remove when vectfit supports installation with build isolation) + pip install wheel + # Install vectfit + cd $HOME + git clone https://github.com/liangjg/vectfit.git + pip install --no-build-isolation ./vectfit - - name: install + - name: Optional apt dependencies for LIBMESH shell: bash + if: ${{ matrix.libmesh == 'y' }} run: | - echo "$HOME/NJOY2016/build" >> $GITHUB_PATH - $GITHUB_WORKSPACE/tools/ci/gha-install.sh + set -ex + # libMESH install + pushd $HOME + mkdir LIBMESH && cd LIBMESH + git clone https://github.com/libmesh/libmesh -b v1.7.1 --recurse-submodules + mkdir build && cd build + export METHODS="opt" + + - name: Optional apt dependencies for LIBMESH + shell: bash + if: ${{ matrix.libmesh == 'y' && matrix.mpi == 'y' }} + run: | + ../libmesh/configure --prefix=$HOME/LIBMESH CXX=mpicxx CC=mpicc FC=mpifort F77=mpif77 \ + --enable-exodus --disable-netcdf-4 --disable-eigen --disable-lapack + + - name: Optional apt dependencies for LIBMESH + shell: bash + if: ${{ matrix.libmesh == 'y' && matrix.mpi == 'n' }} + run: | + ../libmesh/configure --prefix=$HOME/LIBMESH --enable-exodus --disable-netcdf-4 --disable-eigen --disable-lapack --disable-mpi + + - name: Optional apt dependencies for LIBMESH + shell: bash + if: ${{ matrix.libmesh == 'y' }} + run: | + make -j2 install + export LIBMESH_PC=$HOME/LIBMESH/lib/pkgconfig/ + rm -rf $HOME/LIBMESH/build + popd + + - name: Install MCPL + shell: bash + run: | + set -ex + cd $HOME + git clone https://github.com/mctools/mcpl + cd mcpl + mkdir build && cd build + cmake .. && make 2>/dev/null && sudo make install + + # For MPI configurations, make sure mpi4py and h5py are built against the + # correct version of MPI + - name: Optional apt dependencies for MPI + shell: bash + if: ${{ matrix.mpi == 'y' }} + run: | + pip install --no-binary=mpi4py mpi4py + export CC=mpicc + export HDF5_MPI=ON + export HDF5_DIR=/usr/lib/x86_64-linux-gnu/hdf5/mpich + pip install wheel "cython<3.0" + pip install --no-binary=h5py --no-build-isolation h5py - name: cache-xs uses: actions/cache@v4 @@ -146,7 +313,9 @@ jobs: - name: before shell: bash - run: $GITHUB_WORKSPACE/tools/ci/gha-before-script.sh + run: + wget -q -O - https://anl.box.com/shared/static/teaup95cqv8s9nn56hfn7ku8mmelr95p.xz | tar -C $HOME -xJ + wget -q -O - https://anl.box.com/shared/static/4kd2gxnf4gtk4w1c8eua5fsua22kvgjb.xz | tar -C $HOME -xJ - name: test shell: bash diff --git a/tools/ci/gha-install-dagmc.sh b/tools/ci/gha-install-dagmc.sh deleted file mode 100755 index 82759c9bcce..00000000000 --- a/tools/ci/gha-install-dagmc.sh +++ /dev/null @@ -1,35 +0,0 @@ - -#!/bin/bash -set -ex - -# MOAB Variables -MOAB_BRANCH='Version5.1.0' -MOAB_REPO='https://bitbucket.org/fathomteam/moab/' -MOAB_INSTALL_DIR=$HOME/MOAB/ - -# DAGMC Variables -DAGMC_BRANCH='develop' -DAGMC_REPO='https://github.com/svalinn/dagmc' -DAGMC_INSTALL_DIR=$HOME/DAGMC/ - -CURRENT_DIR=$(pwd) - -# MOAB Install -cd $HOME -mkdir MOAB && cd MOAB -git clone -b $MOAB_BRANCH $MOAB_REPO -mkdir build && cd build -cmake ../moab -DENABLE_HDF5=ON -DENABLE_NETCDF=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=$MOAB_INSTALL_DIR -DENABLE_BLASLAPACK=OFF -make -j && make -j install -rm -rf $HOME/MOAB/moab $HOME/MOAB/build - -# DAGMC Install -cd $HOME -mkdir DAGMC && cd DAGMC -git clone -b $DAGMC_BRANCH $DAGMC_REPO -mkdir build && cd build -cmake ../dagmc -DBUILD_TALLY=ON -DCMAKE_INSTALL_PREFIX=$DAGMC_INSTALL_DIR -DBUILD_STATIC_LIBS=OFF -DMOAB_DIR=$MOAB_INSTALL_DIR -make -j install -rm -rf $HOME/DAGMC/dagmc $HOME/DAGMC/build - -cd $CURRENT_DIR diff --git a/tools/ci/gha-install-libmesh.sh b/tools/ci/gha-install-libmesh.sh deleted file mode 100755 index cb808ae5b3b..00000000000 --- a/tools/ci/gha-install-libmesh.sh +++ /dev/null @@ -1,22 +0,0 @@ - -#!/bin/bash -set -ex - -# libMESH install -pushd $HOME -mkdir LIBMESH && cd LIBMESH -git clone https://github.com/libmesh/libmesh -b v1.7.1 --recurse-submodules -mkdir build && cd build -export METHODS="opt" - -if [[ $MPI == 'y' ]]; then - ../libmesh/configure --prefix=$HOME/LIBMESH CXX=mpicxx CC=mpicc FC=mpifort F77=mpif77 \ - --enable-exodus --disable-netcdf-4 --disable-eigen --disable-lapack -else - ../libmesh/configure --prefix=$HOME/LIBMESH --enable-exodus --disable-netcdf-4 --disable-eigen --disable-lapack --disable-mpi -fi -make -j4 install -export LIBMESH_PC=$HOME/LIBMESH/lib/pkgconfig/ -rm -rf $HOME/LIBMESH/build - -popd diff --git a/tools/ci/gha-install-mcpl.sh b/tools/ci/gha-install-mcpl.sh deleted file mode 100755 index 9b8609398a0..00000000000 --- a/tools/ci/gha-install-mcpl.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -ex -cd $HOME -git clone https://github.com/mctools/mcpl -cd mcpl -mkdir build && cd build -cmake .. && make 2>/dev/null && sudo make install diff --git a/tools/ci/gha-install-ncrystal.sh b/tools/ci/gha-install-ncrystal.sh deleted file mode 100755 index 16f77e13e2b..00000000000 --- a/tools/ci/gha-install-ncrystal.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -set -ex -cd $HOME - -#Use the NCrystal develop branch (in the near future we can move this to master): -git clone https://github.com/mctools/ncrystal --branch develop --single-branch --depth 1 ncrystal_src - -SRC_DIR="$PWD/ncrystal_src" -BLD_DIR="$PWD/ncrystal_bld" -INST_DIR="$PWD/ncrystal_inst" -PYTHON=$(which python3) - -CPU_COUNT=1 - -mkdir "$BLD_DIR" -cd ncrystal_bld - -cmake \ - "${SRC_DIR}" \ - -DBUILD_SHARED_LIBS=ON \ - -DNCRYSTAL_NOTOUCH_CMAKE_BUILD_TYPE=ON \ - -DNCRYSTAL_MODIFY_RPATH=OFF \ - -DCMAKE_BUILD_TYPE=Release \ - -DNCRYSTAL_ENABLE_EXAMPLES=OFF \ - -DNCRYSTAL_ENABLE_SETUPSH=OFF \ - -DNCRYSTAL_ENABLE_DATA=EMBED \ - -DCMAKE_INSTALL_PREFIX="${INST_DIR}" \ - -DPython3_EXECUTABLE="$PYTHON" - -make -j${CPU_COUNT:-1} -make install - -#Note: There is no "make test" or "make ctest" functionality for NCrystal -# yet. If it appears in the future, we should add it here. - -# Output the configuration to the log - -"${INST_DIR}/bin/ncrystal-config" --setup - -# Change environmental variables - -eval $( "${INST_DIR}/bin/ncrystal-config" --setup ) - -# Check installation worked - -nctool --test diff --git a/tools/ci/gha-install-njoy.sh b/tools/ci/gha-install-njoy.sh deleted file mode 100755 index 8255ffea836..00000000000 --- a/tools/ci/gha-install-njoy.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -ex -cd $HOME -git clone https://github.com/njoy/NJOY2016 -cd NJOY2016 -mkdir build && cd build -cmake -Dstatic=on .. && make 2>/dev/null && sudo make install diff --git a/tools/ci/gha-install-vectfit.sh b/tools/ci/gha-install-vectfit.sh deleted file mode 100755 index bd38e1ea8cd..00000000000 --- a/tools/ci/gha-install-vectfit.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -set -ex - -PYBIND_BRANCH='master' -PYBIND_REPO='https://github.com/pybind/pybind11' - -XTL_BRANCH='0.6.13' -XTL_REPO='https://github.com/xtensor-stack/xtl' - -XTENSOR_BRANCH='0.21.3' -XTENSOR_REPO='https://github.com/xtensor-stack/xtensor' - -XTENSOR_PYTHON_BRANCH='0.24.1' -XTENSOR_PYTHON_REPO='https://github.com/xtensor-stack/xtensor-python' - -XTENSOR_BLAS_BRANCH='0.17.1' -XTENSOR_BLAS_REPO='https://github.com/xtensor-stack/xtensor-blas' - -cd $HOME -git clone -b $PYBIND_BRANCH $PYBIND_REPO -cd pybind11 && mkdir build && cd build && cmake .. && sudo make install -pip install $HOME/pybind11 - -cd $HOME -git clone -b $XTL_BRANCH $XTL_REPO -cd xtl && mkdir build && cd build && cmake .. && sudo make install - -cd $HOME -git clone -b $XTENSOR_BRANCH $XTENSOR_REPO -cd xtensor && mkdir build && cd build && cmake .. && sudo make install - -cd $HOME -git clone -b $XTENSOR_PYTHON_BRANCH $XTENSOR_PYTHON_REPO -cd xtensor-python && mkdir build && cd build && cmake .. && sudo make install - -cd $HOME -git clone -b $XTENSOR_BLAS_BRANCH $XTENSOR_BLAS_REPO -cd xtensor-blas && mkdir build && cd build && cmake .. && sudo make install - -# Install wheel (remove when vectfit supports installation with build isolation) -pip install wheel - -# Install vectfit -cd $HOME -git clone https://github.com/liangjg/vectfit.git -pip install --no-build-isolation ./vectfit From 88f48b8a8ba2042b9634be4ed462083beff8ad2a Mon Sep 17 00:00:00 2001 From: Jonathan Shimwell Date: Mon, 22 Apr 2024 00:00:02 +0100 Subject: [PATCH 3/3] leaving downloads out of ci --- .github/workflows/ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b31952a1179..f44a570d9a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -313,9 +313,7 @@ jobs: - name: before shell: bash - run: - wget -q -O - https://anl.box.com/shared/static/teaup95cqv8s9nn56hfn7ku8mmelr95p.xz | tar -C $HOME -xJ - wget -q -O - https://anl.box.com/shared/static/4kd2gxnf4gtk4w1c8eua5fsua22kvgjb.xz | tar -C $HOME -xJ + run: $GITHUB_WORKSPACE/tools/ci/gha-before-script.sh - name: test shell: bash