diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c41a663..2e0e07d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,3 +163,61 @@ jobs: cd containers/spindle-slurm-ubuntu/testing docker compose down + spindle-slurm-plugin-ubuntu: + name: Testsuite (Slurm Plugin, Ubuntu) + environment: Spindle CI + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out Spindle + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 + + - name: Setup Docker Compose + uses: docker/setup-compose-action@364cc21a5de5b1ee4a7f5f9d3fa374ce0ccde746 + with: + version: latest + + - name: Login to GitHub Container Registry + if: ${{ !env.ACT }} + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate MariaDB configuration + id: slurm-ubuntu-mariadb + run: | + cd containers/spindle-slurm-ubuntu/testing-plugin + ./generate_config.sh + + - name: Build spindle-slurm-plugin-ubuntu image + id: slurm-ubuntu-build + run: | + cd containers/spindle-slurm-ubuntu/testing-plugin + docker compose --progress=plain build + + - name: Bring spindle-slurm-plugin-ubuntu up + id: slurm-ubuntu-up + run: | + cd containers/spindle-slurm-ubuntu/testing-plugin + docker compose up -d --wait --wait-timeout 120 + + - name: Verify munge works in spindle-slurm-plugin-ubuntu + id: slurm-ubuntu-munge + run: | + docker exec slurm-plugin-head bash -c 'munge -n | unmunge' + + - name: Run spindle-slurm-plugin-ubuntu testsuite + id: slurm-ubuntu-testsuite + run: | + docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && salloc -n${workers} -N${workers} ./runTests ${workers}' + + - name: Bring spindle-slurm-plugin-ubuntu down + id: slurm-ubuntu-down + if: ${{ always() }} + continue-on-error: true + run: | + cd containers/spindle-slurm-ubuntu/testing-plugin + docker compose down + diff --git a/Makefile.in b/Makefile.in index c4aac527..9e4b21b3 100644 --- a/Makefile.in +++ b/Makefile.in @@ -315,6 +315,7 @@ SCONTROL_ABSPATH = @SCONTROL_ABSPATH@ SED = @SED@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ +SINFO_ABSPATH = @SINFO_ABSPATH@ SRUN_PATH = @SRUN_PATH@ STATICFLAG = @STATICFLAG@ STRIP = @STRIP@ diff --git a/config.h.in b/config.h.in index a346ae96..921fe5d8 100644 --- a/config.h.in +++ b/config.h.in @@ -6,6 +6,12 @@ /* Whether we are using a broken srun */ #undef BROKEN_SRUN +/* Colon-separated list of potential back-end cache directories */ +#undef CACHEPATHS + +/* Back-end directory for communication and housekeeping */ +#undef COMMPATH + /* Define if were using biter for client/server communication */ #undef COMM_BITER @@ -33,6 +39,10 @@ /* Slurm with via plugin is enabled */ #undef ENABLE_SLURM_PLUGIN +/* Define to 1 if you have the declaration of `spank_prepend_task_argv', and + to 0 if you don't. */ +#undef HAVE_DECL_SPANK_PREPEND_TASK_ARGV + /* Define to 1 if you have the header file. */ #undef HAVE_DLFCN_H @@ -127,9 +137,6 @@ /* Default mode for slurm launch */ #undef SLURMLAUNCH_ENABLED -/* The default local directory for Spindle */ -#undef SPINDLE_LOC - /* The default colon-separated list of directories that Spindle will not cache files out of */ #undef SPINDLE_LOCAL_PREFIX diff --git a/configure b/configure index aaa2f552..d1b936c0 100755 --- a/configure +++ b/configure @@ -638,6 +638,7 @@ LIBOBJS PKGSYSCONF_DIR BLD_SLURMPLUGIN_FALSE BLD_SLURMPLUGIN_TRUE +SINFO_ABSPATH SCONTROL_ABSPATH BE_host BE_CXXCPP @@ -847,6 +848,8 @@ enable_maintainer_mode with_default_port with_default_num_ports with_localstorage +with_cachepaths +with_commpath with_default_local_prefix with_testrm with_rm @@ -1589,7 +1592,12 @@ Optional Packages: --with-default-numports=NUM Number of TCP/IP ports to scan for Spindle server communication - --with-localstorage=DIR Directory on back-ends for storing relocated files + --with-localstorage=DIR (obsolete) + Use --with-cachepaths and --with-commpath instead. + --with-cachepaths=DIR Colon-separated list of potential back-end cache + directories + --with-compath=DIR Back-end directory for communication and + housekeeping --with-default-local-prefix=DIRS Colon-seperated list of directories that Spindle will not cache files out of @@ -2212,6 +2220,52 @@ fi eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno } # ac_fn_c_check_header_mongrel + +# ac_fn_c_check_decl LINENO SYMBOL VAR INCLUDES +# --------------------------------------------- +# Tests whether SYMBOL is declared in INCLUDES, setting cache variable VAR +# accordingly. +ac_fn_c_check_decl () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + as_decl_name=`echo $2|sed 's/ *(.*//'` + as_decl_use=`echo $2|sed -e 's/(/((/' -e 's/)/) 0&/' -e 's/,/) 0& (/g'` + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $as_decl_name is declared" >&5 +$as_echo_n "checking whether $as_decl_name is declared... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +int +main () +{ +#ifndef $as_decl_name +#ifdef __cplusplus + (void) $as_decl_use; +#else + (void) $as_decl_name; +#endif +#endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + eval "$3=yes" +else + eval "$3=no" +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + +} # ac_fn_c_check_decl cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. @@ -16617,9 +16671,23 @@ fi # Check whether --with-localstorage was given. if test "${with_localstorage+set}" = set; then : - withval=$with_localstorage; SPINDLE_LOC=${withval} + withval=$with_localstorage; as_fn_error $? "requested obsolete option --with-localstorage. Use --with-cachepaths and --with-commpath instead." "$LINENO" 5 +fi + + +# Check whether --with-cachepaths was given. +if test "${with_cachepaths+set}" = set; then : + withval=$with_cachepaths; CACHEPATHS=${withval} +else + CACHEPATHS=$DEFAULT_LOC +fi + + +# Check whether --with-commpath was given. +if test "${with_commpath+set}" = set; then : + withval=$with_commpath; COMMPATH=${withval} else - SPINDLE_LOC=$DEFAULT_LOC + COMMPATH=$DEFAULT_LOC fi @@ -16627,7 +16695,7 @@ fi if test "${with_default_local_prefix+set}" = set; then : withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval} else - SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$SPINDLE_LOC" + SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH" fi @@ -16647,7 +16715,12 @@ _ACEOF cat >>confdefs.h <<_ACEOF -#define SPINDLE_LOC "$SPINDLE_LOC" +#define COMMPATH "$COMMPATH" +_ACEOF + + +cat >>confdefs.h <<_ACEOF +#define CACHEPATHS "$CACHEPATHS" _ACEOF @@ -16911,7 +16984,7 @@ $as_echo "yes" >&6; } $as_echo "$as_me: WARNING: Slurm launching was explicitly requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected. This version of slurm breaks spindle daemon launch. You might still be able to get spindle to work by running jobs with srun's --overlap option. Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster." >&2;} fi - if [ "x$ENABLE_RSH_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_LAUNCH" != "x1" ] ; then + if [ "x$ENABLE_RSH_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_PLUGIN" != "xtrue" ] ; then if test "x$BROKEN_SRUN" == "x1"; then as_fn_error $? "Slurm support was requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected. This version of slurm breaks spindle daemon launch. You can disable this error message and build spindle with slurm-based daemon launching anyways by explicitly passing the --with-slurm-launch option (you might still be able to get spindle to work by running jobs with srun's --overlap option). Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster." "$LINENO" 5 else @@ -18658,6 +18731,18 @@ fi CPPFLAGS=$OCPPFLAGS + ac_fn_c_check_decl "$LINENO" "spank_prepend_task_argv" "ac_cv_have_decl_spank_prepend_task_argv" "#include +" +if test "x$ac_cv_have_decl_spank_prepend_task_argv" = xyes; then : + ac_have_decl=1 +else + ac_have_decl=0 +fi + +cat >>confdefs.h <<_ACEOF +#define HAVE_DECL_SPANK_PREPEND_TASK_ARGV $ac_have_decl +_ACEOF + OPATH=$PATH if test "x$SLURM_DIR" != "x"; then @@ -18707,6 +18792,50 @@ fi if test "x$SCONTROL_ABSPATH" == "xnotfound"; then as_fn_error $? "Could not find scontrol" "$LINENO" 5 fi + # Extract the first word of "sinfo", so it can be a program name with args. +set dummy sinfo; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_SINFO_ABSPATH+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $SINFO_ABSPATH in + [\\/]* | ?:[\\/]*) + ac_cv_path_SINFO_ABSPATH="$SINFO_ABSPATH" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_SINFO_ABSPATH="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + test -z "$ac_cv_path_SINFO_ABSPATH" && ac_cv_path_SINFO_ABSPATH="notfound" + ;; +esac +fi +SINFO_ABSPATH=$ac_cv_path_SINFO_ABSPATH +if test -n "$SINFO_ABSPATH"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $SINFO_ABSPATH" >&5 +$as_echo "$SINFO_ABSPATH" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + if test "x$SINFO_ABSPATH" == "xnotfound"; then + as_fn_error $? "Could not find sinfo" "$LINENO" 5 + fi PATH=$OPATH fi @@ -18720,6 +18849,16 @@ fi SCONTROL_ABSPATH=$SCONTROL_ABSPATH +SINFO_ABSPATH=$SINFO_ABSPATH + + + if test "x$ENABLE_FLUX_PLUGIN" = "xtrue"; then + BLD_FLUXPLUGIN_TRUE= + BLD_FLUXPLUGIN_FALSE='#' +else + BLD_FLUXPLUGIN_TRUE='#' + BLD_FLUXPLUGIN_FALSE= +fi cat >confcache <<\_ACEOF @@ -18935,6 +19074,10 @@ if test -z "${BLD_SLURMPLUGIN_TRUE}" && test -z "${BLD_SLURMPLUGIN_FALSE}"; then as_fn_error $? "conditional \"BLD_SLURMPLUGIN\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${BLD_FLUXPLUGIN_TRUE}" && test -z "${BLD_FLUXPLUGIN_FALSE}"; then + as_fn_error $? "conditional \"BLD_FLUXPLUGIN\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi : "${CONFIG_STATUS=./config.status}" ac_write_fail=0 diff --git a/configure.ac b/configure.ac index 637fe9de..b2c519d4 100644 --- a/configure.ac +++ b/configure.ac @@ -76,6 +76,7 @@ if test "x$ENABLE_SLURM_PLUGIN" == "xtrue"; then [], [AC_MSG_ERROR([Could not find slurm/spank.h])]) CPPFLAGS=$OCPPFLAGS + AC_CHECK_DECLS([spank_prepend_task_argv], [], [], [[#include]]) OPATH=$PATH if test "x$SLURM_DIR" != "x"; then @@ -85,12 +86,18 @@ if test "x$ENABLE_SLURM_PLUGIN" == "xtrue"; then if test "x$SCONTROL_ABSPATH" == "xnotfound"; then AC_MSG_ERROR([Could not find scontrol]) fi + AC_PATH_PROG([SINFO_ABSPATH], [sinfo], [notfound]) + if test "x$SINFO_ABSPATH" == "xnotfound"; then + AC_MSG_ERROR([Could not find sinfo]) + fi PATH=$OPATH fi AM_CONDITIONAL([BLD_SLURMPLUGIN], [test "x$ENABLE_SLURM_PLUGIN" == "xtrue"]) AC_SUBST(SCONTROL_ABSPATH, $SCONTROL_ABSPATH) +AC_SUBST(SINFO_ABSPATH, $SINFO_ABSPATH) AC_SUBST(PKGSYSCONF_DIR) +AM_CONDITIONAL([BLD_FLUXPLUGIN], [test "x$ENABLE_FLUX_PLUGIN" = "xtrue"]) AC_OUTPUT diff --git a/configure.common.ac b/configure.common.ac index 780aef00..c507b053 100644 --- a/configure.common.ac +++ b/configure.common.ac @@ -18,17 +18,26 @@ AC_ARG_WITH(default-num-ports, [NUM_COBO_PORTS=${withval}], [NUM_COBO_PORTS=$DEFAULT_NUM_COBO_PORTS]) AC_ARG_WITH(localstorage, - [AS_HELP_STRING([--with-localstorage=DIR],[Directory on back-ends for storing relocated files])], - [SPINDLE_LOC=${withval}], - [SPINDLE_LOC=$DEFAULT_LOC]) + [AS_HELP_STRING([--with-localstorage=DIR (obsolete)],[Use --with-cachepaths and --with-commpath instead.])], + [AC_MSG_ERROR(requested obsolete option --with-localstorage. Use --with-cachepaths and --with-commpath instead.)], + []) +AC_ARG_WITH(cachepaths, + [AS_HELP_STRING([--with-cachepaths=DIR],[Colon-separated list of potential back-end cache directories])], + [CACHEPATHS=${withval}], + [CACHEPATHS=$DEFAULT_LOC]) +AC_ARG_WITH(commpath, + [AS_HELP_STRING([--with-compath=DIR],[Back-end directory for communication and housekeeping])], + [COMMPATH=${withval}], + [COMMPATH=$DEFAULT_LOC]) AC_ARG_WITH(default-local-prefix, [AS_HELP_STRING([--with-default-local-prefix=DIRS],[Colon-seperated list of directories that Spindle will not cache files out of])], [SPINDLE_LOCAL_PREFIX=${withval}], - [SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$SPINDLE_LOC"]) + [SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH"]) AC_DEFINE_UNQUOTED([SPINDLE_PORT],[$SPINDLE_PORT],[The default port for Spindle]) AC_DEFINE_UNQUOTED([NUM_COBO_PORTS],[$NUM_COBO_PORTS],[Number of ports for COBO to search for an open port]) AC_DEFINE_UNQUOTED([SPINDLE_MAX_PORT],[$(($SPINDLE_PORT + $NUM_COBO_PORTS - 1))],[The maximum port value]) -AC_DEFINE_UNQUOTED([SPINDLE_LOC],"[$SPINDLE_LOC]",[The default local directory for Spindle]) +AC_DEFINE_UNQUOTED([COMMPATH],"[$COMMPATH]",[Back-end directory for communication and housekeeping]) +AC_DEFINE_UNQUOTED([CACHEPATHS],"[$CACHEPATHS]",[Colon-separated list of potential back-end cache directories]) AC_DEFINE_UNQUOTED([SPINDLE_LOCAL_PREFIX],"[$SPINDLE_LOCAL_PREFIX]",[The default colon-separated list of directories that Spindle will not cache files out of]) TESTRM=unknown @@ -168,7 +177,7 @@ if test "x$ENABLE_SLURM" == "xtrue"; then AC_MSG_WARN([Slurm launching was explicitly requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected. This version of slurm breaks spindle daemon launch. You might still be able to get spindle to work by running jobs with srun's --overlap option. Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster.]) fi - if [[ "x$ENABLE_RSH_LAUNCH" != "x1" ]] && [[ "x$ENABLE_SLURM_LAUNCH" != "x1" ]] ; then + if [[ "x$ENABLE_RSH_LAUNCH" != "x1" ]] && [[ "x$ENABLE_SLURM_LAUNCH" != "x1" ]] && [[ "x$ENABLE_SLURM_PLUGIN" != "xtrue" ]] ; then if test "x$BROKEN_SRUN" == "x1"; then AC_MSG_ERROR([Slurm support was requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected. This version of slurm breaks spindle daemon launch. You can disable this error message and build spindle with slurm-based daemon launching anyways by explicitly passing the --with-slurm-launch option (you might still be able to get spindle to work by running jobs with srun's --overlap option). Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster.]) else diff --git a/containers/spindle-flux-ubuntu/Dockerfile b/containers/spindle-flux-ubuntu/Dockerfile index 3af607a8..57badbfc 100644 --- a/containers/spindle-flux-ubuntu/Dockerfile +++ b/containers/spindle-flux-ubuntu/Dockerfile @@ -5,6 +5,8 @@ FROM fluxrm/flux-sched:${flux_sched_version} AS builder ARG replicas=4 ENV workers=${replicas} USER root +ENV TMPDIR=/tmp +RUN echo 'TMPDIR="/tmp"' >> /etc/environment RUN DEBIAN_FRONTEND="noninteractive" apt-get update \ && apt-get -qq install -y --no-install-recommends \ diff --git a/containers/spindle-flux-ubuntu/scripts/build_spindle.sh b/containers/spindle-flux-ubuntu/scripts/build_spindle.sh index 9257f85e..283e4451 100755 --- a/containers/spindle-flux-ubuntu/scripts/build_spindle.sh +++ b/containers/spindle-flux-ubuntu/scripts/build_spindle.sh @@ -4,7 +4,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=flux --enable-flux-plugin --with-localstorage=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=flux --enable-flux-plugin --with-cachepaths=/tmp --with-commpath=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/containers/spindle-serial-ubuntu/Dockerfile b/containers/spindle-serial-ubuntu/Dockerfile index 3070596e..62c0cbf5 100644 --- a/containers/spindle-serial-ubuntu/Dockerfile +++ b/containers/spindle-serial-ubuntu/Dockerfile @@ -1,6 +1,8 @@ ARG ubuntu_version=noble FROM ubuntu:${ubuntu_version} USER root +ENV TMPDIR=/tmp +RUN echo 'TMPDIR="/tmp"' >> /etc/environment RUN DEBIAN_FRONTEND="noninteractive" apt-get update \ # install latest pkg utils: diff --git a/containers/spindle-serial-ubuntu/scripts/build_spindle.sh b/containers/spindle-serial-ubuntu/scripts/build_spindle.sh index 37b6491a..4fee85b4 100755 --- a/containers/spindle-serial-ubuntu/scripts/build_spindle.sh +++ b/containers/spindle-serial-ubuntu/scripts/build_spindle.sh @@ -4,7 +4,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=serial --with-localstorage=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=serial --with-cachepaths=/tmp --with-commpath=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/containers/spindle-slurm-ubuntu/base/Dockerfile b/containers/spindle-slurm-ubuntu/base/Dockerfile index d4724276..d6eee9f2 100644 --- a/containers/spindle-slurm-ubuntu/base/Dockerfile +++ b/containers/spindle-slurm-ubuntu/base/Dockerfile @@ -1,6 +1,8 @@ ARG UBUNTU_VERSION=noble FROM ubuntu:${UBUNTU_VERSION} USER root +ENV TMPDIR=/tmp +RUN echo 'TMPDIR="/tmp"' >> /etc/environment RUN apt-get update \ && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile b/containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile new file mode 100644 index 00000000..7b66a155 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile @@ -0,0 +1,42 @@ +ARG BASE_VERSION=latest +FROM ghcr.io/llnl/spindle-slurm-base:${BASE_VERSION} +ARG replicas=4 +ENV workers=${replicas} + +ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing-plugin + +# Slurm daemons run as $SLURM_USER +ARG SLURM_USER=slurm + +# Applications run as $USER +ARG USER=slurmuser +ARG UID=1001 + +# Set up the Slurm install already present in the base image +COPY ${BUILD_ROOT}/scripts/setup_slurm.sh /setup_slurm.sh +COPY ${BUILD_ROOT}/conf/slurm.conf /home/${SLURM_USER}/slurm.conf +COPY ${BUILD_ROOT}/conf/slurmdbd.conf /home/${SLURM_USER}/slurmdbd.conf +COPY ${BUILD_ROOT}/conf/cgroup.conf /home/${SLURM_USER}/cgroup.conf +RUN /setup_slurm.sh + +USER ${USER} +WORKDIR /home/${USER} + +# Copy the Spindle repo into the container and build it +RUN mkdir -p /home/${USER}/Spindle +COPY . /home/${USER}/Spindle +COPY ${BUILD_ROOT}/scripts/build_spindle.sh /home/${USER}/build_spindle.sh +RUN ./build_spindle.sh + +USER root +COPY ${BUILD_ROOT}/scripts/setup_spank_plugin.sh /setup_spank_plugin.sh +COPY ${BUILD_ROOT}/conf/plugstack.conf /home/${SLURM_USER}/plugstack.conf +RUN /setup_spank_plugin.sh + +USER ${USER} + +COPY ${BUILD_ROOT}/scripts/entrypoint.sh /home/${USER}/entrypoint.sh +ENV PATH /home/${USER}/Spindle-inst/bin:$PATH + +ENTRYPOINT /bin/bash ./entrypoint.sh + diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/conf/cgroup.conf b/containers/spindle-slurm-ubuntu/testing-plugin/conf/cgroup.conf new file mode 100644 index 00000000..e59e9aee --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-plugin/conf/cgroup.conf @@ -0,0 +1 @@ +CgroupPlugin=cgroup/v1 diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/conf/plugstack.conf b/containers/spindle-slurm-ubuntu/testing-plugin/conf/plugstack.conf new file mode 100644 index 00000000..a291c1c0 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-plugin/conf/plugstack.conf @@ -0,0 +1 @@ +required /home/slurmuser/Spindle-inst/lib/libspindleslurm.so diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/conf/slurm.conf b/containers/spindle-slurm-ubuntu/testing-plugin/conf/slurm.conf new file mode 100644 index 00000000..abf060d5 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-plugin/conf/slurm.conf @@ -0,0 +1,42 @@ +ClusterName=linux +ControlMachine=slurm-head +ControlAddr=slurm-head +SlurmUser=slurm +SlurmctldPort=6817 +SlurmdPort=6818 +AuthType=auth/munge +StateSaveLocation=/var/lib/slurmd +SlurmdSpoolDir=/var/spool/slurmd +SwitchType=switch/none +MpiDefault=none +SlurmctldPidFile=/var/run/slurmd/slurmctld.pid +SlurmdPidFile=/var/run/slurmd/slurmd.pid +ProctrackType=proctrack/linuxproc +TaskPlugin=task/affinity +ReturnToService=2 +SlurmctldTimeout=300 +SlurmdTimeout=300 +InactiveLimit=0 +MinJobAge=300 +KillWait=30 +Waittime=0 +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core_Memory +SlurmctldDebug=3 +SlurmctldLogFile=/var/log/slurm/slurmctld.log +SlurmdDebug=3 +SlurmdLogFile=/var/log/slurm/slurmd.log +JobCompType=jobcomp/filetxt +JobCompLoc=/var/log/slurm/jobcomp.log +JobAcctGatherType=jobacct_gather/linux +JobAcctGatherFrequency=30 +AccountingStorageType=accounting_storage/slurmdbd +AccountingStorageHost=slurm-db +AccountingStoragePort=6819 +NodeName=slurm-node-1 NodeAddr=slurm-node-1 CPUs=3 RealMemory=1000 State=UNKNOWN +NodeName=slurm-node-2 NodeAddr=slurm-node-2 CPUs=3 RealMemory=1000 State=UNKNOWN +NodeName=slurm-node-3 NodeAddr=slurm-node-3 CPUs=3 RealMemory=1000 State=UNKNOWN +NodeName=slurm-node-4 NodeAddr=slurm-node-4 CPUs=3 RealMemory=1000 State=UNKNOWN +PartitionName=debug Nodes=ALL Default=YES MaxTime=INFINITE State=UP + diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/conf/slurmdbd.conf.template b/containers/spindle-slurm-ubuntu/testing-plugin/conf/slurmdbd.conf.template new file mode 100644 index 00000000..0e274118 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-plugin/conf/slurmdbd.conf.template @@ -0,0 +1,10 @@ +AuthType=auth/munge +DbdAddr=slurm-db +DbdHost=slurm-db +SlurmUser=slurm +DebugLevel=4 +LogFile=/var/log/slurm/slurmdbd.log +PidFile=/var/run/slurmdbd/slurmdbd.pid +StorageType=accounting_storage/mysql +StorageHost=slurm-mariadb +StorageUser=slurm diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/docker-compose.yml b/containers/spindle-slurm-ubuntu/testing-plugin/docker-compose.yml new file mode 100644 index 00000000..2751a71a --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-plugin/docker-compose.yml @@ -0,0 +1,126 @@ +# `replicas` must match the number of nodes defined in the services section +x-shared-workers: + &workers + replicas: 4 + +# Base image version to use +x-shared-build-args: &shared-build-args + BASE_VERSION: latest + <<: *workers + +# Docker prohibits copying files from outside of the build context. +# In order to be able to copy the whole repo into the container, +# we have to set the context to be the root of the repo. +# We then have to specify the path from there to the Dockerfile. +x-shared-build-context: &shared-build-context + context: ../../.. + dockerfile: containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile + args: *shared-build-args + +# Name of the head node +x-shared-environment: &shared-environment + SLURM_HEAD_NODE: slurm-head + <<: *workers + +# The entrypoint runs different services depending +# on the node's role. Valid options are: +# - worker: runs slurmd +# - db: runs slurmdbd +# - ctl: runs slurmctld +x-worker-environment: &worker-environment + SLURM_ROLE: worker + <<: *shared-environment + +networks: + slurm: + driver: bridge + +# Common parameters for all nodes. +x-shared-node-parameters: &shared-node-parameters + build: *shared-build-context + networks: + - slurm + cap_add: + - SYS_NICE # Required for libnuma + +x-healthcheck-parameters: &healthcheck-parameters + start_period: 3s + interval: 3s + timeout: 5s + retries: 5 + +x-worker-parameters: &worker-node-parameters + <<: *shared-node-parameters + environment: *worker-environment + depends_on: + slurm-head: + condition: service_healthy + healthcheck: + test: ["CMD", "stat", "/var/run/slurmd/slurmd.pid"] + <<: *healthcheck-parameters + +services: + slurm-mariadb: + image: mariadb:12 + networks: + - slurm + hostname: slurm-mariadb + container_name: slurm-plugin-mariadb + env_file: mariadb.env + environment: + MYSQL_RANDOM_ROOT_PASSWORD: "yes" + MYSQL_DATABASE: "slurm_acct_db" + MYSQL_USER: "slurm" + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + <<: *healthcheck-parameters + + slurm-db: + <<: *shared-node-parameters + hostname: slurm-db + container_name: slurm-plugin-db + environment: + SLURM_ROLE: db + <<: *shared-environment + depends_on: + slurm-mariadb: + condition: service_healthy + healthcheck: + test: ["CMD", "stat", "/var/run/slurmdbd/slurmdbd.pid"] + <<: *healthcheck-parameters + + slurm-head: + <<: *shared-node-parameters + hostname: slurm-head + container_name: slurm-plugin-head + tty: true + environment: + SLURM_ROLE: ctl + <<: *shared-environment + depends_on: + slurm-db: + condition: service_healthy + healthcheck: + test: ["CMD", "stat", "/var/run/slurmd/slurmctld.pid"] + <<: *healthcheck-parameters + + slurm-node-1: + <<: *worker-node-parameters + hostname: slurm-node-1 + container_name: slurm-plugin-node-1 + + slurm-node-2: + <<: *worker-node-parameters + hostname: slurm-node-2 + container_name: slurm-plugin-node-2 + + slurm-node-3: + <<: *worker-node-parameters + hostname: slurm-node-3 + container_name: slurm-plugin-node-3 + + slurm-node-4: + <<: *worker-node-parameters + hostname: slurm-node-4 + container_name: slurm-plugin-node-4 + diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/generate_config.sh b/containers/spindle-slurm-ubuntu/testing-plugin/generate_config.sh new file mode 100755 index 00000000..eadb1c8a --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-plugin/generate_config.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Generate random password for the MariaDB slurm user +# and set it in config files + +APP_USER=${APP_USER:-slurmuser} +MARIADB_PASS=$(openssl rand --base64 16 | head -c -3) +echo "MARIADB_PASSWORD: \"${MARIADB_PASS}\"" > mariadb.env +cp conf/slurmdbd.conf.template conf/slurmdbd.conf +echo "StoragePass=${MARIADB_PASS}" >> conf/slurmdbd.conf + +# Enable Spindle SPANK plugin + +echo "required /home/${APP_USER}/Spindle-inst/lib/libspindleslurm.so" > conf/plugstack.conf diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/add_docker_user.sh b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/add_docker_user.sh new file mode 100755 index 00000000..ace8c619 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/add_docker_user.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euxo pipefail + +sudo groupadd -g ${UID} ${USER} +sudo useradd -g ${USER} -u ${UID} -d /home/${USER} -m ${USER} +# Allow user to run as other users so that munge can be started as the munge user +sudo sh -c "printf \"${USER} ALL=(ALL) NOPASSWD: ALL\\n\" >> /etc/sudoers" +sudo adduser ${USER} sudo +sudo usermod -s /bin/bash ${USER} + diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh new file mode 100755 index 00000000..879a28c6 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euxo pipefail + +mkdir -p /home/${USER}/Spindle-build +cd /home/${USER}/Spindle-build +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-commpath=/tmp --with-cachepath=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +make -j$(nproc) +make install + diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/entrypoint.sh b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/entrypoint.sh new file mode 100755 index 00000000..54d40e48 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/entrypoint.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +echo "SLURM_ROLE: ${SLURM_ROLE}" + +echo "Starting sshd..." +sudo service ssh start +echo "Starting munged..." +sudo -u munge /usr/sbin/munged + +if [ "${SLURM_ROLE}" = "db" ]; then + echo "Starting slurmdbd..." + sudo -u slurm /usr/sbin/slurmdbd -Dvvv +elif [ "${SLURM_ROLE}" = "ctl" ] ; then + echo "Starting slurmctld..." + sudo -u slurm /usr/sbin/slurmctld -i -Dvvv +elif [ "${SLURM_ROLE}" = "worker" ] ; then + echo "Starting slurmd..." + sudo /usr/sbin/slurmd -Dvvv +fi + +sleep inf diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/setup_slurm.sh b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/setup_slurm.sh new file mode 100755 index 00000000..186beea0 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/setup_slurm.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euxo pipefail + +mkdir -p /etc/slurm /etc/sysconfig/slurm /var/spool/slurmd /var/spool/slurmctld /var/run/slurmd /var/run/slurmdbd /var/lib/slurmd /var/log/slurm +touch /var/lib/slurmd/node_state /var/lib/slurmd/front_end_state /var/lib/slurmd/job_state /var/lib/slurmd/resv_state /var/lib/slurmd/trigger_state /var/lib/slurmd/assoc_mgr_state /var/lib/slurmd/assoc_usage /var/lib/slurmd/qos_usage /var/lib/slurmd/fed_mgr_state +cp /home/${SLURM_USER}/slurm.conf /etc/slurm/slurm.conf +cp /home/${SLURM_USER}/slurmdbd.conf /etc/slurm/slurmdbd.conf +cp /home/${SLURM_USER}/cgroup.conf /etc/slurm/cgroup.conf +chown -R slurm:slurm /etc/slurm /etc/sysconfig/slurm /var/spool/slurmd /var/spool/slurmctld /var/run/slurmd /var/run/slurmdbd /var/lib/slurmd /var/log/slurm +chmod 600 /etc/slurm/slurmdbd.conf diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/setup_spank_plugin.sh b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/setup_spank_plugin.sh new file mode 100755 index 00000000..7b6ea089 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/setup_spank_plugin.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euxo pipefail + +cp /home/${SLURM_USER}/plugstack.conf /etc/slurm/plugstack.conf +chown -R slurm:slurm /etc/slurm diff --git a/containers/spindle-slurm-ubuntu/testing/Dockerfile b/containers/spindle-slurm-ubuntu/testing/Dockerfile index 91d6901c..99768535 100644 --- a/containers/spindle-slurm-ubuntu/testing/Dockerfile +++ b/containers/spindle-slurm-ubuntu/testing/Dockerfile @@ -2,6 +2,8 @@ ARG BASE_VERSION=latest FROM ghcr.io/llnl/spindle-slurm-base:${BASE_VERSION} ARG replicas=4 ENV workers=${replicas} +ENV TMPDIR=/tmp +RUN echo 'TMPDIR="/tmp"' >> /etc/environment ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing diff --git a/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh b/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh index 6943e49a..17e7197f 100755 --- a/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh +++ b/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh @@ -3,7 +3,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-rsh-launch --with-rsh-cmd=/usr/bin/ssh --with-localstorage=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-rsh-launch --with-rsh-cmd=/usr/bin/ssh --with-cachepaths=/tmp --with-commpath=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/doc/Makefile.in b/doc/Makefile.in index 3dc35cbb..78079922 100644 --- a/doc/Makefile.in +++ b/doc/Makefile.in @@ -253,6 +253,7 @@ SCONTROL_ABSPATH = @SCONTROL_ABSPATH@ SED = @SED@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ +SINFO_ABSPATH = @SINFO_ABSPATH@ SRUN_PATH = @SRUN_PATH@ STATICFLAG = @STATICFLAG@ STRIP = @STRIP@ diff --git a/src/client/auditclient/redirect.c b/src/client/auditclient/redirect.c index b237c841..51ce5d7c 100644 --- a/src/client/auditclient/redirect.c +++ b/src/client/auditclient/redirect.c @@ -36,6 +36,12 @@ ElfX_Addr client_call_binding(const char *symname, ElfX_Addr symvalue) if (!binding) return symvalue; + if (!binding->libc_func) { + debug_printf("%s: symname %s has no libc_func container\n", + __func__, symname); + return (ElfX_Addr) binding->spindle_func; + } + if (*binding->libc_func == NULL) *binding->libc_func = (void *) symvalue; @@ -44,4 +50,3 @@ ElfX_Addr client_call_binding(const char *symname, ElfX_Addr symvalue) else return symvalue; } - diff --git a/src/client/beboot/spindle_bootstrap.c b/src/client/beboot/spindle_bootstrap.c index 7805a9db..d0e06439 100644 --- a/src/client/beboot/spindle_bootstrap.c +++ b/src/client/beboot/spindle_bootstrap.c @@ -53,7 +53,7 @@ static int rankinfo[4]={-1,-1,-1,-1}; number_t number; static int use_cache; static unsigned int cachesize; -static char *location, *number_s, *orig_location, *symbolic_location; +static char *commpath, *number_s, *symbolic_commpath; static char **cmdline; static char *executable; static char *client_lib; @@ -91,7 +91,7 @@ extern char *realize(char *path); static int establish_connection() { debug_printf2("Opening connection to server\n"); - ldcsid = client_open_connection(location, number); + ldcsid = client_open_connection(commpath, number); if (ldcsid == -1) return -1; @@ -107,14 +107,17 @@ static void setup_environment() { char rankinfo_str[256]; snprintf(rankinfo_str, 256, "%d %d %d %d %d", ldcsid, rankinfo[0], rankinfo[1], rankinfo[2], rankinfo[3]); - + char *connection_str = NULL; if (opts & OPT_RELOCAOUT) connection_str = client_get_connection_string(ldcsid); + char *chosen_parsed_cachepath; + send_cachepath_query( ldcsid , NULL, &chosen_parsed_cachepath); + setenv("LD_AUDIT", client_lib, 1); - setenv("LDCS_LOCATION", location, 1); - setenv("LDCS_ORIG_LOCATION", orig_location, 1); + setenv("LDCS_COMMPATH", commpath, 1); + setenv("LDCS_CHOSEN_PARSED_CACHEPATH", chosen_parsed_cachepath, 1); setenv("LDCS_NUMBER", number_s, 1); setenv("LDCS_RANKINFO", rankinfo_str, 1); if (connection_str) @@ -160,7 +163,8 @@ static int parse_cmdline(int argc, char *argv[]) daemon_args[i - 3] = NULL; } - symbolic_location = argv[i++]; + symbolic_commpath = argv[i++]; + i++; // Skip over candidate_cachepaths. number_s = argv[i++]; number = (number_t) strtoul(number_s, NULL, 0); opts_s = argv[i++]; @@ -173,7 +177,7 @@ static int parse_cmdline(int argc, char *argv[]) return 0; } -static void launch_daemon(char *location) +static void launch_daemon(char *commpath) { /*grand-child fork, then execv daemon. By grand-child forking we ensure that the app won't get confused by seeing an unknown process as a child. */ @@ -183,12 +187,12 @@ static void launch_daemon(char *location) char unique_file[MAX_PATH_LEN+1]; char buffer[32]; - result = spindle_mkdir(location); + result = spindle_mkdir(commpath); if (result == -1) { debug_printf("Exiting due to spindle_mkdir error\n"); exit(-1); } - snprintf(unique_file, MAX_PATH_LEN, "%s/spindle_daemon_pid", location); + snprintf(unique_file, MAX_PATH_LEN, "%s/spindle_daemon_pid", commpath); unique_file[MAX_PATH_LEN] = '\0'; fd = open(unique_file, O_CREAT | O_EXCL | O_WRONLY, 0600); if (fd == -1) { @@ -343,14 +347,14 @@ int main(int argc, char *argv[]) } } - orig_location = parse_location(symbolic_location, number); - if (!orig_location) { + char *orig_commpath = parse_location(symbolic_commpath, number); + if (!orig_commpath) { return -1; } - location = realize(orig_location); + commpath = realize(orig_commpath); if (daemon_args) { - launch_daemon(location); + launch_daemon(commpath); } result = establish_connection(); @@ -374,7 +378,7 @@ int main(int argc, char *argv[]) #else shm_cache_limit = cachesize; #endif - shmcache_init(location, number, cachesize, shm_cache_limit); + shmcache_init(commpath, number, cachesize, shm_cache_limit); use_cache = 1; } diff --git a/src/client/client/client.c b/src/client/client/client.c index a50bb228..0f0b6047 100644 --- a/src/client/client/client.c +++ b/src/client/client/client.c @@ -41,6 +41,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "exec_util.h" #include "intercept.h" #include "fixlocale.h" +#include "should_intercept.h" errno_location_t app_errno_location; @@ -70,11 +71,8 @@ static const ElfW(Phdr) *libc_phdrs, *interp_phdrs; static int num_libc_phdrs, num_interp_phdrs; ElfW(Addr) libc_loadoffset, interp_loadoffset; -/* location has the realize'd path to the local file cache. orig_location is not realized and - * may contain symlinks - */ -char *location; -char *orig_location; +static char *commpath; +char *chosen_realized_cachepath, *chosen_parsed_cachepath; number_t number; static int have_stat_patches; @@ -200,8 +198,7 @@ static int init_server_connection() if (!use_ldcs) return 0; - location = getenv("LDCS_LOCATION"); - orig_location = getenv("LDCS_ORIG_LOCATION"); + commpath = getenv("LDCS_COMMPATH"); number = (number_t) strtoul(getenv("LDCS_NUMBER"), NULL, 0); connection = getenv("LDCS_CONNECTION"); rankinfo_s = getenv("LDCS_RANKINFO"); @@ -210,9 +207,9 @@ static int init_server_connection() opts = strtoul(opts_s, NULL, 10); shm_cachesize = atoi(cachesize_s) * 1024; - if (strchr(location, '$')) { - location = parse_location(location, number); - if (!location) { + if (strchr(commpath, '$')) { + commpath = parse_location(commpath, number); + if (!commpath) { exit(-1); } } @@ -220,8 +217,7 @@ static int init_server_connection() if (!(opts & OPT_FOLLOWFORK)) { debug_printf("Disabling environment variables because we're not following forks\n"); unsetenv("LD_AUDIT"); - unsetenv("LDCS_LOCATION"); - unsetenv("LDCS_ORIG_LOCATION"); + unsetenv("LDCS_COMMPATH"); unsetenv("LDCS_NUMBER"); unsetenv("LDCS_CONNECTION"); unsetenv("LDCS_RANKINFO"); @@ -235,14 +231,14 @@ static int init_server_connection() #else shm_cache_limit = shm_cachesize; #endif - shmcache_init(location, number, shm_cachesize, shm_cache_limit); + shmcache_init(commpath, number, shm_cachesize, shm_cache_limit); } if (connection) { /* boostrapper established the connection for us. Reuse it. */ debug_printf("Recreating existing connection to server\n"); - debug_printf3("location = %s, number = %lu, connection = %s, rankinfo = %s\n", - location, (unsigned long) number, connection, rankinfo_s); + debug_printf3("commpath = %s, number = %lu, connection = %s, rankinfo = %s\n", + commpath, (unsigned long) number, connection, rankinfo_s); ldcsid = client_register_connection(connection); if (ldcsid == -1) return -1; @@ -252,20 +248,20 @@ static int init_server_connection() } else { /* Establish a new connection */ - debug_printf("open connection to ldcs %s %lu\n", location, (unsigned long) number); - ldcsid = client_open_connection(location, number); + debug_printf("open connection to ldcs %s %lu\n", commpath, (unsigned long) number); + ldcsid = client_open_connection(commpath, number); if (ldcsid == -1) return -1; send_pid(ldcsid); - send_location(ldcsid, location); + send_location(ldcsid, commpath); send_rankinfo_query(ldcsid, rankinfo+0, rankinfo+1, rankinfo+2, rankinfo+3); #if defined(LIBNUMA) if (opts & OPT_NUMA) send_cpu(ldcsid, get_cur_cpu()); #endif } - + send_cachepath_query( ldcsid, &chosen_realized_cachepath, &chosen_parsed_cachepath ); snprintf(debugging_name, 32, "Client.%d", rankinfo[0]); LOGGING_INIT(debugging_name); @@ -472,7 +468,7 @@ char *client_library_load(const char *name) char *orig_file_name = (char *) name; if (is_in_spindle_cache(name)) { - debug_printf2("Library %s is in spindle cache (%s). Translating request\n", name, location); + debug_printf2("Library %s is in spindle cache (%s). Translating request\n", name, chosen_realized_cachepath); memset(fixed_name, 0, MAX_PATH_LEN+1); send_orig_path_request(ldcsid, orig_file_name, fixed_name); orig_file_name = fixed_name; diff --git a/src/client/client/intercept_exec.c b/src/client/client/intercept_exec.c index 14b555ed..bae9eba3 100644 --- a/src/client/client/intercept_exec.c +++ b/src/client/client/intercept_exec.c @@ -141,8 +141,7 @@ static char **removeEnvironmentStrs(char **envp) continue; if (strIsPrefix("LD", envp[i])) { if (strIsPrefix("LD_AUDIT=", envp[i]) || - strIsPrefix("LDCS_LOCATION=", envp[i]) || - strIsPrefix("LDCS_ORIG_LOCATION=", envp[i]) || + strIsPrefix("LDCS_COMMPATH=", envp[i]) || strIsPrefix("LDCS_CONNECTION=", envp[i]) || strIsPrefix("LDCS_RANKINFO=", envp[i]) || strIsPrefix("LDCS_OPTIONS=", envp[i]) || @@ -176,8 +175,7 @@ static char **updateEnvironment(char **envp, int *num_modified, int propogate_sp unsetf = orig_unsetenv ? orig_unsetenv : unsetenv; unsetf("SPINDLE"); unsetf("LD_AUDIT"); - unsetf("LDCS_LOCATION"); - unsetf("LDCS_ORIG_LOCATION"); + unsetf("LDCS_COMMPATH"); unsetf("LDCS_CONNECTION"); unsetf("LDCS_RANKINFO"); unsetf("LDCS_OPTIONS"); @@ -198,13 +196,12 @@ static char **updateEnvironment(char **envp, int *num_modified, int propogate_sp if (envp) { debug_printf2("Propogating spindle environment by copying it to new envp list\n"); for (cur = (char **) envp; *cur; cur++, orig_size++); - new_size = orig_size + 10; + new_size = orig_size + 9; newenv = (char **) malloc(new_size * sizeof(char*)); propogateEnvironmentStr(envp, newenv, &pos, "SPINDLE"); propogateEnvironmentStr(envp, newenv, &pos, "LD_AUDIT"); - propogateEnvironmentStr(envp, newenv, &pos, "LDCS_LOCATION"); - propogateEnvironmentStr(envp, newenv, &pos, "LDCS_ORIG_LOCATION"); + propogateEnvironmentStr(envp, newenv, &pos, "LDCS_COMMPATH"); propogateEnvironmentStr(envp, newenv, &pos, "LDCS_CONNECTION"); propogateEnvironmentStr(envp, newenv, &pos, "LDCS_RANKINFO"); propogateEnvironmentStr(envp, newenv, &pos, "LDCS_OPTIONS"); diff --git a/src/client/client/intercept_readlink.c b/src/client/client/intercept_readlink.c index 0a326330..37d440e4 100644 --- a/src/client/client/intercept_readlink.c +++ b/src/client/client/intercept_readlink.c @@ -31,19 +31,19 @@ Place, Suite 330, Boston, MA 02111-1307 USA ssize_t (*orig_readlink)(const char *path, char *buf, size_t bufsiz); ssize_t (*orig_readlinkat)(int dirfd, const char *pathname, char *buf, size_t bufsiz); -extern char *location; static int fix_local_readlink(char *buf, size_t bufsiz) { char spindle_id[32]; - int location_len, result; + int cachepath_len, result; char tmp[MAX_PATH_LEN+1]; + extern char *chosen_realized_cachepath; - location_len = strlen(location); + cachepath_len = strlen(chosen_realized_cachepath); snprintf(spindle_id, sizeof(spindle_id), "spindle.%lx", number); - if (strstr(buf, spindle_id) && strncmp(location, buf, location_len) == 0) { + if (strstr(buf, spindle_id) && strncmp(chosen_realized_cachepath, buf, cachepath_len) == 0) { debug_printf2("readlink received spindle cache path %s. Translating\n", buf); - result = send_orig_path_request(ldcsid, buf+location_len+1, tmp); + result = send_orig_path_request(ldcsid, buf+cachepath_len+1, tmp); if (result == -1) return -1; debug_printf2("readlink translated spindle local path %s to %s\n", buf, tmp); diff --git a/src/client/client/lookup_libc.c b/src/client/client/lookup_libc.c index 14f2408a..f25497be 100644 --- a/src/client/client/lookup_libc.c +++ b/src/client/client/lookup_libc.c @@ -212,7 +212,7 @@ int lookup_libc_symbols() } else { mallocfunc = (malloc_sig_t) (symtab[result].st_value + libc->l_addr); - debug_printf3("Bound errno_location to %p\n", app_errno_location); + debug_printf3("Bound mallocfunc to %p\n", mallocfunc); found++; } } diff --git a/src/client/client/should_intercept.c b/src/client/client/should_intercept.c index bfabbbf8..cee4e43c 100644 --- a/src/client/client/should_intercept.c +++ b/src/client/client/should_intercept.c @@ -30,21 +30,19 @@ extern int relocate_spindleapi(); -extern char *location; -extern char *orig_location; - int is_in_spindle_cache(const char *pathname) { - static int location_size = 0; - static int orig_location_size = 0; - if (!location_size) { - location_size = strlen(location); + static int cachepath_size = 0; + static int orig_cachepath_size = 0; + extern char *chosen_realized_cachepath, *chosen_parsed_cachepath; + if (!cachepath_size) { + cachepath_size = strlen(chosen_realized_cachepath); } - if (!orig_location_size) { - orig_location_size = strlen(orig_location); + if (!orig_cachepath_size) { + orig_cachepath_size = strlen(chosen_parsed_cachepath); } - return ((strncmp(pathname, location, location_size) == 0) || - (strncmp(pathname, orig_location, orig_location_size) == 0)); + return ((strncmp(pathname, chosen_realized_cachepath, cachepath_size) == 0) || + (strncmp(pathname, chosen_parsed_cachepath, orig_cachepath_size) == 0)); } extern int is_local_prefix(const char *path, char **cached_local_prefixes); diff --git a/src/client/client_comlib/client_api.c b/src/client/client_comlib/client_api.c index 96390dca..f5e07cc0 100644 --- a/src/client/client_comlib/client_api.c +++ b/src/client/client_comlib/client_api.c @@ -36,7 +36,41 @@ static struct lock_t comm_lock; #define COMM_LOCK do { if (lock(&comm_lock) == -1) return -1; } while (0) #define COMM_UNLOCK unlock(&comm_lock) - + + +int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chosen_parsed_cachepath){ + ldcs_message_t message; + char buffer[MAX_PATH_LEN+1]; + buffer[MAX_PATH_LEN] = '\0'; + + message.header.type = LDCS_MSG_CHOSEN_CACHEPATH_REQUEST; + message.header.len = MAX_PATH_LEN; + message.data = buffer; + + COMM_LOCK; + + debug_printf3("sending message of type: request_location_path.\n" ); + client_send_msg(fd, &message); + client_recv_msg_static(fd, &message, LDCS_READ_BLOCK); + + COMM_UNLOCK; + + if (message.header.type != LDCS_MSG_CHOSEN_CACHEPATH || message.header.len > MAX_PATH_LEN) { + err_printf("Got unexpected message of type %d\n", (int) message.header.type); + return -1; + } + char *local_crc = strdup( buffer ); + char *local_cpc = strdup( &buffer[ strlen(local_crc) + 1 ] ); + if( chosen_realized_cachepath ){ + *chosen_realized_cachepath = local_crc; + } + if( chosen_parsed_cachepath ){ + *chosen_parsed_cachepath = local_cpc; + } + + return 0; +} + int send_file_query(int fd, char* path, int dso, char** newpath, int *errcode) { ldcs_message_t message; char buffer[MAX_PATH_LEN+1+sizeof(int)]; @@ -68,7 +102,7 @@ int send_file_query(int fd, char* path, int dso, char** newpath, int *errcode) { if (message.header.type != LDCS_MSG_FILE_QUERY_ANSWER) { err_printf("Got unexpected message of type %d\n", (int) message.header.type); - assert(0); + return -1; } if (message.header.len > sizeof(int)) { @@ -161,7 +195,7 @@ int send_existance_test(int fd, char *path, int *exists) if (message.header.type != LDCS_MSG_EXISTS_ANSWER || message.header.len != sizeof(uint32_t)) { err_printf("Got unexpected message after existance test: %d\n", (int) message.header.type); - assert(0); + return -1; } memcpy(exists, buffer, sizeof(*exists)); @@ -198,7 +232,7 @@ int send_orig_path_request(int fd, const char *path, char *newpath) if (message.header.type != LDCS_MSG_ORIGPATH_ANSWER || message.header.len > MAX_PATH_LEN) { err_printf("Got unexpected message after existance test: %d\n", (int) message.header.type); - assert(0); + return -1; } strncpy(newpath, buffer, MAX_PATH_LEN+1); @@ -315,7 +349,7 @@ int send_cpu(int fd, int cpu) { int send_location(int fd, char *location) { ldcs_message_t message; - message.header.type = LDCS_MSG_LOCATION; + message.header.type = LDCS_MSG_COMMPATH; message.header.len = strlen(location)+1; message.data = location; @@ -346,7 +380,7 @@ int send_ldso_info_request(int fd, const char *ldso_path, char *result_path) if (message.header.type != LDCS_MSG_LOADER_DATA_RESP) { err_printf("Got unexpected message after ldso req: %d\n", (int) message.header.type); - assert(0); + return -1; } return 0; } @@ -388,7 +422,7 @@ int send_rankinfo_query(int fd, int *mylrank, int *mylsize, int *mymdrank, int * if (message.header.type != LDCS_MSG_MYRANKINFO_QUERY_ANSWER || message.header.len != 4*sizeof(int)) { err_printf("Received incorrect response to rankinfo query %d\n", message.header.type); *mylrank = *mylsize = *mymdrank = *mymdsize = -1; - assert(0); + return -1; } p = (int *) message.data; @@ -423,7 +457,7 @@ int send_procmaps_query(int fd, int pid, char *result) if (message.header.type != LDCS_MSG_PROCMAPS_RESP) { err_printf("Received incorrect response to procmaps query %d\n", message.header.type); - assert(0); + return -1; } memcpy(result, buffer, MAX_PATH_LEN); @@ -454,7 +488,7 @@ int send_pickone_query(int fd, char *key, int *result) if (message.header.type != LDCS_MSG_PICKONE_RESP) { err_printf("Received incorrect response to procmaps query %d\n", message.header.type); - assert(0); + return -1; } *result = *((int *) message.data); diff --git a/src/client/client_comlib/client_api.h b/src/client/client_comlib/client_api.h index 74f82346..3d7c41be 100644 --- a/src/client/client_comlib/client_api.h +++ b/src/client/client_comlib/client_api.h @@ -42,6 +42,7 @@ int send_orig_path_request(int fd, const char *path, char *newpath); int send_dirlists_request(int fd, char **local_result, char **exece_result, char **to_free); int send_procmaps_query(int fd, int pid, char *result); int send_pickone_query(int fd, char *key, int *result); +int send_cachepath_query( int fd, char **chosen_symbolic_cachepath, char **chosen_parsed_cachepath); int get_python_prefix(int fd, char **prefix); diff --git a/src/client/config.h.in b/src/client/config.h.in index 6c5eae90..d78a3e7f 100644 --- a/src/client/config.h.in +++ b/src/client/config.h.in @@ -6,6 +6,12 @@ /* Whether we are using a broken srun */ #undef BROKEN_SRUN +/* Colon-separated list of potential back-end cache directories */ +#undef CACHEPATHS + +/* Back-end directory for communication and housekeeping */ +#undef COMMPATH + /* Define if were using biter for client/server communication */ #undef COMM_BITER @@ -118,9 +124,6 @@ /* Default mode for slurm launch */ #undef SLURMLAUNCH_ENABLED -/* The default local directory for Spindle */ -#undef SPINDLE_LOC - /* The default colon-separated list of directories that Spindle will not cache files out of */ #undef SPINDLE_LOCAL_PREFIX diff --git a/src/client/configure b/src/client/configure index dbd69e1f..b854bc84 100755 --- a/src/client/configure +++ b/src/client/configure @@ -811,6 +811,8 @@ enable_maintainer_mode with_default_port with_default_num_ports with_localstorage +with_cachepaths +with_commpath with_default_local_prefix with_testrm with_rm @@ -1532,7 +1534,12 @@ Optional Packages: --with-default-numports=NUM Number of TCP/IP ports to scan for Spindle server communication - --with-localstorage=DIR Directory on back-ends for storing relocated files + --with-localstorage=DIR (obsolete) + Use --with-cachepaths and --with-commpath instead. + --with-cachepaths=DIR Colon-separated list of potential back-end cache + directories + --with-compath=DIR Back-end directory for communication and + housekeeping --with-default-local-prefix=DIRS Colon-seperated list of directories that Spindle will not cache files out of @@ -12589,9 +12596,23 @@ fi # Check whether --with-localstorage was given. if test "${with_localstorage+set}" = set; then : - withval=$with_localstorage; SPINDLE_LOC=${withval} + withval=$with_localstorage; as_fn_error $? "requested obsolete option --with-localstorage. Use --with-cachepaths and --with-commpath instead." "$LINENO" 5 +fi + + +# Check whether --with-cachepaths was given. +if test "${with_cachepaths+set}" = set; then : + withval=$with_cachepaths; CACHEPATHS=${withval} else - SPINDLE_LOC=$DEFAULT_LOC + CACHEPATHS=$DEFAULT_LOC +fi + + +# Check whether --with-commpath was given. +if test "${with_commpath+set}" = set; then : + withval=$with_commpath; COMMPATH=${withval} +else + COMMPATH=$DEFAULT_LOC fi @@ -12599,7 +12620,7 @@ fi if test "${with_default_local_prefix+set}" = set; then : withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval} else - SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$SPINDLE_LOC" + SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH" fi @@ -12619,7 +12640,12 @@ _ACEOF cat >>confdefs.h <<_ACEOF -#define SPINDLE_LOC "$SPINDLE_LOC" +#define COMMPATH "$COMMPATH" +_ACEOF + + +cat >>confdefs.h <<_ACEOF +#define CACHEPATHS "$CACHEPATHS" _ACEOF @@ -12883,7 +12909,7 @@ $as_echo "yes" >&6; } $as_echo "$as_me: WARNING: Slurm launching was explicitly requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected. This version of slurm breaks spindle daemon launch. You might still be able to get spindle to work by running jobs with srun's --overlap option. Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster." >&2;} fi - if [ "x$ENABLE_RSH_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_LAUNCH" != "x1" ] ; then + if [ "x$ENABLE_RSH_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_PLUGIN" != "xtrue" ] ; then if test "x$BROKEN_SRUN" == "x1"; then as_fn_error $? "Slurm support was requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected. This version of slurm breaks spindle daemon launch. You can disable this error message and build spindle with slurm-based daemon launching anyways by explicitly passing the --with-slurm-launch option (you might still be able to get spindle to work by running jobs with srun's --overlap option). Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster." "$LINENO" 5 else diff --git a/src/cobo/cobo.c b/src/cobo/cobo.c index 785a5d03..2d1a53a9 100644 --- a/src/cobo/cobo.c +++ b/src/cobo/cobo.c @@ -160,7 +160,7 @@ static char* cobo_getenv(char* envvar, int type) char* str = getenv(envvar); if (str == NULL && type == ENV_REQUIRED) { err_printf("Missing required environment variable: %s\n", envvar); - exit(1); + return NULL; } return str; } @@ -171,7 +171,7 @@ static void* cobo_malloc(size_t n, char* msg) void* p = malloc(n); if (!p) { err_printf("Call to malloc(%lu) failed: %s (%m errno %d)\n", n, msg, errno); - exit(1); + return NULL; } return p; } @@ -513,7 +513,7 @@ static int cobo_connect_hostname(char* hostname, int rank) break; case HSHAKE_INTERNAL_ERROR: err_printf("Internal error doing handshake: %s", spindle_handshake_last_error_str()); - exit(-1); + return -1; break; case HSHAKE_DROP_CONNECTION: debug_printf3("Handshake said to drop connection\n"); @@ -768,7 +768,7 @@ static int cobo_open_tree() if (sockfd < 0) { err_printf("Creating parent socket (socket() %m errno=%d)\n", errno); - exit(1); + return -1; } setCloseOnExec(sockfd); @@ -817,7 +817,7 @@ static int cobo_open_tree() if (!port_is_bound) { /* TODO: would like to send an abort back to server */ err_printf("Failed to open socket on any port\n"); - exit(1); + return -1; } /* accept a connection from parent and receive socket table */ @@ -837,7 +837,7 @@ static int cobo_open_tree() break; case HSHAKE_INTERNAL_ERROR: err_printf("Internal error doing handshake: %s", spindle_handshake_last_error_str()); - exit(-1); + return -1; break; case HSHAKE_DROP_CONNECTION: debug_printf3("Handshake said to drop connection\n"); @@ -907,26 +907,26 @@ static int cobo_open_tree() /* read our rank number */ if (cobo_read_fd(cobo_parent_fd, &cobo_me, sizeof(int)) < 0) { err_printf("Receiving my rank from parent failed\n"); - exit(1); + return -1; } /* discover how many ranks are in our world */ if (cobo_read_fd(cobo_parent_fd, &cobo_nprocs, sizeof(int)) < 0) { err_printf("Receiving number of tasks from parent failed\n"); - exit(1); + return -1; } /* read the size of the hostlist (in bytes) */ if (cobo_read_fd(cobo_parent_fd, &cobo_hostlist_size, sizeof(int)) < 0) { err_printf("Receiving size of hostname table from parent failed\n"); - exit(1); + return -1; } /* allocate space for the hostlist and read it in */ cobo_hostlist = (void*) cobo_malloc(cobo_hostlist_size, "Hostlist data buffer"); if (cobo_read_fd(cobo_parent_fd, cobo_hostlist, cobo_hostlist_size) < 0) { err_printf("Receiving hostname table from parent failed\n"); - exit(1); + return -1; } /* @@ -969,7 +969,7 @@ static int cobo_open_tree() if (cobo_child_fd[i] == -1) { err_printf("Failed to connect to child (rank %d) on %s failed\n", c, child_hostname); - exit(1); + return -1; } /* tell child what rank he is and forward the hostname table to him */ @@ -978,7 +978,7 @@ static int cobo_open_tree() if (forward != COBO_SUCCESS) { err_printf("Failed to forward hostname table to child (rank %d) on %s failed\n", c, child_hostname); - exit(1); + return -1; } /* free the child hostname string */ @@ -1033,7 +1033,7 @@ static int cobo_bcast_tree(void* buf, int size) if (cobo_me != 0) { if (cobo_read_fd(cobo_parent_fd, buf, size) < 0) { err_printf("Receiving broadcast data from parent failed\n"); - exit(1); + return -1; } } @@ -1042,7 +1042,7 @@ static int cobo_bcast_tree(void* buf, int size) if (cobo_write_fd(cobo_child_fd[i], buf, size) < 0) { err_printf("Broadcasting data to child (rank %d) failed\n", cobo_child[i]); - exit(1); + return -1; } } @@ -1058,7 +1058,7 @@ int cobo_bcast_down(void* buf, int size) if (cobo_write_fd(cobo_child_fd[i], buf, size) < 0) { err_printf("Broadcasting data to child (rank %d) failed\n", cobo_child[i]); - exit(1); + return -1; } } return rc; @@ -1080,7 +1080,7 @@ static int cobo_allreduce_max_int_tree(int* sendbuf, int* recvbuf) if (cobo_read_fd(cobo_child_fd[i], &child_val, sizeof(child_val)) < 0) { err_printf("Reducing data from child (rank %d) failed\n", cobo_child[i]); - exit(1); + return -1; } /* compare child's max to our current max */ @@ -1094,7 +1094,7 @@ static int cobo_allreduce_max_int_tree(int* sendbuf, int* recvbuf) /* not the root, so forward our reduction result to our parent */ if (cobo_write_fd(cobo_parent_fd, &max_val, sizeof(max_val)) < 0) { err_printf("Sending reduced data to parent failed\n"); - exit(1); + return -1; } } else { /* we're the root, got the result, set the recvbuf */ @@ -1129,7 +1129,7 @@ static int cobo_gather_tree(void* sendbuf, int sendcount, void* recvbuf) if (cobo_read_fd(cobo_child_fd[i], (char*)bigbuf + offset, sendcount * cobo_child_incl[i]) < 0) { err_printf("Gathering data from child (rank %d) failed\n", cobo_child[i]); - exit(1); + return -1; } offset += sendcount * cobo_child_incl[i]; } @@ -1138,7 +1138,7 @@ static int cobo_gather_tree(void* sendbuf, int sendcount, void* recvbuf) if (cobo_me != 0) { if (cobo_write_fd(cobo_parent_fd, bigbuf, bigcount) < 0) { err_printf("Sending gathered data to parent failed\n"); - exit(1); + return -1; } cobo_free(bigbuf); } @@ -1158,7 +1158,7 @@ static int cobo_scatter_tree(void* sendbuf, int sendcount, void* recvbuf) bigbuf = (void*) cobo_malloc(bigcount, "Temporary scatter buffer in cobo_scatter_tree"); if (cobo_read_fd(cobo_parent_fd, bigbuf, bigcount) < 0) { err_printf("Receiving scatter data from parent failed\n"); - exit(1); + return -1; } } @@ -1169,7 +1169,7 @@ static int cobo_scatter_tree(void* sendbuf, int sendcount, void* recvbuf) if (cobo_write_fd(cobo_child_fd[i], (char*)bigbuf + offset, sendcount * cobo_child_incl[i]) < 0) { err_printf("Scattering data to child (rank %d) failed\n", cobo_child[i]); - exit(1); + return -1; } offset += sendcount * cobo_child_incl[i]; } @@ -1257,7 +1257,7 @@ int cobo_bcast(void* buf, int sendcount, int root) rc = cobo_bcast_tree(buf, sendcount); } else { err_printf("Cannot execute bcast from non-zero root\n"); - exit(1); + return -1; } cobo_gettimeofday(&end); @@ -1283,7 +1283,7 @@ int cobo_gather(void* sendbuf, int sendcount, void* recvbuf, int root) rc = cobo_gather_tree(sendbuf, sendcount, recvbuf); } else { err_printf("Cannot execute gather to non-zero root\n"); - exit(1); + return -1; } cobo_gettimeofday(&end); @@ -1309,7 +1309,7 @@ int cobo_scatter(void* sendbuf, int sendcount, void* recvbuf, int root) rc = cobo_scatter_tree(sendbuf, sendcount, recvbuf); } else { err_printf("Cannot execute scatter from non-zero root\n"); - exit(1); + return -1; } cobo_gettimeofday(&end); @@ -1354,7 +1354,7 @@ int cobo_alltoall(void* sendbuf, int sendcount, void* recvbuf) int rc = COBO_SUCCESS; err_printf("Cannot execute alltoall\n"); - exit(1); + return -1; cobo_gettimeofday(&end); debug_printf3("Exiting cobo_alltoall(), took %f seconds for %d procs\n", cobo_getsecs(&end,&start), cobo_nprocs); @@ -1434,6 +1434,48 @@ int cobo_allgather_str(char* sendstr, char*** recvstr, char** recvbuf) return COBO_SUCCESS; } +int cobo_allreduce( uint64_t *pval, cobo_op_t op ){ + + /* if i have any children, receive their data */ + uint64_t child_val; + for(int i=cobo_num_child-1; i>=0; i--) { + /* read int64_t from child */ + if (cobo_read_fd(cobo_child_fd[i], &child_val, sizeof(int64_t)) < 0) { + err_printf("Reducing data from child (rank %d) failed\n", cobo_child[i]); + return -1; + } + + /* compare child's val to our current val */ + switch( op ){ + case COBO_OP_MIN: if( child_val < *pval ) *pval = child_val; break; + case COBO_OP_MAX: if( child_val > *pval ) *pval = child_val; break; + case COBO_OP_BITWISE_AND: *pval &= child_val; break; + case COBO_OP_BITWISE_OR: *pval |= child_val; break; + case COBO_OP_LOGICAL_AND: *pval = *pval && child_val; break; + case COBO_OP_LOGICAL_OR: *pval = *pval || child_val; break; + case COBO_OP_SUM: *pval += child_val; break; + case COBO_OP_NOOP: break; + default: + err_printf("Illegal op (%d). Ignoring.\n", op); + break; + } + } + + /* forward data to parent if we're not rank 0, otherwise set the recvbuf */ + if (cobo_me != 0) { + /* not the root, so forward our reduction result to our parent */ + if (cobo_write_fd(cobo_parent_fd, pval, sizeof(*pval)) < 0) { + err_printf("Sending reduced data to parent failed\n"); + return -1; + } + } + + /* broadcast result of reduction from rank 0 to all tasks */ + cobo_bcast_tree(pval, sizeof(int64_t)); + + return COBO_SUCCESS; +} + /* provide list of ports and number of ports as input, get number of tasks and my rank as output */ int cobo_open(uint64_t sessionid, int* portlist, int num_ports, int* rank, int* num_ranks) { @@ -1482,7 +1524,7 @@ int cobo_open(uint64_t sessionid, int* portlist, int num_ports, int* rank, int* cobo_ports = cobo_int_dup(portlist, num_ports); if (cobo_ports == NULL) { err_printf("Failed to copy port list\n"); - exit(1); + return -1; } /* open the tree */ @@ -1491,7 +1533,7 @@ int cobo_open(uint64_t sessionid, int* portlist, int num_ports, int* rank, int* /* need to check that tree opened successfully before returning, so do a barrier */ if (cobo_barrier() != COBO_SUCCESS) { err_printf("Failed to open tree\n"); - exit(1); + return -1; } if (cobo_me == 0) { diff --git a/src/cobo/ldcs_cobo.h b/src/cobo/ldcs_cobo.h index edacd4b1..fafbda6a 100644 --- a/src/cobo/ldcs_cobo.h +++ b/src/cobo/ldcs_cobo.h @@ -55,6 +55,7 @@ extern "C" { #define cobo_allgather COMBINE(COBO_NAMESPACE, cobo_allgather) #define cobo_alltoall COMBINE(COBO_NAMESPACE, cobo_alltoall ) #define cobo_allgather_str COMBINE(COBO_NAMESPACE, cobo_allgather_str) +#define cobo_allreduce COMBINE(COBO_NAMESPACE, cobo_allreduce) #define cobo_server_open COMBINE(COBO_NAMESPACE, cobo_server_open) #define cobo_server_close COMBINE(COBO_NAMESPACE, cobo_server_close) #define cobo_server_get_root_socket COMBINE(COBO_NAMESPACE, cobo_server_get_root_socket) @@ -67,6 +68,19 @@ extern "C" { #define cobo_register_preconnect_cb COMBINE(COBO_NAMESPACE, cobo_register_preconnect_cb) #endif +// Used for cobo_allreduce(). +typedef enum{ + COBO_OP_MIN, + COBO_OP_MAX, + COBO_OP_BITWISE_AND, + COBO_OP_BITWISE_OR, + COBO_OP_LOGICAL_AND, + COBO_OP_LOGICAL_OR, + COBO_OP_SUM, + COBO_OP_NOOP, + NUM_COBO_OP +} cobo_op_t; + /* * ========================================================================== * ========================================================================== @@ -128,6 +142,8 @@ int cobo_alltoall (void* sendbuf, int sendcount, void* recvbuf); */ int cobo_allgather_str(char* sendstr, char*** recvstr, char** recvbuf); +int cobo_allreduce(uint64_t *pval, cobo_op_t op); + /* * ========================================================================== * ========================================================================== diff --git a/src/fe/config.h.in b/src/fe/config.h.in index 357feea0..3ac96e42 100644 --- a/src/fe/config.h.in +++ b/src/fe/config.h.in @@ -6,6 +6,12 @@ /* Whether we are using a broken srun */ #undef BROKEN_SRUN +/* Colon-separated list of potential back-end cache directories */ +#undef CACHEPATHS + +/* Back-end directory for communication and housekeeping */ +#undef COMMPATH + /* Define if were using biter for client/server communication */ #undef COMM_BITER @@ -163,9 +169,6 @@ /* Default mode for slurm launch */ #undef SLURMLAUNCH_ENABLED -/* The default local directory for Spindle */ -#undef SPINDLE_LOC - /* The default colon-separated list of directories that Spindle will not cache files out of */ #undef SPINDLE_LOCAL_PREFIX diff --git a/src/fe/configure b/src/fe/configure index 95e99784..7384873e 100755 --- a/src/fe/configure +++ b/src/fe/configure @@ -832,6 +832,8 @@ enable_maintainer_mode with_default_port with_default_num_ports with_localstorage +with_cachepaths +with_commpath with_default_local_prefix with_testrm with_rm @@ -1570,7 +1572,12 @@ Optional Packages: --with-default-numports=NUM Number of TCP/IP ports to scan for Spindle server communication - --with-localstorage=DIR Directory on back-ends for storing relocated files + --with-localstorage=DIR (obsolete) + Use --with-cachepaths and --with-commpath instead. + --with-cachepaths=DIR Colon-separated list of potential back-end cache + directories + --with-compath=DIR Back-end directory for communication and + housekeeping --with-default-local-prefix=DIRS Colon-seperated list of directories that Spindle will not cache files out of @@ -16439,9 +16446,23 @@ fi # Check whether --with-localstorage was given. if test "${with_localstorage+set}" = set; then : - withval=$with_localstorage; SPINDLE_LOC=${withval} + withval=$with_localstorage; as_fn_error $? "requested obsolete option --with-localstorage. Use --with-cachepaths and --with-commpath instead." "$LINENO" 5 +fi + + +# Check whether --with-cachepaths was given. +if test "${with_cachepaths+set}" = set; then : + withval=$with_cachepaths; CACHEPATHS=${withval} else - SPINDLE_LOC=$DEFAULT_LOC + CACHEPATHS=$DEFAULT_LOC +fi + + +# Check whether --with-commpath was given. +if test "${with_commpath+set}" = set; then : + withval=$with_commpath; COMMPATH=${withval} +else + COMMPATH=$DEFAULT_LOC fi @@ -16449,7 +16470,7 @@ fi if test "${with_default_local_prefix+set}" = set; then : withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval} else - SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$SPINDLE_LOC" + SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH" fi @@ -16469,7 +16490,12 @@ _ACEOF cat >>confdefs.h <<_ACEOF -#define SPINDLE_LOC "$SPINDLE_LOC" +#define COMMPATH "$COMMPATH" +_ACEOF + + +cat >>confdefs.h <<_ACEOF +#define CACHEPATHS "$CACHEPATHS" _ACEOF @@ -16733,7 +16759,7 @@ $as_echo "yes" >&6; } $as_echo "$as_me: WARNING: Slurm launching was explicitly requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected. This version of slurm breaks spindle daemon launch. You might still be able to get spindle to work by running jobs with srun's --overlap option. Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster." >&2;} fi - if [ "x$ENABLE_RSH_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_LAUNCH" != "x1" ] ; then + if [ "x$ENABLE_RSH_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_PLUGIN" != "xtrue" ] ; then if test "x$BROKEN_SRUN" == "x1"; then as_fn_error $? "Slurm support was requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected. This version of slurm breaks spindle daemon launch. You can disable this error message and build spindle with slurm-based daemon launching anyways by explicitly passing the --with-slurm-launch option (you might still be able to get spindle to work by running jobs with srun's --overlap option). Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster." "$LINENO" 5 else diff --git a/src/fe/startup/config_mgr.cc b/src/fe/startup/config_mgr.cc index 36a30b20..1b9d9ade 100644 --- a/src/fe/startup/config_mgr.cc +++ b/src/fe/startup/config_mgr.cc @@ -50,10 +50,16 @@ using namespace std; #define SPINDLE_NUM_PORTS_STR "250" #endif -#if defined(SPINDLE_LOC) -#define SPINDLE_LOC_STR SPINDLE_LOC +#if defined(COMMPATH) +#define SPINDLE_COMMPATH_STR COMMPATH #else -#define SPINDLE_LOC_STR "$TMPDIR" +#define SPINDLE_COMMPATH_STR "$TMPDIR" +#endif + +#if defined(CACHEPATHS) +#define SPINDLE_CACHEPATHS_STR CACHEPATHS +#else +#define SPINDLE_CACHEPATHS_STR "$TMPDIR" #endif #if defined(SPINDLE_LOCAL_PREFIX) @@ -267,8 +273,10 @@ void initOptionsList() "Provides a text file containing a white-space separated list of files that should be relocated to each node before execution begins" }, { confStrip, "strip", shortStrip, groupMisc, cvBool, {}, "true", "Strip debug and symbol information from binaries before distributing them." }, - { confLocation, "location", shortLocation, groupMisc, cvString, {}, SPINDLE_LOC_STR, - "Back-end directory for storing relocated files. Should be a non-shared location such as a ramdisk." }, + { confCommPath, "commpath", shortCommPath, groupMisc, cvString, {}, SPINDLE_COMMPATH_STR, + "Back-end directory communication and housekeeping. Should be a non-shared location such as a ramdisk." }, + { confCachePaths, "cachepaths", shortCachePaths, groupMisc, cvString, {}, SPINDLE_CACHEPATHS_STR, + "Colon-separated list of candidate paths for cached libraries."}, { confNoclean, "noclean", shortNoClean, groupMisc, cvBool, {}, "false", "Don't remove local file cache after execution." }, { confDisableLogging, "disable-logging", shortDisableLogging, groupMisc, cvBool, {}, DISABLE_LOGGING_STR, @@ -735,9 +743,24 @@ bool ConfigMap::toSpindleArgs(spindle_args_t &args, bool alloc_strs) const case confNumPorts: args.num_ports = numresult; break; - case confLocation: { - string loc = strresult + "/spindle.$NUMBER"; - args.location = strdup(loc.c_str()); + case confCommPath: { + string path = strresult + "/spindle.$NUMBER"; + args.commpath = strdup(path.c_str()); + break; + } + case confCachePaths:{ + // Paramemter values are colon-separated lists of paths. + // Append "/spindle.$NUMBER" to each path in the list. + string paths = strresult; + size_t idx = paths.find(":"); + string number_var_with_colon("/spindle.$NUMBER:"); + string number_var_without_colon("/spindle.$NUMBER"); + while( idx != string::npos ){ + paths.replace(idx, 1, number_var_with_colon); + idx = paths.find(":", idx + number_var_with_colon.size()); + }; + paths += number_var_without_colon; + args.candidate_cachepaths = strdup(paths.c_str()); break; } case confCachePrefix: diff --git a/src/fe/startup/config_mgr.h b/src/fe/startup/config_mgr.h index 8e70daa6..ec3c8135 100644 --- a/src/fe/startup/config_mgr.h +++ b/src/fe/startup/config_mgr.h @@ -29,7 +29,8 @@ enum SpindleConfigID { confCmdlineNewgroup, confPort, confNumPorts, - confLocation, + confCommPath, + confCachePaths, confCachePrefix, confPythonPrefix, confLocalPrefix, @@ -82,7 +83,7 @@ enum CmdlineShortOptions { shortAuditType = 'k', shortRelocSO = 'l', shortNoClean = 'n', - shortLocation = 'o', + shortCommPath = 'o', shortPush = 'p', shortPull = 'q', shortPythonPrefix = 'r', @@ -125,7 +126,8 @@ enum CmdlineShortOptions { shortSpindleLevel = 296, shortLocalPrefix = 297, shortExecExcludes = 298, - shortPatchLdso + shortPatchLdso = 299, + shortCachePaths = 300, }; enum CmdlineGroups { diff --git a/src/fe/startup/parse_launcher.cc b/src/fe/startup/parse_launcher.cc index f05ba18a..4484e998 100644 --- a/src/fe/startup/parse_launcher.cc +++ b/src/fe/startup/parse_launcher.cc @@ -292,7 +292,7 @@ void ModifyArgv::modifyCmdLine() snprintf(options_str, 32, "%lu", (unsigned long) params->opts); string options(options_str); - string location(params->location); + string commpath(params->commpath); char number_str[32]; snprintf(number_str, 32, "%lu", (unsigned long) params->number); @@ -319,7 +319,7 @@ void ModifyArgv::modifyCmdLine() if (p == parser->appExecutableAt()) { #if defined(os_bluegene) string bg_env_str = parser->getParser()->getBGString(); - parser->getParser()->addBGEnvStr(n, new_argv, bg_env_str, default_libstr, intercept_libstr, location, number, options, shmcache_size); + parser->getParser()->addBGEnvStr(n, new_argv, bg_env_str, default_libstr, intercept_libstr, commpath, number, options, shmcache_size); #else char **a_argv; int a_argc; diff --git a/src/fe/startup/parse_launcher_args.cc b/src/fe/startup/parse_launcher_args.cc index 84cf7f23..710b009d 100644 --- a/src/fe/startup/parse_launcher_args.cc +++ b/src/fe/startup/parse_launcher_args.cc @@ -127,7 +127,7 @@ static cmdoption_t openmpi_options[] = { }; -static const char *openmpi_bg_env_str = "-x LD_AUDIT=%s -x LDCS_LOCATION=%s -x LDCS_NUMBER=%s -x LDCS_OPTIONS=%s LDCS_CACHESIZE=%s"; +static const char *openmpi_bg_env_str = "-x LD_AUDIT=%s -x LDCS_COMMPATH=%s -x LDCS_NUMBER=%s -x LDCS_OPTIONS=%s LDCS_CACHESIZE=%s"; static const unsigned int openmpi_size = (sizeof(openmpi_options) / sizeof(cmdoption_t)); static cmdoption_t srun_options[] = { @@ -218,11 +218,11 @@ static cmdoption_t srun_options[] = { { NULL, "--usage", 0 }, { "-V", "--version", 0 } }; -static const char *srun_bg_env_str = "--runjob-opts=--envs LD_AUDIT=%s LD_PRELOAD=%s LDCS_LOCATION=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s"; -static const char *srun_bg_env_str_nopreload = "--runjob-opts=--envs LD_AUDIT=%s%s LDCS_LOCATION=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s"; -static const char *srun_bg_env_bare_str = "%s --envs LD_AUDIT=%s LD_PRELOAD=%s LDCS_LOCATION=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s"; -static const char *srun_bg_env_bare_str_preload = "%sLD_PRELOAD=%s:%s LD_AUDIT=%s LDCS_LOCATION=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s"; -static const char *srun_bg_env_bare_str_nopreload = "%s LD_AUDIT=%s LDCS_LOCATION=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s"; +static const char *srun_bg_env_str = "--runjob-opts=--envs LD_AUDIT=%s LD_PRELOAD=%s LDCS_COMMPATH=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s"; +static const char *srun_bg_env_str_nopreload = "--runjob-opts=--envs LD_AUDIT=%s%s LDCS_COMMPATH=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s"; +static const char *srun_bg_env_bare_str = "%s --envs LD_AUDIT=%s LD_PRELOAD=%s LDCS_COMMPATH=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s"; +static const char *srun_bg_env_bare_str_preload = "%sLD_PRELOAD=%s:%s LD_AUDIT=%s LDCS_COMMPATH=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s"; +static const char *srun_bg_env_bare_str_nopreload = "%s LD_AUDIT=%s LDCS_COMMPATH=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s"; static const unsigned int srun_size (sizeof(srun_options) / sizeof(cmdoption_t)); diff --git a/src/fe/startup/spindle_fe.cc b/src/fe/startup/spindle_fe.cc index 31b29cf7..a038e201 100644 --- a/src/fe/startup/spindle_fe.cc +++ b/src/fe/startup/spindle_fe.cc @@ -41,6 +41,7 @@ static const char *logging_file = NULL; #endif static const char spindle_bootstrap[] = LIBEXECDIR "/spindle_bootstrap"; static bool sendAndWaitForAlive(); +static void determineCachepathConsensus(); #define STARTUP_TIMEOUT 60 @@ -70,7 +71,8 @@ static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size) buffer_size += sizeof(number_t); buffer_size += sizeof(opt_t); buffer_size += sizeof(unique_id_t); - buffer_size += args->location ? strlen(args->location) + 1 : 1; + buffer_size += args->commpath ? strlen(args->commpath) + 1 : 1; + buffer_size += args->candidate_cachepaths ? strlen(args->candidate_cachepaths) + 1 : 1; buffer_size += args->pythonprefix ? strlen(args->pythonprefix) + 1 : 1; buffer_size += args->preloadfile ? strlen(args->preloadfile) + 1 : 1; buffer_size += args->numa_files ? strlen(args->numa_files) + 1 : 1; @@ -90,7 +92,8 @@ static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size) pack_param(args->use_launcher, buf, pos); pack_param(args->startup_type, buf, pos); pack_param(args->shm_cache_size, buf, pos); - pack_param(args->location, buf, pos); + pack_param(args->commpath, buf, pos); + pack_param(args->candidate_cachepaths, buf, pos); pack_param(args->pythonprefix, buf, pos); pack_param(args->preloadfile, buf, pos); pack_param(args->bundle_timeout_ms, buf, pos); @@ -229,7 +232,8 @@ int getApplicationArgsFE(spindle_args_t *params, int *spindle_argc, char ***spin (*spindle_argv)[n++] = strdup(numports_s); (*spindle_argv)[n++] = strdup(uniqueid_s); } - (*spindle_argv)[n++] = strdup(params->location); + (*spindle_argv)[n++] = strdup(params->commpath); + (*spindle_argv)[n++] = strdup(params->candidate_cachepaths); (*spindle_argv)[n++] = strdup(number_s); (*spindle_argv)[n++] = strdup(opt_s); (*spindle_argv)[n++] = strdup(cachesize_s); @@ -258,9 +262,9 @@ int fillInSpindleArgsCmdlineFE(spindle_args_t *params, unsigned int options, int for (i = 0; i < sargc && sargv && sargv[i] != NULL; i++) { mod_argv[i+1] = sargv[i]; } - i++; - mod_argv[i++] = const_cast("launcher"); - mod_argv[i] = NULL; + mod_argv[i+1] = const_cast("launcher"); + mod_argv[i+2] = NULL; + mod_argc = i+2; string errmsg; bool result = gatherAllConfigInfo(mod_argc, mod_argv, false, config, errmsg); @@ -394,10 +398,12 @@ int spindleInitFE(const char **hosts, spindle_args_t *params) /* Start FE server */ debug_printf("spindle_args_t { number = %lu; port = %u; num_ports = %u; opts = %lu; unique_id = %lu; " - "use_launcher = %u; startup_type = %u; shm_cache_size = %u; location = %s; " + "use_launcher = %u; startup_type = %u; shm_cache_size = %u; commpath = %s; " + "cachepaths = %s; " "pythonprefix = %s; preloadfile = %s; bundle_timeout_ms = %u; bundle_cachesize_kb = %u }\n", (unsigned long) params->number, params->port, params->num_ports, params->opts, params->unique_id, - params->use_launcher, params->startup_type, params->shm_cache_size, params->location, + params->use_launcher, params->startup_type, params->shm_cache_size, params->commpath, + params->candidate_cachepaths, params->pythonprefix, params->preloadfile, params->bundle_timeout_ms, params->bundle_cachesize_kb); printSpindleFlags(params->opts); @@ -427,6 +433,7 @@ int spindleInitFE(const char **hosts, spindle_args_t *params) /* Wait for servers to indicate startup */ sendAndWaitForAlive(); + determineCachepathConsensus(); return 0; } @@ -483,6 +490,17 @@ void markRSHPidReapedFE() clear_fe_rsh_pid(); } +static void determineCachepathConsensus( void ){ + ldcs_message_t consensus_req_msg; + consensus_req_msg.header.type = LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS; + consensus_req_msg.header.len = 0; + consensus_req_msg.data = NULL; + int result = ldcs_audit_server_fe_broadcast(&consensus_req_msg, NULL); + if (result == -1) { + debug_printf("Failure sending cachepath consensus message\n"); + } +} + static bool sendAndWaitForAlive() { int result; diff --git a/src/flux/Makefile.in b/src/flux/Makefile.in index 18199543..e6733b24 100644 --- a/src/flux/Makefile.in +++ b/src/flux/Makefile.in @@ -361,6 +361,7 @@ SCONTROL_ABSPATH = @SCONTROL_ABSPATH@ SED = @SED@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ +SINFO_ABSPATH = @SINFO_ABSPATH@ SRUN_PATH = @SRUN_PATH@ STATICFLAG = @STATICFLAG@ STRIP = @STRIP@ diff --git a/src/flux/flux-spindle.c b/src/flux/flux-spindle.c index 6c99341a..101bb6ac 100644 --- a/src/flux/flux-spindle.c +++ b/src/flux/flux-spindle.c @@ -381,8 +381,8 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) int numa = 0; const char *relocaout = NULL, *reloclibs = NULL, *relocexec = NULL, *relocpython = NULL; const char *followfork = NULL, *preload = NULL, *level = NULL; - const char *pyprefix = NULL, *location = NULL; - char *numafiles = NULL; + const char *pyprefix = NULL, *commpath = NULL; + char *numafiles = NULL, *cachepaths = NULL; if (flux_shell_getopt_unpack (shell, "spindle", "o", &opts) < 0) return -1; @@ -404,7 +404,7 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) * supplied by the user, but not unpacked (This handles typos, etc). */ if (json_unpack_ex (opts, &error, JSON_STRICT, - "{s?i s?i s?i s?i s?s s?s s?s s?s s?s s?s s?s s?i s?s s?s s?s}", + "{s?i s?i s?i s?i s?s s?s s?s s?s s?s s?s s?s s?i s?s s?s s?s s?s}", "noclean", &noclean, "nostrip", &nostrip, "push", &push, @@ -415,11 +415,12 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) "reloc-exec", &relocexec, "reloc-python", &relocpython, "python-prefix", &pyprefix, - "location", &location, + "commpath", &commpath, "numa", &numa, "numa-files", &numafiles, "preload", &preload, - "level", &level) < 0) + "level", &level, + "cachepaths", &cachepaths) < 0) logerrno_printf_and_return(1, "Error in spindle option: %s\n", error.text); if (noclean) @@ -462,8 +463,11 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) free (ctx->params.pythonprefix); ctx->params.pythonprefix = tmp; } - if (location) { - ctx->params.location = (char *) location; + if( cachepaths ){ + ctx->params.candidate_cachepaths = cachepaths; + } + if (commpath) { + ctx->params.commpath = (char *) commpath; } if (level) { if (strcmp(level, "high") == 0) { diff --git a/src/flux/sessionmgr.c b/src/flux/sessionmgr.c index 49324a2f..17027163 100644 --- a/src/flux/sessionmgr.c +++ b/src/flux/sessionmgr.c @@ -109,16 +109,16 @@ char **strip_start_from_argv(int argc, char **argv) extern char *parse_location(char *loc, int number); extern int spindle_mkdir(char *orig_path); -#if !defined(SPINDLE_LOC) -#error SPINDLE_LOC must be defined in config.h +#if !defined(COMMPATH) +#error COMMPATH must be defined in config.h #endif const char *get_session_dir() { int result; char *dir; - dir = parse_location((char *) (SPINDLE_LOC "/spindle_session"), 0); + dir = parse_location((char *) (COMMPATH "/spindle_session"), 0); if (!dir) { - spindle_debug_printf(1, "ERROR: Could not parse directory for spindle session location from %s/spindle_session\n", SPINDLE_LOC); + spindle_debug_printf(1, "ERROR: Could not parse directory for spindle session location from %s/spindle_session\n", COMMPATH); return NULL; } diff --git a/src/include/ldcs_api.h b/src/include/ldcs_api.h index e8ffa43d..49ba45e3 100644 --- a/src/include/ldcs_api.h +++ b/src/include/ldcs_api.h @@ -45,7 +45,7 @@ typedef enum { LDCS_MSG_END, LDCS_MSG_CWD, LDCS_MSG_PID, - LDCS_MSG_LOCATION, + LDCS_MSG_COMMPATH, LDCS_MSG_CPU, LDCS_MSG_MYRANKINFO_QUERY, LDCS_MSG_MYRANKINFO_QUERY_ANSWER, @@ -85,6 +85,9 @@ typedef enum { LDCS_MSG_PICKONE_RESP, LDCS_MSG_ALIVE_REQ, LDCS_MSG_ALIVE_RESP, + LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS, + LDCS_MSG_CHOSEN_CACHEPATH_REQUEST, + LDCS_MSG_CHOSEN_CACHEPATH, LDCS_MSG_UNKNOWN } ldcs_message_ids_t; diff --git a/src/include/spindle_launch.h b/src/include/spindle_launch.h index 47a4d92e..ca7b8d3d 100644 --- a/src/include/spindle_launch.h +++ b/src/include/spindle_launch.h @@ -123,8 +123,11 @@ typedef struct { /* Size of client shared memory cache */ unsigned int shm_cache_size; - /* The local-disk location where Spindle will store its cache */ - char *location; + /* The local-disk location for communication and housekeeping. */ + char *commpath; + + /* Path[s] for cached libraries. */ + char *candidate_cachepaths; /* Colon-separated list of candidate paths (max 64) */ /* Colon-seperated list of directories where Python is installed */ char *pythonprefix; diff --git a/src/logging/spindle_logc.c b/src/logging/spindle_logc.c index 9e5edb1f..4e82300f 100644 --- a/src/logging/spindle_logc.c +++ b/src/logging/spindle_logc.c @@ -76,6 +76,7 @@ void spawnLogDaemon(char *tempdir) if (result == 0) { char *params[7]; int cur = 0; + setpgid(0, 0); /* escape to own process group */ params[cur++] = spindle_log_daemon_name; params[cur++] = tempdir; if (spindle_debug_prints) { diff --git a/src/logging/spindle_logd.cc b/src/logging/spindle_logd.cc index 53eb5dc0..45d58a7d 100644 --- a/src/logging/spindle_logd.cc +++ b/src/logging/spindle_logd.cc @@ -202,7 +202,7 @@ class TestVerifier std::vector err_strings; std::set > target_libs; std::set > libs_loaded; - char *location; + char *cachepath; void logerror(std::string s) { @@ -249,7 +249,8 @@ class TestVerifier tmp_s = getenv("TEMPDIR"); if (!tmp_s) tmp_s = "/tmp"; - location = strdup(tmp_s); + // These are reasonable fallbacks that should be replaced via messages, below. + cachepath = strdup(tmp_s); } ~TestVerifier() @@ -268,7 +269,7 @@ class TestVerifier strstr(filename, "retzero") == NULL && strstr(filename, ".py") == NULL) return true; - bool is_from_temp = (strstr(filename, location) != NULL) && (strncmp(filename, "/__not_exist", 12) != 0); + bool is_from_temp = (strstr(filename, cachepath) != NULL) && (strncmp(filename, "/__not_exist", 12) != 0); bool is_local_test = strstr(filename, "liblocal") != NULL; if (is_from_temp && !is_local_test && ret_code == -1) { @@ -294,12 +295,12 @@ class TestVerifier char buffer[4096]; int ret; - if (strstr(s, " location=" ) == s ){ - free( location ); - const char *loc_start = strstr( s, "=") + 1; - size_t loc_len = strlen( loc_start ); - location = strdup( loc_start ); - location[ loc_len - 1 ] = '\0'; // Remove trailing '\n'. + if (strstr(s, " cachepath=" ) == s ){ + free( cachepath ); + const char *cachepath_start = strstr( s, "=") + 1; + size_t cachepath_len = strlen( cachepath_start ); + cachepath = strdup( cachepath_start ); + cachepath[ cachepath_len - 1 ] = '\0'; // Remove trailing '\n'. } if (strstr(s, "open(") == s) { const char *first_quote, *last_quote, *equals; diff --git a/src/server/auditserver/cleanup_proc.cc b/src/server/auditserver/cleanup_proc.cc index c23a9f7d..a3d3ddcc 100644 --- a/src/server/auditserver/cleanup_proc.cc +++ b/src/server/auditserver/cleanup_proc.cc @@ -72,7 +72,8 @@ static void rmDirSet(const set &dirs, const char *prefix_dir) continue; if (strncmp(prefix_dir, componentpath.c_str(), prefix_size) != 0) { - err_printf("Tried to clean a file %s that wasn't in our prefix %s\n", componentpath.c_str(), prefix_dir); + // We have multiple directory roots. Not a problem if the directory + // we're looking for isn't in this one. continue; } unlink(componentpath.c_str()); @@ -83,7 +84,6 @@ static void rmDirSet(const set &dirs, const char *prefix_dir) sort(ordered_dirs.begin(), ordered_dirs.end(), longest_str_first); for (vector::iterator i = ordered_dirs.begin(); i != ordered_dirs.end(); i++) { if (strncmp(prefix_dir, i->c_str(), prefix_size) != 0) { - err_printf("Tried to rmdir directory %s that wasn't in our prefix %s\n", i->c_str(), prefix_dir); continue; } rmdir(i->c_str()); diff --git a/src/server/auditserver/ldcs_audit_server_handlers.c b/src/server/auditserver/ldcs_audit_server_handlers.c index 8a18de24..0129c25d 100644 --- a/src/server/auditserver/ldcs_audit_server_handlers.c +++ b/src/server/auditserver/ldcs_audit_server_handlers.c @@ -178,7 +178,10 @@ static int handle_setup_alias(ldcs_process_data_t *procdata, char *pathname, cha static int handle_client_dirlists_req(ldcs_process_data_t *procdata, int nc); static int handle_close_client_query(ldcs_process_data_t *procdata, int nc); static int handle_alive_msg(ldcs_process_data_t *procdata, ldcs_message_t *msg); +static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_message_t *msg); +static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc); +extern void getValidCachePathByIndex( uint64_t validBitIdx, char **realizedCachePath, char **parsedCachePath, char **symbolicCachePath ); /** * Query from client to server. Returns info about client's rank in server data structures. **/ @@ -196,10 +199,10 @@ static int handle_client_info_msg(ldcs_process_data_t *procdata, int nc, ldcs_me client->remote_pid=mypid; debug_printf2("Server recvd pid %d from %d\n", mypid, nc); } - else if(msg->header.type == LDCS_MSG_LOCATION) { + else if(msg->header.type == LDCS_MSG_COMMPATH) { strncpy(client->remote_location, msg->data, sizeof(client->remote_location)-1); client->remote_location[sizeof(client->remote_location)-1] = '\0'; - debug_printf2("Server recvd location %s from %d\n", msg->data, nc); + debug_printf2("Server recvd remote_location %s from %d\n", msg->data, nc); } else if (msg->header.type == LDCS_MSG_CPU) { int clientcpu; @@ -1863,7 +1866,7 @@ int handle_client_message(ldcs_process_data_t *procdata, int nc, ldcs_message_t switch (msg->header.type) { case LDCS_MSG_CWD: case LDCS_MSG_PID: - case LDCS_MSG_LOCATION: + case LDCS_MSG_COMMPATH: case LDCS_MSG_CPU: return handle_client_info_msg(procdata, nc, msg); case LDCS_MSG_PYTHONPREFIX_REQ: @@ -1890,6 +1893,8 @@ int handle_client_message(ldcs_process_data_t *procdata, int nc, ldcs_message_t return handle_client_pickone_msg(procdata, nc, msg); case LDCS_MSG_END: return handle_client_end(procdata, nc); + case LDCS_MSG_CHOSEN_CACHEPATH_REQUEST: + return handle_chosen_cachepath_request(procdata, nc); default: err_printf("Received unexpected message from client %d: %d\n", nc, (int) msg->header.type); assert(0); @@ -1987,6 +1992,8 @@ int handle_server_message(ldcs_process_data_t *procdata, node_peer_t peer, ldcs_ case LDCS_MSG_ALIVE_REQ: case LDCS_MSG_ALIVE_RESP: return handle_alive_msg(procdata, msg); + case LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS: + return handle_cachepath_consensus(procdata, msg); default: err_printf("Received unexpected message from node: %d\n", (int) msg->header.type); assert(0); @@ -2945,6 +2952,70 @@ static int handle_client_pickone_msg(ldcs_process_data_t *procdata, int nc, ldcs } } +/** + * Handle LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS to determine which cachepaths are + * available across all of the servers. + */ + +static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_message_t *msg){ + + int num_children = ldcs_audit_server_md_get_num_children(procdata); + + if (num_children) { + spindle_broadcast(procdata, msg); + msgbundle_force_flush(procdata); + } + + ldcs_audit_server_md_allreduce_AND( &procdata->cachepath_bitidx ); + + if( procdata->cachepath_bitidx == 0 ){ + err_printf("No valid cachepath path available. Falling back to \"commpath\" path (%s).\n", procdata->commpath); + procdata->cachepath = procdata->commpath; + }else{ + // ldcs_audit_server_filemngt_init() does it's own realize() pass. + getValidCachePathByIndex( procdata->cachepath_bitidx, + &procdata->cachepath, + &procdata->parsed_cachepath, + &procdata->symbolic_cachepath); + } + + debug_printf3("Initializing file cache cachepath %s\n", procdata->cachepath); + ldcs_audit_server_filemngt_init(procdata->cachepath); + + test_printf(" cachepath=%s\n", procdata->cachepath); + return 0; +} + +/** + * Handle LDCS_MSG_CHOSEN_CACHEPATH_REQUEST + */ +static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc){ + ldcs_message_t msg; + int connid; + ldcs_client_t *client; + + assert(nc != -1); + client = procdata->client_table + nc; + connid = client->connid; + if (client->state != LDCS_CLIENT_STATUS_ACTIVE || connid < 0) + return 0; + + + msg.header.type = LDCS_MSG_CHOSEN_CACHEPATH; + + msg.header.len = strlen(procdata->cachepath) + 1 + strlen(procdata->parsed_cachepath) + 1; + msg.data = calloc( 1, msg.header.len ); + strcpy( msg.data, procdata->cachepath ); + strcpy( &msg.data[ strlen(procdata->cachepath)+1 ], procdata->parsed_cachepath ); + ldcs_send_msg(connid, &msg); + free( msg.data ); + procdata->server_stat.clientmsg.cnt++; + procdata->server_stat.clientmsg.time += ldcs_get_time() - client->query_arrival_time; + + return 0; +} + + /** * Handle alive message, which is a broadcast/response ping through all servers */ @@ -3153,7 +3224,7 @@ int exit_note_cb(int fd, int serverid, void *data) eresult = -1; } - result = handleExitNote(fd, procdata->location); + result = handleExitNote(fd, procdata->commpath); if (result == -1) { debug_printf("handleExitNote failed\n"); eresult = -1; diff --git a/src/server/auditserver/ldcs_audit_server_md.h b/src/server/auditserver/ldcs_audit_server_md.h index eb5bf9f6..a4640370 100644 --- a/src/server/auditserver/ldcs_audit_server_md.h +++ b/src/server/auditserver/ldcs_audit_server_md.h @@ -107,6 +107,9 @@ int ldcs_audit_server_md_broadcast_noncontig(ldcs_process_data_t *ldcs_process_d int ldcs_audit_server_md_get_num_children(ldcs_process_data_t *procdata); int ldcs_audit_server_md_is_parent(node_peer_t peer); + +void ldcs_audit_server_md_allreduce_AND( uint64_t *val ); + #if defined(__cplusplus) } diff --git a/src/server/auditserver/ldcs_audit_server_md_cobo.c b/src/server/auditserver/ldcs_audit_server_md_cobo.c index 08c9b952..27393a55 100644 --- a/src/server/auditserver/ldcs_audit_server_md_cobo.c +++ b/src/server/auditserver/ldcs_audit_server_md_cobo.c @@ -401,3 +401,7 @@ int ldcs_audit_server_md_get_num_children(ldcs_process_data_t *procdata) cobo_get_num_childs(&num_childs); return num_childs; } + +void ldcs_audit_server_md_allreduce_AND( uint64_t *val ){ + cobo_allreduce( val, COBO_OP_BITWISE_AND ); +} diff --git a/src/server/auditserver/ldcs_audit_server_md_msocket.c b/src/server/auditserver/ldcs_audit_server_md_msocket.c index c9d616c2..6db18bd4 100644 --- a/src/server/auditserver/ldcs_audit_server_md_msocket.c +++ b/src/server/auditserver/ldcs_audit_server_md_msocket.c @@ -62,7 +62,7 @@ int ldcs_audit_server_md_init ( ldcs_process_data_t *ldcs_process_data ) { int rc=0; char* ldcs_nportsstr=getenv("LDCS_NPORTS"); - char* ldcs_locmodstr=getenv("LDCS_LOCATION_MOD"); + char* ldcs_locmodstr=getenv("LDCS_COMMPATH_MOD"); int usedport; int serverfd, serverid, i; diff --git a/src/server/auditserver/ldcs_audit_server_process.c b/src/server/auditserver/ldcs_audit_server_process.c index 312095be..a73a7b7f 100644 --- a/src/server/auditserver/ldcs_audit_server_process.c +++ b/src/server/auditserver/ldcs_audit_server_process.c @@ -37,6 +37,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "msgbundle.h" #include "exitnote.h" #include "cleanup_proc.h" +#include "parseloc.h" //#define GPERFTOOLS #if defined(GPERFTOOLS) @@ -112,7 +113,7 @@ void startprofile(spindle_args_t *args) char hostname[257]; char *home = getenv("HOME"); if (!home || !*home) - home = ldcs_process_data.location; + home = ldcs_process_data.commpath; gethostname(hostname, sizeof(hostname)); snprintf(filename, 4096, "%s/spindled.%lu.%s.%d.prof", home, (unsigned long) args->number, hostname, getpid()); ProfilerStart(filename); @@ -141,7 +142,10 @@ int ldcs_audit_server_process(spindle_args_t *args) startprofile(args); debug_printf3("Initializing server data structures\n"); - ldcs_process_data.location = args->location; + ldcs_process_data.commpath = args->commpath; + ldcs_process_data.cachepaths = args->candidate_cachepaths; + ldcs_process_data.cachepath = NULL; + ldcs_process_data.cachepath_bitidx = 0; ldcs_process_data.number = args->number; ldcs_process_data.pythonprefix = args->pythonprefix; ldcs_process_data.localprefix = args->local_prefixes; @@ -191,14 +195,12 @@ int ldcs_audit_server_process(spindle_args_t *args) } ldcs_process_data.server_stat.hostname=ldcs_process_data.hostname; - debug_printf3("Initializing file cache location %s\n", ldcs_process_data.location); - ldcs_audit_server_filemngt_init(ldcs_process_data.location); if (ldcs_process_data.opts & OPT_PROCCLEAN) - init_cleanup_proc(ldcs_process_data.location); + init_cleanup_proc(ldcs_process_data.commpath); debug_printf3("Initializing connections for clients at %s and %lu\n", - ldcs_process_data.location, (unsigned long) ldcs_process_data.number); - serverid = ldcs_create_server(ldcs_process_data.location, ldcs_process_data.number); + ldcs_process_data.commpath, (unsigned long) ldcs_process_data.number); + serverid = ldcs_create_server(ldcs_process_data.commpath, ldcs_process_data.number); if (serverid == -1) { err_printf("Unable to setup area for client connections\n"); return -1; @@ -214,7 +216,7 @@ int ldcs_audit_server_process(spindle_args_t *args) ldcs_listen_register_fd(fd, serverid, &_ldcs_server_CB, (void *) &ldcs_process_data); if (args->opts & OPT_BEEXIT) { - fd = createExitNote(args->location); + fd = createExitNote(args->commpath); if (fd != -1) { ldcs_listen_register_fd(fd, serverid, exit_note_cb, (void *) &ldcs_process_data); } @@ -229,6 +231,10 @@ int ldcs_audit_server_process(spindle_args_t *args) if (fd != -1) { ldcs_listen_register_fd(fd, serverid, forceExitCB, (void *) &ldcs_process_data); } + determineValidCachePaths( + &ldcs_process_data.cachepath_bitidx, + ldcs_process_data.cachepaths, + ldcs_process_data.number ); return 0; } @@ -248,7 +254,7 @@ int ldcs_audit_server_run() _ldcs_server_stat_print(&ldcs_process_data.server_stat); - debug_printf("destroy server (%s,%lu)\n", ldcs_process_data.location, (unsigned long) ldcs_process_data.number); + debug_printf("destroy server (%s,%lu)\n", ldcs_process_data.commpath, (unsigned long) ldcs_process_data.number); ldcs_destroy_server(ldcs_process_data.serverid); /* destroy md support (multi-daemon) */ diff --git a/src/server/auditserver/ldcs_audit_server_process.h b/src/server/auditserver/ldcs_audit_server_process.h index 18b3320a..82b60023 100644 --- a/src/server/auditserver/ldcs_audit_server_process.h +++ b/src/server/auditserver/ldcs_audit_server_process.h @@ -125,7 +125,15 @@ struct ldcs_process_data_struct int exit_readys_recvd; ldcs_dist_model_t dist_model; ldcs_client_t* client_table; - char *location; + char *commpath; /* Single user-specified path for fifo, daemons, etc. */ + /* (Everything except the cachepath.) */ + char *cachepaths; /* Up to 64 colon-separated list of candidate cachepaths. */ + char *cachepath; /* The earliest path in the list available to all servers. */ + /* (Environment variables replaced, symbolic links realized.) */ + char *symbolic_cachepath; /* The original representation of the cachepath. */ + char *parsed_cachepath; /* The cachepath with environment variables replaced. */ + /* (Symbolic links, if any, remain.) */ + uint64_t cachepath_bitidx; /* Bit index of valid cachepaths on a given server. */ char *hostname; char *pythonprefix; char *localprefix; diff --git a/src/server/comlib/ldcs_api_util.c b/src/server/comlib/ldcs_api_util.c index 2bc2455d..5ea51ab8 100644 --- a/src/server/comlib/ldcs_api_util.c +++ b/src/server/comlib/ldcs_api_util.c @@ -51,7 +51,7 @@ char* _message_type_to_str (ldcs_message_ids_t type) { STR_CASE(LDCS_MSG_END); STR_CASE(LDCS_MSG_CWD); STR_CASE(LDCS_MSG_PID); - STR_CASE(LDCS_MSG_LOCATION); + STR_CASE(LDCS_MSG_COMMPATH); STR_CASE(LDCS_MSG_CPU); STR_CASE(LDCS_MSG_MYRANKINFO_QUERY); STR_CASE(LDCS_MSG_MYRANKINFO_QUERY_ANSWER); @@ -91,6 +91,9 @@ char* _message_type_to_str (ldcs_message_ids_t type) { STR_CASE(LDCS_MSG_PICKONE_RESP); STR_CASE(LDCS_MSG_ALIVE_REQ); STR_CASE(LDCS_MSG_ALIVE_RESP); + STR_CASE(LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS); + STR_CASE(LDCS_MSG_CHOSEN_CACHEPATH_REQUEST); + STR_CASE(LDCS_MSG_CHOSEN_CACHEPATH); STR_CASE(LDCS_MSG_UNKNOWN); } return "unknown"; diff --git a/src/server/config.h.in b/src/server/config.h.in index 6eea236a..87fbf990 100644 --- a/src/server/config.h.in +++ b/src/server/config.h.in @@ -6,6 +6,12 @@ /* Whether we are using a broken srun */ #undef BROKEN_SRUN +/* Colon-separated list of potential back-end cache directories */ +#undef CACHEPATHS + +/* Back-end directory for communication and housekeeping */ +#undef COMMPATH + /* Define if were using biter for client/server communication */ #undef COMM_BITER @@ -145,9 +151,6 @@ /* Default mode for slurm launch */ #undef SLURMLAUNCH_ENABLED -/* The default local directory for Spindle */ -#undef SPINDLE_LOC - /* The default colon-separated list of directories that Spindle will not cache files out of */ #undef SPINDLE_LOCAL_PREFIX diff --git a/src/server/configure b/src/server/configure index e32c1cfe..adac8311 100755 --- a/src/server/configure +++ b/src/server/configure @@ -838,6 +838,8 @@ enable_maintainer_mode with_default_port with_default_num_ports with_localstorage +with_cachepaths +with_commpath with_default_local_prefix with_testrm with_rm @@ -1567,7 +1569,12 @@ Optional Packages: --with-default-numports=NUM Number of TCP/IP ports to scan for Spindle server communication - --with-localstorage=DIR Directory on back-ends for storing relocated files + --with-localstorage=DIR (obsolete) + Use --with-cachepaths and --with-commpath instead. + --with-cachepaths=DIR Colon-separated list of potential back-end cache + directories + --with-compath=DIR Back-end directory for communication and + housekeeping --with-default-local-prefix=DIRS Colon-seperated list of directories that Spindle will not cache files out of @@ -16436,9 +16443,23 @@ fi # Check whether --with-localstorage was given. if test "${with_localstorage+set}" = set; then : - withval=$with_localstorage; SPINDLE_LOC=${withval} + withval=$with_localstorage; as_fn_error $? "requested obsolete option --with-localstorage. Use --with-cachepaths and --with-commpath instead." "$LINENO" 5 +fi + + +# Check whether --with-cachepaths was given. +if test "${with_cachepaths+set}" = set; then : + withval=$with_cachepaths; CACHEPATHS=${withval} else - SPINDLE_LOC=$DEFAULT_LOC + CACHEPATHS=$DEFAULT_LOC +fi + + +# Check whether --with-commpath was given. +if test "${with_commpath+set}" = set; then : + withval=$with_commpath; COMMPATH=${withval} +else + COMMPATH=$DEFAULT_LOC fi @@ -16446,7 +16467,7 @@ fi if test "${with_default_local_prefix+set}" = set; then : withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval} else - SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$SPINDLE_LOC" + SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH" fi @@ -16466,7 +16487,12 @@ _ACEOF cat >>confdefs.h <<_ACEOF -#define SPINDLE_LOC "$SPINDLE_LOC" +#define COMMPATH "$COMMPATH" +_ACEOF + + +cat >>confdefs.h <<_ACEOF +#define CACHEPATHS "$CACHEPATHS" _ACEOF @@ -16730,7 +16756,7 @@ $as_echo "yes" >&6; } $as_echo "$as_me: WARNING: Slurm launching was explicitly requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected. This version of slurm breaks spindle daemon launch. You might still be able to get spindle to work by running jobs with srun's --overlap option. Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster." >&2;} fi - if [ "x$ENABLE_RSH_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_LAUNCH" != "x1" ] ; then + if [ "x$ENABLE_RSH_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_PLUGIN" != "xtrue" ] ; then if test "x$BROKEN_SRUN" == "x1"; then as_fn_error $? "Slurm support was requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected. This version of slurm breaks spindle daemon launch. You can disable this error message and build spindle with slurm-based daemon launching anyways by explicitly passing the --with-slurm-launch option (you might still be able to get spindle to work by running jobs with srun's --overlap option). Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster." "$LINENO" 5 else diff --git a/src/server/startup/spindle_be.cc b/src/server/startup/spindle_be.cc index 7493c020..c3e7bbd9 100644 --- a/src/server/startup/spindle_be.cc +++ b/src/server/startup/spindle_be.cc @@ -58,7 +58,8 @@ static int unpack_data(spindle_args_t *args, void *buffer, int buffer_size) unpack_param(args->use_launcher, buf, pos); unpack_param(args->startup_type, buf, pos); unpack_param(args->shm_cache_size, buf, pos); - unpack_param(args->location, buf, pos); + unpack_param(args->commpath, buf, pos); + unpack_param(args->candidate_cachepaths, buf, pos); unpack_param(args->pythonprefix, buf, pos); unpack_param(args->preloadfile, buf, pos); unpack_param(args->bundle_timeout_ms, buf, pos); @@ -143,16 +144,15 @@ int spindleRunBE(unsigned int port, unsigned int num_ports, unique_id_t unique_i assert(args.port == port); - /* Expand environment variables in location. */ - char *new_location = parse_location(args.location, args.number); - if (!new_location) { - err_printf("Failed to convert location %s\n", args.location); + /* Expand environment variables in commpath. */ + char *new_commpath = parse_location(args.commpath, args.number); + if (!new_commpath) { + err_printf("Failed to convert commpath %s\n", args.commpath); return -1; } - debug_printf("Translated location from %s to %s\n", args.location, new_location); - free(args.location); - args.location = new_location; - test_printf(" location=%s\n", args.location); + debug_printf("Translated commpath from %s to %s\n", args.commpath, new_commpath); + free(args.commpath); + args.commpath = strdup(new_commpath); result = ldcs_audit_server_process(&args); if (result == -1) { diff --git a/src/slurm_plugin/Makefile.am b/src/slurm_plugin/Makefile.am index a9638b65..489dffcb 100644 --- a/src/slurm_plugin/Makefile.am +++ b/src/slurm_plugin/Makefile.am @@ -3,7 +3,7 @@ lib_LTLIBRARIES = libspindleslurm.la libver=`$(top_srcdir)/LIB_VERSION spindleslurm` libspindleslurm_la_SOURCES = encode_decode.c plugin_utils.c slurm_plugin.c $(top_srcdir)/src/utils/spindle_mkdir.c $(top_srcdir)/src/utils/parseloc.c -libspindleslurm_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/include -I$(top_srcdir)/src/logging -DUSE_PLUGIN_DEBUG -DDEBUG -DCUSTOM_GETENV -DCUSTOM_GETENV_FREE -DSCONTROL_BIN="$(SCONTROL_ABSPATH)" -DSPINDLE_DO_EXPORT +libspindleslurm_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/include -I$(top_srcdir)/src/logging -DUSE_PLUGIN_DEBUG -DDEBUG -DCUSTOM_GETENV -DCUSTOM_GETENV_FREE -DSCONTROL_BIN="$(SCONTROL_ABSPATH)" -DSINFO_BIN="$(SINFO_ABSPATH)" -DSPINDLE_DO_EXPORT libspindleslurm_la_LDFLAGS = $(AM_LDFLAGS) -ldl -version-info $(libver) libspindleslurm_la_LIBADD = $(top_builddir)/src/server/startup/libspindlebe.la $(top_builddir)/src/fe/startup/libspindlefe.la libspindleslurm_la_CFLAGS = $(CFLAGS) -fvisibility=hidden diff --git a/src/slurm_plugin/Makefile.in b/src/slurm_plugin/Makefile.in index c8488a96..a7fe188f 100644 --- a/src/slurm_plugin/Makefile.in +++ b/src/slurm_plugin/Makefile.in @@ -315,6 +315,7 @@ SCONTROL_ABSPATH = @SCONTROL_ABSPATH@ SED = @SED@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ +SINFO_ABSPATH = @SINFO_ABSPATH@ SRUN_PATH = @SRUN_PATH@ STATICFLAG = @STATICFLAG@ STRIP = @STRIP@ @@ -382,7 +383,7 @@ top_srcdir = @top_srcdir@ lib_LTLIBRARIES = libspindleslurm.la libver = `$(top_srcdir)/LIB_VERSION spindleslurm` libspindleslurm_la_SOURCES = encode_decode.c plugin_utils.c slurm_plugin.c $(top_srcdir)/src/utils/spindle_mkdir.c $(top_srcdir)/src/utils/parseloc.c -libspindleslurm_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/include -I$(top_srcdir)/src/logging -DUSE_PLUGIN_DEBUG -DDEBUG -DCUSTOM_GETENV -DCUSTOM_GETENV_FREE -DSCONTROL_BIN="$(SCONTROL_ABSPATH)" -DSPINDLE_DO_EXPORT +libspindleslurm_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/include -I$(top_srcdir)/src/logging -DUSE_PLUGIN_DEBUG -DDEBUG -DCUSTOM_GETENV -DCUSTOM_GETENV_FREE -DSCONTROL_BIN="$(SCONTROL_ABSPATH)" -DSINFO_BIN="$(SINFO_ABSPATH)" -DSPINDLE_DO_EXPORT libspindleslurm_la_LDFLAGS = $(AM_LDFLAGS) -ldl -version-info $(libver) libspindleslurm_la_LIBADD = $(top_builddir)/src/server/startup/libspindlebe.la $(top_builddir)/src/fe/startup/libspindlefe.la libspindleslurm_la_CFLAGS = $(CFLAGS) -fvisibility=hidden diff --git a/src/slurm_plugin/plugin_utils.c b/src/slurm_plugin/plugin_utils.c index 7112e090..392677c4 100644 --- a/src/slurm_plugin/plugin_utils.c +++ b/src/slurm_plugin/plugin_utils.c @@ -22,6 +22,11 @@ #else #define SLURM_SCONTROL_BIN "scontrol" #endif +#if defined SINFO_BIN +#define SLURM_SINFO_BIN STR(SINFO_BIN) +#else +#define SLURM_SINFO_BIN "sinfo" +#endif extern char *parse_location(char *loc, number_t number); @@ -46,7 +51,6 @@ char **getHostsScontrol(unsigned int num_hosts, const char *hoststr) scontrol_cmdline_len = strlen(scontrol_path) + strlen(scontrol_args) + strlen(hoststr) + strlen(scontrol_suffix) + 6; scontrol_cmdline = (char *) malloc(scontrol_cmdline_len); - sdprintf(2, "Running scontrol to get host list: %s\n", scontrol_cmdline); result = snprintf(scontrol_cmdline, scontrol_cmdline_len, "%s %s \"%s\" %s", scontrol_path, scontrol_args, hoststr, scontrol_suffix); if (result >= scontrol_cmdline_len) { @@ -54,6 +58,7 @@ char **getHostsScontrol(unsigned int num_hosts, const char *hoststr) scontrol_cmdline, result, scontrol_cmdline_len); goto done; } + sdprintf(2, "Running scontrol to get host list: %s\n", scontrol_cmdline); f = popen(scontrol_cmdline, "r"); if (!f) { @@ -119,6 +124,86 @@ char **getHostsScontrol(unsigned int num_hosts, const char *hoststr) return ret; } +char **getHostAddrSinfo(unsigned int num_hosts, char **hostlist) +{ + const char *sinfo_path = SLURM_SINFO_BIN; + const char *sinfo_args = "-O NodeAddr -h -n"; + const char *sinfo_suffix = "2> /dev/null"; + FILE *f = NULL; + char **hostaddrlist = NULL, *s, *sinfo_cmdline = NULL, **ret = NULL; + int i, j, hostnamelen; + int result; + size_t maxHostLen = 0, sinfo_cmdlineLen, len; + + hostaddrlist = calloc(num_hosts+1, sizeof(char*)); + + for (i = 0; i < num_hosts; i++) { + if (hostlist[i]) { + size_t thisLen = strlen(hostlist[i]); + if (thisLen > maxHostLen) maxHostLen = thisLen; + } + } + + sinfo_cmdlineLen = strlen(sinfo_path) + strlen(sinfo_args) + + maxHostLen + strlen(sinfo_suffix) + 6; + sinfo_cmdline = (char *) malloc(sinfo_cmdlineLen); + + for (i = 0; i < num_hosts; i++) { + if (!hostlist[i] || !strlen(hostlist[i])) goto done; + result = snprintf(sinfo_cmdline, sinfo_cmdlineLen, "%s %s \"%s\" %s", + sinfo_path, sinfo_args, hostlist[i], sinfo_suffix); + if (result >= sinfo_cmdlineLen) { + sdprintf(1, "ERROR: Formatting error creating sinfo cmdline '%s' (%d)\n", + sinfo_cmdline, result); + goto done; + } + sdprintf(2, "Running sinfo to get host address: %s\n", sinfo_cmdline); + + f = popen(sinfo_cmdline, "r"); + if (!f) { + sdprintf(1, "ERROR: Could not run sinfo: %s\n", sinfo_cmdline); + goto done; + } + + len = 0; + result = getline(&(hostaddrlist[i]), &len, f); + pclose(f); + if (result == -1) { + int error = errno; + sdprintf(1, "ERROR: Resolving '%s' failed: %s\n", hostlist[i], strerror(error)); + (void) error; + goto done; + } + + s = hostaddrlist[i]; + hostnamelen = strlen(s); + for (j = 0; j < hostnamelen; j++) { + if (!((s[j] >= '0' && s[j] <= '9') || + (s[j] >= 'a' && s[j] <= 'z') || + (s[j] >= 'A' && s[j] <= 'Z') || + (s[j] == '-' || s[j] == '_' || s[j] == '.'))) { + s[j] = '\0'; + break; + } + } + sdprintf(3, "sinfo returned hostaddr %s for %s\n", s, hostlist[i]); + } + if (i != num_hosts) { + sdprintf(1, "ERROR: expected %d hosts from sinfo. Got %d\n", num_hosts, i); + goto done; + } + + ret = hostaddrlist; + done: + if (sinfo_cmdline) + free(sinfo_cmdline); + if (!ret && hostaddrlist) { + for (i = 0; i < num_hosts; i++) free(hostaddrlist[i]); + free(hostlist); + } + return ret; +} + int isFEHost(char **hostlist, unsigned int num_hosts) { char host[256]; @@ -172,18 +257,19 @@ int isFEHost(char **hostlist, unsigned int num_hosts) return feresult; } +char *unique_file = NULL; + #define UNIQUE_FILE_NAME "spindle_unique" -int isBEProc(spindle_args_t *params) +int isBEProc(spindle_args_t *params, unsigned int exit_phase) { - char *dir = NULL, *expanded_dir = NULL, *realized_dir = NULL; - char *unique_file = NULL; + char *dir = NULL, *expanded_dir = NULL, *realized_dir = NULL, *phase_name = NULL; char hostname[256], session_id_str[32]; size_t unique_file_len; int beproc_result = -1; int fd = -1, error; - - dir = params->location; + + dir = params->commpath; if (!dir) { sdprintf(1, "ERROR: Location not filled in\n"); goto done; @@ -203,13 +289,16 @@ int isBEProc(spindle_args_t *params) snprintf(session_id_str, sizeof(session_id_str), "%lu", (unsigned long) params->number); + phase_name = exit_phase ? "exit" : "launch"; + unique_file_len = strlen(realized_dir) + 1 + strlen(UNIQUE_FILE_NAME) + 1 + + strlen(phase_name) + 1 + strlen(hostname) + 1 + strlen(session_id_str) + 1; unique_file = (char *) malloc(sizeof(char*) * unique_file_len); - snprintf(unique_file, unique_file_len, "%s/%s.%s.%s", realized_dir, UNIQUE_FILE_NAME, hostname, session_id_str); + snprintf(unique_file, unique_file_len, "%s/%s.%s.%s.%s", realized_dir, UNIQUE_FILE_NAME, phase_name, hostname, session_id_str); spindle_mkdir(realized_dir); @@ -218,9 +307,11 @@ int isBEProc(spindle_args_t *params) sdprintf(2, "Opened %s to result %d\n", unique_file, fd); if (fd != -1) beproc_result = 1; - else if (error == EEXIST) + else if (error == EEXIST) { beproc_result = 0; - else { + free(unique_file); + unique_file = NULL; + } else { sdprintf(1, "ERROR: Could not create spindle unique_file %s: %s\n", unique_file, strerror(error)); goto done; } @@ -230,8 +321,6 @@ int isBEProc(spindle_args_t *params) free(expanded_dir); if (realized_dir) free(realized_dir); - if (unique_file) - free(unique_file); if (fd != -1) close(fd); sdprintf(2, "returning %d\n", beproc_result); @@ -526,7 +615,7 @@ int registerFEPid(pid_t pid, spindle_args_t *args) int fd; int result; - snprintf(pid_file, sizeof(pid_file), "%s/fepid", args->location); + snprintf(pid_file, sizeof(pid_file), "%s/fepid", args->commpath); pid_file[sizeof(pid_file)-1] = '\0'; snprintf(pid_s, sizeof(pid_s), "%d\n", (int) pid); @@ -557,7 +646,7 @@ int readFEPid(pid_t *pid, spindle_args_t *args) pid_t pid_result; int fd, result; - snprintf(pid_file, sizeof(pid_file), "%s/fepid", args->location); + snprintf(pid_file, sizeof(pid_file), "%s/fepid", args->commpath); pid_file[sizeof(pid_file)-1] = '\0'; sdprintf(2, "Reading FE pid from %s\n", pid_file); @@ -700,6 +789,7 @@ pid_t grandchild_fork() exit(result == sizeof(grandchild_pid) ? 0 : -1); } //In grandchild + setpgid(0, 0); /* escape to own process group */ fork_result = 0; goto done; diff --git a/src/slurm_plugin/plugin_utils.h b/src/slurm_plugin/plugin_utils.h index 971d5707..acf2ac5d 100644 --- a/src/slurm_plugin/plugin_utils.h +++ b/src/slurm_plugin/plugin_utils.h @@ -39,10 +39,12 @@ int decodeSpindleConfig(const char *encodedstr, int *spindle_argc, char ***spindle_argv); char **getHostsScontrol(unsigned int num_hosts, const char *hoststr); +char **getHostAddrSinfo(unsigned int num_hosts, char **hostlist); char **getHostsParse(unsigned int num_hosts, const char *shortlist); int isFEHost(char **hostlist, unsigned int num_hosts); -int isBEProc(spindle_args_t *params); +extern char *unique_file; +int isBEProc(spindle_args_t *params, unsigned int exit_phase); char *encodeCmdArgs(int sargc, char **sargv); void decodeCmdArgs(char *cmd, int *sargc, char ***sargv); diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c index 58ed086f..6aadfa6d 100644 --- a/src/slurm_plugin/slurm_plugin.c +++ b/src/slurm_plugin/slurm_plugin.c @@ -29,14 +29,17 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "spindle_launch.h" #include "plugin_utils.h" +#include "config.h" SPINDLE_EXPORT extern const char plugin_name[]; SPINDLE_EXPORT extern const char plugin_type[]; SPINDLE_EXPORT extern const unsigned int plugin_version; SPINDLE_EXPORT extern struct spank_option spank_options[]; SPINDLE_EXPORT int slurm_spank_task_init(spank_t spank, int ac, char *argv[]); +SPINDLE_EXPORT int slurm_spank_task_exit(spank_t spank, int ac, char *argv[]); SPINDLE_EXPORT int slurm_spank_exit(spank_t spank, int site_argc, char *site_argv[]); + SPANK_PLUGIN(spindle, 1) typedef struct { @@ -67,6 +70,7 @@ static int enable_spindle = 0; extern char **environ; extern char *parse_location(char *loc, number_t number); +extern char *realize(char *path); struct spank_option spank_options[] = { @@ -83,10 +87,8 @@ int slurm_spank_task_init(spank_t spank, int site_argc, char *site_argv[]) int result, func_result = -1; saved_env_t *env = NULL; static int initialized = 0; - spindle_args_t params; - int combined_argc; - char **combined_argv; - + spindle_args_t params = {0}; + if (!enable_spindle) return 0; @@ -109,12 +111,16 @@ int slurm_spank_task_init(spank_t spank, int site_argc, char *site_argv[]) push_env(spank, &env); sdprintf(1, "Beginning spindle plugin\n"); - result = process_spindle_args(spank, site_argc, site_argv, ¶ms, &combined_argc, &combined_argv); + result = process_spindle_args(spank, site_argc, site_argv, ¶ms, NULL, NULL); if (result == -1) { sdprintf(1, "Error processesing spindle arguments. Aborting spindle\n"); goto done; } + if (params.opts & OPT_OFF) { + return 0; + } + result = launch_spindle(spank, ¶ms); if (result == -1) { sdprintf(1, "Error launching spindle. Aborting spindle\n"); @@ -137,7 +143,7 @@ int slurm_spank_task_init(spank_t spank, int site_argc, char *site_argv[]) return func_result; } -int slurm_spank_exit(spank_t spank, int site_argc, char *site_argv[]) +int slurm_spank_task_exit(spank_t spank, int site_argc, char *site_argv[]) { spank_context_t context; char *result_str; @@ -223,11 +229,11 @@ static unique_id_t getUniqueID(spank_t spank) static int fillInArgs(spank_t spank, spindle_args_t *args, int argc, char **argv, unique_id_t unique_id) { int result; - char *oldlocation; + char *symbolic_commpath, *orig_commpath; char *err_string; args->unique_id = unique_id; - args->number = (unsigned long) args->unique_id; + args->number = (number_t) args->unique_id; result = fillInSpindleArgsCmdlineFE(args, SPINDLE_FILLARGS_NOUNIQUEID | SPINDLE_FILLARGS_NONUMBER, argc, argv, &err_string); if (result == -1) { @@ -239,11 +245,17 @@ static int fillInArgs(spank_t spank, spindle_args_t *args, int argc, char **argv return -1; } args->opts |= OPT_BEEXIT; + args->use_launcher = slurm_plugin_launcher; + args->startup_type = startup_external; + + symbolic_commpath = args->commpath; + orig_commpath = parse_location(symbolic_commpath, args->number); + if( !orig_commpath ){ + return -1; + } + args->commpath = realize(orig_commpath); - oldlocation = args->location; current_spank = spank; - args->location = parse_location(oldlocation, args->number); - free(oldlocation); return 0; } @@ -273,9 +285,9 @@ static int process_spindle_args(spank_t spank, int site_argc, char *site_argv[], site_options_size = strlen(site_options); user_options_size = strlen(user_options); - combined_options_size = site_options_size + user_options_size + 2; + combined_options_size = site_options_size + user_options_size + 3; combined_options = (char *) malloc(combined_options_size); - snprintf(combined_options, combined_options_size, "%s%s%s", + snprintf(combined_options, combined_options_size, "%s%s%s ", site_options, (site_options_size && user_options_size) ? " " : "", user_options); @@ -301,7 +313,7 @@ static int process_spindle_args(spank_t spank, int site_argc, char *site_argv[], if (spindle_config) free(spindle_config); - if (combined_argv) { + if (!out_argv && combined_argv) { for (i = 0; i < combined_argc; i++) { if (combined_argv[i]) free(combined_argv[i]); @@ -337,15 +349,17 @@ static char **get_hostlist(spank_t spank, unsigned int num_hosts) { char *short_hosts, **hostlist;; - short_hosts = readSpankEnv(spank, "SLURM_JOB_NODELIST"); + short_hosts = readSpankEnv(spank, "SLURM_STEP_NODELIST"); + if (!short_hosts) + short_hosts = readSpankEnv(spank, "SLURM_JOB_NODELIST"); if (!short_hosts) short_hosts = readSpankEnv(spank, "SLURM_NODELIST"); if (!short_hosts) { - sdprintf(1, "ERROR: SLURM_JOB_NODELIST not set.\n"); + sdprintf(1, "ERROR: None of SLURM_STEP_NODELIST, SLURM_JOB_NODELIST, SLURM_NODELIST set.\n"); return NULL; } -#if defined(USE_SCONTROL) +#if defined(SCONTROL_BIN) hostlist = getHostsScontrol(num_hosts, short_hosts); #else hostlist = getHostsParse(num_hosts, short_hosts); @@ -428,11 +442,11 @@ static int get_spindle_args(spank_t spank, spindle_args_t *params) static int launch_spindle(spank_t spank, spindle_args_t *params) { - char **hostlist = NULL; + char **hostlist = NULL, **hostaddrlist = NULL; int result; int is_fe_host = 0; int is_be_leader = 0; - unsigned int num_hosts; + unsigned int i, num_hosts; int num_hosts_result; int launch_result = -1; @@ -445,7 +459,7 @@ static int launch_spindle(spank_t spank, spindle_args_t *params) if (!hostlist) goto done; - is_be_leader = isBEProc(params); + is_be_leader = isBEProc(params, 0); if (is_be_leader == -1) goto done; @@ -462,7 +476,14 @@ static int launch_spindle(spank_t spank, spindle_args_t *params) } if (is_fe_host && is_be_leader) { +#if defined(SINFO_BIN) + hostaddrlist = getHostAddrSinfo(num_hosts, hostlist); + if (!hostaddrlist) + goto done; + result = launchFE(hostaddrlist, params); +#else result = launchFE(hostlist, params); +#endif if (result == -1) goto done; } @@ -470,8 +491,14 @@ static int launch_spindle(spank_t spank, spindle_args_t *params) launch_result = 0; done: - if (hostlist) + if (hostlist) { + for (i = 0; i < num_hosts; i++) free(hostlist[i]); free(hostlist); + } + if (hostaddrlist) { + for (i = 0; i < num_hosts; i++) free(hostaddrlist[i]); + free(hostaddrlist); + } return launch_result; } @@ -499,13 +526,12 @@ static int launchFE(char **hostlist, spindle_args_t *params) } if (pidFE) { sdprintf(2, "Forked FE as pid %d\n", pidFE); - registerFEPid(pidFE, params); return 0; } superclose(); - sdprintf(1, "Initializing FE on pid %d with unqiue_id %lu\n", (int) getpid(), params->unique_id); + sdprintf(1, "Initializing FE on pid %d with unique_id %lu\n", (int) getpid(), params->unique_id); result = spindleInitFE((const char **) hostlist, params); if (result == -1) { sdprintf(1, "ERROR: Could not launch FE. Spindle will likely hang.\n"); @@ -556,6 +582,11 @@ static int launchBE(spank_t spank, spindle_args_t *params) sdprintf(1, "ERROR: spindleRunBE failed\n"); else sdprintf(1, "spindleRunBE completed. Session finishing.\n"); + + if (unique_file) unlink(unique_file); + free(unique_file); + unique_file = NULL; + exit(result); return 0; @@ -563,9 +594,13 @@ static int launchBE(spank_t spank, spindle_args_t *params) static int prepApp(spank_t spank, spindle_args_t *params) { +#if HAVE_DECL_SPANK_PREPEND_TASK_ARGV == 1 + int result; +#else int app_argc, result; char **app_argv; char *app_exe_name, *last_slash; +#endif spank_err_t err; int bootstrap_argc; char **bootstrap_argv; @@ -576,7 +611,17 @@ static int prepApp(spank_t spank, spindle_args_t *params) return -1; } - sdprintf(2, "Prepping app process %d to run spindle\n", getpid()); +#if HAVE_DECL_SPANK_PREPEND_TASK_ARGV == 1 + sdprintf(2, "Prepping task process %d to run spindle using spank_prepend_task_argv method\n", getpid()); + + const char **filter_argv = (const char **)bootstrap_argv; + err = spank_prepend_task_argv(spank, bootstrap_argc, filter_argv); + if (err != ESPANK_SUCCESS) { + sdprintf(1, "WARNING: Could not prepend spindle filter.\n"); + result = -1; + } +#else + sdprintf(2, "Prepping app process %d to run spindle using spindleHookSpindleArgsIntoExecBE method\n", getpid()); err = spank_get_item(spank, S_JOB_ARGV, &app_argc, &app_argv); if (err != ESPANK_SUCCESS) { @@ -590,6 +635,7 @@ static int prepApp(spank_t spank, spindle_args_t *params) } result = spindleHookSpindleArgsIntoExecBE(bootstrap_argc, bootstrap_argv, app_exe_name); +#endif if (result == -1) { sdprintf(1, "ERROR setting up app to run spindle. Spindle won't work\n"); return -1; @@ -602,9 +648,9 @@ static int handleExit(void *params, char **output_str) { exit_params_t *exit_params; spank_t spank; - int site_argc, result; + int site_argc, result, is_be_leader; char **site_argv; - spindle_args_t args; + spindle_args_t args = {0}; exit_params = (exit_params_t *) params; spank = exit_params->spank; @@ -617,12 +663,24 @@ static int handleExit(void *params, char **output_str) sdprintf(1, "ERROR: Could not process spindle args in handleExit\n"); return -1; } - - result = spindleExitBE(args.location); - if (result == -1) { - sdprintf(1, "ERROR: spindleExitBE returned and error on location %s\n", args.location); - return -1; + + if (args.opts & OPT_OFF) { + return 0; } + if (!args.commpath) { + sdprintf(2, "WARNING: spindleExitBE not called since commpath is NULL\n"); + } else { + // The task_exit callback is run for _each proc_, so we use + // isBEProc to pick only one proc per node to call spindleExitBE. + is_be_leader = isBEProc(&args, 1); + if (is_be_leader) { + result = spindleExitBE(args.commpath); + if (result == -1) { + sdprintf(1, "ERROR: spindleExitBE returned an error on commpath %s\n", args.commpath); + return -1; + } + } + } return 0; } diff --git a/src/utils/parseloc.c b/src/utils/parseloc.c index 0d8dcc81..3ec20047 100644 --- a/src/utils/parseloc.c +++ b/src/utils/parseloc.c @@ -22,6 +22,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include #include +#include #if !defined(USE_PLUGIN_DEBUG) #include "spindle_debug.h" @@ -34,13 +35,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "ccwarns.h" #include "spindle_launch.h" -#if defined(__cplusplus) -extern "C" { -#endif - char *parse_location(char *loc, number_t number); -#if defined(__cplusplus) -} -#endif +extern int spindle_mkdir(char *orig_path); #if defined(CUSTOM_GETENV) extern char *custom_getenv(); @@ -134,7 +129,9 @@ static char *parse_location_impl(char *loc, number_t number, int print_on_error) i += envvar_len + 1; j += env_value_len; #if defined(CUSTOM_GETENV) && defined(CUSTOM_GETENV_FREE) - free(env_value); + if(env_value != env_value_str) { + free(env_value); + } #endif } else { @@ -166,38 +163,64 @@ char *parse_location_noerr(char *loc, number_t number) **/ char *realize(char *path) { + int local_errno; char *result; - char *origpath, *cur_slash = NULL, *trailing; - struct stat buf; + char *origpath, *cur_slash = NULL, *prev_slash = NULL; + struct stat *buf = calloc( 1, sizeof( struct stat ) ); char newpath[MAX_PATH_LEN+1]; int lastpos; newpath[MAX_PATH_LEN] = '\0'; origpath = strdup(path); - for (;;) { - if (stat(origpath, &buf) != -1) - break; - if (cur_slash) - *cur_slash = '/'; + errno=0; + while( stat( origpath, buf ) == -1 ){ + local_errno = errno; + debug_printf("Failed to stat '%s' (%s).\n", origpath, strerror(local_errno)); + prev_slash = cur_slash; cur_slash = strrchr(origpath, '/'); - if (!cur_slash) - break; - *cur_slash = '\0'; + if( prev_slash ) + *prev_slash = '/'; + if( cur_slash ) + *cur_slash = '\0'; + else{ + debug_printf("Nothing in the original path can be stat'ed. (%s)\n", path); + return NULL; + } } - if (cur_slash) - trailing = cur_slash + 1; - else - trailing = ""; + errno = 0; result = realpath(origpath, newpath); if (!result) { + local_errno = errno; + err_printf( + "Error: realpath(3) failed to create canonical version of '%s' (%s). Returning '%s'.\n", + origpath, strerror(local_errno), path ); + errno = 0; + int rc = stat( origpath, buf ); + local_errno = errno; + err_printf( + " Statting that path results in rc=%d, errno=%d, error='%s'.\n", + rc, local_errno, strerror(local_errno)); free(origpath); - return path; + return NULL; } + free(buf); - strncat(newpath, "/", MAX_PATH_LEN); - strncat(newpath, trailing, MAX_PATH_LEN); - newpath[MAX_PATH_LEN] = '\0'; + if( cur_slash ){ + if( strlen( newpath ) + strlen( cur_slash+1 ) > MAX_PATH_LEN ){ + err_printf( + "Error: The realized path exceeds MAX_PATH_LEN (%d).\n" + " Original path: '%s'\n" + " Statable part: '%s'\n" + " Canonical version: '%s'\n" + " Returning original path.\n", + MAX_PATH_LEN, path, origpath, newpath); + free(origpath); + return path; + } + strncat(newpath, "/", 2); + strncat(newpath, cur_slash+1, MAX_PATH_LEN - strlen( newpath )); + } free(origpath); lastpos = strlen(newpath)-1; @@ -278,3 +301,77 @@ int is_local_prefix(const char *path, char **local_prefixes) { return 0; } +/* validateCandidatePath determines if candidatePath passes parse_location(), realize(), and spindle_mkdir(), which is to say, can + * spindle create a directory from this path? + * + * If not NULL, then realizedPath, parsedPath, and/or symbolicPath will hold the respective intermediate/final results. + * + * Return 1 if the candidatePath is valid, otherwise 0. + */ +static int validateCandidatePath( char *candidatePath, char **realizedPath, char **parsedPath, char **symbolicPath, number_t number ){ + int rc; + char *parsedCandidatePath, *realizedCandidatePath; + parsedCandidatePath = parse_location( candidatePath, number ); + if( parsedCandidatePath ){ + realizedCandidatePath = realize( parsedCandidatePath ); + if( realizedCandidatePath ){ + rc = spindle_mkdir( parsedCandidatePath ); + if( 0 == rc ){ + if( symbolicPath) *symbolicPath = candidatePath; + if( parsedPath ) *parsedPath = parsedCandidatePath; + if( realizedPath) *realizedPath = realizedCandidatePath; + return 1; + }else{ + debug_printf2("Unable to create directory %s, moving on to the next candidate.\n", realizedCandidatePath ); + } + }else{ + debug_printf2( "Unable to realize candidate %s, moving on to the next candidate.\n", parsedCandidatePath ); + } + }else{ + debug_printf2("Unable to parse candidate %s, moving on to the next candidate.\n", candidatePath ); + } + return 0; +} + +/** + * determineValidCachePaths() works exclusively with the cachepaths parameter. Because not all paths may be valid on all + * compute nodes, and because we want to have all nodes reach a consensus on which cache path to use, we + * determine the validity of all paths in the origPathList, save the intermediate results, and return a bit + * index to the user. Via allReduce() all nodes reach a consensus on the set of valid paths, and retrieves + * that informatino via getValidPathByIndex(). + */ +static char *realizedCachePaths[64], *parsedCachePaths[64], *symbolicCachePaths[64]; + +void determineValidCachePaths( uint64_t *validBitIdx, char *origPathList, number_t number ){ + + char *saveptr, *candidatePath, *pathList = strdup( origPathList ); + uint64_t bitoffset = 0; + + *validBitIdx = 0; + debug_printf2("origPathList='%s', number='%lu'.\n", origPathList, number ); + + candidatePath = strtok_r( pathList, ":", &saveptr ); + while( NULL != candidatePath && bitoffset < 64 ){ + *validBitIdx |= validateCandidatePath( + candidatePath, + &realizedCachePaths[bitoffset], + &parsedCachePaths[bitoffset], + &symbolicCachePaths[bitoffset], number ) << bitoffset; + bitoffset++; + candidatePath = strtok_r( NULL, ":", &saveptr ); + } + free( pathList ); +} + +void getValidCachePathByIndex( uint64_t validBitIdx, char **realizedCachePath, char **parsedCachePath, char **symbolicCachePath ){ + uint64_t bitoffset = 0; + if (!validBitIdx){ + return; + } + while( (bitoffset < 64) && (((1 << bitoffset) & validBitIdx) == 0) ){ + bitoffset++; + } + if( realizedCachePath ) *realizedCachePath = realizedCachePaths[bitoffset]; + if( parsedCachePath ) *parsedCachePath = parsedCachePaths[bitoffset]; + if( symbolicCachePath ) *symbolicCachePath = symbolicCachePaths[bitoffset]; +} diff --git a/src/client/beboot/parseloc.h b/src/utils/parseloc.h similarity index 65% rename from src/client/beboot/parseloc.h rename to src/utils/parseloc.h index c5362e2e..a99409c3 100644 --- a/src/client/beboot/parseloc.h +++ b/src/utils/parseloc.h @@ -24,6 +24,13 @@ extern "C" { #include "spindle_launch.h" char *parse_location(char *loc, number_t number); +char *parse_location_noerr(char *loc, number_t number); +char *realize(char *path); +char **parse_colonsep_prefixes(char *colonsep_list, number_t number); +int is_local_prefix(const char *path, char **local_prefixes); +int validateCandidatePath( char *candidatePath, char **realizedPath, char **parsedPath, char **symbolicPath, number_t number ); +void determineValidCachePaths( uint64_t *validBitIdx, char *origPathList, number_t number ); +void getValidCachePathByIndex( uint64_t validBitIdx, char **realizedCachePath, char **parsedCachePath, char **symbolicCachePath ); #if defined(__cplusplus) } diff --git a/testsuite/Makefile.in b/testsuite/Makefile.in index 9c225322..511a6aca 100644 --- a/testsuite/Makefile.in +++ b/testsuite/Makefile.in @@ -273,6 +273,7 @@ SCONTROL_ABSPATH = @SCONTROL_ABSPATH@ SED = @SED@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ +SINFO_ABSPATH = @SINFO_ABSPATH@ SRUN_PATH = @SRUN_PATH@ STATICFLAG = @STATICFLAG@ STRIP = @STRIP@ diff --git a/testsuite/run_driver_slurm-plugin b/testsuite/run_driver_slurm-plugin new file mode 100644 index 00000000..6d06cbbf --- /dev/null +++ b/testsuite/run_driver_slurm-plugin @@ -0,0 +1,20 @@ +#!/bin/bash + +if [ x$SLURM_CPUS_ON_NODE = x ] ; then +export SLURM_CPUS_ON_NODE=1 +fi + +if [ x$SPINDLE_TEST_ARGS = x ] ; then +((PROCS=$SLURM_NNODES*$SLURM_CPUS_ON_NODE)) +else +PROCS=$SPINDLE_TEST_ARGS +fi +export PROCS + +if [ "x$SPINDLE_BGQ_LD_PRELOAD" = "xtrue" ] ; then +PRELOAD_ARGS="--runjob-opts=--envs LD_PRELOAD=$LIBRARY_LIST" +elif [ "x$SPINDLE_LD_PRELOAD" != "x" ] ; then +PRELOAD_ARGS="--export=ALL,LD_PRELOAD=$SPINDLE_LD_PRELOAD" +fi + +exec srun $PRELOAD_ARGS --spindle="$SPINDLE_FLAGS $SPINDLE_OPTS" -n $PROCS $* diff --git a/testsuite/test_driver.c b/testsuite/test_driver.c index 13f63fea..aae6c5c8 100644 --- a/testsuite/test_driver.c +++ b/testsuite/test_driver.c @@ -1169,7 +1169,7 @@ static char* getCacheLocation(char *env_var) return strdup(last_slash); } -static int checkLinkForLeak(const char *path, const char *spindle_loc) +static int checkLinkForLeak(const char *path, const char *cachepath) { char link_target[4096]; int result, error; @@ -1182,18 +1182,19 @@ static int checkLinkForLeak(const char *path, const char *spindle_loc) return -1; } - if (strstr(link_target, spindle_loc)) { - err_printf("Link at '%s' has path '%s', which leaks spindle path with '%s'\n", path, link_target, spindle_loc); + if (strstr(link_target, cachepath)) { + err_printf("Link at '%s' has path '%s', which leaks spindle path with '%s'\n", path, link_target, cachepath); return -1; } return 0; } -static int checkPathForLeak(const char *what, const char *path, const char *spindle_loc) + +static int checkPathForLeak(const char *what, const char *path, const char *cachepath) { - if (strstr(path, spindle_loc)) { - err_printf("%s: Path '%s' leaks spindle path with '%s'\n", what, path, spindle_loc); + if (strstr(path, cachepath)) { + err_printf("%s: Path '%s' leaks spindle path with '%s'\n", what, path, cachepath); return -1; } return 0; @@ -1201,14 +1202,14 @@ static int checkPathForLeak(const char *what, const char *path, const char *spin static int leak_check_cb(struct dl_phdr_info *p, size_t psize, void *opaque) { - char *spindle_loc = (char *) opaque; + char *cachepath = (char *) opaque; if (!p->dlpi_name || p->dlpi_name[0] == '\0') return 0; - checkPathForLeak("dl_iterate_phdr", p->dlpi_name, spindle_loc); + checkPathForLeak("dl_iterate_phdr", p->dlpi_name, cachepath); return 0; } -static int check_proc_maps(char *path, char *spindle_loc) +static int check_proc_maps(char *path, char *cachepath) { int fd, error, result; struct stat statbuf; @@ -1247,8 +1248,8 @@ static int check_proc_maps(char *path, char *spindle_loc) maps[filesize] = '\0'; close(fd); - if (strstr(maps, spindle_loc)) { - err_printf("Found leaked spindle path '%s' in maps '%s'\n", spindle_loc, path); + if (strstr(maps, cachepath)) { + err_printf("Found leaked spindle path '%s' in maps '%s'\n", cachepath, path); return -1; } @@ -1258,17 +1259,15 @@ static int check_proc_maps(char *path, char *spindle_loc) void check_for_path_leaks() { - char *spindle_loc = NULL; + char *cachepath = NULL; DIR *proc_fds = NULL; struct dirent *d; char path[4096]; struct link_map *lm; char *dlerr_msg = NULL; - spindle_loc = getCacheLocation("LDCS_LOCATION"); - if (!spindle_loc) - spindle_loc = getCacheLocation("LDCS_ORIG_LOCATION"); - if (!spindle_loc) { + cachepath = getCacheLocation("LDCS_CHOSEN_PARSED_CACHEPATH"); + if (!cachepath) { err_printf("Failed to calculate cache location"); goto done; } @@ -1284,11 +1283,16 @@ void check_for_path_leaks() for (d = readdir(proc_fds); d != NULL; d = readdir(proc_fds)) { if (d->d_name[0] == '.') continue; + // Ignore Spindle fifo files for now. + if ( strncmp( "315", d->d_name, 3 ) == 0 ) + continue; + if ( strncmp( "316", d->d_name, 3 ) == 0 ) + continue; strncpy(path, "/proc/self/fd/", sizeof(path)); strncat(path, d->d_name, sizeof(path)-1); - checkLinkForLeak(path, spindle_loc); + checkLinkForLeak(path, cachepath); } - checkLinkForLeak("/proc/self/exe", spindle_loc); + checkLinkForLeak("/proc/self/exe", cachepath); /** * Check link_maps for leaked spindle paths @@ -1296,22 +1300,22 @@ void check_for_path_leaks() for (lm = _r_debug.r_map; lm != NULL; lm = lm->l_next) { if (!lm->l_name || lm->l_name[0] == '\0') continue; - checkPathForLeak("link_map", lm->l_name, spindle_loc); + checkPathForLeak("link_map", lm->l_name, cachepath); } /** * Check libraries in dl_iterate_phdr for leaked paths **/ - dl_iterate_phdr(leak_check_cb, spindle_loc); + dl_iterate_phdr(leak_check_cb, cachepath); /** * Check /proc/pid/maps under various aliases for leaked names **/ - check_proc_maps("/proc/self/maps", spindle_loc); + check_proc_maps("/proc/self/maps", cachepath); snprintf(path, sizeof(path), "/proc/self/task/%d/maps", getpid()); - check_proc_maps(path, spindle_loc); + check_proc_maps(path, cachepath); snprintf(path, sizeof(path), "/proc/%d/maps", getpid()); - check_proc_maps(path, spindle_loc); + check_proc_maps(path, cachepath); /** * Check that dlerror doesn't leak the /__not_exists/ prefix @@ -1323,8 +1327,8 @@ void check_for_path_leaks() } done: - if (spindle_loc) - free(spindle_loc); + if (cachepath) + free(cachepath); if (proc_fds) closedir(proc_fds); }