From c98f70f6f0703cacb38bfd5ebbdd98ca14405762 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:50:08 -0700
Subject: [PATCH 01/57] Cachepath:  parse_loc utilities.

---
 src/client/beboot/parseloc.h |   7 ++
 src/utils/parseloc.c         | 145 +++++++++++++++++++++++++++++------
 2 files changed, 127 insertions(+), 25 deletions(-)

diff --git a/src/client/beboot/parseloc.h b/src/client/beboot/parseloc.h
index c5362e2e..1731906a 100644
--- a/src/client/beboot/parseloc.h
+++ b/src/client/beboot/parseloc.h
@@ -24,6 +24,13 @@ extern "C" {
 #include "spindle_launch.h"
    
 char *parse_location(char *loc, number_t number);
+char *parse_location_noerr(char *loc, number_t number);
+char *realize(char *path);
+char **parse_colonsep_prefixes(char *colonsep_list, number_t number);
+int is_local_prefix(const char *path, char **local_prefixes);
+static int validateCandidatePath( char *candidatePath, char **realizedPath, char **parsedPath, char **symbolicPath, number_t number );
+void determineValidCachePaths( uint64_t *validBitIdx, char *origPathList, number_t number );
+void getValidCachePathByIndex( uint64_t validBitIdx, char **realizedCachePath, char **parsedCachePath, char **symbolicCachePath );
 
 #if defined(__cplusplus)
 }
diff --git a/src/utils/parseloc.c b/src/utils/parseloc.c
index 0d8dcc81..28c446dd 100644
--- a/src/utils/parseloc.c
+++ b/src/utils/parseloc.c
@@ -22,6 +22,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <assert.h>
+#include <errno.h>
 
 #if !defined(USE_PLUGIN_DEBUG)
 #include "spindle_debug.h"
@@ -34,13 +35,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 #include "ccwarns.h"
 #include "spindle_launch.h"
 
-#if defined(__cplusplus)
-extern "C" {
-#endif
-   char *parse_location(char *loc, number_t number);
-#if defined(__cplusplus)
-}
-#endif
+extern int spindle_mkdir(char *orig_path);
 
 #if defined(CUSTOM_GETENV)
 extern char *custom_getenv();
@@ -166,38 +161,64 @@ char *parse_location_noerr(char *loc, number_t number)
  **/
 char *realize(char *path)
 {
+   int local_errno;
    char *result;
-   char *origpath, *cur_slash = NULL, *trailing;
-   struct stat buf;
+   char *origpath, *cur_slash = NULL, *prev_slash = NULL;
+   struct stat *buf = calloc( 1, sizeof( struct stat ) );
    char newpath[MAX_PATH_LEN+1];
    int lastpos;
    newpath[MAX_PATH_LEN] = '\0';
 
    origpath = strdup(path);
-   for (;;) {
-      if (stat(origpath, &buf) != -1)
-         break;
-      if (cur_slash)
-         *cur_slash = '/';
+   errno=0;
+   while( stat( origpath, buf ) == -1 ){
+      local_errno = errno;
+      debug_printf("Failed to stat '%s' (%s).\n", origpath, strerror(local_errno));
+      prev_slash = cur_slash;
       cur_slash = strrchr(origpath, '/');
-      if (!cur_slash)
-         break;
-      *cur_slash = '\0';
+      if( prev_slash )
+          *prev_slash = '/';
+      if( cur_slash )
+          *cur_slash = '\0';
+      else{
+          debug_printf("Nothing in the original path can be stat'ed.  (%s)\n", path);
+          return NULL;
+      }
    }
-   if (cur_slash)
-      trailing = cur_slash + 1;
-   else
-      trailing = "";
 
+   errno = 0;
    result = realpath(origpath, newpath);
    if (!result) {
+      local_errno = errno;
+      err_printf(
+          "Error:  realpath(3) failed to create canonical version of '%s' (%s).  Returning '%s'.\n",
+          origpath, strerror(local_errno), path );
+      errno = 0;
+      int rc = stat( origpath, buf );
+      local_errno = errno;
+      err_printf(
+          "        Statting that path results in rc=%d, errno=%d, error='%s'.\n",
+          rc, local_errno, strerror(local_errno));
       free(origpath);
-      return path;
+      return NULL;
    }
+   free(buf);
 
-   strncat(newpath, "/", MAX_PATH_LEN);
-   strncat(newpath, trailing, MAX_PATH_LEN);
-   newpath[MAX_PATH_LEN] = '\0';
+   if( cur_slash ){
+       if( strlen( newpath ) + strlen( cur_slash+1 ) > MAX_PATH_LEN ){
+            err_printf(
+                    "Error:  The realized path exceeds MAX_PATH_LEN (%d).\n"
+                    "  Original path:     '%s'\n"
+                    "  Statable part:     '%s'\n"
+                    "  Canonical version: '%s'\n"
+                    "  Returning original path.\n",
+                    MAX_PATH_LEN, path, origpath, newpath);
+            free(origpath);
+            return path;
+       }
+       strncat(newpath, "/",         2);
+       strncat(newpath, cur_slash+1, MAX_PATH_LEN - strlen( newpath ));
+   }
    free(origpath);
 
    lastpos = strlen(newpath)-1;
@@ -278,3 +299,77 @@ int is_local_prefix(const char *path, char **local_prefixes) {
    return 0;
 }
 
+/* validateCandidatePath determines if candidatePath passes parse_location(), realize(), and spindle_mkdir(), which is to say, can
+ * spindle create a directory from this path?
+ *
+ * If not NULL, then realizedPath, parsedPath, and/or symbolicPath will hold the respective intermediate/final results.
+ *
+ * Return 1 if the candidatePath is valid, otherwise 0.
+ */
+static int validateCandidatePath( char *candidatePath, char **realizedPath, char **parsedPath, char **symbolicPath, number_t number ){
+    int rc;
+    char *parsedCandidatePath, *realizedCandidatePath;
+    parsedCandidatePath = parse_location( candidatePath, number );
+    if( parsedCandidatePath ){
+       realizedCandidatePath = realize( parsedCandidatePath );
+       if( realizedCandidatePath ){
+           rc = spindle_mkdir( parsedCandidatePath );
+           if( 0 == rc ){
+               if( symbolicPath) *symbolicPath = candidatePath;
+               if( parsedPath  ) *parsedPath   = parsedCandidatePath;
+               if( realizedPath) *realizedPath = realizedCandidatePath;
+               return 1;
+           }else{
+               debug_printf2("Unable to create directory %s, moving on to the next candidate.\n", realizedCandidatePath );
+           }
+        }else{
+            debug_printf2( "Unable to realize candidate %s, moving on to the next candidate.\n", parsedCandidatePath );
+        }
+    }else{
+        debug_printf2("Unable to parse candidate %s, moving on to the next candidate.\n", candidatePath );
+    }
+    return 0;
+}
+
+/**
+ * determineValidCachePaths()  works exclusively with the cachepaths parameter.  Because not all paths may be valid on all
+ * compute nodes, and because we want to have all nodes reach a consensus on which cache path to use, we
+ * determine the validity of all paths in the origPathList, save the intermediate results, and return a bit
+ * index to the user.  Via allReduce() all nodes reach a consensus on the set of valid paths, and retrieves
+ * that informatino via getValidPathByIndex().
+ */
+static char *realizedCachePaths[64], *parsedCachePaths[64], *symbolicCachePaths[64];
+
+void determineValidCachePaths( uint64_t *validBitIdx, char *origPathList, number_t number ){
+
+    char *saveptr, *candidatePath, *pathList = strdup( origPathList );
+    uint64_t bitoffset = 0;
+
+    *validBitIdx = 0;
+    debug_printf2("origPathList='%s', number='%lu'.\n", origPathList, number );
+
+    candidatePath = strtok_r( pathList, ":", &saveptr );
+    while( NULL != candidatePath && bitoffset < 64 ){
+        *validBitIdx |= validateCandidatePath(
+                            candidatePath,
+                            &realizedCachePaths[bitoffset],
+                            &parsedCachePaths[bitoffset],
+                            &symbolicCachePaths[bitoffset], number ) << bitoffset;
+        bitoffset++;
+        candidatePath = strtok_r( NULL, ":", &saveptr );
+    }
+    free( pathList );
+}
+
+void getValidCachePathByIndex( uint64_t validBitIdx, char **realizedCachePath, char **parsedCachePath, char **symbolicCachePath ){
+    uint64_t bitoffset = 0;
+    if (!validBitIdx){
+        return;
+    }
+    while( (bitoffset < 64) && (((1 << bitoffset) & validBitIdx) == 0) ){
+        bitoffset++;
+    }
+    if( realizedCachePath ) *realizedCachePath = realizedCachePaths[bitoffset];
+    if( parsedCachePath   ) *parsedCachePath   = parsedCachePaths[bitoffset];
+    if( symbolicCachePath ) *symbolicCachePath = symbolicCachePaths[bitoffset];
+}

From 77fd7d3cca429c5d39c7b0879dce426d79fe0379 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:50:45 -0700
Subject: [PATCH 02/57] Cachepath:  remove/rename [orig_]location.

---
 src/client/beboot/spindle_bootstrap.c  |  2 +-
 src/client/client/client.c             | 15 +++++++--------
 src/client/client/intercept.h          |  1 +
 src/client/client/intercept_exec.c     |  5 +----
 src/client/client/intercept_readlink.c | 16 +++++++++++-----
 src/client/client/should_intercept.c   | 25 +++++++++++++++----------
 src/client/client/should_intercept.h   |  1 +
 7 files changed, 37 insertions(+), 28 deletions(-)

diff --git a/src/client/beboot/spindle_bootstrap.c b/src/client/beboot/spindle_bootstrap.c
index 7805a9db..98424d50 100644
--- a/src/client/beboot/spindle_bootstrap.c
+++ b/src/client/beboot/spindle_bootstrap.c
@@ -114,7 +114,6 @@ static void setup_environment()
 
    setenv("LD_AUDIT", client_lib, 1);
    setenv("LDCS_LOCATION", location, 1);
-   setenv("LDCS_ORIG_LOCATION", orig_location, 1);
    setenv("LDCS_NUMBER", number_s, 1);
    setenv("LDCS_RANKINFO", rankinfo_str, 1);
    if (connection_str)
@@ -161,6 +160,7 @@ static int parse_cmdline(int argc, char *argv[])
    }
 
    symbolic_location = argv[i++];
+   i++; // Skip over candidate_cachepaths.
    number_s = argv[i++];
    number = (number_t) strtoul(number_s, NULL, 0);
    opts_s = argv[i++];
diff --git a/src/client/client/client.c b/src/client/client/client.c
index a50bb228..d7827d4b 100644
--- a/src/client/client/client.c
+++ b/src/client/client/client.c
@@ -41,6 +41,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 #include "exec_util.h"
 #include "intercept.h"
 #include "fixlocale.h"
+#include "should_intercept.h"
 
 errno_location_t app_errno_location;
 
@@ -70,11 +71,8 @@ static const ElfW(Phdr) *libc_phdrs, *interp_phdrs;
 static int num_libc_phdrs, num_interp_phdrs;
 ElfW(Addr) libc_loadoffset, interp_loadoffset;
 
-/* location has the realize'd path to the local file cache. orig_location is not realized and
- * may contain symlinks
- */
-char *location;
-char *orig_location;
+static char *location;
+static char *chosen_realized_cachepath, *chosen_parsed_cachepath, *chosen_symbolic_cachepath;
 number_t number;
 static int have_stat_patches;
 
@@ -201,7 +199,6 @@ static int init_server_connection()
       return 0;
 
    location = getenv("LDCS_LOCATION");
-   orig_location = getenv("LDCS_ORIG_LOCATION");
    number = (number_t) strtoul(getenv("LDCS_NUMBER"), NULL, 0);
    connection = getenv("LDCS_CONNECTION");
    rankinfo_s = getenv("LDCS_RANKINFO");
@@ -221,7 +218,6 @@ static int init_server_connection()
       debug_printf("Disabling environment variables because we're not following forks\n");
       unsetenv("LD_AUDIT");
       unsetenv("LDCS_LOCATION");
-      unsetenv("LDCS_ORIG_LOCATION");
       unsetenv("LDCS_NUMBER");
       unsetenv("LDCS_CONNECTION");
       unsetenv("LDCS_RANKINFO");
@@ -265,6 +261,9 @@ static int init_server_connection()
          send_cpu(ldcsid, get_cur_cpu());
 #endif
    }
+   send_cachepath_query( ldcsid, &chosen_realized_cachepath, &chosen_parsed_cachepath, &chosen_symbolic_cachepath );
+   set_should_intercept_cachepath(   chosen_realized_cachepath, chosen_parsed_cachepath, chosen_symbolic_cachepath );
+   set_intercept_readlink_cachepath( chosen_realized_cachepath, chosen_parsed_cachepath, chosen_symbolic_cachepath );
    
    snprintf(debugging_name, 32, "Client.%d", rankinfo[0]);
    LOGGING_INIT(debugging_name);
@@ -472,7 +471,7 @@ char *client_library_load(const char *name)
 
    char *orig_file_name = (char *) name;
    if (is_in_spindle_cache(name)) {
-      debug_printf2("Library %s is in spindle cache (%s). Translating request\n", name, location);
+      debug_printf2("Library %s is in spindle cache (%s). Translating request\n", name, chosen_realized_cachepath);
       memset(fixed_name, 0, MAX_PATH_LEN+1);
       send_orig_path_request(ldcsid, orig_file_name, fixed_name);
       orig_file_name = fixed_name;
diff --git a/src/client/client/intercept.h b/src/client/client/intercept.h
index 4ace2328..aae968f7 100644
--- a/src/client/client/intercept.h
+++ b/src/client/client/intercept.h
@@ -89,6 +89,7 @@ int execvpe_wrapper(const char *path, char *const argv[], const char *envp[]);
 pid_t vfork_wrapper();
 char *dlerror_wrapper();
 
+void set_intercept_readlink_cachepath( char *chosen_realized_cachepath, char *chosen_parsed_cachepath, char *chosen_symbolic_cachepath );
 ssize_t readlink_wrapper(const char *path, char *buf, size_t bufsiz);
 ssize_t readlinkat_wrapper(int dirfd, const char *pathname, char *buf, size_t bufsiz);
 
diff --git a/src/client/client/intercept_exec.c b/src/client/client/intercept_exec.c
index 14b555ed..8480b75e 100644
--- a/src/client/client/intercept_exec.c
+++ b/src/client/client/intercept_exec.c
@@ -142,7 +142,6 @@ static char **removeEnvironmentStrs(char **envp)
       if (strIsPrefix("LD", envp[i])) {
          if (strIsPrefix("LD_AUDIT=", envp[i]) ||
              strIsPrefix("LDCS_LOCATION=", envp[i]) ||
-             strIsPrefix("LDCS_ORIG_LOCATION=", envp[i]) ||
              strIsPrefix("LDCS_CONNECTION=", envp[i]) ||
              strIsPrefix("LDCS_RANKINFO=", envp[i]) ||
              strIsPrefix("LDCS_OPTIONS=", envp[i]) ||
@@ -177,7 +176,6 @@ static char **updateEnvironment(char **envp, int *num_modified, int propogate_sp
          unsetf("SPINDLE");
          unsetf("LD_AUDIT");
          unsetf("LDCS_LOCATION");
-         unsetf("LDCS_ORIG_LOCATION");
          unsetf("LDCS_CONNECTION");
          unsetf("LDCS_RANKINFO");
          unsetf("LDCS_OPTIONS");
@@ -198,13 +196,12 @@ static char **updateEnvironment(char **envp, int *num_modified, int propogate_sp
    if (envp) {
       debug_printf2("Propogating spindle environment by copying it to new envp list\n");
       for (cur = (char **) envp; *cur; cur++, orig_size++);
-      new_size = orig_size + 10;
+      new_size = orig_size + 20;
       newenv = (char **) malloc(new_size * sizeof(char*));
       
       propogateEnvironmentStr(envp, newenv, &pos, "SPINDLE");   
       propogateEnvironmentStr(envp, newenv, &pos, "LD_AUDIT");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_LOCATION");
-      propogateEnvironmentStr(envp, newenv, &pos, "LDCS_ORIG_LOCATION");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_CONNECTION");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_RANKINFO");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_OPTIONS");
diff --git a/src/client/client/intercept_readlink.c b/src/client/client/intercept_readlink.c
index 0a326330..2799baee 100644
--- a/src/client/client/intercept_readlink.c
+++ b/src/client/client/intercept_readlink.c
@@ -31,19 +31,25 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 ssize_t (*orig_readlink)(const char *path, char *buf, size_t bufsiz);
 ssize_t (*orig_readlinkat)(int dirfd, const char *pathname, char *buf, size_t bufsiz);
 
-extern char *location;
+static char *cachepath;
+
+void set_intercept_readlink_cachepath( char *chosen_realized_cachepath, char *chosen_parsed_cachepath, char *chosen_symbolic_cachepath ){
+    cachepath = chosen_realized_cachepath;
+    chosen_parsed_cachepath = chosen_parsed_cachepath;
+    chosen_symbolic_cachepath = chosen_symbolic_cachepath;
+}
 
 static int fix_local_readlink(char *buf, size_t bufsiz)
 {
    char spindle_id[32];
-   int location_len, result;
+   int cachepath_len, result;
    char tmp[MAX_PATH_LEN+1];
 
-   location_len = strlen(location);   
+   cachepath_len = strlen(cachepath);   
    snprintf(spindle_id, sizeof(spindle_id), "spindle.%lx", number);
-   if (strstr(buf, spindle_id) && strncmp(location, buf, location_len) == 0) {
+   if (strstr(buf, spindle_id) && strncmp(cachepath, buf, cachepath_len) == 0) {
       debug_printf2("readlink received spindle cache path %s. Translating\n", buf);
-      result = send_orig_path_request(ldcsid, buf+location_len+1, tmp);
+      result = send_orig_path_request(ldcsid, buf+cachepath_len+1, tmp);
       if (result == -1)
          return -1;
       debug_printf2("readlink translated spindle local path %s to %s\n", buf, tmp);
diff --git a/src/client/client/should_intercept.c b/src/client/client/should_intercept.c
index bfabbbf8..3a348d3d 100644
--- a/src/client/client/should_intercept.c
+++ b/src/client/client/should_intercept.c
@@ -29,22 +29,27 @@
 #include "spindle_debug.h"
 
 extern int relocate_spindleapi();
+static char *cachepath, *orig_cachepath;
+
+void set_should_intercept_cachepath( char *chosen_realized_cachepath, char *chosen_parsed_cachepath, char *chosen_symbolic_cachepath ){
+    cachepath = chosen_realized_cachepath;
+    orig_cachepath = chosen_parsed_cachepath;
+    chosen_symbolic_cachepath = chosen_symbolic_cachepath;
+}
 
-extern char *location;
-extern char *orig_location;
 
 int is_in_spindle_cache(const char *pathname)
 {
-   static int location_size = 0;
-   static int orig_location_size = 0;
-   if (!location_size) {
-      location_size = strlen(location);
+   static int cachepath_size = 0;
+   static int orig_cachepath_size = 0;
+   if (!cachepath_size) {
+      cachepath_size = strlen(cachepath);
    }
-   if (!orig_location_size) {
-      orig_location_size = strlen(orig_location);
+   if (!orig_cachepath_size) {
+      orig_cachepath_size = strlen(orig_cachepath);
    }
-   return ((strncmp(pathname, location, location_size) == 0) ||
-           (strncmp(pathname, orig_location, orig_location_size) == 0));
+   return ((strncmp(pathname, cachepath, cachepath_size) == 0) ||
+           (strncmp(pathname, orig_cachepath, orig_cachepath_size) == 0));
 }
 
 extern int is_local_prefix(const char *path, char **cached_local_prefixes);
diff --git a/src/client/client/should_intercept.h b/src/client/client/should_intercept.h
index f6a9b510..6a545913 100644
--- a/src/client/client/should_intercept.h
+++ b/src/client/client/should_intercept.h
@@ -27,6 +27,7 @@
 #define EXCL_OPEN 2
 #define ERR_CALL 3
 
+void set_should_intercept_cachepath( char *chosen_realized_cachepath, char *chosen_parsed_cachepath, char *chosen_symbolic_cachepath );
 int open_filter(const char *fname, int flags);
 int fopen_filter(const char *fname, const char *flags);
 int exec_filter(const char *fname);

From c3dba387739641d8f119cc25241c6283729cdb2d Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:51:51 -0700
Subject: [PATCH 03/57] Cachepath:  Configure-time support.

---
 config.h.in            |  3 +++
 configure              | 16 ++++++++++++++++
 configure.common.ac    |  5 +++++
 src/client/config.h.in |  3 +++
 src/client/configure   | 16 ++++++++++++++++
 src/fe/config.h.in     |  3 +++
 src/fe/configure       | 16 ++++++++++++++++
 src/server/config.h.in |  3 +++
 src/server/configure   | 16 ++++++++++++++++
 9 files changed, 81 insertions(+)

diff --git a/config.h.in b/config.h.in
index a346ae96..6be0335d 100644
--- a/config.h.in
+++ b/config.h.in
@@ -6,6 +6,9 @@
 /* Whether we are using a broken srun */
 #undef BROKEN_SRUN
 
+/* Colon-separated list of potential back-end cache directories */
+#undef CACHEPATHS
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 
diff --git a/configure b/configure
index aaa2f552..c9e517c8 100755
--- a/configure
+++ b/configure
@@ -846,6 +846,7 @@ enable_libtool_lock
 enable_maintainer_mode
 with_default_port
 with_default_num_ports
+with_cachepaths
 with_localstorage
 with_default_local_prefix
 with_testrm
@@ -1589,6 +1590,8 @@ Optional Packages:
   --with-default-numports=NUM
                           Number of TCP/IP ports to scan for Spindle server
                           communication
+  --with-cachepaths=DIR   Colon-separated list of potential back-end cache
+                          directories
   --with-localstorage=DIR Directory on back-ends for storing relocated files
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
@@ -16615,6 +16618,14 @@ else
 fi
 
 
+# Check whether --with-cachepaths was given.
+if test "${with_cachepaths+set}" = set; then :
+  withval=$with_cachepaths; CACHEPATHS=${withval}
+else
+  CACHEPATHS=$DEFAULT_LOC
+fi
+
+
 # Check whether --with-localstorage was given.
 if test "${with_localstorage+set}" = set; then :
   withval=$with_localstorage; SPINDLE_LOC=${withval}
@@ -16651,6 +16662,11 @@ cat >>confdefs.h <<_ACEOF
 _ACEOF
 
 
+cat >>confdefs.h <<_ACEOF
+#define CACHEPATHS "$CACHEPATHS"
+_ACEOF
+
+
 cat >>confdefs.h <<_ACEOF
 #define SPINDLE_LOCAL_PREFIX "$SPINDLE_LOCAL_PREFIX"
 _ACEOF
diff --git a/configure.common.ac b/configure.common.ac
index 780aef00..31a04d4d 100644
--- a/configure.common.ac
+++ b/configure.common.ac
@@ -17,6 +17,10 @@ AC_ARG_WITH(default-num-ports,
             [AS_HELP_STRING([--with-default-numports=NUM],[Number of TCP/IP ports to scan for Spindle server communication])],
             [NUM_COBO_PORTS=${withval}],
             [NUM_COBO_PORTS=$DEFAULT_NUM_COBO_PORTS])
+AC_ARG_WITH(cachepaths,
+            [AS_HELP_STRING([--with-cachepaths=DIR],[Colon-separated list of potential back-end cache directories])],
+            [CACHEPATHS=${withval}],
+            [CACHEPATHS=$DEFAULT_LOC])
 AC_ARG_WITH(localstorage,
             [AS_HELP_STRING([--with-localstorage=DIR],[Directory on back-ends for storing relocated files])],
             [SPINDLE_LOC=${withval}],
@@ -29,6 +33,7 @@ AC_DEFINE_UNQUOTED([SPINDLE_PORT],[$SPINDLE_PORT],[The default port for Spindle]
 AC_DEFINE_UNQUOTED([NUM_COBO_PORTS],[$NUM_COBO_PORTS],[Number of ports for COBO to search for an open port])
 AC_DEFINE_UNQUOTED([SPINDLE_MAX_PORT],[$(($SPINDLE_PORT + $NUM_COBO_PORTS - 1))],[The maximum port value])
 AC_DEFINE_UNQUOTED([SPINDLE_LOC],"[$SPINDLE_LOC]",[The default local directory for Spindle])
+AC_DEFINE_UNQUOTED([CACHEPATHS],"[$CACHEPATHS]",[Colon-separated list of potential back-end cache directories])
 AC_DEFINE_UNQUOTED([SPINDLE_LOCAL_PREFIX],"[$SPINDLE_LOCAL_PREFIX]",[The default colon-separated list of directories that Spindle will not cache files out of])
 
 TESTRM=unknown
diff --git a/src/client/config.h.in b/src/client/config.h.in
index 6c5eae90..a42cc0ab 100644
--- a/src/client/config.h.in
+++ b/src/client/config.h.in
@@ -6,6 +6,9 @@
 /* Whether we are using a broken srun */
 #undef BROKEN_SRUN
 
+/* Colon-separated list of potential back-end cache directories */
+#undef CACHEPATHS
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 
diff --git a/src/client/configure b/src/client/configure
index dbd69e1f..cbae8891 100755
--- a/src/client/configure
+++ b/src/client/configure
@@ -810,6 +810,7 @@ enable_libtool_lock
 enable_maintainer_mode
 with_default_port
 with_default_num_ports
+with_cachepaths
 with_localstorage
 with_default_local_prefix
 with_testrm
@@ -1532,6 +1533,8 @@ Optional Packages:
   --with-default-numports=NUM
                           Number of TCP/IP ports to scan for Spindle server
                           communication
+  --with-cachepaths=DIR   Colon-separated list of potential back-end cache
+                          directories
   --with-localstorage=DIR Directory on back-ends for storing relocated files
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
@@ -12587,6 +12590,14 @@ else
 fi
 
 
+# Check whether --with-cachepaths was given.
+if test "${with_cachepaths+set}" = set; then :
+  withval=$with_cachepaths; CACHEPATHS=${withval}
+else
+  CACHEPATHS=$DEFAULT_LOC
+fi
+
+
 # Check whether --with-localstorage was given.
 if test "${with_localstorage+set}" = set; then :
   withval=$with_localstorage; SPINDLE_LOC=${withval}
@@ -12623,6 +12634,11 @@ cat >>confdefs.h <<_ACEOF
 _ACEOF
 
 
+cat >>confdefs.h <<_ACEOF
+#define CACHEPATHS "$CACHEPATHS"
+_ACEOF
+
+
 cat >>confdefs.h <<_ACEOF
 #define SPINDLE_LOCAL_PREFIX "$SPINDLE_LOCAL_PREFIX"
 _ACEOF
diff --git a/src/fe/config.h.in b/src/fe/config.h.in
index 357feea0..5057ef33 100644
--- a/src/fe/config.h.in
+++ b/src/fe/config.h.in
@@ -6,6 +6,9 @@
 /* Whether we are using a broken srun */
 #undef BROKEN_SRUN
 
+/* Colon-separated list of potential back-end cache directories */
+#undef CACHEPATHS
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 
diff --git a/src/fe/configure b/src/fe/configure
index 95e99784..29db0f61 100755
--- a/src/fe/configure
+++ b/src/fe/configure
@@ -831,6 +831,7 @@ enable_libtool_lock
 enable_maintainer_mode
 with_default_port
 with_default_num_ports
+with_cachepaths
 with_localstorage
 with_default_local_prefix
 with_testrm
@@ -1570,6 +1571,8 @@ Optional Packages:
   --with-default-numports=NUM
                           Number of TCP/IP ports to scan for Spindle server
                           communication
+  --with-cachepaths=DIR   Colon-separated list of potential back-end cache
+                          directories
   --with-localstorage=DIR Directory on back-ends for storing relocated files
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
@@ -16437,6 +16440,14 @@ else
 fi
 
 
+# Check whether --with-cachepaths was given.
+if test "${with_cachepaths+set}" = set; then :
+  withval=$with_cachepaths; CACHEPATHS=${withval}
+else
+  CACHEPATHS=$DEFAULT_LOC
+fi
+
+
 # Check whether --with-localstorage was given.
 if test "${with_localstorage+set}" = set; then :
   withval=$with_localstorage; SPINDLE_LOC=${withval}
@@ -16473,6 +16484,11 @@ cat >>confdefs.h <<_ACEOF
 _ACEOF
 
 
+cat >>confdefs.h <<_ACEOF
+#define CACHEPATHS "$CACHEPATHS"
+_ACEOF
+
+
 cat >>confdefs.h <<_ACEOF
 #define SPINDLE_LOCAL_PREFIX "$SPINDLE_LOCAL_PREFIX"
 _ACEOF
diff --git a/src/server/config.h.in b/src/server/config.h.in
index 6eea236a..dc9439b3 100644
--- a/src/server/config.h.in
+++ b/src/server/config.h.in
@@ -6,6 +6,9 @@
 /* Whether we are using a broken srun */
 #undef BROKEN_SRUN
 
+/* Colon-separated list of potential back-end cache directories */
+#undef CACHEPATHS
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 
diff --git a/src/server/configure b/src/server/configure
index e32c1cfe..55e9c6f5 100755
--- a/src/server/configure
+++ b/src/server/configure
@@ -837,6 +837,7 @@ enable_libtool_lock
 enable_maintainer_mode
 with_default_port
 with_default_num_ports
+with_cachepaths
 with_localstorage
 with_default_local_prefix
 with_testrm
@@ -1567,6 +1568,8 @@ Optional Packages:
   --with-default-numports=NUM
                           Number of TCP/IP ports to scan for Spindle server
                           communication
+  --with-cachepaths=DIR   Colon-separated list of potential back-end cache
+                          directories
   --with-localstorage=DIR Directory on back-ends for storing relocated files
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
@@ -16434,6 +16437,14 @@ else
 fi
 
 
+# Check whether --with-cachepaths was given.
+if test "${with_cachepaths+set}" = set; then :
+  withval=$with_cachepaths; CACHEPATHS=${withval}
+else
+  CACHEPATHS=$DEFAULT_LOC
+fi
+
+
 # Check whether --with-localstorage was given.
 if test "${with_localstorage+set}" = set; then :
   withval=$with_localstorage; SPINDLE_LOC=${withval}
@@ -16470,6 +16481,11 @@ cat >>confdefs.h <<_ACEOF
 _ACEOF
 
 
+cat >>confdefs.h <<_ACEOF
+#define CACHEPATHS "$CACHEPATHS"
+_ACEOF
+
+
 cat >>confdefs.h <<_ACEOF
 #define SPINDLE_LOCAL_PREFIX "$SPINDLE_LOCAL_PREFIX"
 _ACEOF

From 7e1715d72c1a32a314c7096ee1c19637e5ab86c9 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:52:29 -0700
Subject: [PATCH 04/57] Cachepath:  Internal messaging for path resolution

---
 src/client/client_comlib/client_api.c         | 53 ++++++++++++
 src/client/client_comlib/client_api.h         |  1 +
 src/fe/startup/spindle_fe.cc                  | 18 +++++
 src/include/ldcs_api.h                        |  3 +
 .../auditserver/ldcs_audit_server_handlers.c  | 80 +++++++++++++++++++
 .../auditserver/ldcs_audit_server_md_cobo.c   |  6 ++
 src/server/comlib/ldcs_api_util.c             |  3 +
 7 files changed, 164 insertions(+)

diff --git a/src/client/client_comlib/client_api.c b/src/client/client_comlib/client_api.c
index 96390dca..267b8a93 100644
--- a/src/client/client_comlib/client_api.c
+++ b/src/client/client_comlib/client_api.c
@@ -37,6 +37,59 @@ static struct lock_t comm_lock;
 #define COMM_LOCK do { if (lock(&comm_lock) == -1) return -1; } while (0)
 #define COMM_UNLOCK unlock(&comm_lock)
    
+
+int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chosen_parsed_cachepath, char **chosen_symbolic_cachepath ){
+   ldcs_message_t message;
+   char buffer[MAX_PATH_LEN+1];
+   buffer[MAX_PATH_LEN] = '\0';
+
+   message.header.type = LDCS_MSG_CHOSEN_CACHEPATH_REQUEST;
+   message.header.len = MAX_PATH_LEN;
+   message.data = buffer;
+
+   COMM_LOCK;
+
+   debug_printf3("sending message of type: request_location_path.\n" );
+   client_send_msg(fd, &message);
+   client_recv_msg_static(fd, &message, LDCS_READ_BLOCK);
+
+   COMM_UNLOCK;
+
+   if (message.header.type != LDCS_MSG_CHOSEN_CACHEPATH || message.header.len > MAX_PATH_LEN) {
+      err_printf("Got unexpected message of type %d\n", (int) message.header.type);
+      assert(0);
+   }
+   if( chosen_realized_cachepath ){
+       *chosen_realized_cachepath = strdup( buffer );
+   }
+
+   COMM_LOCK;
+   client_recv_msg_static(fd, &message, LDCS_READ_BLOCK);
+   COMM_UNLOCK;
+
+   if (message.header.type != LDCS_MSG_CHOSEN_CACHEPATH || message.header.len > MAX_PATH_LEN) {
+      err_printf("Got unexpected message of type %d\n", (int) message.header.type);
+      assert(0);
+   }
+   if( chosen_parsed_cachepath ){
+       *chosen_parsed_cachepath = strdup( buffer );
+   }
+
+   COMM_LOCK;
+   client_recv_msg_static(fd, &message, LDCS_READ_BLOCK);
+   COMM_UNLOCK;
+
+   if (message.header.type != LDCS_MSG_CHOSEN_CACHEPATH || message.header.len > MAX_PATH_LEN) {
+      err_printf("Got unexpected message of type %d\n", (int) message.header.type);
+      assert(0);
+   }
+   if( chosen_symbolic_cachepath ){
+       *chosen_symbolic_cachepath = strdup( buffer );
+   }
+
+   return 0;
+}
+
 int send_file_query(int fd, char* path, int dso, char** newpath, int *errcode) {
    ldcs_message_t message;
    char buffer[MAX_PATH_LEN+1+sizeof(int)];
diff --git a/src/client/client_comlib/client_api.h b/src/client/client_comlib/client_api.h
index 74f82346..982c4b1c 100644
--- a/src/client/client_comlib/client_api.h
+++ b/src/client/client_comlib/client_api.h
@@ -42,6 +42,7 @@ int send_orig_path_request(int fd, const char *path, char *newpath);
 int send_dirlists_request(int fd, char **local_result, char **exece_result, char **to_free);
 int send_procmaps_query(int fd, int pid, char *result);
 int send_pickone_query(int fd, char *key, int *result);
+int send_cachepath_query( int fd, char **chosen_symbolic_cachepath, char **chosen_parsed_cachepath, char **chosen_realized_cachepath );
 
 int get_python_prefix(int fd, char **prefix);
 
diff --git a/src/fe/startup/spindle_fe.cc b/src/fe/startup/spindle_fe.cc
index 31b29cf7..43e8c6ba 100644
--- a/src/fe/startup/spindle_fe.cc
+++ b/src/fe/startup/spindle_fe.cc
@@ -41,6 +41,7 @@ static const char *logging_file = NULL;
 #endif
 static const char spindle_bootstrap[] = LIBEXECDIR "/spindle_bootstrap";
 static bool sendAndWaitForAlive();
+static void determineCachepathConsensus();
 
 #define STARTUP_TIMEOUT 60
 
@@ -71,6 +72,7 @@ static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size)
    buffer_size += sizeof(opt_t);
    buffer_size += sizeof(unique_id_t);
    buffer_size += args->location ? strlen(args->location) + 1 : 1;
+   buffer_size += args->candidate_cachepaths ? strlen(args->candidate_cachepaths) + 1 : 1;
    buffer_size += args->pythonprefix ? strlen(args->pythonprefix) + 1 : 1;
    buffer_size += args->preloadfile ? strlen(args->preloadfile) + 1 : 1;
    buffer_size += args->numa_files ? strlen(args->numa_files) + 1 : 1;
@@ -91,6 +93,7 @@ static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size)
    pack_param(args->startup_type, buf, pos);
    pack_param(args->shm_cache_size, buf, pos);
    pack_param(args->location, buf, pos);
+   pack_param(args->candidate_cachepaths, buf, pos);
    pack_param(args->pythonprefix, buf, pos);
    pack_param(args->preloadfile, buf, pos);
    pack_param(args->bundle_timeout_ms, buf, pos);
@@ -230,6 +233,7 @@ int getApplicationArgsFE(spindle_args_t *params, int *spindle_argc, char ***spin
       (*spindle_argv)[n++] = strdup(uniqueid_s);
    }
    (*spindle_argv)[n++] = strdup(params->location);
+   (*spindle_argv)[n++] = strdup(params->candidate_cachepaths);
    (*spindle_argv)[n++] = strdup(number_s);
    (*spindle_argv)[n++] = strdup(opt_s);
    (*spindle_argv)[n++] = strdup(cachesize_s);
@@ -395,9 +399,11 @@ int spindleInitFE(const char **hosts, spindle_args_t *params)
    /* Start FE server */
    debug_printf("spindle_args_t { number = %lu; port = %u; num_ports = %u; opts = %lu; unique_id = %lu; "
                 "use_launcher = %u; startup_type = %u; shm_cache_size = %u; location = %s; "
+                "cachepaths = %s; "
                 "pythonprefix = %s; preloadfile = %s; bundle_timeout_ms = %u; bundle_cachesize_kb = %u }\n",
                 (unsigned long) params->number, params->port, params->num_ports, params->opts, params->unique_id,
                 params->use_launcher, params->startup_type, params->shm_cache_size, params->location,
+                params->candidate_cachepaths,
                 params->pythonprefix, params->preloadfile, params->bundle_timeout_ms,
                 params->bundle_cachesize_kb);
    printSpindleFlags(params->opts);
@@ -427,6 +433,7 @@ int spindleInitFE(const char **hosts, spindle_args_t *params)
 
    /* Wait for servers to indicate startup */
    sendAndWaitForAlive();
+   determineCachepathConsensus();
 
    return 0;   
 }
@@ -483,6 +490,17 @@ void markRSHPidReapedFE()
    clear_fe_rsh_pid();
 }
 
+static void determineCachepathConsensus( void ){
+   ldcs_message_t consensus_req_msg;
+   consensus_req_msg.header.type = LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS;
+   consensus_req_msg.header.len = 0;
+   consensus_req_msg.data = NULL;
+   int result = ldcs_audit_server_fe_broadcast(&consensus_req_msg, NULL);
+   if (result == -1) {
+      debug_printf("Failure sending cachepath consensus message\n");
+   }
+}
+
 static bool sendAndWaitForAlive()
 {
    int result;
diff --git a/src/include/ldcs_api.h b/src/include/ldcs_api.h
index e8ffa43d..e6ccbafb 100644
--- a/src/include/ldcs_api.h
+++ b/src/include/ldcs_api.h
@@ -85,6 +85,9 @@ typedef enum {
    LDCS_MSG_PICKONE_RESP,
    LDCS_MSG_ALIVE_REQ,
    LDCS_MSG_ALIVE_RESP,
+   LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS,
+   LDCS_MSG_CHOSEN_CACHEPATH_REQUEST,
+   LDCS_MSG_CHOSEN_CACHEPATH,
    LDCS_MSG_UNKNOWN
 } ldcs_message_ids_t;
 
diff --git a/src/server/auditserver/ldcs_audit_server_handlers.c b/src/server/auditserver/ldcs_audit_server_handlers.c
index 8a18de24..98c6807a 100644
--- a/src/server/auditserver/ldcs_audit_server_handlers.c
+++ b/src/server/auditserver/ldcs_audit_server_handlers.c
@@ -178,7 +178,10 @@ static int handle_setup_alias(ldcs_process_data_t *procdata, char *pathname, cha
 static int handle_client_dirlists_req(ldcs_process_data_t *procdata, int nc);
 static int handle_close_client_query(ldcs_process_data_t *procdata, int nc);
 static int handle_alive_msg(ldcs_process_data_t *procdata, ldcs_message_t *msg);
+static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_message_t *msg);
+static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc);
 
+extern void getValidCachePathByIndex( uint64_t validBitIdx, char **realizedCachePath, char **parsedCachePath, char **symbolicCachePath );
 /**
  * Query from client to server.  Returns info about client's rank in server data structures. 
  **/
@@ -1890,6 +1893,8 @@ int handle_client_message(ldcs_process_data_t *procdata, int nc, ldcs_message_t
          return handle_client_pickone_msg(procdata, nc, msg);
       case LDCS_MSG_END:
          return handle_client_end(procdata, nc);
+      case LDCS_MSG_CHOSEN_CACHEPATH_REQUEST:
+         return handle_chosen_cachepath_request(procdata, nc);
       default:
          err_printf("Received unexpected message from client %d: %d\n", nc, (int) msg->header.type);
          assert(0);
@@ -1987,6 +1992,8 @@ int handle_server_message(ldcs_process_data_t *procdata, node_peer_t peer, ldcs_
       case LDCS_MSG_ALIVE_REQ:
       case LDCS_MSG_ALIVE_RESP:
          return handle_alive_msg(procdata, msg);
+      case LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS:
+         return handle_cachepath_consensus(procdata, msg);
       default:
          err_printf("Received unexpected message from node: %d\n", (int) msg->header.type);
          assert(0);
@@ -2945,6 +2952,79 @@ static int handle_client_pickone_msg(ldcs_process_data_t *procdata, int nc, ldcs
    }
 }
 
+/**
+ * Handle LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS to determine which of the locations, commpaths, and cachepaths are
+ * available across all of the servers.
+ */
+
+static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_message_t *msg){
+
+    int num_children = ldcs_audit_server_md_get_num_children(procdata);
+
+    if (num_children) {
+        spindle_broadcast(procdata, msg);
+        msgbundle_force_flush(procdata);
+    }
+
+    ldcs_audit_server_md_consensus(procdata, msg);
+
+    if( procdata->cachepath_bitidx == 0 ){
+       err_printf("No valid cachepath path available.  Falling back to \"location\" path (%s).\n", procdata->location);
+       procdata->cachepath = procdata->location;
+    }else{
+        // ldcs_audit_server_filemngt_init() does it's own realize() pass.
+        getValidCachePathByIndex( procdata->cachepath_bitidx,
+                &procdata->cachepath,
+                &procdata->parsed_cachepath,
+                &procdata->symbolic_cachepath);
+    }
+
+    debug_printf3("Initializing file cache location %s\n", procdata->location);
+    ldcs_audit_server_filemngt_init(procdata->cachepath);
+
+    test_printf("<internal> cachepath=%s\n", procdata->cachepath);
+    return 0;
+}
+
+/**
+ * Handle LDCS_MSG_CHOSEN_CACHEPATH_REQUEST
+ */
+static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc){
+   ldcs_message_t msg;
+   int connid;
+   ldcs_client_t *client;
+
+   assert(nc != -1);
+   client = procdata->client_table + nc;
+   connid = client->connid;
+   if (client->state != LDCS_CLIENT_STATUS_ACTIVE || connid < 0)
+      return 0;
+
+
+   msg.header.type = LDCS_MSG_CHOSEN_CACHEPATH;
+
+   msg.header.len = strlen(procdata->cachepath) + 1;
+   msg.data = procdata->cachepath;
+   ldcs_send_msg(connid, &msg);
+   procdata->server_stat.clientmsg.cnt++;
+   procdata->server_stat.clientmsg.time += ldcs_get_time() - client->query_arrival_time;
+
+   msg.header.len = strlen(procdata->parsed_cachepath) + 1;
+   msg.data = procdata->parsed_cachepath;
+   ldcs_send_msg(connid, &msg);
+   procdata->server_stat.clientmsg.cnt++;
+   procdata->server_stat.clientmsg.time += ldcs_get_time() - client->query_arrival_time;
+
+   msg.header.len = strlen(procdata->symbolic_cachepath) + 1;
+   msg.data = procdata->symbolic_cachepath;
+   ldcs_send_msg(connid, &msg);
+   procdata->server_stat.clientmsg.cnt++;
+   procdata->server_stat.clientmsg.time += ldcs_get_time() - client->query_arrival_time;
+
+   return 0;
+}
+
+
 /**
  * Handle alive message, which is a broadcast/response ping through all servers
  */
diff --git a/src/server/auditserver/ldcs_audit_server_md_cobo.c b/src/server/auditserver/ldcs_audit_server_md_cobo.c
index 08c9b952..d8b5442f 100644
--- a/src/server/auditserver/ldcs_audit_server_md_cobo.c
+++ b/src/server/auditserver/ldcs_audit_server_md_cobo.c
@@ -401,3 +401,9 @@ int ldcs_audit_server_md_get_num_children(ldcs_process_data_t *procdata)
    cobo_get_num_childs(&num_childs);
    return num_childs;
 }
+
+void ldcs_audit_server_md_consensus(ldcs_process_data_t *ldcs_process_data, ldcs_message_t *msg){
+    if( msg->header.type == LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS ){
+        cobo_allreduce( &ldcs_process_data->cachepath_bitidx, COBO_OP_BITWISE_AND );
+    }
+}
diff --git a/src/server/comlib/ldcs_api_util.c b/src/server/comlib/ldcs_api_util.c
index 2bc2455d..b6beb56d 100644
--- a/src/server/comlib/ldcs_api_util.c
+++ b/src/server/comlib/ldcs_api_util.c
@@ -91,6 +91,9 @@ char* _message_type_to_str (ldcs_message_ids_t type) {
       STR_CASE(LDCS_MSG_PICKONE_RESP);
       STR_CASE(LDCS_MSG_ALIVE_REQ);
       STR_CASE(LDCS_MSG_ALIVE_RESP);      
+      STR_CASE(LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS);
+      STR_CASE(LDCS_MSG_CHOSEN_CACHEPATH_REQUEST);
+      STR_CASE(LDCS_MSG_CHOSEN_CACHEPATH);
       STR_CASE(LDCS_MSG_UNKNOWN);
    }
    return "unknown";

From 2020634e2943a5941767c6380b695f76e430db54 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:53:25 -0700
Subject: [PATCH 05/57] Cachepath:  Adds cobo_allreduce()

---
 src/cobo/cobo.c      | 42 ++++++++++++++++++++++++++++++++++++++++++
 src/cobo/ldcs_cobo.h | 16 ++++++++++++++++
 2 files changed, 58 insertions(+)

diff --git a/src/cobo/cobo.c b/src/cobo/cobo.c
index 785a5d03..ad2ea878 100644
--- a/src/cobo/cobo.c
+++ b/src/cobo/cobo.c
@@ -1434,6 +1434,48 @@ int cobo_allgather_str(char* sendstr, char*** recvstr, char** recvbuf)
     return COBO_SUCCESS;
 }
 
+int cobo_allreduce( int64_t *pval, cobo_op_t op ){
+
+    /* if i have any children, receive their data */
+    int64_t child_val;
+    for(int i=cobo_num_child-1; i>=0; i--) {
+        /* read int64_t from child */
+        if (cobo_read_fd(cobo_child_fd[i], &child_val, sizeof(int64_t)) < 0) {
+            err_printf("Reducing data from child (rank %d) failed\n", cobo_child[i]);
+            exit(1);
+        }
+
+        /* compare child's val to our current val */
+        switch( op ){
+            case COBO_OP_MIN:           if( child_val < *pval ) *pval = child_val; break;
+            case COBO_OP_MAX:           if( child_val > *pval ) *pval = child_val; break;
+            case COBO_OP_BITWISE_AND:   *pval &= child_val; break;
+            case COBO_OP_BITWISE_OR:    *pval |= child_val; break;
+            case COBO_OP_LOGICAL_AND:   *pval = *pval && child_val; break;
+            case COBO_OP_LOGICAL_OR:    *pval = *pval || child_val; break;
+            case COBO_OP_SUM:           *pval += child_val; break;
+            case COBO_OP_NOOP:          break;
+            default:
+                err_printf("Illegal op (%d).  Ignoring.\n", op);
+                break;
+        }
+    }
+
+    /* forward data to parent if we're not rank 0, otherwise set the recvbuf */
+    if (cobo_me != 0) {
+        /* not the root, so forward our reduction result to our parent */
+        if (cobo_write_fd(cobo_parent_fd, pval, sizeof(*pval)) < 0) {
+            err_printf("Sending reduced data to parent failed\n");
+            exit(1);
+        }
+    }
+
+    /* broadcast result of reduction from rank 0 to all tasks */
+    cobo_bcast_tree(pval, sizeof(int64_t));
+
+    return COBO_SUCCESS;
+}
+
 /* provide list of ports and number of ports as input, get number of tasks and my rank as output */
 int cobo_open(uint64_t sessionid, int* portlist, int num_ports, int* rank, int* num_ranks)
 {
diff --git a/src/cobo/ldcs_cobo.h b/src/cobo/ldcs_cobo.h
index edacd4b1..30cc9673 100644
--- a/src/cobo/ldcs_cobo.h
+++ b/src/cobo/ldcs_cobo.h
@@ -55,6 +55,7 @@ extern "C" {
 #define cobo_allgather COMBINE(COBO_NAMESPACE, cobo_allgather)
 #define cobo_alltoall  COMBINE(COBO_NAMESPACE, cobo_alltoall )
 #define cobo_allgather_str COMBINE(COBO_NAMESPACE, cobo_allgather_str)
+#define cobo_allreduce COMBINE(COBO_NAMESPACE, cobo_allreduce)
 #define cobo_server_open COMBINE(COBO_NAMESPACE, cobo_server_open)
 #define cobo_server_close COMBINE(COBO_NAMESPACE, cobo_server_close)
 #define cobo_server_get_root_socket COMBINE(COBO_NAMESPACE, cobo_server_get_root_socket)
@@ -67,6 +68,19 @@ extern "C" {
 #define cobo_register_preconnect_cb COMBINE(COBO_NAMESPACE, cobo_register_preconnect_cb)
 #endif
 
+// Used for cobo_allreduce().
+typedef enum{
+    COBO_OP_MIN,
+    COBO_OP_MAX,
+    COBO_OP_BITWISE_AND,
+    COBO_OP_BITWISE_OR,
+    COBO_OP_LOGICAL_AND,
+    COBO_OP_LOGICAL_OR,
+    COBO_OP_SUM,
+    COBO_OP_NOOP,
+    NUM_COBO_OP
+} cobo_op_t;
+
 /*
  * ==========================================================================
  * ==========================================================================
@@ -128,6 +142,8 @@ int cobo_alltoall (void* sendbuf, int sendcount, void* recvbuf);
  */
 int cobo_allgather_str(char* sendstr, char*** recvstr, char** recvbuf);
 
+int cobo_allreduce(int64_t *pval, cobo_op_t op);
+
 /*
  * ==========================================================================
  * ==========================================================================

From dfdf1665aa0c2d6e8d56b904aac0182a4fc24ee2 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:54:01 -0700
Subject: [PATCH 06/57] Cachepath:  Adds parameters to config_mgr

---
 src/fe/startup/config_mgr.cc | 23 +++++++++++++++++++++++
 src/fe/startup/config_mgr.h  |  4 +++-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/src/fe/startup/config_mgr.cc b/src/fe/startup/config_mgr.cc
index 36a30b20..eb710b36 100644
--- a/src/fe/startup/config_mgr.cc
+++ b/src/fe/startup/config_mgr.cc
@@ -56,6 +56,12 @@ using namespace std;
 #define SPINDLE_LOC_STR "$TMPDIR"
 #endif
 
+#if defined(CACHEPATHS)
+#define SPINDLE_CACHEPATHS_STR CACHEPATHS
+#else
+#define SPINDLE_CACHEPATHS_STR "$TMPDIR"
+#endif
+
 #if defined(SPINDLE_LOCAL_PREFIX)
 #define SPINDLE_LOCAL_PREFIX_STR SPINDLE_LOCAL_PREFIX
 #else
@@ -269,6 +275,8 @@ void initOptionsList()
      "Strip debug and symbol information from binaries before distributing them." },
    { confLocation, "location", shortLocation, groupMisc, cvString, {}, SPINDLE_LOC_STR,
      "Back-end directory for storing relocated files.  Should be a non-shared location such as a ramdisk." },
+   { confCachePaths, "cachepaths", shortCachePaths, groupMisc, cvString, {}, SPINDLE_CACHEPATHS_STR,
+     "Colon-separated list of candidate paths for cached libraries."},
    { confNoclean, "noclean", shortNoClean, groupMisc, cvBool, {}, "false",
      "Don't remove local file cache after execution." },
    { confDisableLogging, "disable-logging", shortDisableLogging, groupMisc, cvBool, {}, DISABLE_LOGGING_STR,
@@ -740,6 +748,21 @@ bool ConfigMap::toSpindleArgs(spindle_args_t &args, bool alloc_strs) const
             args.location = strdup(loc.c_str());
             break;
          }
+         case confCachePaths:{
+            // Paramemter values are colon-separated lists of paths.
+            // Append "/spindle.$NUMBER" to each path in the list.
+            string paths = strresult;
+            size_t idx = paths.find(":");
+            string number_var_with_colon("/spindle.$NUMBER:");
+            string number_var_without_colon("/spindle.$NUMBER");
+            while( idx != string::npos ){
+               paths.replace(idx, 1, number_var_with_colon);
+               idx = paths.find(":", idx + number_var_with_colon.size());
+            };
+            paths += number_var_without_colon;
+            args.candidate_cachepaths = strdup(paths.c_str());
+            break;
+         }
          case confCachePrefix:
          case confPythonPrefix:
             if (args.pythonprefix)
diff --git a/src/fe/startup/config_mgr.h b/src/fe/startup/config_mgr.h
index 8e70daa6..27be1ae8 100644
--- a/src/fe/startup/config_mgr.h
+++ b/src/fe/startup/config_mgr.h
@@ -30,6 +30,7 @@ enum SpindleConfigID {
    confPort,
    confNumPorts,
    confLocation,
+   confCachePaths,
    confCachePrefix,
    confPythonPrefix,
    confLocalPrefix,
@@ -125,7 +126,8 @@ enum CmdlineShortOptions {
    shortSpindleLevel = 296,
    shortLocalPrefix = 297,
    shortExecExcludes = 298,
-   shortPatchLdso
+   shortPatchLdso,
+   shortCachePaths,
 };
 
 enum CmdlineGroups {

From 38bacd3ae153f2a7b03f8614659fbcff0723d61d Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:55:39 -0700
Subject: [PATCH 07/57] Cachepath:  adds flux parameter support

---
 src/flux/flux-spindle.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/flux/flux-spindle.c b/src/flux/flux-spindle.c
index 6c99341a..65de5ed9 100644
--- a/src/flux/flux-spindle.c
+++ b/src/flux/flux-spindle.c
@@ -382,7 +382,7 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx)
     const char *relocaout = NULL, *reloclibs = NULL, *relocexec = NULL, *relocpython = NULL;
     const char *followfork = NULL, *preload = NULL, *level = NULL;
     const char *pyprefix = NULL, *location = NULL;
-    char *numafiles = NULL;
+    char *numafiles = NULL, *cachepaths = NULL;
 
     if (flux_shell_getopt_unpack (shell, "spindle", "o", &opts) < 0)
         return -1;
@@ -404,7 +404,7 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx)
      *  supplied by the user, but not unpacked (This handles typos, etc).
      */
     if (json_unpack_ex (opts, &error, JSON_STRICT,
-                        "{s?i s?i s?i s?i s?s s?s s?s s?s s?s s?s s?s s?i s?s s?s s?s}",
+                        "{s?i s?i s?i s?i s?s s?s s?s s?s s?s s?s s?s s?i s?s s?s s?s s?s}",
                         "noclean", &noclean,
                         "nostrip", &nostrip,
                         "push", &push,
@@ -419,7 +419,8 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx)
                         "numa", &numa,
                         "numa-files", &numafiles,
                         "preload", &preload,
-                        "level", &level) < 0)
+                        "level", &level,
+                        "cachepaths", &cachepaths) < 0)
        logerrno_printf_and_return(1, "Error in spindle option: %s\n", error.text);
 
     if (noclean)
@@ -462,6 +463,9 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx)
         free (ctx->params.pythonprefix);
         ctx->params.pythonprefix = tmp;
     }
+    if( cachepaths ){
+        ctx->params.candidate_cachepaths = cachepaths;
+    }
     if (location) {
        ctx->params.location = (char *) location;
     }

From 6d34e1848f9561e0aef7a896872c0d3db725f4cd Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:56:05 -0700
Subject: [PATCH 08/57] Cachepath:  Adds logging support.

---
 src/logging/spindle_logd.cc | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/logging/spindle_logd.cc b/src/logging/spindle_logd.cc
index 53eb5dc0..45d58a7d 100644
--- a/src/logging/spindle_logd.cc
+++ b/src/logging/spindle_logd.cc
@@ -202,7 +202,7 @@ class TestVerifier
    std::vector<std::string> err_strings;
    std::set<std::pair<int, std::string> > target_libs;
    std::set<std::pair<int, std::string> > libs_loaded;
-   char *location;
+   char *cachepath;
 
    void logerror(std::string s)
    {
@@ -249,7 +249,8 @@ class TestVerifier
          tmp_s = getenv("TEMPDIR");
       if (!tmp_s)
          tmp_s = "/tmp";
-      location = strdup(tmp_s);
+      // These are reasonable fallbacks that should be replaced via <internal> messages, below.
+      cachepath  = strdup(tmp_s);
    }
 
    ~TestVerifier()
@@ -268,7 +269,7 @@ class TestVerifier
           strstr(filename, "retzero") == NULL &&
           strstr(filename, ".py") == NULL)
          return true;
-      bool is_from_temp = (strstr(filename, location) != NULL) && (strncmp(filename, "/__not_exist", 12) != 0);
+      bool is_from_temp = (strstr(filename, cachepath) != NULL) && (strncmp(filename, "/__not_exist", 12) != 0);
       bool is_local_test = strstr(filename, "liblocal") != NULL;
 
       if (is_from_temp && !is_local_test && ret_code == -1) {
@@ -294,12 +295,12 @@ class TestVerifier
       char buffer[4096];
       int ret;
 
-      if (strstr(s, "<internal> location=" ) == s ){
-        free( location );
-        const char *loc_start = strstr( s, "=") + 1;
-        size_t loc_len = strlen( loc_start );
-        location = strdup( loc_start );
-        location[ loc_len - 1 ] = '\0'; // Remove trailing '\n'.
+      if (strstr(s, "<internal> cachepath=" ) == s ){
+        free( cachepath );
+        const char *cachepath_start = strstr( s, "=") + 1;
+        size_t cachepath_len = strlen( cachepath_start );
+        cachepath = strdup( cachepath_start );
+        cachepath[ cachepath_len - 1 ] = '\0'; // Remove trailing '\n'.
       }
       if (strstr(s, "open(") == s) {
          const char *first_quote, *last_quote, *equals;

From 69f03deeb031adc629bf2698327bebd71599704c Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:56:34 -0700
Subject: [PATCH 09/57] Cachepath:  Removes out-of-root cleanup checks.

---
 src/server/auditserver/cleanup_proc.cc |  4 ++--
 testsuite/test_driver.c                | 23 -----------------------
 2 files changed, 2 insertions(+), 25 deletions(-)

diff --git a/src/server/auditserver/cleanup_proc.cc b/src/server/auditserver/cleanup_proc.cc
index c23a9f7d..a3d3ddcc 100644
--- a/src/server/auditserver/cleanup_proc.cc
+++ b/src/server/auditserver/cleanup_proc.cc
@@ -72,7 +72,8 @@ static void rmDirSet(const set<string> &dirs, const char *prefix_dir)
             continue;
 
          if (strncmp(prefix_dir, componentpath.c_str(), prefix_size) != 0) {
-            err_printf("Tried to clean a file %s that wasn't in our prefix %s\n", componentpath.c_str(), prefix_dir);
+            // We have multiple directory roots.  Not a problem if the directory
+            // we're looking for isn't in this one.
             continue;
          }
          unlink(componentpath.c_str());
@@ -83,7 +84,6 @@ static void rmDirSet(const set<string> &dirs, const char *prefix_dir)
    sort(ordered_dirs.begin(), ordered_dirs.end(), longest_str_first);
    for (vector<string>::iterator i = ordered_dirs.begin(); i != ordered_dirs.end(); i++) {
       if (strncmp(prefix_dir, i->c_str(), prefix_size) != 0) {
-         err_printf("Tried to rmdir directory %s that wasn't in our prefix %s\n", i->c_str(), prefix_dir);
          continue;
       }      
       rmdir(i->c_str());
diff --git a/testsuite/test_driver.c b/testsuite/test_driver.c
index 13f63fea..a0b46a75 100644
--- a/testsuite/test_driver.c
+++ b/testsuite/test_driver.c
@@ -1169,27 +1169,6 @@ static char* getCacheLocation(char *env_var)
    return strdup(last_slash);
 }
 
-static int checkLinkForLeak(const char *path, const char *spindle_loc)
-{
-   char link_target[4096];
-   int result, error;
-   memset(link_target, 0, sizeof(link_target));
-
-   result = readlink(path, link_target, sizeof(link_target));
-   if (result == -1) {
-      error = errno;
-      err_printf("Failed to read link %s: %s\n", path, strerror(error));
-      return -1;
-   }
-
-   if (strstr(link_target, spindle_loc)) {
-      err_printf("Link at '%s' has path '%s', which leaks spindle path with '%s'\n", path, link_target, spindle_loc);
-      return -1;
-   }
-
-   return 0;
-}
-
 static int checkPathForLeak(const char *what, const char *path, const char *spindle_loc)
 {
    if (strstr(path, spindle_loc)) {
@@ -1286,9 +1265,7 @@ void check_for_path_leaks()
          continue;
       strncpy(path, "/proc/self/fd/", sizeof(path));
       strncat(path, d->d_name, sizeof(path)-1);
-      checkLinkForLeak(path, spindle_loc);
    }
-   checkLinkForLeak("/proc/self/exe", spindle_loc);
 
    /**
     * Check link_maps for leaked spindle paths

From 36540bec1006bef965fb72475357e7e15a79ce21 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:57:06 -0700
Subject: [PATCH 10/57] Cachepath:  Set of small, miscellaneous patches.

---
 src/include/spindle_launch.h                       | 5 +++++
 src/server/auditserver/ldcs_audit_server_md.h      | 3 +++
 src/server/auditserver/ldcs_audit_server_process.c | 5 +++--
 src/server/auditserver/ldcs_audit_server_process.h | 5 +++++
 src/server/startup/spindle_be.cc                   | 4 +++-
 5 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/src/include/spindle_launch.h b/src/include/spindle_launch.h
index 47a4d92e..5476734d 100644
--- a/src/include/spindle_launch.h
+++ b/src/include/spindle_launch.h
@@ -126,6 +126,11 @@ typedef struct {
    /* The local-disk location where Spindle will store its cache */
    char *location;
 
+   /* Path[s] for cached libraries. */
+   char *candidate_cachepaths;  /* Colon-separated list of candidate paths (max 64) */
+   char *chosen_cachepath;      /* The consensus path (same across all nodes). */
+   uint64_t cachepath_bitidx;   /* Bit index used by allReduce() to arrive at consensus. */
+
    /* Colon-seperated list of directories where Python is installed */
    char *pythonprefix;
 
diff --git a/src/server/auditserver/ldcs_audit_server_md.h b/src/server/auditserver/ldcs_audit_server_md.h
index eb5bf9f6..ba7943e2 100644
--- a/src/server/auditserver/ldcs_audit_server_md.h
+++ b/src/server/auditserver/ldcs_audit_server_md.h
@@ -107,6 +107,9 @@ int ldcs_audit_server_md_broadcast_noncontig(ldcs_process_data_t *ldcs_process_d
 int ldcs_audit_server_md_get_num_children(ldcs_process_data_t *procdata);
 
 int ldcs_audit_server_md_is_parent(node_peer_t peer);
+
+void ldcs_audit_server_md_consensus(ldcs_process_data_t *ldcs_process_data, ldcs_message_t *msg);
+
    
 #if defined(__cplusplus)
 }
diff --git a/src/server/auditserver/ldcs_audit_server_process.c b/src/server/auditserver/ldcs_audit_server_process.c
index 312095be..dca91c20 100644
--- a/src/server/auditserver/ldcs_audit_server_process.c
+++ b/src/server/auditserver/ldcs_audit_server_process.c
@@ -142,6 +142,9 @@ int ldcs_audit_server_process(spindle_args_t *args)
 
    debug_printf3("Initializing server data structures\n");
    ldcs_process_data.location = args->location;
+   ldcs_process_data.cachepaths = args->candidate_cachepaths;
+   ldcs_process_data.cachepath = args->chosen_cachepath;
+   ldcs_process_data.cachepath_bitidx = args->cachepath_bitidx;
    ldcs_process_data.number = args->number;
    ldcs_process_data.pythonprefix = args->pythonprefix;
    ldcs_process_data.localprefix = args->local_prefixes;
@@ -191,8 +194,6 @@ int ldcs_audit_server_process(spindle_args_t *args)
    }
    ldcs_process_data.server_stat.hostname=ldcs_process_data.hostname;
 
-   debug_printf3("Initializing file cache location %s\n", ldcs_process_data.location);
-   ldcs_audit_server_filemngt_init(ldcs_process_data.location);
    if (ldcs_process_data.opts & OPT_PROCCLEAN)
       init_cleanup_proc(ldcs_process_data.location);
 
diff --git a/src/server/auditserver/ldcs_audit_server_process.h b/src/server/auditserver/ldcs_audit_server_process.h
index 18b3320a..9ba1675e 100644
--- a/src/server/auditserver/ldcs_audit_server_process.h
+++ b/src/server/auditserver/ldcs_audit_server_process.h
@@ -126,6 +126,11 @@ struct ldcs_process_data_struct
   ldcs_dist_model_t dist_model;
   ldcs_client_t* client_table;
   char *location;
+  char *cachepaths;
+  char *cachepath;
+  char *symbolic_cachepath;
+  char *parsed_cachepath;
+  int64_t cachepath_bitidx;
   char *hostname;
   char *pythonprefix;
   char *localprefix;
diff --git a/src/server/startup/spindle_be.cc b/src/server/startup/spindle_be.cc
index 7493c020..4f583756 100644
--- a/src/server/startup/spindle_be.cc
+++ b/src/server/startup/spindle_be.cc
@@ -59,6 +59,7 @@ static int unpack_data(spindle_args_t *args, void *buffer, int buffer_size)
    unpack_param(args->startup_type, buf, pos);
    unpack_param(args->shm_cache_size, buf, pos);
    unpack_param(args->location, buf, pos);
+   unpack_param(args->candidate_cachepaths, buf, pos);
    unpack_param(args->pythonprefix, buf, pos);
    unpack_param(args->preloadfile, buf, pos);
    unpack_param(args->bundle_timeout_ms, buf, pos);
@@ -152,7 +153,8 @@ int spindleRunBE(unsigned int port, unsigned int num_ports, unique_id_t unique_i
    debug_printf("Translated location from %s to %s\n", args.location, new_location);
    free(args.location);
    args.location = new_location;
-   test_printf("<internal> location=%s\n", args.location);
+
+   determineValidCachePaths( &args.cachepath_bitidx, args.candidate_cachepaths, args.number);
 
    result = ldcs_audit_server_process(&args);
    if (result == -1) {

From 0eb779ad3fe1d2d8e4bbd96d1c157cc616412996 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Tue, 21 Oct 2025 12:21:37 -0700
Subject: [PATCH 11/57] Fixes per Matt's comments.

---
 src/client/client/intercept_exec.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/client/client/intercept_exec.c b/src/client/client/intercept_exec.c
index 8480b75e..27928d35 100644
--- a/src/client/client/intercept_exec.c
+++ b/src/client/client/intercept_exec.c
@@ -196,7 +196,7 @@ static char **updateEnvironment(char **envp, int *num_modified, int propogate_sp
    if (envp) {
       debug_printf2("Propogating spindle environment by copying it to new envp list\n");
       for (cur = (char **) envp; *cur; cur++, orig_size++);
-      new_size = orig_size + 20;
+      new_size = orig_size + 9;
       newenv = (char **) malloc(new_size * sizeof(char*));
       
       propogateEnvironmentStr(envp, newenv, &pos, "SPINDLE");   

From 61ba31cb937b6e55e11b723cbcf2f1f02a69c88f Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Wed, 22 Oct 2025 12:20:35 -0700
Subject: [PATCH 12/57] Single source of truth for client cachepath.

Previously,

chosen_realized_cachepath was copied into
set_intercept_readlink_cachepath()

chosen_realized_cachepath and chosen_parsed_cachepath were copied
into set_should_intercept_cachepath()

This PR removes both setter functions and makes the original
pointers global.
---
 src/client/client/client.c             |  7 ++-----
 src/client/client/intercept.h          |  1 -
 src/client/client/intercept_readlink.c | 12 +++---------
 src/client/client/should_intercept.c   | 17 +++++------------
 src/client/client/should_intercept.h   |  1 -
 5 files changed, 10 insertions(+), 28 deletions(-)

diff --git a/src/client/client/client.c b/src/client/client/client.c
index d7827d4b..f803e75b 100644
--- a/src/client/client/client.c
+++ b/src/client/client/client.c
@@ -72,7 +72,7 @@ static int num_libc_phdrs, num_interp_phdrs;
 ElfW(Addr) libc_loadoffset, interp_loadoffset;
 
 static char *location;
-static char *chosen_realized_cachepath, *chosen_parsed_cachepath, *chosen_symbolic_cachepath;
+char *chosen_realized_cachepath, *chosen_parsed_cachepath;
 number_t number;
 static int have_stat_patches;
 
@@ -261,10 +261,7 @@ static int init_server_connection()
          send_cpu(ldcsid, get_cur_cpu());
 #endif
    }
-   send_cachepath_query( ldcsid, &chosen_realized_cachepath, &chosen_parsed_cachepath, &chosen_symbolic_cachepath );
-   set_should_intercept_cachepath(   chosen_realized_cachepath, chosen_parsed_cachepath, chosen_symbolic_cachepath );
-   set_intercept_readlink_cachepath( chosen_realized_cachepath, chosen_parsed_cachepath, chosen_symbolic_cachepath );
-   
+   send_cachepath_query( ldcsid, &chosen_realized_cachepath, &chosen_parsed_cachepath, NULL);
    snprintf(debugging_name, 32, "Client.%d", rankinfo[0]);
    LOGGING_INIT(debugging_name);
 
diff --git a/src/client/client/intercept.h b/src/client/client/intercept.h
index aae968f7..4ace2328 100644
--- a/src/client/client/intercept.h
+++ b/src/client/client/intercept.h
@@ -89,7 +89,6 @@ int execvpe_wrapper(const char *path, char *const argv[], const char *envp[]);
 pid_t vfork_wrapper();
 char *dlerror_wrapper();
 
-void set_intercept_readlink_cachepath( char *chosen_realized_cachepath, char *chosen_parsed_cachepath, char *chosen_symbolic_cachepath );
 ssize_t readlink_wrapper(const char *path, char *buf, size_t bufsiz);
 ssize_t readlinkat_wrapper(int dirfd, const char *pathname, char *buf, size_t bufsiz);
 
diff --git a/src/client/client/intercept_readlink.c b/src/client/client/intercept_readlink.c
index 2799baee..37d440e4 100644
--- a/src/client/client/intercept_readlink.c
+++ b/src/client/client/intercept_readlink.c
@@ -31,23 +31,17 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 ssize_t (*orig_readlink)(const char *path, char *buf, size_t bufsiz);
 ssize_t (*orig_readlinkat)(int dirfd, const char *pathname, char *buf, size_t bufsiz);
 
-static char *cachepath;
-
-void set_intercept_readlink_cachepath( char *chosen_realized_cachepath, char *chosen_parsed_cachepath, char *chosen_symbolic_cachepath ){
-    cachepath = chosen_realized_cachepath;
-    chosen_parsed_cachepath = chosen_parsed_cachepath;
-    chosen_symbolic_cachepath = chosen_symbolic_cachepath;
-}
 
 static int fix_local_readlink(char *buf, size_t bufsiz)
 {
    char spindle_id[32];
    int cachepath_len, result;
    char tmp[MAX_PATH_LEN+1];
+   extern char *chosen_realized_cachepath;
 
-   cachepath_len = strlen(cachepath);   
+   cachepath_len = strlen(chosen_realized_cachepath);   
    snprintf(spindle_id, sizeof(spindle_id), "spindle.%lx", number);
-   if (strstr(buf, spindle_id) && strncmp(cachepath, buf, cachepath_len) == 0) {
+   if (strstr(buf, spindle_id) && strncmp(chosen_realized_cachepath, buf, cachepath_len) == 0) {
       debug_printf2("readlink received spindle cache path %s. Translating\n", buf);
       result = send_orig_path_request(ldcsid, buf+cachepath_len+1, tmp);
       if (result == -1)
diff --git a/src/client/client/should_intercept.c b/src/client/client/should_intercept.c
index 3a348d3d..cee4e43c 100644
--- a/src/client/client/should_intercept.c
+++ b/src/client/client/should_intercept.c
@@ -29,27 +29,20 @@
 #include "spindle_debug.h"
 
 extern int relocate_spindleapi();
-static char *cachepath, *orig_cachepath;
-
-void set_should_intercept_cachepath( char *chosen_realized_cachepath, char *chosen_parsed_cachepath, char *chosen_symbolic_cachepath ){
-    cachepath = chosen_realized_cachepath;
-    orig_cachepath = chosen_parsed_cachepath;
-    chosen_symbolic_cachepath = chosen_symbolic_cachepath;
-}
-
 
 int is_in_spindle_cache(const char *pathname)
 {
    static int cachepath_size = 0;
    static int orig_cachepath_size = 0;
+   extern char *chosen_realized_cachepath, *chosen_parsed_cachepath;
    if (!cachepath_size) {
-      cachepath_size = strlen(cachepath);
+      cachepath_size = strlen(chosen_realized_cachepath);
    }
    if (!orig_cachepath_size) {
-      orig_cachepath_size = strlen(orig_cachepath);
+      orig_cachepath_size = strlen(chosen_parsed_cachepath);
    }
-   return ((strncmp(pathname, cachepath, cachepath_size) == 0) ||
-           (strncmp(pathname, orig_cachepath, orig_cachepath_size) == 0));
+   return ((strncmp(pathname, chosen_realized_cachepath, cachepath_size) == 0) ||
+           (strncmp(pathname, chosen_parsed_cachepath, orig_cachepath_size) == 0));
 }
 
 extern int is_local_prefix(const char *path, char **cached_local_prefixes);
diff --git a/src/client/client/should_intercept.h b/src/client/client/should_intercept.h
index 6a545913..f6a9b510 100644
--- a/src/client/client/should_intercept.h
+++ b/src/client/client/should_intercept.h
@@ -27,7 +27,6 @@
 #define EXCL_OPEN 2
 #define ERR_CALL 3
 
-void set_should_intercept_cachepath( char *chosen_realized_cachepath, char *chosen_parsed_cachepath, char *chosen_symbolic_cachepath );
 int open_filter(const char *fname, int flags);
 int fopen_filter(const char *fname, const char *flags);
 int exec_filter(const char *fname);

From 78ee5ff02f3f902b26a3655a78f2a89d8282c5d7 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Wed, 22 Oct 2025 14:39:23 -0700
Subject: [PATCH 13/57] Comments the cachepath variables.

---
 .../auditserver/ldcs_audit_server_process.h       | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/server/auditserver/ldcs_audit_server_process.h b/src/server/auditserver/ldcs_audit_server_process.h
index 9ba1675e..f9f4c30c 100644
--- a/src/server/auditserver/ldcs_audit_server_process.h
+++ b/src/server/auditserver/ldcs_audit_server_process.h
@@ -125,12 +125,15 @@ struct ldcs_process_data_struct
   int exit_readys_recvd;
   ldcs_dist_model_t dist_model;
   ldcs_client_t* client_table;
-  char *location;
-  char *cachepaths;
-  char *cachepath;
-  char *symbolic_cachepath;
-  char *parsed_cachepath;
-  int64_t cachepath_bitidx;
+  char *location;               /* Single user-specified path for fifo, daemons, etc. */
+                                /*   (Everything except the cachepath.) */
+  char *cachepaths;             /* Up to 64 colon-separated list of candidate cachepaths. */
+  char *cachepath;              /* The earliest path in the list available to all servers. */
+                                /*   (Environment variables replaced, symbolic links realized.) */
+  char *symbolic_cachepath;     /* The original representation of the cachepath. */
+  char *parsed_cachepath;       /* The cachepath with environment variables replaced. */
+                                /*   (Symbolic links, if any, remain.) */
+  int64_t cachepath_bitidx;     /* Bit index of valid cachepaths on a given server. */
   char *hostname;
   char *pythonprefix;
   char *localprefix;

From f9aa472a55bb411c796107043a26d8abca5997e2 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 23 Oct 2025 13:07:21 -0700
Subject: [PATCH 14/57] Removes internal vars from spindle_launch.h

Removes chosen_cachepath and cachepath_bitindex from
  spindle_launch.h

Updates initialization of matching variables in ldcs_process_data.

determineValidCachePaths() moved from spindle_be.cc to
  ldcs_audit_server_process.c to get ldcs_process_data visibility.

Added #include "parseloc.h" to ldcs_audit_server_process.c to get
  declaration of determineValidCachePaths().

Relocated "parseloc.h" to src/util so ldcs_audit_server_process.c
  could find it.

Trued up signedness of types caused my making "parseloc.h" more
  visible, e.g., cachepath_bitidx is now uint64_t everywhere.
---
 src/cobo/cobo.c                                    | 2 +-
 src/cobo/ldcs_cobo.h                               | 2 +-
 src/include/spindle_launch.h                       | 2 --
 src/server/auditserver/ldcs_audit_server_process.c | 9 +++++++--
 src/server/auditserver/ldcs_audit_server_process.h | 2 +-
 src/server/startup/spindle_be.cc                   | 2 --
 src/{client/beboot => utils}/parseloc.h            | 2 +-
 7 files changed, 11 insertions(+), 10 deletions(-)
 rename src/{client/beboot => utils}/parseloc.h (91%)

diff --git a/src/cobo/cobo.c b/src/cobo/cobo.c
index ad2ea878..59c2f809 100644
--- a/src/cobo/cobo.c
+++ b/src/cobo/cobo.c
@@ -1434,7 +1434,7 @@ int cobo_allgather_str(char* sendstr, char*** recvstr, char** recvbuf)
     return COBO_SUCCESS;
 }
 
-int cobo_allreduce( int64_t *pval, cobo_op_t op ){
+int cobo_allreduce( uint64_t *pval, cobo_op_t op ){
 
     /* if i have any children, receive their data */
     int64_t child_val;
diff --git a/src/cobo/ldcs_cobo.h b/src/cobo/ldcs_cobo.h
index 30cc9673..fafbda6a 100644
--- a/src/cobo/ldcs_cobo.h
+++ b/src/cobo/ldcs_cobo.h
@@ -142,7 +142,7 @@ int cobo_alltoall (void* sendbuf, int sendcount, void* recvbuf);
  */
 int cobo_allgather_str(char* sendstr, char*** recvstr, char** recvbuf);
 
-int cobo_allreduce(int64_t *pval, cobo_op_t op);
+int cobo_allreduce(uint64_t *pval, cobo_op_t op);
 
 /*
  * ==========================================================================
diff --git a/src/include/spindle_launch.h b/src/include/spindle_launch.h
index 5476734d..81c0728e 100644
--- a/src/include/spindle_launch.h
+++ b/src/include/spindle_launch.h
@@ -128,8 +128,6 @@ typedef struct {
 
    /* Path[s] for cached libraries. */
    char *candidate_cachepaths;  /* Colon-separated list of candidate paths (max 64) */
-   char *chosen_cachepath;      /* The consensus path (same across all nodes). */
-   uint64_t cachepath_bitidx;   /* Bit index used by allReduce() to arrive at consensus. */
 
    /* Colon-seperated list of directories where Python is installed */
    char *pythonprefix;
diff --git a/src/server/auditserver/ldcs_audit_server_process.c b/src/server/auditserver/ldcs_audit_server_process.c
index dca91c20..566cad01 100644
--- a/src/server/auditserver/ldcs_audit_server_process.c
+++ b/src/server/auditserver/ldcs_audit_server_process.c
@@ -37,6 +37,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 #include "msgbundle.h"
 #include "exitnote.h"
 #include "cleanup_proc.h"
+#include "parseloc.h"
 
 //#define GPERFTOOLS
 #if defined(GPERFTOOLS)
@@ -143,8 +144,8 @@ int ldcs_audit_server_process(spindle_args_t *args)
    debug_printf3("Initializing server data structures\n");
    ldcs_process_data.location = args->location;
    ldcs_process_data.cachepaths = args->candidate_cachepaths;
-   ldcs_process_data.cachepath = args->chosen_cachepath;
-   ldcs_process_data.cachepath_bitidx = args->cachepath_bitidx;
+   ldcs_process_data.cachepath = NULL;
+   ldcs_process_data.cachepath_bitidx = 0;
    ldcs_process_data.number = args->number;
    ldcs_process_data.pythonprefix = args->pythonprefix;
    ldcs_process_data.localprefix = args->local_prefixes;
@@ -230,6 +231,10 @@ int ldcs_audit_server_process(spindle_args_t *args)
    if (fd != -1) {
       ldcs_listen_register_fd(fd, serverid, forceExitCB, (void *) &ldcs_process_data);
    }
+   determineValidCachePaths(
+           &ldcs_process_data.cachepath_bitidx,
+           ldcs_process_data.cachepaths,
+           ldcs_process_data.number );
    return 0;
 }  
 
diff --git a/src/server/auditserver/ldcs_audit_server_process.h b/src/server/auditserver/ldcs_audit_server_process.h
index f9f4c30c..1495cebd 100644
--- a/src/server/auditserver/ldcs_audit_server_process.h
+++ b/src/server/auditserver/ldcs_audit_server_process.h
@@ -133,7 +133,7 @@ struct ldcs_process_data_struct
   char *symbolic_cachepath;     /* The original representation of the cachepath. */
   char *parsed_cachepath;       /* The cachepath with environment variables replaced. */
                                 /*   (Symbolic links, if any, remain.) */
-  int64_t cachepath_bitidx;     /* Bit index of valid cachepaths on a given server. */
+  uint64_t cachepath_bitidx;    /* Bit index of valid cachepaths on a given server. */
   char *hostname;
   char *pythonprefix;
   char *localprefix;
diff --git a/src/server/startup/spindle_be.cc b/src/server/startup/spindle_be.cc
index 4f583756..733d3244 100644
--- a/src/server/startup/spindle_be.cc
+++ b/src/server/startup/spindle_be.cc
@@ -154,8 +154,6 @@ int spindleRunBE(unsigned int port, unsigned int num_ports, unique_id_t unique_i
    free(args.location);
    args.location = new_location;
 
-   determineValidCachePaths( &args.cachepath_bitidx, args.candidate_cachepaths, args.number);
-
    result = ldcs_audit_server_process(&args);
    if (result == -1) {
       err_printf("Error in ldcs_audit_server_process\n");
diff --git a/src/client/beboot/parseloc.h b/src/utils/parseloc.h
similarity index 91%
rename from src/client/beboot/parseloc.h
rename to src/utils/parseloc.h
index 1731906a..a99409c3 100644
--- a/src/client/beboot/parseloc.h
+++ b/src/utils/parseloc.h
@@ -28,7 +28,7 @@ char *parse_location_noerr(char *loc, number_t number);
 char *realize(char *path);
 char **parse_colonsep_prefixes(char *colonsep_list, number_t number);
 int is_local_prefix(const char *path, char **local_prefixes);
-static int validateCandidatePath( char *candidatePath, char **realizedPath, char **parsedPath, char **symbolicPath, number_t number );
+int validateCandidatePath( char *candidatePath, char **realizedPath, char **parsedPath, char **symbolicPath, number_t number );
 void determineValidCachePaths( uint64_t *validBitIdx, char *origPathList, number_t number );
 void getValidCachePathByIndex( uint64_t validBitIdx, char **realizedCachePath, char **parsedCachePath, char **symbolicCachePath );
 

From ef75381f8ef4f51ef7d4547d7e35a7a43dba0e20 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 23 Oct 2025 14:18:59 -0700
Subject: [PATCH 15/57] Client cachepath message now uses single response.

The three-message-reply response is now a single message with
two strings.  The symbolic version of the cachepath is no longer
communicated as it was not being used.
---
 src/client/client/client.c                    |  2 +-
 src/client/client_comlib/client_api.c         | 31 ++++---------------
 src/client/client_comlib/client_api.h         |  2 +-
 .../auditserver/ldcs_audit_server_handlers.c  | 19 +++---------
 4 files changed, 13 insertions(+), 41 deletions(-)

diff --git a/src/client/client/client.c b/src/client/client/client.c
index f803e75b..ac8d4b63 100644
--- a/src/client/client/client.c
+++ b/src/client/client/client.c
@@ -261,7 +261,7 @@ static int init_server_connection()
          send_cpu(ldcsid, get_cur_cpu());
 #endif
    }
-   send_cachepath_query( ldcsid, &chosen_realized_cachepath, &chosen_parsed_cachepath, NULL);
+   send_cachepath_query( ldcsid, &chosen_realized_cachepath, &chosen_parsed_cachepath );
    snprintf(debugging_name, 32, "Client.%d", rankinfo[0]);
    LOGGING_INIT(debugging_name);
 
diff --git a/src/client/client_comlib/client_api.c b/src/client/client_comlib/client_api.c
index 267b8a93..cdff2bb5 100644
--- a/src/client/client_comlib/client_api.c
+++ b/src/client/client_comlib/client_api.c
@@ -36,9 +36,9 @@ static struct lock_t comm_lock;
 
 #define COMM_LOCK do { if (lock(&comm_lock) == -1) return -1; } while (0)
 #define COMM_UNLOCK unlock(&comm_lock)
-   
 
-int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chosen_parsed_cachepath, char **chosen_symbolic_cachepath ){
+
+int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chosen_parsed_cachepath){
    ldcs_message_t message;
    char buffer[MAX_PATH_LEN+1];
    buffer[MAX_PATH_LEN] = '\0';
@@ -59,32 +59,13 @@ int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chose
       err_printf("Got unexpected message of type %d\n", (int) message.header.type);
       assert(0);
    }
+   char *local_crc = strdup( buffer );
+   char *local_cpc = strdup( &buffer[ strlen(local_crc) + 1 ] );
    if( chosen_realized_cachepath ){
-       *chosen_realized_cachepath = strdup( buffer );
-   }
-
-   COMM_LOCK;
-   client_recv_msg_static(fd, &message, LDCS_READ_BLOCK);
-   COMM_UNLOCK;
-
-   if (message.header.type != LDCS_MSG_CHOSEN_CACHEPATH || message.header.len > MAX_PATH_LEN) {
-      err_printf("Got unexpected message of type %d\n", (int) message.header.type);
-      assert(0);
+       *chosen_realized_cachepath = local_crc;
    }
    if( chosen_parsed_cachepath ){
-       *chosen_parsed_cachepath = strdup( buffer );
-   }
-
-   COMM_LOCK;
-   client_recv_msg_static(fd, &message, LDCS_READ_BLOCK);
-   COMM_UNLOCK;
-
-   if (message.header.type != LDCS_MSG_CHOSEN_CACHEPATH || message.header.len > MAX_PATH_LEN) {
-      err_printf("Got unexpected message of type %d\n", (int) message.header.type);
-      assert(0);
-   }
-   if( chosen_symbolic_cachepath ){
-       *chosen_symbolic_cachepath = strdup( buffer );
+       *chosen_parsed_cachepath = local_cpc;
    }
 
    return 0;
diff --git a/src/client/client_comlib/client_api.h b/src/client/client_comlib/client_api.h
index 982c4b1c..3d7c41be 100644
--- a/src/client/client_comlib/client_api.h
+++ b/src/client/client_comlib/client_api.h
@@ -42,7 +42,7 @@ int send_orig_path_request(int fd, const char *path, char *newpath);
 int send_dirlists_request(int fd, char **local_result, char **exece_result, char **to_free);
 int send_procmaps_query(int fd, int pid, char *result);
 int send_pickone_query(int fd, char *key, int *result);
-int send_cachepath_query( int fd, char **chosen_symbolic_cachepath, char **chosen_parsed_cachepath, char **chosen_realized_cachepath );
+int send_cachepath_query( int fd, char **chosen_symbolic_cachepath, char **chosen_parsed_cachepath);
 
 int get_python_prefix(int fd, char **prefix);
 
diff --git a/src/server/auditserver/ldcs_audit_server_handlers.c b/src/server/auditserver/ldcs_audit_server_handlers.c
index 98c6807a..c8e76d54 100644
--- a/src/server/auditserver/ldcs_audit_server_handlers.c
+++ b/src/server/auditserver/ldcs_audit_server_handlers.c
@@ -3003,21 +3003,12 @@ static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc
 
    msg.header.type = LDCS_MSG_CHOSEN_CACHEPATH;
 
-   msg.header.len = strlen(procdata->cachepath) + 1;
-   msg.data = procdata->cachepath;
-   ldcs_send_msg(connid, &msg);
-   procdata->server_stat.clientmsg.cnt++;
-   procdata->server_stat.clientmsg.time += ldcs_get_time() - client->query_arrival_time;
-
-   msg.header.len = strlen(procdata->parsed_cachepath) + 1;
-   msg.data = procdata->parsed_cachepath;
-   ldcs_send_msg(connid, &msg);
-   procdata->server_stat.clientmsg.cnt++;
-   procdata->server_stat.clientmsg.time += ldcs_get_time() - client->query_arrival_time;
-
-   msg.header.len = strlen(procdata->symbolic_cachepath) + 1;
-   msg.data = procdata->symbolic_cachepath;
+   msg.header.len = strlen(procdata->cachepath) + 1 + strlen(procdata->parsed_cachepath) + 1;
+   msg.data = calloc( 1, msg.header.len );
+   strcpy( msg.data, procdata->cachepath );
+   strcpy( &msg.data[ strlen(procdata->cachepath)+1 ], procdata->parsed_cachepath );
    ldcs_send_msg(connid, &msg);
+   free( msg.data );
    procdata->server_stat.clientmsg.cnt++;
    procdata->server_stat.clientmsg.time += ldcs_get_time() - client->query_arrival_time;
 

From 3c83f6c4af9caa00c73efaffd256dc3294d336ee Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 23 Oct 2025 14:33:07 -0700
Subject: [PATCH 16/57] Removes assert(0) in network error paths.

---
 src/client/client_comlib/client_api.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/client/client_comlib/client_api.c b/src/client/client_comlib/client_api.c
index cdff2bb5..4b0f9ded 100644
--- a/src/client/client_comlib/client_api.c
+++ b/src/client/client_comlib/client_api.c
@@ -57,7 +57,7 @@ int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chose
 
    if (message.header.type != LDCS_MSG_CHOSEN_CACHEPATH || message.header.len > MAX_PATH_LEN) {
       err_printf("Got unexpected message of type %d\n", (int) message.header.type);
-      assert(0);
+      return -1;
    }
    char *local_crc = strdup( buffer );
    char *local_cpc = strdup( &buffer[ strlen(local_crc) + 1 ] );
@@ -102,7 +102,7 @@ int send_file_query(int fd, char* path, int dso, char** newpath, int *errcode) {
 
    if (message.header.type != LDCS_MSG_FILE_QUERY_ANSWER) {
       err_printf("Got unexpected message of type %d\n", (int) message.header.type);
-      assert(0);
+      return -1;
    }
    
    if (message.header.len > sizeof(int)) {
@@ -195,7 +195,7 @@ int send_existance_test(int fd, char *path, int *exists)
 
    if (message.header.type != LDCS_MSG_EXISTS_ANSWER || message.header.len != sizeof(uint32_t)) {
       err_printf("Got unexpected message after existance test: %d\n", (int) message.header.type);
-      assert(0);
+      return -1;
    }
 
    memcpy(exists, buffer, sizeof(*exists));
@@ -232,7 +232,7 @@ int send_orig_path_request(int fd, const char *path, char *newpath)
 
    if (message.header.type != LDCS_MSG_ORIGPATH_ANSWER || message.header.len > MAX_PATH_LEN) {
       err_printf("Got unexpected message after existance test: %d\n", (int) message.header.type);
-      assert(0);
+      return -1;
    }
    strncpy(newpath, buffer, MAX_PATH_LEN+1);
 
@@ -380,7 +380,7 @@ int send_ldso_info_request(int fd, const char *ldso_path, char *result_path)
 
    if (message.header.type != LDCS_MSG_LOADER_DATA_RESP) {
       err_printf("Got unexpected message after ldso req: %d\n", (int) message.header.type);
-      assert(0);
+      return -1;
    }
    return 0;
 }
@@ -422,7 +422,7 @@ int send_rankinfo_query(int fd, int *mylrank, int *mylsize, int *mymdrank, int *
    if (message.header.type != LDCS_MSG_MYRANKINFO_QUERY_ANSWER || message.header.len != 4*sizeof(int)) {
       err_printf("Received incorrect response to rankinfo query %d\n", message.header.type);
       *mylrank = *mylsize = *mymdrank = *mymdsize = -1;
-      assert(0);
+      return -1;
    }
 
    p = (int *) message.data;
@@ -457,7 +457,7 @@ int send_procmaps_query(int fd, int pid, char *result)
 
    if (message.header.type != LDCS_MSG_PROCMAPS_RESP) {
       err_printf("Received incorrect response to procmaps query %d\n", message.header.type);
-      assert(0);
+      return -1;
    }
 
    memcpy(result, buffer, MAX_PATH_LEN);
@@ -488,7 +488,7 @@ int send_pickone_query(int fd, char *key, int *result)
 
    if (message.header.type != LDCS_MSG_PICKONE_RESP) {
       err_printf("Received incorrect response to procmaps query %d\n", message.header.type);
-      assert(0);
+      return -1;
    }
 
    *result = *((int *) message.data);

From 1134a1c47072b13cf19e036c0d80323b674898e9 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 23 Oct 2025 15:18:40 -0700
Subject: [PATCH 17/57] Renames ldcs_audit_server_md_consensus().

New name is ldcs_audit_server_md_allreduce_AND().

If we get to the point where we're using other allreduce operations
we can solve the problem of duplicating the op list in md-land and
cobo-land.  For now, we're only using one op in md-land, so the
op can go into the function name.
---
 src/server/auditserver/ldcs_audit_server_handlers.c | 2 +-
 src/server/auditserver/ldcs_audit_server_md.h       | 2 +-
 src/server/auditserver/ldcs_audit_server_md_cobo.c  | 6 ++----
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/server/auditserver/ldcs_audit_server_handlers.c b/src/server/auditserver/ldcs_audit_server_handlers.c
index c8e76d54..29107bad 100644
--- a/src/server/auditserver/ldcs_audit_server_handlers.c
+++ b/src/server/auditserver/ldcs_audit_server_handlers.c
@@ -2966,7 +2966,7 @@ static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_messag
         msgbundle_force_flush(procdata);
     }
 
-    ldcs_audit_server_md_consensus(procdata, msg);
+    ldcs_audit_server_md_allreduce_AND( &procdata->cachepath_bitidx );
 
     if( procdata->cachepath_bitidx == 0 ){
        err_printf("No valid cachepath path available.  Falling back to \"location\" path (%s).\n", procdata->location);
diff --git a/src/server/auditserver/ldcs_audit_server_md.h b/src/server/auditserver/ldcs_audit_server_md.h
index ba7943e2..a4640370 100644
--- a/src/server/auditserver/ldcs_audit_server_md.h
+++ b/src/server/auditserver/ldcs_audit_server_md.h
@@ -108,7 +108,7 @@ int ldcs_audit_server_md_get_num_children(ldcs_process_data_t *procdata);
 
 int ldcs_audit_server_md_is_parent(node_peer_t peer);
 
-void ldcs_audit_server_md_consensus(ldcs_process_data_t *ldcs_process_data, ldcs_message_t *msg);
+void ldcs_audit_server_md_allreduce_AND( uint64_t *val );
 
    
 #if defined(__cplusplus)
diff --git a/src/server/auditserver/ldcs_audit_server_md_cobo.c b/src/server/auditserver/ldcs_audit_server_md_cobo.c
index d8b5442f..27393a55 100644
--- a/src/server/auditserver/ldcs_audit_server_md_cobo.c
+++ b/src/server/auditserver/ldcs_audit_server_md_cobo.c
@@ -402,8 +402,6 @@ int ldcs_audit_server_md_get_num_children(ldcs_process_data_t *procdata)
    return num_childs;
 }
 
-void ldcs_audit_server_md_consensus(ldcs_process_data_t *ldcs_process_data, ldcs_message_t *msg){
-    if( msg->header.type == LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS ){
-        cobo_allreduce( &ldcs_process_data->cachepath_bitidx, COBO_OP_BITWISE_AND );
-    }
+void ldcs_audit_server_md_allreduce_AND( uint64_t *val ){
+    cobo_allreduce( val, COBO_OP_BITWISE_AND );
 }

From 57eb3fe752797f39a1f7da3721385eee210dd1d5 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 23 Oct 2025 15:27:27 -0700
Subject: [PATCH 18/57] Adds explicit enum values to CmdlineShortOptions.

---
 src/fe/startup/config_mgr.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/fe/startup/config_mgr.h b/src/fe/startup/config_mgr.h
index 27be1ae8..c2d3cd7e 100644
--- a/src/fe/startup/config_mgr.h
+++ b/src/fe/startup/config_mgr.h
@@ -126,8 +126,8 @@ enum CmdlineShortOptions {
    shortSpindleLevel = 296,
    shortLocalPrefix = 297,
    shortExecExcludes = 298,
-   shortPatchLdso,
-   shortCachePaths,
+   shortPatchLdso = 299,
+   shortCachePaths = 300,
 };
 
 enum CmdlineGroups {

From 4a438758f4d8e51c0e4089829547f4ad95f39da3 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 23 Oct 2025 15:38:19 -0700
Subject: [PATCH 19/57] Return instead of exit on network errors.

---
 src/cobo/cobo.c | 58 ++++++++++++++++++++++++-------------------------
 1 file changed, 29 insertions(+), 29 deletions(-)

diff --git a/src/cobo/cobo.c b/src/cobo/cobo.c
index 59c2f809..65741008 100644
--- a/src/cobo/cobo.c
+++ b/src/cobo/cobo.c
@@ -160,7 +160,7 @@ static char* cobo_getenv(char* envvar, int type)
     char* str = getenv(envvar);
     if (str == NULL && type == ENV_REQUIRED) {
         err_printf("Missing required environment variable: %s\n", envvar);
-        exit(1);
+        return NULL;
     }
     return str;
 }
@@ -171,7 +171,7 @@ static void* cobo_malloc(size_t n, char* msg)
     void* p = malloc(n);
     if (!p) {
         err_printf("Call to malloc(%lu) failed: %s (%m errno %d)\n", n, msg, errno);
-        exit(1);
+        return NULL;
     }
     return p;
 }
@@ -513,7 +513,7 @@ static int cobo_connect_hostname(char* hostname, int rank)
                       break;
                    case HSHAKE_INTERNAL_ERROR:
                       err_printf("Internal error doing handshake: %s", spindle_handshake_last_error_str());
-                      exit(-1);
+                      return -1;
                       break;
                    case HSHAKE_DROP_CONNECTION:
                       debug_printf3("Handshake said to drop connection\n");
@@ -768,7 +768,7 @@ static int cobo_open_tree()
     if (sockfd < 0) {
         err_printf("Creating parent socket (socket() %m errno=%d)\n",
                    errno);
-        exit(1);
+        return -1;
     }
     setCloseOnExec(sockfd);
 
@@ -817,7 +817,7 @@ static int cobo_open_tree()
     if (!port_is_bound) {
         /* TODO: would like to send an abort back to server */
         err_printf("Failed to open socket on any port\n");
-        exit(1);
+        return -1;
     }
 
     /* accept a connection from parent and receive socket table */
@@ -837,7 +837,7 @@ static int cobo_open_tree()
               break;
            case HSHAKE_INTERNAL_ERROR:
               err_printf("Internal error doing handshake: %s", spindle_handshake_last_error_str());
-              exit(-1);
+              return -1;
               break;
            case HSHAKE_DROP_CONNECTION:
               debug_printf3("Handshake said to drop connection\n");
@@ -907,26 +907,26 @@ static int cobo_open_tree()
     /* read our rank number */
     if (cobo_read_fd(cobo_parent_fd, &cobo_me, sizeof(int)) < 0) {
         err_printf("Receiving my rank from parent failed\n");
-        exit(1);
+        return -1;
     }
 
     /* discover how many ranks are in our world */
     if (cobo_read_fd(cobo_parent_fd, &cobo_nprocs, sizeof(int)) < 0) {
         err_printf("Receiving number of tasks from parent failed\n");
-        exit(1);
+        return -1;
     }
 
     /* read the size of the hostlist (in bytes) */
     if (cobo_read_fd(cobo_parent_fd, &cobo_hostlist_size, sizeof(int)) < 0) {
         err_printf("Receiving size of hostname table from parent failed\n");
-        exit(1);
+        return -1;
     }
 
     /* allocate space for the hostlist and read it in */
     cobo_hostlist = (void*) cobo_malloc(cobo_hostlist_size, "Hostlist data buffer");
     if (cobo_read_fd(cobo_parent_fd, cobo_hostlist, cobo_hostlist_size) < 0) {
         err_printf("Receiving hostname table from parent failed\n");
-        exit(1);
+        return -1;
     }
 
 /*
@@ -969,7 +969,7 @@ static int cobo_open_tree()
         if (cobo_child_fd[i] == -1) {
             err_printf("Failed to connect to child (rank %d) on %s failed\n",
                        c, child_hostname);
-            exit(1);
+            return -1;
         }
 
         /* tell child what rank he is and forward the hostname table to him */
@@ -978,7 +978,7 @@ static int cobo_open_tree()
         if (forward != COBO_SUCCESS) {
             err_printf("Failed to forward hostname table to child (rank %d) on %s failed\n",
                        c, child_hostname);
-            exit(1);
+            return -1;
         }
 
         /* free the child hostname string */
@@ -1033,7 +1033,7 @@ static int cobo_bcast_tree(void* buf, int size)
     if (cobo_me != 0) {
         if (cobo_read_fd(cobo_parent_fd, buf, size) < 0) {
             err_printf("Receiving broadcast data from parent failed\n");
-            exit(1);
+            return -1;
         }
     }
 
@@ -1042,7 +1042,7 @@ static int cobo_bcast_tree(void* buf, int size)
         if (cobo_write_fd(cobo_child_fd[i], buf, size) < 0) {
             err_printf("Broadcasting data to child (rank %d) failed\n",
                        cobo_child[i]);
-            exit(1);
+            return -1;
         }
     }
 
@@ -1058,7 +1058,7 @@ int cobo_bcast_down(void* buf, int size)
       if (cobo_write_fd(cobo_child_fd[i], buf, size) < 0) {
          err_printf("Broadcasting data to child (rank %d) failed\n",
                     cobo_child[i]);
-         exit(1);
+         return -1;
       }
    }   
    return rc;
@@ -1080,7 +1080,7 @@ static int cobo_allreduce_max_int_tree(int* sendbuf, int* recvbuf)
         if (cobo_read_fd(cobo_child_fd[i], &child_val, sizeof(child_val)) < 0) {
             err_printf("Reducing data from child (rank %d) failed\n",
                        cobo_child[i]);
-            exit(1);
+            return -1;
         }
 
         /* compare child's max to our current max */
@@ -1094,7 +1094,7 @@ static int cobo_allreduce_max_int_tree(int* sendbuf, int* recvbuf)
         /* not the root, so forward our reduction result to our parent */
         if (cobo_write_fd(cobo_parent_fd, &max_val, sizeof(max_val)) < 0) {
             err_printf("Sending reduced data to parent failed\n");
-            exit(1);
+            return -1;
         }
     } else {
         /* we're the root, got the result, set the recvbuf */
@@ -1129,7 +1129,7 @@ static int cobo_gather_tree(void* sendbuf, int sendcount, void* recvbuf)
         if (cobo_read_fd(cobo_child_fd[i], (char*)bigbuf + offset, sendcount * cobo_child_incl[i]) < 0) {
             err_printf("Gathering data from child (rank %d) failed\n",
                        cobo_child[i]);
-            exit(1);
+            return -1;
         }
         offset += sendcount * cobo_child_incl[i];
     }
@@ -1138,7 +1138,7 @@ static int cobo_gather_tree(void* sendbuf, int sendcount, void* recvbuf)
     if (cobo_me != 0) {
         if (cobo_write_fd(cobo_parent_fd, bigbuf, bigcount) < 0) {
             err_printf("Sending gathered data to parent failed\n");
-            exit(1);
+            return -1;
         }
         cobo_free(bigbuf);
     }
@@ -1158,7 +1158,7 @@ static int cobo_scatter_tree(void* sendbuf, int sendcount, void* recvbuf)
         bigbuf = (void*) cobo_malloc(bigcount, "Temporary scatter buffer in cobo_scatter_tree");
         if (cobo_read_fd(cobo_parent_fd, bigbuf, bigcount) < 0) {
             err_printf("Receiving scatter data from parent failed\n");
-            exit(1);
+            return -1;
         }
     }
 
@@ -1169,7 +1169,7 @@ static int cobo_scatter_tree(void* sendbuf, int sendcount, void* recvbuf)
         if (cobo_write_fd(cobo_child_fd[i], (char*)bigbuf + offset, sendcount * cobo_child_incl[i]) < 0) {
             err_printf("Scattering data to child (rank %d) failed\n",
                        cobo_child[i]);
-            exit(1);
+            return -1;
         }
         offset += sendcount * cobo_child_incl[i];
     }
@@ -1257,7 +1257,7 @@ int cobo_bcast(void* buf, int sendcount, int root)
         rc = cobo_bcast_tree(buf, sendcount);
     } else {
         err_printf("Cannot execute bcast from non-zero root\n");
-        exit(1);
+        return -1;
     }
 
     cobo_gettimeofday(&end);
@@ -1283,7 +1283,7 @@ int cobo_gather(void* sendbuf, int sendcount, void* recvbuf, int root)
         rc = cobo_gather_tree(sendbuf, sendcount, recvbuf);
     } else {
         err_printf("Cannot execute gather to non-zero root\n");
-        exit(1);
+        return -1;
     }
 
     cobo_gettimeofday(&end);
@@ -1309,7 +1309,7 @@ int cobo_scatter(void* sendbuf, int sendcount, void* recvbuf, int root)
         rc = cobo_scatter_tree(sendbuf, sendcount, recvbuf);
     } else {
         err_printf("Cannot execute scatter from non-zero root\n");
-        exit(1);
+        return -1;
     }
 
     cobo_gettimeofday(&end);
@@ -1354,7 +1354,7 @@ int cobo_alltoall(void* sendbuf, int sendcount, void* recvbuf)
     int rc = COBO_SUCCESS;
 
     err_printf("Cannot execute alltoall\n");
-    exit(1);
+    return -1;
 
     cobo_gettimeofday(&end);
     debug_printf3("Exiting cobo_alltoall(), took %f seconds for %d procs\n", cobo_getsecs(&end,&start), cobo_nprocs);
@@ -1442,7 +1442,7 @@ int cobo_allreduce( uint64_t *pval, cobo_op_t op ){
         /* read int64_t from child */
         if (cobo_read_fd(cobo_child_fd[i], &child_val, sizeof(int64_t)) < 0) {
             err_printf("Reducing data from child (rank %d) failed\n", cobo_child[i]);
-            exit(1);
+            return -1;
         }
 
         /* compare child's val to our current val */
@@ -1466,7 +1466,7 @@ int cobo_allreduce( uint64_t *pval, cobo_op_t op ){
         /* not the root, so forward our reduction result to our parent */
         if (cobo_write_fd(cobo_parent_fd, pval, sizeof(*pval)) < 0) {
             err_printf("Sending reduced data to parent failed\n");
-            exit(1);
+            return -1;
         }
     }
 
@@ -1524,7 +1524,7 @@ int cobo_open(uint64_t sessionid, int* portlist, int num_ports, int* rank, int*
     cobo_ports = cobo_int_dup(portlist, num_ports);
     if (cobo_ports == NULL) {
         err_printf("Failed to copy port list\n");
-        exit(1);
+        return -1;
     }
 
     /* open the tree */
@@ -1533,7 +1533,7 @@ int cobo_open(uint64_t sessionid, int* portlist, int num_ports, int* rank, int*
     /* need to check that tree opened successfully before returning, so do a barrier */
     if (cobo_barrier() != COBO_SUCCESS) {
         err_printf("Failed to open tree\n");
-        exit(1);
+        return -1;
     }
 
     if (cobo_me == 0) {

From 97ae725c4c204865ab5055b5deef4d50206f0945 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 24 Oct 2025 06:54:47 -0700
Subject: [PATCH 20/57] Apply rename to configuration and parameters.

---
 config.h.in                  |  6 +++---
 configure                    | 17 +++++++++--------
 configure.common.ac          | 12 ++++++------
 src/client/config.h.in       |  6 +++---
 src/client/configure         | 17 +++++++++--------
 src/fe/config.h.in           |  6 +++---
 src/fe/configure             | 17 +++++++++--------
 src/fe/startup/config_mgr.cc | 16 ++++++++--------
 src/fe/startup/config_mgr.h  |  4 ++--
 src/flux/sessionmgr.c        |  8 ++++----
 src/server/config.h.in       |  6 +++---
 src/server/configure         | 17 +++++++++--------
 12 files changed, 68 insertions(+), 64 deletions(-)

diff --git a/config.h.in b/config.h.in
index 6be0335d..f85aef75 100644
--- a/config.h.in
+++ b/config.h.in
@@ -9,6 +9,9 @@
 /* Colon-separated list of potential back-end cache directories */
 #undef CACHEPATHS
 
+/* Back-end directory for communication and housekeeping */
+#undef COMMPATH
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 
@@ -130,9 +133,6 @@
 /* Default mode for slurm launch */
 #undef SLURMLAUNCH_ENABLED
 
-/* The default local directory for Spindle */
-#undef SPINDLE_LOC
-
 /* The default colon-separated list of directories that Spindle will not cache
    files out of */
 #undef SPINDLE_LOCAL_PREFIX
diff --git a/configure b/configure
index c9e517c8..897d9d5f 100755
--- a/configure
+++ b/configure
@@ -847,7 +847,7 @@ enable_maintainer_mode
 with_default_port
 with_default_num_ports
 with_cachepaths
-with_localstorage
+with_commpath
 with_default_local_prefix
 with_testrm
 with_rm
@@ -1592,7 +1592,8 @@ Optional Packages:
                           communication
   --with-cachepaths=DIR   Colon-separated list of potential back-end cache
                           directories
-  --with-localstorage=DIR Directory on back-ends for storing relocated files
+  --with-compath=DIR      Back-end directory for communication and
+                          housekeeping
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
                           will not cache files out of
@@ -16626,11 +16627,11 @@ else
 fi
 
 
-# Check whether --with-localstorage was given.
-if test "${with_localstorage+set}" = set; then :
-  withval=$with_localstorage; SPINDLE_LOC=${withval}
+# Check whether --with-commpath was given.
+if test "${with_commpath+set}" = set; then :
+  withval=$with_commpath; COMMPATH=${withval}
 else
-  SPINDLE_LOC=$DEFAULT_LOC
+  COMMPATH=$DEFAULT_LOC
 fi
 
 
@@ -16638,7 +16639,7 @@ fi
 if test "${with_default_local_prefix+set}" = set; then :
   withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval}
 else
-  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$SPINDLE_LOC"
+  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH"
 fi
 
 
@@ -16658,7 +16659,7 @@ _ACEOF
 
 
 cat >>confdefs.h <<_ACEOF
-#define SPINDLE_LOC "$SPINDLE_LOC"
+#define COMMPATH "$COMMPATH"
 _ACEOF
 
 
diff --git a/configure.common.ac b/configure.common.ac
index 31a04d4d..b25bc3af 100644
--- a/configure.common.ac
+++ b/configure.common.ac
@@ -21,18 +21,18 @@ AC_ARG_WITH(cachepaths,
             [AS_HELP_STRING([--with-cachepaths=DIR],[Colon-separated list of potential back-end cache directories])],
             [CACHEPATHS=${withval}],
             [CACHEPATHS=$DEFAULT_LOC])
-AC_ARG_WITH(localstorage,
-            [AS_HELP_STRING([--with-localstorage=DIR],[Directory on back-ends for storing relocated files])],
-            [SPINDLE_LOC=${withval}],
-            [SPINDLE_LOC=$DEFAULT_LOC])
+AC_ARG_WITH(commpath,
+            [AS_HELP_STRING([--with-compath=DIR],[Back-end directory for communication and housekeeping])],
+            [COMMPATH=${withval}],
+            [COMMPATH=$DEFAULT_LOC])
 AC_ARG_WITH(default-local-prefix,
             [AS_HELP_STRING([--with-default-local-prefix=DIRS],[Colon-seperated list of directories that Spindle will not cache files out of])],
             [SPINDLE_LOCAL_PREFIX=${withval}],
-            [SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$SPINDLE_LOC"])
+            [SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH"])
 AC_DEFINE_UNQUOTED([SPINDLE_PORT],[$SPINDLE_PORT],[The default port for Spindle])
 AC_DEFINE_UNQUOTED([NUM_COBO_PORTS],[$NUM_COBO_PORTS],[Number of ports for COBO to search for an open port])
 AC_DEFINE_UNQUOTED([SPINDLE_MAX_PORT],[$(($SPINDLE_PORT + $NUM_COBO_PORTS - 1))],[The maximum port value])
-AC_DEFINE_UNQUOTED([SPINDLE_LOC],"[$SPINDLE_LOC]",[The default local directory for Spindle])
+AC_DEFINE_UNQUOTED([COMMPATH],"[$COMMPATH]",[Back-end directory for communication and housekeeping])
 AC_DEFINE_UNQUOTED([CACHEPATHS],"[$CACHEPATHS]",[Colon-separated list of potential back-end cache directories])
 AC_DEFINE_UNQUOTED([SPINDLE_LOCAL_PREFIX],"[$SPINDLE_LOCAL_PREFIX]",[The default colon-separated list of directories that Spindle will not cache files out of])
 
diff --git a/src/client/config.h.in b/src/client/config.h.in
index a42cc0ab..d78a3e7f 100644
--- a/src/client/config.h.in
+++ b/src/client/config.h.in
@@ -9,6 +9,9 @@
 /* Colon-separated list of potential back-end cache directories */
 #undef CACHEPATHS
 
+/* Back-end directory for communication and housekeeping */
+#undef COMMPATH
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 
@@ -121,9 +124,6 @@
 /* Default mode for slurm launch */
 #undef SLURMLAUNCH_ENABLED
 
-/* The default local directory for Spindle */
-#undef SPINDLE_LOC
-
 /* The default colon-separated list of directories that Spindle will not cache
    files out of */
 #undef SPINDLE_LOCAL_PREFIX
diff --git a/src/client/configure b/src/client/configure
index cbae8891..b7dbffc0 100755
--- a/src/client/configure
+++ b/src/client/configure
@@ -811,7 +811,7 @@ enable_maintainer_mode
 with_default_port
 with_default_num_ports
 with_cachepaths
-with_localstorage
+with_commpath
 with_default_local_prefix
 with_testrm
 with_rm
@@ -1535,7 +1535,8 @@ Optional Packages:
                           communication
   --with-cachepaths=DIR   Colon-separated list of potential back-end cache
                           directories
-  --with-localstorage=DIR Directory on back-ends for storing relocated files
+  --with-compath=DIR      Back-end directory for communication and
+                          housekeeping
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
                           will not cache files out of
@@ -12598,11 +12599,11 @@ else
 fi
 
 
-# Check whether --with-localstorage was given.
-if test "${with_localstorage+set}" = set; then :
-  withval=$with_localstorage; SPINDLE_LOC=${withval}
+# Check whether --with-commpath was given.
+if test "${with_commpath+set}" = set; then :
+  withval=$with_commpath; COMMPATH=${withval}
 else
-  SPINDLE_LOC=$DEFAULT_LOC
+  COMMPATH=$DEFAULT_LOC
 fi
 
 
@@ -12610,7 +12611,7 @@ fi
 if test "${with_default_local_prefix+set}" = set; then :
   withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval}
 else
-  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$SPINDLE_LOC"
+  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH"
 fi
 
 
@@ -12630,7 +12631,7 @@ _ACEOF
 
 
 cat >>confdefs.h <<_ACEOF
-#define SPINDLE_LOC "$SPINDLE_LOC"
+#define COMMPATH "$COMMPATH"
 _ACEOF
 
 
diff --git a/src/fe/config.h.in b/src/fe/config.h.in
index 5057ef33..3ac96e42 100644
--- a/src/fe/config.h.in
+++ b/src/fe/config.h.in
@@ -9,6 +9,9 @@
 /* Colon-separated list of potential back-end cache directories */
 #undef CACHEPATHS
 
+/* Back-end directory for communication and housekeeping */
+#undef COMMPATH
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 
@@ -166,9 +169,6 @@
 /* Default mode for slurm launch */
 #undef SLURMLAUNCH_ENABLED
 
-/* The default local directory for Spindle */
-#undef SPINDLE_LOC
-
 /* The default colon-separated list of directories that Spindle will not cache
    files out of */
 #undef SPINDLE_LOCAL_PREFIX
diff --git a/src/fe/configure b/src/fe/configure
index 29db0f61..d983778a 100755
--- a/src/fe/configure
+++ b/src/fe/configure
@@ -832,7 +832,7 @@ enable_maintainer_mode
 with_default_port
 with_default_num_ports
 with_cachepaths
-with_localstorage
+with_commpath
 with_default_local_prefix
 with_testrm
 with_rm
@@ -1573,7 +1573,8 @@ Optional Packages:
                           communication
   --with-cachepaths=DIR   Colon-separated list of potential back-end cache
                           directories
-  --with-localstorage=DIR Directory on back-ends for storing relocated files
+  --with-compath=DIR      Back-end directory for communication and
+                          housekeeping
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
                           will not cache files out of
@@ -16448,11 +16449,11 @@ else
 fi
 
 
-# Check whether --with-localstorage was given.
-if test "${with_localstorage+set}" = set; then :
-  withval=$with_localstorage; SPINDLE_LOC=${withval}
+# Check whether --with-commpath was given.
+if test "${with_commpath+set}" = set; then :
+  withval=$with_commpath; COMMPATH=${withval}
 else
-  SPINDLE_LOC=$DEFAULT_LOC
+  COMMPATH=$DEFAULT_LOC
 fi
 
 
@@ -16460,7 +16461,7 @@ fi
 if test "${with_default_local_prefix+set}" = set; then :
   withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval}
 else
-  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$SPINDLE_LOC"
+  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH"
 fi
 
 
@@ -16480,7 +16481,7 @@ _ACEOF
 
 
 cat >>confdefs.h <<_ACEOF
-#define SPINDLE_LOC "$SPINDLE_LOC"
+#define COMMPATH "$COMMPATH"
 _ACEOF
 
 
diff --git a/src/fe/startup/config_mgr.cc b/src/fe/startup/config_mgr.cc
index eb710b36..16c4a0f8 100644
--- a/src/fe/startup/config_mgr.cc
+++ b/src/fe/startup/config_mgr.cc
@@ -50,10 +50,10 @@ using namespace std;
 #define SPINDLE_NUM_PORTS_STR "250"
 #endif
 
-#if defined(SPINDLE_LOC)
-#define SPINDLE_LOC_STR SPINDLE_LOC
+#if defined(COMMPATH)
+#define SPINDLE_COMMPATH_STR COMMPATH
 #else
-#define SPINDLE_LOC_STR "$TMPDIR"
+#define SPINDLE_COMMPATH_STR "$TMPDIR"
 #endif
 
 #if defined(CACHEPATHS)
@@ -273,8 +273,8 @@ void initOptionsList()
      "Provides a text file containing a white-space separated list of files that should be relocated to each node before execution begins" },
    { confStrip, "strip", shortStrip, groupMisc, cvBool, {}, "true", 
      "Strip debug and symbol information from binaries before distributing them." },
-   { confLocation, "location", shortLocation, groupMisc, cvString, {}, SPINDLE_LOC_STR,
-     "Back-end directory for storing relocated files.  Should be a non-shared location such as a ramdisk." },
+   { confCommPath, "commpath", shortCommPath, groupMisc, cvString, {}, SPINDLE_COMMPATH_STR,
+     "Back-end directory communication and housekeeping.  Should be a non-shared location such as a ramdisk." },
    { confCachePaths, "cachepaths", shortCachePaths, groupMisc, cvString, {}, SPINDLE_CACHEPATHS_STR,
      "Colon-separated list of candidate paths for cached libraries."},
    { confNoclean, "noclean", shortNoClean, groupMisc, cvBool, {}, "false",
@@ -743,9 +743,9 @@ bool ConfigMap::toSpindleArgs(spindle_args_t &args, bool alloc_strs) const
          case confNumPorts:
             args.num_ports = numresult;
             break;
-         case confLocation: {
-            string loc = strresult + "/spindle.$NUMBER";
-            args.location = strdup(loc.c_str());
+         case confCommPath: {
+            string path = strresult + "/spindle.$NUMBER";
+            args.location = strdup(path.c_str());
             break;
          }
          case confCachePaths:{
diff --git a/src/fe/startup/config_mgr.h b/src/fe/startup/config_mgr.h
index c2d3cd7e..ec3c8135 100644
--- a/src/fe/startup/config_mgr.h
+++ b/src/fe/startup/config_mgr.h
@@ -29,7 +29,7 @@ enum SpindleConfigID {
    confCmdlineNewgroup,
    confPort,
    confNumPorts,
-   confLocation,
+   confCommPath,
    confCachePaths,
    confCachePrefix,
    confPythonPrefix,
@@ -83,7 +83,7 @@ enum CmdlineShortOptions {
    shortAuditType = 'k',
    shortRelocSO = 'l',
    shortNoClean = 'n',
-   shortLocation = 'o',
+   shortCommPath = 'o',
    shortPush = 'p',
    shortPull = 'q',
    shortPythonPrefix = 'r',
diff --git a/src/flux/sessionmgr.c b/src/flux/sessionmgr.c
index 49324a2f..17027163 100644
--- a/src/flux/sessionmgr.c
+++ b/src/flux/sessionmgr.c
@@ -109,16 +109,16 @@ char **strip_start_from_argv(int argc, char **argv)
 extern char *parse_location(char *loc, int number);
 extern int spindle_mkdir(char *orig_path);
 
-#if !defined(SPINDLE_LOC)
-#error SPINDLE_LOC must be defined in config.h
+#if !defined(COMMPATH)
+#error COMMPATH must be defined in config.h
 #endif
 const char *get_session_dir()
 {
    int result;
    char *dir;
-   dir = parse_location((char *) (SPINDLE_LOC "/spindle_session"), 0);
+   dir = parse_location((char *) (COMMPATH "/spindle_session"), 0);
    if (!dir) {
-      spindle_debug_printf(1, "ERROR: Could not parse directory for spindle session location from %s/spindle_session\n", SPINDLE_LOC);
+      spindle_debug_printf(1, "ERROR: Could not parse directory for spindle session location from %s/spindle_session\n", COMMPATH);
       return NULL;
    }
       
diff --git a/src/server/config.h.in b/src/server/config.h.in
index dc9439b3..87fbf990 100644
--- a/src/server/config.h.in
+++ b/src/server/config.h.in
@@ -9,6 +9,9 @@
 /* Colon-separated list of potential back-end cache directories */
 #undef CACHEPATHS
 
+/* Back-end directory for communication and housekeeping */
+#undef COMMPATH
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 
@@ -148,9 +151,6 @@
 /* Default mode for slurm launch */
 #undef SLURMLAUNCH_ENABLED
 
-/* The default local directory for Spindle */
-#undef SPINDLE_LOC
-
 /* The default colon-separated list of directories that Spindle will not cache
    files out of */
 #undef SPINDLE_LOCAL_PREFIX
diff --git a/src/server/configure b/src/server/configure
index 55e9c6f5..b71bdd82 100755
--- a/src/server/configure
+++ b/src/server/configure
@@ -838,7 +838,7 @@ enable_maintainer_mode
 with_default_port
 with_default_num_ports
 with_cachepaths
-with_localstorage
+with_commpath
 with_default_local_prefix
 with_testrm
 with_rm
@@ -1570,7 +1570,8 @@ Optional Packages:
                           communication
   --with-cachepaths=DIR   Colon-separated list of potential back-end cache
                           directories
-  --with-localstorage=DIR Directory on back-ends for storing relocated files
+  --with-compath=DIR      Back-end directory for communication and
+                          housekeeping
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
                           will not cache files out of
@@ -16445,11 +16446,11 @@ else
 fi
 
 
-# Check whether --with-localstorage was given.
-if test "${with_localstorage+set}" = set; then :
-  withval=$with_localstorage; SPINDLE_LOC=${withval}
+# Check whether --with-commpath was given.
+if test "${with_commpath+set}" = set; then :
+  withval=$with_commpath; COMMPATH=${withval}
 else
-  SPINDLE_LOC=$DEFAULT_LOC
+  COMMPATH=$DEFAULT_LOC
 fi
 
 
@@ -16457,7 +16458,7 @@ fi
 if test "${with_default_local_prefix+set}" = set; then :
   withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval}
 else
-  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$SPINDLE_LOC"
+  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH"
 fi
 
 
@@ -16477,7 +16478,7 @@ _ACEOF
 
 
 cat >>confdefs.h <<_ACEOF
-#define SPINDLE_LOC "$SPINDLE_LOC"
+#define COMMPATH "$COMMPATH"
 _ACEOF
 
 

From 97566886e9f374a0c3b7922cf473e4590923ae31 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 24 Oct 2025 12:29:41 -0700
Subject: [PATCH 21/57] Renaming location variables/fields to commpath.

---
 src/fe/startup/config_mgr.cc                     |  2 +-
 src/fe/startup/parse_launcher.cc                 |  4 ++--
 src/fe/startup/spindle_fe.cc                     | 10 +++++-----
 src/flux/flux-spindle.c                          |  8 ++++----
 src/include/spindle_launch.h                     |  4 ++--
 .../auditserver/ldcs_audit_server_handlers.c     | 12 ++++++------
 .../auditserver/ldcs_audit_server_process.c      | 14 +++++++-------
 .../auditserver/ldcs_audit_server_process.h      |  2 +-
 src/server/startup/spindle_be.cc                 | 16 ++++++++--------
 9 files changed, 36 insertions(+), 36 deletions(-)

diff --git a/src/fe/startup/config_mgr.cc b/src/fe/startup/config_mgr.cc
index 16c4a0f8..1b9d9ade 100644
--- a/src/fe/startup/config_mgr.cc
+++ b/src/fe/startup/config_mgr.cc
@@ -745,7 +745,7 @@ bool ConfigMap::toSpindleArgs(spindle_args_t &args, bool alloc_strs) const
             break;
          case confCommPath: {
             string path = strresult + "/spindle.$NUMBER";
-            args.location = strdup(path.c_str());
+            args.commpath = strdup(path.c_str());
             break;
          }
          case confCachePaths:{
diff --git a/src/fe/startup/parse_launcher.cc b/src/fe/startup/parse_launcher.cc
index f05ba18a..4484e998 100644
--- a/src/fe/startup/parse_launcher.cc
+++ b/src/fe/startup/parse_launcher.cc
@@ -292,7 +292,7 @@ void ModifyArgv::modifyCmdLine()
    snprintf(options_str, 32, "%lu", (unsigned long) params->opts);
    string options(options_str);
    
-   string location(params->location);
+   string commpath(params->commpath);
    
    char number_str[32];
    snprintf(number_str, 32, "%lu", (unsigned long) params->number);
@@ -319,7 +319,7 @@ void ModifyArgv::modifyCmdLine()
       if (p == parser->appExecutableAt()) {
 #if defined(os_bluegene)
          string bg_env_str = parser->getParser()->getBGString();
-         parser->getParser()->addBGEnvStr(n, new_argv, bg_env_str, default_libstr, intercept_libstr, location, number, options, shmcache_size);
+         parser->getParser()->addBGEnvStr(n, new_argv, bg_env_str, default_libstr, intercept_libstr, commpath, number, options, shmcache_size);
 #else
          char **a_argv;
          int a_argc;
diff --git a/src/fe/startup/spindle_fe.cc b/src/fe/startup/spindle_fe.cc
index 43e8c6ba..95bf970b 100644
--- a/src/fe/startup/spindle_fe.cc
+++ b/src/fe/startup/spindle_fe.cc
@@ -71,7 +71,7 @@ static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size)
    buffer_size += sizeof(number_t);
    buffer_size += sizeof(opt_t);
    buffer_size += sizeof(unique_id_t);
-   buffer_size += args->location ? strlen(args->location) + 1 : 1;
+   buffer_size += args->commpath ? strlen(args->commpath) + 1 : 1;
    buffer_size += args->candidate_cachepaths ? strlen(args->candidate_cachepaths) + 1 : 1;
    buffer_size += args->pythonprefix ? strlen(args->pythonprefix) + 1 : 1;
    buffer_size += args->preloadfile ? strlen(args->preloadfile) + 1 : 1;
@@ -92,7 +92,7 @@ static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size)
    pack_param(args->use_launcher, buf, pos);
    pack_param(args->startup_type, buf, pos);
    pack_param(args->shm_cache_size, buf, pos);
-   pack_param(args->location, buf, pos);
+   pack_param(args->commpath, buf, pos);
    pack_param(args->candidate_cachepaths, buf, pos);
    pack_param(args->pythonprefix, buf, pos);
    pack_param(args->preloadfile, buf, pos);
@@ -232,7 +232,7 @@ int getApplicationArgsFE(spindle_args_t *params, int *spindle_argc, char ***spin
       (*spindle_argv)[n++] = strdup(numports_s);
       (*spindle_argv)[n++] = strdup(uniqueid_s);
    }
-   (*spindle_argv)[n++] = strdup(params->location);
+   (*spindle_argv)[n++] = strdup(params->commpath);
    (*spindle_argv)[n++] = strdup(params->candidate_cachepaths);
    (*spindle_argv)[n++] = strdup(number_s);
    (*spindle_argv)[n++] = strdup(opt_s);
@@ -398,11 +398,11 @@ int spindleInitFE(const char **hosts, spindle_args_t *params)
    
    /* Start FE server */
    debug_printf("spindle_args_t { number = %lu; port = %u; num_ports = %u; opts = %lu; unique_id = %lu; "
-                "use_launcher = %u; startup_type = %u; shm_cache_size = %u; location = %s; "
+                "use_launcher = %u; startup_type = %u; shm_cache_size = %u; commpath = %s; "
                 "cachepaths = %s; "
                 "pythonprefix = %s; preloadfile = %s; bundle_timeout_ms = %u; bundle_cachesize_kb = %u }\n",
                 (unsigned long) params->number, params->port, params->num_ports, params->opts, params->unique_id,
-                params->use_launcher, params->startup_type, params->shm_cache_size, params->location,
+                params->use_launcher, params->startup_type, params->shm_cache_size, params->commpath,
                 params->candidate_cachepaths,
                 params->pythonprefix, params->preloadfile, params->bundle_timeout_ms,
                 params->bundle_cachesize_kb);
diff --git a/src/flux/flux-spindle.c b/src/flux/flux-spindle.c
index 65de5ed9..101bb6ac 100644
--- a/src/flux/flux-spindle.c
+++ b/src/flux/flux-spindle.c
@@ -381,7 +381,7 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx)
     int numa = 0;
     const char *relocaout = NULL, *reloclibs = NULL, *relocexec = NULL, *relocpython = NULL;
     const char *followfork = NULL, *preload = NULL, *level = NULL;
-    const char *pyprefix = NULL, *location = NULL;
+    const char *pyprefix = NULL, *commpath = NULL;
     char *numafiles = NULL, *cachepaths = NULL;
 
     if (flux_shell_getopt_unpack (shell, "spindle", "o", &opts) < 0)
@@ -415,7 +415,7 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx)
                         "reloc-exec", &relocexec,
                         "reloc-python", &relocpython,
                         "python-prefix", &pyprefix,
-                        "location", &location,
+                        "commpath", &commpath,
                         "numa", &numa,
                         "numa-files", &numafiles,
                         "preload", &preload,
@@ -466,8 +466,8 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx)
     if( cachepaths ){
         ctx->params.candidate_cachepaths = cachepaths;
     }
-    if (location) {
-       ctx->params.location = (char *) location;
+    if (commpath) {
+       ctx->params.commpath = (char *) commpath;
     }
     if (level) {
        if (strcmp(level, "high") == 0) {
diff --git a/src/include/spindle_launch.h b/src/include/spindle_launch.h
index 81c0728e..ca7b8d3d 100644
--- a/src/include/spindle_launch.h
+++ b/src/include/spindle_launch.h
@@ -123,8 +123,8 @@ typedef struct {
    /* Size of client shared memory cache */
    unsigned int shm_cache_size;
 
-   /* The local-disk location where Spindle will store its cache */
-   char *location;
+   /* The local-disk location for communication and housekeeping. */
+   char *commpath;
 
    /* Path[s] for cached libraries. */
    char *candidate_cachepaths;  /* Colon-separated list of candidate paths (max 64) */
diff --git a/src/server/auditserver/ldcs_audit_server_handlers.c b/src/server/auditserver/ldcs_audit_server_handlers.c
index 29107bad..91d8779c 100644
--- a/src/server/auditserver/ldcs_audit_server_handlers.c
+++ b/src/server/auditserver/ldcs_audit_server_handlers.c
@@ -202,7 +202,7 @@ static int handle_client_info_msg(ldcs_process_data_t *procdata, int nc, ldcs_me
    else if(msg->header.type == LDCS_MSG_LOCATION) {
       strncpy(client->remote_location, msg->data, sizeof(client->remote_location)-1);
       client->remote_location[sizeof(client->remote_location)-1] = '\0';
-      debug_printf2("Server recvd location %s from %d\n", msg->data, nc);
+      debug_printf2("Server recvd remote_location %s from %d\n", msg->data, nc);
    }
    else if (msg->header.type == LDCS_MSG_CPU) {
       int clientcpu;
@@ -2953,7 +2953,7 @@ static int handle_client_pickone_msg(ldcs_process_data_t *procdata, int nc, ldcs
 }
 
 /**
- * Handle LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS to determine which of the locations, commpaths, and cachepaths are
+ * Handle LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS to determine which cachepaths are
  * available across all of the servers.
  */
 
@@ -2969,8 +2969,8 @@ static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_messag
     ldcs_audit_server_md_allreduce_AND( &procdata->cachepath_bitidx );
 
     if( procdata->cachepath_bitidx == 0 ){
-       err_printf("No valid cachepath path available.  Falling back to \"location\" path (%s).\n", procdata->location);
-       procdata->cachepath = procdata->location;
+       err_printf("No valid cachepath path available.  Falling back to \"commpath\" path (%s).\n", procdata->commpath);
+       procdata->cachepath = procdata->commpath;
     }else{
         // ldcs_audit_server_filemngt_init() does it's own realize() pass.
         getValidCachePathByIndex( procdata->cachepath_bitidx,
@@ -2979,7 +2979,7 @@ static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_messag
                 &procdata->symbolic_cachepath);
     }
 
-    debug_printf3("Initializing file cache location %s\n", procdata->location);
+    debug_printf3("Initializing file cache cachepath %s\n", procdata->cachepath);
     ldcs_audit_server_filemngt_init(procdata->cachepath);
 
     test_printf("<internal> cachepath=%s\n", procdata->cachepath);
@@ -3224,7 +3224,7 @@ int exit_note_cb(int fd, int serverid, void *data)
       eresult = -1;
    }
 
-   result = handleExitNote(fd, procdata->location);
+   result = handleExitNote(fd, procdata->commpath);
    if (result == -1) {
       debug_printf("handleExitNote failed\n");
       eresult = -1;
diff --git a/src/server/auditserver/ldcs_audit_server_process.c b/src/server/auditserver/ldcs_audit_server_process.c
index 566cad01..a73a7b7f 100644
--- a/src/server/auditserver/ldcs_audit_server_process.c
+++ b/src/server/auditserver/ldcs_audit_server_process.c
@@ -113,7 +113,7 @@ void startprofile(spindle_args_t *args)
    char hostname[257];
    char *home = getenv("HOME");
    if (!home || !*home)
-      home = ldcs_process_data.location;
+      home = ldcs_process_data.commpath;
    gethostname(hostname, sizeof(hostname));
    snprintf(filename, 4096, "%s/spindled.%lu.%s.%d.prof", home, (unsigned long) args->number, hostname, getpid());
    ProfilerStart(filename);
@@ -142,7 +142,7 @@ int ldcs_audit_server_process(spindle_args_t *args)
    startprofile(args);
 
    debug_printf3("Initializing server data structures\n");
-   ldcs_process_data.location = args->location;
+   ldcs_process_data.commpath = args->commpath;
    ldcs_process_data.cachepaths = args->candidate_cachepaths;
    ldcs_process_data.cachepath = NULL;
    ldcs_process_data.cachepath_bitidx = 0;
@@ -196,11 +196,11 @@ int ldcs_audit_server_process(spindle_args_t *args)
    ldcs_process_data.server_stat.hostname=ldcs_process_data.hostname;
 
    if (ldcs_process_data.opts & OPT_PROCCLEAN)
-      init_cleanup_proc(ldcs_process_data.location);
+      init_cleanup_proc(ldcs_process_data.commpath);
 
    debug_printf3("Initializing connections for clients at %s and %lu\n",
-                 ldcs_process_data.location, (unsigned long) ldcs_process_data.number);
-   serverid = ldcs_create_server(ldcs_process_data.location, ldcs_process_data.number);
+                 ldcs_process_data.commpath, (unsigned long) ldcs_process_data.number);
+   serverid = ldcs_create_server(ldcs_process_data.commpath, ldcs_process_data.number);
    if (serverid == -1) {
       err_printf("Unable to setup area for client connections\n");
       return -1;
@@ -216,7 +216,7 @@ int ldcs_audit_server_process(spindle_args_t *args)
       ldcs_listen_register_fd(fd, serverid, &_ldcs_server_CB, (void *) &ldcs_process_data);
 
    if (args->opts & OPT_BEEXIT) {
-      fd = createExitNote(args->location);
+      fd = createExitNote(args->commpath);
       if (fd != -1) {
          ldcs_listen_register_fd(fd, serverid, exit_note_cb, (void *) &ldcs_process_data);
       }
@@ -254,7 +254,7 @@ int ldcs_audit_server_run()
 
    _ldcs_server_stat_print(&ldcs_process_data.server_stat);
   
-   debug_printf("destroy server (%s,%lu)\n", ldcs_process_data.location, (unsigned long) ldcs_process_data.number);
+   debug_printf("destroy server (%s,%lu)\n", ldcs_process_data.commpath, (unsigned long) ldcs_process_data.number);
    ldcs_destroy_server(ldcs_process_data.serverid);
   
    /* destroy md support (multi-daemon) */
diff --git a/src/server/auditserver/ldcs_audit_server_process.h b/src/server/auditserver/ldcs_audit_server_process.h
index 1495cebd..82b60023 100644
--- a/src/server/auditserver/ldcs_audit_server_process.h
+++ b/src/server/auditserver/ldcs_audit_server_process.h
@@ -125,7 +125,7 @@ struct ldcs_process_data_struct
   int exit_readys_recvd;
   ldcs_dist_model_t dist_model;
   ldcs_client_t* client_table;
-  char *location;               /* Single user-specified path for fifo, daemons, etc. */
+  char *commpath;               /* Single user-specified path for fifo, daemons, etc. */
                                 /*   (Everything except the cachepath.) */
   char *cachepaths;             /* Up to 64 colon-separated list of candidate cachepaths. */
   char *cachepath;              /* The earliest path in the list available to all servers. */
diff --git a/src/server/startup/spindle_be.cc b/src/server/startup/spindle_be.cc
index 733d3244..fa0eccb3 100644
--- a/src/server/startup/spindle_be.cc
+++ b/src/server/startup/spindle_be.cc
@@ -58,7 +58,7 @@ static int unpack_data(spindle_args_t *args, void *buffer, int buffer_size)
    unpack_param(args->use_launcher, buf, pos);
    unpack_param(args->startup_type, buf, pos);
    unpack_param(args->shm_cache_size, buf, pos);
-   unpack_param(args->location, buf, pos);
+   unpack_param(args->commpath, buf, pos);
    unpack_param(args->candidate_cachepaths, buf, pos);
    unpack_param(args->pythonprefix, buf, pos);
    unpack_param(args->preloadfile, buf, pos);
@@ -144,15 +144,15 @@ int spindleRunBE(unsigned int port, unsigned int num_ports, unique_id_t unique_i
    assert(args.port == port);
    
    
-   /* Expand environment variables in location. */
-   char *new_location = parse_location(args.location, args.number);
-   if (!new_location) {
-      err_printf("Failed to convert location %s\n", args.location);
+   /* Expand environment variables in commpath. */
+   char *new_commpath = parse_location(args.commpath, args.number);
+   if (!new_commpath) {
+      err_printf("Failed to convert commpath %s\n", args.commpath);
       return -1;
    }
-   debug_printf("Translated location from %s to %s\n", args.location, new_location);
-   free(args.location);
-   args.location = new_location;
+   debug_printf("Translated commpath from %s to %s\n", args.commpath, new_commpath);
+   free(args.commpath);
+   args.commpath = new_commpath;
 
    result = ldcs_audit_server_process(&args);
    if (result == -1) {

From 2b0acc013ef84ef2f0da1ae10a447fd19aeebe20 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 24 Oct 2025 12:45:10 -0700
Subject: [PATCH 22/57] Renames enums.

---
 src/client/client/client.c                           |  4 ++--
 src/client/client/intercept_exec.c                   |  6 +++---
 src/client/client_comlib/client_api.c                |  2 +-
 src/fe/startup/parse_launcher_args.cc                | 12 ++++++------
 src/include/ldcs_api.h                               |  2 +-
 src/server/auditserver/ldcs_audit_server_handlers.c  |  4 ++--
 .../auditserver/ldcs_audit_server_md_msocket.c       |  2 +-
 src/server/comlib/ldcs_api_util.c                    |  2 +-
 8 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/src/client/client/client.c b/src/client/client/client.c
index ac8d4b63..1635e060 100644
--- a/src/client/client/client.c
+++ b/src/client/client/client.c
@@ -198,7 +198,7 @@ static int init_server_connection()
    if (!use_ldcs)
       return 0;
 
-   location = getenv("LDCS_LOCATION");
+   location = getenv("LDCS_COMMPATH");
    number = (number_t) strtoul(getenv("LDCS_NUMBER"), NULL, 0);
    connection = getenv("LDCS_CONNECTION");
    rankinfo_s = getenv("LDCS_RANKINFO");
@@ -217,7 +217,7 @@ static int init_server_connection()
    if (!(opts & OPT_FOLLOWFORK)) {
       debug_printf("Disabling environment variables because we're not following forks\n");
       unsetenv("LD_AUDIT");
-      unsetenv("LDCS_LOCATION");
+      unsetenv("LDCS_COMMPATH");
       unsetenv("LDCS_NUMBER");
       unsetenv("LDCS_CONNECTION");
       unsetenv("LDCS_RANKINFO");
diff --git a/src/client/client/intercept_exec.c b/src/client/client/intercept_exec.c
index 27928d35..bae9eba3 100644
--- a/src/client/client/intercept_exec.c
+++ b/src/client/client/intercept_exec.c
@@ -141,7 +141,7 @@ static char **removeEnvironmentStrs(char **envp)
          continue;
       if (strIsPrefix("LD", envp[i])) {
          if (strIsPrefix("LD_AUDIT=", envp[i]) ||
-             strIsPrefix("LDCS_LOCATION=", envp[i]) ||
+             strIsPrefix("LDCS_COMMPATH=", envp[i]) ||
              strIsPrefix("LDCS_CONNECTION=", envp[i]) ||
              strIsPrefix("LDCS_RANKINFO=", envp[i]) ||
              strIsPrefix("LDCS_OPTIONS=", envp[i]) ||
@@ -175,7 +175,7 @@ static char **updateEnvironment(char **envp, int *num_modified, int propogate_sp
          unsetf = orig_unsetenv ? orig_unsetenv : unsetenv;
          unsetf("SPINDLE");
          unsetf("LD_AUDIT");
-         unsetf("LDCS_LOCATION");
+         unsetf("LDCS_COMMPATH");
          unsetf("LDCS_CONNECTION");
          unsetf("LDCS_RANKINFO");
          unsetf("LDCS_OPTIONS");
@@ -201,7 +201,7 @@ static char **updateEnvironment(char **envp, int *num_modified, int propogate_sp
       
       propogateEnvironmentStr(envp, newenv, &pos, "SPINDLE");   
       propogateEnvironmentStr(envp, newenv, &pos, "LD_AUDIT");
-      propogateEnvironmentStr(envp, newenv, &pos, "LDCS_LOCATION");
+      propogateEnvironmentStr(envp, newenv, &pos, "LDCS_COMMPATH");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_CONNECTION");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_RANKINFO");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_OPTIONS");
diff --git a/src/client/client_comlib/client_api.c b/src/client/client_comlib/client_api.c
index 4b0f9ded..f5e07cc0 100644
--- a/src/client/client_comlib/client_api.c
+++ b/src/client/client_comlib/client_api.c
@@ -349,7 +349,7 @@ int send_cpu(int fd, int cpu) {
 int send_location(int fd, char *location) {
    ldcs_message_t message;
 
-   message.header.type = LDCS_MSG_LOCATION;
+   message.header.type = LDCS_MSG_COMMPATH;
    message.header.len = strlen(location)+1;
    message.data = location;
 
diff --git a/src/fe/startup/parse_launcher_args.cc b/src/fe/startup/parse_launcher_args.cc
index 84cf7f23..710b009d 100644
--- a/src/fe/startup/parse_launcher_args.cc
+++ b/src/fe/startup/parse_launcher_args.cc
@@ -127,7 +127,7 @@ static cmdoption_t openmpi_options[] = {
 };
 
 
-static const char *openmpi_bg_env_str = "-x LD_AUDIT=%s -x LDCS_LOCATION=%s -x LDCS_NUMBER=%s -x LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
+static const char *openmpi_bg_env_str = "-x LD_AUDIT=%s -x LDCS_COMMPATH=%s -x LDCS_NUMBER=%s -x LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
 static const unsigned int openmpi_size = (sizeof(openmpi_options) / sizeof(cmdoption_t));
 
 static cmdoption_t srun_options[] = {
@@ -218,11 +218,11 @@ static cmdoption_t srun_options[] = {
    { NULL,   "--usage",              0 },
    { "-V",   "--version",            0 }
 };
-static const char *srun_bg_env_str = "--runjob-opts=--envs LD_AUDIT=%s LD_PRELOAD=%s LDCS_LOCATION=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
-static const char *srun_bg_env_str_nopreload = "--runjob-opts=--envs LD_AUDIT=%s%s LDCS_LOCATION=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
-static const char *srun_bg_env_bare_str = "%s --envs LD_AUDIT=%s LD_PRELOAD=%s LDCS_LOCATION=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
-static const char *srun_bg_env_bare_str_preload = "%sLD_PRELOAD=%s:%s LD_AUDIT=%s LDCS_LOCATION=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
-static const char *srun_bg_env_bare_str_nopreload = "%s LD_AUDIT=%s LDCS_LOCATION=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
+static const char *srun_bg_env_str = "--runjob-opts=--envs LD_AUDIT=%s LD_PRELOAD=%s LDCS_COMMPATH=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
+static const char *srun_bg_env_str_nopreload = "--runjob-opts=--envs LD_AUDIT=%s%s LDCS_COMMPATH=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
+static const char *srun_bg_env_bare_str = "%s --envs LD_AUDIT=%s LD_PRELOAD=%s LDCS_COMMPATH=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
+static const char *srun_bg_env_bare_str_preload = "%sLD_PRELOAD=%s:%s LD_AUDIT=%s LDCS_COMMPATH=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
+static const char *srun_bg_env_bare_str_nopreload = "%s LD_AUDIT=%s LDCS_COMMPATH=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
 
 static const unsigned int srun_size (sizeof(srun_options) / sizeof(cmdoption_t));
 
diff --git a/src/include/ldcs_api.h b/src/include/ldcs_api.h
index e6ccbafb..49ba45e3 100644
--- a/src/include/ldcs_api.h
+++ b/src/include/ldcs_api.h
@@ -45,7 +45,7 @@ typedef enum {
    LDCS_MSG_END,
    LDCS_MSG_CWD,
    LDCS_MSG_PID,
-   LDCS_MSG_LOCATION,
+   LDCS_MSG_COMMPATH,
    LDCS_MSG_CPU,
    LDCS_MSG_MYRANKINFO_QUERY,
    LDCS_MSG_MYRANKINFO_QUERY_ANSWER,
diff --git a/src/server/auditserver/ldcs_audit_server_handlers.c b/src/server/auditserver/ldcs_audit_server_handlers.c
index 91d8779c..0129c25d 100644
--- a/src/server/auditserver/ldcs_audit_server_handlers.c
+++ b/src/server/auditserver/ldcs_audit_server_handlers.c
@@ -199,7 +199,7 @@ static int handle_client_info_msg(ldcs_process_data_t *procdata, int nc, ldcs_me
       client->remote_pid=mypid;
       debug_printf2("Server recvd pid %d from %d\n", mypid, nc);
    } 
-   else if(msg->header.type == LDCS_MSG_LOCATION) {
+   else if(msg->header.type == LDCS_MSG_COMMPATH) {
       strncpy(client->remote_location, msg->data, sizeof(client->remote_location)-1);
       client->remote_location[sizeof(client->remote_location)-1] = '\0';
       debug_printf2("Server recvd remote_location %s from %d\n", msg->data, nc);
@@ -1866,7 +1866,7 @@ int handle_client_message(ldcs_process_data_t *procdata, int nc, ldcs_message_t
    switch (msg->header.type) {
       case LDCS_MSG_CWD:
       case LDCS_MSG_PID:
-      case LDCS_MSG_LOCATION:
+      case LDCS_MSG_COMMPATH:
       case LDCS_MSG_CPU:
          return handle_client_info_msg(procdata, nc, msg);
       case LDCS_MSG_PYTHONPREFIX_REQ:
diff --git a/src/server/auditserver/ldcs_audit_server_md_msocket.c b/src/server/auditserver/ldcs_audit_server_md_msocket.c
index c9d616c2..6db18bd4 100644
--- a/src/server/auditserver/ldcs_audit_server_md_msocket.c
+++ b/src/server/auditserver/ldcs_audit_server_md_msocket.c
@@ -62,7 +62,7 @@ int ldcs_audit_server_md_init ( ldcs_process_data_t *ldcs_process_data ) {
   int rc=0;
 
   char* ldcs_nportsstr=getenv("LDCS_NPORTS");
-  char* ldcs_locmodstr=getenv("LDCS_LOCATION_MOD");
+  char* ldcs_locmodstr=getenv("LDCS_COMMPATH_MOD");
 
   int usedport;
   int serverfd, serverid, i;
diff --git a/src/server/comlib/ldcs_api_util.c b/src/server/comlib/ldcs_api_util.c
index b6beb56d..5ea51ab8 100644
--- a/src/server/comlib/ldcs_api_util.c
+++ b/src/server/comlib/ldcs_api_util.c
@@ -51,7 +51,7 @@ char* _message_type_to_str (ldcs_message_ids_t type) {
       STR_CASE(LDCS_MSG_END);
       STR_CASE(LDCS_MSG_CWD);
       STR_CASE(LDCS_MSG_PID);
-      STR_CASE(LDCS_MSG_LOCATION);
+      STR_CASE(LDCS_MSG_COMMPATH);
       STR_CASE(LDCS_MSG_CPU);      
       STR_CASE(LDCS_MSG_MYRANKINFO_QUERY);
       STR_CASE(LDCS_MSG_MYRANKINFO_QUERY_ANSWER);

From fa47cf1c9f963f0ab4e9d8534d2f86d50aa99470 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Wed, 5 Nov 2025 15:28:52 -0800
Subject: [PATCH 23/57] Use strdup() for commpath instead of stack var.

Unlikely it would ever make a difference, but this is much more
correct.
---
 src/server/startup/spindle_be.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server/startup/spindle_be.cc b/src/server/startup/spindle_be.cc
index fa0eccb3..c3e7bbd9 100644
--- a/src/server/startup/spindle_be.cc
+++ b/src/server/startup/spindle_be.cc
@@ -152,7 +152,7 @@ int spindleRunBE(unsigned int port, unsigned int num_ports, unique_id_t unique_i
    }
    debug_printf("Translated commpath from %s to %s\n", args.commpath, new_commpath);
    free(args.commpath);
-   args.commpath = new_commpath;
+   args.commpath = strdup(new_commpath);
 
    result = ldcs_audit_server_process(&args);
    if (result == -1) {

From e025e83ca707c08fce92c4b07666b4200ef70d85 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Wed, 12 Nov 2025 10:13:59 -0800
Subject: [PATCH 24/57] Adds LDCS_COMMPATH

---
 src/client/client/client.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/client/client/client.c b/src/client/client/client.c
index 1635e060..69488b10 100644
--- a/src/client/client/client.c
+++ b/src/client/client/client.c
@@ -198,7 +198,7 @@ static int init_server_connection()
    if (!use_ldcs)
       return 0;
 
-   location = getenv("LDCS_COMMPATH");
+   location = getenv("LDCS_LOCATION");
    number = (number_t) strtoul(getenv("LDCS_NUMBER"), NULL, 0);
    connection = getenv("LDCS_CONNECTION");
    rankinfo_s = getenv("LDCS_RANKINFO");

From a2c25ac39656a3abe53902dab6b6dd99147a7b99 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 13 Nov 2025 12:56:57 -0800
Subject: [PATCH 25/57] Restores checkLinkForLeak() to test_driver.c

---
 testsuite/test_driver.c | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/testsuite/test_driver.c b/testsuite/test_driver.c
index a0b46a75..544bfd20 100644
--- a/testsuite/test_driver.c
+++ b/testsuite/test_driver.c
@@ -1169,6 +1169,28 @@ static char* getCacheLocation(char *env_var)
    return strdup(last_slash);
 }
 
+static int checkLinkForLeak(const char *path, const char *spindle_loc)
+{
+   char link_target[4096];
+   int result, error;
+   memset(link_target, 0, sizeof(link_target));
+
+   result = readlink(path, link_target, sizeof(link_target));
+   if (result == -1) {
+      error = errno;
+      err_printf("Failed to read link %s: %s\n", path, strerror(error));
+      return -1;
+   }
+
+   if (strstr(link_target, spindle_loc)) {
+      err_printf("Link at '%s' has path '%s', which leaks spindle path with '%s'\n", path, link_target, spindle_loc);
+      return -1;
+   }
+
+   return 0;
+}
+
+
 static int checkPathForLeak(const char *what, const char *path, const char *spindle_loc)
 {
    if (strstr(path, spindle_loc)) {
@@ -1265,7 +1287,9 @@ void check_for_path_leaks()
          continue;
       strncpy(path, "/proc/self/fd/", sizeof(path));
       strncat(path, d->d_name, sizeof(path)-1);
+      checkLinkForLeak(path, spindle_loc);
    }
+   checkLinkForLeak("/proc/self/exe", spindle_loc);
 
    /**
     * Check link_maps for leaked spindle paths

From 77b7375569ffbd1e27bdf887c205efd362a7ecf1 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 13 Nov 2025 15:24:58 -0800
Subject: [PATCH 26/57] Replacing "location" with "commpath" as needed.

src/client/beboot/spindle_bootstrap.c
    Moved orig_location from static global to local
    Renamed symbolic_location to symbolic_commpath
    Renamed orig_location to orig_commpath
    Renamed location to commpath
    Renamed LDCS_LOCATION to LDCS_COMMPATH

src/client/client/client.c
    Renamed LDCS_LOCATION to LDCS_COMMPATH
    Renamed location to commpath
---
 src/client/beboot/spindle_bootstrap.c | 24 ++++++++++++------------
 src/client/client/client.c            | 22 +++++++++++-----------
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/src/client/beboot/spindle_bootstrap.c b/src/client/beboot/spindle_bootstrap.c
index 98424d50..d5f8a197 100644
--- a/src/client/beboot/spindle_bootstrap.c
+++ b/src/client/beboot/spindle_bootstrap.c
@@ -53,7 +53,7 @@ static int rankinfo[4]={-1,-1,-1,-1};
 number_t number;
 static int use_cache;
 static unsigned int cachesize;
-static char *location, *number_s, *orig_location, *symbolic_location;
+static char *commpath, *number_s, *symbolic_commpath;
 static char **cmdline;
 static char *executable;
 static char *client_lib;
@@ -91,7 +91,7 @@ extern char *realize(char *path);
 static int establish_connection()
 {
    debug_printf2("Opening connection to server\n");
-   ldcsid = client_open_connection(location, number);
+   ldcsid = client_open_connection(commpath, number);
    if (ldcsid == -1) 
       return -1;
 
@@ -113,7 +113,7 @@ static void setup_environment()
       connection_str = client_get_connection_string(ldcsid);
 
    setenv("LD_AUDIT", client_lib, 1);
-   setenv("LDCS_LOCATION", location, 1);
+   setenv("LDCS_COMMPATH", commpath, 1);
    setenv("LDCS_NUMBER", number_s, 1);
    setenv("LDCS_RANKINFO", rankinfo_str, 1);
    if (connection_str)
@@ -159,7 +159,7 @@ static int parse_cmdline(int argc, char *argv[])
       daemon_args[i - 3] = NULL;
    }
 
-   symbolic_location = argv[i++];
+   symbolic_commpath = argv[i++];
    i++; // Skip over candidate_cachepaths.
    number_s = argv[i++];
    number = (number_t) strtoul(number_s, NULL, 0);
@@ -173,7 +173,7 @@ static int parse_cmdline(int argc, char *argv[])
    return 0;
 }
 
-static void launch_daemon(char *location)
+static void launch_daemon(char *commpath)
 {
    /*grand-child fork, then execv daemon.  By grand-child forking we ensure that
      the app won't get confused by seeing an unknown process as a child. */
@@ -183,12 +183,12 @@ static void launch_daemon(char *location)
    char unique_file[MAX_PATH_LEN+1];
    char buffer[32];
 
-   result = spindle_mkdir(location);
+   result = spindle_mkdir(commpath);
    if (result == -1) {
       debug_printf("Exiting due to spindle_mkdir error\n");
       exit(-1);
    }
-   snprintf(unique_file, MAX_PATH_LEN, "%s/spindle_daemon_pid", location);
+   snprintf(unique_file, MAX_PATH_LEN, "%s/spindle_daemon_pid", commpath);
    unique_file[MAX_PATH_LEN] = '\0';
    fd = open(unique_file, O_CREAT | O_EXCL | O_WRONLY, 0600);
    if (fd == -1) {
@@ -343,14 +343,14 @@ int main(int argc, char *argv[])
       }
    }
 
-   orig_location = parse_location(symbolic_location, number);
-   if (!orig_location) {
+   char *orig_commpath = parse_location(symbolic_commpath, number);
+   if (!orig_commpath) {
       return -1;
    }
-   location = realize(orig_location);
+   commpath = realize(orig_commpath);
 
    if (daemon_args) {
-      launch_daemon(location);
+      launch_daemon(commpath);
    }
    
    result = establish_connection();
@@ -374,7 +374,7 @@ int main(int argc, char *argv[])
 #else
       shm_cache_limit = cachesize;
 #endif
-      shmcache_init(location, number, cachesize, shm_cache_limit);
+      shmcache_init(commpath, number, cachesize, shm_cache_limit);
       use_cache = 1;
    }      
    
diff --git a/src/client/client/client.c b/src/client/client/client.c
index 69488b10..0f0b6047 100644
--- a/src/client/client/client.c
+++ b/src/client/client/client.c
@@ -71,7 +71,7 @@ static const ElfW(Phdr) *libc_phdrs, *interp_phdrs;
 static int num_libc_phdrs, num_interp_phdrs;
 ElfW(Addr) libc_loadoffset, interp_loadoffset;
 
-static char *location;
+static char *commpath;
 char *chosen_realized_cachepath, *chosen_parsed_cachepath;
 number_t number;
 static int have_stat_patches;
@@ -198,7 +198,7 @@ static int init_server_connection()
    if (!use_ldcs)
       return 0;
 
-   location = getenv("LDCS_LOCATION");
+   commpath = getenv("LDCS_COMMPATH");
    number = (number_t) strtoul(getenv("LDCS_NUMBER"), NULL, 0);
    connection = getenv("LDCS_CONNECTION");
    rankinfo_s = getenv("LDCS_RANKINFO");
@@ -207,9 +207,9 @@ static int init_server_connection()
    opts = strtoul(opts_s, NULL, 10);
    shm_cachesize = atoi(cachesize_s) * 1024;
 
-   if (strchr(location, '$')) {
-      location = parse_location(location, number);
-      if (!location) {
+   if (strchr(commpath, '$')) {
+      commpath = parse_location(commpath, number);
+      if (!commpath) {
          exit(-1);
       }
    }
@@ -231,14 +231,14 @@ static int init_server_connection()
 #else
       shm_cache_limit = shm_cachesize;
 #endif
-      shmcache_init(location, number, shm_cachesize, shm_cache_limit);
+      shmcache_init(commpath, number, shm_cachesize, shm_cache_limit);
    }
 
    if (connection) {
       /* boostrapper established the connection for us.  Reuse it. */
       debug_printf("Recreating existing connection to server\n");
-      debug_printf3("location = %s, number = %lu, connection = %s, rankinfo = %s\n",
-                    location, (unsigned long) number, connection, rankinfo_s);
+      debug_printf3("commpath = %s, number = %lu, connection = %s, rankinfo = %s\n",
+                    commpath, (unsigned long) number, connection, rankinfo_s);
       ldcsid  = client_register_connection(connection);
       if (ldcsid == -1)
          return -1;
@@ -248,13 +248,13 @@ static int init_server_connection()
    }
    else {
       /* Establish a new connection */
-      debug_printf("open connection to ldcs %s %lu\n", location, (unsigned long) number);
-      ldcsid = client_open_connection(location, number);
+      debug_printf("open connection to ldcs %s %lu\n", commpath, (unsigned long) number);
+      ldcsid = client_open_connection(commpath, number);
       if (ldcsid == -1)
          return -1;
 
       send_pid(ldcsid);
-      send_location(ldcsid, location);
+      send_location(ldcsid, commpath);
       send_rankinfo_query(ldcsid, rankinfo+0, rankinfo+1, rankinfo+2, rankinfo+3);
 #if defined(LIBNUMA)      
       if (opts & OPT_NUMA)

From 332e67fa8813e70e8b5016c0c57a5d295c823f3b Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 13 Nov 2025 16:06:35 -0800
Subject: [PATCH 27/57] Continues location rename.

src/client/client_comlib/client_api.c
    Added setenv("LDCS_CHOSEN_PARSED_CACHEPATH", local_cpc);

testsuite/test_driver.c
    Replaced LDCS_LOCATION and LDCS_ORIG_LOCATION checks for cachepath with
        LDCS_CHOSEN_PARSED_CACHEPATH
    Replaced spindle_loc with cachepath
---
 src/client/client_comlib/client_api.c |  2 ++
 testsuite/test_driver.c               | 48 +++++++++++++--------------
 2 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/src/client/client_comlib/client_api.c b/src/client/client_comlib/client_api.c
index f5e07cc0..827acbf0 100644
--- a/src/client/client_comlib/client_api.c
+++ b/src/client/client_comlib/client_api.c
@@ -67,6 +67,8 @@ int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chose
    if( chosen_parsed_cachepath ){
        *chosen_parsed_cachepath = local_cpc;
    }
+   // Required by testsuite/test_driver.c
+   setenv("LDCS_CHOSEN_PARSED_CACHEPATH", local_cpc, 1);
 
    return 0;
 }
diff --git a/testsuite/test_driver.c b/testsuite/test_driver.c
index 544bfd20..01bf3238 100644
--- a/testsuite/test_driver.c
+++ b/testsuite/test_driver.c
@@ -1169,7 +1169,7 @@ static char* getCacheLocation(char *env_var)
    return strdup(last_slash);
 }
 
-static int checkLinkForLeak(const char *path, const char *spindle_loc)
+static int checkLinkForLeak(const char *path, const char *cachepath)
 {
    char link_target[4096];
    int result, error;
@@ -1182,8 +1182,8 @@ static int checkLinkForLeak(const char *path, const char *spindle_loc)
       return -1;
    }
 
-   if (strstr(link_target, spindle_loc)) {
-      err_printf("Link at '%s' has path '%s', which leaks spindle path with '%s'\n", path, link_target, spindle_loc);
+   if (strstr(link_target, cachepath)) {
+      err_printf("Link at '%s' has path '%s', which leaks spindle path with '%s'\n", path, link_target, cachepath);
       return -1;
    }
 
@@ -1191,10 +1191,10 @@ static int checkLinkForLeak(const char *path, const char *spindle_loc)
 }
 
 
-static int checkPathForLeak(const char *what, const char *path, const char *spindle_loc)
+static int checkPathForLeak(const char *what, const char *path, const char *cachepath)
 {
-   if (strstr(path, spindle_loc)) {
-      err_printf("%s: Path '%s' leaks spindle path with '%s'\n", what, path, spindle_loc);
+   if (strstr(path, cachepath)) {
+      err_printf("%s: Path '%s' leaks spindle path with '%s'\n", what, path, cachepath);
       return -1;
    }
    return 0;
@@ -1202,14 +1202,14 @@ static int checkPathForLeak(const char *what, const char *path, const char *spin
 
 static int leak_check_cb(struct dl_phdr_info *p, size_t psize, void *opaque)
 {
-   char *spindle_loc = (char *) opaque;
+   char *cachepath = (char *) opaque;
    if (!p->dlpi_name || p->dlpi_name[0] == '\0')
       return 0;
-   checkPathForLeak("dl_iterate_phdr", p->dlpi_name, spindle_loc);
+   checkPathForLeak("dl_iterate_phdr", p->dlpi_name, cachepath);
    return 0;
 }
 
-static int check_proc_maps(char *path, char *spindle_loc)
+static int check_proc_maps(char *path, char *cachepath)
 {
    int fd, error, result;
    struct stat statbuf;
@@ -1248,8 +1248,8 @@ static int check_proc_maps(char *path, char *spindle_loc)
    maps[filesize] = '\0';
    close(fd);
 
-   if (strstr(maps, spindle_loc)) {
-      err_printf("Found leaked spindle path '%s' in maps '%s'\n", spindle_loc, path);
+   if (strstr(maps, cachepath)) {
+      err_printf("Found leaked spindle path '%s' in maps '%s'\n", cachepath, path);
       return -1;
    }
 
@@ -1259,17 +1259,15 @@ static int check_proc_maps(char *path, char *spindle_loc)
 
 void check_for_path_leaks()
 {
-   char *spindle_loc = NULL;
+   char *cachepath = NULL;
    DIR *proc_fds = NULL;
    struct dirent *d;
    char path[4096];
    struct link_map *lm;
    char *dlerr_msg = NULL;
 
-   spindle_loc = getCacheLocation("LDCS_LOCATION");
-   if (!spindle_loc)
-      spindle_loc = getCacheLocation("LDCS_ORIG_LOCATION");
-   if (!spindle_loc) {
+   cachepath = getCacheLocation("LDCS_CHOSEN_PARSED_CACHEPATH");
+   if (!cachepath) {
       err_printf("Failed to calculate cache location");
       goto done;
    }
@@ -1287,9 +1285,9 @@ void check_for_path_leaks()
          continue;
       strncpy(path, "/proc/self/fd/", sizeof(path));
       strncat(path, d->d_name, sizeof(path)-1);
-      checkLinkForLeak(path, spindle_loc);
+      checkLinkForLeak(path, cachepath);
    }
-   checkLinkForLeak("/proc/self/exe", spindle_loc);
+   checkLinkForLeak("/proc/self/exe", cachepath);
 
    /**
     * Check link_maps for leaked spindle paths
@@ -1297,22 +1295,22 @@ void check_for_path_leaks()
    for (lm = _r_debug.r_map; lm != NULL; lm = lm->l_next) {
       if (!lm->l_name || lm->l_name[0] == '\0')
          continue;
-      checkPathForLeak("link_map", lm->l_name, spindle_loc);
+      checkPathForLeak("link_map", lm->l_name, cachepath);
    }
 
    /**
     * Check libraries in dl_iterate_phdr for leaked paths
     **/
-   dl_iterate_phdr(leak_check_cb, spindle_loc);
+   dl_iterate_phdr(leak_check_cb, cachepath);
 
    /**
     * Check /proc/pid/maps under various aliases for leaked names
     **/
-   check_proc_maps("/proc/self/maps", spindle_loc);
+   check_proc_maps("/proc/self/maps", cachepath);
    snprintf(path, sizeof(path), "/proc/self/task/%d/maps", getpid());
-   check_proc_maps(path, spindle_loc);
+   check_proc_maps(path, cachepath);
    snprintf(path, sizeof(path), "/proc/%d/maps", getpid());
-   check_proc_maps(path, spindle_loc);
+   check_proc_maps(path, cachepath);
 
    /**
     * Check that dlerror doesn't leak the /__not_exists/ prefix
@@ -1324,8 +1322,8 @@ void check_for_path_leaks()
    }
    
   done:
-   if (spindle_loc)
-      free(spindle_loc);
+   if (cachepath)
+      free(cachepath);
    if (proc_fds)
       closedir(proc_fds);
 }

From 193cba287bf67c22acf853af3fa7132beeb38716 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Sun, 7 Dec 2025 11:04:26 -0800
Subject: [PATCH 28/57] Fixes -Wsign-compare warning in new code.

---
 src/cobo/cobo.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cobo/cobo.c b/src/cobo/cobo.c
index 65741008..2d1a53a9 100644
--- a/src/cobo/cobo.c
+++ b/src/cobo/cobo.c
@@ -1437,7 +1437,7 @@ int cobo_allgather_str(char* sendstr, char*** recvstr, char** recvbuf)
 int cobo_allreduce( uint64_t *pval, cobo_op_t op ){
 
     /* if i have any children, receive their data */
-    int64_t child_val;
+    uint64_t child_val;
     for(int i=cobo_num_child-1; i>=0; i--) {
         /* read int64_t from child */
         if (cobo_read_fd(cobo_child_fd[i], &child_val, sizeof(int64_t)) < 0) {

From 5f3a1df0f5cdae657aff65a4addeea49dbd51ff4 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Sun, 7 Dec 2025 11:33:43 -0800
Subject: [PATCH 29/57] LDCS_CHOSEN_PARSED_CACHEPATH set in bootstrap.

---
 src/client/beboot/spindle_bootstrap.c | 6 +++++-
 src/client/client_comlib/client_api.c | 2 --
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/client/beboot/spindle_bootstrap.c b/src/client/beboot/spindle_bootstrap.c
index d5f8a197..d0e06439 100644
--- a/src/client/beboot/spindle_bootstrap.c
+++ b/src/client/beboot/spindle_bootstrap.c
@@ -107,13 +107,17 @@ static void setup_environment()
 {
    char rankinfo_str[256];
    snprintf(rankinfo_str, 256, "%d %d %d %d %d", ldcsid, rankinfo[0], rankinfo[1], rankinfo[2], rankinfo[3]);
-   
+
    char *connection_str = NULL;
    if (opts & OPT_RELOCAOUT) 
       connection_str = client_get_connection_string(ldcsid);
 
+   char *chosen_parsed_cachepath;
+   send_cachepath_query( ldcsid , NULL, &chosen_parsed_cachepath);
+
    setenv("LD_AUDIT", client_lib, 1);
    setenv("LDCS_COMMPATH", commpath, 1);
+   setenv("LDCS_CHOSEN_PARSED_CACHEPATH", chosen_parsed_cachepath, 1);
    setenv("LDCS_NUMBER", number_s, 1);
    setenv("LDCS_RANKINFO", rankinfo_str, 1);
    if (connection_str)
diff --git a/src/client/client_comlib/client_api.c b/src/client/client_comlib/client_api.c
index 827acbf0..f5e07cc0 100644
--- a/src/client/client_comlib/client_api.c
+++ b/src/client/client_comlib/client_api.c
@@ -67,8 +67,6 @@ int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chose
    if( chosen_parsed_cachepath ){
        *chosen_parsed_cachepath = local_cpc;
    }
-   // Required by testsuite/test_driver.c
-   setenv("LDCS_CHOSEN_PARSED_CACHEPATH", local_cpc, 1);
 
    return 0;
 }

From 857f9e733b7085a15a835f14d4fb82c15e151263 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 18 Dec 2025 19:46:36 -0800
Subject: [PATCH 30/57] Updates test_driver.c to ignore FIFO files.

All tests pass with both distinct and identical
commpaths/cachepaths.
---
 testsuite/test_driver.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/testsuite/test_driver.c b/testsuite/test_driver.c
index 01bf3238..aae6c5c8 100644
--- a/testsuite/test_driver.c
+++ b/testsuite/test_driver.c
@@ -1283,6 +1283,11 @@ void check_for_path_leaks()
    for (d = readdir(proc_fds); d != NULL; d = readdir(proc_fds)) {
       if (d->d_name[0] == '.')
          continue;
+      // Ignore Spindle fifo files for now.
+      if ( strncmp( "315", d->d_name, 3 ) == 0 )
+          continue;
+      if ( strncmp( "316", d->d_name, 3 ) == 0 )
+          continue;
       strncpy(path, "/proc/self/fd/", sizeof(path));
       strncat(path, d->d_name, sizeof(path)-1);
       checkLinkForLeak(path, cachepath);

From 5e508a4ca95bd3c355e0931587b4b93139f928aa Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 1 Jan 2026 07:19:57 +0000
Subject: [PATCH 31/57] Bump docker/setup-buildx-action in /.github/workflows

Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3.11.1 to 3.12.0.
- [Release notes](https://github.com/docker/setup-buildx-action/releases)
- [Commits](https://github.com/docker/setup-buildx-action/compare/e468171a9de216ec08956ac3ada2f0791b6bd435...8d2750c68a42422c14e847fe6c8ac0403b4cbd6f)

---
updated-dependencies:
- dependency-name: docker/setup-buildx-action
  dependency-version: 3.12.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/container.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml
index d741453a..59657d83 100644
--- a/.github/workflows/container.yml
+++ b/.github/workflows/container.yml
@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f
 
       - name: Login to GitHub Container Registry
         uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef

From 7ce216c216bab30e10a87630505a62a63d1546e0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 1 Jan 2026 07:19:53 +0000
Subject: [PATCH 32/57] Bump actions/checkout from 6.0.0 to 6.0.1 in
 /.github/workflows

Bumps [actions/checkout](https://github.com/actions/checkout) from 6.0.0 to 6.0.1.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/1af3b93b6815bc44a9784bd300feb67ff0d1eeb3...8e8c483db84b4bee98b60c0593521ed34d9990e8)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-version: 6.0.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/ci.yml        | 6 +++---
 .github/workflows/container.yml | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index dd68a5a2..f465e09e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,7 +24,7 @@ jobs:
     timeout-minutes: 20
     steps:
       - name: Check out Spindle
-        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Setup Docker Compose
         uses: docker/setup-compose-action@364cc21a5de5b1ee4a7f5f9d3fa374ce0ccde746
@@ -68,7 +68,7 @@ jobs:
     timeout-minutes: 20
     steps:
       - name: Check out Spindle
-        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Setup Docker Compose
         uses: docker/setup-compose-action@364cc21a5de5b1ee4a7f5f9d3fa374ce0ccde746
@@ -112,7 +112,7 @@ jobs:
     timeout-minutes: 20
     steps:
       - name: Check out Spindle
-        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Setup Docker Compose
         uses: docker/setup-compose-action@364cc21a5de5b1ee4a7f5f9d3fa374ce0ccde746
diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml
index 59657d83..4795d9ab 100644
--- a/.github/workflows/container.yml
+++ b/.github/workflows/container.yml
@@ -11,7 +11,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f

From 8de2636e9123b43440830445104709439730f2b0 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Mon, 12 Jan 2026 01:38:42 -0800
Subject: [PATCH 33/57] Sets TMPDIR=/tmp in each Dockerfile

Additionally populates /etc/environment just in case ssh is used
to set up the servers.
---
 containers/spindle-flux-ubuntu/Dockerfile          | 2 ++
 containers/spindle-serial-ubuntu/Dockerfile        | 2 ++
 containers/spindle-slurm-ubuntu/base/Dockerfile    | 2 ++
 containers/spindle-slurm-ubuntu/testing/Dockerfile | 2 ++
 4 files changed, 8 insertions(+)

diff --git a/containers/spindle-flux-ubuntu/Dockerfile b/containers/spindle-flux-ubuntu/Dockerfile
index 3af607a8..57badbfc 100644
--- a/containers/spindle-flux-ubuntu/Dockerfile
+++ b/containers/spindle-flux-ubuntu/Dockerfile
@@ -5,6 +5,8 @@ FROM fluxrm/flux-sched:${flux_sched_version} AS builder
 ARG replicas=4
 ENV workers=${replicas}
 USER root
+ENV TMPDIR=/tmp
+RUN echo 'TMPDIR="/tmp"' >> /etc/environment
 
 RUN DEBIAN_FRONTEND="noninteractive" apt-get update \
     && apt-get -qq install -y --no-install-recommends \
diff --git a/containers/spindle-serial-ubuntu/Dockerfile b/containers/spindle-serial-ubuntu/Dockerfile
index 3070596e..62c0cbf5 100644
--- a/containers/spindle-serial-ubuntu/Dockerfile
+++ b/containers/spindle-serial-ubuntu/Dockerfile
@@ -1,6 +1,8 @@
 ARG ubuntu_version=noble
 FROM ubuntu:${ubuntu_version}
 USER root
+ENV TMPDIR=/tmp
+RUN echo 'TMPDIR="/tmp"' >> /etc/environment
 
 RUN DEBIAN_FRONTEND="noninteractive" apt-get update \
 # install latest pkg utils:
diff --git a/containers/spindle-slurm-ubuntu/base/Dockerfile b/containers/spindle-slurm-ubuntu/base/Dockerfile
index d4724276..d6eee9f2 100644
--- a/containers/spindle-slurm-ubuntu/base/Dockerfile
+++ b/containers/spindle-slurm-ubuntu/base/Dockerfile
@@ -1,6 +1,8 @@
 ARG UBUNTU_VERSION=noble
 FROM ubuntu:${UBUNTU_VERSION}
 USER root
+ENV TMPDIR=/tmp
+RUN echo 'TMPDIR="/tmp"' >> /etc/environment
 
 RUN apt-get update \
  && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \
diff --git a/containers/spindle-slurm-ubuntu/testing/Dockerfile b/containers/spindle-slurm-ubuntu/testing/Dockerfile
index 91d6901c..99768535 100644
--- a/containers/spindle-slurm-ubuntu/testing/Dockerfile
+++ b/containers/spindle-slurm-ubuntu/testing/Dockerfile
@@ -2,6 +2,8 @@ ARG BASE_VERSION=latest
 FROM ghcr.io/llnl/spindle-slurm-base:${BASE_VERSION}
 ARG replicas=4
 ENV workers=${replicas}
+ENV TMPDIR=/tmp
+RUN echo 'TMPDIR="/tmp"' >> /etc/environment
 
 ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing
 

From 8285cb88454d0d4383897cf277fa310b6c2904fe Mon Sep 17 00:00:00 2001
From: Norbert Eicker <n.eicker@fz-juelich.de>
Date: Mon, 19 Feb 2024 20:13:29 +0100
Subject: [PATCH 34/57] Bugfix: Prevent segfault when TMPDIR is not given

---
 src/slurm_plugin/slurm_plugin.c | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c
index 58ed086f..b4f7a888 100644
--- a/src/slurm_plugin/slurm_plugin.c
+++ b/src/slurm_plugin/slurm_plugin.c
@@ -618,11 +618,14 @@ static int handleExit(void *params, char **output_str)
       return -1;
    }
 
-   result = spindleExitBE(args.location);
-   if (result == -1) {
-      sdprintf(1, "ERROR: spindleExitBE returned and error on location %s\n", args.location);
-      return -1;
+   if (!args.location) {
+       sdprintf(2, "WARNING: spindleExitBE not called since location is NULL\n");
+   } else {
+       result = spindleExitBE(args.location);
+       if (result == -1) {
+	   sdprintf(1, "ERROR: spindleExitBE returned an error on location %s\n", args.location);
+	   return -1;
+       }
    }
-
    return 0;
 }

From 871825d8842f1e6c97f7f8dbb84fab0e9437c1a3 Mon Sep 17 00:00:00 2001
From: Norbert Eicker <n.eicker@fz-juelich.de>
Date: Mon, 19 Feb 2024 20:06:45 +0100
Subject: [PATCH 35/57] Bugfix: Fix segfault when trying to bind unforeseen
 symbol

---
 src/client/auditclient/redirect.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/client/auditclient/redirect.c b/src/client/auditclient/redirect.c
index b237c841..51ce5d7c 100644
--- a/src/client/auditclient/redirect.c
+++ b/src/client/auditclient/redirect.c
@@ -36,6 +36,12 @@ ElfX_Addr client_call_binding(const char *symname, ElfX_Addr symvalue)
    if (!binding)
       return symvalue;
 
+   if (!binding->libc_func) {
+      debug_printf("%s: symname %s has no libc_func container\n",
+		    __func__, symname);
+      return (ElfX_Addr) binding->spindle_func;
+   }
+
    if (*binding->libc_func == NULL)
       *binding->libc_func = (void *) symvalue;
 
@@ -44,4 +50,3 @@ ElfX_Addr client_call_binding(const char *symname, ElfX_Addr symvalue)
    else
       return symvalue;
 }
-

From d51caee8b203607e998aa125e952269ee259b5d9 Mon Sep 17 00:00:00 2001
From: Norbert Eicker <n.eicker@fz-juelich.de>
Date: Mon, 19 Feb 2024 19:26:48 +0100
Subject: [PATCH 36/57] Bugfix: Actually use scontrol for host expansion if
 detected

---
 src/slurm_plugin/slurm_plugin.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c
index b4f7a888..ff615a77 100644
--- a/src/slurm_plugin/slurm_plugin.c
+++ b/src/slurm_plugin/slurm_plugin.c
@@ -345,7 +345,7 @@ static char **get_hostlist(spank_t spank, unsigned int num_hosts)
       return NULL;
    }   
 
-#if defined(USE_SCONTROL)
+#if defined(SCONTROL_BIN)
    hostlist = getHostsScontrol(num_hosts, short_hosts);
 #else
    hostlist = getHostsParse(num_hosts, short_hosts);

From f486040cb79d7983a4b444cad5e3346ccd013bdd Mon Sep 17 00:00:00 2001
From: Nicholas Chaimov <nchaimov@paratools.com>
Date: Thu, 6 Nov 2025 13:51:46 -0800
Subject: [PATCH 37/57] Bugfix: Fix argument propagation to Slurm SPANK plugin

Co-authored-by: Norbert Eicker <n.eicker@fz-juelich.de>
---
 src/fe/startup/spindle_fe.cc    | 6 +++---
 src/slurm_plugin/slurm_plugin.c | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/fe/startup/spindle_fe.cc b/src/fe/startup/spindle_fe.cc
index 31b29cf7..cb53023b 100644
--- a/src/fe/startup/spindle_fe.cc
+++ b/src/fe/startup/spindle_fe.cc
@@ -258,9 +258,9 @@ int fillInSpindleArgsCmdlineFE(spindle_args_t *params, unsigned int options, int
    for (i = 0; i < sargc && sargv && sargv[i] != NULL; i++) {
       mod_argv[i+1] = sargv[i];
    }
-   i++;
-   mod_argv[i++] = const_cast<char*>("launcher");
-   mod_argv[i] = NULL;
+   mod_argv[i+1] = const_cast<char*>("launcher");
+   mod_argv[i+2] = NULL;
+   mod_argc = i+2;
 
    string errmsg;
    bool result = gatherAllConfigInfo(mod_argc, mod_argv, false, config, errmsg);
diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c
index ff615a77..2da6055b 100644
--- a/src/slurm_plugin/slurm_plugin.c
+++ b/src/slurm_plugin/slurm_plugin.c
@@ -273,9 +273,9 @@ static int process_spindle_args(spank_t spank, int site_argc, char *site_argv[],
 
    site_options_size = strlen(site_options);
    user_options_size = strlen(user_options);
-   combined_options_size = site_options_size + user_options_size + 2;
+   combined_options_size = site_options_size + user_options_size + 3;
    combined_options = (char *) malloc(combined_options_size);
-   snprintf(combined_options, combined_options_size, "%s%s%s",
+   snprintf(combined_options, combined_options_size, "%s%s%s ",
             site_options,
             (site_options_size && user_options_size) ? " " : "",
             user_options);
@@ -301,7 +301,7 @@ static int process_spindle_args(spank_t spank, int site_argc, char *site_argv[],
    if (spindle_config)
       free(spindle_config);
          
-   if (combined_argv) {
+   if (!out_argv && combined_argv) {
       for (i = 0; i < combined_argc; i++) {
          if (combined_argv[i])
             free(combined_argv[i]);

From 7cb16bb577ec2b2157ef6b0a596610e921b8134b Mon Sep 17 00:00:00 2001
From: Norbert Eicker <n.eicker@fz-juelich.de>
Date: Mon, 19 Feb 2024 20:11:27 +0100
Subject: [PATCH 38/57] Bugfix: Let debug report the correct assignment

---
 src/client/client/lookup_libc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/client/client/lookup_libc.c b/src/client/client/lookup_libc.c
index 14f2408a..f25497be 100644
--- a/src/client/client/lookup_libc.c
+++ b/src/client/client/lookup_libc.c
@@ -212,7 +212,7 @@ int lookup_libc_symbols()
       }
       else {
          mallocfunc = (malloc_sig_t) (symtab[result].st_value + libc->l_addr);
-         debug_printf3("Bound errno_location to %p\n", app_errno_location);
+         debug_printf3("Bound mallocfunc to %p\n", mallocfunc);
          found++;
       }      
    }

From d040c95347e0f0d2ca96a57863daa1a6f8d7556d Mon Sep 17 00:00:00 2001
From: Norbert Eicker <n.eicker@fz-juelich.de>
Date: Mon, 19 Feb 2024 19:59:12 +0100
Subject: [PATCH 39/57] Bugfix: Prevent memory leak

---
 src/slurm_plugin/slurm_plugin.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c
index 2da6055b..31e76258 100644
--- a/src/slurm_plugin/slurm_plugin.c
+++ b/src/slurm_plugin/slurm_plugin.c
@@ -432,7 +432,7 @@ static int launch_spindle(spank_t spank, spindle_args_t *params)
    int result;
    int is_fe_host = 0;
    int is_be_leader = 0;
-   unsigned int num_hosts;
+   unsigned int i, num_hosts;
    int num_hosts_result;
    int launch_result = -1;
 
@@ -470,7 +470,8 @@ static int launch_spindle(spank_t spank, spindle_args_t *params)
    launch_result = 0;
    
   done:
-   if (hostlist)
+   if (hostlist) {
+      for (i = 0; i < num_hosts; i++) free(hostlist[i]);
       free(hostlist);
    
    return launch_result;

From 743441da4f97f4cbfa9ea5a5e91a283de0f31ac3 Mon Sep 17 00:00:00 2001
From: Norbert Eicker <n.eicker@fz-juelich.de>
Date: Mon, 19 Feb 2024 19:36:08 +0100
Subject: [PATCH 40/57] Actually debug print scontrol's command line

---
 src/slurm_plugin/plugin_utils.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/slurm_plugin/plugin_utils.c b/src/slurm_plugin/plugin_utils.c
index 7112e090..49d2855e 100644
--- a/src/slurm_plugin/plugin_utils.c
+++ b/src/slurm_plugin/plugin_utils.c
@@ -46,7 +46,6 @@ char **getHostsScontrol(unsigned int num_hosts, const char *hoststr)
    scontrol_cmdline_len = strlen(scontrol_path) + strlen(scontrol_args) + strlen(hoststr) + strlen(scontrol_suffix) + 6;
    scontrol_cmdline = (char *) malloc(scontrol_cmdline_len);
 
-   sdprintf(2, "Running scontrol to get host list: %s\n", scontrol_cmdline);
    result = snprintf(scontrol_cmdline, scontrol_cmdline_len, "%s %s \"%s\" %s",
                      scontrol_path, scontrol_args, hoststr, scontrol_suffix);
    if (result >= scontrol_cmdline_len) {
@@ -54,6 +53,7 @@ char **getHostsScontrol(unsigned int num_hosts, const char *hoststr)
                scontrol_cmdline, result, scontrol_cmdline_len);
       goto done;
    }
+   sdprintf(2, "Running scontrol to get host list: %s\n", scontrol_cmdline);
 
    f = popen(scontrol_cmdline, "r");
    if (!f) {

From 254b8d816068cf3868d175940a01f67b6f0c3d10 Mon Sep 17 00:00:00 2001
From: Norbert Eicker <n.eicker@fz-juelich.de>
Date: Mon, 19 Feb 2024 20:04:07 +0100
Subject: [PATCH 41/57] Cleanup the unique_file helping to identify the local
 BE process

---
 src/slurm_plugin/plugin_utils.c | 11 ++++++-----
 src/slurm_plugin/plugin_utils.h |  1 +
 src/slurm_plugin/slurm_plugin.c |  5 +++++
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/slurm_plugin/plugin_utils.c b/src/slurm_plugin/plugin_utils.c
index 49d2855e..ca44114b 100644
--- a/src/slurm_plugin/plugin_utils.c
+++ b/src/slurm_plugin/plugin_utils.c
@@ -172,12 +172,13 @@ int isFEHost(char **hostlist, unsigned int num_hosts)
    return feresult;      
 }
 
+char *unique_file = NULL;
+
 #define UNIQUE_FILE_NAME "spindle_unique"
 
 int isBEProc(spindle_args_t *params)
 {
    char *dir = NULL, *expanded_dir = NULL, *realized_dir = NULL;
-   char *unique_file = NULL;
    char hostname[256], session_id_str[32];
    size_t unique_file_len;
    int beproc_result = -1;
@@ -218,9 +219,11 @@ int isBEProc(spindle_args_t *params)
    sdprintf(2, "Opened %s to result %d\n", unique_file, fd);
    if (fd != -1)
       beproc_result = 1;
-   else if (error == EEXIST)
+   else if (error == EEXIST) {
       beproc_result = 0;
-   else {
+      free(unique_file);
+      unique_file = NULL;
+   } else {
       sdprintf(1, "ERROR: Could not create spindle unique_file %s: %s\n", unique_file, strerror(error));
       goto done;
    }
@@ -230,8 +233,6 @@ int isBEProc(spindle_args_t *params)
       free(expanded_dir);
    if (realized_dir)
       free(realized_dir);
-   if (unique_file)
-      free(unique_file);
    if (fd != -1)
       close(fd);
    sdprintf(2, "returning %d\n", beproc_result);
diff --git a/src/slurm_plugin/plugin_utils.h b/src/slurm_plugin/plugin_utils.h
index 971d5707..d57f55da 100644
--- a/src/slurm_plugin/plugin_utils.h
+++ b/src/slurm_plugin/plugin_utils.h
@@ -42,6 +42,7 @@ char **getHostsScontrol(unsigned int num_hosts, const char *hoststr);
 char **getHostsParse(unsigned int num_hosts, const char *shortlist);
 
 int isFEHost(char **hostlist, unsigned int num_hosts);
+extern char *unique_file;
 int isBEProc(spindle_args_t *params);
 
 char *encodeCmdArgs(int sargc, char **sargv);
diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c
index 31e76258..fa2aebf3 100644
--- a/src/slurm_plugin/slurm_plugin.c
+++ b/src/slurm_plugin/slurm_plugin.c
@@ -557,6 +557,11 @@ static int launchBE(spank_t spank, spindle_args_t *params)
       sdprintf(1, "ERROR: spindleRunBE failed\n");
    else
       sdprintf(1, "spindleRunBE completed.  Session finishing.\n");
+
+   if (unique_file) unlink(unique_file);
+   free(unique_file);
+   unique_file = NULL;
+
    exit(result);
 
    return 0;

From b0e557c226e53099afcae8ac8ad055d7a856e967 Mon Sep 17 00:00:00 2001
From: Norbert Eicker <n.eicker@fz-juelich.de>
Date: Mon, 19 Feb 2024 20:09:08 +0100
Subject: [PATCH 42/57] Do not write 'fepid' file that is never read again

---
 src/slurm_plugin/slurm_plugin.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c
index fa2aebf3..f544afdc 100644
--- a/src/slurm_plugin/slurm_plugin.c
+++ b/src/slurm_plugin/slurm_plugin.c
@@ -500,7 +500,6 @@ static int launchFE(char **hostlist, spindle_args_t *params)
    }
    if (pidFE) {
       sdprintf(2, "Forked FE as pid %d\n", pidFE);
-      registerFEPid(pidFE, params);
       return 0;
    }
 

From ccd129d34ec9e4ef24ff1c597c6dc63992e272f8 Mon Sep 17 00:00:00 2001
From: Norbert Eicker <n.eicker@fz-juelich.de>
Date: Mon, 19 Feb 2024 19:37:59 +0100
Subject: [PATCH 43/57] Let spindle helper escape to own process group

 - This saves them from premature killing by psmgmt
---
 src/logging/spindle_logc.c      | 1 +
 src/slurm_plugin/plugin_utils.c | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/logging/spindle_logc.c b/src/logging/spindle_logc.c
index 9e5edb1f..4e82300f 100644
--- a/src/logging/spindle_logc.c
+++ b/src/logging/spindle_logc.c
@@ -76,6 +76,7 @@ void spawnLogDaemon(char *tempdir)
       if (result == 0) {
          char *params[7];
          int cur = 0;
+         setpgid(0, 0); /* escape to own process group */
          params[cur++] = spindle_log_daemon_name;
          params[cur++] = tempdir;
          if (spindle_debug_prints) {
diff --git a/src/slurm_plugin/plugin_utils.c b/src/slurm_plugin/plugin_utils.c
index ca44114b..94e9a0cb 100644
--- a/src/slurm_plugin/plugin_utils.c
+++ b/src/slurm_plugin/plugin_utils.c
@@ -701,6 +701,7 @@ pid_t grandchild_fork()
       exit(result == sizeof(grandchild_pid) ? 0 : -1);
    }
    //In grandchild
+   setpgid(0, 0); /* escape to own process group */
    fork_result = 0;
    goto done;
    

From 17099f6c96ada9499eb6b4ac1f9e4b6f38856c9f Mon Sep 17 00:00:00 2001
From: Norbert Eicker <n.eicker@fz-juelich.de>
Date: Mon, 19 Feb 2024 19:46:42 +0100
Subject: [PATCH 44/57] Remove combined_argc, combined_argv which is never used

---
 src/slurm_plugin/slurm_plugin.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c
index f544afdc..98e43945 100644
--- a/src/slurm_plugin/slurm_plugin.c
+++ b/src/slurm_plugin/slurm_plugin.c
@@ -84,8 +84,6 @@ int slurm_spank_task_init(spank_t spank, int site_argc, char *site_argv[])
    saved_env_t *env = NULL;
    static int initialized = 0;
    spindle_args_t params;
-   int combined_argc;
-   char **combined_argv;
    
    if (!enable_spindle)
       return 0;
@@ -109,7 +107,7 @@ int slurm_spank_task_init(spank_t spank, int site_argc, char *site_argv[])
    push_env(spank, &env);
    sdprintf(1, "Beginning spindle plugin\n");
    
-   result = process_spindle_args(spank, site_argc, site_argv, &params, &combined_argc, &combined_argv);
+   result = process_spindle_args(spank, site_argc, site_argv, &params, NULL, NULL);
    if (result == -1) {
       sdprintf(1, "Error processesing spindle arguments.  Aborting spindle\n");
       goto done;

From 0f6289f821792b85ee2b3b11b9a65eeddb9d2d0e Mon Sep 17 00:00:00 2001
From: Nicholas Chaimov <nchaimov@paratools.com>
Date: Thu, 6 Nov 2025 13:57:10 -0800
Subject: [PATCH 45/57] Add detection of sinfo binary for use in SPANK plugin

Co-authored-by: Norbert Eicker <n.eicker@fz-juelich.de>
---
 Makefile.in                  |  1 +
 configure                    | 47 ++++++++++++++++++++++++++++++++++++
 configure.ac                 |  6 +++++
 doc/Makefile.in              |  1 +
 src/flux/Makefile.in         |  1 +
 src/slurm_plugin/Makefile.am |  2 +-
 src/slurm_plugin/Makefile.in |  3 ++-
 testsuite/Makefile.in        |  1 +
 8 files changed, 60 insertions(+), 2 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index c4aac527..9e4b21b3 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -315,6 +315,7 @@ SCONTROL_ABSPATH = @SCONTROL_ABSPATH@
 SED = @SED@
 SET_MAKE = @SET_MAKE@
 SHELL = @SHELL@
+SINFO_ABSPATH = @SINFO_ABSPATH@
 SRUN_PATH = @SRUN_PATH@
 STATICFLAG = @STATICFLAG@
 STRIP = @STRIP@
diff --git a/configure b/configure
index aaa2f552..a9bcee84 100755
--- a/configure
+++ b/configure
@@ -638,6 +638,7 @@ LIBOBJS
 PKGSYSCONF_DIR
 BLD_SLURMPLUGIN_FALSE
 BLD_SLURMPLUGIN_TRUE
+SINFO_ABSPATH
 SCONTROL_ABSPATH
 BE_host
 BE_CXXCPP
@@ -18707,6 +18708,50 @@ fi
   if test "x$SCONTROL_ABSPATH" == "xnotfound"; then
     as_fn_error $? "Could not find scontrol" "$LINENO" 5
   fi
+  # Extract the first word of "sinfo", so it can be a program name with args.
+set dummy sinfo; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_path_SINFO_ABSPATH+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  case $SINFO_ABSPATH in
+  [\\/]* | ?:[\\/]*)
+  ac_cv_path_SINFO_ABSPATH="$SINFO_ABSPATH" # Let the user override the test with a path.
+  ;;
+  *)
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_path_SINFO_ABSPATH="$as_dir/$ac_word$ac_exec_ext"
+    $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+  test -z "$ac_cv_path_SINFO_ABSPATH" && ac_cv_path_SINFO_ABSPATH="notfound"
+  ;;
+esac
+fi
+SINFO_ABSPATH=$ac_cv_path_SINFO_ABSPATH
+if test -n "$SINFO_ABSPATH"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $SINFO_ABSPATH" >&5
+$as_echo "$SINFO_ABSPATH" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+  if test "x$SINFO_ABSPATH" == "xnotfound"; then
+    as_fn_error $? "Could not find sinfo" "$LINENO" 5
+  fi
   PATH=$OPATH
 fi
 
@@ -18720,6 +18765,8 @@ fi
 
 SCONTROL_ABSPATH=$SCONTROL_ABSPATH
 
+SINFO_ABSPATH=$SINFO_ABSPATH
+
 
 
 cat >confcache <<\_ACEOF
diff --git a/configure.ac b/configure.ac
index 637fe9de..fa553700 100644
--- a/configure.ac
+++ b/configure.ac
@@ -85,12 +85,18 @@ if test "x$ENABLE_SLURM_PLUGIN" == "xtrue"; then
   if test "x$SCONTROL_ABSPATH" == "xnotfound"; then
     AC_MSG_ERROR([Could not find scontrol])
   fi
+  AC_PATH_PROG([SINFO_ABSPATH], [sinfo], [notfound])
+  if test "x$SINFO_ABSPATH" == "xnotfound"; then
+    AC_MSG_ERROR([Could not find sinfo])
+  fi
   PATH=$OPATH
 fi
 
 AM_CONDITIONAL([BLD_SLURMPLUGIN], [test "x$ENABLE_SLURM_PLUGIN" == "xtrue"])
 AC_SUBST(SCONTROL_ABSPATH, $SCONTROL_ABSPATH)
+AC_SUBST(SINFO_ABSPATH, $SINFO_ABSPATH)
 AC_SUBST(PKGSYSCONF_DIR)
+AM_CONDITIONAL([BLD_FLUXPLUGIN], [test "x$ENABLE_FLUX_PLUGIN" = "xtrue"])
 
 AC_OUTPUT
 
diff --git a/doc/Makefile.in b/doc/Makefile.in
index 3dc35cbb..78079922 100644
--- a/doc/Makefile.in
+++ b/doc/Makefile.in
@@ -253,6 +253,7 @@ SCONTROL_ABSPATH = @SCONTROL_ABSPATH@
 SED = @SED@
 SET_MAKE = @SET_MAKE@
 SHELL = @SHELL@
+SINFO_ABSPATH = @SINFO_ABSPATH@
 SRUN_PATH = @SRUN_PATH@
 STATICFLAG = @STATICFLAG@
 STRIP = @STRIP@
diff --git a/src/flux/Makefile.in b/src/flux/Makefile.in
index 18199543..e6733b24 100644
--- a/src/flux/Makefile.in
+++ b/src/flux/Makefile.in
@@ -361,6 +361,7 @@ SCONTROL_ABSPATH = @SCONTROL_ABSPATH@
 SED = @SED@
 SET_MAKE = @SET_MAKE@
 SHELL = @SHELL@
+SINFO_ABSPATH = @SINFO_ABSPATH@
 SRUN_PATH = @SRUN_PATH@
 STATICFLAG = @STATICFLAG@
 STRIP = @STRIP@
diff --git a/src/slurm_plugin/Makefile.am b/src/slurm_plugin/Makefile.am
index a9638b65..489dffcb 100644
--- a/src/slurm_plugin/Makefile.am
+++ b/src/slurm_plugin/Makefile.am
@@ -3,7 +3,7 @@ lib_LTLIBRARIES = libspindleslurm.la
 libver=`$(top_srcdir)/LIB_VERSION spindleslurm`
 
 libspindleslurm_la_SOURCES = encode_decode.c plugin_utils.c slurm_plugin.c $(top_srcdir)/src/utils/spindle_mkdir.c $(top_srcdir)/src/utils/parseloc.c
-libspindleslurm_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/include -I$(top_srcdir)/src/logging -DUSE_PLUGIN_DEBUG -DDEBUG -DCUSTOM_GETENV -DCUSTOM_GETENV_FREE -DSCONTROL_BIN="$(SCONTROL_ABSPATH)" -DSPINDLE_DO_EXPORT
+libspindleslurm_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/include -I$(top_srcdir)/src/logging -DUSE_PLUGIN_DEBUG -DDEBUG -DCUSTOM_GETENV -DCUSTOM_GETENV_FREE -DSCONTROL_BIN="$(SCONTROL_ABSPATH)" -DSINFO_BIN="$(SINFO_ABSPATH)" -DSPINDLE_DO_EXPORT
 libspindleslurm_la_LDFLAGS = $(AM_LDFLAGS) -ldl -version-info $(libver)
 libspindleslurm_la_LIBADD = $(top_builddir)/src/server/startup/libspindlebe.la $(top_builddir)/src/fe/startup/libspindlefe.la
 libspindleslurm_la_CFLAGS = $(CFLAGS) -fvisibility=hidden
diff --git a/src/slurm_plugin/Makefile.in b/src/slurm_plugin/Makefile.in
index c8488a96..a7fe188f 100644
--- a/src/slurm_plugin/Makefile.in
+++ b/src/slurm_plugin/Makefile.in
@@ -315,6 +315,7 @@ SCONTROL_ABSPATH = @SCONTROL_ABSPATH@
 SED = @SED@
 SET_MAKE = @SET_MAKE@
 SHELL = @SHELL@
+SINFO_ABSPATH = @SINFO_ABSPATH@
 SRUN_PATH = @SRUN_PATH@
 STATICFLAG = @STATICFLAG@
 STRIP = @STRIP@
@@ -382,7 +383,7 @@ top_srcdir = @top_srcdir@
 lib_LTLIBRARIES = libspindleslurm.la
 libver = `$(top_srcdir)/LIB_VERSION spindleslurm`
 libspindleslurm_la_SOURCES = encode_decode.c plugin_utils.c slurm_plugin.c $(top_srcdir)/src/utils/spindle_mkdir.c $(top_srcdir)/src/utils/parseloc.c
-libspindleslurm_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/include -I$(top_srcdir)/src/logging -DUSE_PLUGIN_DEBUG -DDEBUG -DCUSTOM_GETENV -DCUSTOM_GETENV_FREE -DSCONTROL_BIN="$(SCONTROL_ABSPATH)" -DSPINDLE_DO_EXPORT
+libspindleslurm_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/include -I$(top_srcdir)/src/logging -DUSE_PLUGIN_DEBUG -DDEBUG -DCUSTOM_GETENV -DCUSTOM_GETENV_FREE -DSCONTROL_BIN="$(SCONTROL_ABSPATH)" -DSINFO_BIN="$(SINFO_ABSPATH)" -DSPINDLE_DO_EXPORT
 libspindleslurm_la_LDFLAGS = $(AM_LDFLAGS) -ldl -version-info $(libver)
 libspindleslurm_la_LIBADD = $(top_builddir)/src/server/startup/libspindlebe.la $(top_builddir)/src/fe/startup/libspindlefe.la
 libspindleslurm_la_CFLAGS = $(CFLAGS) -fvisibility=hidden
diff --git a/testsuite/Makefile.in b/testsuite/Makefile.in
index 9c225322..511a6aca 100644
--- a/testsuite/Makefile.in
+++ b/testsuite/Makefile.in
@@ -273,6 +273,7 @@ SCONTROL_ABSPATH = @SCONTROL_ABSPATH@
 SED = @SED@
 SET_MAKE = @SET_MAKE@
 SHELL = @SHELL@
+SINFO_ABSPATH = @SINFO_ABSPATH@
 SRUN_PATH = @SRUN_PATH@
 STATICFLAG = @STATICFLAG@
 STRIP = @STRIP@

From 3755c18f315257996094cdc5024535cc4f471ec9 Mon Sep 17 00:00:00 2001
From: Norbert Eicker <n.eicker@fz-juelich.de>
Date: Mon, 19 Feb 2024 19:58:06 +0100
Subject: [PATCH 46/57] Translate Slurm NodeName to NodeAddr for server
 communication setup

---
 src/slurm_plugin/plugin_utils.c | 85 +++++++++++++++++++++++++++++++++
 src/slurm_plugin/plugin_utils.h |  1 +
 src/slurm_plugin/slurm_plugin.c | 14 +++++-
 3 files changed, 99 insertions(+), 1 deletion(-)

diff --git a/src/slurm_plugin/plugin_utils.c b/src/slurm_plugin/plugin_utils.c
index 94e9a0cb..47d553cf 100644
--- a/src/slurm_plugin/plugin_utils.c
+++ b/src/slurm_plugin/plugin_utils.c
@@ -22,6 +22,11 @@
 #else
 #define SLURM_SCONTROL_BIN "scontrol"
 #endif
+#if defined SINFO_BIN
+#define SLURM_SINFO_BIN STR(SINFO_BIN)
+#else
+#define SLURM_SINFO_BIN "sinfo"
+#endif
 
 
 extern char *parse_location(char *loc, number_t number);
@@ -119,6 +124,86 @@ char **getHostsScontrol(unsigned int num_hosts, const char *hoststr)
    return ret;   
 }
 
+char **getHostAddrSinfo(unsigned int num_hosts, char **hostlist)
+{
+   const char *sinfo_path = SLURM_SINFO_BIN;
+   const char *sinfo_args = "-O NodeAddr -h -n";
+   const char *sinfo_suffix = "2> /dev/null";
+   FILE *f = NULL;
+   char **hostaddrlist = NULL, *s, *sinfo_cmdline = NULL, **ret = NULL;
+   int i, j, hostnamelen;
+   int result;
+   size_t maxHostLen = 0, sinfo_cmdlineLen, len;
+
+   hostaddrlist = calloc(num_hosts+1, sizeof(char*));
+
+   for (i = 0; i < num_hosts; i++) {
+       if (hostlist[i]) {
+	   size_t thisLen = strlen(hostlist[i]);
+	   if (thisLen > maxHostLen) maxHostLen = thisLen;
+       }
+   }
+
+   sinfo_cmdlineLen = strlen(sinfo_path) + strlen(sinfo_args)
+       + maxHostLen + strlen(sinfo_suffix) + 6;
+   sinfo_cmdline = (char *) malloc(sinfo_cmdlineLen);
+
+   for (i = 0; i < num_hosts; i++) {
+      if (!hostlist[i] || !strlen(hostlist[i])) goto done;
+      result = snprintf(sinfo_cmdline, sinfo_cmdlineLen, "%s %s \"%s\" %s",
+			sinfo_path, sinfo_args, hostlist[i], sinfo_suffix);
+      if (result >= sinfo_cmdlineLen) {
+	 sdprintf(1, "ERROR: Formatting error creating sinfo cmdline '%s' (%d)\n",
+		  sinfo_cmdline, result);
+	 goto done;
+      }
+      sdprintf(2, "Running sinfo to get host address: %s\n", sinfo_cmdline);
+
+      f = popen(sinfo_cmdline, "r");
+      if (!f) {
+	 sdprintf(1, "ERROR: Could not run sinfo: %s\n", sinfo_cmdline);
+	 goto done;
+      }
+
+      len = 0;
+      result = getline(&(hostaddrlist[i]), &len, f);
+      pclose(f);
+      if (result == -1) {
+         int error = errno;
+         sdprintf(1, "ERROR: Resolving '%s' failed: %s\n", hostlist[i], strerror(error));
+         (void) error;
+         goto done;
+      }
+
+      s = hostaddrlist[i];
+      hostnamelen = strlen(s);
+      for (j = 0; j < hostnamelen; j++) {
+         if (!((s[j] >= '0' && s[j] <= '9') ||
+               (s[j] >= 'a' && s[j] <= 'z') ||
+               (s[j] >= 'A' && s[j] <= 'Z') ||
+               (s[j] == '-' || s[j] == '_' || s[j] == '.'))) {
+            s[j] = '\0';
+            break;
+         }
+      }
+      sdprintf(3, "sinfo returned hostaddr %s for %s\n", s, hostlist[i]);
+   }
+   if (i != num_hosts) {
+      sdprintf(1, "ERROR: expected %d hosts from sinfo.  Got %d\n",  num_hosts, i);
+      goto done;
+   }
+
+   ret = hostaddrlist;
+  done:
+   if (sinfo_cmdline)
+      free(sinfo_cmdline);
+   if (!ret && hostaddrlist) {
+      for (i = 0; i < num_hosts; i++) free(hostaddrlist[i]);
+      free(hostlist);
+   }
+   return ret;
+}
+
 int isFEHost(char **hostlist, unsigned int num_hosts)
 {
    char host[256];
diff --git a/src/slurm_plugin/plugin_utils.h b/src/slurm_plugin/plugin_utils.h
index d57f55da..eaa0da12 100644
--- a/src/slurm_plugin/plugin_utils.h
+++ b/src/slurm_plugin/plugin_utils.h
@@ -39,6 +39,7 @@ int decodeSpindleConfig(const char *encodedstr,
                         int *spindle_argc, char ***spindle_argv);
 
 char **getHostsScontrol(unsigned int num_hosts, const char *hoststr);
+char **getHostAddrSinfo(unsigned int num_hosts, char **hostlist);
 char **getHostsParse(unsigned int num_hosts, const char *shortlist);
 
 int isFEHost(char **hostlist, unsigned int num_hosts);
diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c
index 98e43945..700008a9 100644
--- a/src/slurm_plugin/slurm_plugin.c
+++ b/src/slurm_plugin/slurm_plugin.c
@@ -426,7 +426,7 @@ static int get_spindle_args(spank_t spank, spindle_args_t *params)
 
 static int launch_spindle(spank_t spank, spindle_args_t *params)
 {
-   char **hostlist = NULL;
+   char **hostlist = NULL, **hostaddrlist = NULL;
    int result;
    int is_fe_host = 0;
    int is_be_leader = 0;
@@ -460,7 +460,14 @@ static int launch_spindle(spank_t spank, spindle_args_t *params)
    }
 
    if (is_fe_host && is_be_leader) {
+#if defined(SINFO_BIN)
+      hostaddrlist = getHostAddrSinfo(num_hosts, hostlist);
+      if (!hostaddrlist)
+	  goto done;
+      result = launchFE(hostaddrlist, params);
+#else
       result = launchFE(hostlist, params);
+#endif
       if (result == -1)
          goto done;
    }
@@ -471,6 +478,11 @@ static int launch_spindle(spank_t spank, spindle_args_t *params)
    if (hostlist) {
       for (i = 0; i < num_hosts; i++) free(hostlist[i]);
       free(hostlist);
+   }
+   if (hostaddrlist) {
+      for (i = 0; i < num_hosts; i++) free(hostaddrlist[i]);
+      free(hostaddrlist);
+   }
    
    return launch_result;
 }

From e62ad6706ed668e2969bad159e3cc9f4f7266b63 Mon Sep 17 00:00:00 2001
From: Nicholas Chaimov <nchaimov@paratools.com>
Date: Thu, 6 Nov 2025 13:58:36 -0800
Subject: [PATCH 47/57] Check for SPANK's spank_prepend_task_argv() in
 configure

Co-authored-by: Norbert Eicker <n.eicker@fz-juelich.de>
---
 config.h.in  |  4 ++++
 configure    | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 configure.ac |  1 +
 3 files changed, 63 insertions(+)

diff --git a/config.h.in b/config.h.in
index a346ae96..8c7a94db 100644
--- a/config.h.in
+++ b/config.h.in
@@ -33,6 +33,10 @@
 /* Slurm with via plugin is enabled */
 #undef ENABLE_SLURM_PLUGIN
 
+/* Define to 1 if you have the declaration of `spank_prepend_task_argv', and
+   to 0 if you don't. */
+#undef HAVE_DECL_SPANK_PREPEND_TASK_ARGV
+
 /* Define to 1 if you have the <dlfcn.h> header file. */
 #undef HAVE_DLFCN_H
 
diff --git a/configure b/configure
index a9bcee84..f04e5c2c 100755
--- a/configure
+++ b/configure
@@ -2213,6 +2213,52 @@ fi
   eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
 
 } # ac_fn_c_check_header_mongrel
+
+# ac_fn_c_check_decl LINENO SYMBOL VAR INCLUDES
+# ---------------------------------------------
+# Tests whether SYMBOL is declared in INCLUDES, setting cache variable VAR
+# accordingly.
+ac_fn_c_check_decl ()
+{
+  as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+  as_decl_name=`echo $2|sed 's/ *(.*//'`
+  as_decl_use=`echo $2|sed -e 's/(/((/' -e 's/)/) 0&/' -e 's/,/) 0& (/g'`
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $as_decl_name is declared" >&5
+$as_echo_n "checking whether $as_decl_name is declared... " >&6; }
+if eval \${$3+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+$4
+int
+main ()
+{
+#ifndef $as_decl_name
+#ifdef __cplusplus
+  (void) $as_decl_use;
+#else
+  (void) $as_decl_name;
+#endif
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+  eval "$3=yes"
+else
+  eval "$3=no"
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+eval ac_res=\$$3
+	       { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
+$as_echo "$ac_res" >&6; }
+  eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
+
+} # ac_fn_c_check_decl
 cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
@@ -18659,6 +18705,18 @@ fi
 
 
   CPPFLAGS=$OCPPFLAGS
+  ac_fn_c_check_decl "$LINENO" "spank_prepend_task_argv" "ac_cv_have_decl_spank_prepend_task_argv" "#include<slurm/spank.h>
+"
+if test "x$ac_cv_have_decl_spank_prepend_task_argv" = xyes; then :
+  ac_have_decl=1
+else
+  ac_have_decl=0
+fi
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_DECL_SPANK_PREPEND_TASK_ARGV $ac_have_decl
+_ACEOF
+
 
   OPATH=$PATH
   if test "x$SLURM_DIR" != "x"; then
diff --git a/configure.ac b/configure.ac
index fa553700..b2c519d4 100644
--- a/configure.ac
+++ b/configure.ac
@@ -76,6 +76,7 @@ if test "x$ENABLE_SLURM_PLUGIN" == "xtrue"; then
                   [],
                   [AC_MSG_ERROR([Could not find slurm/spank.h])])
   CPPFLAGS=$OCPPFLAGS
+  AC_CHECK_DECLS([spank_prepend_task_argv], [], [], [[#include<slurm/spank.h>]])
 
   OPATH=$PATH
   if test "x$SLURM_DIR" != "x"; then

From 608ea09634356acc2189d447b45f5c8cdfcc1990 Mon Sep 17 00:00:00 2001
From: Norbert Eicker <n.eicker@fz-juelich.de>
Date: Mon, 19 Feb 2024 20:35:56 +0100
Subject: [PATCH 48/57] Use spank_prepend_task_argv() to tweak task's argv

 - Requires Slurm 23.11 or later or a backport of this function
---
 src/slurm_plugin/slurm_plugin.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c
index 700008a9..643467eb 100644
--- a/src/slurm_plugin/slurm_plugin.c
+++ b/src/slurm_plugin/slurm_plugin.c
@@ -29,6 +29,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 #include "spindle_launch.h"
 #include "plugin_utils.h"
 
+#include "config.h"
 
 SPINDLE_EXPORT extern const char plugin_name[];
 SPINDLE_EXPORT extern const char plugin_type[];
@@ -578,9 +579,13 @@ static int launchBE(spank_t spank, spindle_args_t *params)
 
 static int prepApp(spank_t spank, spindle_args_t *params)
 {
+#if HAVE_DECL_SPANK_PREPEND_TASK_ARGV == 1
+   int result;
+#else
    int app_argc, result;
    char **app_argv;
    char *app_exe_name, *last_slash;
+#endif
    spank_err_t err;
    int bootstrap_argc;
    char **bootstrap_argv;
@@ -591,6 +596,16 @@ static int prepApp(spank_t spank, spindle_args_t *params)
       return -1;
    }
       
+#if HAVE_DECL_SPANK_PREPEND_TASK_ARGV == 1
+   sdprintf(2, "Prepping task process %d to run spindle\n", getpid());
+
+   const char **filter_argv = (const char **)bootstrap_argv;
+   err = spank_prepend_task_argv(spank, bootstrap_argc, filter_argv);
+   if (err != ESPANK_SUCCESS) {
+      sdprintf(1, "WARNING: Could not prepend spindle filter.\n");
+      result = -1;
+   }
+#else
    sdprintf(2, "Prepping app process %d to run spindle\n", getpid());
 
    err = spank_get_item(spank, S_JOB_ARGV, &app_argc, &app_argv);
@@ -605,6 +620,7 @@ static int prepApp(spank_t spank, spindle_args_t *params)
    }
 
    result = spindleHookSpindleArgsIntoExecBE(bootstrap_argc, bootstrap_argv, app_exe_name);
+#endif
    if (result == -1) {
       sdprintf(1, "ERROR setting up app to run spindle.  Spindle won't work\n");
       return -1;

From 58f433fb808715d0d5f46d93c04bc94fda84381a Mon Sep 17 00:00:00 2001
From: Nicholas Chaimov <nchaimov@paratools.com>
Date: Fri, 14 Nov 2025 10:27:03 -0800
Subject: [PATCH 49/57] Only free env_value if it was actually allocated

---
 src/utils/parseloc.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/utils/parseloc.c b/src/utils/parseloc.c
index 0d8dcc81..00cdcf24 100644
--- a/src/utils/parseloc.c
+++ b/src/utils/parseloc.c
@@ -134,7 +134,9 @@ static char *parse_location_impl(char *loc, number_t number, int print_on_error)
          i += envvar_len + 1;
          j += env_value_len;
 #if defined(CUSTOM_GETENV) && defined(CUSTOM_GETENV_FREE)
-         free(env_value);
+         if(env_value != env_value_str) {
+            free(env_value);
+         }
 #endif
       }
       else {

From e50f2eb33366ec0e8dacf07a0ff3931cb863e297 Mon Sep 17 00:00:00 2001
From: Nicholas Chaimov <nchaimov@paratools.com>
Date: Tue, 30 Dec 2025 17:06:33 -0800
Subject: [PATCH 50/57] Don't require rsh launch when SPANK plugin is built;
 ensure shutdown is run exactly once on every proc

---
 configure                       | 14 ++++++++++-
 configure.common.ac             |  2 +-
 src/client/configure            |  2 +-
 src/fe/configure                |  2 +-
 src/server/configure            |  2 +-
 src/slurm_plugin/plugin_utils.c | 11 +++++----
 src/slurm_plugin/plugin_utils.h |  2 +-
 src/slurm_plugin/slurm_plugin.c | 41 ++++++++++++++++++++-------------
 8 files changed, 50 insertions(+), 26 deletions(-)

diff --git a/configure b/configure
index f04e5c2c..e9a4f95a 100755
--- a/configure
+++ b/configure
@@ -16958,7 +16958,7 @@ $as_echo "yes" >&6; }
 $as_echo "$as_me: WARNING: Slurm launching was explicitly requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected.  This version of slurm breaks spindle daemon launch.  You might still be able to get spindle to work by running jobs with srun's --overlap option.  Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster." >&2;}
   fi
 
-  if [ "x$ENABLE_RSH_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_LAUNCH" != "x1" ] ; then
+  if [ "x$ENABLE_RSH_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_PLUGIN" != "xtrue" ] ; then
     if test "x$BROKEN_SRUN" == "x1"; then
       as_fn_error $? "Slurm support was requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected.  This version of slurm breaks spindle daemon launch.  You can disable this error message and build spindle with slurm-based daemon launching anyways by explicitly passing the --with-slurm-launch option (you might still be able to get spindle to work by running jobs with srun's --overlap option).  Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster." "$LINENO" 5
     else
@@ -18826,6 +18826,14 @@ SCONTROL_ABSPATH=$SCONTROL_ABSPATH
 SINFO_ABSPATH=$SINFO_ABSPATH
 
 
+ if test "x$ENABLE_FLUX_PLUGIN" = "xtrue"; then
+  BLD_FLUXPLUGIN_TRUE=
+  BLD_FLUXPLUGIN_FALSE='#'
+else
+  BLD_FLUXPLUGIN_TRUE='#'
+  BLD_FLUXPLUGIN_FALSE=
+fi
+
 
 cat >confcache <<\_ACEOF
 # This file is a shell script that caches the results of configure
@@ -19040,6 +19048,10 @@ if test -z "${BLD_SLURMPLUGIN_TRUE}" && test -z "${BLD_SLURMPLUGIN_FALSE}"; then
   as_fn_error $? "conditional \"BLD_SLURMPLUGIN\" was never defined.
 Usually this means the macro was only invoked conditionally." "$LINENO" 5
 fi
+if test -z "${BLD_FLUXPLUGIN_TRUE}" && test -z "${BLD_FLUXPLUGIN_FALSE}"; then
+  as_fn_error $? "conditional \"BLD_FLUXPLUGIN\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
 
 : "${CONFIG_STATUS=./config.status}"
 ac_write_fail=0
diff --git a/configure.common.ac b/configure.common.ac
index 780aef00..e8d311f4 100644
--- a/configure.common.ac
+++ b/configure.common.ac
@@ -168,7 +168,7 @@ if test "x$ENABLE_SLURM" == "xtrue"; then
     AC_MSG_WARN([Slurm launching was explicitly requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected.  This version of slurm breaks spindle daemon launch.  You might still be able to get spindle to work by running jobs with srun's --overlap option.  Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster.])
   fi
   
-  if [[ "x$ENABLE_RSH_LAUNCH" != "x1" ]] && [[ "x$ENABLE_SLURM_LAUNCH" != "x1" ]] ; then
+  if [[ "x$ENABLE_RSH_LAUNCH" != "x1" ]] && [[ "x$ENABLE_SLURM_LAUNCH" != "x1" ]] && [[ "x$ENABLE_SLURM_PLUGIN" != "xtrue" ]] ; then
     if test "x$BROKEN_SRUN" == "x1"; then
       AC_MSG_ERROR([Slurm support was requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected.  This version of slurm breaks spindle daemon launch.  You can disable this error message and build spindle with slurm-based daemon launching anyways by explicitly passing the --with-slurm-launch option (you might still be able to get spindle to work by running jobs with srun's --overlap option).  Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster.])
     else
diff --git a/src/client/configure b/src/client/configure
index dbd69e1f..236528aa 100755
--- a/src/client/configure
+++ b/src/client/configure
@@ -12883,7 +12883,7 @@ $as_echo "yes" >&6; }
 $as_echo "$as_me: WARNING: Slurm launching was explicitly requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected.  This version of slurm breaks spindle daemon launch.  You might still be able to get spindle to work by running jobs with srun's --overlap option.  Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster." >&2;}
   fi
 
-  if [ "x$ENABLE_RSH_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_LAUNCH" != "x1" ] ; then
+  if [ "x$ENABLE_RSH_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_PLUGIN" != "xtrue" ] ; then
     if test "x$BROKEN_SRUN" == "x1"; then
       as_fn_error $? "Slurm support was requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected.  This version of slurm breaks spindle daemon launch.  You can disable this error message and build spindle with slurm-based daemon launching anyways by explicitly passing the --with-slurm-launch option (you might still be able to get spindle to work by running jobs with srun's --overlap option).  Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster." "$LINENO" 5
     else
diff --git a/src/fe/configure b/src/fe/configure
index 95e99784..75088c9a 100755
--- a/src/fe/configure
+++ b/src/fe/configure
@@ -16733,7 +16733,7 @@ $as_echo "yes" >&6; }
 $as_echo "$as_me: WARNING: Slurm launching was explicitly requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected.  This version of slurm breaks spindle daemon launch.  You might still be able to get spindle to work by running jobs with srun's --overlap option.  Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster." >&2;}
   fi
 
-  if [ "x$ENABLE_RSH_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_LAUNCH" != "x1" ] ; then
+  if [ "x$ENABLE_RSH_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_PLUGIN" != "xtrue" ] ; then
     if test "x$BROKEN_SRUN" == "x1"; then
       as_fn_error $? "Slurm support was requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected.  This version of slurm breaks spindle daemon launch.  You can disable this error message and build spindle with slurm-based daemon launching anyways by explicitly passing the --with-slurm-launch option (you might still be able to get spindle to work by running jobs with srun's --overlap option).  Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster." "$LINENO" 5
     else
diff --git a/src/server/configure b/src/server/configure
index e32c1cfe..a747c739 100755
--- a/src/server/configure
+++ b/src/server/configure
@@ -16730,7 +16730,7 @@ $as_echo "yes" >&6; }
 $as_echo "$as_me: WARNING: Slurm launching was explicitly requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected.  This version of slurm breaks spindle daemon launch.  You might still be able to get spindle to work by running jobs with srun's --overlap option.  Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster." >&2;}
   fi
 
-  if [ "x$ENABLE_RSH_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_LAUNCH" != "x1" ] ; then
+  if [ "x$ENABLE_RSH_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_LAUNCH" != "x1" ] && [ "x$ENABLE_SLURM_PLUGIN" != "xtrue" ] ; then
     if test "x$BROKEN_SRUN" == "x1"; then
       as_fn_error $? "Slurm support was requested, but slurm $srun_version, which is later than $bad_srun_major.$bad_srun_minor, was detected.  This version of slurm breaks spindle daemon launch.  You can disable this error message and build spindle with slurm-based daemon launching anyways by explicitly passing the --with-slurm-launch option (you might still be able to get spindle to work by running jobs with srun's --overlap option).  Or you could switch to having spindle launch daemons with rsh/ssh by passing the --with-rsh-launch option, and ensuring that rsh/ssh to nodes works on your cluster." "$LINENO" 5
     else
diff --git a/src/slurm_plugin/plugin_utils.c b/src/slurm_plugin/plugin_utils.c
index 47d553cf..3be45742 100644
--- a/src/slurm_plugin/plugin_utils.c
+++ b/src/slurm_plugin/plugin_utils.c
@@ -261,14 +261,14 @@ char *unique_file = NULL;
 
 #define UNIQUE_FILE_NAME "spindle_unique"
 
-int isBEProc(spindle_args_t *params)
+int isBEProc(spindle_args_t *params, unsigned int exit_phase)
 {
-   char *dir = NULL, *expanded_dir = NULL, *realized_dir = NULL;
+   char *dir = NULL, *expanded_dir = NULL, *realized_dir = NULL, *phase_name = NULL;
    char hostname[256], session_id_str[32];
    size_t unique_file_len;
    int beproc_result = -1;
    int fd = -1, error;
-
+   
    dir = params->location;
    if (!dir) {
       sdprintf(1, "ERROR: Location not filled in\n");
@@ -289,13 +289,16 @@ int isBEProc(spindle_args_t *params)
 
    snprintf(session_id_str, sizeof(session_id_str), "%lu", (unsigned long) params->number);
 
+   phase_name = exit_phase ? "exit" : "launch";
+
    unique_file_len = strlen(realized_dir) + 1 +
       strlen(UNIQUE_FILE_NAME) + 1 +
+      strlen(phase_name) + 1 +
       strlen(hostname) + 1 +
       strlen(session_id_str) + 1;
 
    unique_file = (char *) malloc(sizeof(char*) * unique_file_len);
-   snprintf(unique_file, unique_file_len, "%s/%s.%s.%s", realized_dir, UNIQUE_FILE_NAME, hostname, session_id_str);
+   snprintf(unique_file, unique_file_len, "%s/%s.%s.%s.%s", realized_dir, UNIQUE_FILE_NAME, phase_name, hostname, session_id_str);
 
    spindle_mkdir(realized_dir);
 
diff --git a/src/slurm_plugin/plugin_utils.h b/src/slurm_plugin/plugin_utils.h
index eaa0da12..acf2ac5d 100644
--- a/src/slurm_plugin/plugin_utils.h
+++ b/src/slurm_plugin/plugin_utils.h
@@ -44,7 +44,7 @@ char **getHostsParse(unsigned int num_hosts, const char *shortlist);
 
 int isFEHost(char **hostlist, unsigned int num_hosts);
 extern char *unique_file;
-int isBEProc(spindle_args_t *params);
+int isBEProc(spindle_args_t *params, unsigned int exit_phase);
 
 char *encodeCmdArgs(int sargc, char **sargv);
 void decodeCmdArgs(char *cmd, int *sargc, char ***sargv);
diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c
index 643467eb..740b48ff 100644
--- a/src/slurm_plugin/slurm_plugin.c
+++ b/src/slurm_plugin/slurm_plugin.c
@@ -36,8 +36,10 @@ SPINDLE_EXPORT extern const char plugin_type[];
 SPINDLE_EXPORT extern const unsigned int plugin_version;
 SPINDLE_EXPORT extern struct spank_option spank_options[];
 SPINDLE_EXPORT int slurm_spank_task_init(spank_t spank, int ac, char *argv[]);
+SPINDLE_EXPORT int slurm_spank_task_exit(spank_t spank, int ac, char *argv[]);
 SPINDLE_EXPORT int slurm_spank_exit(spank_t spank, int site_argc, char *site_argv[]);
 
+
 SPANK_PLUGIN(spindle, 1)
 
 typedef struct {
@@ -84,8 +86,8 @@ int slurm_spank_task_init(spank_t spank, int site_argc, char *site_argv[])
    int result, func_result = -1;
    saved_env_t *env = NULL;
    static int initialized = 0;
-   spindle_args_t params;
-   
+   spindle_args_t params = {0};
+
    if (!enable_spindle)
       return 0;
 
@@ -136,7 +138,7 @@ int slurm_spank_task_init(spank_t spank, int site_argc, char *site_argv[])
    return func_result;
 }
 
-int slurm_spank_exit(spank_t spank, int site_argc, char *site_argv[])
+int slurm_spank_task_exit(spank_t spank, int site_argc, char *site_argv[])
 {
    spank_context_t context;
    char *result_str;
@@ -226,7 +228,7 @@ static int fillInArgs(spank_t spank, spindle_args_t *args, int argc, char **argv
    char *err_string;
 
    args->unique_id = unique_id;
-   args->number = (unsigned long) args->unique_id;
+   args->number = (number_t) args->unique_id;
    result = fillInSpindleArgsCmdlineFE(args, SPINDLE_FILLARGS_NOUNIQUEID | SPINDLE_FILLARGS_NONUMBER,
                                        argc, argv, &err_string);
    if (result == -1) {
@@ -238,6 +240,8 @@ static int fillInArgs(spank_t spank, spindle_args_t *args, int argc, char **argv
       return -1;
    }
    args->opts |= OPT_BEEXIT;
+   args->use_launcher = slurm_plugin_launcher;
+   args->startup_type = startup_external;
 
    oldlocation = args->location;
    current_spank = spank;
@@ -444,7 +448,7 @@ static int launch_spindle(spank_t spank, spindle_args_t *params)
    if (!hostlist)
       goto done;
    
-   is_be_leader = isBEProc(params);
+   is_be_leader = isBEProc(params, 0);
    if (is_be_leader == -1)
       goto done;
    
@@ -516,7 +520,7 @@ static int launchFE(char **hostlist, spindle_args_t *params)
 
    superclose();
    
-   sdprintf(1, "Initializing FE on pid %d with unqiue_id %lu\n", (int) getpid(), params->unique_id);
+   sdprintf(1, "Initializing FE on pid %d with unique_id %lu\n", (int) getpid(), params->unique_id);
    result = spindleInitFE((const char **) hostlist, params);
    if (result == -1) {
       sdprintf(1, "ERROR: Could not launch FE.  Spindle will likely hang.\n");
@@ -597,7 +601,7 @@ static int prepApp(spank_t spank, spindle_args_t *params)
    }
       
 #if HAVE_DECL_SPANK_PREPEND_TASK_ARGV == 1
-   sdprintf(2, "Prepping task process %d to run spindle\n", getpid());
+   sdprintf(2, "Prepping task process %d to run spindle using spank_prepend_task_argv method\n", getpid());
 
    const char **filter_argv = (const char **)bootstrap_argv;
    err = spank_prepend_task_argv(spank, bootstrap_argc, filter_argv);
@@ -606,7 +610,7 @@ static int prepApp(spank_t spank, spindle_args_t *params)
       result = -1;
    }
 #else
-   sdprintf(2, "Prepping app process %d to run spindle\n", getpid());
+   sdprintf(2, "Prepping app process %d to run spindle using spindleHookSpindleArgsIntoExecBE method\n", getpid());
 
    err = spank_get_item(spank, S_JOB_ARGV, &app_argc, &app_argv);
    if (err != ESPANK_SUCCESS) {
@@ -633,9 +637,9 @@ static int handleExit(void *params, char **output_str)
 {
    exit_params_t *exit_params;
    spank_t spank;
-   int site_argc, result;
+   int site_argc, result, is_be_leader;
    char **site_argv;
-   spindle_args_t args;
+   spindle_args_t args = {0};
 
    exit_params = (exit_params_t *) params;
    spank = exit_params->spank;
@@ -650,13 +654,18 @@ static int handleExit(void *params, char **output_str)
    }
 
    if (!args.location) {
-       sdprintf(2, "WARNING: spindleExitBE not called since location is NULL\n");
+      sdprintf(2, "WARNING: spindleExitBE not called since location is NULL\n");
    } else {
-       result = spindleExitBE(args.location);
-       if (result == -1) {
-	   sdprintf(1, "ERROR: spindleExitBE returned an error on location %s\n", args.location);
-	   return -1;
-       }
+      // The task_exit callback is run for _each proc_, so we use
+      // isBEProc to pick only one proc per node to call spindleExitBE.
+      is_be_leader = isBEProc(&args, 1);
+      if (is_be_leader) {
+         result = spindleExitBE(args.location);
+         if (result == -1) {
+             sdprintf(1, "ERROR: spindleExitBE returned an error on location %s\n", args.location);
+             return -1;
+         }
+      }
    }
    return 0;
 }

From 495286bb6df3fc75bd64dd2023ff2c804584f122 Mon Sep 17 00:00:00 2001
From: Nicholas Chaimov <nchaimov@paratools.com>
Date: Thu, 8 Jan 2026 15:52:17 -0800
Subject: [PATCH 51/57] Handle --level=off in SPANK plugin

---
 src/slurm_plugin/slurm_plugin.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c
index 740b48ff..c7bbcef2 100644
--- a/src/slurm_plugin/slurm_plugin.c
+++ b/src/slurm_plugin/slurm_plugin.c
@@ -116,6 +116,10 @@ int slurm_spank_task_init(spank_t spank, int site_argc, char *site_argv[])
       goto done;
    }
 
+   if (params.opts & OPT_OFF) {
+     return 0;
+   }
+
    result = launch_spindle(spank, &params);
    if (result == -1) {
       sdprintf(1, "Error launching spindle.  Aborting spindle\n");
@@ -652,6 +656,10 @@ static int handleExit(void *params, char **output_str)
       sdprintf(1, "ERROR: Could not process spindle args in handleExit\n");
       return -1;
    }
+   
+   if (args.opts & OPT_OFF) {
+      return 0;
+   }
 
    if (!args.location) {
       sdprintf(2, "WARNING: spindleExitBE not called since location is NULL\n");

From 2d6d5a827ebd3368e87b7a887c871a760f43ac96 Mon Sep 17 00:00:00 2001
From: Nicholas Chaimov <nchaimov@paratools.com>
Date: Mon, 12 Jan 2026 15:05:33 -0800
Subject: [PATCH 52/57] CI testing for SPANK plugin

---
 .github/workflows/ci.yml                      |  58 ++++++++
 .../testing-plugin/Dockerfile                 |  42 ++++++
 .../testing-plugin/conf/cgroup.conf           |   1 +
 .../testing-plugin/conf/plugstack.conf        |   1 +
 .../testing-plugin/conf/slurm.conf            |  42 ++++++
 .../conf/slurmdbd.conf.template               |  10 ++
 .../testing-plugin/docker-compose.yml         | 126 ++++++++++++++++++
 .../testing-plugin/generate_config.sh         |  14 ++
 .../testing-plugin/scripts/add_docker_user.sh |  10 ++
 .../testing-plugin/scripts/build_spindle.sh   |   9 ++
 .../testing-plugin/scripts/entrypoint.sh      |  21 +++
 .../testing-plugin/scripts/setup_slurm.sh     |  10 ++
 .../scripts/setup_spank_plugin.sh             |   5 +
 testsuite/run_driver_slurm-plugin             |  20 +++
 14 files changed, 369 insertions(+)
 create mode 100644 containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile
 create mode 100644 containers/spindle-slurm-ubuntu/testing-plugin/conf/cgroup.conf
 create mode 100644 containers/spindle-slurm-ubuntu/testing-plugin/conf/plugstack.conf
 create mode 100644 containers/spindle-slurm-ubuntu/testing-plugin/conf/slurm.conf
 create mode 100644 containers/spindle-slurm-ubuntu/testing-plugin/conf/slurmdbd.conf.template
 create mode 100644 containers/spindle-slurm-ubuntu/testing-plugin/docker-compose.yml
 create mode 100755 containers/spindle-slurm-ubuntu/testing-plugin/generate_config.sh
 create mode 100755 containers/spindle-slurm-ubuntu/testing-plugin/scripts/add_docker_user.sh
 create mode 100755 containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh
 create mode 100755 containers/spindle-slurm-ubuntu/testing-plugin/scripts/entrypoint.sh
 create mode 100755 containers/spindle-slurm-ubuntu/testing-plugin/scripts/setup_slurm.sh
 create mode 100755 containers/spindle-slurm-ubuntu/testing-plugin/scripts/setup_spank_plugin.sh
 create mode 100644 testsuite/run_driver_slurm-plugin

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8c41a663..2e0e07d5 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -163,3 +163,61 @@ jobs:
           cd containers/spindle-slurm-ubuntu/testing
           docker compose down
 
+  spindle-slurm-plugin-ubuntu:
+    name: Testsuite (Slurm Plugin, Ubuntu)
+    environment: Spindle CI
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Check out Spindle
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
+
+      - name: Setup Docker Compose
+        uses: docker/setup-compose-action@364cc21a5de5b1ee4a7f5f9d3fa374ce0ccde746
+        with:
+          version: latest
+
+      - name: Login to GitHub Container Registry
+        if: ${{ !env.ACT }}
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Generate MariaDB configuration
+        id: slurm-ubuntu-mariadb
+        run: |
+          cd containers/spindle-slurm-ubuntu/testing-plugin
+          ./generate_config.sh
+
+      - name: Build spindle-slurm-plugin-ubuntu image
+        id: slurm-ubuntu-build
+        run: |
+          cd containers/spindle-slurm-ubuntu/testing-plugin
+          docker compose --progress=plain build
+
+      - name: Bring spindle-slurm-plugin-ubuntu up
+        id: slurm-ubuntu-up
+        run: |
+          cd containers/spindle-slurm-ubuntu/testing-plugin
+          docker compose up -d --wait --wait-timeout 120
+
+      - name: Verify munge works in spindle-slurm-plugin-ubuntu
+        id: slurm-ubuntu-munge
+        run: |
+          docker exec slurm-plugin-head bash -c 'munge -n | unmunge'
+
+      - name: Run spindle-slurm-plugin-ubuntu testsuite
+        id: slurm-ubuntu-testsuite
+        run: |
+          docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && salloc -n${workers} -N${workers} ./runTests ${workers}'
+
+      - name: Bring spindle-slurm-plugin-ubuntu down
+        id: slurm-ubuntu-down
+        if: ${{ always() }}
+        continue-on-error: true
+        run: |
+          cd containers/spindle-slurm-ubuntu/testing-plugin
+          docker compose down
+
diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile b/containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile
new file mode 100644
index 00000000..7b66a155
--- /dev/null
+++ b/containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile
@@ -0,0 +1,42 @@
+ARG BASE_VERSION=latest
+FROM ghcr.io/llnl/spindle-slurm-base:${BASE_VERSION}
+ARG replicas=4
+ENV workers=${replicas}
+
+ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing-plugin
+
+# Slurm daemons run as $SLURM_USER
+ARG SLURM_USER=slurm
+
+# Applications run as $USER
+ARG USER=slurmuser
+ARG UID=1001
+
+# Set up the Slurm install already present in the base image
+COPY ${BUILD_ROOT}/scripts/setup_slurm.sh /setup_slurm.sh
+COPY ${BUILD_ROOT}/conf/slurm.conf /home/${SLURM_USER}/slurm.conf
+COPY ${BUILD_ROOT}/conf/slurmdbd.conf /home/${SLURM_USER}/slurmdbd.conf
+COPY ${BUILD_ROOT}/conf/cgroup.conf /home/${SLURM_USER}/cgroup.conf
+RUN /setup_slurm.sh
+
+USER ${USER}
+WORKDIR /home/${USER}
+
+# Copy the Spindle repo into the container and build it
+RUN mkdir -p /home/${USER}/Spindle
+COPY . /home/${USER}/Spindle
+COPY ${BUILD_ROOT}/scripts/build_spindle.sh /home/${USER}/build_spindle.sh
+RUN ./build_spindle.sh
+
+USER root
+COPY ${BUILD_ROOT}/scripts/setup_spank_plugin.sh /setup_spank_plugin.sh
+COPY ${BUILD_ROOT}/conf/plugstack.conf /home/${SLURM_USER}/plugstack.conf
+RUN /setup_spank_plugin.sh
+
+USER ${USER}
+
+COPY ${BUILD_ROOT}/scripts/entrypoint.sh /home/${USER}/entrypoint.sh
+ENV PATH /home/${USER}/Spindle-inst/bin:$PATH
+
+ENTRYPOINT /bin/bash ./entrypoint.sh
+
diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/conf/cgroup.conf b/containers/spindle-slurm-ubuntu/testing-plugin/conf/cgroup.conf
new file mode 100644
index 00000000..e59e9aee
--- /dev/null
+++ b/containers/spindle-slurm-ubuntu/testing-plugin/conf/cgroup.conf
@@ -0,0 +1 @@
+CgroupPlugin=cgroup/v1
diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/conf/plugstack.conf b/containers/spindle-slurm-ubuntu/testing-plugin/conf/plugstack.conf
new file mode 100644
index 00000000..a291c1c0
--- /dev/null
+++ b/containers/spindle-slurm-ubuntu/testing-plugin/conf/plugstack.conf
@@ -0,0 +1 @@
+required /home/slurmuser/Spindle-inst/lib/libspindleslurm.so
diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/conf/slurm.conf b/containers/spindle-slurm-ubuntu/testing-plugin/conf/slurm.conf
new file mode 100644
index 00000000..abf060d5
--- /dev/null
+++ b/containers/spindle-slurm-ubuntu/testing-plugin/conf/slurm.conf
@@ -0,0 +1,42 @@
+ClusterName=linux
+ControlMachine=slurm-head
+ControlAddr=slurm-head
+SlurmUser=slurm
+SlurmctldPort=6817
+SlurmdPort=6818
+AuthType=auth/munge
+StateSaveLocation=/var/lib/slurmd
+SlurmdSpoolDir=/var/spool/slurmd
+SwitchType=switch/none
+MpiDefault=none
+SlurmctldPidFile=/var/run/slurmd/slurmctld.pid
+SlurmdPidFile=/var/run/slurmd/slurmd.pid
+ProctrackType=proctrack/linuxproc
+TaskPlugin=task/affinity
+ReturnToService=2
+SlurmctldTimeout=300
+SlurmdTimeout=300
+InactiveLimit=0
+MinJobAge=300
+KillWait=30
+Waittime=0
+SchedulerType=sched/backfill
+SelectType=select/cons_tres
+SelectTypeParameters=CR_Core_Memory
+SlurmctldDebug=3
+SlurmctldLogFile=/var/log/slurm/slurmctld.log
+SlurmdDebug=3
+SlurmdLogFile=/var/log/slurm/slurmd.log
+JobCompType=jobcomp/filetxt
+JobCompLoc=/var/log/slurm/jobcomp.log
+JobAcctGatherType=jobacct_gather/linux
+JobAcctGatherFrequency=30
+AccountingStorageType=accounting_storage/slurmdbd
+AccountingStorageHost=slurm-db
+AccountingStoragePort=6819
+NodeName=slurm-node-1 NodeAddr=slurm-node-1 CPUs=3 RealMemory=1000 State=UNKNOWN
+NodeName=slurm-node-2 NodeAddr=slurm-node-2 CPUs=3 RealMemory=1000 State=UNKNOWN
+NodeName=slurm-node-3 NodeAddr=slurm-node-3 CPUs=3 RealMemory=1000 State=UNKNOWN
+NodeName=slurm-node-4 NodeAddr=slurm-node-4 CPUs=3 RealMemory=1000 State=UNKNOWN
+PartitionName=debug Nodes=ALL Default=YES MaxTime=INFINITE State=UP
+
diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/conf/slurmdbd.conf.template b/containers/spindle-slurm-ubuntu/testing-plugin/conf/slurmdbd.conf.template
new file mode 100644
index 00000000..0e274118
--- /dev/null
+++ b/containers/spindle-slurm-ubuntu/testing-plugin/conf/slurmdbd.conf.template
@@ -0,0 +1,10 @@
+AuthType=auth/munge
+DbdAddr=slurm-db
+DbdHost=slurm-db
+SlurmUser=slurm
+DebugLevel=4
+LogFile=/var/log/slurm/slurmdbd.log
+PidFile=/var/run/slurmdbd/slurmdbd.pid
+StorageType=accounting_storage/mysql
+StorageHost=slurm-mariadb
+StorageUser=slurm
diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/docker-compose.yml b/containers/spindle-slurm-ubuntu/testing-plugin/docker-compose.yml
new file mode 100644
index 00000000..2751a71a
--- /dev/null
+++ b/containers/spindle-slurm-ubuntu/testing-plugin/docker-compose.yml
@@ -0,0 +1,126 @@
+# `replicas` must match the number of nodes defined in the services section
+x-shared-workers:
+  &workers
+  replicas: 4 
+
+# Base image version to use
+x-shared-build-args: &shared-build-args
+  BASE_VERSION: latest
+  <<: *workers
+
+# Docker prohibits copying files from outside of the build context.
+# In order to be able to copy the whole repo into the container,
+# we have to set the context to be the root of the repo.
+# We then have to specify the path from there to the Dockerfile.
+x-shared-build-context: &shared-build-context
+  context: ../../..
+  dockerfile: containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile
+  args: *shared-build-args
+
+# Name of the head node
+x-shared-environment: &shared-environment
+  SLURM_HEAD_NODE: slurm-head
+  <<: *workers
+
+# The entrypoint runs different services depending
+# on the node's role. Valid options are:
+#   - worker: runs slurmd
+#   - db:     runs slurmdbd
+#   - ctl:    runs slurmctld
+x-worker-environment: &worker-environment
+  SLURM_ROLE: worker
+  <<: *shared-environment
+
+networks:
+  slurm:
+    driver: bridge
+
+# Common parameters for all nodes.
+x-shared-node-parameters: &shared-node-parameters
+  build: *shared-build-context
+  networks:
+    - slurm
+  cap_add:
+    - SYS_NICE # Required for libnuma
+
+x-healthcheck-parameters: &healthcheck-parameters
+  start_period: 3s
+  interval: 3s
+  timeout: 5s
+  retries: 5
+
+x-worker-parameters: &worker-node-parameters
+  <<: *shared-node-parameters
+  environment: *worker-environment
+  depends_on:
+    slurm-head:
+      condition: service_healthy
+  healthcheck:
+    test: ["CMD", "stat", "/var/run/slurmd/slurmd.pid"]
+    <<: *healthcheck-parameters
+
+services:
+  slurm-mariadb:
+    image: mariadb:12
+    networks:
+      - slurm
+    hostname: slurm-mariadb
+    container_name: slurm-plugin-mariadb
+    env_file: mariadb.env
+    environment:
+      MYSQL_RANDOM_ROOT_PASSWORD: "yes"
+      MYSQL_DATABASE: "slurm_acct_db"
+      MYSQL_USER: "slurm"
+    healthcheck:
+      test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]
+      <<: *healthcheck-parameters
+
+  slurm-db:
+    <<: *shared-node-parameters
+    hostname: slurm-db
+    container_name: slurm-plugin-db
+    environment:
+      SLURM_ROLE: db
+      <<: *shared-environment
+    depends_on:
+      slurm-mariadb:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "stat", "/var/run/slurmdbd/slurmdbd.pid"]
+      <<: *healthcheck-parameters
+
+  slurm-head:
+    <<: *shared-node-parameters
+    hostname: slurm-head
+    container_name: slurm-plugin-head
+    tty: true
+    environment:
+      SLURM_ROLE: ctl
+      <<: *shared-environment
+    depends_on:
+      slurm-db:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "stat", "/var/run/slurmd/slurmctld.pid"]
+      <<: *healthcheck-parameters
+
+  slurm-node-1:
+    <<: *worker-node-parameters
+    hostname: slurm-node-1
+    container_name: slurm-plugin-node-1
+
+  slurm-node-2:
+    <<: *worker-node-parameters
+    hostname: slurm-node-2
+    container_name: slurm-plugin-node-2
+
+  slurm-node-3:
+    <<: *worker-node-parameters
+    hostname: slurm-node-3
+    container_name: slurm-plugin-node-3
+
+  slurm-node-4:
+    <<: *worker-node-parameters
+    hostname: slurm-node-4
+    container_name: slurm-plugin-node-4
+
diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/generate_config.sh b/containers/spindle-slurm-ubuntu/testing-plugin/generate_config.sh
new file mode 100755
index 00000000..eadb1c8a
--- /dev/null
+++ b/containers/spindle-slurm-ubuntu/testing-plugin/generate_config.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# Generate random password for the MariaDB slurm user
+# and set it in config files
+
+APP_USER=${APP_USER:-slurmuser}
+MARIADB_PASS=$(openssl rand --base64 16 | head -c -3)
+echo "MARIADB_PASSWORD: \"${MARIADB_PASS}\"" > mariadb.env
+cp conf/slurmdbd.conf.template conf/slurmdbd.conf
+echo "StoragePass=${MARIADB_PASS}" >> conf/slurmdbd.conf
+
+# Enable Spindle SPANK plugin
+
+echo "required /home/${APP_USER}/Spindle-inst/lib/libspindleslurm.so" > conf/plugstack.conf
diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/add_docker_user.sh b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/add_docker_user.sh
new file mode 100755
index 00000000..ace8c619
--- /dev/null
+++ b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/add_docker_user.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+sudo groupadd -g ${UID} ${USER}
+sudo useradd -g ${USER} -u ${UID} -d /home/${USER} -m ${USER}
+# Allow user to run as other users so that munge can be started as the munge user
+sudo sh -c "printf \"${USER} ALL=(ALL) NOPASSWD: ALL\\n\" >> /etc/sudoers"
+sudo adduser ${USER} sudo
+sudo usermod -s /bin/bash ${USER}
+
diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh
new file mode 100755
index 00000000..2a252b32
--- /dev/null
+++ b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+mkdir -p /home/${USER}/Spindle-build
+cd /home/${USER}/Spindle-build
+/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-localstorage=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
+make -j$(nproc)
+make install
+
diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/entrypoint.sh b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/entrypoint.sh
new file mode 100755
index 00000000..54d40e48
--- /dev/null
+++ b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/entrypoint.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+echo "SLURM_ROLE: ${SLURM_ROLE}"
+
+echo "Starting sshd..."
+sudo service ssh start
+echo "Starting munged..."
+sudo -u munge /usr/sbin/munged 
+
+if [ "${SLURM_ROLE}" = "db" ]; then
+    echo "Starting slurmdbd..."
+    sudo -u slurm /usr/sbin/slurmdbd -Dvvv
+elif [ "${SLURM_ROLE}" = "ctl" ] ; then
+    echo "Starting slurmctld..."
+    sudo -u slurm /usr/sbin/slurmctld -i -Dvvv
+elif [ "${SLURM_ROLE}" = "worker" ] ; then
+    echo "Starting slurmd..."
+    sudo /usr/sbin/slurmd -Dvvv
+fi
+
+sleep inf
diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/setup_slurm.sh b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/setup_slurm.sh
new file mode 100755
index 00000000..186beea0
--- /dev/null
+++ b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/setup_slurm.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+mkdir -p /etc/slurm /etc/sysconfig/slurm /var/spool/slurmd /var/spool/slurmctld /var/run/slurmd /var/run/slurmdbd /var/lib/slurmd /var/log/slurm
+touch /var/lib/slurmd/node_state /var/lib/slurmd/front_end_state /var/lib/slurmd/job_state /var/lib/slurmd/resv_state /var/lib/slurmd/trigger_state /var/lib/slurmd/assoc_mgr_state /var/lib/slurmd/assoc_usage /var/lib/slurmd/qos_usage /var/lib/slurmd/fed_mgr_state 
+cp /home/${SLURM_USER}/slurm.conf /etc/slurm/slurm.conf
+cp /home/${SLURM_USER}/slurmdbd.conf /etc/slurm/slurmdbd.conf
+cp /home/${SLURM_USER}/cgroup.conf /etc/slurm/cgroup.conf
+chown -R slurm:slurm /etc/slurm /etc/sysconfig/slurm /var/spool/slurmd /var/spool/slurmctld /var/run/slurmd /var/run/slurmdbd /var/lib/slurmd /var/log/slurm 
+chmod 600 /etc/slurm/slurmdbd.conf
diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/setup_spank_plugin.sh b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/setup_spank_plugin.sh
new file mode 100755
index 00000000..7b6ea089
--- /dev/null
+++ b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/setup_spank_plugin.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+cp /home/${SLURM_USER}/plugstack.conf /etc/slurm/plugstack.conf
+chown -R slurm:slurm /etc/slurm
diff --git a/testsuite/run_driver_slurm-plugin b/testsuite/run_driver_slurm-plugin
new file mode 100644
index 00000000..6d06cbbf
--- /dev/null
+++ b/testsuite/run_driver_slurm-plugin
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [ x$SLURM_CPUS_ON_NODE = x ] ; then
+export SLURM_CPUS_ON_NODE=1
+fi
+
+if [ x$SPINDLE_TEST_ARGS = x ] ; then
+((PROCS=$SLURM_NNODES*$SLURM_CPUS_ON_NODE))
+else
+PROCS=$SPINDLE_TEST_ARGS
+fi
+export PROCS
+
+if [ "x$SPINDLE_BGQ_LD_PRELOAD" = "xtrue" ] ; then
+PRELOAD_ARGS="--runjob-opts=--envs LD_PRELOAD=$LIBRARY_LIST"
+elif [ "x$SPINDLE_LD_PRELOAD" != "x" ] ; then
+PRELOAD_ARGS="--export=ALL,LD_PRELOAD=$SPINDLE_LD_PRELOAD"
+fi
+
+exec srun $PRELOAD_ARGS --spindle="$SPINDLE_FLAGS $SPINDLE_OPTS" -n $PROCS $*

From 4f8d30bc9143564743f86246660660d1de2c91ca Mon Sep 17 00:00:00 2001
From: Nicholas Chaimov <nchaimov@paratools.com>
Date: Tue, 10 Feb 2026 16:11:46 -0800
Subject: [PATCH 53/57] Use SLURM_STEP_NODELIST to account for change in NNODES
 between salloc/sbatch and srun

---
 src/slurm_plugin/slurm_plugin.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c
index c7bbcef2..5401b337 100644
--- a/src/slurm_plugin/slurm_plugin.c
+++ b/src/slurm_plugin/slurm_plugin.c
@@ -344,11 +344,13 @@ static char **get_hostlist(spank_t spank, unsigned int num_hosts)
 {
    char *short_hosts, **hostlist;;
 
-   short_hosts = readSpankEnv(spank, "SLURM_JOB_NODELIST");
+   short_hosts = readSpankEnv(spank, "SLURM_STEP_NODELIST");
+   if (!short_hosts)
+      short_hosts = readSpankEnv(spank, "SLURM_JOB_NODELIST");
    if (!short_hosts)
       short_hosts = readSpankEnv(spank, "SLURM_NODELIST");
    if (!short_hosts) {
-      sdprintf(1, "ERROR: SLURM_JOB_NODELIST not set.\n");
+      sdprintf(1, "ERROR: None of SLURM_STEP_NODELIST, SLURM_JOB_NODELIST, SLURM_NODELIST set.\n");
       return NULL;
    }   
 

From 84ef9e70ba8e3fdd14068957d6da5eef3abbbd2e Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 12 Feb 2026 10:41:36 -0800
Subject: [PATCH 54/57] Restores --with-localstorage to generate error.

The option is marked as obsolete in configure --help and will
cause an error in configure if it is specified.

As updates the CI configure scripts to use --with-cachepaths and
--with-commpath instead of --with-localstorage.
---
 configure                                                | 9 +++++++++
 configure.common.ac                                      | 4 ++++
 containers/spindle-flux-ubuntu/scripts/build_spindle.sh  | 2 +-
 .../spindle-serial-ubuntu/scripts/build_spindle.sh       | 2 +-
 .../testing/scripts/build_spindle.sh                     | 2 +-
 src/client/configure                                     | 9 +++++++++
 src/fe/configure                                         | 9 +++++++++
 src/server/configure                                     | 9 +++++++++
 8 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/configure b/configure
index 897d9d5f..10b41e90 100755
--- a/configure
+++ b/configure
@@ -846,6 +846,7 @@ enable_libtool_lock
 enable_maintainer_mode
 with_default_port
 with_default_num_ports
+with_localstorage
 with_cachepaths
 with_commpath
 with_default_local_prefix
@@ -1590,6 +1591,8 @@ Optional Packages:
   --with-default-numports=NUM
                           Number of TCP/IP ports to scan for Spindle server
                           communication
+  --with-localstorage=DIR (obsolete)
+                          Use --with-cachepaths and --with-commpath instead.
   --with-cachepaths=DIR   Colon-separated list of potential back-end cache
                           directories
   --with-compath=DIR      Back-end directory for communication and
@@ -16619,6 +16622,12 @@ else
 fi
 
 
+# Check whether --with-localstorage was given.
+if test "${with_localstorage+set}" = set; then :
+  withval=$with_localstorage; as_fn_error $? "requested obsolete option --with-localstorage.  Use --with-cachepaths and --with-commpath instead." "$LINENO" 5
+fi
+
+
 # Check whether --with-cachepaths was given.
 if test "${with_cachepaths+set}" = set; then :
   withval=$with_cachepaths; CACHEPATHS=${withval}
diff --git a/configure.common.ac b/configure.common.ac
index b25bc3af..aa3d55e4 100644
--- a/configure.common.ac
+++ b/configure.common.ac
@@ -17,6 +17,10 @@ AC_ARG_WITH(default-num-ports,
             [AS_HELP_STRING([--with-default-numports=NUM],[Number of TCP/IP ports to scan for Spindle server communication])],
             [NUM_COBO_PORTS=${withval}],
             [NUM_COBO_PORTS=$DEFAULT_NUM_COBO_PORTS])
+AC_ARG_WITH(localstorage,
+            [AS_HELP_STRING([--with-localstorage=DIR (obsolete)],[Use --with-cachepaths and --with-commpath instead.])],
+            [AC_MSG_ERROR(requested obsolete option --with-localstorage.  Use --with-cachepaths and --with-commpath instead.)],
+            [])
 AC_ARG_WITH(cachepaths,
             [AS_HELP_STRING([--with-cachepaths=DIR],[Colon-separated list of potential back-end cache directories])],
             [CACHEPATHS=${withval}],
diff --git a/containers/spindle-flux-ubuntu/scripts/build_spindle.sh b/containers/spindle-flux-ubuntu/scripts/build_spindle.sh
index 9257f85e..283e4451 100755
--- a/containers/spindle-flux-ubuntu/scripts/build_spindle.sh
+++ b/containers/spindle-flux-ubuntu/scripts/build_spindle.sh
@@ -4,7 +4,7 @@ set -euxo pipefail
 
 mkdir -p /home/${USER}/Spindle-build
 cd /home/${USER}/Spindle-build
-/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=flux --enable-flux-plugin --with-localstorage=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
+/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=flux --enable-flux-plugin --with-cachepaths=/tmp --with-commpath=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
 make -j$(nproc)
 make install
 
diff --git a/containers/spindle-serial-ubuntu/scripts/build_spindle.sh b/containers/spindle-serial-ubuntu/scripts/build_spindle.sh
index 37b6491a..4fee85b4 100755
--- a/containers/spindle-serial-ubuntu/scripts/build_spindle.sh
+++ b/containers/spindle-serial-ubuntu/scripts/build_spindle.sh
@@ -4,7 +4,7 @@ set -euxo pipefail
 
 mkdir -p /home/${USER}/Spindle-build
 cd /home/${USER}/Spindle-build
-/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=serial --with-localstorage=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
+/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=serial --with-cachepaths=/tmp --with-commpath=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
 make -j$(nproc)
 make install
 
diff --git a/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh b/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh
index 6943e49a..17e7197f 100755
--- a/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh
+++ b/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh
@@ -3,7 +3,7 @@ set -euxo pipefail
 
 mkdir -p /home/${USER}/Spindle-build
 cd /home/${USER}/Spindle-build
-/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-rsh-launch --with-rsh-cmd=/usr/bin/ssh --with-localstorage=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
+/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-rsh-launch --with-rsh-cmd=/usr/bin/ssh --with-cachepaths=/tmp --with-commpath=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
 make -j$(nproc)
 make install
 
diff --git a/src/client/configure b/src/client/configure
index b7dbffc0..94bd62cc 100755
--- a/src/client/configure
+++ b/src/client/configure
@@ -810,6 +810,7 @@ enable_libtool_lock
 enable_maintainer_mode
 with_default_port
 with_default_num_ports
+with_localstorage
 with_cachepaths
 with_commpath
 with_default_local_prefix
@@ -1533,6 +1534,8 @@ Optional Packages:
   --with-default-numports=NUM
                           Number of TCP/IP ports to scan for Spindle server
                           communication
+  --with-localstorage=DIR (obsolete)
+                          Use --with-cachepaths and --with-commpath instead.
   --with-cachepaths=DIR   Colon-separated list of potential back-end cache
                           directories
   --with-compath=DIR      Back-end directory for communication and
@@ -12591,6 +12594,12 @@ else
 fi
 
 
+# Check whether --with-localstorage was given.
+if test "${with_localstorage+set}" = set; then :
+  withval=$with_localstorage; as_fn_error $? "requested obsolete option --with-localstorage.  Use --with-cachepaths and --with-commpath instead." "$LINENO" 5
+fi
+
+
 # Check whether --with-cachepaths was given.
 if test "${with_cachepaths+set}" = set; then :
   withval=$with_cachepaths; CACHEPATHS=${withval}
diff --git a/src/fe/configure b/src/fe/configure
index d983778a..aa0b544e 100755
--- a/src/fe/configure
+++ b/src/fe/configure
@@ -831,6 +831,7 @@ enable_libtool_lock
 enable_maintainer_mode
 with_default_port
 with_default_num_ports
+with_localstorage
 with_cachepaths
 with_commpath
 with_default_local_prefix
@@ -1571,6 +1572,8 @@ Optional Packages:
   --with-default-numports=NUM
                           Number of TCP/IP ports to scan for Spindle server
                           communication
+  --with-localstorage=DIR (obsolete)
+                          Use --with-cachepaths and --with-commpath instead.
   --with-cachepaths=DIR   Colon-separated list of potential back-end cache
                           directories
   --with-compath=DIR      Back-end directory for communication and
@@ -16441,6 +16444,12 @@ else
 fi
 
 
+# Check whether --with-localstorage was given.
+if test "${with_localstorage+set}" = set; then :
+  withval=$with_localstorage; as_fn_error $? "requested obsolete option --with-localstorage.  Use --with-cachepaths and --with-commpath instead." "$LINENO" 5
+fi
+
+
 # Check whether --with-cachepaths was given.
 if test "${with_cachepaths+set}" = set; then :
   withval=$with_cachepaths; CACHEPATHS=${withval}
diff --git a/src/server/configure b/src/server/configure
index b71bdd82..62611580 100755
--- a/src/server/configure
+++ b/src/server/configure
@@ -837,6 +837,7 @@ enable_libtool_lock
 enable_maintainer_mode
 with_default_port
 with_default_num_ports
+with_localstorage
 with_cachepaths
 with_commpath
 with_default_local_prefix
@@ -1568,6 +1569,8 @@ Optional Packages:
   --with-default-numports=NUM
                           Number of TCP/IP ports to scan for Spindle server
                           communication
+  --with-localstorage=DIR (obsolete)
+                          Use --with-cachepaths and --with-commpath instead.
   --with-cachepaths=DIR   Colon-separated list of potential back-end cache
                           directories
   --with-compath=DIR      Back-end directory for communication and
@@ -16438,6 +16441,12 @@ else
 fi
 
 
+# Check whether --with-localstorage was given.
+if test "${with_localstorage+set}" = set; then :
+  withval=$with_localstorage; as_fn_error $? "requested obsolete option --with-localstorage.  Use --with-cachepaths and --with-commpath instead." "$LINENO" 5
+fi
+
+
 # Check whether --with-cachepaths was given.
 if test "${with_cachepaths+set}" = set; then :
   withval=$with_cachepaths; CACHEPATHS=${withval}

From 123ba20c01d5e5ba66ce336e300e96a86557515c Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 12 Feb 2026 16:27:54 -0800
Subject: [PATCH 55/57] Updates spank plugin to use commpath.

Replaces args->location with args->commpath.
---
 src/slurm_plugin/slurm_plugin.c | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c
index 5401b337..2dd70bc3 100644
--- a/src/slurm_plugin/slurm_plugin.c
+++ b/src/slurm_plugin/slurm_plugin.c
@@ -70,6 +70,7 @@ static int enable_spindle = 0;
 
 extern char **environ;
 extern char *parse_location(char *loc, number_t number);
+extern char *realize(char *path);
 
 struct spank_option spank_options[] =
 {
@@ -228,7 +229,7 @@ static unique_id_t getUniqueID(spank_t spank)
 static int fillInArgs(spank_t spank, spindle_args_t *args, int argc, char **argv, unique_id_t unique_id)
 {
    int result;
-   char *oldlocation;
+   char *symbolic_commpath, *orig_commpath;
    char *err_string;
 
    args->unique_id = unique_id;
@@ -247,10 +248,14 @@ static int fillInArgs(spank_t spank, spindle_args_t *args, int argc, char **argv
    args->use_launcher = slurm_plugin_launcher;
    args->startup_type = startup_external;
 
-   oldlocation = args->location;
+   symbolic_commpath = args->commpath;
+   orig_commpath = parse_location(xmbolic_commpath, args->number);
+   if( !orig_commpath ){
+       return -1;
+   }
+   args->commpath = realize(orig_commpath)
+
    current_spank = spank;
-   args->location = parse_location(oldlocation, args->number);
-   free(oldlocation);
 
    return 0;
 }
@@ -663,16 +668,16 @@ static int handleExit(void *params, char **output_str)
       return 0;
    }
 
-   if (!args.location) {
-      sdprintf(2, "WARNING: spindleExitBE not called since location is NULL\n");
+   if (!args.commpath) {
+      sdprintf(2, "WARNING: spindleExitBE not called since commpath is NULL\n");
    } else {
       // The task_exit callback is run for _each proc_, so we use
       // isBEProc to pick only one proc per node to call spindleExitBE.
       is_be_leader = isBEProc(&args, 1);
       if (is_be_leader) {
-         result = spindleExitBE(args.location);
+         result = spindleExitBE(args.commpath);
          if (result == -1) {
-             sdprintf(1, "ERROR: spindleExitBE returned an error on location %s\n", args.location);
+             sdprintf(1, "ERROR: spindleExitBE returned an error on commpath %s\n", args.commpath);
              return -1;
          }
       }

From 007b2c740a5b4f5811b358f41a7c0a0c368a442d Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 12 Feb 2026 17:15:21 -0800
Subject: [PATCH 56/57] Updates a configure script to use commpath.

---
 .../testing-plugin/scripts/build_spindle.sh                     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh
index 2a252b32..879a28c6 100755
--- a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh
+++ b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh
@@ -3,7 +3,7 @@ set -euxo pipefail
 
 mkdir -p /home/${USER}/Spindle-build
 cd /home/${USER}/Spindle-build
-/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-localstorage=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
+/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-commpath=/tmp --with-cachepath=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
 make -j$(nproc)
 make install
 

From 3e02e46cd92b107f0894bece9d4fdf02dbdb01c0 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 12 Feb 2026 20:12:23 -0800
Subject: [PATCH 57/57] Additional integration for commpath + spank-plugin.

---
 src/slurm_plugin/plugin_utils.c | 6 +++---
 src/slurm_plugin/slurm_plugin.c | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/slurm_plugin/plugin_utils.c b/src/slurm_plugin/plugin_utils.c
index 3be45742..392677c4 100644
--- a/src/slurm_plugin/plugin_utils.c
+++ b/src/slurm_plugin/plugin_utils.c
@@ -269,7 +269,7 @@ int isBEProc(spindle_args_t *params, unsigned int exit_phase)
    int beproc_result = -1;
    int fd = -1, error;
    
-   dir = params->location;
+   dir = params->commpath;
    if (!dir) {
       sdprintf(1, "ERROR: Location not filled in\n");
       goto done;
@@ -615,7 +615,7 @@ int registerFEPid(pid_t pid, spindle_args_t *args)
    int fd;
    int result;
 
-   snprintf(pid_file, sizeof(pid_file), "%s/fepid", args->location);
+   snprintf(pid_file, sizeof(pid_file), "%s/fepid", args->commpath);
    pid_file[sizeof(pid_file)-1] = '\0';
 
    snprintf(pid_s, sizeof(pid_s), "%d\n", (int) pid);
@@ -646,7 +646,7 @@ int readFEPid(pid_t *pid, spindle_args_t *args)
    pid_t pid_result;
    int fd, result;
 
-   snprintf(pid_file, sizeof(pid_file), "%s/fepid", args->location);
+   snprintf(pid_file, sizeof(pid_file), "%s/fepid", args->commpath);
    pid_file[sizeof(pid_file)-1] = '\0';
 
    sdprintf(2, "Reading FE pid from %s\n", pid_file);
diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c
index 2dd70bc3..6aadfa6d 100644
--- a/src/slurm_plugin/slurm_plugin.c
+++ b/src/slurm_plugin/slurm_plugin.c
@@ -249,11 +249,11 @@ static int fillInArgs(spank_t spank, spindle_args_t *args, int argc, char **argv
    args->startup_type = startup_external;
 
    symbolic_commpath = args->commpath;
-   orig_commpath = parse_location(xmbolic_commpath, args->number);
+   orig_commpath = parse_location(symbolic_commpath, args->number);
    if( !orig_commpath ){
        return -1;
    }
-   args->commpath = realize(orig_commpath)
+   args->commpath = realize(orig_commpath);
 
    current_spank = spank;