Ctest locks for limited parallel execution

.gitignore

@@ -37,3 +37,4 @@ CMakeLists.txt.user
 CMakeUserPresets.json
 testing/data/tmp*
 testing/data/*-tmp-*
+testing/data/.mrtrix_testing.lock

cmake/BashTests.cmake

@@ -6,11 +6,12 @@ function(add_bash_test)
         message(FATAL_ERROR "bash not found")
     endif()
 
+    set(options NO_FILESYSTEM_LOCK)
     set(singleValueArgs FILE_PATH PREFIX WORKING_DIRECTORY ENVIRONMENT)
     set(multiValueArgs EXEC_DIRECTORIES LABELS)
     cmake_parse_arguments(
         ARG
-        ""
+        "${options}"
         "${singleValueArgs}"
         "${multiValueArgs}"
         ${ARGN}
@@ -34,22 +35,50 @@ function(add_bash_test)
 
     string(REPLACE ";" ":" exec_directories "${exec_directories}")
 
-    set(cleanup_cmd "rm -rf ${working_directory}/tmp* ${working_directory}/*-tmp-*")
+    # Bash tests sharing a working directory write, overwrite and delete temporary
+    # filesystem content prefixed "tmp"; running two such tests concurrently lets
+    # one test's cleanup clobber another's in-flight temporaries.
+    # Tests are serialised per working directory by two complementary mechanisms:
+    #   - the CTest RESOURCE_LOCK property (within a single ctest invocation), and
+    #   - a filesystem-level lock in RunTest.cmake (across invocations/instances).
+    # A test that the developer certifies never writes to the filesystem can opt
+    # out of both via NO_FILESYSTEM_LOCK, leaving it free to run in parallel.
+    set(run_test_args
+        -D BASH=${BASH}
+        -D FILE_PATH=${file_path}
+        -D WORKING_DIRECTORY=${working_directory}
+    )
+    if(NOT ARG_NO_FILESYSTEM_LOCK)
+        # The lock file lives inside the guarded directory itself, so every test
+        # sharing that directory (including across build trees that share it)
+        # agrees on the same lock; its fixed name avoids the "tmp" cleanup glob.
+        set(cleanup_cmd "rm -rf ${working_directory}/tmp* ${working_directory}/*-tmp-*")
+        set(lock_file "${working_directory}/.mrtrix_testing.lock")
+        list(APPEND run_test_args
+            -D CLEANUP_CMD=${cleanup_cmd}
+            -D LOCKFILE=${lock_file}
+        )
+    endif()
 
     add_test(
         NAME ${test_name}
         COMMAND
             ${CMAKE_COMMAND}
-            -D BASH=${BASH}
-            -D FILE_PATH=${file_path}
-            -D CLEANUP_CMD=${cleanup_cmd}
-            -D WORKING_DIRECTORY=${working_directory}
+            ${run_test_args}
             -P ${PROJECT_SOURCE_DIR}/cmake/RunTest.cmake
     )
     set_tests_properties(${test_name}
         PROPERTIES
         ENVIRONMENT "PATH=${exec_directories};${environment}"
     )
+    if(NOT ARG_NO_FILESYSTEM_LOCK)
+        # One resource per unique working directory: tests holding the same lock
+        # string are never scheduled concurrently by a single ctest invocation.
+        set_tests_properties(${test_name}
+            PROPERTIES
+            RESOURCE_LOCK "${working_directory}"
+        )
+    endif()
     if(labels)
         set_tests_properties(${test_name} PROPERTIES LABELS "${labels}")
     endif()

cmake/RunTest.cmake

@@ -1,15 +1,30 @@
 # This script can be invoked by calling ${CMAKE_COMMAND} -P <script_name>.cmake
 # Runs a bash test described by the FILE_PATH variable
 # The test is run in the WORKING_DIRECTORY directory
-# The CLEANUP_CMD command is run after the test is finished
+# The CLEANUP_CMD command, if provided, is run after the test is finished
+# If LOCKFILE is provided, an exclusive filesystem-level lock is held across both
+# the test and its cleanup, serialising tests that share a working directory even
+# across separate ctest invocations or instances. The lock is released
+# automatically when this process exits (including on crash), so a failed test
+# cannot deadlock subsequent runs.
+
+if(DEFINED LOCKFILE AND NOT LOCKFILE STREQUAL "")
+    # file(LOCK) yields "0" on success or an error message otherwise.
+    file(LOCK ${LOCKFILE} GUARD PROCESS RESULT_VARIABLE lock_result)
+    if(NOT lock_result STREQUAL "0")
+        message(FATAL_ERROR "Failed to acquire test lock ${LOCKFILE}: ${lock_result}")
+    endif()
+endif()
 
 execute_process(COMMAND ${BASH} -e ${FILE_PATH}
     RESULT_VARIABLE test_result_${FILE_PATH}
     WORKING_DIRECTORY ${WORKING_DIRECTORY}
 )
-execute_process(COMMAND ${BASH} -c ${CLEANUP_CMD}
-    WORKING_DIRECTORY ${WORKING_DIRECTORY}
-)
+if(DEFINED CLEANUP_CMD AND NOT CLEANUP_CMD STREQUAL "")
+    execute_process(COMMAND ${BASH} -c ${CLEANUP_CMD}
+        WORKING_DIRECTORY ${WORKING_DIRECTORY}
+    )
+endif()
 
 if(test_result_${FILE_PATH} EQUAL 0)
     message(STATUS "Test ${FILE_PATH} passed")