Feature/oshmem

.github/workflows/github_actions.yml

@@ -61,14 +61,6 @@ jobs:
             config_opts: --enable-i4 --without-blas --enable-cxx --disable-f77
             f77: gfortran
             cc: gcc
-          - os: macos-13
-            experimental: true
-            mpi_impl: mpich
-            armci_network: mpi-pr
-            f77: ifort
-            cc: icc
-            config_opts: LIBS=-lifcore
-            oneapi: /Users/runner/apps/oneapi
           - os: ubuntu-22.04
             experimental: true 
             mpi_impl: mpich
@@ -83,14 +75,14 @@ jobs:
             config_opts: "--disable-f77 --enable-cxx"
             f77: gfortran-10
             cc: gcc-10
-          - os: macos-13
+          - os: macos-15
             experimental: true 
             mpi_impl: mpich
             armci_network: mpi-ts
             config_opts: "--disable-static --enable-shared"
             f77: gfortran-13
             cc: clang
-          - os: macos-13
+          - os: macos-15
             experimental: true
             use_cmake: "Y"
             mpi_impl: mpich
@@ -181,6 +173,15 @@ jobs:
           macos*)
           brew update
           brew install coreutils automake || true
+          if [[ "$F77" =~ gfortran-[0-9][0-9] ]] || [[ "$CC" =~ gcc-[0-9][0-9] ]]; then
+          if [[ "$CC" =~ gcc-[0-9][0-9] ]]; then
+          version=$(echo "$CC" | cut -d - -f 2 )
+          fi
+          if [[ "$F77" =~ gfortran-[0-9][0-9] ]]; then
+          version=$(echo "$F77" | cut -d - -f 2 )
+          fi
+          brew reinstall gcc@${version}
+          fi
           ;;
           esac
           if [[ "$F77" == "ifort" ]] || [[ "$CC" == "icc" ]] || [[ "$CC" == "icx" ]]; then ./travis/install-intel.sh; source ${{ matrix.oneapi }}/setvars.sh --force;  fi

.gitignore

@@ -1,3 +1,32 @@
+# Ignore build directories and their contents
+build_shmem/
+build/
+bld*/
+
+# Ignore CMake build directories and files
+CMakeFiles/
+cmake_install.cmake
+CMakeCache.txt
+CTestTestfile.cmake
+Makefile
+compile_commands.json
+globalarrays-config.cmake
+globalarrays-config-version.cmake
+globalarrays-targets.cmake
+
+# Ignore all files in any CMakeFiles directory recursively
+**/CMakeFiles/
+**/CMakeFiles/**
+
+# Ignore test and object files
+*.x
+*.o
+*.log
+
+# Ignore any .o, .x, .log files in subdirectories
+**/*.o
+**/*.x
+**/*.log
 # Compiled Object files
 *.slo
 *.lo

CMakeLists.txt

@@ -500,6 +500,14 @@ install(TARGETS ga
     ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
 )
 
+if(ENABLE_CXX)
+  install(TARGETS ga++
+      EXPORT globalarrays-targets
+      LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+      ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  )
+endif()
+
 #Export the targets to a script
 install(EXPORT globalarrays-targets
     FILE

Makefile.am

@@ -88,6 +88,12 @@ endif
 
 LDADD += libga.la
 
+# When using OpenSHMEM, add MPI libraries explicitly for test programs
+# that call MPI functions (like ffflush.F with mpi_finalize)
+if WITH_OSHMEM
+LDADD += -lmpi_mpifh -lmpi
+endif
+
 # Certain trickery when turning Fortran support on or off.
 if ENABLE_F77
 MAYBE_FLIBS = $(FLIBS)
@@ -523,6 +529,7 @@ check_PROGRAMS += global/testing/testeig
 check_PROGRAMS += global/testing/testmatmult
 check_PROGRAMS += global/testing/testsolve
 check_PROGRAMS += global/testing/test
+check_PROGRAMS += global/testing/accTest
 check_PROGRAMS += global/testing/overlay
 check_PROGRAMS += global/testing/test_mirrored
 check_PROGRAMS += global/testing/types_test
@@ -721,6 +728,7 @@ GLOBAL_PARALLEL_TESTS += global/testing/testeig$(EXEEXT)
 GLOBAL_PARALLEL_TESTS += global/testing/testmatmult$(EXEEXT)
 GLOBAL_PARALLEL_TESTS += global/testing/testsolve$(EXEEXT)
 GLOBAL_PARALLEL_TESTS += global/testing/test$(EXEEXT)
+GLOBAL_PARALLEL_TESTS += global/testing/accTest$(EXEEXT)
 GLOBAL_PARALLEL_TESTS += global/testing/overlay$(EXEEXT)
 if HAVE_SCALAPACK
 GLOBAL_PARALLEL_TESTS += global/testing/testspd$(EXEEXT)
@@ -967,6 +975,7 @@ global_testing_sprs_testf_SOURCES          = global/testing/sprs_testf.F $(gtsrc
 global_testing_stride_SOURCES              = global/testing/stride.F $(gtsrcf)
 global_testing_testabstract_ops_SOURCES    = global/testing/testabstract_ops.c
 global_testing_test_SOURCES                = global/testing/test.F $(gtsrcf)
+global_testing_accTest_SOURCES             = global/testing/accTest.F $(gtsrcf)
 global_testing_test_mirrored_SOURCES       = global/testing/test.F $(gtsrcf)
 global_testing_overlay_SOURCES             = global/testing/overlay.F $(gtsrcf)
 global_testing_testc_SOURCES               = global/testing/testc.c
@@ -1134,7 +1143,8 @@ global/examples/boltzmann/printdat.F      \
 global/examples/boltzmann/properties.F    \
 global/examples/boltzmann/setup.F         \
 global/examples/boltzmann/timestep.F      \
-global/examples/boltzmann/vorticity.F
+global/examples/boltzmann/vorticity.F     \
+global/testing/ffflush.F
 global_examples_boltzmann_boltz_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/global/examples/boltzmann
 
 global_examples_conjugate_gradient_ga_cg_SOURCES = \

armci/src/xfer/strided.c

@@ -746,6 +746,14 @@ int PARMCI_GetS( void *src_ptr,  	/* pointer to 1st segment at source*/
 
   ORDER(GET,proc);
   ARMCI_INIT_HANDLE(&nbh);
+  /* Diagnostic: print seg_count and stride_levels to check units (bytes vs elements) */
+  {
+    int _i;
+    fprintf(stderr, "[ARMCI PARMCI_GetS] proc=%d stride_levels=%d seg_count:", proc, stride_levels);
+    for (_i = 0; _i <= stride_levels; _i++) fprintf(stderr, " %d", seg_count[_i]);
+    fprintf(stderr, "\n");
+    fflush(stderr);
+  }
   PARMCI_NbGetS(src_ptr,src_stride_arr,dst_ptr,dst_stride_arr,seg_count,stride_levels,proc,&nbh);
   PARMCI_Wait(&nbh);
   return 0;

cmake/ga-compiler-options.cmake

@@ -15,25 +15,28 @@ if(CMAKE_C_COMPILER_ID STREQUAL "Clang")
     if(GCCROOT)
       set(__GA_GCC_INSTALL_PREFIX ${GCCROOT})
       set(GA_GCC_TOOLCHAIN_FLAG "--gcc-toolchain=${GCCROOT}")
-    else()
-      get_filename_component(__GA_GCC_INSTALL_PREFIX "${CMAKE_Fortran_COMPILER}/../.." ABSOLUTE)
-      if(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
-        set(GA_GCC_TOOLCHAIN_FLAG "--gcc-toolchain=${__GA_GCC_INSTALL_PREFIX}")
-      else()
-        message(FATAL_ERROR "GCCROOT cmake option not set when using clang compilers. \
-                Please set a valid path to the GCC installation.")
-      endif()
+    # else()
+    #   get_filename_component(__GA_GCC_INSTALL_PREFIX "${CMAKE_Fortran_COMPILER}/../.." ABSOLUTE)
+    #   if(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+    #     set(GA_GCC_TOOLCHAIN_FLAG "--gcc-toolchain=${__GA_GCC_INSTALL_PREFIX}")
+    #   else()
+    #     message(FATAL_ERROR "GCCROOT cmake option not set when using clang compilers. \
+    #             Please set a valid path to the GCC installation.")
+    #   endif()
     endif()
     #Check GCC installation
-    if(NOT 
+    if(GCCROOT)
+      if(NOT 
        (EXISTS ${__GA_GCC_INSTALL_PREFIX}/bin AND
         EXISTS ${__GA_GCC_INSTALL_PREFIX}/include AND
         EXISTS ${__GA_GCC_INSTALL_PREFIX}/lib)
       )
       message(FATAL_ERROR "GCC installation path found ${__GA_GCC_INSTALL_PREFIX} seems to be incorrect. \
       Please set the GCCROOT cmake option to the correct GCC installation prefix.")
-    endif()
+      endif()
     message(STATUS "GA_GCC_TOOLCHAIN_FLAG: ${GA_GCC_TOOLCHAIN_FLAG}")
+    endif()
+
     set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${GA_GCC_TOOLCHAIN_FLAG}")
     set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${GA_GCC_TOOLCHAIN_FLAG}")
     set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} ${GA_GCC_TOOLCHAIN_FLAG}")

cmake/linalg-modules/FindStandardFortran.cmake

@@ -21,7 +21,10 @@ if(CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
 elseif(CMAKE_Fortran_COMPILER_ID MATCHES "Intel")
     set(STANDARDFORTRAN_LIBS ifcore)
 elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
-    set(STANDARDFORTRAN_LIBS FortranRuntime FortranDecimal)
+  set(STANDARDFORTRAN_LIBS FortranRuntime FortranDecimal)
+  if(CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL "21.1")
+    set(STANDARDFORTRAN_LIBS flang_rt.runtime FortranDecimal)
+  endif()
 elseif(CMAKE_Fortran_COMPILER_ID MATCHES "Flang")
     set(STANDARDFORTRAN_LIBS flang flangrti pgmath)
     #CMAKE_Fortran_COMPILER_ID does not give "ArmFlang"

comex/CMakeLists.txt

@@ -36,8 +36,64 @@ option (COMEX_NETWORK_MPI3 "use MPI RMA protocols for communication" OFF)
 option (COMEX_NETWORK_MPI_MT "use MPI multi-threading protocol for communication" OFF)
 option (COMEX_NETWORK_MPI_PT "use MPI progress threads protocol for communication" OFF)
 
+# Allow selecting an alternate runtime. When GA_RUNTIME is set to OPEN_SHMEM
+# the src-oshmem backend will be used instead of the MPI-based runtime.
+set(GA_RUNTIME "MPI" CACHE STRING "Runtime to build: MPI (default) or OPEN_SHMEM")
+set_property(CACHE GA_RUNTIME PROPERTY STRINGS MPI OPEN_SHMEM)
+
 include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR} src-common)
-if (MPI_TS)
+if (GA_RUNTIME STREQUAL "OPEN_SHMEM")
+  set(COMEX_DEVICE
+    src-oshmem/comex.c
+    src-oshmem/groups.c
+      src-oshmem/nb.c
+      src-oshmem/locks.c
+  )
+  set (COMEX_NETWORK_OSHMEM ON)
+  include_directories(AFTER src-oshmem)
+  # Allow user to point to an OpenSHMEM installation root. If provided,
+  # we'll search under that prefix for headers and libraries.
+  option(OSHMEM_ROOT "Path to OpenSHMEM installation root" "")
+
+  # Find headers (shmem.h or openshmem.h) and library (libshmem / libopenshmem).
+  if (OSHMEM_ROOT)
+    set(_oshmem_hints ${OSHMEM_ROOT}/include ${OSHMEM_ROOT}/usr/include)
+    set(_oshmem_lib_hints ${OSHMEM_ROOT}/lib ${OSHMEM_ROOT}/lib64 ${OSHMEM_ROOT}/usr/lib ${OSHMEM_ROOT}/usr/lib64)
+  else()
+    set(_oshmem_hints /usr/include /usr/local/include)
+    set(_oshmem_lib_hints /usr/lib /usr/lib64 /usr/local/lib /usr/local/lib64 /opt/openmpi/lib)
+  endif()
+
+  find_path(SHMEM_INCLUDE
+    NAMES shmem.h openshmem.h
+    HINTS ${_oshmem_hints}
+  )
+
+  find_library(SHMEM_LIB
+    NAMES oshmem shmem openshmem liboshmem libshmem
+    HINTS ${_oshmem_lib_hints}
+  )
+
+  if (SHMEM_INCLUDE)
+    include_directories(AFTER ${SHMEM_INCLUDE})
+    message(STATUS "OpenSHMEM headers found: ${SHMEM_INCLUDE}")
+  else()
+    message(STATUS "OpenSHMEM headers NOT found; continuing but build may fail until OSHMEM_ROOT is set")
+  endif()
+
+  if (NOT SHMEM_LIB)
+    message(STATUS "OpenSHMEM library not found by CMake; will try linker name 'oshmem' then 'shmem' at link time")
+    # prefer 'oshmem' as the linker name for Open MPI's implementation
+    set(SHMEM_LIB oshmem)
+  else()
+    message(STATUS "OpenSHMEM library found: ${SHMEM_LIB}")
+  endif()
+  # Export the discovered library to the parent scope so top-level targets
+  # (e.g., the 'ga' library) can link against OpenSHMEM.
+  set(SHMEM_LIB ${SHMEM_LIB} CACHE STRING "OpenSHMEM library")
+  # Append to GA_EXTRA_LIBS in the parent scope so 'ga' picks it up
+  set(GA_EXTRA_LIBS ${GA_EXTRA_LIBS} ${SHMEM_LIB} PARENT_SCOPE)
+elseif (MPI_TS)
   set(COMEX_DEVICE
     src-mpi/comex.c
     src-mpi/groups.c
@@ -124,6 +180,10 @@ add_library(armci_comex OBJECT
 )
 target_compile_definitions(armci_comex PRIVATE HAVE_CONFIG_H)
 target_include_directories(armci_comex PRIVATE ${MPI_C_INCLUDE_DIRS})
+if (GA_RUNTIME STREQUAL "OPEN_SHMEM")
+  # Ensure C sources see the OpenSHMEM build mode
+  target_compile_definitions(armci_comex PRIVATE _OPENSHMEM)
+endif()
 
 add_library(armci
   ${ARMCI_FILES}
@@ -134,6 +194,22 @@ target_compile_definitions(armci PRIVATE HAVE_CONFIG_H)
 add_library(GlobalArrays::armci ALIAS armci)
 
 target_include_directories(armci PRIVATE ${MPI_C_INCLUDE_DIRS})
+if (GA_RUNTIME STREQUAL "OPEN_SHMEM")
+  target_compile_definitions(armci PRIVATE _OPENSHMEM)
+endif()
+if (GA_RUNTIME STREQUAL "OPEN_SHMEM")
+  # Link OpenSHMEM into armci & comex libraries
+  target_link_libraries(armci PRIVATE ${SHMEM_LIB})
+endif()
+
+if (GA_RUNTIME STREQUAL "OPEN_SHMEM")
+  # Also link the top-level 'ga' target if it exists so the final
+  # executable link commands get the SHMEM library. If 'ga' is not yet
+  # defined, we already appended SHMEM_LIB to GA_EXTRA_LIBS above.
+  if (TARGET ga)
+    target_link_libraries(ga PRIVATE ${SHMEM_LIB})
+  endif()
+endif()
 
 install(TARGETS armci
     EXPORT globalarrays-targets
@@ -148,6 +224,10 @@ add_library(comex
 add_library(GlobalArrays::comex ALIAS comex)
 
 target_include_directories(comex PRIVATE ${MPI_C_INCLUDE_DIRS})
+if (GA_RUNTIME STREQUAL "OPEN_SHMEM")
+  target_link_libraries(comex PRIVATE ${SHMEM_LIB})
+  target_compile_definitions(comex PRIVATE _OPENSHMEM)
+endif()
 
 install(TARGETS comex
     EXPORT globalarrays-targets

comex/Makefile.am

@@ -35,10 +35,15 @@ AM_LDFLAGS =
 LDADD =
 
 lib_LTLIBRARIES += libcomex.la
-# later Makefile fragments append to this
+if COMEX_NETWORK_OSHMEM
+libcomex_la_SOURCES = src-oshmem/comex.c src-oshmem/groups.c src-oshmem/locks.c src-oshmem/nb.c
+else
 libcomex_la_SOURCES =
+endif
 libcomex_la_LIBADD =
+if !COMEX_NETWORK_OSHMEM
 libcomex_la_LIBADD += $(MPI_LIBS)
+endif
 libcomex_la_LIBADD += $(COMEX_NETWORK_LIBS)
 libcomex_la_LIBADD += $(BLAS_LIBS)
 libcomex_la_LIBADD += $(LIBS)
@@ -47,12 +52,16 @@ libcomex_la_LIBADD += $(LIBADD_DLOPEN)
 endif
 include_HEADERS += $(top_srcdir)/src-common/comex.h
 
+if !COMEX_NETWORK_OSHMEM
 AM_CPPFLAGS += $(MPI_CPPFLAGS)
+endif
 AM_CPPFLAGS += $(COMEX_NETWORK_CPPFLAGS)
 AM_CPPFLAGS += $(BLAS_CPPFLAGS)
 AM_CPPFLAGS += -I$(top_srcdir)/src-common
 
+if !COMEX_NETWORK_OSHMEM
 AM_LDFLAGS += $(MPI_LDFLAGS)
+endif
 AM_LDFLAGS += $(COMEX_NETWORK_LDFLAGS)
 AM_LDFLAGS += $(BLAS_LDFLAGS)
 
@@ -62,7 +71,9 @@ LDADD += libcomex.la
 noinst_LTLIBRARIES += libcomexi.la
 libcomexi_la_SOURCES = $(libcomex_la_SOURCES)
 libcomexi_la_LIBADD =
+if !COMEX_NETWORK_OSHMEM
 libcomexi_la_LIBADD += $(MPI_LIBS)
+endif
 libcomexi_la_LIBADD += $(COMEX_NETWORK_LIBS)
 libcomexi_la_LIBADD += $(BLAS_LIBS)
 libcomexi_la_LIBADD += $(LIBS)
@@ -143,6 +154,13 @@ if COMEX_NETWORK_MPI3
 include $(top_srcdir)/src-mpi3/Makefile.inc
 endif
 
+##############################################################################
+# src-oshmem (OpenSHMEM runtime)
+##############################################################################
+if COMEX_NETWORK_OSHMEM
+include $(top_srcdir)/src-oshmem/Makefile.inc
+endif
+
 ##############################################################################
 # src-ofi
 #
@@ -206,6 +224,8 @@ testing_perf_strided_SOURCES = testing/perf_strided.c
 testing_shift_SOURCES        = testing/shift.c
 testing_test_SOURCES         = testing/test.c
 
+
+
 ##############################################################################
 # the end
 #