diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7e0beeda..99af3309 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -123,7 +123,7 @@ jobs:
         feature: ${{ fromJSON(needs.features-matrix.outputs.features) }}
     with:
       name: "{0}"
-      args: "-f ${{ matrix.feature }} -i ubuntu:22.04 --skip-scenarios"
+      args: "-f ${{ matrix.feature }} -i ubuntu:24.04 --skip-scenarios"
 
   image-matrix:
     name: Determine image matrix
diff --git a/features/src/nvhpc/.bashrc b/features/src/nvhpc/.bashrc
index 75e78075..584bd333 100644
--- a/features/src/nvhpc/.bashrc
+++ b/features/src/nvhpc/.bashrc
@@ -13,3 +13,5 @@ if [ -z "${LIBRARY_PATH:-}" ] \
 || [ -n "${LIBRARY_PATH##*"${NVHPC_CUDA_HOME}/lib64/stubs"*}" ]; then
     export LIBRARY_PATH="${NVHPC_CUDA_HOME}/lib64/stubs${LIBRARY_PATH:+:$LIBRARY_PATH}"
 fi
+
+. /etc/profile.d/load-nvhpc._sh
diff --git a/features/src/nvhpc/devcontainer-feature.json b/features/src/nvhpc/devcontainer-feature.json
index 9fc79b56..97ed553d 100644
--- a/features/src/nvhpc/devcontainer-feature.json
+++ b/features/src/nvhpc/devcontainer-feature.json
@@ -1,7 +1,7 @@
 {
   "name": "NVHPC SDK",
   "id": "nvhpc",
-  "version": "26.4.2",
+  "version": "26.4.3",
   "description": "A feature to install the NVHPC SDK",
   "options": {
     "version": {
diff --git a/features/src/nvhpc/etc/profile.d/nvhpc.sh b/features/src/nvhpc/etc/profile.d/nvhpc.sh
deleted file mode 100755
index 7d7da673..00000000
--- a/features/src/nvhpc/etc/profile.d/nvhpc.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-if ! command -V module 2>&1 | grep -q function; then
-    . /etc/profile.d/lmod._sh;
-fi
-
-if [ -n "${PATH##*"${NVHPC_ROOT}/compilers/bin"*}" ]; then
-    for NVHPC_MODULEFILES_DIR in "${NVHPC_MODULEFILE_DIRS[@]}"; do
-        if [ -n "${MODULEPATH##*"${NVHPC_MODULEFILES_DIR}"*}" ]; then
-            module use -a "${NVHPC_MODULEFILES_DIR}";
-        fi
-    done
-    unset NVHPC_MODULEFILES_DIR;
-    for NVHPC_MODULE_NAME in "nvhpc-hpcx-cuda${NVHPC_CUDA_VERSION_MAJOR}/${NVHPC_VERSION}" \
-                             "nvhpc-hpcx/${NVHPC_VERSION}"; do
-        if ! module list "${NVHPC_MODULE_NAME}" 2>&1 | grep -q 'None found.'; then
-            if ! module list 2>&1 | grep -q "${NVHPC_MODULE_NAME}"; then
-                module try-load "${NVHPC_MODULE_NAME}" >/dev/null 2>&1;
-            fi
-        fi
-    done
-    unset NVHPC_MODULE_NAME;
-
-    # Have to source and manually call hpcx_load for nvhpc>=25.7
-    if [ "${NVHPC_VERSION_MAJOR}" -ge 25 ] \
-    && [ "${NVHPC_VERSION_MINOR}" -ge 5 ]; then
-        HPCX_INIT="$(find -L "$NVHPC_ROOT"/comm_libs/ -path '*/latest/hpcx-init.sh' -print -quit)";
-        if [ -n "${HPCX_INIT:+x}" ] && [ -s "${HPCX_INIT}" ]; then
-            . "$HPCX_INIT";
-            hpcx_load;
-        fi
-        unset HPCX_INIT;
-    fi
-fi
diff --git a/features/src/nvhpc/install.sh b/features/src/nvhpc/install.sh
index 2349d0dc..c2466218 100644
--- a/features/src/nvhpc/install.sh
+++ b/features/src/nvhpc/install.sh
@@ -68,7 +68,7 @@ if ! test -L /usr/local/cuda; then
     ln -s "${NVHPC_CUDA_HOME}" /usr/local/cuda;
 fi
 
-cuda_ver=$(grep "#define CUDA_VERSION" "${NVHPC_CUDA_HOME}/include/cuda.h" | cut -d' ' -f3);
+cuda_ver="$(grep "#define CUDA_VERSION" "${NVHPC_CUDA_HOME}/include/cuda.h" | cut -d' ' -f3)";
 export NVHPC_CUDA_VERSION_MAJOR="$((cuda_ver / 1000))";
 export NVHPC_CUDA_VERSION_MINOR="$((cuda_ver / 10 % 100))";
 export NVHPC_CUDA_VERSION_PATCH="$((cuda_ver % 10))";
@@ -94,11 +94,13 @@ vars_+=('$NVHPC_CUDA_HOME');
 vars_+=('$NVHPC_MODULEFILE_DIRS');
 printf -v vars_ '%s,' "${vars_[@]}";
 
+cp load-nvhpc.sh /etc/profile.d/load-nvhpc._sh
+
 # export envvars in bashrc files
-append_to_etc_bashrc "$(cat <(cat .bashrc | envsubst "${vars_%,}") etc/profile.d/nvhpc.sh)";
-append_to_all_bashrcs "$(cat <(cat .bashrc | envsubst "${vars_%,}") etc/profile.d/nvhpc.sh)";
+append_to_etc_bashrc "$(cat <(cat .bashrc | envsubst "${vars_%,}"))";
+append_to_all_bashrcs "$(cat <(cat .bashrc | envsubst "${vars_%,}"))";
 # export envvars in /etc/profile.d
-add_etc_profile_d_script nvhpc "$(cat <(cat .bashrc | envsubst "${vars_%,}") etc/profile.d/nvhpc.sh)";
+add_etc_profile_d_script nvhpc "$(cat <(cat .bashrc | envsubst "${vars_%,}"))";
 
 # Clean up
 # rm -rf /tmp/*;
diff --git a/features/src/nvhpc/load-nvhpc.sh b/features/src/nvhpc/load-nvhpc.sh
new file mode 100755
index 00000000..bd2bccd2
--- /dev/null
+++ b/features/src/nvhpc/load-nvhpc.sh
@@ -0,0 +1,48 @@
+DEVCONTAINERS_NVHPC_LOADED_BY="${DEVCONTAINERS_NVHPC_LOADED_BY:-}"
+
+if ! test "${DEVCONTAINERS_NVHPC_LOADED_BY:-}" = "$(whoami)"; then
+
+    # "unload" lmod so all the modules are loaded from scratch
+    for __module_unload_func_name in clearMT clearLmod; do
+        if ! command -V $__module_unload_func_name 2>&1 | grep -q function; then
+            . /etc/profile.d/lmod._sh
+        fi
+        $__module_unload_func_name >/dev/null 2>&1
+    done
+    unset __module_unload_func_name
+
+    if ! command -V module 2>&1 | grep -q function; then
+        . /etc/profile.d/lmod._sh;
+    fi
+
+    # Restore MODULEPATH since it is cleared by `clearLmod`
+    for NVHPC_MODULEFILES_DIR in "${NVHPC_MODULEFILE_DIRS[@]}"; do
+        if [ -n "${MODULEPATH##*"${NVHPC_MODULEFILES_DIR}"*}" ]; then
+            module use -a "${NVHPC_MODULEFILES_DIR}";
+        fi
+    done
+    unset NVHPC_MODULEFILES_DIR;
+
+    # Load the NHVPC modules again
+    for NVHPC_MODULE_NAME in "nvhpc-hpcx-cuda${NVHPC_CUDA_VERSION_MAJOR}/${NVHPC_VERSION}" \
+                             "nvhpc-hpcx/${NVHPC_VERSION}"; do
+        if ! module list "${NVHPC_MODULE_NAME}" 2>&1 | grep -q 'None found.'; then
+            if ! module list 2>&1 | grep -q "${NVHPC_MODULE_NAME}"; then
+                module try-load "${NVHPC_MODULE_NAME}" >/dev/null 2>&1;
+            fi
+        fi
+    done
+    unset NVHPC_MODULE_NAME;
+
+    # Have to source and manually call hpcx_load for nvhpc>=25.7
+    HPCX_INIT="$(find -L "$NVHPC_ROOT"/comm_libs/ -path '*/latest/hpcx-init.sh' -print -quit)";
+    if [ -n "${HPCX_INIT:+x}" ] && [ -s "${HPCX_INIT}" ]; then
+        . "$HPCX_INIT";
+        hpcx_load;
+    fi
+    unset HPCX_INIT;
+
+    DEVCONTAINERS_NVHPC_LOADED_BY="$(whoami)"
+fi
+
+export DEVCONTAINERS_NVHPC_LOADED_BY