From 9ca90fa6ea7047060a190e55384aeef07e180789 Mon Sep 17 00:00:00 2001 From: "DUPRAT, JULIEN" Date: Wed, 22 Oct 2025 14:47:59 +0200 Subject: [PATCH 1/3] [UBCL] No bxi endpoints if we don't need them Signed-off-by: Brelle Emmanuel --- ompi/mca/pml/ubcl/pml_ubcl_endpoint.c | 55 ++++++++++++++------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/ompi/mca/pml/ubcl/pml_ubcl_endpoint.c b/ompi/mca/pml/ubcl/pml_ubcl_endpoint.c index 8f17e8f3bea..509ad505011 100644 --- a/ompi/mca/pml/ubcl/pml_ubcl_endpoint.c +++ b/ompi/mca/pml/ubcl/pml_ubcl_endpoint.c @@ -111,7 +111,7 @@ static int mca_pml_ubcl_export_local_endpoint_handle(const int type) err = ubcl_export_local_endpoint_handle(type, endpoint_h, &remote_rank_u64); if (UBCL_SUCCESS != err) { - return OMPI_ERROR; + return ubcl_error_to_ompi(err); } mca_pml_ubcl_endpoint_modex_put(type, (void *) endpoint_h, size); @@ -120,10 +120,10 @@ static int mca_pml_ubcl_export_local_endpoint_handle(const int type) * The actual recv rank will be allocated during add_procs calls */ err = ubcl_close_local_endpoint_channel(type, remote_rank_u64); if (UBCL_SUCCESS != err) { - mca_pml_ubcl_warn(OMPI_ERROR, + mca_pml_ubcl_warn(ubcl_error_to_ompi(err), "PML/UBCL failed to clean local endpoint (very unlikely error)." " For safety reason PML will be disabled."); - return OMPI_ERROR; + return ubcl_error_to_ompi(err); } return OMPI_SUCCESS; @@ -133,35 +133,31 @@ int mca_pml_ubcl_create_local_endpoint(void) { int type; ubcl_error_t err; - int ompi_error; type = UBCL_ENDPOINT_TYPE_SELF; err = ubcl_create_local_endpoint(type); if (UBCL_SUCCESS != err) { - mca_pml_ubcl_error(OMPI_ERROR, "Failed ubcl_create_local_endpoint %d (%d)", type, err); + mca_pml_ubcl_warn(ubcl_error_to_ompi(err), "Failed ubcl_create_local_endpoint %d (%d)", type, err); } - /* UBCL_ENDPOINT_SHM */ if (!mca_pml_ubcl_component.force_intranode_bxi) { type = UBCL_ENDPOINT_TYPE_SHMEM; err = ubcl_create_local_endpoint(type); - if (UBCL_SUCCESS != err) { - mca_pml_ubcl_error(OMPI_ERROR, "Failed ubcl_create_local_endpoint %d (%d)", type, err); + if (UBCL_SUCCESS == err) { + err = mca_pml_ubcl_export_local_endpoint_handle(type); } - ompi_error = mca_pml_ubcl_export_local_endpoint_handle(type); - if (OMPI_SUCCESS != ompi_error) { - return ompi_error; + if (UBCL_SUCCESS != err) { + mca_pml_ubcl_warn(ubcl_error_to_ompi(err), "Failed ubcl_create_local_endpoint %d (%d)", type, err); } } type = UBCL_ENDPOINT_TYPE_BXI; err = ubcl_create_local_endpoint(type); - if (UBCL_SUCCESS != err) { - mca_pml_ubcl_error(OMPI_ERROR, "Failed ubcl_create_local_endpoint %d (%d)", type, err); + if (UBCL_SUCCESS == err) { + err = mca_pml_ubcl_export_local_endpoint_handle(type); } - ompi_error = mca_pml_ubcl_export_local_endpoint_handle(type); - if (OMPI_SUCCESS != ompi_error) { - return ompi_error; + if (UBCL_SUCCESS != err) { + mca_pml_ubcl_warn(ubcl_error_to_ompi(err), "Failed ubcl_create_local_endpoint %d (%d)", type, err); } return OMPI_SUCCESS; @@ -170,20 +166,23 @@ int mca_pml_ubcl_create_local_endpoint(void) int mca_pml_ubcl_free_local_endpoints() { int ret; - /* Finalize BXI */ ret = ubcl_free_local_endpoint(UBCL_ENDPOINT_TYPE_BXI); - if (UBCL_SUCCESS != ret) { - return OMPI_ERROR; + if (UBCL_SUCCESS != ret && UBCL_ERR_NOT_AVAILABLE != ret) { + /* If the transport was unavailable we silence the error, + * we're closing it anyway */ + return ubcl_error_to_ompi(ret); } + if (!mca_pml_ubcl_component.force_intranode_bxi) { ret = ubcl_free_local_endpoint(UBCL_ENDPOINT_TYPE_SHMEM); - if (UBCL_SUCCESS != ret) { - return OMPI_ERROR; + if (UBCL_SUCCESS != ret && UBCL_ERR_NOT_AVAILABLE != ret) { + return ubcl_error_to_ompi(ret); } } + ret = ubcl_free_local_endpoint(UBCL_ENDPOINT_TYPE_SELF); - if (UBCL_SUCCESS != ret) { - return OMPI_ERROR; + if (UBCL_SUCCESS != ret && UBCL_ERR_NOT_AVAILABLE != ret) { + return ubcl_error_to_ompi(ret); } return OMPI_SUCCESS; @@ -331,14 +330,16 @@ static int mca_pml_ubcl_create_endpoints(ompi_proc_t *proc) err = mca_pml_ubcl_create_recv_endpoint(new_endpoint->rank, new_endpoint->type); if (OMPI_SUCCESS != err) { - mca_pml_ubcl_error(err, "Failed to create recv endpoint for rank %zu\n", - new_endpoint->rank); + mca_pml_ubcl_warn(err, "Failed to create recv endpoint for rank %zu\n", + new_endpoint->rank); + return err; } err = mca_pml_ubcl_create_send_endpoint(proc, new_endpoint->rank, new_endpoint->type); if (OMPI_SUCCESS != err) { - mca_pml_ubcl_error(err, "Failed to create send endpoint for rank %zu\n", - new_endpoint->rank); + mca_pml_ubcl_warn(err, "Failed to create send endpoint for rank %zu\n", + new_endpoint->rank); + return err; } end: From 959dc301d77e426a9a5b2f6f8d1c4e715cb6b19b Mon Sep 17 00:00:00 2001 From: "DUPRAT, JULIEN" Date: Mon, 3 Nov 2025 12:05:28 +0100 Subject: [PATCH 2/3] [PML/UBCL] add_procs fallbacks on higher transport when required Signed-off-by: Brelle Emmanuel --- ompi/mca/pml/ubcl/pml_ubcl_endpoint.c | 71 +++++++++++++++++++++------ 1 file changed, 57 insertions(+), 14 deletions(-) diff --git a/ompi/mca/pml/ubcl/pml_ubcl_endpoint.c b/ompi/mca/pml/ubcl/pml_ubcl_endpoint.c index 509ad505011..400384696e6 100644 --- a/ompi/mca/pml/ubcl/pml_ubcl_endpoint.c +++ b/ompi/mca/pml/ubcl/pml_ubcl_endpoint.c @@ -254,7 +254,7 @@ static int mca_pml_ubcl_create_recv_endpoint(uint64_t sender_rank, const int typ err = ubcl_export_local_endpoint_handle(type, endpoint_h, &remote_rank_u64); if (UBCL_SUCCESS != err) { - return OMPI_ERROR; + return ubcl_error_to_ompi(err); } return OMPI_SUCCESS; @@ -269,11 +269,11 @@ static int mca_pml_ubcl_create_self_endpoints(uint64_t remote_rank) err = ubcl_export_local_endpoint_handle(type, endpoint_h, &my_rank); if (UBCL_SUCCESS != err) { - return OMPI_ERROR; + return ubcl_error_to_ompi(err); } err = ubcl_create_remote_endpoint(my_rank, my_rank, type, endpoint_h); if (UBCL_SUCCESS != err) { - return OMPI_ERROR; + return ubcl_error_to_ompi(err); } return OMPI_SUCCESS; @@ -295,6 +295,25 @@ static int get_endpoint_type(ompi_proc_t *proc) } } +static enum ubcl_endpoint_type_t mca_pml_ubcl_get_higher_transport( + enum ubcl_endpoint_type_t type) +{ + switch ((int) type) { + case UBCL_ENDPOINT_TYPE_SELF: + case UBCL_ENDPOINT_TYPE_SHMEM: + type++; + break; + /* There are no valid higher transport */ + case UBCL_ENDPOINT_TYPE_BXI: + default: + type = UBCL_ENDPOINT_TYPE_NONE; + /* Not a valid transport */ + break; + } + + return type; +} + void mca_pml_ubcl_endpoint_retain(ompi_proc_t *proc) { mca_common_ubcl_endpoint_t *endpoint = NULL; @@ -311,6 +330,7 @@ void mca_pml_ubcl_endpoint_retain(ompi_proc_t *proc) static int mca_pml_ubcl_create_endpoints(ompi_proc_t *proc) { int err = OMPI_SUCCESS; + enum ubcl_endpoint_type_t type; mca_common_ubcl_endpoint_t *new_endpoint; new_endpoint = malloc(sizeof(mca_common_ubcl_endpoint_t)); @@ -321,21 +341,43 @@ static int mca_pml_ubcl_create_endpoints(ompi_proc_t *proc) new_endpoint->refcount = 0; //we increment it to 1 in endpoint_retain new_endpoint->rank = mca_pml_forge_rank(proc); - new_endpoint->type = get_endpoint_type(proc); + type = get_endpoint_type(proc); - if (UBCL_ENDPOINT_TYPE_SELF == new_endpoint->type) { + if (UBCL_ENDPOINT_TYPE_SELF == type) { err = mca_pml_ubcl_create_self_endpoints((uint64_t) new_endpoint->rank); - goto end; - } - err = mca_pml_ubcl_create_recv_endpoint(new_endpoint->rank, new_endpoint->type); - if (OMPI_SUCCESS != err) { - mca_pml_ubcl_warn(err, "Failed to create recv endpoint for rank %zu\n", - new_endpoint->rank); - return err; + /* If the transport is unvailable (either explicitely disabled, + * or just unavailable) we do not return any error + * If UBCL encountered another error we return it */ + if (OMPI_SUCCESS == err) { + goto end; + } else if (OMPI_ERR_NOT_AVAILABLE != err) { + return err; + } } - err = mca_pml_ubcl_create_send_endpoint(proc, new_endpoint->rank, new_endpoint->type); + /* If a transport is unavailable only a higher transport can take its place, + * ie. if SHM is unavailable, SELF cannot replace it but BXI can */ + do { + err = mca_pml_ubcl_create_recv_endpoint(new_endpoint->rank, type); + + if (OMPI_ERR_NOT_AVAILABLE == err) { + type = mca_pml_ubcl_get_higher_transport(type); + if (UBCL_ENDPOINT_TYPE_NONE == type) { + mca_pml_ubcl_warn(err, "Failed to create recv endpoint for rank %zu\n", + new_endpoint->rank); + return err; + } + } else if (OMPI_SUCCESS != err) { + mca_pml_ubcl_warn(err, "Failed to create recv endpoint for rank %zu\n", + new_endpoint->rank); + return err; + } + } while (OMPI_SUCCESS != err); + + /* No need to loop again, if the transport became unavailable between + * the last operation and this one we can consider this a error */ + err = mca_pml_ubcl_create_send_endpoint(proc, new_endpoint->rank, type); if (OMPI_SUCCESS != err) { mca_pml_ubcl_warn(err, "Failed to create send endpoint for rank %zu\n", new_endpoint->rank); @@ -343,10 +385,11 @@ static int mca_pml_ubcl_create_endpoints(ompi_proc_t *proc) } end: + new_endpoint->type = type; (proc)->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PML] = new_endpoint; mca_pml_ubcl_endpoint_retain(proc); - return err; + return UBCL_SUCCESS; } int mca_pml_ubcl_add_procs(ompi_proc_t **procs, size_t nprocs) From 90532de15fa149801bf8b88aa9f4071a6b404f94 Mon Sep 17 00:00:00 2001 From: "DUPRAT, JULIEN" Date: Mon, 24 Nov 2025 14:37:15 +0100 Subject: [PATCH 3/3] [PML/UBCL] free endpoints when transport init failed Signed-off-by: Brelle Emmanuel --- ompi/mca/pml/ubcl/pml_ubcl_endpoint.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ompi/mca/pml/ubcl/pml_ubcl_endpoint.c b/ompi/mca/pml/ubcl/pml_ubcl_endpoint.c index 400384696e6..8754aabebc6 100644 --- a/ompi/mca/pml/ubcl/pml_ubcl_endpoint.c +++ b/ompi/mca/pml/ubcl/pml_ubcl_endpoint.c @@ -352,7 +352,7 @@ static int mca_pml_ubcl_create_endpoints(ompi_proc_t *proc) if (OMPI_SUCCESS == err) { goto end; } else if (OMPI_ERR_NOT_AVAILABLE != err) { - return err; + goto error; } } @@ -366,12 +366,12 @@ static int mca_pml_ubcl_create_endpoints(ompi_proc_t *proc) if (UBCL_ENDPOINT_TYPE_NONE == type) { mca_pml_ubcl_warn(err, "Failed to create recv endpoint for rank %zu\n", new_endpoint->rank); - return err; + goto error; } } else if (OMPI_SUCCESS != err) { mca_pml_ubcl_warn(err, "Failed to create recv endpoint for rank %zu\n", new_endpoint->rank); - return err; + goto error; } } while (OMPI_SUCCESS != err); @@ -381,7 +381,7 @@ static int mca_pml_ubcl_create_endpoints(ompi_proc_t *proc) if (OMPI_SUCCESS != err) { mca_pml_ubcl_warn(err, "Failed to create send endpoint for rank %zu\n", new_endpoint->rank); - return err; + goto error; } end: @@ -390,6 +390,10 @@ static int mca_pml_ubcl_create_endpoints(ompi_proc_t *proc) mca_pml_ubcl_endpoint_retain(proc); return UBCL_SUCCESS; + +error: + free(new_endpoint); + return err; } int mca_pml_ubcl_add_procs(ompi_proc_t **procs, size_t nprocs)