Integration-test hardening + bootstrap race tolerance + XML docs

src/Hyperbee.Migrations.Providers.OpenSearch/Internal/Bootstrap/Steps/LedgerIndexInitStep.cs

@@ -84,14 +84,35 @@ public async Task<StepOutcome> ExecuteAsync( BootstrapContext context )
 
             logger.LogInformation( "{step} creating ledger index `{idx}` with strict mapping", Name, indexName );
 
-            var createResponse = await context.Client.LowLevel.Indices.CreateAsync<StringResponse>(
-                indexName,
-                PostData.String( DefaultMappingJson ),
-                ctx: context.CancellationToken
-            ).ConfigureAwait( false );
+            StringResponse createResponse;
+            try
+            {
+                createResponse = await context.Client.LowLevel.Indices.CreateAsync<StringResponse>(
+                    indexName,
+                    PostData.String( DefaultMappingJson ),
+                    ctx: context.CancellationToken
+                ).ConfigureAwait( false );
+            }
+            catch ( OpenSearchClientException ex ) when ( IsResourceAlreadyExists( ex.Response ) )
+            {
+                // TOCTOU race: another runner created the index between our Exists()
+                // check and Create(). Verify the mapping and treat as success.
+                logger.LogDebug( "{step} ledger index `{idx}` created concurrently by another runner; verifying mapping", Name, indexName );
+                var verifyDetail = await VerifyMappingAsync( context, indexName, logger ).ConfigureAwait( false );
+                var raceElapsed = context.TimeProvider.GetElapsedTime( start );
+                return StepOutcome.Succeeded( Name, raceElapsed, $"{verifyDetail} (raced)" );
+            }
 
             if ( !createResponse.Success )
             {
+                if ( IsResourceAlreadyExists( createResponse ) )
+                {
+                    logger.LogDebug( "{step} ledger index `{idx}` created concurrently by another runner; verifying mapping", Name, indexName );
+                    var verifyDetail = await VerifyMappingAsync( context, indexName, logger ).ConfigureAwait( false );
+                    var raceElapsed = context.TimeProvider.GetElapsedTime( start );
+                    return StepOutcome.Succeeded( Name, raceElapsed, $"{verifyDetail} (raced)" );
+                }
+
                 var detail = createResponse.OriginalException?.Message ?? createResponse.Body ?? "Unknown create failure";
                 var ex = new OpenSearchProviderException(
                     $"{Name} could not create ledger index `{indexName}`. {detail}",
@@ -155,4 +176,29 @@ private static async Task<string> VerifyMappingAsync( BootstrapContext context,
         logger.LogDebug( "{step} ledger schema verified ({count} required fields present)", "ledger-init", RequiredFields.Length );
         return "verified existing schema";
     }
+
+    // Detects the OpenSearch-specific 400 body that signals a TOCTOU race
+    // between Exists() and Create() — another runner won. Inspect the body
+    // string rather than the status code alone because OS reuses 400 for
+    // genuine bad-request shapes (malformed mapping, invalid settings).
+    private static bool IsResourceAlreadyExists( IApiCallDetails? response )
+    {
+        if ( response is null || response.HttpStatusCode != 400 )
+            return false;
+
+        var body = response.ResponseBodyInBytes is { Length: > 0 } bytes
+            ? System.Text.Encoding.UTF8.GetString( bytes )
+            : null;
+
+        return body is not null && body.Contains( "resource_already_exists_exception", StringComparison.Ordinal );
+    }
+
+    private static bool IsResourceAlreadyExists( StringResponse response )
+    {
+        if ( response.HttpStatusCode != 400 )
+            return false;
+
+        return !string.IsNullOrEmpty( response.Body )
+            && response.Body.Contains( "resource_already_exists_exception", StringComparison.Ordinal );
+    }
 }

src/Hyperbee.Migrations.Providers.OpenSearch/Internal/Bootstrap/Steps/LockIndexInitStep.cs

@@ -67,14 +67,33 @@ public async Task<StepOutcome> ExecuteAsync( BootstrapContext context )
 
             logger.LogInformation( "{step} creating lock index `{idx}` (replicas=0)", Name, indexName );
 
-            var createResponse = await context.Client.LowLevel.Indices.CreateAsync<StringResponse>(
-                indexName,
-                PostData.String( DefaultMappingJson ),
-                ctx: context.CancellationToken
-            ).ConfigureAwait( false );
+            StringResponse createResponse;
+            try
+            {
+                createResponse = await context.Client.LowLevel.Indices.CreateAsync<StringResponse>(
+                    indexName,
+                    PostData.String( DefaultMappingJson ),
+                    ctx: context.CancellationToken
+                ).ConfigureAwait( false );
+            }
+            catch ( OpenSearchClientException ex ) when ( IsResourceAlreadyExists( ex.Response ) )
+            {
+                // TOCTOU race: another runner created the lock index between our
+                // Exists() check and Create(). Treat as success.
+                logger.LogDebug( "{step} lock index `{idx}` created concurrently by another runner", Name, indexName );
+                var raceElapsed = context.TimeProvider.GetElapsedTime( start );
+                return StepOutcome.Succeeded( Name, raceElapsed, "exists (raced)" );
+            }
 
             if ( !createResponse.Success )
             {
+                if ( IsResourceAlreadyExists( createResponse ) )
+                {
+                    logger.LogDebug( "{step} lock index `{idx}` created concurrently by another runner", Name, indexName );
+                    var raceElapsed = context.TimeProvider.GetElapsedTime( start );
+                    return StepOutcome.Succeeded( Name, raceElapsed, "exists (raced)" );
+                }
+
                 var detail = createResponse.OriginalException?.Message ?? createResponse.Body ?? "Unknown create failure";
                 var ex = new OpenSearchProviderException(
                     $"{Name} could not create lock index `{indexName}`. {detail}",
@@ -97,4 +116,29 @@ public async Task<StepOutcome> ExecuteAsync( BootstrapContext context )
                 $"{Name} threw an unexpected exception. {ex.Message}", ex ) );
         }
     }
+
+    // Detects the OpenSearch-specific 400 body that signals a TOCTOU race
+    // between Exists() and Create() — another runner won. Inspect the body
+    // string rather than the status code alone because OS reuses 400 for
+    // genuine bad-request shapes (malformed mapping, invalid settings).
+    private static bool IsResourceAlreadyExists( IApiCallDetails? response )
+    {
+        if ( response is null || response.HttpStatusCode != 400 )
+            return false;
+
+        var body = response.ResponseBodyInBytes is { Length: > 0 } bytes
+            ? System.Text.Encoding.UTF8.GetString( bytes )
+            : null;
+
+        return body is not null && body.Contains( "resource_already_exists_exception", StringComparison.Ordinal );
+    }
+
+    private static bool IsResourceAlreadyExists( StringResponse response )
+    {
+        if ( response.HttpStatusCode != 400 )
+            return false;
+
+        return !string.IsNullOrEmpty( response.Body )
+            && response.Body.Contains( "resource_already_exists_exception", StringComparison.Ordinal );
+    }
 }