[codex] Add Firebase E2E runtime drift all mode

tests/E2E/Firebase.Foundation/FirebaseFoundationE2E/FirebaseFoundationE2E.csproj

@@ -17,6 +17,7 @@
     <EnableNullabilityValidation>false</EnableNullabilityValidation>
     <RuntimeDriftCase></RuntimeDriftCase>
     <RuntimeDriftCaseMethod></RuntimeDriftCaseMethod>
+    <RuntimeDriftCases></RuntimeDriftCases>
     <RuntimeDriftCasePropsPath></RuntimeDriftCasePropsPath>
     <BindingSurfaceCoverageTarget></BindingSurfaceCoverageTarget>
     <BindingSurfaceCoveragePropsPath></BindingSurfaceCoveragePropsPath>
@@ -65,6 +66,10 @@
       <_Parameter1>RuntimeDriftCaseMethod</_Parameter1>
       <_Parameter2>$(RuntimeDriftCaseMethod)</_Parameter2>
     </AssemblyAttribute>
+    <AssemblyAttribute Include="System.Reflection.AssemblyMetadataAttribute" Condition="'$(RuntimeDriftCases)' != ''">
+      <_Parameter1>RuntimeDriftCases</_Parameter1>
+      <_Parameter2>$(RuntimeDriftCases)</_Parameter2>
+    </AssemblyAttribute>
   </ItemGroup>
 
   <ItemGroup Condition="'$(BindingSurfaceCoverageTarget)' != ''">

tests/E2E/Firebase.Foundation/FirebaseFoundationE2E/FirebaseRuntimeDriftCases.cs

@@ -1,119 +1,43 @@
 using System.Reflection;
-
-#if ENABLE_RUNTIME_DRIFT_CASE_CORE_CONFIGURATION_LOGGERLEVEL
-using Firebase.Core;
 using Foundation;
 using ObjCRuntime;
-#endif
 
-#if ENABLE_RUNTIME_DRIFT_CASE_ANALYTICS_SESSIONIDWITHCOMPLETION
-using Firebase.Analytics;
-using Foundation;
-using ObjCRuntime;
+#if ENABLE_RUNTIME_DRIFT_CASE_CORE_CONFIGURATION_LOGGERLEVEL
+using Firebase.Core;
 #endif
 
-#if ENABLE_RUNTIME_DRIFT_CASE_ANALYTICS_ONDEVICECONVERSION
+#if ENABLE_RUNTIME_DRIFT_CASE_ANALYTICS_SESSIONIDWITHCOMPLETION || ENABLE_RUNTIME_DRIFT_CASE_ANALYTICS_ONDEVICECONVERSION
 using Firebase.Analytics;
-using Foundation;
-using ObjCRuntime;
 #endif
 
 #if ENABLE_RUNTIME_DRIFT_CASE_APPCHECK_LIMITED_USE_TOKENS
 using Firebase.AppCheck;
 using FirebaseCoreApp = Firebase.Core.App;
-using Foundation;
-using ObjCRuntime;
 #endif
 
 #if ENABLE_RUNTIME_DRIFT_CASE_REMOTECONFIG_REALTIME_CUSTOMSIGNALS
 using Firebase.RemoteConfig;
-using Foundation;
-using ObjCRuntime;
 #endif
 
 #if ENABLE_RUNTIME_DRIFT_CASE_DATABASE_SERVERVALUE_INCREMENT || ENABLE_RUNTIME_DRIFT_CASE_DATABASE_QUERY_GETDATA
 using Firebase.Database;
 using FirebaseCoreOptions = Firebase.Core.Options;
-using Foundation;
-using ObjCRuntime;
-#endif
-
-#if ENABLE_RUNTIME_DRIFT_CASE_ABTESTING_UPDATEEXPERIMENTS
-using Firebase.ABTesting;
-using Foundation;
-using ObjCRuntime;
-#endif
-
-#if ENABLE_RUNTIME_DRIFT_CASE_ABTESTING_ACTIVATEEXPERIMENT
-using Firebase.ABTesting;
-using Foundation;
-using ObjCRuntime;
 #endif
 
-#if ENABLE_RUNTIME_DRIFT_CASE_ABTESTING_VALIDATERUNNINGEXPERIMENTS
+#if ENABLE_RUNTIME_DRIFT_CASE_ABTESTING_UPDATEEXPERIMENTS || ENABLE_RUNTIME_DRIFT_CASE_ABTESTING_ACTIVATEEXPERIMENT || ENABLE_RUNTIME_DRIFT_CASE_ABTESTING_VALIDATERUNNINGEXPERIMENTS
 using Firebase.ABTesting;
-using Foundation;
-using ObjCRuntime;
-#endif
-
-#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_GETQUERYNAMED
-using Firebase.CloudFirestore;
-using Foundation;
-using ObjCRuntime;
-#endif
-
-#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_FIELDVALUE_VECTORWITHARRAY
-using Firebase.CloudFirestore;
-using Foundation;
-using ObjCRuntime;
-#endif
-
-#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_AGGREGATE_QUERY
-using Firebase.CloudFirestore;
-using Foundation;
-using ObjCRuntime;
-#endif
-
-#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_QUERY_FILTERS
-using Firebase.CloudFirestore;
-using Foundation;
-using ObjCRuntime;
-#endif
-
-#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_SNAPSHOT_LISTEN_OPTIONS
-using Firebase.CloudFirestore;
-using Foundation;
-using ObjCRuntime;
-#endif
-
-#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_NAMED_DATABASE
-using Firebase.CloudFirestore;
-using Foundation;
-using ObjCRuntime;
 #endif
 
-#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_CACHE_SETTINGS
-using Firebase.CloudFirestore;
-using Foundation;
-using ObjCRuntime;
-#endif
-
-#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_INDEX_CONFIGURATION
+#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_GETQUERYNAMED || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_FIELDVALUE_VECTORWITHARRAY || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_AGGREGATE_QUERY || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_QUERY_FILTERS || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_SNAPSHOT_LISTEN_OPTIONS || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_NAMED_DATABASE || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_CACHE_SETTINGS || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_INDEX_CONFIGURATION
 using Firebase.CloudFirestore;
-using Foundation;
-using ObjCRuntime;
 #endif
 
 #if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFUNCTIONS_USEFUNCTIONSEMULATORORIGIN
 using Firebase.CloudFunctions;
-using Foundation;
-using ObjCRuntime;
 #endif
 
 #if ENABLE_RUNTIME_DRIFT_CASE_CRASHLYTICS_STACKFRAMEWITHADDRESS || ENABLE_RUNTIME_DRIFT_CASE_CRASHLYTICS_RECORD_ERROR_USER_INFO
 using Firebase.Crashlytics;
-using Foundation;
-using ObjCRuntime;
 #endif
 
 namespace FirebaseFoundationE2E;
@@ -122,17 +46,28 @@ static partial class FirebaseRuntimeDriftCases
 {
     static readonly TimeSpan AsyncTimeout = TimeSpan.FromSeconds(5);
 
+    public sealed record RuntimeDriftCase(string Id, string MethodName);
+
     public static string? GetConfiguredCaseId()
     {
         return GetAssemblyMetadataValue("RuntimeDriftCase");
     }
 
-    public static async Task<string> ExecuteConfiguredCaseAsync()
+    public static IReadOnlyList<RuntimeDriftCase> GetConfiguredCases()
     {
+        var configuredCases = GetAssemblyMetadataValue("RuntimeDriftCases");
+        if (!string.IsNullOrWhiteSpace(configuredCases))
+        {
+            return configuredCases
+                .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
+                .Select(ParseConfiguredCase)
+                .ToArray();
+        }
+
         var caseId = GetConfiguredCaseId();
         if (string.IsNullOrWhiteSpace(caseId))
         {
-            throw new InvalidOperationException("Runtime drift mode was requested without a RuntimeDriftCase value.");
+            return Array.Empty<RuntimeDriftCase>();
         }
 
         var methodName = GetAssemblyMetadataValue("RuntimeDriftCaseMethod");
@@ -141,20 +76,54 @@ public static async Task<string> ExecuteConfiguredCaseAsync()
             throw new InvalidOperationException($"Runtime drift case '{caseId}' is missing RuntimeDriftCaseMethod metadata.");
         }
 
-        var method = typeof(FirebaseRuntimeDriftCases).GetMethod(methodName, BindingFlags.Static | BindingFlags.NonPublic);
+        return new[] { new RuntimeDriftCase(caseId, methodName) };
+    }
+
+    public static async Task<string> ExecuteConfiguredCaseAsync()
+    {
+        var caseId = GetConfiguredCaseId();
+        if (string.IsNullOrWhiteSpace(caseId))
+        {
+            throw new InvalidOperationException("Runtime drift mode was requested without a RuntimeDriftCase value.");
+        }
+
+        var methodName = GetAssemblyMetadataValue("RuntimeDriftCaseMethod");
+        var configuredCase = new RuntimeDriftCase(caseId, methodName ?? string.Empty);
+        return await ExecuteConfiguredCaseAsync(configuredCase);
+    }
+
+    public static async Task<string> ExecuteConfiguredCaseAsync(RuntimeDriftCase configuredCase)
+    {
+        if (string.IsNullOrWhiteSpace(configuredCase.MethodName))
+        {
+            throw new InvalidOperationException($"Runtime drift case '{configuredCase.Id}' is missing RuntimeDriftCaseMethod metadata.");
+        }
+
+        var method = typeof(FirebaseRuntimeDriftCases).GetMethod(configuredCase.MethodName, BindingFlags.Static | BindingFlags.NonPublic);
         if (method is null)
         {
-            throw new InvalidOperationException($"Runtime drift case '{caseId}' points at missing method '{methodName}'.");
+            throw new InvalidOperationException($"Runtime drift case '{configuredCase.Id}' points at missing method '{configuredCase.MethodName}'.");
         }
 
         if (method.Invoke(null, null) is not Task<string> task)
         {
-            throw new InvalidOperationException($"Runtime drift case '{caseId}' method '{methodName}' did not return Task<string>.");
+            throw new InvalidOperationException($"Runtime drift case '{configuredCase.Id}' method '{configuredCase.MethodName}' did not return Task<string>.");
         }
 
         return await task;
     }
 
+    static RuntimeDriftCase ParseConfiguredCase(string value)
+    {
+        var separatorIndex = value.IndexOf('=');
+        if (separatorIndex <= 0 || separatorIndex == value.Length - 1)
+        {
+            throw new InvalidOperationException($"Runtime drift case metadata entry '{value}' must use '<id>=<method>' format.");
+        }
+
+        return new RuntimeDriftCase(value[..separatorIndex], value[(separatorIndex + 1)..]);
+    }
+
     static string? GetAssemblyMetadataValue(string key)
     {
         return typeof(FirebaseRuntimeDriftCases)

tests/E2E/Firebase.Foundation/FirebaseFoundationE2E/FirebaseSelfTestRunner.cs

@@ -29,10 +29,14 @@ public static async Task RunAsync(StatusViewController statusViewController)
 #if ENABLE_BINDING_SURFACE_COVERAGE
         await statusViewController.AppendLineAsync("Firebase binding surface coverage mode enabled.");
 #endif
-        var runtimeDriftCase = FirebaseRuntimeDriftCases.GetConfiguredCaseId();
-        if (!string.IsNullOrWhiteSpace(runtimeDriftCase))
+        var runtimeDriftCases = FirebaseRuntimeDriftCases.GetConfiguredCases();
+        if (runtimeDriftCases.Count > 0)
         {
-            await statusViewController.AppendLineAsync($"Runtime drift case mode enabled: {runtimeDriftCase}");
+            var configuredCaseId = FirebaseRuntimeDriftCases.GetConfiguredCaseId();
+            var caseDisplay = string.Equals(configuredCaseId, "all", StringComparison.OrdinalIgnoreCase)
+                ? $"all ({runtimeDriftCases.Count} cases)"
+                : runtimeDriftCases[0].Id;
+            await statusViewController.AppendLineAsync($"Runtime drift case mode enabled: {caseDisplay}");
         }
 
         try
@@ -52,10 +56,13 @@ await ExecuteCaseAsync(result, statusViewController, "ConfigureApp", async () =>
                 return $"Configured app '{defaultApp.Name}'.";
             });
 
-            if (!string.IsNullOrWhiteSpace(runtimeDriftCase))
+            if (runtimeDriftCases.Count > 0)
             {
-                await ExecuteCaseAsync(result, statusViewController, $"RuntimeDrift:{runtimeDriftCase}", () =>
-                    FirebaseRuntimeDriftCases.ExecuteConfiguredCaseAsync());
+                foreach (var runtimeDriftCase in runtimeDriftCases)
+                {
+                    await ExecuteCaseAsync(result, statusViewController, $"RuntimeDrift:{runtimeDriftCase.Id}", () =>
+                        FirebaseRuntimeDriftCases.ExecuteConfiguredCaseAsync(runtimeDriftCase));
+                }
             }
 #if ENABLE_BINDING_SURFACE_COVERAGE
             else if (!string.IsNullOrWhiteSpace(FirebaseBindingSurfaceCoverage.GetConfiguredTarget()))

tests/E2E/Firebase.Foundation/README.md

@@ -81,16 +81,17 @@ dotnet tool run dotnet-cake -- --target=nuget --names="Firebase.Analytics,Fireba
 tools/e2e/run-firebase-foundation.sh --package-dir output --configuration Debug --enable-nullability-validation
 ```
 
-## Targeted runtime drift mode
+## Runtime drift mode
 
-The harness also supports a targeted runtime-drift lane for one binding drift at a time. This mode runs only `ConfigureApp` plus the selected drift case, so each remediation PR can prove the unfixed runtime failure locally, then keep only the success-oriented regression test in the final diff.
+The harness also supports a runtime-drift lane for binding-layer checks that need simulator execution. This mode runs only `ConfigureApp` plus the selected drift case, so each remediation PR can prove the unfixed runtime failure locally, then keep only the success-oriented regression test in the final diff. Use `all` to build once and run every checked-in drift case in the same app launch.
 
 The checked-in case manifest lives at [`runtime-drift-cases.json`](./runtime-drift-cases.json), and the backlog/queue is tracked in [`docs/firebase-runtime-failure-backlog.md`](../../docs/firebase-runtime-failure-backlog.md).
 
 Run a specific drift case with:
 
 ```sh
 tools/e2e/run-firebase-foundation.sh --package-dir output --configuration Debug --runtime-drift-case cloudfirestore-getquerynamed
+tools/e2e/run-firebase-foundation.sh --package-dir output --configuration Debug --runtime-drift-case all
 ```
 
 ## Binding surface coverage mode