diff --git a/tests/E2E/Firebase.Foundation/FirebaseFoundationE2E/FirebaseFoundationE2E.csproj b/tests/E2E/Firebase.Foundation/FirebaseFoundationE2E/FirebaseFoundationE2E.csproj index aea2a8a7..ed580a6e 100644 --- a/tests/E2E/Firebase.Foundation/FirebaseFoundationE2E/FirebaseFoundationE2E.csproj +++ b/tests/E2E/Firebase.Foundation/FirebaseFoundationE2E/FirebaseFoundationE2E.csproj @@ -17,6 +17,7 @@ false + @@ -65,6 +66,10 @@ <_Parameter1>RuntimeDriftCaseMethod <_Parameter2>$(RuntimeDriftCaseMethod) + + <_Parameter1>RuntimeDriftCases + <_Parameter2>$(RuntimeDriftCases) + diff --git a/tests/E2E/Firebase.Foundation/FirebaseFoundationE2E/FirebaseRuntimeDriftCases.cs b/tests/E2E/Firebase.Foundation/FirebaseFoundationE2E/FirebaseRuntimeDriftCases.cs index 259a436e..f4537b42 100644 --- a/tests/E2E/Firebase.Foundation/FirebaseFoundationE2E/FirebaseRuntimeDriftCases.cs +++ b/tests/E2E/Firebase.Foundation/FirebaseFoundationE2E/FirebaseRuntimeDriftCases.cs @@ -1,119 +1,43 @@ using System.Reflection; - -#if ENABLE_RUNTIME_DRIFT_CASE_CORE_CONFIGURATION_LOGGERLEVEL -using Firebase.Core; using Foundation; using ObjCRuntime; -#endif -#if ENABLE_RUNTIME_DRIFT_CASE_ANALYTICS_SESSIONIDWITHCOMPLETION -using Firebase.Analytics; -using Foundation; -using ObjCRuntime; +#if ENABLE_RUNTIME_DRIFT_CASE_CORE_CONFIGURATION_LOGGERLEVEL +using Firebase.Core; #endif -#if ENABLE_RUNTIME_DRIFT_CASE_ANALYTICS_ONDEVICECONVERSION +#if ENABLE_RUNTIME_DRIFT_CASE_ANALYTICS_SESSIONIDWITHCOMPLETION || ENABLE_RUNTIME_DRIFT_CASE_ANALYTICS_ONDEVICECONVERSION using Firebase.Analytics; -using Foundation; -using ObjCRuntime; #endif #if ENABLE_RUNTIME_DRIFT_CASE_APPCHECK_LIMITED_USE_TOKENS using Firebase.AppCheck; using FirebaseCoreApp = Firebase.Core.App; -using Foundation; -using ObjCRuntime; #endif #if ENABLE_RUNTIME_DRIFT_CASE_REMOTECONFIG_REALTIME_CUSTOMSIGNALS using Firebase.RemoteConfig; -using Foundation; -using ObjCRuntime; #endif #if ENABLE_RUNTIME_DRIFT_CASE_DATABASE_SERVERVALUE_INCREMENT || ENABLE_RUNTIME_DRIFT_CASE_DATABASE_QUERY_GETDATA using Firebase.Database; using FirebaseCoreOptions = Firebase.Core.Options; -using Foundation; -using ObjCRuntime; -#endif - -#if ENABLE_RUNTIME_DRIFT_CASE_ABTESTING_UPDATEEXPERIMENTS -using Firebase.ABTesting; -using Foundation; -using ObjCRuntime; -#endif - -#if ENABLE_RUNTIME_DRIFT_CASE_ABTESTING_ACTIVATEEXPERIMENT -using Firebase.ABTesting; -using Foundation; -using ObjCRuntime; #endif -#if ENABLE_RUNTIME_DRIFT_CASE_ABTESTING_VALIDATERUNNINGEXPERIMENTS +#if ENABLE_RUNTIME_DRIFT_CASE_ABTESTING_UPDATEEXPERIMENTS || ENABLE_RUNTIME_DRIFT_CASE_ABTESTING_ACTIVATEEXPERIMENT || ENABLE_RUNTIME_DRIFT_CASE_ABTESTING_VALIDATERUNNINGEXPERIMENTS using Firebase.ABTesting; -using Foundation; -using ObjCRuntime; -#endif - -#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_GETQUERYNAMED -using Firebase.CloudFirestore; -using Foundation; -using ObjCRuntime; -#endif - -#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_FIELDVALUE_VECTORWITHARRAY -using Firebase.CloudFirestore; -using Foundation; -using ObjCRuntime; -#endif - -#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_AGGREGATE_QUERY -using Firebase.CloudFirestore; -using Foundation; -using ObjCRuntime; -#endif - -#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_QUERY_FILTERS -using Firebase.CloudFirestore; -using Foundation; -using ObjCRuntime; -#endif - -#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_SNAPSHOT_LISTEN_OPTIONS -using Firebase.CloudFirestore; -using Foundation; -using ObjCRuntime; -#endif - -#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_NAMED_DATABASE -using Firebase.CloudFirestore; -using Foundation; -using ObjCRuntime; #endif -#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_CACHE_SETTINGS -using Firebase.CloudFirestore; -using Foundation; -using ObjCRuntime; -#endif - -#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_INDEX_CONFIGURATION +#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_GETQUERYNAMED || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_FIELDVALUE_VECTORWITHARRAY || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_AGGREGATE_QUERY || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_QUERY_FILTERS || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_SNAPSHOT_LISTEN_OPTIONS || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_NAMED_DATABASE || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_CACHE_SETTINGS || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_INDEX_CONFIGURATION using Firebase.CloudFirestore; -using Foundation; -using ObjCRuntime; #endif #if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFUNCTIONS_USEFUNCTIONSEMULATORORIGIN using Firebase.CloudFunctions; -using Foundation; -using ObjCRuntime; #endif #if ENABLE_RUNTIME_DRIFT_CASE_CRASHLYTICS_STACKFRAMEWITHADDRESS || ENABLE_RUNTIME_DRIFT_CASE_CRASHLYTICS_RECORD_ERROR_USER_INFO using Firebase.Crashlytics; -using Foundation; -using ObjCRuntime; #endif namespace FirebaseFoundationE2E; @@ -122,17 +46,28 @@ static partial class FirebaseRuntimeDriftCases { static readonly TimeSpan AsyncTimeout = TimeSpan.FromSeconds(5); + public sealed record RuntimeDriftCase(string Id, string MethodName); + public static string? GetConfiguredCaseId() { return GetAssemblyMetadataValue("RuntimeDriftCase"); } - public static async Task ExecuteConfiguredCaseAsync() + public static IReadOnlyList GetConfiguredCases() { + var configuredCases = GetAssemblyMetadataValue("RuntimeDriftCases"); + if (!string.IsNullOrWhiteSpace(configuredCases)) + { + return configuredCases + .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(ParseConfiguredCase) + .ToArray(); + } + var caseId = GetConfiguredCaseId(); if (string.IsNullOrWhiteSpace(caseId)) { - throw new InvalidOperationException("Runtime drift mode was requested without a RuntimeDriftCase value."); + return Array.Empty(); } var methodName = GetAssemblyMetadataValue("RuntimeDriftCaseMethod"); @@ -141,20 +76,54 @@ public static async Task ExecuteConfiguredCaseAsync() throw new InvalidOperationException($"Runtime drift case '{caseId}' is missing RuntimeDriftCaseMethod metadata."); } - var method = typeof(FirebaseRuntimeDriftCases).GetMethod(methodName, BindingFlags.Static | BindingFlags.NonPublic); + return new[] { new RuntimeDriftCase(caseId, methodName) }; + } + + public static async Task ExecuteConfiguredCaseAsync() + { + var caseId = GetConfiguredCaseId(); + if (string.IsNullOrWhiteSpace(caseId)) + { + throw new InvalidOperationException("Runtime drift mode was requested without a RuntimeDriftCase value."); + } + + var methodName = GetAssemblyMetadataValue("RuntimeDriftCaseMethod"); + var configuredCase = new RuntimeDriftCase(caseId, methodName ?? string.Empty); + return await ExecuteConfiguredCaseAsync(configuredCase); + } + + public static async Task ExecuteConfiguredCaseAsync(RuntimeDriftCase configuredCase) + { + if (string.IsNullOrWhiteSpace(configuredCase.MethodName)) + { + throw new InvalidOperationException($"Runtime drift case '{configuredCase.Id}' is missing RuntimeDriftCaseMethod metadata."); + } + + var method = typeof(FirebaseRuntimeDriftCases).GetMethod(configuredCase.MethodName, BindingFlags.Static | BindingFlags.NonPublic); if (method is null) { - throw new InvalidOperationException($"Runtime drift case '{caseId}' points at missing method '{methodName}'."); + throw new InvalidOperationException($"Runtime drift case '{configuredCase.Id}' points at missing method '{configuredCase.MethodName}'."); } if (method.Invoke(null, null) is not Task task) { - throw new InvalidOperationException($"Runtime drift case '{caseId}' method '{methodName}' did not return Task."); + throw new InvalidOperationException($"Runtime drift case '{configuredCase.Id}' method '{configuredCase.MethodName}' did not return Task."); } return await task; } + static RuntimeDriftCase ParseConfiguredCase(string value) + { + var separatorIndex = value.IndexOf('='); + if (separatorIndex <= 0 || separatorIndex == value.Length - 1) + { + throw new InvalidOperationException($"Runtime drift case metadata entry '{value}' must use '=' format."); + } + + return new RuntimeDriftCase(value[..separatorIndex], value[(separatorIndex + 1)..]); + } + static string? GetAssemblyMetadataValue(string key) { return typeof(FirebaseRuntimeDriftCases) diff --git a/tests/E2E/Firebase.Foundation/FirebaseFoundationE2E/FirebaseSelfTestRunner.cs b/tests/E2E/Firebase.Foundation/FirebaseFoundationE2E/FirebaseSelfTestRunner.cs index e705b5be..46203400 100644 --- a/tests/E2E/Firebase.Foundation/FirebaseFoundationE2E/FirebaseSelfTestRunner.cs +++ b/tests/E2E/Firebase.Foundation/FirebaseFoundationE2E/FirebaseSelfTestRunner.cs @@ -29,10 +29,14 @@ public static async Task RunAsync(StatusViewController statusViewController) #if ENABLE_BINDING_SURFACE_COVERAGE await statusViewController.AppendLineAsync("Firebase binding surface coverage mode enabled."); #endif - var runtimeDriftCase = FirebaseRuntimeDriftCases.GetConfiguredCaseId(); - if (!string.IsNullOrWhiteSpace(runtimeDriftCase)) + var runtimeDriftCases = FirebaseRuntimeDriftCases.GetConfiguredCases(); + if (runtimeDriftCases.Count > 0) { - await statusViewController.AppendLineAsync($"Runtime drift case mode enabled: {runtimeDriftCase}"); + var configuredCaseId = FirebaseRuntimeDriftCases.GetConfiguredCaseId(); + var caseDisplay = string.Equals(configuredCaseId, "all", StringComparison.OrdinalIgnoreCase) + ? $"all ({runtimeDriftCases.Count} cases)" + : runtimeDriftCases[0].Id; + await statusViewController.AppendLineAsync($"Runtime drift case mode enabled: {caseDisplay}"); } try @@ -52,10 +56,13 @@ await ExecuteCaseAsync(result, statusViewController, "ConfigureApp", async () => return $"Configured app '{defaultApp.Name}'."; }); - if (!string.IsNullOrWhiteSpace(runtimeDriftCase)) + if (runtimeDriftCases.Count > 0) { - await ExecuteCaseAsync(result, statusViewController, $"RuntimeDrift:{runtimeDriftCase}", () => - FirebaseRuntimeDriftCases.ExecuteConfiguredCaseAsync()); + foreach (var runtimeDriftCase in runtimeDriftCases) + { + await ExecuteCaseAsync(result, statusViewController, $"RuntimeDrift:{runtimeDriftCase.Id}", () => + FirebaseRuntimeDriftCases.ExecuteConfiguredCaseAsync(runtimeDriftCase)); + } } #if ENABLE_BINDING_SURFACE_COVERAGE else if (!string.IsNullOrWhiteSpace(FirebaseBindingSurfaceCoverage.GetConfiguredTarget())) diff --git a/tests/E2E/Firebase.Foundation/README.md b/tests/E2E/Firebase.Foundation/README.md index ec965b9e..0832be9a 100644 --- a/tests/E2E/Firebase.Foundation/README.md +++ b/tests/E2E/Firebase.Foundation/README.md @@ -81,9 +81,9 @@ dotnet tool run dotnet-cake -- --target=nuget --names="Firebase.Analytics,Fireba tools/e2e/run-firebase-foundation.sh --package-dir output --configuration Debug --enable-nullability-validation ``` -## Targeted runtime drift mode +## Runtime drift mode -The harness also supports a targeted runtime-drift lane for one binding drift at a time. This mode runs only `ConfigureApp` plus the selected drift case, so each remediation PR can prove the unfixed runtime failure locally, then keep only the success-oriented regression test in the final diff. +The harness also supports a runtime-drift lane for binding-layer checks that need simulator execution. This mode runs only `ConfigureApp` plus the selected drift case, so each remediation PR can prove the unfixed runtime failure locally, then keep only the success-oriented regression test in the final diff. Use `all` to build once and run every checked-in drift case in the same app launch. The checked-in case manifest lives at [`runtime-drift-cases.json`](./runtime-drift-cases.json), and the backlog/queue is tracked in [`docs/firebase-runtime-failure-backlog.md`](../../docs/firebase-runtime-failure-backlog.md). @@ -91,6 +91,7 @@ Run a specific drift case with: ```sh tools/e2e/run-firebase-foundation.sh --package-dir output --configuration Debug --runtime-drift-case cloudfirestore-getquerynamed +tools/e2e/run-firebase-foundation.sh --package-dir output --configuration Debug --runtime-drift-case all ``` ## Binding surface coverage mode diff --git a/tools/e2e/run-firebase-foundation.sh b/tools/e2e/run-firebase-foundation.sh index d5ff8e83..370480c0 100755 --- a/tools/e2e/run-firebase-foundation.sh +++ b/tools/e2e/run-firebase-foundation.sh @@ -3,7 +3,7 @@ set -euo pipefail usage() { cat <<'EOF' -Usage: tools/e2e/run-firebase-foundation.sh [--package-dir output] [--configuration Release] [--enable-nullability-validation] [--runtime-drift-case ] [--binding-surface-target ] +Usage: tools/e2e/run-firebase-foundation.sh [--package-dir output] [--configuration Release] [--enable-nullability-validation] [--runtime-drift-case ] [--binding-surface-target ] EOF } @@ -27,6 +27,7 @@ runtime_drift_props="$artifacts_dir/runtime-drift-case.generated.props" runtime_drift_info="$artifacts_dir/runtime-drift-case.info" runtime_drift_method="" runtime_drift_binding_package="" +runtime_drift_case_count="" binding_surface_manifest="$repo_root/tests/E2E/Firebase.Foundation/binding-surface-coverage.json" binding_surface_document="$artifacts_dir/binding-surface-coverage.generated.json" binding_surface_props="$artifacts_dir/binding-surface-coverage.generated.props" @@ -127,48 +128,113 @@ manifest_path = pathlib.Path(sys.argv[1]) case_id = sys.argv[2] props_path = pathlib.Path(sys.argv[3]) manifest = json.loads(manifest_path.read_text()) - -case = next((entry for entry in manifest.get("cases", []) if entry.get("id") == case_id), None) -if case is None: - available = ", ".join(sorted(entry.get("id", "") for entry in manifest.get("cases", []))) - raise SystemExit(f"Unknown runtime drift case '{case_id}'. Available cases: {available}") - -method = case.get("method") -binding_package = case.get("bindingPackage") -packages = case.get("packages", []) - -if not method or not binding_package or packages is None: - raise SystemExit(f"Runtime drift case '{case_id}' is missing required manifest fields.") - -symbol = "ENABLE_RUNTIME_DRIFT_CASE_" + re.sub(r"[^A-Za-z0-9]+", "_", case_id).strip("_").upper() +cases = manifest.get("cases", []) + +if case_id == "all": + selected_cases = cases +else: + case = next((entry for entry in cases if entry.get("id") == case_id), None) + if case is None: + available = ", ".join(sorted(entry.get("id", "") for entry in cases)) + raise SystemExit(f"Unknown runtime drift case '{case_id}'. Available cases: {available}, all") + selected_cases = [case] + +if not selected_cases: + raise SystemExit("Runtime drift manifest does not contain any cases.") + +package_versions = {} +required_package_ids = set() + +def add_package(package): + package_id = package.get("id") + version = package.get("version") + if not package_id or not version: + raise SystemExit("Runtime drift package entries must include id and version.") + + existing = package_versions.get(package_id) + if existing is not None and existing != version: + raise SystemExit( + f"Runtime drift manifest contains conflicting versions for '{package_id}': '{existing}' and '{version}'." + ) + + package_versions[package_id] = version + required_package_ids.add(package_id) + +case_entries = [] +symbols = [] +binding_packages = [] + +for selected_case in selected_cases: + selected_case_id = selected_case.get("id") + method = selected_case.get("method") + binding_package = selected_case.get("bindingPackage") + packages = selected_case.get("packages", []) + + if not selected_case_id or not method or not binding_package or packages is None: + raise SystemExit(f"Runtime drift case '{selected_case_id or ''}' is missing required manifest fields.") + + binding_packages.append(binding_package) + required_package_ids.add(binding_package) + case_entries.append(f"{selected_case_id}={method}") + symbols.append("ENABLE_RUNTIME_DRIFT_CASE_" + re.sub(r"[^A-Za-z0-9]+", "_", selected_case_id).strip("_").upper()) + + for package in packages: + add_package(package) + +runtime_cases = ",".join(case_entries) +define_constants = "$(DefineConstants);ENABLE_RUNTIME_DRIFT_CASE;" + ";".join(symbols) +single_method = selected_cases[0].get("method") if len(selected_cases) == 1 else "" props_path.write_text( "\n" " \n" f" {escape(case_id)}\n" - f" {escape(method)}\n" - f" $(DefineConstants);ENABLE_RUNTIME_DRIFT_CASE;{escape(symbol)}\n" + f" {escape(single_method)}\n" + f" {escape(runtime_cases)}\n" + f" {escape(define_constants)}\n" " \n" " \n" + "".join( - f" \n" - for package in packages + f" \n" + for package_id in sorted(package_versions) ) + " \n" "\n" ) -print(method) -print(binding_package) -for package in packages: - print(package["id"]) +print(f"case-count|{len(selected_cases)}") +if len(selected_cases) == 1: + print(f"method|{selected_cases[0]['method']}") +for selected_case in selected_cases: + print(f"case|{selected_case['id']}") +for binding_package in sorted(set(binding_packages)): + print(f"binding-package|{binding_package}") +for package_id in sorted(required_package_ids): + print(f"package|{package_id}") PY runtime_drift_details=("${(@f)$(<"$runtime_drift_info")}") - runtime_drift_method="${runtime_drift_details[1]}" - runtime_drift_binding_package="${runtime_drift_details[2]}" - required_packages+=("$runtime_drift_binding_package") - for (( i = 3; i <= ${#runtime_drift_details[@]}; i++ )); do - required_packages+=("${runtime_drift_details[$i]}") + for runtime_drift_detail in "${runtime_drift_details[@]}"; do + runtime_drift_key="${runtime_drift_detail%%|*}" + runtime_drift_value="${runtime_drift_detail#*|}" + case "$runtime_drift_key" in + case-count) + runtime_drift_case_count="$runtime_drift_value" + ;; + method) + runtime_drift_method="$runtime_drift_value" + ;; + binding-package) + if [[ -z "$runtime_drift_binding_package" ]]; then + runtime_drift_binding_package="$runtime_drift_value" + else + runtime_drift_binding_package="$runtime_drift_binding_package,$runtime_drift_value" + fi + required_packages+=("$runtime_drift_value") + ;; + package) + required_packages+=("$runtime_drift_value") + ;; + esac done msbuild_args+=( @@ -178,7 +244,11 @@ PY ) restore_args+=("--force-evaluate") - echo "Runtime drift case: $runtime_drift_case ($runtime_drift_binding_package)" + if [[ "$runtime_drift_case" == "all" ]]; then + echo "Runtime drift cases: all ($runtime_drift_case_count cases)" + else + echo "Runtime drift case: $runtime_drift_case ($runtime_drift_binding_package)" + fi fi if [[ -n "$binding_surface_target" ]]; then @@ -332,7 +402,13 @@ xcrun simctl launch --terminate-running-process "$simulator_udid" "$bundle_id" > data_container="$(xcrun simctl get_app_container "$simulator_udid" "$bundle_id" data)" container_result_file="$data_container/Library/Caches/firebase-foundation-e2e-result.json" -timeout_seconds="${E2E_TIMEOUT_SECONDS:-90}" +default_timeout_seconds=90 +timeout_seconds="${E2E_TIMEOUT_SECONDS:-$default_timeout_seconds}" +if [[ -z "${E2E_TIMEOUT_SECONDS:-}" && "$runtime_drift_case" == "all" && "$runtime_drift_case_count" == <-> ]]; then + timeout_seconds=$((default_timeout_seconds + runtime_drift_case_count * 10)) +fi +echo "Waiting up to $timeout_seconds seconds for E2E result" + elapsed=0 while [[ ! -f "$container_result_file" ]]; do if (( elapsed >= timeout_seconds )); then