From 85703c06c92af5e68e618db6757574fb52250e5b Mon Sep 17 00:00:00 2001 From: Cristiano Veiga Date: Fri, 27 Mar 2026 17:30:57 -0400 Subject: [PATCH 1/2] fix(hypershift/gcp): correct DNS zone name and surface cleanup errors The e2e-gke workflow had HYPERSHIFT_GCP_CI_DNS_ZONE set to "hypershift-ci-zone" but the actual zone is "hypershift-ci-gcp-hcp-openshiftapps-com". This caused the deprovision step's DNS cleanup to silently fail. Additionally, the gcloud dns list command had 2>/dev/null || true which swallowed permission errors (403 Forbidden), making it appear that no DNS records existed. Replace with explicit error handling that logs failures instead of hiding them. Co-Authored-By: Claude Opus 4.6 --- .../hypershift-gcp-gke-deprovision-commands.sh | 13 +++++++++---- .../gke/e2e/hypershift-gcp-gke-e2e-workflow.yaml | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ci-operator/step-registry/hypershift/gcp/gke/deprovision/hypershift-gcp-gke-deprovision-commands.sh b/ci-operator/step-registry/hypershift/gcp/gke/deprovision/hypershift-gcp-gke-deprovision-commands.sh index 6c36a8754bd7a..fa9ef95881671 100644 --- a/ci-operator/step-registry/hypershift/gcp/gke/deprovision/hypershift-gcp-gke-deprovision-commands.sh +++ b/ci-operator/step-registry/hypershift/gcp/gke/deprovision/hypershift-gcp-gke-deprovision-commands.sh @@ -101,20 +101,25 @@ EXTERNAL_DNS_GSA="external-dns@${HYPERSHIFT_GCP_CI_PROJECT}.iam.gserviceaccount. if [[ -n "${HC_CLUSTER_NAME}" ]]; then echo "Cleaning up DNS records for hosted cluster ${HC_CLUSTER_NAME}..." DNS_SUFFIX="in.${HC_CLUSTER_NAME}.${HYPERSHIFT_GCP_CI_DNS_DOMAIN}." - DNS_RECORDS=$(gcloud dns record-sets list \ + if ! DNS_RECORDS=$(gcloud dns record-sets list \ --zone="${HYPERSHIFT_GCP_CI_DNS_ZONE}" \ --project="${HYPERSHIFT_GCP_CI_PROJECT}" \ --filter="name ~ ${DNS_SUFFIX}" \ - --format="csv[no-heading](name,type)" 2>/dev/null || true) + --format="csv[no-heading](name,type)"); then + echo "WARNING: Failed to list DNS records - check service account permissions" + DNS_RECORDS="" + fi if [[ -n "${DNS_RECORDS}" ]]; then while IFS=, read -r name type; do [[ -z "${name}" ]] && continue echo "Deleting DNS record: ${name} ${type}" - gcloud dns record-sets delete "${name}" \ + if ! gcloud dns record-sets delete "${name}" \ --type="${type}" \ --zone="${HYPERSHIFT_GCP_CI_DNS_ZONE}" \ - --project="${HYPERSHIFT_GCP_CI_PROJECT}" --quiet || true + --project="${HYPERSHIFT_GCP_CI_PROJECT}" --quiet; then + echo "WARNING: Failed to delete DNS record ${name} ${type}" + fi done <<< "${DNS_RECORDS}" else echo "No DNS records found matching ${DNS_SUFFIX}" diff --git a/ci-operator/step-registry/hypershift/gcp/gke/e2e/hypershift-gcp-gke-e2e-workflow.yaml b/ci-operator/step-registry/hypershift/gcp/gke/e2e/hypershift-gcp-gke-e2e-workflow.yaml index 37b47fd57d826..4a5a1c678d306 100644 --- a/ci-operator/step-registry/hypershift/gcp/gke/e2e/hypershift-gcp-gke-e2e-workflow.yaml +++ b/ci-operator/step-registry/hypershift/gcp/gke/e2e/hypershift-gcp-gke-e2e-workflow.yaml @@ -37,5 +37,5 @@ workflow: GKE_RELEASE_CHANNEL: "stable" TECH_PREVIEW_NO_UPGRADE: "true" HYPERSHIFT_GCP_CI_PROJECT: "gcp-hcp-hypershift-ci" - HYPERSHIFT_GCP_CI_DNS_ZONE: "hypershift-ci-zone" + HYPERSHIFT_GCP_CI_DNS_ZONE: "hypershift-ci-gcp-hcp-openshiftapps-com" HYPERSHIFT_GCP_CI_DNS_DOMAIN: "hypershift-ci.gcp-hcp.openshiftapps.com" From ba4ed1fbee78fc47ea17b884c5857f3c7fe384d2 Mon Sep 17 00:00:00 2001 From: Cristiano Veiga Date: Mon, 30 Mar 2026 14:35:35 -0400 Subject: [PATCH 2/2] fix(hypershift/gcp): fail deprovision step on DNS cleanup errors DNS cleanup failures were logged as warnings but the step still exited 0, making orphaned DNS records invisible. Since the step has best_effort: true, failing it won't block the job but will surface the issue in the Prow UI. Co-Authored-By: Claude Opus 4.6 --- .../hypershift-gcp-gke-deprovision-commands.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ci-operator/step-registry/hypershift/gcp/gke/deprovision/hypershift-gcp-gke-deprovision-commands.sh b/ci-operator/step-registry/hypershift/gcp/gke/deprovision/hypershift-gcp-gke-deprovision-commands.sh index fa9ef95881671..80b4e09a84f13 100644 --- a/ci-operator/step-registry/hypershift/gcp/gke/deprovision/hypershift-gcp-gke-deprovision-commands.sh +++ b/ci-operator/step-registry/hypershift/gcp/gke/deprovision/hypershift-gcp-gke-deprovision-commands.sh @@ -98,6 +98,7 @@ gcloud projects delete "${CP_PROJECT_ID}" --quiet || true EXTERNAL_DNS_GSA="external-dns@${HYPERSHIFT_GCP_CI_PROJECT}.iam.gserviceaccount.com" # Clean up DNS records from the CI zone (DNS records use the hosted cluster name) +DNS_CLEANUP_FAILED=false if [[ -n "${HC_CLUSTER_NAME}" ]]; then echo "Cleaning up DNS records for hosted cluster ${HC_CLUSTER_NAME}..." DNS_SUFFIX="in.${HC_CLUSTER_NAME}.${HYPERSHIFT_GCP_CI_DNS_DOMAIN}." @@ -106,7 +107,8 @@ if [[ -n "${HC_CLUSTER_NAME}" ]]; then --project="${HYPERSHIFT_GCP_CI_PROJECT}" \ --filter="name ~ ${DNS_SUFFIX}" \ --format="csv[no-heading](name,type)"); then - echo "WARNING: Failed to list DNS records - check service account permissions" + echo "ERROR: Failed to list DNS records - check service account permissions" + DNS_CLEANUP_FAILED=true DNS_RECORDS="" fi @@ -118,7 +120,8 @@ if [[ -n "${HC_CLUSTER_NAME}" ]]; then --type="${type}" \ --zone="${HYPERSHIFT_GCP_CI_DNS_ZONE}" \ --project="${HYPERSHIFT_GCP_CI_PROJECT}" --quiet; then - echo "WARNING: Failed to delete DNS record ${name} ${type}" + echo "ERROR: Failed to delete DNS record ${name} ${type}" + DNS_CLEANUP_FAILED=true fi done <<< "${DNS_RECORDS}" else @@ -142,4 +145,9 @@ gcloud iam service-accounts remove-iam-policy-binding "${EXTERNAL_DNS_GSA}" \ --project="${HYPERSHIFT_GCP_CI_PROJECT}" || true set -x +if [[ "${DNS_CLEANUP_FAILED}" == "true" ]]; then + echo "Cleanup complete but DNS cleanup failed - orphaned DNS records may remain" + exit 1 +fi + echo "Cleanup complete"