diff --git a/.github/actions/checkout-and-sync/action.yml b/.github/actions/checkout-and-sync/action.yml new file mode 100644 index 000000000000..930626a1543d --- /dev/null +++ b/.github/actions/checkout-and-sync/action.yml @@ -0,0 +1,62 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: 'Checkout and Sync' +description: >- + Check out apache/spark at a pinned ref and, on forks, squash-merge + the fork branch on top so the build tests the combined result. + Callers must add a bare 'actions/checkout@v6' step before invoking + this action so that the latest action definition is present in the workspace. +inputs: + ref: + description: 'Git ref to check out from apache/spark' + required: true + set-safe-directory: + description: 'Add GITHUB_WORKSPACE to git safe.directory (needed inside containers)' + required: false + default: 'false' +outputs: + head_sha: + description: 'The apache/spark HEAD SHA checked out (before the fork merge)' + value: ${{ steps.resolve-sha.outputs.head_sha }} +runs: + using: 'composite' + steps: + - name: Checkout Spark repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + repository: apache/spark + ref: ${{ inputs.ref }} + - name: Resolve apache/spark HEAD SHA + id: resolve-sha + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + - name: Add GITHUB_WORKSPACE to git trust safe.directory + if: inputs.set-safe-directory == 'true' + shell: bash + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Sync the current branch with the latest in Apache Spark + if: github.repository != 'apache/spark' + shell: bash + run: | + echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV + git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} + git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD + git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index b0ae21f9801b..fc8f61af014d 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -69,7 +69,7 @@ jobs: outputs: required: ${{ steps.set-outputs.outputs.required }} # Pinned so every downstream job checks out the same snapshot, even if `master` advances mid-run. - head_sha: ${{ steps.resolve-sha.outputs.head_sha }} + head_sha: ${{ steps.checkout.outputs.head_sha }} image_url: ${{ steps.infra-image-outputs.outputs.image_url }} image_docs_url: ${{ steps.infra-image-docs-outputs.outputs.image_docs_url }} image_docs_url_link: ${{ steps.infra-image-link.outputs.image_docs_url_link }} @@ -80,22 +80,13 @@ jobs: image_pyspark_url: ${{ steps.infra-image-pyspark-outputs.outputs.image_pyspark_url }} image_pyspark_url_link: ${{ steps.infra-image-link.outputs.image_pyspark_url_link }} steps: - - name: Checkout Spark repository + - name: Bootstrap composite actions uses: actions/checkout@v6 + - name: Checkout and sync Spark repository + id: checkout + uses: ./.github/actions/checkout-and-sync with: - fetch-depth: 0 - repository: apache/spark ref: ${{ inputs.branch }} - - name: Resolve apache/spark HEAD SHA - id: resolve-sha - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - name: Sync the current branch with the latest in Apache Spark - if: github.repository != 'apache/spark' - run: | - echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV - git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty - name: Check all modules id: set-outputs run: | @@ -353,20 +344,12 @@ jobs: SKIP_MIMA: true SKIP_PACKAGING: true steps: - - name: Checkout Spark repository + - name: Bootstrap composite actions uses: actions/checkout@v6 - # In order to fetch changed files + - name: Checkout and sync Spark repository + uses: ./.github/actions/checkout-and-sync with: - fetch-depth: 0 - repository: apache/spark ref: ${{ needs.precondition.outputs.head_sha }} - - name: Sync the current branch with the latest in Apache Spark - if: github.repository != 'apache/spark' - run: | - echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV - git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty # Cache local repositories. Note that GitHub Actions cache has a 10G limit. - name: Cache SBT and Maven uses: actions/cache@v5 @@ -490,20 +473,12 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Checkout Spark repository + - name: Bootstrap composite actions uses: actions/checkout@v6 - # In order to fetch changed files + - name: Checkout and sync Spark repository + uses: ./.github/actions/checkout-and-sync with: - fetch-depth: 0 - repository: apache/spark ref: ${{ needs.precondition.outputs.head_sha }} - - name: Sync the current branch with the latest in Apache Spark - if: github.repository != 'apache/spark' - run: | - echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV - git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty - name: Set up QEMU uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a - name: Set up Docker Buildx @@ -589,19 +564,12 @@ jobs: HIVE_PROFILE: hive2.3 GITHUB_PREV_SHA: ${{ github.event.before }} steps: - - name: Checkout Spark repository + - name: Bootstrap composite actions uses: actions/checkout@v6 + - name: Checkout and sync Spark repository + uses: ./.github/actions/checkout-and-sync with: - fetch-depth: 0 - repository: apache/spark ref: ${{ needs.precondition.outputs.head_sha }} - - name: Sync the current branch with the latest in Apache Spark - if: github.repository != 'apache/spark' - run: | - echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV - git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty - name: Cache SBT and Maven uses: actions/cache@v5 with: @@ -713,23 +681,13 @@ jobs: BRANCH: ${{ inputs.branch }} PYSPARK_TEST_TIMEOUT: 450 steps: - - name: Checkout Spark repository + - name: Bootstrap composite actions uses: actions/checkout@v6 - # In order to fetch changed files + - name: Checkout and sync Spark repository + uses: ./.github/actions/checkout-and-sync with: - fetch-depth: 0 - repository: apache/spark ref: ${{ needs.precondition.outputs.head_sha }} - - name: Add GITHUB_WORKSPACE to git trust safe.directory - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Sync the current branch with the latest in Apache Spark - if: github.repository != 'apache/spark' - run: | - echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV - git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty + set-safe-directory: 'true' # Cache local repositories. Note that GitHub Actions cache has a 10G limit. - name: Cache SBT and Maven uses: actions/cache@v5 @@ -899,23 +857,13 @@ jobs: SKIP_MIMA: true SKIP_PACKAGING: true steps: - - name: Checkout Spark repository + - name: Bootstrap composite actions uses: actions/checkout@v6 - # In order to fetch changed files + - name: Checkout and sync Spark repository + uses: ./.github/actions/checkout-and-sync with: - fetch-depth: 0 - repository: apache/spark ref: ${{ needs.precondition.outputs.head_sha }} - - name: Add GITHUB_WORKSPACE to git trust safe.directory - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Sync the current branch with the latest in Apache Spark - if: github.repository != 'apache/spark' - run: | - echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV - git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty + set-safe-directory: 'true' # Cache local repositories. Note that GitHub Actions cache has a 10G limit. - name: Cache SBT and Maven uses: actions/cache@v5 @@ -989,18 +937,12 @@ jobs: name: Protobuf breaking change detection and Python CodeGen check runs-on: ubuntu-latest steps: - - name: Checkout Spark repository + - name: Bootstrap composite actions uses: actions/checkout@v6 + - name: Checkout and sync Spark repository + uses: ./.github/actions/checkout-and-sync with: - fetch-depth: 0 - repository: apache/spark ref: ${{ needs.precondition.outputs.head_sha }} - - name: Sync the current branch with the latest in Apache Spark - if: github.repository != 'apache/spark' - run: | - git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty - name: Install Buf uses: bufbuild/buf-setup-action@a47c93e0b1648d5651a065437926377d060baa99 # v1 with: @@ -1051,22 +993,13 @@ jobs: container: image: ${{ needs.precondition.outputs.image_lint_url_link }} steps: - - name: Checkout Spark repository + - name: Bootstrap composite actions uses: actions/checkout@v6 + - name: Checkout and sync Spark repository + uses: ./.github/actions/checkout-and-sync with: - fetch-depth: 0 - repository: apache/spark ref: ${{ needs.precondition.outputs.head_sha }} - - name: Add GITHUB_WORKSPACE to git trust safe.directory - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Sync the current branch with the latest in Apache Spark - if: github.repository != 'apache/spark' - run: | - echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV - git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty + set-safe-directory: 'true' # Cache local repositories. Note that GitHub Actions cache has a 10G limit. - name: Cache SBT and Maven uses: actions/cache@v5 @@ -1259,22 +1192,13 @@ jobs: container: image: ${{ needs.precondition.outputs.image_docs_url_link }} steps: - - name: Checkout Spark repository + - name: Bootstrap composite actions uses: actions/checkout@v6 + - name: Checkout and sync Spark repository + uses: ./.github/actions/checkout-and-sync with: - fetch-depth: 0 - repository: apache/spark ref: ${{ needs.precondition.outputs.head_sha }} - - name: Add GITHUB_WORKSPACE to git trust safe.directory - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Sync the current branch with the latest in Apache Spark - if: github.repository != 'apache/spark' - run: | - echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV - git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty + set-safe-directory: 'true' # Cache local repositories. Note that GitHub Actions cache has a 10G limit. - name: Cache SBT and Maven uses: actions/cache@v5 @@ -1459,18 +1383,12 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - name: Checkout Spark repository + - name: Bootstrap composite actions uses: actions/checkout@v6 + - name: Checkout and sync Spark repository + uses: ./.github/actions/checkout-and-sync with: - fetch-depth: 0 - repository: apache/spark ref: ${{ needs.precondition.outputs.head_sha }} - - name: Sync the current branch with the latest in Apache Spark - if: github.repository != 'apache/spark' - run: | - git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty - name: Cache SBT and Maven uses: actions/cache@v5 with: @@ -1590,19 +1508,12 @@ jobs: SKIP_MIMA: true SKIP_PACKAGING: true steps: - - name: Checkout Spark repository + - name: Bootstrap composite actions uses: actions/checkout@v6 + - name: Checkout and sync Spark repository + uses: ./.github/actions/checkout-and-sync with: - fetch-depth: 0 - repository: apache/spark ref: ${{ needs.precondition.outputs.head_sha }} - - name: Sync the current branch with the latest in Apache Spark - if: github.repository != 'apache/spark' - run: | - echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV - git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty - name: Cache SBT and Maven uses: actions/cache@v5 with: @@ -1676,19 +1587,12 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 steps: - - name: Checkout Spark repository + - name: Bootstrap composite actions uses: actions/checkout@v6 + - name: Checkout and sync Spark repository + uses: ./.github/actions/checkout-and-sync with: - fetch-depth: 0 - repository: apache/spark ref: ${{ needs.precondition.outputs.head_sha }} - - name: Sync the current branch with the latest in Apache Spark - if: github.repository != 'apache/spark' - run: | - echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV - git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty - name: Cache SBT and Maven uses: actions/cache@v5 with: diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index f8edd0b72336..69af2bca6f55 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -63,29 +63,20 @@ jobs: continue-on-error: true outputs: # Pinned so the build job checks out the same snapshot, even if the branch advances mid-run. - head_sha: ${{ steps.resolve-sha.outputs.head_sha }} + head_sha: ${{ steps.checkout.outputs.head_sha }} env: HADOOP_PROFILE: ${{ inputs.hadoop }} HIVE_PROFILE: hive2.3 SPARK_LOCAL_IP: localhost GITHUB_PREV_SHA: ${{ github.event.before }} steps: - - name: Checkout Spark repository + - name: Bootstrap composite actions uses: actions/checkout@v6 + - name: Checkout and sync Spark repository + id: checkout + uses: ./.github/actions/checkout-and-sync with: - fetch-depth: 0 - repository: apache/spark ref: ${{ inputs.branch }} - - name: Resolve apache/spark HEAD SHA - id: resolve-sha - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - name: Sync the current branch with the latest in Apache Spark - if: github.repository != 'apache/spark' - run: | - echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV - git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty - name: Cache SBT and Maven # TODO(SPARK-54466): https://github.com/actions/runner-images/issues/13341 if: ${{ runner.os != 'macOS' }} @@ -222,21 +213,12 @@ jobs: SPARK_LOCAL_IP: localhost GITHUB_PREV_SHA: ${{ github.event.before }} steps: - - name: Checkout Spark repository + - name: Bootstrap composite actions uses: actions/checkout@v6 - # In order to fetch changed files + - name: Checkout and sync Spark repository + uses: ./.github/actions/checkout-and-sync with: - fetch-depth: 0 - repository: apache/spark - # Fall back to the branch when the precompile job failed before resolving the SHA. ref: ${{ needs.precompile-maven.outputs.head_sha || inputs.branch }} - - name: Sync the current branch with the latest in Apache Spark - if: github.repository != 'apache/spark' - run: | - echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV - git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty # Cache local repositories. Note that GitHub Actions cache has a 10G limit. - name: Cache SBT and Maven # TODO(SPARK-54466): https://github.com/actions/runner-images/issues/13341 diff --git a/.github/workflows/python_hosted_runner_test.yml b/.github/workflows/python_hosted_runner_test.yml index 7488a0b7b9d1..960267c2dabd 100644 --- a/.github/workflows/python_hosted_runner_test.yml +++ b/.github/workflows/python_hosted_runner_test.yml @@ -67,29 +67,20 @@ jobs: continue-on-error: true outputs: # Pinned so the build job checks out the same snapshot, even if the branch advances mid-run. - head_sha: ${{ steps.resolve-sha.outputs.head_sha }} + head_sha: ${{ steps.checkout.outputs.head_sha }} env: HADOOP_PROFILE: ${{ inputs.hadoop }} HIVE_PROFILE: hive2.3 SPARK_LOCAL_IP: localhost GITHUB_PREV_SHA: ${{ github.event.before }} steps: - - name: Checkout Spark repository + - name: Bootstrap composite actions uses: actions/checkout@v6 + - name: Checkout and sync Spark repository + id: checkout + uses: ./.github/actions/checkout-and-sync with: - fetch-depth: 0 - repository: apache/spark ref: ${{ inputs.branch }} - - name: Resolve apache/spark HEAD SHA - id: resolve-sha - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - name: Sync the current branch with the latest in Apache Spark - if: github.repository != 'apache/spark' - run: | - echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV - git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty - name: Cache SBT and Maven # TODO(SPARK-54466): https://github.com/actions/runner-images/issues/13341 if: ${{ runner.os != 'macOS' }} @@ -185,21 +176,12 @@ jobs: BRANCH: ${{ inputs.branch }} PYSPARK_TEST_TIMEOUT: 450 steps: - - name: Checkout Spark repository + - name: Bootstrap composite actions uses: actions/checkout@v6 - # In order to fetch changed files + - name: Checkout and sync Spark repository + uses: ./.github/actions/checkout-and-sync with: - fetch-depth: 0 - repository: apache/spark - # Fall back to the branch when the precompile job failed before resolving the SHA. ref: ${{ needs.precompile.outputs.head_sha || inputs.branch }} - - name: Sync the current branch with the latest in Apache Spark - if: github.repository != 'apache/spark' - run: | - echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV - git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty # Cache local repositories. Note that GitHub Actions cache has a 10G limit. - name: Cache SBT and Maven # TODO(SPARK-54466): https://github.com/actions/runner-images/issues/13341