diff --git a/.changeset/proud-owls-lick.md b/.changeset/proud-owls-lick.md new file mode 100644 index 000000000..5323f6f71 --- /dev/null +++ b/.changeset/proud-owls-lick.md @@ -0,0 +1,5 @@ +--- +"ctf-build-image": minor +--- + +Adds inputs and tweaks docker caching strategy for better performance diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml index cd6afc6d2..fc6d331cf 100644 --- a/.github/workflows/run-e2e-tests.yml +++ b/.github/workflows/run-e2e-tests.yml @@ -427,7 +427,7 @@ jobs: persist-credentials: false - name: Setup Go - uses: actions/setup-go@v5.0.2 + uses: actions/setup-go@v6 with: go-version: "1.24.0" check-latest: true @@ -955,13 +955,13 @@ jobs: - name: Upload trace data as artifact if: inputs.enable_otel_traces_for_ocr2_plugins && matrix.tests.test_env_vars.ENABLE_OTEL_TRACES == 'true' - uses: actions/upload-artifact@v4.4.3 + uses: actions/upload-artifact@v7 with: name: trace-data path: ./integration-tests/smoke/traces/trace-data.json - name: Upload test log as artifact - uses: actions/upload-artifact@v4.4.3 + uses: actions/upload-artifact@v7 if: failure() with: name: test_log_${{ env.TEST_ID }} @@ -971,7 +971,7 @@ jobs: - name: Upload cl node coverage data as artifact if: inputs.upload_cl_node_coverage_artifact - uses: actions/upload-artifact@v4.4.3 + uses: actions/upload-artifact@v7 timeout-minutes: 2 continue-on-error: true with: @@ -988,7 +988,7 @@ jobs: - name: Upload test result as artifact if: ${{ always() }} - uses: actions/upload-artifact@v4.4.3 + uses: actions/upload-artifact@v7 with: name: test_result_${{ needs.load-test-configurations.outputs.workflow_id }}_${{ env.TEST_ID }} @@ -997,7 +997,7 @@ jobs: - name: Upload custom test artifacts if: failure() && matrix.tests.test_artifacts_on_failure != '' - uses: actions/upload-artifact@v4.4.3 + uses: actions/upload-artifact@v7 with: name: custom_test_artifacts_${{ env.TEST_ID }}_${{ needs.load-test-configurations.outputs.workflow_id }} @@ -1256,7 +1256,7 @@ jobs: test_suite: ${{ matrix.tests.test_env_vars.TEST_SUITE }} - name: Upload test log as artifact - uses: actions/upload-artifact@v4.4.3 + uses: actions/upload-artifact@v7 if: failure() with: name: test_log_${{ env.TEST_ID }} @@ -1266,7 +1266,7 @@ jobs: - name: Upload custom test artifacts if: failure() && matrix.tests.test_artifacts_on_failure != '' - uses: actions/upload-artifact@v4.4.3 + uses: actions/upload-artifact@v7 with: name: ${{ format('custom_test_artifacts_{0}_{1}', env.TEST_ID, needs.load-test-configurations.outputs.workflow_id) }} path: ${{ matrix.tests.test_artifacts_on_failure }} diff --git a/.github/workflows/solidity-review-artifacts.yml b/.github/workflows/solidity-review-artifacts.yml index bb4013cf1..f3499786a 100644 --- a/.github/workflows/solidity-review-artifacts.yml +++ b/.github/workflows/solidity-review-artifacts.yml @@ -184,7 +184,7 @@ jobs: done - name: Upload basic info and modified contracts list - uses: actions/upload-artifact@v4.6.2 + uses: actions/upload-artifact@v7 timeout-minutes: 2 continue-on-error: true with: @@ -284,7 +284,7 @@ jobs: inputs.foundry_profile_override || inputs.product }} - name: Upload Artifacts for product contracts - uses: actions/upload-artifact@v4.6.2 + uses: actions/upload-artifact@v7 timeout-minutes: 2 continue-on-error: true with: @@ -401,7 +401,7 @@ jobs: ./dot_github/tools/scripts/solidity/generate_slither_report.sh "${{ github.server_url }}/${{ github.repository }}/blob/${{ env.head_ref }}/" "$SLITHER_CONFIG_FILE_PATH" "$CONTRACTS_DIRECTORY" "$contract_list" "${{ env.artifacts_dir }}/slither-reports" "--solc-remaps @=$CONTRACTS_DIRECTORY/node_modules/@" - name: Upload UMLs and Slither reports - uses: actions/upload-artifact@v4.6.2 + uses: actions/upload-artifact@v7 timeout-minutes: 10 continue-on-error: true with: @@ -437,7 +437,7 @@ jobs: merge-multiple: true - name: Upload all artifacts as single package - uses: actions/upload-artifact@v4.6.2 + uses: actions/upload-artifact@v7 with: name: review-artifacts-${{ inputs.product }}-${{ inputs.base_ref }}-${{ env.head_ref }} diff --git a/actions/apidiff-go/README.md b/actions/apidiff-go/README.md index ac4df704e..4937ef6f3 100644 --- a/actions/apidiff-go/README.md +++ b/actions/apidiff-go/README.md @@ -67,7 +67,7 @@ jobs: fetch-depth: 0 - name: Set up Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version-file: "go.mod" cache: false @@ -140,7 +140,7 @@ jobs: fetch-depth: 0 - name: Set up Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 # uses: ./.github/actions/setup-go with: go-version-file: ${{matrix.modules}}/go.mod diff --git a/actions/build-push-docker/action.yml b/actions/build-push-docker/action.yml index 04f0fed76..53d3f759e 100644 --- a/actions/build-push-docker/action.yml +++ b/actions/build-push-docker/action.yml @@ -203,14 +203,14 @@ runs: - name: Login to private ECR registries for base images if: ${{ steps.dockerfile-ecr-parse.outputs.needs-ecr-login == 'true' }} - uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 + uses: aws-actions/amazon-ecr-login@183a1442edf41672e66566b7fc560e297a290896 # v2.1.1 with: registries: ${{ steps.dockerfile-ecr-parse.outputs.ecr-registries }} - name: Login to ECR for publishing if: ${{ inputs.docker-push == 'true' }} id: login-ecr - uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 + uses: aws-actions/amazon-ecr-login@183a1442edf41672e66566b7fc560e297a290896 # v2.1.1 with: registry-type: >- ${{ @@ -221,13 +221,13 @@ runs: registries: ${{ inputs.aws-account-number }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v3.9.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 with: version: latest - name: Docker meta id: docker-meta - uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1 + uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 with: images: ${{ format('{0}/{1}', inputs.docker-registry-url, diff --git a/actions/ctf-build-image/action.yml b/actions/ctf-build-image/action.yml index 5493da3a0..f132b795d 100644 --- a/actions/ctf-build-image/action.yml +++ b/actions/ctf-build-image/action.yml @@ -108,6 +108,21 @@ inputs: go get github.com/smartcontractkit/chainlink-solana@abc123 go get github.com/smartcontractkit/chainlink-evm@def456 + cache-scope: + required: false + description: | + Custom scope for Docker build cache. Separates caches when multiple + Dockerfiles build on the same architecture (e.g. "core" vs "plugins"). + If not set, defaults to runner OS and architecture. + default: "" + + free-disk-space: + required: false + description: | + Whether to run the free-disk-space step before building. Set to "false" + on runners with sufficient disk (e.g. RunsOn with 100GB+) to save ~30-60s. + default: "true" + outputs: docker-image-sha-digest-amd64: description: "Docker image SHA digest for platform: amd64" @@ -119,12 +134,6 @@ outputs: runs: using: composite steps: - - uses: actions/setup-go@v6 - with: - go-version-file: "go.mod" - check-latest: true - cache: false - - name: Setup GitHub token using GATI if: inputs.gati-role-arn != '' && inputs.gati-lambda-url != '' id: github-token @@ -136,13 +145,6 @@ runs: aws-role-duration-seconds: "1800" set-git-config: "true" - - name: Process go get overrides - shell: bash - env: - GO_OVERRIDES: ${{ inputs.go-get-overrides }} - ACTIONS_PATH: ${{ github.action_path }} - run: ${ACTIONS_PATH}/scripts/go-get-overrides.sh - - name: Process plugin manifest overrides (public) shell: bash env: @@ -152,18 +154,52 @@ runs: ACTIONS_PATH: ${{ github.action_path }} run: ${ACTIONS_PATH}/scripts/plugin-overrides.sh - - name: Tidy and Output go.mod + - name: Setup Go for dependency overrides + if: inputs.go-get-overrides != '' + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: false + + - name: Apply go-get dependency overrides + if: inputs.go-get-overrides != '' shell: bash + env: + GO_OVERRIDES: ${{ inputs.go-get-overrides }} run: | - echo "::group::Tidy go.mod" + set -e + while IFS= read -r line; do + [ -z "$line" ] && continue + dep="${line%%=*}" + sha="${line#*=}" + [ -z "$dep" ] || [ -z "$sha" ] && continue + echo "Overriding: github.com/smartcontractkit/${dep}@${sha}" + go get "github.com/smartcontractkit/${dep}@${sha}" + done <<< "$GO_OVERRIDES" go mod tidy - echo "::endgroup::" - echo "::group::cat go.mod" - cat go.mod - echo "::endgroup::" + - name: Compute remote plugin cache key + id: plugin-cache + shell: bash + run: | + HASH=$(cat \ + plugins/plugins.public.yaml \ + plugins/plugins.private.yaml \ + plugins/plugins.testing.yaml \ + plugins/scripts/* \ + | sha256sum | cut -d' ' -f1) + echo "key=remote-plugins-${HASH}" >> "$GITHUB_OUTPUT" + mkdir -p .plugin-cache + + - name: Restore cached remote plugin binaries + id: plugin-cache-restore + uses: actions/cache/restore@v5 + with: + key: ${{ steps.plugin-cache.outputs.key }} + path: .plugin-cache/ - name: Free up disk space (to avoid 'no space left on device' errors) + if: inputs.free-disk-space == 'true' uses: smartcontractkit/.github/actions/free-disk-space@free-disk-space/v1 - name: Build push docker image @@ -178,22 +214,26 @@ runs: COMMIT_SHA=${{ github.sha }} CHAINLINK_USER=chainlink ${{ inputs.docker-additional-build-args }} + docker-build-contexts: >- + ${{ steps.plugin-cache-restore.outputs.cache-hit == 'true' + && 'build-remote-plugins=.plugin-cache/' + || '' }} docker-attestations: "false" docker-registry-url: ${{ inputs.docker-registry-url }} docker-repository-name: ${{ inputs.docker-repository-name }} # only save on events which are expected to be from the default branch - docker-save-cache: - ${{ github.event_name == 'schedule' || github.event_name == 'push' }} + docker-save-cache: ${{ github.event_name == 'schedule' || + github.event_name == 'push' || github.event_name == 'pull_request' }} # TODO: Remove pull_request after testing # dont use cache on events which are expected to be from the default branch # this is to create a fresh cache/snapshot unpolluted by previous cache entries docker-restore-cache: ${{ github.event_name != 'schedule' && github.event_name != 'push' }} docker-build-cache-to: - "type=gha,timeout=10m,mode=min,ignore-error=true,scope=ctf-build-image-${{ - runner.os }}-${{ runner.arch }}" + "type=gha,timeout=10m,mode=max,ignore-error=true,compression=zstd,compression-level=3,scope=ctf-build-image-${{ + inputs.cache-scope || format('{0}-{1}', runner.os, runner.arch) }}" docker-build-cache-from: - "type=gha,timeout=10m,scope=ctf-build-image-${{ runner.os }}-${{ - runner.arch }}" + "type=gha,timeout=10m,scope=ctf-build-image-${{ inputs.cache-scope || + format('{0}-{1}', runner.os, runner.arch) }}" tags: type=raw,value=${{ inputs.image-tag }} aws-account-number: ${{ inputs.aws-account-number }} @@ -201,3 +241,26 @@ runs: aws-region: ${{ inputs.aws-region }} github-token: ${{ steps.github-token.outputs.access-token || '' }} + + - name: Extract remote plugin binaries for caching + if: steps.plugin-cache-restore.outputs.cache-hit != 'true' && + (github.event_name == 'schedule' || github.event_name == 'push' || + github.event_name == 'pull_request') # TODO: Remove pull_request after testing + shell: bash + env: + DOCKERFILE: ${{ inputs.dockerfile }} + run: | + echo "Extracting remote plugin binaries for caching..." + docker buildx build \ + --target export-remote-plugins \ + --output type=local,dest=.plugin-cache \ + -f "$DOCKERFILE" . + + - name: Save remote plugin cache + if: steps.plugin-cache-restore.outputs.cache-hit != 'true' && + (github.event_name == 'schedule' || github.event_name == 'push' || + github.event_name == 'pull_request') # TODO: Remove pull_request after testing + uses: actions/cache/save@v5 + with: + key: ${{ steps.plugin-cache.outputs.key }} + path: .plugin-cache/ diff --git a/actions/ctf-build-image/scripts/go-get-overrides.sh b/actions/ctf-build-image/scripts/go-get-overrides.sh deleted file mode 100755 index 888c221b8..000000000 --- a/actions/ctf-build-image/scripts/go-get-overrides.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -set -e - -# Check if running in dry run mode -DRY_RUN=${DRY_RUN:-false} - -# Check for dependencies -if ! command -v go &> /dev/null; then - echo "::error::'go' command not found." - exit 1 -fi - -# Validate environment variables -if [[ -z "${GO_OVERRIDES}" ]]; then - echo "::info:: No go get overrides specified, skipping." - exit 0 -fi - -echo "::info:: Processing go get overrides..." - -while IFS= read -r line || [[ -n "$line" ]]; do - # Skip empty lines - [[ -z "$line" ]] && continue - - # Check if line contains '=' character - if [[ "$line" != *"="* ]]; then - echo "::warning::Invalid format for line '$line', expected 'dependency=sha', skipping." - continue - fi - - # Extract dependency name and SHA - dependency="${line%%=*}" - sha="${line#*=}" - - # Skip if SHA is empty - if [[ -z "$sha" ]]; then - echo "::warning::Empty SHA for dependency $dependency, skipping." - continue - fi - - echo "::info:: Replacing $dependency dependency with SHA: $sha" - - if [[ "$DRY_RUN" == "true" ]]; then - echo "[DRY RUN] Would execute: go get \"github.com/smartcontractkit/${dependency}@${sha}\"" - else - # Set GOPRIVATE if provided - if [[ -n "${GOPRIVATE}" ]]; then - export GOPRIVATE="${GOPRIVATE}" - fi - - go get "github.com/smartcontractkit/${dependency}@${sha}" || { - echo "Error: Failed to get dependency github.com/smartcontractkit/${dependency}@${sha}" - exit 1 - } - echo "::info::Successfully updated ${dependency} to ${sha}" - fi -done <<< "$GO_OVERRIDES" - -echo "Go get overrides processing completed."