diff --git a/.github/workflows/01-make-dist-worker.yml b/.github/workflows/01-make-dist-worker.yml new file mode 100644 index 0000000..207e286 --- /dev/null +++ b/.github/workflows/01-make-dist-worker.yml @@ -0,0 +1,191 @@ +# Adapted from NUT 01-make-dist-worker.yml +# Separated from 01-make-dist.yml to touch untrusted sources +# in a separate job without particular permissions to anything. +# Triggered by a step in that job. +# +# See also: +# https://github.com/actions/upload-artifact +# https://docs.github.com/en/actions/reference/workflows-and-actions/variables +# https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax + +name: "GHA-01: Tarballs Build Worker" + +on: + workflow_call: + inputs: + ref: + required: true + type: string + artifact_name: + required: true + type: string + workflow_dispatch: + inputs: + ref: + required: true + type: string + artifact_name: + required: true + type: string + +# This untrusted part of the job may only read Git +# (and upload artifacts, that's allowed by default) +permissions: + contents: read + +jobs: + make-dist-tarballs: + name: "Make Dist and Docs Tarballs, see workflow page for links" + # FIXME: Prepare/maintain a container image with pre-installed + # WMNut build/tooling prereqs (save about 3 minutes per run!) + # Maybe https://aschmelyun.com/blog/using-docker-run-inside-of-github-actions/ + # => https://github.com/addnab/docker-run-action can help + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + # NOTE: This is the unprivileged part of the GHA-01-tarballs ritual, + # running potentially untrusted shell code from the PR source branch. + # We are still dealing with the upstream repository (not a forked + # repo), and in case of pull requests - fetch a separate scheme of + # ephemerally provided refs for PR "merge" or "head" commits. + # + # 0 => all commit history for the PR + fetch-depth: 0 + # Get the git tags to construct NUT SEMVER version strings. + fetch-tags: true + ref: ${{ inputs.ref }} + persist-credentials: false + + # Make build identification more useful (so we use no fallbacks in script) + - name: Try to get more Git metadata + run: | + git describe || { + git remote -v || true + git branch -a || true + for R in `git remote` ; do git fetch $R master ; done || true + git fetch --tags + pwd ; ls -la + echo "=== Known commits in history:" + git log --oneline | wc -l + echo "=== Recent commits in history:" + git log -2 || true + echo "=== Known tags:" + git tag || true + echo "=== Try to ensure 'git describe' works:" + git describe || { + git fetch --all && for R in `git remote` ; do for T in `git tag` ; do git fetch $R $T ; done ; done + git describe || { + TEST_REF="`git symbolic-ref --short HEAD 2>/dev/null || cat .git/HEAD`" && [ -n "${TEST_REF}" ] && git checkout master && git pull --all && git checkout "${TEST_REF}" + git describe || true + } + } + } + + # Using hints from https://askubuntu.com/questions/272248/processing-triggers-for-man-db + # and our own docs/config-prereqs.txt + # NOTE: Currently installing the MAX prerequisite footprint, + # which for building just the docs may be a bit of an overkill. + - name: WMNut CI Prerequisite packages (Ubuntu, GCC) + run: | + echo "set man-db/auto-update false" | sudo debconf-communicate + sudo dpkg-reconfigure man-db + sudo apt update + sudo apt install \ + gcc g++ clang \ + libxpm-dev libxext-dev libupsclient-dev libc6-dev-amd64-cross libgcc-s1-amd64-cross ccache \ + || exit + date > .timestamp-init + + - name: Prepare ccache + # Based on https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#example-using-the-cache-action example + id: cache-ccache + uses: actions/cache@v4 + env: + compiler: 'CC=gcc CXX=g++' + cache-name: cache-ccache-${{ env.compiler }} + with: + path: | + ~/.ccache + ~/.cache/ccache + ~/.config/ccache/ccache.conf + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/.timestamp-init') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}- + ${{ runner.os }}-build- + ${{ runner.os }}- + + - name: CCache stats before build + run: | + ccache -sv || ccache -s || echo "FAILED to read ccache info, oh well" + rm -f .timestamp-init + + #- name: Debug gitlog2version processing + # run: bash -x ./tools/gitlog2version.sh || true + + - name: WMNut CI Build Configuration + env: + compiler: 'CC=gcc CXX=g++' + run: | + PATH="/usr/lib/ccache:$PATH" ; export PATH + CCACHE_COMPRESS=true; export CCACHE_COMPRESS + ccache --version || true + ( ${{env.compiler}} ; echo "=== CC: $CC => `command -v $CC` =>" ; $CC --version ; echo "=== CXX: $CXX => `command -v $CXX` =>" ; $CXX --version ) || true + ./autogen.sh && \ + ./configure ${{env.compiler}} --enable-debug --enable-Werror + + # NOTE: In this scenario we do not build actually WMNut in the main + # checkout directory, at least not explicitly (recipe may generate + # some files like man pages to fulfill the "dist" requirements; + # for now this may generate some libs to figure out their IDs). + # We do `make docs` to provide them as a separate tarball just + # in case, later. + # DO NOT `make dist-files` here as it includes `dist-sig` and + # needs a GPG keychain with maintainers' secrets deployed locally. + - name: WMNut CI Build to create "dist" tarball and related files + env: + compiler: 'CC=gcc CXX=g++' + run: | + PATH="/usr/lib/ccache:$PATH" ; export PATH + CCACHE_COMPRESS=true; export CCACHE_COMPRESS + ccache --version || true + ( ${{env.compiler}} ; echo "=== CC: $CC => `command -v $CC` =>" ; $CC --version ; echo "=== CXX: $CXX => `command -v $CXX` =>" ; $CXX --version ) || true + make -s -j 8 dist dist-hash + + - name: WMNut CI Build to verify "dist" tarball build + env: + compiler: 'CC=gcc CXX=g++' + run: | + PATH="/usr/lib/ccache:$PATH" ; export PATH + CCACHE_COMPRESS=true; export CCACHE_COMPRESS + ccache --version || true + ( ${{env.compiler}} ; echo "=== CC: $CC => `command -v $CC` =>" ; $CC --version ; echo "=== CXX: $CXX => `command -v $CXX` =>" ; $CXX --version ) || true + make -s -j 8 distcheck + + - name: WMNut CI Build to verify "dist" tarball build self-reproducibility + env: + compiler: 'CC=gcc CXX=g++' + run: | + PATH="/usr/lib/ccache:$PATH" ; export PATH + CCACHE_COMPRESS=true; export CCACHE_COMPRESS + ccache --version || true + ( ${{env.compiler}} ; echo "=== CC: $CC => `command -v $CC` =>" ; $CC --version ; echo "=== CXX: $CXX => `command -v $CXX` =>" ; $CXX --version ) || true + make -s -j 8 distcheck-completeness + + - name: CCache stats after distcheck + run: ccache -sv || ccache -s || echo "FAILED to read ccache info, oh well" + + # NOTE: A `.zip` is added to the base `$name` automatically + - name: Upload tarball and its checksum artifacts + uses: actions/upload-artifact@v4 + id: upload_artifact + with: + name: ${{ inputs.artifact_name }} + path: | + wmnut-*.tar* + compression-level: 0 + overwrite: true + + # FINISH: Pass control back to the main pipeline diff --git a/.github/workflows/01-make-dist.yml b/.github/workflows/01-make-dist.yml index f54c7b0..75d54e9 100644 --- a/.github/workflows/01-make-dist.yml +++ b/.github/workflows/01-make-dist.yml @@ -1,11 +1,18 @@ # Adapted from NUT 01-make-dist.yml with inspiration taken from # https://javahelps.com/manage-github-artifact-storage-quota # regarding uploads of artifacts and clearing the way for them. +# +# This workflow is triggered on pushes to a repository (PRs, +# branch increments, or tags). It has higher permissions to +# perform management of comments and checks, and launches a +# separate "Worker" workflow for tarball generation touching +# possibly untrusted code base. +# # See also: # https://github.com/actions/upload-artifact # https://docs.github.com/en/actions/reference/workflows-and-actions/variables # https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax -#name: "GHA-01: Make dist and docs tarballs, see workflow page for links" + name: "GHA-01: Tarballs" on: @@ -28,42 +35,21 @@ on: permissions: checks: write - contents: write + actions: write issues: write pull-requests: write jobs: - make-dist-tarballs: - name: "Make Dist and Docs Tarballs, see workflow page for links" - # FIXME: Prepare/maintain a container image with pre-installed - # WMNut build/tooling prereqs (save about 3 minutes per run!) - # Maybe https://aschmelyun.com/blog/using-docker-run-inside-of-github-actions/ - # => https://github.com/addnab/docker-run-action can help + arrange-dist-tarballs: + name: "Arrange making Dist and Docs Tarballs, see workflow page for links" runs-on: ubuntu-latest - strategy: - fail-fast: false - steps: - - name: Checkout repository - uses: actions/checkout@v5 - with: - fetch-depth: 0 - fetch-tags: true - # NOTE: pull_request_target protects us by using the workflow definition - # from target branch (so trusted operations like artifact management can - # be used), but it also uses the code from that branch too, so fix it back. - # Maybe use "github.event.pull_request.merge_commit_sha" instead?.. though - # it may be null if the PR is not mergeable or is still checked for that, - # or may return the commit for PREVIOUS pushed iteration's merge to target. - # See more ideas in https://github.com/actions/checkout/issues/518 : - ref: "${{ github.event.pull_request.number > 0 && github.event.pull_request.head.sha || github.ref_name }}" - persist-credentials: false # https://github.com/marketplace/actions/substitute-string # See also examples in https://github.com/dhimmel/dump-actions-context/ # Note it warns about "unexpected input(s)" with replacement tokens below, - # as they are by design not predefined, as far as actions API is concened. + # as they are by design not predefined, as far as actions API is concerned. # They still work for substitutions though. - uses: bluwy/substitute-string-action@v3 id: subst-github-ref-name @@ -72,6 +58,10 @@ jobs: " ": _ "/": _ + - name: Define Artifact Name + id: set-artifact-name + run: echo "artifact_name=WMNUT-tarballs-${{ steps.subst-github-ref-name.outputs.result }}" >> "$GITHUB_OUTPUT" + - name: Debug PR/branch identification run: | echo "steps.subst-github-ref-name.outputs.result='${{ steps.subst-github-ref-name.outputs.result }}'" || true @@ -81,125 +71,36 @@ jobs: echo "github.ref='${{ github.ref }}'" || true echo "github.ref_name='${{ github.ref_name }}'" || true - # Make build identification more useful (so we use no fallbacks in script) - - name: Try to get more Git metadata - run: | - git describe || { - git remote -v || true - git branch -a || true - for R in `git remote` ; do git fetch $R master ; done || true - git fetch --tags - pwd ; ls -la - echo "=== Known commits in history:" - git log --oneline | wc -l - echo "=== Recent commits in history:" - git log -2 || true - echo "=== Known tags:" - git tag || true - echo "=== Try to ensure 'git describe' works:" - git describe || { - git fetch --all && for R in `git remote` ; do for T in `git tag` ; do git fetch $R $T ; done ; done - git describe || { - TEST_REF="`git symbolic-ref --short HEAD 2>/dev/null || cat .git/HEAD`" && [ -n "${TEST_REF}" ] && git checkout master && git pull --all && git checkout "${TEST_REF}" - git describe || true - } - } - } - - # Using hints from https://askubuntu.com/questions/272248/processing-triggers-for-man-db - # and our own docs/config-prereqs.txt - # NOTE: Currently installing the MAX prerequisite footprint, - # which for building just the docs may be a bit of an overkill. - - name: WMNut CI Prerequisite packages (Ubuntu, GCC) - run: | - echo "set man-db/auto-update false" | sudo debconf-communicate - sudo dpkg-reconfigure man-db - sudo apt update - sudo apt install \ - gcc g++ clang \ - libxpm-dev libxext-dev libupsclient-dev libc6-dev-amd64-cross libgcc-s1-amd64-cross ccache \ - || exit - date > .timestamp-init - - - name: Prepare ccache - # Based on https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#example-using-the-cache-action example - id: cache-ccache - uses: actions/cache@v4 + - name: "GHA-01: Create/Update new GH PR comment for future link" + if: github.event.pull_request.number > 0 + continue-on-error: true env: - compiler: 'CC=gcc CXX=g++' - cache-name: cache-ccache-${{ env.compiler }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: thollander/actions-comment-pull-request@v3 with: - path: | - ~/.ccache - ~/.cache/ccache - ~/.config/ccache/ccache.conf - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/.timestamp-init') }} - restore-keys: | - ${{ runner.os }}-build-${{ env.cache-name }}- - ${{ runner.os }}-build- - ${{ runner.os }}- - - - name: CCache stats before build - run: | - ccache -sv || ccache -s || echo "FAILED to read ccache info, oh well" - rm -f .timestamp-init - - #- name: Debug gitlog2version processing - # run: bash -x ./tools/gitlog2version.sh || true - - - name: WMNut CI Build Configuration - env: - compiler: 'CC=gcc CXX=g++' - run: | - PATH="/usr/lib/ccache:$PATH" ; export PATH - CCACHE_COMPRESS=true; export CCACHE_COMPRESS - ccache --version || true - ( ${{env.compiler}} ; echo "=== CC: $CC => `command -v $CC` =>" ; $CC --version ; echo "=== CXX: $CXX => `command -v $CXX` =>" ; $CXX --version ) || true - ./autogen.sh && \ - ./configure ${{env.compiler}} --enable-debug --enable-Werror - - # NOTE: In this scenario we do not build actually WMNut in the main - # checkout directory, at least not explicitly (recipe may generate - # some files like man pages to fulfill the "dist" requirements; - # for now this may generate some libs to figure out their IDs). - # We do `make docs` to provide them as a separate tarball just - # in case, later. - # DO NOT `make dist-files` here as it includes `dist-sig` and - # needs a GPG keychain with maintainers' secrets deployed locally. - - name: WMNut CI Build to create "dist" tarball and related files - env: - compiler: 'CC=gcc CXX=g++' - run: | - PATH="/usr/lib/ccache:$PATH" ; export PATH - CCACHE_COMPRESS=true; export CCACHE_COMPRESS - ccache --version || true - ( ${{env.compiler}} ; echo "=== CC: $CC => `command -v $CC` =>" ; $CC --version ; echo "=== CXX: $CXX => `command -v $CXX` =>" ; $CXX --version ) || true - make -s -j 8 dist dist-hash - - - name: WMNut CI Build to verify "dist" tarball build - env: - compiler: 'CC=gcc CXX=g++' - run: | - PATH="/usr/lib/ccache:$PATH" ; export PATH - CCACHE_COMPRESS=true; export CCACHE_COMPRESS - ccache --version || true - ( ${{env.compiler}} ; echo "=== CC: $CC => `command -v $CC` =>" ; $CC --version ; echo "=== CXX: $CXX => `command -v $CXX` =>" ; $CXX --version ) || true - make -s -j 8 distcheck - - - name: WMNut CI Build to verify "dist" tarball build self-reproducibility - env: - compiler: 'CC=gcc CXX=g++' - run: | - PATH="/usr/lib/ccache:$PATH" ; export PATH - CCACHE_COMPRESS=true; export CCACHE_COMPRESS - ccache --version || true - ( ${{env.compiler}} ; echo "=== CC: $CC => `command -v $CC` =>" ; $CC --version ; echo "=== CXX: $CXX => `command -v $CXX` =>" ; $CXX --version ) || true - make -s -j 8 distcheck-completeness - - - name: CCache stats after distcheck - run: ccache -sv || ccache -s || echo "FAILED to read ccache info, oh well" + message: | + Preparing a ZIP file with standard source tarball and another tarball with pre-built docs for commit ${{ env.ref }} ... + comment-tag: tarball-url + github-token: ${{ env.GITHUB_TOKEN }} + + # We do not necessarily trust any code that comes from the PR branch + # (nor whatever is produced by scripts and tools running there), + # so we run that in a separate sandbox and just publish a link to + # the results later. + - name: Launch Build Worker + id: launch_worker + uses: benc-uk/workflow-dispatch@v1 + with: + workflow: "GHA-01: Tarballs Build Worker" + token: ${{ secrets.GITHUB_TOKEN }} + inputs: '{ "ref": "${{ github.event.pull_request.number > 0 && github.event.pull_request.head.sha || github.sha }}", "artifact_name": "${{ steps.set-artifact-name.outputs.artifact_name }}" }' + wait-timeout-seconds: 3600 + wait-for-completion: true + sync-status: true # Inspired by https://javahelps.com/manage-github-artifact-storage-quota + # This is the part which needs "actions: write" permission. # Note that the code below wipes everything matched by the filter! # We may want another script block (after this cleanup of obsolete data) # to iterate clearing the way build by build until there's X MB available @@ -217,7 +118,7 @@ jobs: }) res.data.artifacts - .filter(({ name }) => name === 'WMNUT-tarballs-${{ steps.subst-github-ref-name.outputs.result }}') + .filter(({ name }) => name === '${{ steps.set-artifact-name.outputs.artifact_name }}') .forEach(({ id }) => { github.rest.actions.deleteArtifact({ owner: context.repo.owner, @@ -226,15 +127,46 @@ jobs: }) }) - - name: Upload tarball and its checksum artifacts - uses: actions/upload-artifact@v4 - id: upload_artifact + # The artifact was already uploaded by the "Worker" job. + # We need to find the artifact URL and report it below. + - name: Get Artifact URL + id: get_artifact_url + uses: actions/github-script@v6 + with: + script: | + const res = await github.rest.actions.listArtifactsForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + }) + const artifactName = '${{ steps.set-artifact-name.outputs.artifact_name }}'; + // Find the most recent artifact with this name + const artifacts = res.data.artifacts + .filter(a => a.name === artifactName) + .sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); + + if (artifacts.length > 0) { + const artifact = artifacts[0]; + var workflowRunId = artifact.workflow_run_id || ${{ steps.launch_worker.outputs.runId }}; + core.setOutput('artifact_url', `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${workflowRunId}/artifacts/${artifact.id}`); + core.setOutput('real_artifact_url', `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${workflowRunId}/artifacts/${artifact.id}`); + } else { + core.setFailed(`Artifact ${artifactName} not found`); + } + + - name: "GHA-01: Create/update new GH PR comment with link" + if: github.event.pull_request.number > 0 + continue-on-error: true + env: + artifact_name: "${{ steps.set-artifact-name.outputs.artifact_name }}.zip" + artifact_url: ${{ steps.get_artifact_url.outputs.real_artifact_url }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: thollander/actions-comment-pull-request@v3 with: - name: WMNUT-tarballs-${{ steps.subst-github-ref-name.outputs.result }} - path: | - wmnut-*.tar* - compression-level: 0 - overwrite: true + message: | + A ZIP file with standard source tarball and another tarball with pre-built docs for commit ${{ env.ref }} is temporarily available: [${{ env.artifact_name }}](${{ env.artifact_url }}). + comment-tag: tarball-url + github-token: ${{ env.GITHUB_TOKEN }} # NOTE: Despite the docs examples, due to GH API changes in Mar 2025 # this action can no longer update an existing check, only create a @@ -245,8 +177,8 @@ jobs: if: always() continue-on-error: true env: - artifact_name: "WMNUT-tarballs-${{ steps.subst-github-ref-name.outputs.result }}.zip" - artifact_url: ${{ steps.upload_artifact.outputs.artifact-url }} + artifact_name: "${{ steps.set-artifact-name.outputs.artifact_name }}.zip" + artifact_url: ${{ steps.get_artifact_url.outputs.real_artifact_url }} ref: ${{ github.event.pull_request.head.sha || github.sha }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: |