pullrequest.yml

name: Monorepo pipeline - pull request

on:
  pull_request: {}
  workflow_dispatch: {}

defaults:
  run:
    shell: bash

concurrency:
  # See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-using-a-fallback-value
  group: pullrequest-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

env:
  COMPOSE_HTTP_TIMEOUT: 180
  SKIP_GENERATED_CACHE: ${{ contains(github.event.pull_request.labels.*.name, 'skip-generated-cache') }}
  NX_AFFECTED_ALL: ${{ contains(github.event.pull_request.labels.*.name, 'nx-affected-all') }}
  NX_SKIP_NX_CACHE: ${{ contains(github.event.pull_request.labels.*.name, 'skip-nx-cache') }}
  DISABLE_CHUNKS: 'true'
  DISABLE_GROUPING: 'false'
  DISABLE_PROBLEMATIC: 'false'
  CHUNK_SIZE: '8'
  MAX_JOBS: '3'
  NX_PARALLEL: '2'
  NX_MAX_PARALLEL: '4'
  NX_TASKS_RUNNER: ci
  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
  S3_DOCKER_CACHE_BUCKET: ${{ secrets.S3_DOCKER_CACHE_BUCKET }}
  RUNS_ON_S3_BUCKET_CACHE: ${{ secrets.S3_DOCKER_CACHE_BUCKET }}
  AWS_REGION: eu-west-1
  GENERATED_FILES: ${{ github.sha }}.tar.gz
  AFFECTED_ALL: ${{ secrets.AFFECTED_ALL }}
  SERVERSIDE_FEATURES_ON: ''
  DOCKER_REGISTRY: 821090935708.dkr.ecr.eu-west-1.amazonaws.com
  DOCKER_BASE_IMAGE_REGISTRY: 821090935708.dkr.ecr.eu-west-1.amazonaws.com/ecr-public
  CI_DEBUG: ${{ contains(github.event.pull_request.labels.*.name, 'ci debug') }}
  SKIP_TESTS: ${{ contains(github.event.pull_request.labels.*.name, 'test nothing') }}
  DEPLOY_FEATURE: ${{ contains(github.event.pull_request.labels.*.name, 'deploy-feature') }}
  CYPRESS_CACHE_FOLDER: ${{ github.workspace }}/.cypress-cache
  YARN_ENABLE_HARDENED_MODE: '0'

jobs:
  prepare:
    uses: ./.github/workflows/install.yml
    secrets: inherit
    with:
      main_branch: ${{ github.base_ref }}

  tests:
    needs:
      - prepare
    if: ${{ needs.prepare.outputs.TEST_CHUNKS && needs.prepare.outputs.DEPLOY_FEATURE == 'false' }}
    runs-on: arc-shared
    timeout-minutes: 45
    env:
      AFFECTED_PROJECTS: ${{ matrix.projects }}
      NX_BASE: ${{ needs.prepare.outputs.NX_BASE }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.prepare.outputs.TEST_CHUNKS) }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Setup yarn
        uses: ./.github/actions/setup-yarn
        with:
          RUNS_ON_S3_BUCKET_CACHE: ${{ secrets.S3_DOCKER_CACHE_BUCKET }}

      - name: load-deps
        uses: ./.github/actions/load-deps
      - name: Run unit tests
        uses: ./.github/actions/unit-test
        with:
          dd-api-key: '${{ secrets.DD_API_KEY }}'
          codecov-token: ${{ secrets.CODECOV_TOKEN }}
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          docker-registry: 821090935708.dkr.ecr.eu-west-1.amazonaws.com/

  unicorn-tests:
    name: All projects have some tests
    needs:
      - prepare
    if: ${{ needs.prepare.outputs.TEST_CHUNKS && needs.prepare.outputs.DEPLOY_FEATURE == 'false' && needs.prepare.outputs.is-unicorn == 'true' }}
    runs-on: arc-shared
    env:
      CHUNKS: ${{ needs.prepare.outputs.TEST_CHUNKS }}
      NX_BASE: ${{ needs.prepare.outputs.NX_BASE }}
      NX_HEAD: ${{ needs.prepare.outputs.NX_HEAD }}
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          # Need full repo for Nx to find the commits at NX_* below
          fetch-depth: 0

      - name: Setup yarn
        uses: ./.github/actions/setup-yarn
        with:
          RUNS_ON_S3_BUCKET_CACHE: ${{ secrets.S3_DOCKER_CACHE_BUCKET }}

      - name: load-deps
        uses: ./.github/actions/load-deps

      - name: Find unicorn projects
        id: unicorn
        run: |
          unicorns=$(node scripts/ci/unicorn-utils.mjs show-unicorns --json)
          jq -ncr --argjson chunks "$CHUNKS" --argjson unicorns "$unicorns" '
            [
                $chunks.projects
              | .[]
              | select( . | IN( $unicorns | .[] ) )
            ]
            | "projects=\(. | join(","))"
          ' | tee -a "$GITHUB_OUTPUT"

      - name: Verify that projects have some tests
        env:
          projects: ${{ steps.unicorn.outputs.projects }}
        run: |
          # Get list of tests, with `<project-name>: ` prefixed (stream style, without cache), grouping by unique prefixes (sorted)
          yarn nx run-many -t test --skip-nx-cache --output-style=stream --projects="${projects}" --parallel --verbose -- --listTests |
            grep -P "^(${projects//,/|}): " | cut -d':' -f1 | sort -u > /tmp/projects-with-tests
          # Get affected projects sorted
          echo -e "${projects}" | sed 's/,/\n/g' | sort -u > /tmp/projects-all
          if ! d=$(diff /tmp/projects-with-tests /tmp/projects-all); then
            echo "These projects don't have any tests:"
            echo "${d}"
            exit 1
          fi
          echo "All projects ($projects) have tests."

  e2e:
    needs:
      - prepare
    if: ${{ needs.prepare.outputs.E2E_CHUNKS && needs.prepare.outputs.DEPLOY_FEATURE == 'false' }}
    runs-on: arc-shared
    timeout-minutes: 45
    env:
      AFFECTED_PROJECT: ${{ matrix.projects }}
      CYPRESS_PROJECT_ID: 4q7jz8
      CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
      API_MOCKS: 'true'
      NODE_OPTIONS: --max-old-space-size=4096
      E2E_BUILD_ID: '${{ needs.prepare.outputs.E2E_BUILD_ID }}-${{ github.run_attempt }}'
      NX_BASE: ${{ needs.prepare.outputs.NX_BASE }}
      AFFECTED_PROJECTS: ${{ matrix.projects }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.prepare.outputs.E2E_CHUNKS) }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Setup yarn
        uses: ./.github/actions/setup-yarn
        with:
          RUNS_ON_S3_BUCKET_CACHE: ${{ secrets.S3_DOCKER_CACHE_BUCKET }}

      - name: load-deps
        uses: ./.github/actions/load-deps

      - name: Running e2e tests
        run: yarn nx run-many --target=e2e --projects="${AFFECTED_PROJECTS}" --parallel 3

  linting-workspace:
    needs:
      - prepare
    runs-on: arc-shared
    timeout-minutes: 15
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Setup yarn
        uses: ./.github/actions/setup-yarn
        with:
          RUNS_ON_S3_BUCKET_CACHE: ${{ secrets.S3_DOCKER_CACHE_BUCKET }}

      - name: load-deps
        uses: ./.github/actions/load-deps

      - name: Linting workspace
        run: ./scripts/ci/20_lint-workspace.sh

  run-shellcheck:
    needs:
      - prepare
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@v4
      - uses: reviewdog/action-shellcheck@v1
        with:
          github_token: ${{ secrets.github_token }}
          reporter: github-pr-review
          fail_level: info
          exclude: >-
            */node_modules/*

  formatting:
    needs:
      - prepare
    runs-on: arc-shared
    timeout-minutes: 15
    env:
      NX_BASE: ${{ needs.prepare.outputs.NX_BASE }}
    steps:
      - uses: actions/checkout@v4
        if: ${{ github.event_name == 'pull_request' }}
        with:
          token: ${{ secrets.DIRTY_FIX_BOT_TOKEN }}
          ref: ${{ github.event.pull_request.head.ref }}

      - name: Setup yarn
        uses: ./.github/actions/setup-yarn
        with:
          RUNS_ON_S3_BUCKET_CACHE: ${{ secrets.S3_DOCKER_CACHE_BUCKET }}

      - name: load-deps
        uses: ./.github/actions/load-deps

      - name: NX format:check
        if: ${{ github.ref == 'ref/heads/main' }}
        run: ./scripts/ci/20_check-formatting.sh "check"

      # - name: NX format:write
      #   if: ${{ github.event_name == 'pull_request' }}
      #   run: |
      #     ./scripts/ci/20_check-formatting.sh "write"
      #     ./infra/scripts/ci/git-check-dirty.sh "/" "nx format:write" "dirtybot"

  lintfix-changed-files:
    name: Lintfix Changed Files
    runs-on: arc-shared
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'dirty bypass') }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          # Need full repo for Nx to find the commits at NX_* below
          fetch-depth: 0
          token: ${{ secrets.DIRTY_FIX_BOT_TOKEN }}
          # Need ref to know where to push to
          # https://github.com/actions/checkout#push-a-commit-to-a-pr-using-the-built-in-token
          ref: ${{ github.event.pull_request.head.ref }}

      - name: Setup yarn
        uses: ./.github/actions/setup-yarn
        with:
          RUNS_ON_S3_BUCKET_CACHE: ${{ secrets.S3_DOCKER_CACHE_BUCKET }}

      - name: Run lintfix
        id: lintfix
        # NOTE: We want to use the PR base, not last good, since the lint is not part of "good" build
        # and causes unintended lint changes when using a different commit
        run: |
          yarn nx affected -t lint --base="origin/$GITHUB_BASE_REF" --fix
          if ! git diff "$GITHUB_BASE_REF" --exit-code; then
            echo "FILES_CHANGED=true" | tee -a "$GITHUB_OUTPUT"
          fi

      - name: Commit changed files
        if: ${{ steps.lintfix.outputs.FILES_CHANGED == 'true' }}
        run: ./infra/scripts/ci/git-check-dirty.sh . lintfix dirtybot

  linting:
    needs:
      - prepare
    runs-on: arc-shared
    timeout-minutes: 35
    if: needs.prepare.outputs.LINT_CHUNKS
    env:
      AFFECTED_PROJECTS: ${{ matrix.projects }}
      NODE_OPTIONS: --max-old-space-size=4096
      NX_BASE: ${{ needs.prepare.outputs.NX_BASE }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.prepare.outputs.LINT_CHUNKS) }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Setup yarn
        uses: ./.github/actions/setup-yarn
        with:
          RUNS_ON_S3_BUCKET_CACHE: ${{ secrets.S3_DOCKER_CACHE_BUCKET }}

      - name: load-deps
        uses: ./.github/actions/load-deps

      - name: Lint head
        id: lint-head
        run: |
          set -euo pipefail
          lints=$(mktemp -t lint-head.XXXXXX)
          echo "lints-file=$lints" | tee -a "$GITHUB_OUTPUT"
          yarn nx run-many -t lint --output-style=static --projects "$AFFECTED_PROJECTS" |
            tee -a "$lints" | # Show output when running, but preserve pipe-chaining
            (grep -oP '(?<=✖ )(\d+)(?= problems)' || echo '0') | # Find number of problems
            awk '{ SUM += $1} END { SUM /= 2; printf "lint-count=%d", SUM }' | # sum number of problems, divide by 2 because each problem summary occurs twice
            tee -a "$GITHUB_OUTPUT" # Finally create output

      # Checkout for linting base
      - uses: actions/checkout@v4
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'dirty bypass') }}
        with:
          ref: ${{ github.event.pull_request.base.ref }}
          clean: false

      - name: Lint base
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'dirty bypass') }}
        id: lint-base
        run: |
          set -euo pipefail
          lints=$(mktemp -t lint-base.XXXXXX)
          echo "lints-file=$lints" | tee -a "$GITHUB_OUTPUT"
          yarn nx run-many -t lint --output-style=static --projects "$AFFECTED_PROJECTS" |
            tee -a "$lints" | # Show output when running, but preserve pipe-chaining
            (grep -oP '(?<=✖ )(\d+)(?= problems)' || echo '0') | # Find number of problems
            awk '{ SUM += $1} END { SUM /= 2; printf "lint-count=%d", SUM }' | # sum number of problems, divide by 2 because each problem summary occurs twice
            tee -a "$GITHUB_OUTPUT" # Finally create output

      - name: No new lints
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'dirty bypass') }}
        env:
          HEAD_LINT_COUNT: ${{steps.lint-head.outputs.lint-count}}
          BASE_LINT_COUNT: ${{steps.lint-base.outputs.lint-count}}
          HEAD_LINT_FILE: ${{steps.lint-head.outputs.lints-file}}
          BASE_LINT_FILE: ${{steps.lint-base.outputs.lints-file}}
        run: |
          # Get `<filename> #lints: <n>` lines for easy diff
          for f in "${BASE_LINT_FILE}" "${HEAD_LINT_FILE}"; do
            # Make "Remote cache..." and "Local cache" be the same
            sed -i -E 's/\s*\[[^[]*cache[^]]*\]\s*//g' "${f}" || echo "No cached targets found in $f"
            grep -oP "(?<=$PWD/).*" "${f}" | sort -n | uniq -c | awk '{print($2 " #lints: " $1)}' > "${f}_files.out" || { echo "No lints found in $f"; }
          done
          NEW_LINTS=$(comm -13 "${BASE_LINT_FILE}_files.out" "${HEAD_LINT_FILE}_files.out" | wc -l) || { echo "Failed comparing output files"; exit 1; }
          COUNT_DIFF=$(( HEAD_LINT_COUNT - BASE_LINT_COUNT )) || :
          if (( COUNT_DIFF <= 0 && NEW_LINTS == 0 )); then
            echo "Lint count has not increased (𝚫=$COUNT_DIFF)"
            exit 0
          fi
          echo "::error title=NEW_LINTS::Please fix your lints 🙏 (found $NEW_LINTS additional lints)"
          echo "Files with changed lints:"
          diff "${BASE_LINT_FILE}_files.out" "${HEAD_LINT_FILE}_files.out" || :
          echo "::group::lintdiff"
          diff "${BASE_LINT_FILE}" "${HEAD_LINT_FILE}" || : # Ignore diff exit code
          echo "::endgroup::"
          exit 1

  build:
    needs:
      - prepare
    runs-on: arc-shared
    timeout-minutes: 35
    env:
      AFFECTED_PROJECTS: ${{ matrix.projects }}
      NX_BASE: ${{ needs.prepare.outputs.NX_BASE }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.prepare.outputs.BUILD_CHUNKS) }}
    if: ${{ needs.prepare.outputs.BUILD_CHUNKS && needs.prepare.outputs.DEPLOY_FEATURE == 'false' }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Setup yarn
        uses: ./.github/actions/setup-yarn
        with:
          RUNS_ON_S3_BUCKET_CACHE: ${{ secrets.S3_DOCKER_CACHE_BUCKET }}

      - name: load-deps
        uses: ./.github/actions/load-deps

      - name: Building
        run: ./scripts/ci/run-in-parallel-native.sh build

  success:
    runs-on: arc-shared
    if: ${{ !cancelled() }}
    needs:
      - prepare
      - linting-workspace
      - tests
      - linting
      - run-shellcheck
      - formatting
      - e2e
      - build
      - unicorn-tests
    steps:
      - name: Notify about deploy-feature label
        if: ${{ github.event.pull_request.auto_merge && needs.prepare.outputs.DEPLOY_FEATURE == 'true' }}
        run: |
          echo "- 👉 Remove the \`deploy-feature\` label if you want this PR to be merged automatically" >> $GITHUB_STEP_SUMMARY
          exit 1

      - name: Collect
        env:
          NEEDS: ${{ toJSON(needs) }}
        run: |
          # "failed" here is everything that wasn't successful or skipped (e.g. "failuer", "cancelled")
          failed_dependencies=$(jq -nr --argjson needs "$NEEDS" ' $needs | with_entries(select(.value.result != "success" and .value.result != "skipped")) ')
          echo -n "Did any dependency fail? "
          if jq -ne --argjson failures "$failed_dependencies" '$failures | length > 0'; then
            failure_statuses=$(jq -nr --argjson failures "$failed_dependencies" '[ $failures | to_entries | .[] | "\(.key) (\(.value.result))" ] | join(", ")')
            echo "::error title=NEEDS::Some dependencies failed: $failure_statuses"
            exit 1
          fi
          echo "::info title=NEEDS::No failing dependencies 🎉"
          exit 0

      - name: Announce success
        run: echo "Build is successful"