From 526466e034e8b33bd7e3f8130e1f031253e5e235 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 9 Apr 2025 15:05:11 +0200 Subject: [PATCH 01/14] github-actions: build images --- .github/workflows/build-images.yml | 41 ++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/build-images.yml diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml new file mode 100644 index 000000000..259575eba --- /dev/null +++ b/.github/workflows/build-images.yml @@ -0,0 +1,41 @@ +--- +# This workflow publishes docker images base on .ci/.docker-images.yml`. +# See docs/INTERNAL_DOCKER_IMAGES.md for further information. +name: publish-docker-images + +on: + workflow_dispatch: + +env: + DOCKER_BUILDKIT: 1 + +permissions: + contents: read + +jobs: + + build-test-push: + runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v4 + + - name: Login to docker.elastic.co + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + registry: ${{ secrets.ELASTIC_DOCKER_REGISTRY }} + username: ${{ secrets.ELASTIC_DOCKER_USERNAME }} + password: ${{ secrets.ELASTIC_DOCKER_PASSWORD }} + + - name: Login to dockerhub + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + registry: ${{ secrets.DOCKERHUB_REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - run: ./util.sh --action build + working-directory: .ci/docker + + #- run: ./util.sh --action push + # working-directory: .ci/docker From 0e1ccb69654b648907f307c1e80859b5af7c7523 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 9 Apr 2025 15:07:12 +0200 Subject: [PATCH 02/14] run pr --- .github/workflows/build-images.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 259575eba..7c35266ad 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -1,10 +1,9 @@ --- -# This workflow publishes docker images base on .ci/.docker-images.yml`. -# See docs/INTERNAL_DOCKER_IMAGES.md for further information. -name: publish-docker-images +name: build-images on: - workflow_dispatch: + pull_request: ~ + workflow_dispatch: ~ env: DOCKER_BUILDKIT: 1 From 836abfdd945ec3414bfdbe156878d719711a6d04 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 9 Apr 2025 15:09:20 +0200 Subject: [PATCH 03/14] docs --- .ci/docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker/README.md b/.ci/docker/README.md index 1241a7f05..ba4b8655a 100644 --- a/.ci/docker/README.md +++ b/.ci/docker/README.md @@ -2,7 +2,7 @@ Utility script for building and pushing the images based on `.ci/.matrix_python_full.yml`. -> :information_source: This script is mainly used in [publish-docker-images](https://github.com/elastic/apm-pipeline-library/actions/workflows/publish-docker-images.yml) workflow, +> :information_source: This script is mainly used in [publish-docker-images](https://github.com/elastic/apm-agent-java/actions/workflows/build-images.yml) workflow, which can be triggered safely at any time. ## Options From 6d417ae7815cbcaf83b49bab875c45adb41336ad Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 9 Apr 2025 15:13:10 +0200 Subject: [PATCH 04/14] run manually --- .github/workflows/build-images.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 7c35266ad..a02a9ca17 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -2,7 +2,6 @@ name: build-images on: - pull_request: ~ workflow_dispatch: ~ env: From cb377933f4fff2d1f0391c9be7b6fc215fd83895 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 9 Apr 2025 15:13:35 +0200 Subject: [PATCH 05/14] publihs --- .github/workflows/build-images.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index a02a9ca17..18e1216b0 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -35,5 +35,5 @@ jobs: - run: ./util.sh --action build working-directory: .ci/docker - #- run: ./util.sh --action push - # working-directory: .ci/docker + - run: ./util.sh --action push + working-directory: .ci/docker From d318516f20749e2a321cc82d1ec94780d3d2e260 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 9 Apr 2025 15:31:34 +0200 Subject: [PATCH 06/14] Update .ci/docker/README.md --- .ci/docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker/README.md b/.ci/docker/README.md index ba4b8655a..9ca435c1c 100644 --- a/.ci/docker/README.md +++ b/.ci/docker/README.md @@ -2,7 +2,7 @@ Utility script for building and pushing the images based on `.ci/.matrix_python_full.yml`. -> :information_source: This script is mainly used in [publish-docker-images](https://github.com/elastic/apm-agent-java/actions/workflows/build-images.yml) workflow, +> :information_source: This script is mainly used in [publish-docker-images](https://github.com/elastic/apm-agent-python/actions/workflows/build-images.yml) workflow, which can be triggered safely at any time. ## Options From a815a936bb94fd78d440fbb46225852904c4d097 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Tue, 29 Apr 2025 15:13:42 +0200 Subject: [PATCH 07/14] Use github container registry --- .github/workflows/build-images.yml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 18e1216b0..fa4a576e4 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -8,6 +8,7 @@ env: DOCKER_BUILDKIT: 1 permissions: + packages: write contents: read jobs: @@ -17,20 +18,13 @@ jobs: steps: - uses: actions/checkout@v4 - - - name: Login to docker.elastic.co - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 - with: - registry: ${{ secrets.ELASTIC_DOCKER_REGISTRY }} - username: ${{ secrets.ELASTIC_DOCKER_USERNAME }} - password: ${{ secrets.ELASTIC_DOCKER_PASSWORD }} - - name: Login to dockerhub + - name: Login to ghcr.io uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 with: - registry: ${{ secrets.DOCKERHUB_REGISTRY }} - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - run: ./util.sh --action build working-directory: .ci/docker From da11ebd14f9305e9b25e2ab8d7eb012ba13a581f Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 1 May 2025 10:36:49 +0200 Subject: [PATCH 08/14] github container registry --- .github/workflows/build-images.yml | 18 ++++++++++-------- tests/Dockerfile | 3 +++ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index fa4a576e4..a6a87c19b 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -4,17 +4,19 @@ name: build-images on: workflow_dispatch: ~ -env: - DOCKER_BUILDKIT: 1 - permissions: - packages: write contents: read jobs: - build-test-push: + build-push: runs-on: ubuntu-latest + permissions: + contents: read + packages: write + env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}/apm-agent-python-testing steps: - uses: actions/checkout@v4 @@ -22,12 +24,12 @@ jobs: - name: Login to ghcr.io uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 with: - registry: ghcr.io + registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - run: ./util.sh --action build + - run: ./util.sh --action build --registry ${{ env.REGISTRY }} --image-name ${{ env.IMAGE_NAME }} working-directory: .ci/docker - - run: ./util.sh --action push + - run: ./util.sh --action push --registry ${{ env.REGISTRY }} --image-name ${{ env.IMAGE_NAME }} working-directory: .ci/docker diff --git a/tests/Dockerfile b/tests/Dockerfile index cf1a8e30b..c5cd8050a 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -44,4 +44,7 @@ RUN chmod +x /usr/local/bin/entrypoint.sh WORKDIR /app +# configure the label to help with the GitHub container registry +LABEL org.opencontainers.image.source https://github.com/elastic/apm-agent-python + ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] From 4fa5013e23d65645ca3daf6723ab97bf263e100d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 1 May 2025 10:37:22 +0200 Subject: [PATCH 09/14] enable docker buildx --- .github/workflows/build-images.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index a6a87c19b..1243bbd9f 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -28,6 +28,9 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + - run: ./util.sh --action build --registry ${{ env.REGISTRY }} --image-name ${{ env.IMAGE_NAME }} working-directory: .ci/docker From d1924287c7061fa94cf8833dd97ece975dc8c9ca Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 1 May 2025 10:37:28 +0200 Subject: [PATCH 10/14] Revert "enable docker buildx" This reverts commit 4fa5013e23d65645ca3daf6723ab97bf263e100d. --- .github/workflows/build-images.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 1243bbd9f..a6a87c19b 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -28,9 +28,6 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 - - run: ./util.sh --action build --registry ${{ env.REGISTRY }} --image-name ${{ env.IMAGE_NAME }} working-directory: .ci/docker From e8ae4b0cea518500d4982a04d051ab135225e2ba Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 1 May 2025 10:39:20 +0200 Subject: [PATCH 11/14] fix --- .github/workflows/build-images.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index a6a87c19b..44dcb39be 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -28,8 +28,8 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - run: ./util.sh --action build --registry ${{ env.REGISTRY }} --image-name ${{ env.IMAGE_NAME }} + - run: ./util.sh --action build --registry ${{ env.REGISTRY }} --name ${{ env.IMAGE_NAME }} working-directory: .ci/docker - - run: ./util.sh --action push --registry ${{ env.REGISTRY }} --image-name ${{ env.IMAGE_NAME }} + - run: ./util.sh --action push --registry ${{ env.REGISTRY }} --name ${{ env.IMAGE_NAME }} working-directory: .ci/docker From 0d7feedb811538e1d5f1ffce7078c6ac68dd29ee Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 1 May 2025 10:54:44 +0200 Subject: [PATCH 12/14] use the docker images from the github container registry --- .github/workflows/run-matrix.yml | 4 ++++ tests/docker-compose.yml | 2 +- tests/scripts/docker/run_tests.sh | 13 +++++++++---- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run-matrix.yml b/.github/workflows/run-matrix.yml index 0b31f4318..3dc3befb7 100644 --- a/.github/workflows/run-matrix.yml +++ b/.github/workflows/run-matrix.yml @@ -20,6 +20,10 @@ jobs: max-parallel: 10 matrix: include: ${{ fromJSON(inputs.include) }} + env: + # These env variables are used in the docker-compose.yml and the run_tests.sh script. + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}/apm-agent-python-testing steps: - uses: actions/checkout@v4 - name: Run tests diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index c6598a969..b65a97e9e 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -185,7 +185,7 @@ services: - zookeeper run_tests: - image: elasticobservability/apm-agent-python-testing:${PYTHON_VERSION} + image: ${REGISTRY:-elasticobservability}/${IMAGE_NAME:-apm-agent-python-testing}:${PYTHON_VERSION} environment: ES_8_URL: 'http://elasticsearch8:9200' ES_7_URL: 'http://elasticsearch7:9200' diff --git a/tests/scripts/docker/run_tests.sh b/tests/scripts/docker/run_tests.sh index de518dc32..bf39df6ff 100755 --- a/tests/scripts/docker/run_tests.sh +++ b/tests/scripts/docker/run_tests.sh @@ -2,7 +2,7 @@ set -ex function cleanup { - PYTHON_VERSION=${1} docker compose down -v + PYTHON_VERSION=${1} REGISTRY=${REGISTRY} IMAGE_NAME=${IMAGE_NAME} docker compose down -v if [[ $CODECOV_TOKEN ]]; then cd .. @@ -21,6 +21,8 @@ docker_pip_cache="/tmp/cache/pip" TEST="${1}/${2}" LOCAL_USER_ID=${LOCAL_USER_ID:=$(id -u)} LOCAL_GROUP_ID=${LOCAL_GROUP_ID:=$(id -g)} +IMAGE_NAME=${IMAGE_NAME:-"apm-agent-python-testing"} +REGISTRY=${REGISTRY:-"elasticobservability"} cd tests @@ -42,18 +44,19 @@ echo "Running tests for ${1}/${2}" if [[ -n $DOCKER_DEPS ]] then - PYTHON_VERSION=${1} docker compose up -d ${DOCKER_DEPS} + PYTHON_VERSION=${1} REGISTRY=${REGISTRY} IMAGE_NAME=${IMAGE_NAME} docker compose up -d ${DOCKER_DEPS} fi # CASS_DRIVER_NO_EXTENSIONS is set so we don't build the Cassandra C-extensions, # as this can take several minutes if ! ${CI}; then + full_image_name="${REGISTRY}/${IMAGE_NAME}:${1}" DOCKER_BUILDKIT=1 docker build \ --progress=plain \ - --cache-from="elasticobservability/apm-agent-python-testing:${1}" \ + --cache-from="${full_image_name}" \ --build-arg PYTHON_IMAGE="${1/-/:}" \ - --tag "elasticobservability/apm-agent-python-testing:${1}" \ + --tag "${full_image_name}" \ . fi @@ -67,6 +70,8 @@ PYTHON_VERSION=${1} docker compose run \ -e WITH_COVERAGE=true \ -e CASS_DRIVER_NO_EXTENSIONS=1 \ -e PYTEST_JUNIT="--junitxml=/app/tests/docker-${1}-${2}-python-agent-junit.xml" \ + -e REGISTRY=${REGISTRY} \ + -e IMAGE_NAME=${IMAGE_NAME} \ -v ${pip_cache}:$(dirname ${docker_pip_cache}) \ -v "$(dirname $(pwd))":/app \ --rm run_tests \ From b5bfc3744b86a9867f5a8bef2abdaa8651f89000 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 1 May 2025 11:10:51 +0200 Subject: [PATCH 13/14] less verbose --- .ci/docker/util.sh | 1 + tests/scripts/docker/run_tests.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.ci/docker/util.sh b/.ci/docker/util.sh index 9326a5773..865e4d884 100755 --- a/.ci/docker/util.sh +++ b/.ci/docker/util.sh @@ -44,6 +44,7 @@ for version in $versions; do case $ACTION in build) DOCKER_BUILDKIT=1 docker build \ + --progress=plain \ --cache-from="${full_image_name}" \ -f "${project_root}/tests/Dockerfile" \ --build-arg PYTHON_IMAGE="${version/-/:}" \ diff --git a/tests/scripts/docker/run_tests.sh b/tests/scripts/docker/run_tests.sh index bf39df6ff..75f7dcb85 100755 --- a/tests/scripts/docker/run_tests.sh +++ b/tests/scripts/docker/run_tests.sh @@ -40,11 +40,11 @@ else fi fi -echo "Running tests for ${1}/${2}" +echo "Running tests for ${TEST}" if [[ -n $DOCKER_DEPS ]] then - PYTHON_VERSION=${1} REGISTRY=${REGISTRY} IMAGE_NAME=${IMAGE_NAME} docker compose up -d ${DOCKER_DEPS} + PYTHON_VERSION=${1} REGISTRY=${REGISTRY} IMAGE_NAME=${IMAGE_NAME} docker compose up --quiet-pull -d ${DOCKER_DEPS} fi # CASS_DRIVER_NO_EXTENSIONS is set so we don't build the Cassandra C-extensions, From 6babb37f64170040e6938c68bd11fd1003c3d859 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 1 May 2025 11:12:12 +0200 Subject: [PATCH 14/14] less verbose --- tests/scripts/docker/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/docker/run_tests.sh b/tests/scripts/docker/run_tests.sh index 75f7dcb85..9de251f02 100755 --- a/tests/scripts/docker/run_tests.sh +++ b/tests/scripts/docker/run_tests.sh @@ -60,7 +60,7 @@ if ! ${CI}; then . fi -PYTHON_VERSION=${1} docker compose run \ +PYTHON_VERSION=${1} docker compose run --quiet-pull \ -e PYTHON_FULL_VERSION=${1} \ -e LOCAL_USER_ID=$LOCAL_USER_ID \ -e LOCAL_GROUP_ID=$LOCAL_GROUP_ID \