From a6f3328ed88cb789b3be13386e211ebe668bdec1 Mon Sep 17 00:00:00 2001 From: Matthias Dellweg Date: Mon, 9 Mar 2026 09:11:20 +0100 Subject: [PATCH] EXPERIMENT --- .ci/ansible/Containerfile.j2 | 43 +++++------- .ci/ansible/build_container.yaml | 11 ++-- .ci/ansible/start_container.yaml | 66 ++++++++++--------- .github/workflows/scripts/before_install.sh | 22 ++++--- .github/workflows/scripts/before_script.sh | 2 +- .github/workflows/scripts/install.sh | 4 -- .../workflows/scripts/pre_before_script.sh | 21 ------ template_config.yml | 1 - 8 files changed, 70 insertions(+), 100 deletions(-) delete mode 100644 .github/workflows/scripts/pre_before_script.sh diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index 01469b647..fb9ed4a1c 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -1,34 +1,28 @@ -FROM {{ ci_base | default(pulp_default_container) }} +FROM {{ image.ci_base }} +{%- if image.webserver_snippet %} -# Add source directories to container -{% for item in plugins %} -ADD ./{{ item.name }} ./{{ item.name }} -{% endfor %} +ADD ./{{ plugin_name }}/{{ plugin_name | replace("-", "_") }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ plugin_name }}.conf +{%- endif %} + +{%- for item in extra_files | default([]) %} -{% for item in extra_files | default([]) %} ADD ./{{ item.origin }} {{ item.destination }} -{% endfor %} +{%- endfor %} # This MUST be the ONLY call to pip install in inside the container. RUN pip3 install --upgrade pip setuptools wheel && \ rm -rf /root/.cache/pip && \ - pip3 install -{%- if s3_test | default(false) -%} -{{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue -{%- endif -%} -{%- for item in plugins -%} -{{ " " }}{{ item.source }} -{%- if item.upperbounds | default(false) -%} -{{ " " }}-c ./{{ item.name }}/upperbounds_constraints.txt + pip3 install {{ image.source }} +{%- if image.upperbounds | default(false) -%} +{{ " " }}-c ./{{ plugin_name }}/upperbounds_constraints.txt {%- endif -%} -{%- if item.lowerbounds | default(false) -%} -{{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt +{%- if image.lowerbounds | default(false) -%} +{{ " " }}-c ./{{ plugin_name }}/lowerbounds_constraints.txt {%- endif -%} -{%- if item.ci_requirements | default(false) -%} -{{ " " }}-r ./{{ item.name }}/ci_requirements.txt +{%- if image.ci_requirements | default(false) -%} +{{ " " }}-r ./{{ plugin_name }}/ci_requirements.txt {%- endif -%} -{%- endfor %} -{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \ +{{ " " }}-c ./{{ plugin_name }}/.ci/assets/ci_constraints.txt && \ rm -rf /root/.cache/pip {% if pulp_env is defined and pulp_env %} @@ -46,11 +40,8 @@ ENV {{ key | upper }}={{ value }} USER pulp:pulp RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \ /usr/local/bin/pulpcore-manager collectstatic --clear --noinput --link -USER root:root -{% for item in plugins %} -RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \ - ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true -{% endfor %} +RUN mkdir /var/lib/pulp/.config +USER root:root ENTRYPOINT ["/init"] diff --git a/.ci/ansible/build_container.yaml b/.ci/ansible/build_container.yaml index c380b430a..0a188cba9 100644 --- a/.ci/ansible/build_container.yaml +++ b/.ci/ansible/build_container.yaml @@ -1,15 +1,14 @@ # Ansible playbook to create the pulp service containers image --- -- hosts: localhost +- hosts: "localhost" gather_facts: false vars_files: - - vars/main.yaml + - "vars/main.yaml" tasks: - name: "Generate Containerfile from template" - template: - src: Containerfile.j2 - dest: Containerfile - + ansible.builtin.template: + src: "Containerfile.j2" + dest: "Containerfile" - name: "Build pulp image" # We build from the ../.. (parent dir of pulpcore git repo) Docker build # "context" so that repos like pulp-smash are accessible to Docker diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index 47e5221e5..acdc22ad3 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -1,33 +1,26 @@ # Ansible playbook to start the pulp service container and its supporting services --- -- hosts: localhost +- hosts: "localhost" gather_facts: false vars_files: - - vars/main.yaml + - "vars/main.yaml" tasks: - name: "Create Settings Directories" - file: + ansible.builtin.file: path: "{{ item }}" - state: directory + state: "directory" mode: "0755" loop: - - settings - - ssh - - ~/.config/pulp_smash + - "settings" - name: "Generate Pulp Settings" template: - src: settings.py.j2 - dest: settings/settings.py - - - name: "Configure pulp-smash" - copy: - src: smash-config.json - dest: ~/.config/pulp_smash/settings.json + src: "settings.py.j2" + dest: "settings/settings.py" - name: "Setup docker networking" docker_network: - name: pulp_ci_bridge + name: "pulp_ci_bridge" - name: "Start Service Containers" docker_container: @@ -37,24 +30,24 @@ recreate: true privileged: true networks: - - name: pulp_ci_bridge + - name: "pulp_ci_bridge" aliases: "{{ item.name }}" volumes: "{{ item.volumes | default(omit) }}" env: "{{ item.env | default(omit) }}" command: "{{ item.command | default(omit) }}" - state: started + state: "started" loop: "{{ services | default([]) }}" - name: "Retrieve Docker Network Info" docker_network_info: - name: pulp_ci_bridge - register: pulp_ci_bridge_info + name: "pulp_ci_bridge" + register: "pulp_ci_bridge_info" - name: "Update /etc/hosts" lineinfile: - path: /etc/hosts + path: "/etc/hosts" regexp: "\\s{{ item.value.Name }}\\s*$" - line: "{{ item.value.IPv4Address | ipaddr('address') }}\t{{ item.value.Name }}" + line: "{{ item.value.IPv4Address | ansible.utils.ipaddr('address') }}\t{{ item.value.Name }}" loop: "{{ pulp_ci_bridge_info.network.Containers | dict2items }}" become: true @@ -63,19 +56,19 @@ aws_access_key: "{{ minio_access_key }}" aws_secret_key: "{{ minio_secret_key }}" s3_url: "http://minio:9000" - region: eu-central-1 - name: pulp3 - state: present - when: s3_test | default(false) + region: "eu-central-1" + name: "pulp3" + state: "present" + when: "s3_test | default(false)" - block: - name: "Wait for Pulp" uri: url: "http://pulp{{ lookup('env', 'PULP_API_ROOT') | default('\/pulp\/', True) }}api/v3/status/" - follow_redirects: all - validate_certs: no - register: result - until: result.status == 200 + follow_redirects: "all" + validate_certs: "no" + register: "result" + until: "result.status == 200" retries: 12 delay: 5 rescue: @@ -86,7 +79,7 @@ - name: "Check version of component being tested" assert: that: - - (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] | canonical_semver == (component_version | canonical_semver) + - "(result.json.versions | items2dict(key_name='component', value_name='version'))[item.app_label] | canonical_semver == (component_version | canonical_semver)" fail_msg: | Component {{ item.app_label }} was expected to be installed in version {{ component_version }}. Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] }}. @@ -100,9 +93,20 @@ login admin password password -- hosts: pulp +- hosts: "pulp" gather_facts: false tasks: + - name: "Create directory for pulp-smash config" + ansible.builtin.file: + path: "/var/lib/pulp/.config/pulp_smash/" + state: "directory" + mode: "0755" + + - name: "Configure pulp-smash" + ansible.builtin.copy: + src: "smash-config.json" + dest: "/var/lib/pulp/.config/pulp_smash/settings.json" + - name: "Set pulp admin password" command: cmd: "pulpcore-manager reset-admin-password --password password" diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index d3779b666..56abed453 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -26,7 +26,7 @@ fi COMPONENT_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')" COMPONENT_SOURCE="./pulp_container/dist/pulp_container-${COMPONENT_VERSION}-py3-none-any.whl" if [ "$TEST" = "s3" ]; then - COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[s3]" + COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[s3] git+https://github.com/gerrod3/botocore.git@fix-100-continue" fi if [ "$TEST" = "azure" ]; then COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[azure]" @@ -38,6 +38,7 @@ fi if [[ "$TEST" = "lowerbounds" ]]; then python3 .ci/scripts/calc_constraints.py pyproject.toml > lowerbounds_constraints.txt fi + export PULP_API_ROOT=$(test "${TEST}" = "s3" && echo "/rerouted/djnd/" || echo "/pulp/") echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV" @@ -48,30 +49,31 @@ mkdir -p .ci/ansible/vars cat > .ci/ansible/vars/main.yaml << VARSYAML --- scenario: "${TEST}" +plugin_name: "pulp_container" legacy_component_name: "pulp_container" component_name: "container" component_version: "${COMPONENT_VERSION}" pulp_env: {} pulp_settings: {"allowed_content_checksums": ["sha1", "sha224", "sha256", "sha384", "sha512"], "allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"], "flatpak_index": true} pulp_scheme: "https" -pulp_default_container: "ghcr.io/pulp/pulp-ci-centos9:latest" api_root: "${PULP_API_ROOT}" image: name: "pulp" tag: "ci_build" -plugins: - - name: "pulp_container" - source: "${COMPONENT_SOURCE}" - ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false) - upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false) - lowerbounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false) + ci_base: "ghcr.io/pulp/pulp-ci-centos9:latest" + source: "${COMPONENT_SOURCE}" + ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false) + upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false) + lowerbounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false) + webserver_snippet: $(test -f pulp_container/app/webserver_snippets/nginx.conf && echo -n true || echo -n false ) +extra_files: + - origin: "pulp_container" + destination: "pulp_container" services: - name: "pulp" image: "pulp:ci_build" volumes: - "./settings:/etc/pulp" - - "./ssh:/keys/" - - "~/.config:/var/lib/pulp/.config" - "../../../pulp-openapi-generator:/root/pulp-openapi-generator" env: PULP_WORKERS: "4" diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh index defdad73c..b6a8d13dd 100755 --- a/.github/workflows/scripts/before_script.sh +++ b/.github/workflows/scripts/before_script.sh @@ -23,7 +23,7 @@ fi # Developers often want to know the final pulp config echo echo "# Pulp config:" -tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json +tail -v -n +1 .ci/ansible/settings/settings.* echo echo "# Containerfile:" diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 2b6fb96a4..188ac6103 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -44,10 +44,6 @@ cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml" ansible-playbook build_container.yaml ansible-playbook start_container.yaml -# .config needs to be accessible by the pulp user in the container, but some -# files will likely be modified on the host by post/pre scripts. -chmod 777 ~/.config/pulp_smash/ -chmod 666 ~/.config/pulp_smash/settings.json # Plugins often write to ~/.config/pulp/cli.toml from the host chmod 777 ~/.config/pulp chmod 666 ~/.config/pulp/cli.toml diff --git a/.github/workflows/scripts/pre_before_script.sh b/.github/workflows/scripts/pre_before_script.sh deleted file mode 100644 index d2670805e..000000000 --- a/.github/workflows/scripts/pre_before_script.sh +++ /dev/null @@ -1,21 +0,0 @@ -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -set -mveuo pipefail - -# add pulp.example.com ot the /etc/hosts -# docker clients cannot identify 'pulp' as host -cat /etc/hosts | grep pulp -PULP_HOSTNAME=$(cat /etc/hosts | sed -En "s/pulp/pulp.example.com/p") -echo $PULP_HOSTNAME | sudo tee -a /etc/hosts -cat /etc/hosts | grep pulp - -echo $PULP_HOSTNAME | docker exec -i pulp bash -c "cat >> /etc/hosts" - -echo "machine pulp.example.com -login admin -password password -" >> ~/.netrc - -sed -i 's/https:\/\/pulp/https:\/\/pulp.example.com/g' $PWD/.github/workflows/scripts/script.sh -sed -i 's/\"hostname\": \"pulp\",/\"hostname\": \"pulp.example.com\",/g' ~/.config/pulp_smash/settings.json diff --git a/template_config.yml b/template_config.yml index 023b494d0..b1b577c43 100644 --- a/template_config.yml +++ b/template_config.yml @@ -116,7 +116,6 @@ test_deprecations: true test_gcp: false test_lowerbounds: true test_performance: false -test_reroute: true test_s3: true use_issue_template: true ...