From 46461e3fee7fcaa717e4f9e5c8686e7fa91d8415 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 24 Oct 2023 17:29:07 +0200 Subject: [PATCH 01/10] Move manifests to toplevel directory --- {sdk_container/.repo/manifests => manifests}/mantle-container | 0 {sdk_container/.repo/manifests => manifests}/os-release | 0 {sdk_container/.repo/manifests => manifests}/version.txt | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {sdk_container/.repo/manifests => manifests}/mantle-container (100%) rename {sdk_container/.repo/manifests => manifests}/os-release (100%) rename {sdk_container/.repo/manifests => manifests}/version.txt (100%) diff --git a/sdk_container/.repo/manifests/mantle-container b/manifests/mantle-container similarity index 100% rename from sdk_container/.repo/manifests/mantle-container rename to manifests/mantle-container diff --git a/sdk_container/.repo/manifests/os-release b/manifests/os-release similarity index 100% rename from sdk_container/.repo/manifests/os-release rename to manifests/os-release diff --git a/sdk_container/.repo/manifests/version.txt b/manifests/version.txt similarity index 100% rename from sdk_container/.repo/manifests/version.txt rename to manifests/version.txt From 6843fb701e32fe0aadb0f3c6a0347f99bb2eebad Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 24 Oct 2023 17:31:05 +0200 Subject: [PATCH 02/10] .github: Use new location for manifests --- .github/workflows/ci.yaml | 4 ++-- .github/workflows/mantle-releases-main.yml | 6 +++--- .github/workflows/run-kola-tests.yaml | 4 ++-- .github/workflows/setup-flatcar-sdk.sh | 2 +- .github/workflows/update-sdk.yaml | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 55126730322..64274d58ff5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -116,7 +116,7 @@ jobs: # This is also done again in run-kola-tests.yaml because these changes here disappear source ci-automation/ci_automation_common.sh - source sdk_container/.repo/manifests/version.txt + source manifests/version.txt version="alpha-$FLATCAR_VERSION_ID" check_version_string "$version" @@ -196,7 +196,7 @@ jobs: source ci-automation/image_changes.sh channel=alpha - vernum=$(source sdk_container/.repo/manifests/version.txt; echo "${FLATCAR_VERSION}") + vernum=$(source manifests/version.txt; echo "${FLATCAR_VERSION}") board="${arch}-usr" package_diff_env=( diff --git a/.github/workflows/mantle-releases-main.yml b/.github/workflows/mantle-releases-main.yml index a45fc0f3c22..e14f04712fb 100644 --- a/.github/workflows/mantle-releases-main.yml +++ b/.github/workflows/mantle-releases-main.yml @@ -63,9 +63,9 @@ jobs: set -euo pipefail set -x commit=${{ steps.fetch-latest-mantle.outputs.COMMIT }} - if ! grep -q "ghcr.io/flatcar/mantle:git-${commit}" sdk_container/.repo/manifests/mantle-container; then - echo "ghcr.io/flatcar/mantle:git-${commit}" > sdk_container/.repo/manifests/mantle-container - git add sdk_container/.repo/manifests/mantle-container + if ! grep -q "ghcr.io/flatcar/mantle:git-${commit}" manifests/mantle-container; then + echo "ghcr.io/flatcar/mantle:git-${commit}" > manifests/mantle-container + git add manifests/mantle-container fi - name: Create pull request for branch if: ${{ steps.figure-out-branch.outputs.SKIP == 0 }} diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml index 816ad60e0c8..be33e84c7ec 100644 --- a/.github/workflows/run-kola-tests.yaml +++ b/.github/workflows/run-kola-tests.yaml @@ -179,7 +179,7 @@ jobs: pushd scripts source ci-automation/ci_automation_common.sh - source sdk_container/.repo/manifests/version.txt + source manifests/version.txt version="alpha-$FLATCAR_VERSION_ID" check_version_string "$version" @@ -324,7 +324,7 @@ jobs: done done - source sdk_container/.repo/manifests/version.txt + source manifests/version.txt tap_generate_report "${all_archs}" "${FLATCAR_VERSION}" "md" "true" > test-results.md cat test-results.md >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/setup-flatcar-sdk.sh b/.github/workflows/setup-flatcar-sdk.sh index d362ba23594..77aa15c76f5 100755 --- a/.github/workflows/setup-flatcar-sdk.sh +++ b/.github/workflows/setup-flatcar-sdk.sh @@ -24,7 +24,7 @@ sudo apt-get install -y docker-ce docker-ce-cli containerd.io \ pushd "${WORK_SCRIPTS_DIR}" source ci-automation/ci_automation_common.sh -source sdk_container/.repo/manifests/version.txt +source manifests/version.txt # run_sdk_container requires a tag to exist in the repo it resides, # which may not be the case for forked repos. Add some fake tag in diff --git a/.github/workflows/update-sdk.yaml b/.github/workflows/update-sdk.yaml index 4d8e2b7bd9b..ca38417e38a 100644 --- a/.github/workflows/update-sdk.yaml +++ b/.github/workflows/update-sdk.yaml @@ -93,7 +93,7 @@ jobs: set -euo pipefail source ci-automation/ci_automation_common.sh - source sdk_container/.repo/manifests/version.txt + source manifests/version.txt version="alpha-$FLATCAR_VERSION_ID" sdk_version="${SOURCE_SDK_VERSION:-$FLATCAR_SDK_VERSION}" @@ -124,7 +124,7 @@ jobs: echo "setting sdk_version=${target_version} as a github output" echo "sdk_version=${target_version}" >> "$GITHUB_OUTPUT" - # This also updates sdk_container/.repo/manifests/version.txt with the new SDK version. + # This also updates manifests/version.txt with the new SDK version. ./update_sdk_container_image "${target_version}" - name: Upload the SDK container and binary packages to bincache @@ -144,7 +144,7 @@ jobs: echo " User ${BUILDCACHE_USER}" >> ~/.ssh/config echo " IdentityFile ~/.ssh/bincache" >> ~/.ssh/config - source sdk_container/.repo/manifests/version.txt + source manifests/version.txt vernum="${FLATCAR_SDK_VERSION}" docker_vernum="$(vernum_to_docker_image_version "${vernum}")" From 051e5e980ad692aa9b07582e8acd930cd6392fb1 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 25 Oct 2023 10:17:01 +0200 Subject: [PATCH 03/10] *: Use new manifests location --- build_toolchains | 5 +++-- common.sh | 2 +- retag-for-jenkins | 8 ++++++-- run_local_tests.sh | 4 ++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/build_toolchains b/build_toolchains index ebef08c8cfc..c667aba0301 100755 --- a/build_toolchains +++ b/build_toolchains @@ -41,8 +41,9 @@ create_provenance_overlay() { read scripts_hash<"${scripts_git}/${scripts_hash#ref: }" echo "${scripts_hash}" >"${root_overlay}${scripts_git}/HEAD" fi - mkdir -p "${root_overlay}/mnt/host/source/.repo/manifests" - cp "${REPO_MANIFESTS_DIR}/version.txt" "${root_overlay}/mnt/host/source/.repo/manifests" + local mdir="${root_overlay}/mnt/host/source/src/scripts/manifests" + mkdir -p "${mdir}" + cp "${REPO_MANIFESTS_DIR}/version.txt" "${mdir}" } catalyst_init "$@" diff --git a/common.sh b/common.sh index 0ca53468782..2f3fee71d74 100644 --- a/common.sh +++ b/common.sh @@ -294,7 +294,7 @@ SRC_INTERNAL="${GCLIENT_ROOT}/src-internal" SCRIPTS_DIR="${SRC_ROOT}/scripts" BUILD_LIBRARY_DIR="${SCRIPTS_DIR}/build_library" REPO_CACHE_DIR="${REPO_ROOT}/.cache" -REPO_MANIFESTS_DIR="${REPO_ROOT}/.repo/manifests" +REPO_MANIFESTS_DIR="${SCRIPTS_DIR}/manifests" # Source FLATCAR_VERSION_ID from manifest. if [[ -f "${REPO_MANIFESTS_DIR}/version.txt" ]]; then diff --git a/retag-for-jenkins b/retag-for-jenkins index 52f97371f60..4c6430c0a3a 100755 --- a/retag-for-jenkins +++ b/retag-for-jenkins @@ -26,8 +26,12 @@ BUILD_PATCH=$(git format-patch --output=/dev/stdout "${TAG}~1..${TAG}") git checkout --recurse-submodules "${TAG}" git reset --hard "${BRANCH}" echo "${BUILD_PATCH}" | git am -3 || { - git checkout "${TAG}" -- sdk_container/.repo/manifests/version.txt - git add sdk_container/.repo/manifests/version.txt + vertxt='manifest/version.txt' + if ! git checkout "${TAG}" -- "${vertxt}"; then + vertxt='sdk_container/.repo/manifest/version.txt' + git checkout "${TAG}" -- "${vertxt}" + fi + git add "${vertxt}" git am --continue # This does not handle submodule conflicts: It should use the one # from the TAG (similar to version.txt) unless an explicit new diff --git a/run_local_tests.sh b/run_local_tests.sh index 85670e2810f..cd3aad3cb72 100755 --- a/run_local_tests.sh +++ b/run_local_tests.sh @@ -26,7 +26,7 @@ # ./run_local_tests.sh # # Optional prerequisites: -# - Custom Mantle container image / version in sdk_container/.repo/manifests/mantle-container. +# - Custom Mantle container image / version in manifests/mantle-container. # This comes in handy if you've built a local mantle/kola which you want to test. # Just edit the file and put in the whole containerr image name and version. # @@ -70,7 +70,7 @@ function run_local_tests() ( rm -f results.* - local mantle_container="$(cat "sdk_container/.repo/manifests/mantle-container")" + local mantle_container="$(cat "manifests/mantle-container")" local tests="" local update_tests=false From 2074fcd62fd24824655352a8c1e026eeede04183 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 24 Oct 2023 17:39:12 +0200 Subject: [PATCH 04/10] ci-automation: Use new manifests location --- ci-automation/ci_automation_common.sh | 2 +- ci-automation/garbage_collect.sh | 11 ++++++++--- ci-automation/image.sh | 4 ++-- ci-automation/image_changes.sh | 2 +- ci-automation/packages-tag.sh | 6 +++--- ci-automation/packages.sh | 6 +++--- ci-automation/push_pkgs.sh | 4 ++-- ci-automation/release.sh | 10 +++++----- ci-automation/sdk_bootstrap.sh | 4 ++-- ci-automation/sdk_container.sh | 4 ++-- ci-automation/test.sh | 8 ++++---- ci-automation/util/fetch_image.sh | 2 +- ci-automation/vms.sh | 4 ++-- 13 files changed, 36 insertions(+), 31 deletions(-) diff --git a/ci-automation/ci_automation_common.sh b/ci-automation/ci_automation_common.sh index a0745b25ecd..2a7408b6d52 100644 --- a/ci-automation/ci_automation_common.sh +++ b/ci-automation/ci_automation_common.sh @@ -34,7 +34,7 @@ function update_and_push_version() { fi # Add and commit local changes - git add "sdk_container/.repo/manifests/version.txt" + git add "manifests/version.txt" git commit --allow-empty -m "New version: ${version}" diff --git a/ci-automation/garbage_collect.sh b/ci-automation/garbage_collect.sh index 3e0d0a8d581..c36f92fd897 100644 --- a/ci-automation/garbage_collect.sh +++ b/ci-automation/garbage_collect.sh @@ -82,8 +82,13 @@ function _garbage_collect_impl() { echo "#### Processing version '${version}' ####" echo - git checkout "${version}" -- sdk_container/.repo/manifests/version.txt - source sdk_container/.repo/manifests/version.txt + vertxt=manifests/version.txt + if ! git checkout "${version}" -- "${vertxt}"; then + echo 'Trying to check out the old location of the version.txt then.' + vertxt=sdk_container/.repo/manifests/version.txt + git checkout "${version}" -- "${vertxt}" + fi + source "${vertxt}" # Assuming that the SDK build version also has the same OS version local os_vernum="${FLATCAR_VERSION}" @@ -151,7 +156,7 @@ function _garbage_collect_impl() { echo local mantle_ref - mantle_ref=$(cat sdk_container/.repo/manifests/mantle-container) + mantle_ref=$(cat manifests/mantle-container) docker run --pull always --rm --net host \ --env AZURE_AUTH_CREDENTIALS --env AZURE_PROFILE \ --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY \ diff --git a/ci-automation/image.sh b/ci-automation/image.sh index 0cb5b7e1185..4a1608d1832 100644 --- a/ci-automation/image.sh +++ b/ci-automation/image.sh @@ -13,7 +13,7 @@ # # PREREQUISITES: # -# 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt +# 1. SDK version and OS image version are recorded in manifests/version.txt # 2. Scripts repo version tag of OS image version to be built is available and checked out. # 3. Flatcar packages container is available via build cache server # from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz" @@ -63,7 +63,7 @@ function _image_build_impl() { source ci-automation/ci_automation_common.sh source ci-automation/gpg_setup.sh - source sdk_container/.repo/manifests/version.txt + source manifests/version.txt local vernum="${FLATCAR_VERSION}" local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh index 82d9857c7ee..23af10c70da 100644 --- a/ci-automation/image_changes.sh +++ b/ci-automation/image_changes.sh @@ -40,7 +40,7 @@ function image_changes() ( if [ "${channel}" = "developer" ]; then channel="alpha" fi - vernum=$(source sdk_container/.repo/manifests/version.txt; echo "${FLATCAR_VERSION}") + vernum=$(source manifests/version.txt; echo "${FLATCAR_VERSION}") local -a package_diff_env package_diff_params_b local -a size_changes_env size_changes_params_b diff --git a/ci-automation/packages-tag.sh b/ci-automation/packages-tag.sh index 053f8935460..f8acbc511b3 100644 --- a/ci-automation/packages-tag.sh +++ b/ci-automation/packages-tag.sh @@ -14,7 +14,7 @@ # # PREREQUISITES: # -# 1. SDK version is recorded in sdk_container/.repo/manifests/version.txt +# 1. SDK version is recorded in manifests/version.txt # 2. SDK container is either # - available via ghcr.io/flatcar/flatcar-sdk-[ARCH]:[VERSION] (official SDK release) # OR @@ -41,7 +41,7 @@ # # 1. Updated scripts repository # - version tag -# - sdk_container/.repo/manifests/version.txt denotes new FLATCAR OS version +# - manifests/version.txt denotes new FLATCAR OS version # 2. "./skip-build" as flag file to signal that the build should stop function packages_tag() { @@ -63,7 +63,7 @@ function _packages_tag_impl() { check_version_string "${version}" - source sdk_container/.repo/manifests/version.txt + source manifests/version.txt local sdk_version="${FLATCAR_SDK_VERSION}" # Create new tag in scripts repo w/ updated versionfile diff --git a/ci-automation/packages.sh b/ci-automation/packages.sh index e6e984f7f38..4ef2538aa22 100644 --- a/ci-automation/packages.sh +++ b/ci-automation/packages.sh @@ -13,7 +13,7 @@ # # PREREQUISITES: # -# 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt +# 1. SDK version and OS image version are recorded in manifests/version.txt # 2. Scripts repo version tag of OS image version to be built is available and checked out. # 3. SDK container is either # - available via ghcr.io/flatcar/flatcar-sdk-[ARCH]:[VERSION] (official SDK release) @@ -64,7 +64,7 @@ function _packages_build_impl() { source ci-automation/ci_automation_common.sh source ci-automation/gpg_setup.sh - source sdk_container/.repo/manifests/version.txt + source manifests/version.txt local sdk_version="${FLATCAR_SDK_VERSION}" # Get SDK from either the registry or import from build cache @@ -90,7 +90,7 @@ function _packages_build_impl() { ./build_packages --board="${arch}-usr" # run_sdk_container updates the version file, use that version from here on - source sdk_container/.repo/manifests/version.txt + source manifests/version.txt local vernum="${FLATCAR_VERSION}" local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" local packages_image="flatcar-packages-${arch}" diff --git a/ci-automation/push_pkgs.sh b/ci-automation/push_pkgs.sh index 0cb1a0187cf..1d4d31259b2 100644 --- a/ci-automation/push_pkgs.sh +++ b/ci-automation/push_pkgs.sh @@ -15,7 +15,7 @@ # # PREREQUISITES: # -# 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt +# 1. SDK version and OS image version are recorded in manifests/version.txt # 2. Scripts repo version tag of OS image version to be built is available and checked out. # 3. Flatcar packages container is available via build cache server # from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz" @@ -79,7 +79,7 @@ function _push_packages_impl() { source ci-automation/ci_automation_common.sh source ci-automation/gpg_setup.sh - source sdk_container/.repo/manifests/version.txt + source manifests/version.txt local vernum="${FLATCAR_VERSION}" local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" diff --git a/ci-automation/release.sh b/ci-automation/release.sh index 90fb1c59db2..bff9ae6f728 100644 --- a/ci-automation/release.sh +++ b/ci-automation/release.sh @@ -13,9 +13,9 @@ # # PREREQUISITES: # -# 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt +# 1. SDK version and OS image version are recorded in manifests/version.txt # 2. Scripts repo version tag of OS image version to be built is available and checked out. -# 3. Mantle container docker image reference is stored in sdk_container/.repo/manifests/mantle-container. +# 3. Mantle container docker image reference is stored in manifests/mantle-container. # 4. Vendor image to run tests for are available on buildcache # ( images/[ARCH]/[FLATCAR_VERSION]/ ) # 5. SDK container is either @@ -74,7 +74,7 @@ function _inside_mantle() { source sdk_lib/sdk_container_common.sh source ci-automation/ci_automation_common.sh - source sdk_container/.repo/manifests/version.txt + source manifests/version.txt # Needed because we are not the SDK container here source sdk_container/.env CHANNEL="$(get_git_channel)" @@ -189,7 +189,7 @@ function _release_build_impl() { source ci-automation/ci_automation_common.sh source ci-automation/gpg_setup.sh - source sdk_container/.repo/manifests/version.txt + source manifests/version.txt # Needed because we are not the SDK container here source sdk_container/.env local sdk_version="${FLATCAR_SDK_VERSION}" @@ -201,7 +201,7 @@ function _release_build_impl() { local container_name="flatcar-publish-${docker_vernum}" local mantle_ref - mantle_ref=$(cat sdk_container/.repo/manifests/mantle-container) + mantle_ref=$(cat manifests/mantle-container) # A job on each worker prunes old mantle images (docker image prune), no need to do it here echo "docker rm -f '${container_name}'" >> ./ci-cleanup.sh diff --git a/ci-automation/sdk_bootstrap.sh b/ci-automation/sdk_bootstrap.sh index 3540ec90239..d3ea8d1691b 100644 --- a/ci-automation/sdk_bootstrap.sh +++ b/ci-automation/sdk_bootstrap.sh @@ -47,7 +47,7 @@ # 1. SDK tarball (gentoo catalyst output) of the new SDK, pushed to buildcache. # 2. Updated scripts repository # - version tag -# - sdk_container/.repo/manifests/version.txt denotes new SDK version +# - manifests/version.txt denotes new SDK version # 3. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. # 4. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. @@ -147,7 +147,7 @@ function _sdk_bootstrap_impl() { # push SDK tarball to buildcache # Get Flatcar version number format (separator is '+' instead of '-', # equal to $(strip_version_prefix "$version") - source sdk_container/.repo/manifests/version.txt + source manifests/version.txt local dest_tarball="flatcar-sdk-${ARCH}-${FLATCAR_SDK_VERSION}.tar.bz2" # change the owner of the files and directories in __build__ back diff --git a/ci-automation/sdk_container.sh b/ci-automation/sdk_container.sh index e3875581912..57be39febd5 100644 --- a/ci-automation/sdk_container.sh +++ b/ci-automation/sdk_container.sh @@ -15,7 +15,7 @@ # # PREREQUISITES: # -# 1. SDK version is recorded in sdk_container/.repo/manifests/version.txt and a matching +# 1. SDK version is recorded in manifests/version.txt and a matching # SDK tarball is available on BUILDCACHE/sdk/[ARCH]/[VERSION]/flatcar-sdk-[ARCH]-[VERSION].tar.bz2 # # OPTIONAL INPUT: @@ -56,7 +56,7 @@ function _sdk_container_build_impl() { source ci-automation/ci_automation_common.sh source ci-automation/gpg_setup.sh - source sdk_container/.repo/manifests/version.txt + source manifests/version.txt local vernum="${FLATCAR_SDK_VERSION}" local sdk_tarball="flatcar-sdk-${ARCH}-${vernum}.tar.bz2" diff --git a/ci-automation/test.sh b/ci-automation/test.sh index bdddb4a3b43..7026b2f7369 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -16,9 +16,9 @@ # # PREREQUISITES: # -# 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt +# 1. SDK version and OS image version are recorded in manifests/version.txt # 2. Scripts repo version tag of OS image version to be built is available and checked out. -# 3. Mantle container docker image reference is stored in sdk_container/.repo/manifests/mantle-container. +# 3. Mantle container docker image reference is stored in manifests/mantle-container. # 4. Vendor image to run tests for are available on buildcache # ( images/[ARCH]/[FLATCAR_VERSION]/ ) # @@ -112,7 +112,7 @@ function _test_run_impl() { source ci-automation/ci_automation_common.sh source sdk_lib/sdk_container_common.sh - source sdk_container/.repo/manifests/version.txt + source manifests/version.txt local vernum="${FLATCAR_VERSION}" local docker_vernum docker_vernum="$(vernum_to_docker_image_version "${vernum}")" @@ -130,7 +130,7 @@ function _test_run_impl() { local container_name="flatcar-tests-${arch}-${docker_vernum}-${image}" local mantle_ref - mantle_ref=$(cat sdk_container/.repo/manifests/mantle-container) + mantle_ref=$(cat manifests/mantle-container) local tap_merged_summary="results-${image}" local tap_merged_detailed="results-${image}-detailed" diff --git a/ci-automation/util/fetch_image.sh b/ci-automation/util/fetch_image.sh index 51c4ef3a424..5b65e13256d 100755 --- a/ci-automation/util/fetch_image.sh +++ b/ci-automation/util/fetch_image.sh @@ -30,7 +30,7 @@ function fetch_image() { local script_root="$(dirname "${BASH_SOURCE[0]}")/../.." source "${script_root}/ci-automation/ci_automation_common.sh" - local vernum="$(source "${script_root}/sdk_container/.repo/manifests/version.txt"; + local vernum="$(source "${script_root}/manifests/version.txt"; echo "${FLATCAR_VERSION}")" local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" diff --git a/ci-automation/vms.sh b/ci-automation/vms.sh index 69f8fd54620..720689453dc 100644 --- a/ci-automation/vms.sh +++ b/ci-automation/vms.sh @@ -13,7 +13,7 @@ # # PREREQUISITES: # -# 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt +# 1. SDK version and OS image version are recorded in manifests/version.txt # 2. Scripts repo version tag of OS image version to be built is available and checked out. # 3. Flatcar packages container is available via build cache server # from "/containers/[VERSION]/flatcar-images-[ARCH]-[FLATCAR_VERSION].tar.gz" @@ -65,7 +65,7 @@ function _vm_build_impl() { source ci-automation/ci_automation_common.sh source ci-automation/gpg_setup.sh - source sdk_container/.repo/manifests/version.txt + source manifests/version.txt local vernum="${FLATCAR_VERSION}" local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" From e5d0cba49ed5e3d2f6d3fb093a10d9b5b45910fa Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 24 Oct 2023 18:31:03 +0200 Subject: [PATCH 05/10] sdk_lib: Use new manifests location --- sdk_lib/Dockerfile.lean-arch | 5 ++++- sdk_lib/sdk_container_common.sh | 2 +- sdk_lib/sdk_entry.sh | 2 +- sdk_lib/sdk_init_selfcontained.sh | 4 +++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/sdk_lib/Dockerfile.lean-arch b/sdk_lib/Dockerfile.lean-arch index ee3bbf68ed8..e9d30153309 100644 --- a/sdk_lib/Dockerfile.lean-arch +++ b/sdk_lib/Dockerfile.lean-arch @@ -13,8 +13,11 @@ RUN if [ -n "$RMARCH" ]; then \ sudo rm -f /usr/local/bin/*-$RMARCH; \ fi -# Note: .repo/manifests/version.txt will survive this. That's intended. +# Note: Preserve manifests. +RUN sudo mv /mnt/host/source/src/scripts/manifests /manifests RUN sudo rm -rf /mnt/host/source/* +RUN sudo mkdir -p /mnt/host/source/src/scripts +RUN sudo mv /manifests /mnt/host/source/src/scripts/manifests FROM scratch diff --git a/sdk_lib/sdk_container_common.sh b/sdk_lib/sdk_container_common.sh index 55fdcc5b7ef..56751663d1f 100644 --- a/sdk_lib/sdk_container_common.sh +++ b/sdk_lib/sdk_container_common.sh @@ -7,7 +7,7 @@ # # globals # -sdk_container_common_versionfile="sdk_container/.repo/manifests/version.txt" +sdk_container_common_versionfile="manifests/version.txt" sdk_container_common_registry="ghcr.io/flatcar" sdk_container_common_env_file="sdk_container/.sdkenv" diff --git a/sdk_lib/sdk_entry.sh b/sdk_lib/sdk_entry.sh index eca47d7fa84..bb86be133b3 100755 --- a/sdk_lib/sdk_entry.sh +++ b/sdk_lib/sdk_entry.sh @@ -15,7 +15,7 @@ chown -R sdk:sdk /home/sdk # chroot portage conf to point to the correct binhost. ( source /etc/lsb-release # SDK version in DISTRIB_RELEASE - source /mnt/host/source/.repo/manifests/version.txt # OS image version in FLATCAR_VERSION_ID + source /mnt/host/source/src/scripts/manifests/version.txt # OS image version in FLATCAR_VERSION_ID version="${FLATCAR_VERSION_ID}" # If this is a nightly build tag we can use pre-built binaries directly from the diff --git a/sdk_lib/sdk_init_selfcontained.sh b/sdk_lib/sdk_init_selfcontained.sh index 76b08b3c21b..2292a4698be 100755 --- a/sdk_lib/sdk_init_selfcontained.sh +++ b/sdk_lib/sdk_init_selfcontained.sh @@ -26,8 +26,10 @@ function clone_version() { } # -- -version="$(source /mnt/host/source/.repo/manifests/version.txt; echo $FLATCAR_VERSION)" +version=$(source /mnt/host/source/src/scripts/manifests/version.txt; echo "$FLATCAR_VERSION") mkdir -p /home/sdk/trunk/src/third_party/ +# Drop the manifests dir, it will be replaced by one from scripts repo. +rm -rf /home/sdk/trunk/src/scripts clone_version scripts /home/sdk/trunk/src/scripts "$version" From 0eb5f2908ec56943681e40437182d053d14837ac Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 24 Oct 2023 18:31:22 +0200 Subject: [PATCH 06/10] overlay: Use new manifests location --- .../profiles/coreos/base/profile.bashrc.slsa-provenance | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/profile.bashrc.slsa-provenance b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/profile.bashrc.slsa-provenance index 76a3c02f7f2..f1eb687a739 100644 --- a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/profile.bashrc.slsa-provenance +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/profile.bashrc.slsa-provenance @@ -143,7 +143,7 @@ __slsa_provenance_report() { fi # FIXME: Supply SDK image ID and sha256 digest along with the version tag - local sdk_version="$(source /mnt/host/source/.repo/manifests/version.txt; echo ${FLATCAR_SDK_VERSION})" + local sdk_version=$(source /mnt/host/source/src/scripts/manifests/version.txt; echo "${FLATCAR_SDK_VERSION}") # FIXME: add builder ID cat < Date: Wed, 25 Oct 2023 07:36:47 +0200 Subject: [PATCH 07/10] retag-for-jenkins: Drop obsolete comment Submodules are not a thing any more. --- retag-for-jenkins | 3 --- 1 file changed, 3 deletions(-) diff --git a/retag-for-jenkins b/retag-for-jenkins index 4c6430c0a3a..450f3537077 100755 --- a/retag-for-jenkins +++ b/retag-for-jenkins @@ -33,9 +33,6 @@ echo "${BUILD_PATCH}" | git am -3 || { fi git add "${vertxt}" git am --continue - # This does not handle submodule conflicts: It should use the one - # from the TAG (similar to version.txt) unless an explicit new - # reference was specified } || { echo "Failed to resolve conflict, continue manually" >&2 ; exit 1 ; } git tag -d "${TAG}" git tag "${TAG}" From 3efb48fd6c73ff3117a3f08d2a59d993682f373b Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 25 Oct 2023 09:42:45 +0200 Subject: [PATCH 08/10] sdk_lib: Fix a quoting issue --- sdk_lib/sdk_init_selfcontained.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk_lib/sdk_init_selfcontained.sh b/sdk_lib/sdk_init_selfcontained.sh index 2292a4698be..50499c04dec 100755 --- a/sdk_lib/sdk_init_selfcontained.sh +++ b/sdk_lib/sdk_init_selfcontained.sh @@ -19,7 +19,7 @@ function clone_version() { local dest="$2" local version="$3" - git clone https://github.com/flatcar/$repo "$dest" + git clone "https://github.com/flatcar/$repo" "$dest" git -C "${dest}" fetch --all local tag=$(git -C "${dest}" tag -l | grep "${version}") git -C "${dest}" checkout "$tag" From cf65da07fa3247a9455fb4444e9a89869e6df702 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 25 Oct 2023 10:17:15 +0200 Subject: [PATCH 09/10] build_toolchains: Make variables local --- build_toolchains | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build_toolchains b/build_toolchains index c667aba0301..dd661fbe3a5 100755 --- a/build_toolchains +++ b/build_toolchains @@ -26,8 +26,10 @@ stage4/root_overlay: ${ROOT_OVERLAY} EOF catalyst_stage_default } + create_provenance_overlay() { local root_overlay="$1" + local f d while read f; do d="${f%/*}" mkdir -p "${root_overlay}${d}/" @@ -36,6 +38,7 @@ create_provenance_overlay() { local scripts_git=/mnt/host/source/src/scripts/.git # `git rev-parse` fails due to the safe.directory setting in SDK container. # Open-code the ref lookup + local scripts_hash read scripts_hash <"${scripts_git}/HEAD" if [[ "${scripts_hash}" == "ref:"* ]]; then read scripts_hash<"${scripts_git}/${scripts_hash#ref: }" From 047d6e085772c2c7b25ac895884c721cbef1839f Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 25 Oct 2023 15:17:48 +0200 Subject: [PATCH 10/10] manifests: Update mantle tag --- manifests/mantle-container | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/mantle-container b/manifests/mantle-container index 3e02248b400..e59f3f9f61a 100644 --- a/manifests/mantle-container +++ b/manifests/mantle-container @@ -1 +1 @@ -ghcr.io/flatcar/mantle:git-99dccdb94e34d02566d4bf155a89763c7dd74e4f +ghcr.io/flatcar/mantle:pr-470