diff --git a/.github/workflows/build_and_push_nf_base_images.yml b/.github/workflows/build_and_push_nf_base_images.yml index 7971b9d5..8a1fcbdc 100644 --- a/.github/workflows/build_and_push_nf_base_images.yml +++ b/.github/workflows/build_and_push_nf_base_images.yml @@ -17,7 +17,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 with: - path: containers + path: containers - name: Checkout other private repository uses: actions/checkout@v3 @@ -46,16 +46,14 @@ jobs: echo "Building an image present in $dir" image_name=nextflow-approved/public tag_name=$(basename "$dir") - docker build -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir - docker push public.ecr.aws/u5x5h6w3/$image_name:$tag_name + docker buildx build --push --platform linux/amd64 -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir echo "Built an image with name --> $image_name:$tag_name" for dir in containers/nextflow-base-images/*/;do echo "Building an image present in $dir" image_name=nextflow-approved/public tag_name=$(basename "$dir") - docker build -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir - docker push public.ecr.aws/u5x5h6w3/$image_name:$tag_name + docker buildx build --push --platform linux/amd64 -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir echo "Built an image with name --> $image_name:$tag_name" done diff --git a/.github/workflows/get_layer_info_for_nf_imgs.yml b/.github/workflows/get_layer_info_for_nf_imgs.yml index b6072c1e..ddb15c02 100644 --- a/.github/workflows/get_layer_info_for_nf_imgs.yml +++ b/.github/workflows/get_layer_info_for_nf_imgs.yml @@ -29,7 +29,7 @@ jobs: if [ -z "$layer_json" ]; then layer_json="{}" fi - echo "Line: $LINENO layer_json=$layer_json" + echo "Line $LINENO: layer_json=$layer_json" # Function to fetch the manifest with retry logic for rate limiting fetch_manifest() { local url=$1 @@ -45,34 +45,67 @@ jobs: echo "Line $LINENO: Rate limit exceeded, waiting for $wait_time seconds before retrying..." >&2 sleep $wait_time wait_time=$((wait_time * 2)) # Exponential backoff - else + continue + fi + + mediaType=$(jq -r '.mediaType // empty' <<<"$response") + + if [ "$mediaType" == "application/vnd.oci.image.index.v1+json" ]; then + # Pick the amd64/linux image manifest (ignore attestations) + digest=$(jq -r ' + .manifests[] + | select(.platform.architecture=="amd64" and .platform.os=="linux") + | .digest + ' <<<"$response") + + if [ -z "$digest" ]; then + echo "Line $LINENO: No amd64/linux manifest found in OCI index" >&2 + return 1 + fi + + # Replace tag with digest to form concrete manifest URL + resolved_url="$(sed "s|/manifests/.*|/manifests/$digest|" <<<"$url")" + + # Fetch and echo the resolved image manifest + response=$(curl -s \ + -H "Authorization: Bearer $TOKEN" \ + -H "Accept: application/vnd.oci.image.manifest.v1+json" \ + "$resolved_url") + echo "$response" return fi + + echo "$response" + return + done echo "Line $LINENO: Failed to fetch manifest after $retries attempts due to rate limiting." >&2 return 1 } + echo "Line $LINENO: started reading lines from script" while IFS= read -r image_url; do # Ignore lines that start with # - if [[ "$image_url" == \#* ]]; then - echo "Line_$LINENO: Skipping this line -- $image_url" - continue - fi + if [[ "$image_url" == \#* ]]; then + echo "Line $LINENO: Skipping this line -- $image_url" + continue + fi # Strip the first * if a line starts with * - if [[ "$image_url" == \** ]]; then - echo "Line_$LINENO: Stripping * from this line -- $image_url" - image_url=$(echo "${image_url:1}"| sed 's/^[ \t]*//;s/[ \t]*$//') - fi - echo "Line $LINENO: started reading lines from script" + if [[ "$image_url" == \** ]]; then + echo "Line $LINENO: Stripping * from this line -- $image_url" + image_url=$(echo "${image_url:1}"| sed 's/^[ \t]*//;s/[ \t]*$//') + fi + manifest_url=$(echo "${image_url}" | sed 's|public\.ecr\.aws/\(.*\):\(.*\)|https://public.ecr.aws/v2/\1/manifests/\2|') tag_name=$(echo "${image_url}" | sed 's|\(.*\):\(.*\)|\2|') - echo "$LINENO: manifest_url = $manifest_url, tag_name = $tag_name" + echo "Line $LINENO: manifest_url = $manifest_url - tag_name = $tag_name" # Fetch the manifest with retry using exponential backoff - response=$(fetch_manifest "$manifest_url" | jq "[.layers[].digest]|.[-1]") + response=$(fetch_manifest "$manifest_url") + echo "Response from manifest_url: $response" + response=$(echo "${response}" | jq "[.layers[].digest]|.[-1]") if [ $? -ne 0 ] || [ -z "$response" ]; then echo "Line $LINENO: Failed to retrieve valid response from manifest_url -- $manifest_url" >&2 @@ -94,7 +127,7 @@ jobs: done <<< "$url_list" if [ -n "$layer_json" ]; then - echo "$layer_json" > $output_file_name + echo "$layer_json" > $output_file_name fi - name: Commit and push changes uses: stefanzweifel/git-auto-commit-action@v5 diff --git a/nextflow-base-images/gen3-cuda-11.8-ubuntu22.04-openssl/Dockerfile b/nextflow-base-images/gen3-cuda-11.8-ubuntu22.04-openssl/Dockerfile index 28d14dad..85778e5c 100644 --- a/nextflow-base-images/gen3-cuda-11.8-ubuntu22.04-openssl/Dockerfile +++ b/nextflow-base-images/gen3-cuda-11.8-ubuntu22.04-openssl/Dockerfile @@ -1,36 +1,45 @@ # Use the specified base image -FROM nvcr.io/nvidia/cuda:11.8.0-base-ubuntu22.04 - -RUN apt-get purge -y --auto-remove openssl && apt-get autoremove && apt-get autoclean +FROM nvcr.io/nvidia/cuda:11.8.0-base-ubuntu22.04 AS openssl-builder -RUN apt-get update && apt-get -y upgrade && apt install -y wget +# Install build deps only +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + wget \ + ca-certificates && \ + rm -rf /var/lib/apt/lists/* -RUN apt install -y build-essential && \ - apt-get install -y python3 && \ - apt-get install -y python3-pip - -# install openssl 3.0.8 as it is required for FIPS compliance. WORKDIR /tmp RUN wget https://www.openssl.org/source/openssl-3.0.8.tar.gz && \ - tar -xzvf openssl-3.0.8.tar.gz && \ - rm openssl-3.0.8.tar.gz + tar -xzf openssl-3.0.8.tar.gz WORKDIR /tmp/openssl-3.0.8 -RUN ./Configure enable-fips && \ - make && \ - make install +RUN ./Configure enable-fips && make -j$(nproc) && make install_sw install_ssldirs -# Changing adding `/usr/local/lib` as a prefix to LD_LIBRARY_PATH will -# give precedence to OpenSSL 3.0.8 library files over the 3.0.2 -ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH +FROM nvcr.io/nvidia/cuda:11.8.0-base-ubuntu22.04 + +ENV DEBIAN_FRONTEND=noninteractive -# Make config changes ti ensure FIPS compliance -RUN sed -i 's$# .include fipsmodule.cnf$.include /usr/local/ssl/fipsmodule.cnf$g' /usr/local/ssl/openssl.cnf -RUN sed -i 's$providers = provider_sect$providers = provider_sect\nalg_section = algorithm_sect$g' /usr/local/ssl/openssl.cnf -RUN sed -i 's$# fips = fips_sect$fips = fips_sect$g' /usr/local/ssl/openssl.cnf -RUN sed -i -e 's$# activate = 1$activate = 1 \n\n[algorithm_sect]\ndefault_properties = fips=yes$g' /usr/local/ssl/openssl.cnf +# Install runtime deps only +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + python3 \ + python3-pip \ + libglib2.0-0 && \ + rm -rf /var/lib/apt/lists/* +# Copy OpenSSL runtime artifacts only +COPY --from=openssl-builder /usr/local/ssl /usr/local/ssl +COPY --from=openssl-builder /usr/local/lib /usr/local/lib +COPY --from=openssl-builder /usr/local/lib64 /usr/local/lib64 +COPY --from=openssl-builder /usr/local/bin/openssl /usr/local/bin/openssl + +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH +ENV OPENSSL_CONF=/usr/local/ssl/openssl.cnf -# Clean up the temporary directory +# FIPS config +RUN sed -i 's$# .include fipsmodule.cnf$.include /usr/local/ssl/fipsmodule.cnf$g' /usr/local/ssl/openssl.cnf && \ + sed -i 's$providers = provider_sect$providers = provider_sect\nalg_section = algorithm_sect$g' /usr/local/ssl/openssl.cnf && \ + sed -i 's$# fips = fips_sect$fips = fips_sect$g' /usr/local/ssl/openssl.cnf && \ + sed -i -e 's$# activate = 1$activate = 1 \n\n[algorithm_sect]\ndefault_properties = fips=yes$g' /usr/local/ssl/openssl.cnf WORKDIR / -RUN rm -rf /tmp/openssl-3.0.8 diff --git a/nextflow-base-images/gen3-cuda-12.3-torch2.2-ubuntu22.04-openssl/Dockerfile b/nextflow-base-images/gen3-cuda-12.3-torch2.2-ubuntu22.04-openssl/Dockerfile index 5ff21418..9b3467f3 100644 --- a/nextflow-base-images/gen3-cuda-12.3-torch2.2-ubuntu22.04-openssl/Dockerfile +++ b/nextflow-base-images/gen3-cuda-12.3-torch2.2-ubuntu22.04-openssl/Dockerfile @@ -3,20 +3,19 @@ FROM public.ecr.aws/u5x5h6w3/nextflow-approved/public:gen3-cuda-12.3-ubuntu22.04 # Set environment variables ENV DEBIAN_FRONTEND=noninteractive -# Install system dependencies +# Install system dependencies + clean up RUN apt-get update && \ - apt-get install -y \ + apt-get install -y --no-install-recommends \ git \ python3-pip \ python3-dev \ python3-opencv \ - libglib2.0-0 - -# Upgrade pip -RUN python3 -m pip install --upgrade pip - -# Install PyTorch and torchvision -RUN pip3 install torch -f https://download.pytorch.org/whl/cu123/torch_stable.html + libglib2.0-0 && \ + rm -rf /var/lib/apt/lists/* +# Upgrade pip and install torch (NO CACHE) +RUN python3 -m pip install --upgrade pip && \ + pip3 install --no-cache-dir torch \ + -f https://download.pytorch.org/whl/cu123/torch_stable.html # Set the working directory WORKDIR /app diff --git a/nextflow-base-images/gen3-cuda-12.3-ubuntu22.04-openssl/Dockerfile b/nextflow-base-images/gen3-cuda-12.3-ubuntu22.04-openssl/Dockerfile index 65e8347f..b66df72a 100644 --- a/nextflow-base-images/gen3-cuda-12.3-ubuntu22.04-openssl/Dockerfile +++ b/nextflow-base-images/gen3-cuda-12.3-ubuntu22.04-openssl/Dockerfile @@ -1,36 +1,45 @@ # Use the specified base image -FROM nvcr.io/nvidia/cuda:12.3.1-base-ubuntu22.04 - -RUN apt-get purge -y --auto-remove openssl && apt-get autoremove && apt-get autoclean +FROM nvcr.io/nvidia/cuda:12.3.1-base-ubuntu22.04 AS openssl-builder -RUN apt-get update && apt-get -y upgrade && apt install -y wget +# Install build deps only +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + wget \ + ca-certificates && \ + rm -rf /var/lib/apt/lists/* -RUN apt install -y build-essential && \ - apt-get install -y python3 && \ - apt-get install -y python3-pip - -# install openssl 3.0.8 as it is required for FIPS compliance. WORKDIR /tmp RUN wget https://www.openssl.org/source/openssl-3.0.8.tar.gz && \ - tar -xzvf openssl-3.0.8.tar.gz && \ - rm openssl-3.0.8.tar.gz + tar -xzf openssl-3.0.8.tar.gz WORKDIR /tmp/openssl-3.0.8 -RUN ./Configure enable-fips && \ - make && \ - make install +RUN ./Configure enable-fips && make -j$(nproc) && make install_sw install_ssldirs -# Changing adding `/usr/local/lib` as a prefix to LD_LIBRARY_PATH will -# give precedence to OpenSSL 3.0.8 library files over the 3.0.2 -ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH +FROM nvcr.io/nvidia/cuda:12.3.1-base-ubuntu22.04 + +ENV DEBIAN_FRONTEND=noninteractive -# Make config changes ti ensure FIPS compliance -RUN sed -i 's$# .include fipsmodule.cnf$.include /usr/local/ssl/fipsmodule.cnf$g' /usr/local/ssl/openssl.cnf -RUN sed -i 's$providers = provider_sect$providers = provider_sect\nalg_section = algorithm_sect$g' /usr/local/ssl/openssl.cnf -RUN sed -i 's$# fips = fips_sect$fips = fips_sect$g' /usr/local/ssl/openssl.cnf -RUN sed -i -e 's$# activate = 1$activate = 1 \n\n[algorithm_sect]\ndefault_properties = fips=yes$g' /usr/local/ssl/openssl.cnf +# Install runtime deps only +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + python3 \ + python3-pip \ + libglib2.0-0 && \ + rm -rf /var/lib/apt/lists/* +# Copy OpenSSL runtime artifacts only +COPY --from=openssl-builder /usr/local/ssl /usr/local/ssl +COPY --from=openssl-builder /usr/local/lib /usr/local/lib +COPY --from=openssl-builder /usr/local/lib64 /usr/local/lib64 +COPY --from=openssl-builder /usr/local/bin/openssl /usr/local/bin/openssl + +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH +ENV OPENSSL_CONF=/usr/local/ssl/openssl.cnf -# Clean up the temporary directory +# FIPS config +RUN sed -i 's$# .include fipsmodule.cnf$.include /usr/local/ssl/fipsmodule.cnf$g' /usr/local/ssl/openssl.cnf && \ + sed -i 's$providers = provider_sect$providers = provider_sect\nalg_section = algorithm_sect$g' /usr/local/ssl/openssl.cnf && \ + sed -i 's$# fips = fips_sect$fips = fips_sect$g' /usr/local/ssl/openssl.cnf && \ + sed -i -e 's$# activate = 1$activate = 1 \n\n[algorithm_sect]\ndefault_properties = fips=yes$g' /usr/local/ssl/openssl.cnf WORKDIR / -RUN rm -rf /tmp/openssl-3.0.8