From 15dc77a6569f4e3217176db8992351e2964546f9 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Sun, 15 Mar 2026 19:44:52 -0700 Subject: [PATCH] fix(ci): keep wheel publishing trigger-only --- .github/workflows/release-dev.yml | 88 ------------------------------ .github/workflows/release-tag.yml | 91 +------------------------------ mise.toml | 3 - tasks/scripts/release.py | 63 --------------------- 4 files changed, 1 insertion(+), 244 deletions(-) diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index c8515e92..ad5c0706 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -96,7 +96,6 @@ jobs: timeout-minutes: 120 outputs: wheel_version: ${{ needs.compute-versions.outputs.python_version }} - s3_prefix: ${{ steps.upload.outputs.s3_prefix }} container: image: ghcr.io/nvidia/openshell/ci:latest credentials: @@ -109,10 +108,6 @@ jobs: MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }} OPENSHELL_IMAGE_TAG: dev - NAV_PYPI_S3_BUCKET: navigator-pypi-artifacts # TODO: rename bucket to openshell-pypi-artifacts - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-west-2 steps: - uses: actions/checkout@v4 with: @@ -137,16 +132,6 @@ jobs: OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run python:build:macos ls -la target/wheels/*.whl - - name: Upload wheels to S3 - id: upload - run: | - set -euo pipefail - WHEEL_VERSION="${{ needs.compute-versions.outputs.python_version }}" - S3_PREFIX="openshell/${WHEEL_VERSION}" - aws s3 cp target/wheels/ "s3://${NAV_PYPI_S3_BUCKET}/${S3_PREFIX}/" --recursive --exclude "*" --include "*.whl" - aws s3 ls "s3://${NAV_PYPI_S3_BUCKET}/${S3_PREFIX}/" - echo "s3_prefix=${S3_PREFIX}" >> "$GITHUB_OUTPUT" - - name: Upload wheel artifacts uses: actions/upload-artifact@v4 with: @@ -154,79 +139,6 @@ jobs: path: target/wheels/*.whl retention-days: 5 - publish-python: - name: Publish Python - needs: [build-python-wheels] - runs-on: [self-hosted, nv] - timeout-minutes: 10 - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - NAV_PYPI_S3_BUCKET: navigator-pypi-artifacts # TODO: rename bucket to openshell-pypi-artifacts - NAV_PYPI_REPOSITORY_URL: https://urm.nvidia.com/artifactory/api/pypi/nv-shared-pypi-local - NAV_PYPI_USERNAME: ${{ secrets.NAV_PYPI_USERNAME }} - NAV_PYPI_PASSWORD: ${{ secrets.NAV_PYPI_PASSWORD }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-west-2 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: actions/setup-python@v5 - with: - python-version: "3.13" - - - name: Install publish dependencies - run: | - set -euo pipefail - python -m pip install --upgrade pip uv - - if ! command -v aws >/dev/null 2>&1; then - ARCH="$(uname -m)" - case "$ARCH" in - x86_64|amd64) AWSCLI_ARCH="x86_64" ;; - aarch64|arm64) AWSCLI_ARCH="aarch64" ;; - *) - echo "Unsupported architecture for AWS CLI installer: $ARCH" >&2 - exit 1 - ;; - esac - - rm -rf aws awscliv2.zip - curl --fail --silent --show-error --location \ - "https://awscli.amazonaws.com/awscli-exe-linux-${AWSCLI_ARCH}.zip" \ - --output awscliv2.zip - unzip -q awscliv2.zip - ./aws/install --install-dir "$HOME/.local/aws-cli" --bin-dir "$HOME/.local/bin" --update - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - export PATH="$HOME/.local/bin:$PATH" - fi - - aws --version - uv --version - - - name: List and download versioned wheels from S3 - run: | - set -euo pipefail - WHEEL_VERSION="${{ needs.build-python-wheels.outputs.wheel_version }}" - S3_PREFIX="${{ needs.build-python-wheels.outputs.s3_prefix }}" - OBJECT_COUNT=$(aws s3api list-objects-v2 --bucket "$NAV_PYPI_S3_BUCKET" --prefix "${S3_PREFIX}/" --query "length(Contents)" --output text) - if [ "$OBJECT_COUNT" = "None" ] || [ "$OBJECT_COUNT" = "0" ]; then - echo "No wheel artifacts found for ${WHEEL_VERSION} at s3://${NAV_PYPI_S3_BUCKET}/${S3_PREFIX}/" >&2 - exit 1 - fi - aws s3api list-objects-v2 --bucket "$NAV_PYPI_S3_BUCKET" --prefix "${S3_PREFIX}/" --query "Contents[].Key" --output text - mkdir -p target/wheels - aws s3 cp "s3://${NAV_PYPI_S3_BUCKET}/${S3_PREFIX}/" target/wheels/ --recursive --exclude "*" --include "*.whl" - ls -la target/wheels/*.whl - - - name: Publish wheels to Artifactory - run: | - set -euo pipefail - WHEEL_VERSION="${{ needs.build-python-wheels.outputs.wheel_version }}" - uv run python tasks/scripts/release.py python-publish --version "$WHEEL_VERSION" - # --------------------------------------------------------------------------- # Build CLI binaries (Linux musl — static, native on each arch) # --------------------------------------------------------------------------- diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 38f29fcb..e36de607 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -116,7 +116,6 @@ jobs: timeout-minutes: 120 outputs: wheel_version: ${{ needs.compute-versions.outputs.python_version }} - s3_prefix: ${{ steps.upload.outputs.s3_prefix }} container: image: ghcr.io/nvidia/openshell/ci:latest credentials: @@ -129,10 +128,6 @@ jobs: MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }} OPENSHELL_IMAGE_TAG: ${{ needs.compute-versions.outputs.semver }} - NAV_PYPI_S3_BUCKET: navigator-pypi-artifacts # TODO: rename bucket to openshell-pypi-artifacts - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-west-2 steps: - uses: actions/checkout@v4 with: @@ -158,16 +153,6 @@ jobs: OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run python:build:macos ls -la target/wheels/*.whl - - name: Upload wheels to S3 - id: upload - run: | - set -euo pipefail - WHEEL_VERSION="${{ needs.compute-versions.outputs.python_version }}" - S3_PREFIX="openshell/${WHEEL_VERSION}" - aws s3 cp target/wheels/ "s3://${NAV_PYPI_S3_BUCKET}/${S3_PREFIX}/" --recursive --exclude "*" --include "*.whl" - aws s3 ls "s3://${NAV_PYPI_S3_BUCKET}/${S3_PREFIX}/" - echo "s3_prefix=${S3_PREFIX}" >> "$GITHUB_OUTPUT" - - name: Upload wheel artifacts uses: actions/upload-artifact@v4 with: @@ -175,80 +160,6 @@ jobs: path: target/wheels/*.whl retention-days: 5 - publish-python: - name: Publish Python - needs: [build-python-wheels, e2e] - runs-on: [self-hosted, nv] - timeout-minutes: 10 - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - NAV_PYPI_S3_BUCKET: navigator-pypi-artifacts # TODO: rename bucket to openshell-pypi-artifacts - NAV_PYPI_REPOSITORY_URL: https://urm.nvidia.com/artifactory/api/pypi/nv-shared-pypi-local - NAV_PYPI_USERNAME: ${{ secrets.NAV_PYPI_USERNAME }} - NAV_PYPI_PASSWORD: ${{ secrets.NAV_PYPI_PASSWORD }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-west-2 - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.tag || github.ref }} - fetch-depth: 0 - - - uses: actions/setup-python@v5 - with: - python-version: "3.13" - - - name: Install publish dependencies - run: | - set -euo pipefail - python -m pip install --upgrade pip uv - - if ! command -v aws >/dev/null 2>&1; then - ARCH="$(uname -m)" - case "$ARCH" in - x86_64|amd64) AWSCLI_ARCH="x86_64" ;; - aarch64|arm64) AWSCLI_ARCH="aarch64" ;; - *) - echo "Unsupported architecture for AWS CLI installer: $ARCH" >&2 - exit 1 - ;; - esac - - rm -rf aws awscliv2.zip - curl --fail --silent --show-error --location \ - "https://awscli.amazonaws.com/awscli-exe-linux-${AWSCLI_ARCH}.zip" \ - --output awscliv2.zip - unzip -q awscliv2.zip - ./aws/install --install-dir "$HOME/.local/aws-cli" --bin-dir "$HOME/.local/bin" --update - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - export PATH="$HOME/.local/bin:$PATH" - fi - - aws --version - uv --version - - - name: List and download versioned wheels from S3 - run: | - set -euo pipefail - WHEEL_VERSION="${{ needs.build-python-wheels.outputs.wheel_version }}" - S3_PREFIX="${{ needs.build-python-wheels.outputs.s3_prefix }}" - OBJECT_COUNT=$(aws s3api list-objects-v2 --bucket "$NAV_PYPI_S3_BUCKET" --prefix "${S3_PREFIX}/" --query "length(Contents)" --output text) - if [ "$OBJECT_COUNT" = "None" ] || [ "$OBJECT_COUNT" = "0" ]; then - echo "No wheel artifacts found for ${WHEEL_VERSION} at s3://${NAV_PYPI_S3_BUCKET}/${S3_PREFIX}/" >&2 - exit 1 - fi - aws s3api list-objects-v2 --bucket "$NAV_PYPI_S3_BUCKET" --prefix "${S3_PREFIX}/" --query "Contents[].Key" --output text - mkdir -p target/wheels - aws s3 cp "s3://${NAV_PYPI_S3_BUCKET}/${S3_PREFIX}/" target/wheels/ --recursive --exclude "*" --include "*.whl" - ls -la target/wheels/*.whl - - - name: Publish wheels to Artifactory - run: | - set -euo pipefail - WHEEL_VERSION="${{ needs.build-python-wheels.outputs.wheel_version }}" - uv run python tasks/scripts/release.py python-publish --version "$WHEEL_VERSION" - # --------------------------------------------------------------------------- # Build CLI binaries (Linux musl — static, native on each arch) # --------------------------------------------------------------------------- @@ -413,7 +324,7 @@ jobs: # --------------------------------------------------------------------------- release: name: Release - needs: [compute-versions, build-cli-linux, build-cli-macos, build-python-wheels, publish-python, tag-ghcr-release] + needs: [compute-versions, build-cli-linux, build-cli-macos, build-python-wheels, tag-ghcr-release] runs-on: build-amd64 timeout-minutes: 10 steps: diff --git a/mise.toml b/mise.toml index 5b6ce835..115b13c0 100644 --- a/mise.toml +++ b/mise.toml @@ -36,9 +36,6 @@ UV_CACHE_DIR = "{{config_root}}/.cache/uv" RUSTC_WRAPPER = "sccache" SCCACHE_DIR = "{{config_root}}/.cache/sccache" -# Artifact registries -NAV_PYPI_REPOSITORY_URL = "https://urm.nvidia.com/artifactory/api/pypi/nv-shared-pypi-local" - # Shared build constants (overridable via environment) DOCKER_BUILDKIT = "1" diff --git a/tasks/scripts/release.py b/tasks/scripts/release.py index e4daef7a..57e8fbf5 100644 --- a/tasks/scripts/release.py +++ b/tasks/scripts/release.py @@ -6,8 +6,6 @@ from __future__ import annotations import argparse -import fnmatch -import os import re import subprocess from dataclasses import dataclass @@ -69,53 +67,6 @@ def _compute_versions() -> Versions: ) -def python_publish( - version: str | None = None, wheel_glob: str = "openshell-*.whl" -) -> None: - if version is None: - version = _compute_versions().python - - repo_url = os.getenv("NAV_PYPI_REPOSITORY_URL") - username = os.getenv("NAV_PYPI_USERNAME") - password = os.getenv("NAV_PYPI_PASSWORD") - - if not repo_url or not username or not password: - raise SystemExit( - "Auth is not set up for publishing to PyPI registry, see CONTRIBUTING.md for details." - ) - - env = dict(os.environ) - env["UV_PUBLISH_USERNAME"] = username - env["UV_PUBLISH_PASSWORD"] = password - - wheels_dir = _repo_root() / "target" / "wheels" - wheels_dir.mkdir(parents=True, exist_ok=True) - - wheel_paths = sorted( - p - for p in wheels_dir.glob("*.whl") - if f"-{version}-" in p.name and fnmatch.fnmatch(p.name, wheel_glob) - ) - - if not wheel_paths: - available = "\n".join(sorted(p.name for p in wheels_dir.glob("*.whl"))) - raise SystemExit( - f"No wheels found for version {version} in {wheels_dir}.\n" - f"Available wheels:\n{available or '(none)'}" - ) - - _run( - [ - "uv", - "publish", - "--publish-url", - repo_url, - *[str(path) for path in wheel_paths], - ], - env=env, - ) - - def get_version(format: str) -> None: versions = _compute_versions() if format == "python": @@ -147,18 +98,6 @@ def build_parser() -> argparse.ArgumentParser: "--docker", action="store_true", help="Print Docker version only." ) - python_publish_parser = sub.add_parser( - "python-publish", help="Publish python wheel." - ) - python_publish_parser.add_argument( - "--version", help="Version to publish (defaults to computed version)." - ) - python_publish_parser.add_argument( - "--wheel-glob", - default="openshell-*.whl", - help="Filename glob for wheels to publish (defaults to all openshell wheels).", - ) - return parser @@ -175,8 +114,6 @@ def main() -> None: get_version("docker") else: get_version("all") - elif args.command == "python-publish": - python_publish(version=args.version, wheel_glob=args.wheel_glob) if __name__ == "__main__":