diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index d392615213..992c0849a7 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -1,3 +1,4 @@ + name: PR build on: @@ -27,13 +28,18 @@ run-name: > || github.event.pull_request.title }} jobs: - validate: + + build_info: runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} steps: - name: Checkout code (Pull Request) if: github.event_name == 'pull_request' uses: actions/checkout@v6 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + fetch-depth: 0 - name: Checkout code (Workflow Dispatch) if: github.event_name == 'workflow_dispatch' @@ -44,7 +50,7 @@ jobs: - name: Install required packages run: | sudo apt update -y - sudo apt-get install -y file + sudo apt-get install -y file jq - name: Install Python dependencies run: | @@ -59,23 +65,553 @@ jobs: python3 -u gha-script/validate_builds.py ${PR_NUMBER:-false} 2>&1 | tee build_log my_pid_status=${PIPESTATUS[0]} - build_size=$(stat -c %s build_log) - if [ "$my_pid_status" -ne 0 ]; then - echo "Script failed for PR #${PR_NUMBER}" - if [ "$build_size" -lt 1800000 ]; then - cat build_log - else - echo "Build log too large, showing last 100 lines" - tail -100 build_log - fi - exit 1 + echo "Script failed for PR #${PR_NUMBER}" + echo "::group::Validation Logs" + tail -200 build_log + echo "::endgroup::" + exit 1 + else + echo "Script completed successfully for PR #${PR_NUMBER}" + fi + + - name: Fetch base branch + run: git fetch origin ${{ github.base_ref }} --depth=1 + + - name: Locate and parse build_info.json + run: | + + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) + + BUILD_INFO_FILE=$(echo "$CHANGED_FILES" | grep 'build_info.json' | head -n 1) + + if [ -z "$BUILD_INFO_FILE" ]; then + echo "No build_info.json modified, trying to detect from changed files..." + + PACKAGE_DIR=$(echo "$CHANGED_FILES" | head -n 1 | cut -d'/' -f1-2) + + BUILD_INFO_FILE="$PACKAGE_DIR/build_info.json" + + if [ ! -f "$BUILD_INFO_FILE" ]; then + echo "Could not locate build_info.json!" + exit 1 + fi + + echo "Using fallback build_info: $BUILD_INFO_FILE" + fi + + PACKAGE_NAME=$(jq -r '.package_name // ""' $BUILD_INFO_FILE) + VERSION=$(jq -r '.version // ""' $BUILD_INFO_FILE) + + echo "BUILD_INFO_FILE=$BUILD_INFO_FILE" >> $GITHUB_ENV + echo "PACKAGE_NAME=$PACKAGE_NAME" >> $GITHUB_ENV + echo "VERSION=$VERSION" >> $GITHUB_ENV + echo "CHANGED_FILES<> $GITHUB_ENV + echo "$CHANGED_FILES" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: Read build_info.json + run: | + chmod +x ./gha-script/read_buildinfo.sh + bash ./gha-script/read_buildinfo.sh + + - name: Create scanner-env.sh + run: | + mkdir -p package-cache + + PACKAGE_DIR=$(jq -r '.package_dir // ""' $BUILD_INFO_FILE) + WHEEL_BUILD=$(jq -r '.wheel_build // "false"' $BUILD_INFO_FILE) + + # Robust docker_build extraction + if jq -e '.docker_build == true or .docker_build == "true"' "$BUILD_INFO_FILE" > /dev/null; then + DOCKER_BUILD="true" else - echo "Script completed successfully for PR #${PR_NUMBER}" - if [ "$build_size" -lt 1800000 ]; then - cat build_log - else - echo "Build log too large, showing last 100 lines" - tail -100 build_log - fi + DOCKER_BUILD="false" + fi + + cat < package-cache/scanner-env.sh + export PACKAGE_NAME=$PACKAGE_NAME + export VERSION=$VERSION + export PACKAGE_DIR=$PACKAGE_DIR + export WHEEL_BUILD=$WHEEL_BUILD + export BUILD_DOCKER=$DOCKER_BUILD + export CHANGED_FILES="$CHANGED_FILES" + EOF + + mv variable.sh package-cache/ + + - name: Archive package cache + run: tar -czf package-cache.tar.gz package-cache/ + + - name: Upload package cache + uses: actions/upload-artifact@v6 + with: + name: package-cache + path: package-cache.tar.gz + + build: + needs: build_info + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + + steps: + - uses: actions/checkout@v6 + + - name: Download package-cache + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Build Package + run: | + source package-cache/variable.sh + source package-cache/scanner-env.sh + + echo "------------------- variable.sh -----------------------------" + cat package-cache/variable.sh + echo "------------------- scanner-env.sh -----------------------------" + cat package-cache/scanner-env.sh + + chmod +x ./gha-script/build_package.sh + bash ./gha-script/build_package.sh + +# ===================== WHEEL JOBS ===================== + + + wheel_build_py39: + needs: build_info + if: ${{ success() }} + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + continue-on-error: true + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + PYTHON_VERSION: "3.9" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install system dependencies + run: | + sudo apt update -y + + - name: Download package-cache from previous step + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Run build_wheels.sh + run: | + ls + echo "---------------------updated cache-----------------------" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + # CONTROL FLAG HERE + if [ "$WHEEL_BUILD" != "true" ]; then + echo "Skipping wheel build as WHEEL_BUILD=false" + exit 0 + fi + + # Check if any .sh build script is modified in this package + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then + echo "Skipping wheel build as no .sh build script changes detected" + exit 0 + fi + + chmod +x ./gha-script/build_wheels.sh + bash ./gha-script/build_wheels.sh + + echo "===========after execution ==================" + sudo apt update -y + sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" + sudo uname -a + + + wheel_build_py310: + needs: build_info + if: ${{ success() }} + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + continue-on-error: false + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + PYTHON_VERSION: "3.10" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install system dependencies + run: | + sudo apt update -y + + - name: Download package-cache from previous step + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Run build_wheels.sh + run: | + ls + echo "---------------------updated cache-----------------------" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + # CONTROL FLAG HERE + if [ "$WHEEL_BUILD" != "true" ]; then + echo "Skipping wheel build as WHEEL_BUILD=false" + exit 0 + fi + + # Check if any .sh build script is modified in this package + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then + echo "Skipping wheel build as no .sh build script changes detected" + exit 0 + fi + + chmod +x ./gha-script/build_wheels.sh + bash ./gha-script/build_wheels.sh + + echo "===========after execution ==================" + sudo apt update -y + sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" + sudo uname -a + + + wheel_build_py311: + needs: build_info + if: ${{ success() }} + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + continue-on-error: false + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + PYTHON_VERSION: "3.11" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install system dependencies + run: | + sudo apt update -y + + - name: Download package-cache from previous step + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Run build_wheels.sh + run: | + ls + echo "---------------------updated cache-----------------------" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + # CONTROL FLAG HERE + if [ "$WHEEL_BUILD" != "true" ]; then + echo "Skipping wheel build as WHEEL_BUILD=false" + exit 0 + fi + + # Check if any .sh build script is modified in this package + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then + echo "Skipping wheel build as no .sh build script changes detected" + exit 0 + fi + + chmod +x ./gha-script/build_wheels.sh + bash ./gha-script/build_wheels.sh + + echo "===========after execution ==================" + sudo apt update -y + sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" + sudo uname -a + + + + wheel_build_py312: + needs: build_info + if: ${{ success() }} + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + continue-on-error: false + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + PYTHON_VERSION: "3.12" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install system dependencies + run: | + sudo apt update -y + + - name: Download package-cache from previous step + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Run build_wheels.sh + run: | + ls + echo "---------------------updated cache-----------------------" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + # CONTROL FLAG HERE + if [ "$WHEEL_BUILD" != "true" ]; then + echo "Skipping wheel build as WHEEL_BUILD=false" + exit 0 fi + + # Check if any .sh build script is modified in this package + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then + echo "Skipping wheel build as no .sh build script changes detected" + exit 0 + fi + + chmod +x ./gha-script/build_wheels.sh + bash ./gha-script/build_wheels.sh + + echo "===========after execution ==================" + sudo apt update -y + sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" + sudo uname -a + + wheel_build_py313: + needs: build_info + if: ${{ success() }} + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + continue-on-error: true + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + PYTHON_VERSION: "3.13" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install system dependencies + run: | + sudo apt update -y + + - name: Download package-cache from previous step + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Run build_wheels.sh + run: | + ls + echo "---------------------updated cache-----------------------" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + # CONTROL FLAG HERE + if [ "$WHEEL_BUILD" != "true" ]; then + echo "Skipping wheel build as WHEEL_BUILD=false" + exit 0 + fi + + # Check if any .sh build script is modified in this package + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then + echo "Skipping wheel build as no .sh build script changes detected" + exit 0 + fi + + chmod +x ./gha-script/build_wheels.sh + bash ./gha-script/build_wheels.sh + + echo "===========after execution ==================" + sudo apt update -y + sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" + sudo uname -a + + wheel_build_py314: + needs: build_info + if: ${{ success() }} + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + continue-on-error: true + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + PYTHON_VERSION: "3.14" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install system dependencies + run: | + sudo apt update -y + + - name: Download package-cache from previous step + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Run build_wheels.sh + run: | + ls + echo "---------------------updated cache-----------------------" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + # CONTROL FLAG HERE + if [ "$WHEEL_BUILD" != "true" ]; then + echo "Skipping wheel build as WHEEL_BUILD=false" + exit 0 + fi + + # Check if any .sh build script is modified in this package + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + + if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then + echo "Skipping wheel build as no .sh build script changes detected" + exit 0 + fi + + chmod +x ./gha-script/build_wheels.sh + bash ./gha-script/build_wheels.sh + + echo "===========after execution ==================" + sudo apt update -y + sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" + sudo uname -a + + + + build_docker: + needs: build_info + if: ${{ success() }} + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Download package-cache + uses: actions/download-artifact@v7 + with: + name: package-cache + + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + + - name: Build Docker Image + run: | + echo "===== Sourcing environment =====" + ls package-cache + + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + + BUILD_DOCKER=$(echo "$BUILD_DOCKER" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]') + DOCKERFILE_CHANGED=$(echo "$CHANGED_FILES" | grep -i 'Dockerfile' || true) + + if [[ "$BUILD_DOCKER" != "true" ]]; then + echo "Skipping Docker build as BUILD_DOCKER=$BUILD_DOCKER" + exit 0 + fi + + if [[ -z "$DOCKERFILE_CHANGED" ]]; then + echo "Skipping Docker build as no Dockerfile changes detected in PR" + exit 0 + fi + + echo "===== Starting Docker build =====" + chmod +x ./gha-script/build_docker.sh + bash ./gha-script/build_docker.sh + + echo "===== Docker images after build =====" + docker images + + echo "===== Saving Docker image =====" + docker save -o package-cache/image.tar "$IMAGE_NAME" + + ls -lh package-cache/image.tar + + + + + + + + + + + + diff --git a/gha-script/validate_builds.py b/gha-script/validate_builds.py index 7677b679e3..3a9ac93daa 100644 --- a/gha-script/validate_builds.py +++ b/gha-script/validate_builds.py @@ -12,7 +12,7 @@ GITHUB_BUILD_SCRIPT_BASE_REPO = "build-scripts" -GITHUB_BUILD_SCRIPT_BASE_OWNER = "ppc64le" +GITHUB_BUILD_SCRIPT_BASE_OWNER = "stutiibm" HOME = os.getcwd() package_data = {} diff --git a/p/param/Dockerfiles/v2.1.0_ubi_9/Dockerfile b/p/param/Dockerfiles/v2.1.0_ubi_9/Dockerfile index 7d758cdc83..a3a99b41b8 100644 --- a/p/param/Dockerfiles/v2.1.0_ubi_9/Dockerfile +++ b/p/param/Dockerfiles/v2.1.0_ubi_9/Dockerfile @@ -16,4 +16,5 @@ RUN yum update -y && \ git checkout $PACKAGE_VERSION && \ python3 -m pip install build + CMD ["/bin/bash"] diff --git a/p/param/build_info.json b/p/param/build_info.json index 1dd96f4800..0a613c9a31 100644 --- a/p/param/build_info.json +++ b/p/param/build_info.json @@ -15,3 +15,4 @@ "build_script": "param_ubi_9.3.sh" } } +