diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index 00f32e096a..8e365dc7d8 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -28,7 +28,55 @@ run-name: > jobs: + check_changes: + runs-on: ubuntu-latest + outputs: + should_build: ${{ steps.filter.outputs.should_build }} + changed_files: ${{ steps.filter.outputs.changed_files }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + fetch-depth: 0 + + - name: Check for relevant file changes + id: filter + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + echo "should_build=true" >> $GITHUB_OUTPUT + echo "Workflow dispatch - proceeding with build" + exit 0 + fi + + git fetch origin ${{ github.base_ref }} --depth=1 + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) + + echo "Changed files:" + echo "$CHANGED_FILES" + + # Store changed files for reuse in later jobs + echo "changed_files<> $GITHUB_OUTPUT + echo "$CHANGED_FILES" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + # Check for build_info.json, .sh scripts, or Dockerfile + RELEVANT_CHANGES=$(echo "$CHANGED_FILES" | grep -E '(build_info\.json|\.sh$|Dockerfile)' || true) + + if [ -n "$RELEVANT_CHANGES" ]; then + echo "should_build=true" >> $GITHUB_OUTPUT + echo "✅ Found relevant changes:" + echo "$RELEVANT_CHANGES" + else + echo "should_build=false" >> $GITHUB_OUTPUT + echo "⏭️ Skipping PR build CI check - no changes related to build_info.json, build scripts (.sh), or Dockerfile" + fi + + build_info: + needs: check_changes + if: needs.check_changes.outputs.should_build == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} steps: @@ -36,6 +84,8 @@ jobs: if: github.event_name == 'pull_request' uses: actions/checkout@v6 with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} fetch-depth: 0 - name: Checkout code (Workflow Dispatch) @@ -77,10 +127,50 @@ jobs: - name: Fetch base branch run: git fetch origin ${{ github.base_ref }} --depth=1 + # - name: Locate and parse build_info.json + # run: | + + # CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) + + # BUILD_INFO_FILE=$(echo "$CHANGED_FILES" | grep 'build_info.json' | head -n 1) + + # if [ -z "$BUILD_INFO_FILE" ]; then + # echo "No build_info.json modified, trying to detect from changed files..." + + # PACKAGE_DIR=$(echo "$CHANGED_FILES" | head -n 1 | cut -d'/' -f1-2) + + # BUILD_INFO_FILE="$PACKAGE_DIR/build_info.json" + + # if [ ! -f "$BUILD_INFO_FILE" ]; then + # echo "Could not locate build_info.json!" + # exit 1 + # fi + + # echo "Using fallback build_info: $BUILD_INFO_FILE" + # fi + + # PACKAGE_NAME=$(jq -r '.package_name // ""' $BUILD_INFO_FILE) + # VERSION=$(jq -r '.version // ""' $BUILD_INFO_FILE) + + # echo "BUILD_INFO_FILE=$BUILD_INFO_FILE" >> $GITHUB_ENV + # echo "PACKAGE_NAME=$PACKAGE_NAME" >> $GITHUB_ENV + # echo "VERSION=$VERSION" >> $GITHUB_ENV + # echo "CHANGED_FILES<> $GITHUB_ENV + # echo "$CHANGED_FILES" >> $GITHUB_ENV + # echo "EOF" >> $GITHUB_ENV + + + - name: Locate and parse build_info.json run: | - - CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) + # Reuse changed files from check_changes job + CHANGED_FILES="${{ needs.check_changes.outputs.changed_files }}" + + # If workflow_dispatch, fetch and compute changed files + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + git fetch origin ${{ github.base_ref }} --depth=1 + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) + fi BUILD_INFO_FILE=$(echo "$CHANGED_FILES" | grep 'build_info.json' | head -n 1) @@ -109,7 +199,6 @@ jobs: echo "$CHANGED_FILES" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - - name: Read build_info.json run: | chmod +x ./gha-script/read_buildinfo.sh diff --git a/gha-script/validate_builds.py b/gha-script/validate_builds.py index 54094dcce1..9b568fab84 100644 --- a/gha-script/validate_builds.py +++ b/gha-script/validate_builds.py @@ -11,7 +11,7 @@ GITHUB_BUILD_SCRIPT_BASE_REPO = "build-scripts" -GITHUB_BUILD_SCRIPT_BASE_OWNER = "ppc64le" +GITHUB_BUILD_SCRIPT_BASE_OWNER = "stutiibm" HOME = os.getcwd() package_data = {} diff --git a/process_bom/run_currency_processor.py b/process_bom/run_currency_processor.py index 962def69b9..62f14a50d7 100644 --- a/process_bom/run_currency_processor.py +++ b/process_bom/run_currency_processor.py @@ -1,25 +1,232 @@ +import json import os -import sys -from process_bom.CurrencyProcessor import CurrencyProcessor - -def main(): - # Read arguments passed from command line or environment - if len(sys.argv) >= 3: - package_name = sys.argv[1] - version = sys.argv[2] - else: - package_name = os.getenv("PACKAGE_NAME") - version = os.getenv("VERSION") - - if not package_name or not version: - print("❌ Error: Both package_name and version must be provided.") - sys.exit(1) - - print(f"Running CurrencyProcessor for {package_name=} and {version=}") - - # Initialize and call the method - cp = CurrencyProcessor() - cp.update_local_build_details_in_database(package_name, version) - -if __name__ == "__main__": - main() +import datetime +import gzip +import re + +from process_bom.ca_config import * +from process_bom.BOMProcessor import BOMProcessor +from process_bom.COSWrapper import COSWrapper +from process_bom.LicensesProcessor import LicensesProcessor + +class CurrencyProcessor: + + bom_processor = BOMProcessor() + licenses_processor = LicensesProcessor() + cos = COSWrapper(CLOUD_OBJECT_CVE_SBOM_BUCKET) + jenkins_jobs_database = 'currency_jenkins_build_history' + + database_name = "package_build_details" + currency_build_logs = "currency_build_logs" + + def update_local_build_details_in_database(self, package_name: str, version: str): + """ + Updates the local build details in the database for a given package name and version. + + Args: + package_name (str): The name of the package. + version (str): The version of the package. + response (dict): The response object containing the updated details. + + Returns: + None + """ + os.mkdir(SBOM_CVE_DIR) + required_package_details = self._get_package_details(package_name, version) + required_package_details["wheel_status"] = self.get_wheel_status(package_name, version) + required_package_details["Created"] = str(datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)) + + result = [] + try: + result.append(required_package_details) + #result = self._update_or_append(result, required_package_details, file_path_to_json) + self.get_image_details_for_package(result,package_name=package_name,version=version) + except Exception as e: + print("Exception occurred:", e) + + def get_wheel_status(self, package_name: str, version: str): + result = {} + cos = COSWrapper(CLOUD_OBJECT_CVE_SBOM_BUCKET) + + python_versions = ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + + for py_version in python_versions: + artifact_name = f"{package_name}/{version}/{package_name}_{version}_wheel_py{py_version}_log.gz" + key = f"wheel build {py_version}" + + try: + zip_path = cos.download_artifacts_gz(artifact_name) + + with gzip.open(zip_path, "rt", errors="ignore") as f: + content = f.read() + result[key] = self._get_build_status_from_log(content) + + except Exception: + result[key] = "failure" + + return result + + def _get_build_status_from_log(self, content: str) -> str: + lines = content.splitlines() + success_messages = [ + "SUCCESS: Wheels post process successfully." + ] + failure_messages = [ + "ERROR: Auditwheel failed.", + "ERROR: Expected exactly 1 wheel but found", + "ERROR: Auditwheel failed to repair wheel:", + "ERROR: Skipped wheel is not universal i.e(*any.whl).", + "Wheel Creation Failed for Python.", + "ERROR: Failed to post process wheels." + ] + # --- Check success first (exact line match) --- + for line in lines: + line_clean = line.strip().lower() + # Remove ===> prefix if present + if line_clean.startswith("===>"): + line_clean = line_clean[4:].strip() + for success in success_messages: + if line_clean == success.lower(): + return "success" + + # --- Check failure next (substring match) --- + content_lower = content.lower() + for failure in failure_messages: + if failure.lower() in content_lower: + return "failure" + # --- Default fallback --- + return "success" + + def _get_package_details(self, package_name: str, version: str): + """ + Retrieve package details from Bill of Materials (BOM) and add creation timestamp. + + Args: + package_name (str): Name of the package. + version (str): Version of the package. + + Returns: + dict: Package details including creation timestamp. + """ + required_package_details = self.bom_processor.get_bom_details_from_cos(package_name, version) + required_package_details["Created"] = str(datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)) + return required_package_details + + + def _update_or_append(self, result, required_package_details, file_path_to_json): + """ + Update or append a required package details to the result list. + + Parameters: + result (list): The list of required package details. + required_package_details (dict): The details of the required package. + file_path_to_json (str): The file path to the JSON file. + + Returns: + None + """ + for i, item in enumerate(result): + if item["Tag"] == required_package_details["Tag"]: + result[i] = required_package_details + self._write_to_file(result, file_path_to_json) + return + result.append(required_package_details) + return result + + def get_image_details_for_package(self, result,package_name: str,version: str ): + """ + Retrieve image details for a given package name. + + Parameters: + package_name (str): The name of the package for which image details are to be retrieved. + + Returns: + dict: A dictionary containing the image details for the specified package. + """ + + try: + new_results = self._normalize_json_response(result) + final_result = self._process_local_data(new_results, package_name=package_name) + with open(f"{package_name}_{version}.json", 'w') as outfile: + json.dump(final_result, outfile, indent=4) + cos = COSWrapper(CLOUD_OBJECT_CVE_SBOM_BUCKET) + artifact_name=f"{package_name}_{version}.json" + response = cos.push_artifacts_sbomcve(artifact_name=artifact_name) + if response: + print(f"Successfully pushed {package_name}_{version}.json to Cloud Object Storage.") + except FileExistsError as e: + print("File does not exist:", e) + return {} + + def _normalize_json_response(self, data): + """ + Ensures JSON response is always a list of dicts. + - If it's a dict ({}), wrap in a list. + - If it's already a list ([{}]), return as-is. + """ + if isinstance(data, dict): + return [data] + elif isinstance(data, list): + return data + else: + raise ValueError("Unsupported JSON structure (must be dict or list)") + + def _process_local_data(self, results, package_name): + """ + Process the local data when _id does not exist in the results. + + Args: + results (list): List of results to process. + package_name (str): Name of the package to process. + + Returns: + dict: Dictionary containing the processed results. + """ + for item in results: + for scan in SCAN_TYPES: + if scan in item: + source = item[scan] + source["SBOM"] = self.licenses_processor.evaluate_licenses(source["SBOM"]) + else: + print("Empty File") + return {} + + # Handle ICR, QUAY, and DOCKER fields + return { + "package": package_name, + "icr": [], + "quay": [], + "local": results + } + + def _get_empty_data(self, package_name): + """ + Return empty data when no response is received. + + Args: + package_name (str): Name of the package. + + Returns: + dict: Empty data dictionary with keys "package", "icr", "quay", and "local". + """ + return { + "package": package_name, + "icr": [], + "quay": [], + "local": [] + } + + def _remove_existing_file(self, filepath: str): + """ + This function removes a file if it exists. + + Parameters: + filepath (str): The path to the file to be removed. + + Returns: + None + """ + if os.path.exists(filepath): + os.remove(filepath) + +