From 31676cfab9888f0ac2b0f78016659eeabf63dc38 Mon Sep 17 00:00:00 2001 From: priti-parate <140157516+priti-parate@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:34:27 +0530 Subject: [PATCH 01/11] fix for adding service_kube_control_plane --- build_stream/api/catalog_roles/service.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/build_stream/api/catalog_roles/service.py b/build_stream/api/catalog_roles/service.py index ed6d9f35c9..20dca50d11 100644 --- a/build_stream/api/catalog_roles/service.py +++ b/build_stream/api/catalog_roles/service.py @@ -204,6 +204,17 @@ def _extract_roles_from_archive( ) roles = sorted(data.keys()) + + # Add service_kube_control_plane_first_x86 if service_kube_control_plane_x86_64 exists + if "service_kube_control_plane_x86_64" in roles and "service_kube_control_plane_first_x86" not in roles: + roles.append("service_kube_control_plane_first_x86") + roles = sorted(roles) + + # Add service_kube_control_plane_first_aarch64 if service_kube_control_plane_aarch64 exists + if "service_kube_control_plane_aarch64" in roles and "service_kube_control_plane_first_aarch64" not in roles: + roles.append("service_kube_control_plane_first_aarch64") + roles = sorted(roles) + return roles def _validate_parse_catalog_completed(self, job_id: JobId) -> None: From 2da1c3c68294acbf70935c1d032ca250588acfd6 Mon Sep 17 00:00:00 2001 From: priti-parate <140157516+priti-parate@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:51:20 +0530 Subject: [PATCH 02/11] removing service_kube_control_plane_aarch64 --- build_stream/api/catalog_roles/service.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/build_stream/api/catalog_roles/service.py b/build_stream/api/catalog_roles/service.py index 20dca50d11..3bc66ebd39 100644 --- a/build_stream/api/catalog_roles/service.py +++ b/build_stream/api/catalog_roles/service.py @@ -210,11 +210,6 @@ def _extract_roles_from_archive( roles.append("service_kube_control_plane_first_x86") roles = sorted(roles) - # Add service_kube_control_plane_first_aarch64 if service_kube_control_plane_aarch64 exists - if "service_kube_control_plane_aarch64" in roles and "service_kube_control_plane_first_aarch64" not in roles: - roles.append("service_kube_control_plane_first_aarch64") - roles = sorted(roles) - return roles def _validate_parse_catalog_completed(self, job_id: JobId) -> None: From ec4ddf64f221dff42a94a351625f8b41b152de1c Mon Sep 17 00:00:00 2001 From: priti-parate <140157516+priti-parate@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:59:43 +0530 Subject: [PATCH 03/11] term should have x86_64 --- build_stream/api/catalog_roles/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_stream/api/catalog_roles/service.py b/build_stream/api/catalog_roles/service.py index 3bc66ebd39..736c292e41 100644 --- a/build_stream/api/catalog_roles/service.py +++ b/build_stream/api/catalog_roles/service.py @@ -207,7 +207,7 @@ def _extract_roles_from_archive( # Add service_kube_control_plane_first_x86 if service_kube_control_plane_x86_64 exists if "service_kube_control_plane_x86_64" in roles and "service_kube_control_plane_first_x86" not in roles: - roles.append("service_kube_control_plane_first_x86") + roles.append("service_kube_control_plane_first_x86_64") roles = sorted(roles) return roles From 4ed6b52fb08912267e1c60a833a0275b07bce439 Mon Sep 17 00:00:00 2001 From: priti-parate <140157516+priti-parate@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:00:34 +0530 Subject: [PATCH 04/11] renaming with x86_64 --- build_stream/api/catalog_roles/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_stream/api/catalog_roles/service.py b/build_stream/api/catalog_roles/service.py index 736c292e41..e102d30d0b 100644 --- a/build_stream/api/catalog_roles/service.py +++ b/build_stream/api/catalog_roles/service.py @@ -206,7 +206,7 @@ def _extract_roles_from_archive( roles = sorted(data.keys()) # Add service_kube_control_plane_first_x86 if service_kube_control_plane_x86_64 exists - if "service_kube_control_plane_x86_64" in roles and "service_kube_control_plane_first_x86" not in roles: + if "service_kube_control_plane_x86_64" in roles and "service_kube_control_plane_first_x86_64" not in roles: roles.append("service_kube_control_plane_first_x86_64") roles = sorted(roles) From b6307779f158d5886fb0a7fc763bf256c402047d Mon Sep 17 00:00:00 2001 From: priti-parate <140157516+priti-parate@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:24:31 +0530 Subject: [PATCH 05/11] fix for prepare_oim input valdiation failure --- .../schema/build_stream_config.json | 4 +-- .../build_stream_validation.py | 31 ++++++++++++++++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/common/library/module_utils/input_validation/schema/build_stream_config.json b/common/library/module_utils/input_validation/schema/build_stream_config.json index 48fa90dbb5..c5e0e8ef81 100644 --- a/common/library/module_utils/input_validation/schema/build_stream_config.json +++ b/common/library/module_utils/input_validation/schema/build_stream_config.json @@ -20,8 +20,8 @@ }, "aarch64_inventory_host_ip": { "type": ["string", "null"], - "pattern": "^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$", - "description": "AArch64 inventory host IP for aarch64 builds. Must be in same subnet as OIM admin IP. Accepted values: valid IPv4 address or null" + "pattern": "^$|^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$", + "description": "AArch64 inventory host IP for aarch64 builds. Must be in same subnet as OIM admin IP. Accepted values: valid IPv4 address, empty string, or null" }, "postgres_db_name": { "type": "string", diff --git a/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py b/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py index f8a695d967..f87ef20ba9 100644 --- a/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py +++ b/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py @@ -247,9 +247,38 @@ def validate_build_stream_config(input_file_path, data, msg.build_stream_host_ip_not_oim_ip_msg(build_stream_host_ip, ethernet_ips) )) - # Validate aarch64_inventory_host_ip + # Validate aarch64_inventory_host_ip (conditional - required if PXE mapping has aarch64 groups) aarch64_inventory_host_ip = data.get("aarch64_inventory_host_ip") + + # Check if PXE mapping file contains aarch64 functional groups + has_aarch64_groups = False + try: + pxe_mapping_path = os.path.join(omnia_base_dir, project_name, "pxe_mapping_file.csv") + if os.path.exists(pxe_mapping_path): + with open(pxe_mapping_path, 'r', encoding='utf-8') as f: + # Skip header and check for aarch64 in functional group names + for line in f: + if line.startswith('FUNCTIONAL_GROUP_NAME'): + continue + if 'aarch64' in line.lower(): + has_aarch64_groups = True + break + logger.debug("PXE mapping contains aarch64 groups: %s", has_aarch64_groups) + except Exception as e: + logger.warning("Failed to check PXE mapping file for aarch64 groups: %s", str(e)) + + # If PXE mapping has aarch64 groups, require aarch64_inventory_host_ip + if has_aarch64_groups: + if not aarch64_inventory_host_ip or aarch64_inventory_host_ip in ["", None]: + errors.append(create_error_msg( + build_stream_yml, + "aarch64_inventory_host_ip", + "Field 'aarch64_inventory_host_ip' is required when PXE mapping file contains aarch64 functional groups. " + "Provide the admin IP of the aarch64 inventory host or remove aarch64 groups from PXE mapping." + )) + return errors + # If aarch64_inventory_host_ip is provided, validate it if aarch64_inventory_host_ip and aarch64_inventory_host_ip not in ["", None]: # Check if it's a valid IP format try: From 617e1acb9ed0a64f48e3b93051809d3c84fe94d4 Mon Sep 17 00:00:00 2001 From: priti-parate <140157516+priti-parate@users.noreply.github.com> Date: Thu, 5 Mar 2026 17:21:54 +0530 Subject: [PATCH 06/11] adding rechability check for aarch64 IP --- .../common_utils/en_us_validation_msg.py | 15 ++++++++ .../build_stream_validation.py | 35 +++++++++++++++++-- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py b/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py index b1071d1372..45bca59b55 100644 --- a/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py +++ b/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py @@ -428,6 +428,21 @@ def server_spec_network_key_fail_msg(nic_device): "Check network_spec.yml for admin network configuration." ) +AARCH64_INVENTORY_HOST_IP_REQUIRED_MSG = ( + "Field 'aarch64_inventory_host_ip' is required when PXE mapping file contains aarch64 functional groups. " + "Provide the admin IP of the aarch64 inventory host or remove aarch64 groups from PXE mapping." +) + +AARCH64_INVENTORY_HOST_IP_NOT_REACHABLE_MSG = ( + "aarch64 inventory host IP {0} is not reachable on SSH port 22. " + "Ensure the host is online, SSH service is running, and accessible from OIM." +) + +AARCH64_INVENTORY_HOST_IP_REACHABILITY_CHECK_FAILED_MSG = ( + "Unable to verify reachability of aarch64 inventory host IP {0}. " + "Ensure network connectivity and SSH service are available on the host." +) + BUILD_STREAM_HOST_IP_REQUIRED_MSG = ( "Field 'build_stream_host_ip' is mandatory in build_stream_config.yml. " "Please provide a valid IPv4 address (OIM admin IP or OIM public IP)." diff --git a/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py b/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py index f87ef20ba9..3abcbf0e8c 100644 --- a/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py +++ b/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py @@ -250,6 +250,7 @@ def validate_build_stream_config(input_file_path, data, # Validate aarch64_inventory_host_ip (conditional - required if PXE mapping has aarch64 groups) aarch64_inventory_host_ip = data.get("aarch64_inventory_host_ip") + ### aarch64_inventory_host_ip check # Check if PXE mapping file contains aarch64 functional groups has_aarch64_groups = False try: @@ -273,8 +274,7 @@ def validate_build_stream_config(input_file_path, data, errors.append(create_error_msg( build_stream_yml, "aarch64_inventory_host_ip", - "Field 'aarch64_inventory_host_ip' is required when PXE mapping file contains aarch64 functional groups. " - "Provide the admin IP of the aarch64 inventory host or remove aarch64 groups from PXE mapping." + msg.AARCH64_INVENTORY_HOST_IP_REQUIRED_MSG )) return errors @@ -301,6 +301,37 @@ def validate_build_stream_config(input_file_path, data, except ValueError as e: logger.error("Failed to validate subnet for aarch64_inventory_host_ip: %s", str(e)) + # Check aarch64 host IP reachability using socket (safer than subprocess) + try: + import socket + # Try to connect to SSH port which is usually open on inventory hosts + ssh_port = 22 # SSH + reachable = False + + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(2) # 2-second timeout + result = sock.connect_ex((str(aarch64_ip), ssh_port)) + if result == 0: + reachable = True + logger.debug(f"aarch64 host {aarch64_ip} reachable on SSH port {ssh_port}") + except (socket.timeout, socket.error): + pass + + if not reachable: + errors.append(create_error_msg( + build_stream_yml, + "aarch64_inventory_host_ip", + msg.AARCH64_INVENTORY_HOST_IP_NOT_REACHABLE_MSG.format(str(aarch64_ip)) + )) + except Exception as e: + logger.warning("Failed to check aarch64 host IP reachability: %s", str(e)) + errors.append(create_error_msg( + build_stream_yml, + "aarch64_inventory_host_ip", + msg.AARCH64_INVENTORY_HOST_IP_REACHABILITY_CHECK_FAILED_MSG.format(str(aarch64_ip)) + )) + # Validate build_stream_port build_stream_port = data.get("build_stream_port") From a86f80aa0f2b9bf1308f86ee760a1fd41f2e17ec Mon Sep 17 00:00:00 2001 From: priti-parate <140157516+priti-parate@users.noreply.github.com> Date: Wed, 11 Mar 2026 12:49:32 +0530 Subject: [PATCH 07/11] Readme document for BuildStreaM --- build_stream/README.md | 121 + build_stream/doc/README.md | 59 + build_stream/doc/build_image.md | 114 + build_stream/doc/catalog.md | 91 + build_stream/doc/developer-guide.md | 319 ++ build_stream/doc/jobs.md | 80 + build_stream/doc/local_repo.md | 105 + build_stream/doc/validation.md | 135 + build_stream/tests/README.md | 478 +- .../catalogs/adapter_policy_test.json | 72 - .../tests/fixtures/catalogs/catalog_rhel.json | 4220 ----------------- .../fixtures/catalogs/functional_layer.json | 145 - .../tests/fixtures/catalogs/sample.py | 81 - .../tests/fixtures/catalogs/sample_catalog.py | 81 - 14 files changed, 1478 insertions(+), 4623 deletions(-) create mode 100644 build_stream/doc/README.md create mode 100644 build_stream/doc/build_image.md create mode 100644 build_stream/doc/catalog.md create mode 100644 build_stream/doc/developer-guide.md create mode 100644 build_stream/doc/jobs.md create mode 100644 build_stream/doc/local_repo.md create mode 100644 build_stream/doc/validation.md delete mode 100644 build_stream/tests/fixtures/catalogs/adapter_policy_test.json delete mode 100644 build_stream/tests/fixtures/catalogs/catalog_rhel.json delete mode 100644 build_stream/tests/fixtures/catalogs/functional_layer.json delete mode 100644 build_stream/tests/fixtures/catalogs/sample.py delete mode 100644 build_stream/tests/fixtures/catalogs/sample_catalog.py diff --git a/build_stream/README.md b/build_stream/README.md index b0f57e8399..623171b3b4 100644 --- a/build_stream/README.md +++ b/build_stream/README.md @@ -1,2 +1,123 @@ # Build Stream +Build Stream is a RESTful API service that orchestrates the creation and management of build jobs for the Omnia infrastructure platform. It provides a centralized interface for managing software catalog parsing, local repository creation, image building, and validation workflows. + +## Architecture Overview + +Build Stream follows a clean architecture pattern with clear separation of concerns: + +- **API Layer** (`api/`): FastAPI routes and HTTP handling +- **Core Layer** (`core/`): Business logic, entities, and domain services +- **Orchestrator Layer** (`orchestrator/`): Use cases that coordinate workflows +- **Infrastructure Layer** (`infra/`): External integrations and data persistence +- **Common Layer** (`common/`): Shared utilities and configuration + +## High-Level Workflow + +1. **Authentication**: JWT-based authentication secures all API endpoints +2. **Job Creation**: Clients submit build requests through the jobs API +3. **Stage Processing**: Jobs are broken into stages (catalog parsing, local repo, build image, validation) +4. **Async Execution**: Stages execute asynchronously with result polling +5. **Artifact Management**: Build artifacts are stored and tracked throughout the process +6. **Audit Trail**: All operations are logged for traceability and compliance + +## Configuration + +Configuration is managed through: +- Environment variables for runtime settings +- `build_stream.ini` for artifact store configuration +- Vault integration for secure credential management +- Database configuration for persistent storage + +Key configuration areas: +- Database connections (PostgreSQL) +- Artifact storage backend (file system or in-memory) +- Vault endpoints and authentication +- CORS and server settings + +## Getting Started + +### For Developers + +**Primary Entry Points:** +- `main.py` - FastAPI application entry point +- `api/router.py` - API route aggregation +- `container.py` - Dependency injection setup + +**Key Workflows:** +- [Jobs Management](./doc/jobs.md) - Job lifecycle and orchestration +- [Catalog Processing](./doc/catalog.md) - Software catalog parsing and role generation +- [Local Repository](./doc/local_repo.md) - Local package repository creation +- [Image Building](./doc/build_image.md) - Container image build workflows +- [Validation](./doc/validation.md) - Input and output validation + +**Development Setup:** +```bash +# Install dependencies +pip install -r requirements.txt +pip install -r requirements-dev.txt + +# Set environment variables +export HOST=0.0.0.0 +export PORT=8000 + +# Run development server +uvicorn main:app --reload + +# Run tests +pytest +``` + +**API Documentation:** +- See Omnia ReadTheDocs for complete API documentation + +### Architecture Components + +**Core Services:** +- **Job Service**: Manages job lifecycle and state transitions +- **Catalog Service**: Parses software catalogs and generates roles +- **Local Repo Service**: Creates and manages local repositories +- **Build Service**: Orchestrates container image builds +- **Validation Service**: Validates inputs and outputs + +**Data Flow:** +1. Client requests → API routes → Use cases → Core services → Repositories +2. Async job processing with stage-based execution +3. Result polling and webhook notifications +4. Artifact storage and metadata tracking + +**Security:** +- JWT token-based authentication +- Vault integration for secret management +- Role-based access control +- Audit logging for compliance + +## Workflow Areas + +Each major workflow area has dedicated documentation: + +- **Jobs** - Job creation, monitoring, and lifecycle management +- **Catalog** - Software catalog parsing and role generation +- **Local Repo** - Local package repository setup and management +- **Build Image** - Container image build orchestration +- **Validation** - Input validation and output verification + +## Dependencies + +Build Stream uses FastAPI with the following key dependencies: +- FastAPI/Uvicorn for web framework +- SQLAlchemy for database ORM +- Dependency Injector for IoC container +- PyJWT for authentication +- Ansible for infrastructure automation +- Vault client for secret management + +## Support + +For troubleshooting and development guidance: +1. Check the workflow-specific documentation in `doc/` +2. Review API logs for error details +3. Consult the audit trail for job execution history +4. Refer to the health check endpoint: `/health` +5. See [tests README](tests/README.md) for pytest guidance and test execution + diff --git a/build_stream/doc/README.md b/build_stream/doc/README.md new file mode 100644 index 0000000000..186bfcdf5d --- /dev/null +++ b/build_stream/doc/README.md @@ -0,0 +1,59 @@ +# Build Stream Documentation + +This directory contains comprehensive documentation for the Build Stream module and its workflows. + +## Documentation Structure + +### Overview Documentation +- **[Developer Guide](./developer-guide.md)** - Complete development guide with architecture deep dive +- **[Main README](../README.md)** - High-level overview and getting started guide + +### Workflow Documentation +- **[Jobs Management](./jobs.md)** - Job lifecycle and orchestration +- **[Catalog Processing](./catalog.md)** - Software catalog parsing and role generation +- **[Local Repository](./local_repo.md)** - Local package repository creation +- **[Image Building](./build_image.md)** - Container image build workflows +- **[Validation](./validation.md)** - Input and output validation + +## Quick Navigation + +### For New Contributors +1. Start with the [main README](../README.md) for architecture overview +2. Read the [Developer Guide](./developer-guide.md) for detailed understanding +3. Explore specific workflow documentation based on your area of focus + +### For Debugging Issues +1. Check the relevant workflow documentation for your issue area +2. Use the Developer Guide for troubleshooting steps +3. Review the audit trail and logging sections + +### For Feature Development +1. Read the Developer Guide for architecture and patterns +2. Review the relevant workflow documentation +3. Follow the contribution guidelines in the Developer Guide + +## Documentation Standards + +All Build Stream documentation follows these standards: +- **No sensitive data** - Never include passwords, tokens, or secrets +- **Developer-focused** - Written for technical contributors +- **Cross-referenced** - Links between related documentation +- **Example-driven** - Includes practical examples and code snippets +- **Maintainable** - Easy to update as the codebase evolves + +## Getting Help + +If you need additional help beyond the documentation: +1. Check the troubleshooting sections in workflow docs +2. Review the audit trail and error handling patterns +3. Consult the architecture diagrams in the Developer Guide +4. Reach out to the Build Stream development team + +## Contributing to Documentation + +When contributing to Build Stream: +1. Update relevant documentation for API changes +2. Add new workflow documentation for new features +3. Keep cross-references up to date +4. Follow the established documentation standards +5. Include examples and troubleshooting information diff --git a/build_stream/doc/build_image.md b/build_stream/doc/build_image.md new file mode 100644 index 0000000000..8a159d95a7 --- /dev/null +++ b/build_stream/doc/build_image.md @@ -0,0 +1,114 @@ +# Image Building + +The Image Building workflow orchestrates container image creation for the Omnia platform components. + +## What It Does + +The Image Building workflow provides: +- Container image build orchestration +- Multi-architecture image support (x86_64, aarch64) +- Docker and Podman integration +- Build context management and optimization +- Image security scanning and validation +- Registry push and distribution management + +## Inputs/Outputs + +**Inputs:** +- Build specifications and Dockerfiles +- Package lists from local repositories +- Build parameters and configurations +- Security scanning requirements + +**Outputs:** +- Built container images +- Image metadata and manifests +- Security scan reports +- Registry push confirmations +- Build logs and artifacts + +## Key Logic Locations + +**Primary Files:** +- `api/build_image/routes.py` - HTTP endpoints for build operations +- `orchestrator/build_image/use_cases/` - Build orchestration logic +- `core/build_image/entities.py` - Build domain entities +- `core/build_image/repositories.py` - Build data access +- `core/build_image/services.py` - Build management services + +**Main Components:** +- **BuildImageUseCase** - Orchestrates image build processes +- **BuildService** - Manages build execution and monitoring +- **MultiArchBuilder** - Handles multi-architecture builds +- **SecurityScanner** - Performs image security analysis + +## Workflow Flow + +1. **Build Request**: Client submits image build request +2. **Build Context Preparation**: Dockerfiles and dependencies assembled +3. **Multi-Arch Setup**: Build configurations prepared for target architectures +4. **Build Execution**: Container images built using Docker/Podman +5. **Security Scanning**: Built images scanned for vulnerabilities +6. **Manifest Creation**: Multi-architecture manifests generated +7. **Registry Push**: Images pushed to container registries +8. **Validation**: Final image validation and testing + +## Architecture Support + +Supports multiple CPU architectures: +- **x86_64** - Standard 64-bit Intel/AMD processors +- **aarch64** - 64-bit ARM processors +- **Multi-arch manifests** - Single image supporting multiple architectures + +## Build Optimization + +Optimizations include: +- **Layer caching** - Reusing unchanged layers across builds +- **Parallel builds** - Concurrent building for multiple architectures +- **Context optimization** - Minimizing build context size +- **Dependency caching** - Caching package downloads + +## Security Features + +Security capabilities include: +- **Vulnerability scanning** - Automated security analysis +- **Base image validation** - Verified base image sources +- **Signature verification** - Package integrity checks +- **Runtime security** - Secure container configurations + +## Integration Points + +- Receives packages from local repository workflow +- Integrates with validation workflow for quality checks +- Uses Vault for registry credentials +- Connects to container registries for distribution + +## Configuration + +Build configuration includes: +- Build parameters and environment variables +- Registry endpoints and credentials +- Security scanning policies +- Architecture-specific settings + +## Error Handling + +- Detailed build error reporting +- Step-by-step build progress tracking +- Rollback capabilities for failed builds +- Automated retry for transient failures + +## Monitoring + +- Real-time build progress monitoring +- Resource usage tracking (CPU, memory, storage) +- Build success/failure metrics +- Security scan result tracking + +## Registry Integration + +Supports multiple container registries: +- **Docker Hub** - Public container registry +- **Harbor** - Enterprise container registry +- **Artifactory** - JFrog container registry +- **Custom registries** - Organization-specific registries diff --git a/build_stream/doc/catalog.md b/build_stream/doc/catalog.md new file mode 100644 index 0000000000..8b64f8e4f1 --- /dev/null +++ b/build_stream/doc/catalog.md @@ -0,0 +1,91 @@ +# Catalog Processing + +The Catalog workflow handles software catalog parsing and role generation for the Omnia platform. + +## What It Does + +The Catalog workflow provides: +- Software catalog parsing from JSON files +- Role generation based on catalog contents +- Package categorization and dependency resolution +- Integration with Ansible for role creation +- Validation of catalog structure and contents + +## Inputs/Outputs + +**Inputs:** +- Software catalog JSON files +- Package configuration mappings +- Role templates and definitions +- Platform-specific parameters + +**Outputs:** +- Generated Ansible roles +- Package dependency mappings +- Validated catalog structures +- Role metadata and documentation + +## Key Logic Locations + +**Primary Files:** +- `api/catalog_roles/routes.py` - HTTP endpoints for catalog operations +- `api/parse_catalog/routes.py` - Catalog parsing endpoints +- `orchestrator/catalog/use_cases/parse_catalog.py` - Catalog parsing logic +- `orchestrator/catalog/use_cases/generate_input_files.py` - Input file generation +- `core/catalog/entities.py` - Catalog domain entities +- `core/catalog/repositories.py` - Catalog data access +- `generate_catalog.py` - Standalone catalog generation script + +**Main Components:** +- **ParseCatalogUseCase** - Handles catalog parsing and validation +- **GenerateInputFilesUseCase** - Creates Ansible input files +- **CatalogRolesService** - Role generation and management +- **CatalogRepository** - Catalog data persistence + +## Workflow Flow + +1. **Catalog Upload**: Client submits catalog via `/api/v1/parse_catalog` endpoint +2. **Structure Validation**: Catalog schema and structure validated +3. **Package Parsing**: Individual packages extracted and categorized +4. **Dependency Resolution**: Package dependencies analyzed and resolved +5. **Role Generation**: Ansible roles generated based on packages +6. **Input File Creation**: Configuration files created for downstream workflows +7. **Validation**: Generated artifacts validated for completeness +8. **Storage**: Results stored in artifact repository + +## Package Categorization + +Packages are categorized into: +- **Functional Bundles**: Core service packages (service_k8s, slurm_custom, additional_packages) +- **Infrastructure Bundles**: CSI and infrastructure packages (csi_driver_powerscale) +- **Miscellaneous**: Additional packages that don't fit other categories + +## Integration Points + +- Feeds into local repository creation workflow +- Provides input for image building workflows +- Integrates with validation workflow for quality checks +- Uses Vault for secure access to package repositories + +## Configuration + +Catalog processing is configured through: +- Package mapping files +- Role templates +- Platform-specific configurations +- Validation rules and schemas + +## Error Handling + +- Comprehensive schema validation for catalogs +- Detailed error reporting for invalid packages +- Graceful handling of missing dependencies +- Rollback capabilities for failed processing + +## Standalone Script + +The `generate_catalog.py` script provides: +- Command-line catalog generation +- Batch processing capabilities +- Integration with external CI/CD pipelines +- Detailed logging and error reporting diff --git a/build_stream/doc/developer-guide.md b/build_stream/doc/developer-guide.md new file mode 100644 index 0000000000..2baa9f9a7a --- /dev/null +++ b/build_stream/doc/developer-guide.md @@ -0,0 +1,319 @@ +# Build Stream Developer Guide + +This guide provides developers with comprehensive documentation for understanding and working with the Build Stream codebase. + +## Quick Start + +**For New Developers:** +1. Read the main [README](../README.md) for architecture overview +2. Review this guide for end-to-end workflow understanding +3. Explore specific workflow documentation for detailed implementation +4. Set up development environment using the instructions below + +**Key Concepts:** +- **Jobs**: Orchestrate multi-stage build processes +- **Stages**: Individual workflow steps (catalog, repo, build, validate) +- **Artifacts**: Generated files and outputs from workflows +- **Audit Trail**: Complete logging of all operations + +## Architecture Deep Dive + +### Layer Structure + +``` +build_stream/ +├── api/ # HTTP layer - FastAPI routes and middleware +├── core/ # Domain layer - Business logic and entities +├── orchestrator/ # Application layer - Use cases and workflow coordination +├── infra/ # Infrastructure layer - External integrations +├── common/ # Shared layer - Utilities and configuration +└── doc/ # Documentation layer - Workflow guides +``` + +### Dependency Flow + +``` +HTTP Request → API Routes → Use Cases → Core Services → Repositories + ↓ ↓ ↓ + Authentication Business Logic Data Persistence + ↓ ↓ ↓ + Authorization Validation External Systems +``` + +## Workflow Tracing Guide + +### End-to-End Job Flow + +To trace a complete job from start to finish: + +1. **Job Creation** (`POST /api/v1/jobs`) + - Entry point: `api/jobs/routes.py` + - Use case: `orchestrator/jobs/use_cases/create_job.py` + - Entity: `core/jobs/entities.py` + +2. **Stage Processing** (Async) + - Each stage runs independently + - Result polling handles completion + - Status updates tracked in database + +3. **Artifact Storage** + - Files stored in configured artifact store + - Metadata tracked in database + - Access controlled through permissions + +### Debugging Workflow Issues + +**Common Debugging Steps:** +1. Check job status via `/api/v1/jobs/{job_id}` +2. Review stage-specific error messages +3. Examine audit trail for detailed execution logs +4. Validate inputs using validation workflow +5. Check external system connectivity (Vault, repositories, registries) + +**Key Debug Files:** +- `api/logging_utils.py` - Centralized logging configuration +- `core/exceptions.py` - Domain-specific error definitions +- `infra/` - External integration points + +## Development Workflow + +### Setting Up Development Environment + +```bash +# Clone repository +git clone +cd build_stream + +# Install dependencies +pip install -r requirements.txt +pip install -r requirements-dev.txt + +# Set up environment variables +cp .env.test .env +# Edit .env with your configuration + +# Run database migrations (if using SQL backend) +alembic upgrade head + +# Start development server +uvicorn main:app --reload --host 0.0.0.0 --port 8000 +``` + +### Adding New Features + +**1. Add New API Endpoint:** +- Create route in appropriate `api/` subdirectory +- Add schema definitions for request/response +- Implement authentication/authorization as needed +- Add to `api/router.py` for registration + +**2. Add New Business Logic:** +- Create entities in `core/` domain +- Implement repositories for data access +- Create use cases in `orchestrator/` +- Wire up in dependency injection container + +**3. Add New Workflow Stage:** +- Define stage type in `core/jobs/value_objects.py` +- Implement stage-specific logic +- Add to job creation workflow +- Update documentation + +### Testing Strategy + +**Unit Tests:** +- Test individual components in isolation +- Mock external dependencies +- Focus on business logic validation + +**Integration Tests:** +- Test workflow end-to-end +- Use test database and artifact store +- Validate cross-component interactions + +**API Tests:** +- Test HTTP endpoints thoroughly +- Validate authentication and authorization +- Test error scenarios and edge cases + +## Common Patterns + +### Dependency Injection + +Build Stream uses `dependency-injector` for IoC: + +```python +# In container.py +from dependency_injector import containers, providers + +class Container(containers.DeclarativeContainer): + config = providers.Configuration() + + # Repository providers + job_repository = providers.Singleton( + SqlJobRepository, + session_factory=SessionLocal, + ) + + # Use case providers + create_job_use_case = providers.Factory( + CreateJobUseCase, + job_repository=job_repository, + # ... other dependencies + ) +``` + +### Error Handling + +Consistent error handling across all layers: + +```python +# Domain exceptions +class JobNotFoundError(Exception): + """Raised when a job is not found.""" + +# API error handling +@app.exception_handler(JobNotFoundError) +async def job_not_found_handler(request, exc): + return JSONResponse( + status_code=404, + content={"error": "Job not found", "details": str(exc)} + ) +``` + +### Logging + +Structured logging with correlation IDs: + +```python +from api.logging_utils import log_secure_info + +log_secure_info( + "Job created successfully", + extra={ + "job_id": str(job_id), + "client_id": str(client_id), + "stages": [stage.name for stage in stages] + } +) +``` + +## Security Considerations + +### Authentication Flow + +1. Client authenticates with JWT token +2. Token validated in middleware +3. User context attached to request +4. Authorization checked per endpoint + +### Secure Data Handling + +- Never log sensitive data (passwords, tokens) +- Use Vault for credential storage +- Implement proper access controls +- Audit all sensitive operations + +### Input Validation + +All inputs validated using: +- Pydantic models for type checking +- JSON schemas for structure validation +- Business rules in domain layer +- Security scanning for malicious content + +## Performance Optimization + +### Database Optimization + +- Use connection pooling +- Implement proper indexing +- Batch operations where possible +- Monitor query performance + +### Async Processing + +- Use async/await for I/O operations +- Implement proper error handling +- Monitor async task completion +- Handle resource cleanup + +### Caching Strategies + +- Cache frequently accessed data +- Use appropriate cache invalidation +- Monitor cache hit rates +- Consider distributed caching for scale + +## Troubleshooting Common Issues + +### Job Failures + +**Symptoms:** Jobs stuck in running state or failing unexpectedly + +**Debug Steps:** +1. Check job status and stage details +2. Review error messages in audit trail +3. Verify external system connectivity +4. Check resource availability + +### Performance Issues + +**Symptoms:** Slow API responses or job processing + +**Debug Steps:** +1. Monitor database query performance +2. Check resource utilization +3. Review async task processing +4. Analyze network connectivity + +### Authentication Problems + +**Symptoms:** 401/403 errors on API calls + +**Debug Steps:** +1. Verify JWT token validity +2. Check user permissions +3. Review token expiration +4. Validate token claims + +## External Dependencies + +### Required Services + +- **PostgreSQL** - Primary data storage +- **Vault** - Secure credential storage +- **Pulp** - Repository management (optional) +- **Container Registry** - Image storage + +### Optional Integrations + +- **External CI/CD** - Build pipeline integration +- **Monitoring Systems** - Metrics and alerting +- **Security Scanners** - Vulnerability assessment +- **Compliance Tools** - Regulatory reporting + +## Contributing Guidelines + +### Code Standards + +- Follow PEP 8 for Python code +- Use type hints for all public functions +- Write comprehensive docstrings +- Include unit tests for new features + +### Documentation Requirements + +- Update README for API changes +- Document new workflows in `doc/` +- Update architecture diagrams +- Include examples in docstrings + +### Review Process + +1. Create feature branch from main +2. Implement changes with tests +3. Update documentation +4. Submit pull request +5. Address review feedback +6. Merge after approval diff --git a/build_stream/doc/jobs.md b/build_stream/doc/jobs.md new file mode 100644 index 0000000000..58c6a27efe --- /dev/null +++ b/build_stream/doc/jobs.md @@ -0,0 +1,80 @@ +# Jobs Management + +The Jobs workflow manages the complete lifecycle of build jobs in Build Stream, from creation through completion and monitoring. + +## What It Does + +The Jobs workflow provides: +- Job creation with idempotency guarantees +- Stage-based execution with state management +- Real-time job monitoring and status tracking +- Audit trail for all job operations +- Result polling and notification handling + +## Inputs/Outputs + +**Inputs:** +- Job creation requests with stage definitions +- Authentication tokens for security +- Optional job parameters and configuration + +**Outputs:** +- Job IDs for tracking +- Stage execution results +- Audit events for compliance +- Error details and diagnostics + +## Key Logic Locations + +**Primary Files:** +- `api/jobs/routes.py` - HTTP endpoints for job operations +- `orchestrator/jobs/use_cases/create_job.py` - Job creation business logic +- `core/jobs/entities.py` - Job and Stage domain entities +- `core/jobs/repositories.py` - Data access layer +- `core/jobs/services.py` - Job-related domain services + +**Main Components:** +- **CreateJobUseCase** - Handles job creation with validation +- **JobRepository** - Manages job persistence +- **StageRepository** - Manages stage state tracking +- **ResultPoller** - Handles async result collection + +## Workflow Flow + +1. **Job Creation**: Client submits job via `/api/v1/jobs` endpoint +2. **Validation**: Request validated for authentication and schema +3. **Idempotency Check**: Prevents duplicate job creation +4. **Stage Initialization**: Job broken into executable stages +5. **Async Execution**: Stages queued for background processing +6. **Status Updates**: Job status tracked through state transitions +7. **Result Collection**: Results polled and stored +8. **Audit Logging**: All operations logged for traceability + +## Stage Types + +Jobs support multiple stage types: +- **catalog_roles** - Software catalog processing +- **local_repo** - Local repository creation +- **build_image** - Container image building +- **validate** - Input/output validation + +## Error Handling + +- Invalid state transitions are rejected +- Failed stages can be retried based on configuration +- Comprehensive error reporting with context +- Audit trail captures all error events + +## Monitoring + +- Job status available via `/api/v1/jobs/{job_id}` endpoint +- List all jobs with filtering options +- Real-time status updates through result polling +- Detailed audit trail for compliance reporting + +## Integration Points + +- Integrates with all other workflows as stages +- Uses Vault for secure credential access +- Stores artifacts in configured artifact store +- Emits events for external system integration diff --git a/build_stream/doc/local_repo.md b/build_stream/doc/local_repo.md new file mode 100644 index 0000000000..0fe1729ef0 --- /dev/null +++ b/build_stream/doc/local_repo.md @@ -0,0 +1,105 @@ +# Local Repository + +The Local Repository workflow manages the creation and configuration of local package repositories for the Omnia platform. + +## What It Does + +The Local Repository workflow provides: +- Local package repository setup and configuration +- Package synchronization from remote sources +- Repository metadata generation and management +- Integration with Pulp for repository management +- Repository validation and health checking + +## Inputs/Outputs + +**Inputs:** +- Package lists from catalog processing +- Repository configuration parameters +- Remote repository URLs and credentials +- Storage and bandwidth constraints + +**Outputs:** +- Configured local repositories +- Synchronized package metadata +- Repository access credentials +- Health check reports and validation results + +## Key Logic Locations + +**Primary Files:** +- `api/local_repo/routes.py` - HTTP endpoints for repository operations +- `orchestrator/local_repo/use_cases/create_local_repo.py` - Repository creation logic +- `core/localrepo/entities.py` - Repository domain entities +- `core/localrepo/repositories.py` - Repository data access +- `core/localrepo/services.py` - Repository management services + +**Main Components:** +- **CreateLocalRepoUseCase** - Handles repository creation and setup +- **LocalRepoService** - Repository management and operations +- **LocalRepoRepository** - Repository configuration persistence +- **PackageSyncService** - Package synchronization from remote sources + +## Workflow Flow + +1. **Repository Request**: Client submits repository creation request +2. **Configuration Validation**: Repository parameters validated +3. **Remote Source Setup**: Remote repository connections configured +4. **Package Synchronization**: Packages synced from remote sources +5. **Metadata Generation**: Repository metadata created and updated +6. **Access Configuration**: User access and permissions configured +7. **Health Validation**: Repository health and accessibility validated +8. **Registration**: Repository registered with downstream systems + +## Repository Types + +Supports multiple repository types: +- **YUM/DNF repositories** - RPM-based package management +- **APT repositories** - Debian-based package management +- **Python repositories** - PyPI-compatible package hosting +- **Custom repositories** - Organization-specific package formats + +## Integration Points + +- Receives package lists from catalog workflow +- Provides packages to image building workflow +- Integrates with validation workflow for quality checks +- Uses Vault for secure credential storage +- Connects to Pulp for advanced repository management + +## Configuration + +Repository configuration includes: +- Storage locations and quotas +- Remote source URLs and credentials +- Synchronization schedules and policies +- Access control and permissions +- Health check parameters + +## Security + +- Secure credential management through Vault +- Access control based on user roles +- Package signature verification +- Audit logging for all repository operations + +## Error Handling + +- Graceful handling of remote source failures +- Retry mechanisms for synchronization errors +- Detailed error reporting and diagnostics +- Rollback capabilities for failed operations + +## Monitoring + +- Repository health status monitoring +- Package synchronization progress tracking +- Storage usage and quota monitoring +- Access logging and audit trails + +## Performance Optimization + +- Incremental synchronization to minimize bandwidth +- Parallel package downloading +- Caching of repository metadata +- Optimized storage layouts for fast access diff --git a/build_stream/doc/validation.md b/build_stream/doc/validation.md new file mode 100644 index 0000000000..8ad5780c83 --- /dev/null +++ b/build_stream/doc/validation.md @@ -0,0 +1,135 @@ +# Validation + +The Validation workflow provides comprehensive input and output validation for all Build Stream operations. + +## What It Does + +The Validation workflow provides: +- Input schema validation for all API requests +- Output validation for generated artifacts +- Cross-workflow dependency validation +- Security and compliance checking +- Quality assurance and testing integration + +## Inputs/Outputs + +**Inputs:** +- API request payloads and parameters +- Generated artifacts and configurations +- Build outputs and container images +- Security scan results and reports + +**Outputs:** +- Validation reports and results +- Error details and correction suggestions +- Compliance status and recommendations +- Quality metrics and measurements + +## Key Logic Locations + +**Primary Files:** +- `api/validate/routes.py` - HTTP endpoints for validation operations +- `orchestrator/validate/use_cases/` - Validation logic implementations +- `core/validate/entities.py` - Validation domain entities +- `core/validate/repositories.py` - Validation data access +- `core/validate/services.py` - Validation processing services + +**Main Components:** +- **ValidateUseCase** - Orchestrates validation processes +- **SchemaValidator** - Handles JSON schema validation +- **SecurityValidator** - Performs security compliance checks +- **QualityValidator** - Assesses output quality + +## Validation Types + +**Input Validation:** +- JSON schema validation for API requests +- Parameter type and range checking +- Authentication and authorization verification +- File format and size validation + +**Output Validation:** +- Generated file structure validation +- Container image security scanning +- Configuration file syntax checking +- Dependency integrity verification + +**Cross-Workflow Validation:** +- Catalog-to-repository dependency validation +- Repository-to-image package validation +- Image-to-deployment compatibility checking +- End-to-end workflow validation + +## Workflow Flow + +1. **Validation Request**: Client submits validation request +2. **Schema Validation**: Input schemas validated against definitions +3. **Security Checking**: Security policies and compliance verified +4. **Quality Assessment**: Output quality metrics evaluated +5. **Dependency Validation**: Cross-component dependencies verified +6. **Report Generation**: Comprehensive validation reports created +7. **Result Storage**: Validation results stored for audit trail +8. **Notification**: Validation status notifications sent + +## Schema Management + +Schema validation includes: +- **JSON Schema** - Standard JSON schema validation +- **Custom Validators** - Business-specific validation rules +- **Version Compatibility** - Schema version compatibility checking +- **Extensible Rules** - Configurable validation policies + +## Security Validation + +Security checks include: +- **Vulnerability Scanning** - Container image vulnerability analysis +- **Credential Validation** - Secure credential verification +- **Access Control** - Permission and authorization checking +- **Compliance Checking** - Regulatory compliance validation + +## Quality Assurance + +Quality metrics include: +- **Code Quality** - Generated code style and structure +- **Configuration Validity** - Configuration file correctness +- **Performance Metrics** - Resource usage and efficiency +- **Reliability Checks** - Error handling and robustness + +## Integration Points + +- Validates inputs for all API endpoints +- Checks outputs from all workflow stages +- Integrates with external security scanning tools +- Connects to compliance and audit systems + +## Configuration + +Validation configuration includes: +- Schema definitions and versions +- Security policies and thresholds +- Quality standards and metrics +- Compliance requirements and rules + +## Error Handling + +- Detailed validation error reporting +- Suggested corrections and fixes +- Error categorization and prioritization +- Automated retry for validation failures + +## Reporting + +Validation reports provide: +- Overall validation status summary +- Detailed error and warning lists +- Security vulnerability assessments +- Quality metrics and trends +- Compliance status and recommendations + +## Continuous Validation + +Ongoing validation includes: +- Automated validation in CI/CD pipelines +- Periodic security scanning +- Continuous quality monitoring +- Regular compliance checking diff --git a/build_stream/tests/README.md b/build_stream/tests/README.md index 47d5270ec7..fcd6ae3aff 100644 --- a/build_stream/tests/README.md +++ b/build_stream/tests/README.md @@ -1,23 +1,80 @@ -# Test Suite for JobId Feature +# Build Stream Test Suite -This directory contains the unit and integration tests for the Jobs API (JobId, correlation/idempotency, CRUD flows). +This directory contains comprehensive unit and integration tests for all Build Stream workflows including Jobs API, Catalog Processing, Local Repository, Image Building, and Validation. ## Test Structure ``` tests/ -├── integration/ -│ └── api/ -│ └── jobs/ -│ ├── conftest.py # Shared fixtures -│ ├── test_create_job_api.py # POST /jobs tests -│ ├── test_get_job_api.py # GET /jobs/{id} tests -│ └── test_delete_job_api.py # DELETE /jobs/{id} tests -└── unit/ - └── api/ - └── jobs/ - ├── test_schemas.py # Pydantic schema tests - └── test_dependencies.py # Dependency injection tests +├── integration/ # Integration tests for end-to-end workflows +│ ├── api/ # API endpoint integration tests +│ │ ├── jobs/ # Jobs API tests +│ │ │ ├── conftest.py # Shared fixtures +│ │ │ ├── test_create_job_api.py # POST /jobs tests +│ │ │ ├── test_get_job_api.py # GET /jobs/{id} tests +│ │ │ └── test_delete_job_api.py # DELETE /jobs/{id} tests +│ │ ├── catalog_roles/ # Catalog processing tests +│ │ │ ├── conftest.py # Shared fixtures +│ │ │ ├── test_get_roles_api.py # GET /catalog_roles tests +│ │ │ └── test_catalog_workflow.py # End-to-end catalog tests +│ │ ├── parse_catalog/ # Catalog parsing tests +│ │ │ ├── conftest.py # Shared fixtures +│ │ │ └── test_parse_catalog_api.py # POST /parse_catalog tests +│ │ ├── local_repo/ # Local repository tests +│ │ │ ├── conftest.py # Shared fixtures +│ │ │ ├── test_create_local_repo_api.py # POST /local_repo tests +│ │ │ └── test_repo_workflow.py # End-to-end repo tests +│ │ ├── build_image/ # Image building tests +│ │ │ ├── conftest.py # Shared fixtures +│ │ │ ├── test_build_image_api.py # POST /build_image tests +│ │ │ └── test_multi_arch_build.py # Multi-architecture tests +│ │ └── validate/ # Validation tests +│ │ ├── conftest.py # Shared fixtures +│ │ └── test_validate_api.py # POST /validate tests +│ ├── core/ # Core domain integration tests +│ │ ├── jobs/ # Job entity integration tests +│ │ ├── catalog/ # Catalog entity integration tests +│ │ └── localrepo/ # Repository entity integration tests +│ └── infra/ # Infrastructure integration tests +│ ├── repositories/ # Repository integration tests +│ └── external/ # External service integration tests +├── unit/ # Unit tests for individual components +│ ├── api/ # API layer unit tests +│ │ ├── jobs/ # Jobs API unit tests +│ │ │ ├── test_schemas.py # Pydantic schema tests +│ │ │ ├── test_dependencies.py # Dependency injection tests +│ │ │ └── test_routes.py # Route handler tests +│ │ ├── catalog_roles/ # Catalog API unit tests +│ │ ├── local_repo/ # Local repo API unit tests +│ │ └── validate/ # Validation API unit tests +│ ├── core/ # Core domain unit tests +│ │ ├── jobs/ # Job entity and value object tests +│ │ ├── catalog/ # Catalog entity tests +│ │ ├── localrepo/ # Repository entity tests +│ │ └── validate/ # Validation entity tests +│ ├── orchestrator/ # Use case unit tests +│ │ ├── jobs/ # Job use case tests +│ │ ├── catalog/ # Catalog use case tests +│ │ ├── local_repo/ # Repository use case tests +│ │ └── validate/ # Validation use case tests +│ └── infra/ # Infrastructure unit tests +│ ├── repositories/ # Repository implementation tests +│ ├── artifact_store/ # Artifact store tests +│ └── db/ # Database layer tests +├── end_to_end/ # Complete workflow tests +│ ├── test_full_job_workflow.py # Complete job lifecycle +│ └── test_catalog_to_image.py # Catalog to image workflow +├── performance/ # Performance and load tests +│ └── test_load.py # Load testing scenarios +├── fixtures/ # Shared test fixtures +│ ├── job_fixtures.py # Job test data +│ └── repo_fixtures.py # Repository test data +├── mocks/ # Mock objects and data +│ ├── mock_vault.py # Vault mock +│ └── mock_registry.py # Registry mock +└── utils/ # Test utilities and helpers + ├── assertions.py # Custom assertions + └── helpers.py # Test helper functions ``` ## Prerequisites @@ -62,14 +119,26 @@ pytest tests/integration/api/ tests/unit/api/ -v ### Run Specific Test Files ```bash -# Create Job API tests +# Jobs API tests pytest tests/integration/api/jobs/test_create_job_api.py -v +# Catalog processing tests +pytest tests/integration/api/catalog_roles/ -v + +# Local repository tests +pytest tests/integration/api/local_repo/ -v + +# Image building tests +pytest tests/integration/api/build_image/ -v + +# Validation tests +pytest tests/integration/api/validate/ -v + # Schema validation tests pytest tests/unit/api/jobs/test_schemas.py -v -# Dependency injection tests -pytest tests/unit/api/jobs/test_dependencies.py -v +# Use case tests +pytest tests/unit/orchestrator/ -v ``` ### Run Specific Test Classes or Functions @@ -85,22 +154,123 @@ pytest tests/integration/api/jobs/test_create_job_api.py::TestCreateJobSuccess:: pytest tests/integration/ -k idempotency -v ``` +## Test Types + +### Unit Tests +Test individual components in isolation: +- **API Layer**: Route handlers, schemas, dependencies +- **Core Layer**: Entities, value objects, domain services +- **Orchestrator Layer**: Use cases and business logic +- **Infrastructure Layer**: Repositories, external integrations + +### Integration Tests +Test component interactions: +- **API Integration**: Full HTTP request/response cycles +- **Database Integration**: Repository operations with real DB +- **External Services**: Vault, Pulp, container registries +- **Cross-Layer**: API → Use Case → Repository flows + +### End-to-End Tests +Test complete workflows from start to finish: +- Full job creation and execution +- Catalog parsing through role generation +- Repository creation and package sync +- Image building and registry push + +### Performance Tests +Test system performance and scalability: +- Load testing for concurrent requests +- Stress testing for resource limits +- Benchmark tests for critical operations + +## Workflow-Specific Tests + +### Jobs Workflow Tests +```bash +# All jobs tests +pytest tests/integration/api/jobs/ tests/unit/orchestrator/jobs/ -v + +# Job creation and idempotency +pytest tests/integration/api/jobs/test_create_job_api.py -v + +# Job lifecycle management +pytest tests/integration/api/jobs/test_get_job_api.py -v +``` + +### Catalog Workflow Tests +```bash +# All catalog tests +pytest tests/integration/api/catalog_roles/ tests/unit/core/catalog/ -v + +# Catalog parsing +pytest tests/integration/api/parse_catalog/ -v + +# Role generation +pytest tests/unit/orchestrator/catalog/ -v +``` + +### Local Repository Workflow Tests +```bash +# All local repo tests +pytest tests/integration/api/local_repo/ tests/unit/core/localrepo/ -v + +# Repository creation +pytest tests/integration/api/local_repo/test_create_local_repo.py -v +``` + +### Image Building Workflow Tests +```bash +# All build image tests +pytest tests/integration/api/build_image/ tests/unit/core/build_image/ -v + +# Multi-architecture builds +pytest tests/integration/api/build_image/ -k multi_arch -v +``` + +### Validation Workflow Tests +```bash +# All validation tests +pytest tests/integration/api/validate/ tests/unit/core/validate/ -v + +# Schema validation +pytest tests/unit/core/validate/ -k schema -v +``` + ## Test Fixtures ### Shared Fixtures (conftest.py) +**Authentication & Authorization:** - `client`: FastAPI TestClient with dev container - `auth_headers`: Standard authentication headers +- `admin_auth_headers`: Admin-level authentication + +**Idempotency & Correlation:** - `unique_idempotency_key`: Unique key per test - `unique_correlation_id`: Unique correlation ID per test +**Database & Storage:** +- `db_session`: Database session for tests +- `clean_db`: Fresh database for each test +- `artifact_store`: Test artifact storage + +**Mock Services:** +- `mock_vault_client`: Mocked Vault integration +- `mock_pulp_client`: Mocked Pulp integration +- `mock_registry_client`: Mocked container registry + ### Usage Example ```python -def test_example(client, auth_headers): - payload = {"catalog_uri": "s3://bucket/catalog.json"} +def test_create_job(client, auth_headers, unique_idempotency_key): + """Test job creation with idempotency.""" + payload = { + "catalog_uri": "s3://bucket/catalog.json", + "idempotency_key": unique_idempotency_key + } response = client.post("/api/v1/jobs", json=payload, headers=auth_headers) assert response.status_code == 201 + assert "job_id" in response.json() ``` ## Coverage Report @@ -134,11 +304,93 @@ Add to GitHub Actions workflow: ## Test Best Practices -1. **Isolation**: Each test is independent (unique idempotency keys) -2. **Fast**: Integration tests complete in <5 seconds each -3. **Deterministic**: No flaky tests, no time-dependent logic -4. **Clear**: Descriptive test names following pattern `test___` -5. **Comprehensive**: Cover happy path, error cases, edge cases, and security +### Test Design Principles + +1. **Isolation**: Each test is independent and can run in any order + - Use unique idempotency keys and correlation IDs + - Clean up resources after each test + - Avoid shared mutable state + +2. **Fast Execution**: Tests should complete quickly + - Unit tests: <100ms each + - Integration tests: <5 seconds each + - Use mocks for external dependencies + +3. **Deterministic**: Tests produce consistent results + - No flaky tests or race conditions + - Avoid time-dependent logic + - Use fixed test data + +4. **Clear Naming**: Follow descriptive naming conventions + - Pattern: `test___` + - Example: `test_create_job_with_invalid_catalog_returns_400` + +5. **Comprehensive Coverage**: Test all scenarios + - Happy path (success cases) + - Error cases (validation failures, exceptions) + - Edge cases (boundary conditions) + - Security (authentication, authorization) + +### Test Organization + +**Arrange-Act-Assert Pattern:** +```python +def test_example(): + # Arrange: Set up test data and preconditions + payload = {"catalog_uri": "s3://bucket/catalog.json"} + + # Act: Execute the operation being tested + response = client.post("/api/v1/jobs", json=payload) + + # Assert: Verify the expected outcome + assert response.status_code == 201 + assert "job_id" in response.json() +``` + +**Test Grouping:** +- Group related tests in classes +- Use descriptive class names (e.g., `TestCreateJobSuccess`, `TestCreateJobValidation`) +- Share setup/teardown logic within classes + +### Security Testing + +**Authentication Tests:** +- Test endpoints without authentication (should return 401) +- Test with invalid tokens (should return 401) +- Test with expired tokens (should return 401) + +**Authorization Tests:** +- Test with insufficient permissions (should return 403) +- Test role-based access control +- Verify resource ownership checks + +**Input Validation:** +- Test SQL injection attempts +- Test XSS payloads +- Test path traversal attempts +- Test oversized inputs + +### Mocking Guidelines + +**When to Mock:** +- External HTTP APIs (Vault, Pulp, registries) +- File system operations (for unit tests) +- Time-dependent operations +- Expensive computations + +**When NOT to Mock:** +- Database operations (use test database) +- Core business logic +- Internal service calls +- Simple utility functions + +### Code Coverage Goals + +- **Overall**: >80% code coverage +- **Core Domain**: >90% coverage +- **API Routes**: >85% coverage +- **Use Cases**: >90% coverage +- **Critical Paths**: 100% coverage ## Troubleshooting @@ -170,3 +422,181 @@ pytest tests/ pip install pytest-xdist pytest tests/ -n auto ``` + +### Database Connection Issues + +```bash +# Ensure PostgreSQL is running +# Check connection settings in environment variables + +# For Windows PowerShell +$env:DATABASE_URL = "postgresql://user:password@localhost:5432/build_stream_test" + +# For Linux/Mac +export DATABASE_URL="postgresql://user:password@localhost:5432/build_stream_test" + +# Run migrations +alembic upgrade head + +# Run tests +pytest tests/ +``` + +### Authentication Failures + +```bash +# Verify Vault is accessible (if using real Vault) +# Or ensure mock Vault is configured + +# Check JWT token configuration +# Verify environment variables are set correctly +``` + +## Environment Configuration + +### Required Environment Variables + +For running tests, configure the following environment variables: + +**Windows PowerShell:** +```powershell +$env:ENV = "dev" +$env:HOST = "0.0.0.0" +$env:PORT = "8000" +$env:DATABASE_URL = "postgresql://user:password@localhost:5432/build_stream_test" +$env:LOG_LEVEL = "DEBUG" +``` + +**Linux/Mac:** +```bash +export ENV=dev +export HOST=0.0.0.0 +export PORT=8000 +export DATABASE_URL=postgresql://user:password@localhost:5432/build_stream_test +export LOG_LEVEL=DEBUG +``` + +### Test Database Setup + +```bash +# Create test database +createdb build_stream_test + +# Run migrations +alembic upgrade head + +# Verify database +psql build_stream_test -c "\dt" +``` + +## Writing New Tests + +### Adding a New Unit Test + +1. Create test file in appropriate `tests/unit/` subdirectory +2. Import required modules and fixtures +3. Write test functions following naming conventions +4. Use mocks for external dependencies +5. Run tests to verify + +**Example:** +```python +# tests/unit/core/jobs/test_job_entity.py +import pytest +from core.jobs.entities import Job +from core.jobs.value_objects import JobId, StageName + +def test_job_creation_with_valid_data(): + """Test job entity creation with valid data.""" + job_id = JobId.generate() + job = Job(job_id=job_id, client_id="test-client") + + assert job.job_id == job_id + assert job.client_id == "test-client" + assert job.status == "pending" +``` + +### Adding a New Integration Test + +1. Create test file in appropriate `tests/integration/` subdirectory +2. Use shared fixtures from conftest.py +3. Test full request/response cycles +4. Verify database state changes +5. Clean up test data + +**Example:** +```python +# tests/integration/api/jobs/test_create_job_integration.py +def test_create_job_integration(client, auth_headers, unique_idempotency_key): + """Test complete job creation flow.""" + payload = { + "catalog_uri": "s3://test-bucket/catalog.json", + "idempotency_key": unique_idempotency_key + } + + response = client.post("/api/v1/jobs", json=payload, headers=auth_headers) + + assert response.status_code == 201 + data = response.json() + assert "job_id" in data + assert data["status"] == "pending" +``` + +## Continuous Integration + +### GitHub Actions Example + +```yaml +name: Test Suite + +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:15 + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: build_stream_test + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install -r requirements-dev.txt + + - name: Run tests + env: + ENV: dev + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/build_stream_test + run: | + pytest tests/ -v --cov=api --cov=orchestrator --cov=core --cov-report=xml + + - name: Upload coverage + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml +``` + +## Additional Resources + +- [Main Build Stream README](../README.md) - Architecture and getting started +- [Developer Guide](../doc/developer-guide.md) - Comprehensive development guide +- [Workflow Documentation](../doc/) - Detailed workflow guides +- [pytest Documentation](https://docs.pytest.org/) - pytest framework reference +- [FastAPI Testing](https://fastapi.tiangolo.com/tutorial/testing/) - FastAPI testing guide diff --git a/build_stream/tests/fixtures/catalogs/adapter_policy_test.json b/build_stream/tests/fixtures/catalogs/adapter_policy_test.json deleted file mode 100644 index d62429d3b2..0000000000 --- a/build_stream/tests/fixtures/catalogs/adapter_policy_test.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "version": "2.0.0", - "description": "Test fixture for v2 target-centric mapping spec.", - "architectures": ["aarch64", "x86_64"], - "targets": { - "default_packages.json": { - "transform": { - "exclude_fields": ["architecture"] - }, - "sources": [ - { - "source_file": "base_os.json", - "pulls": [ - {"source_key": "Base OS", "target_key": "default_packages"} - ] - } - ] - }, - "service_k8s.json": { - "transform": { - "exclude_fields": ["architecture"] - }, - "sources": [ - { - "source_file": "functional_layer.json", - "pulls": [ - {"source_key": "K8S Controller", "target_key": "service_kube_control_plane"}, - {"source_key": "K8S Worker", "target_key": "service_kube_node"} - ] - } - ], - "derived": [ - { - "target_key": "service_k8s", - "operation": { - "type": "extract_common", - "from_keys": ["service_kube_control_plane", "service_kube_node"], - "min_occurrences": 2, - "remove_from_sources": true - } - } - ] - }, - "slurm_custom.json": { - "transform": { - "exclude_fields": ["architecture"] - }, - "sources": [ - { - "source_file": "functional_layer.json", - "pulls": [ - {"source_key": "Login Node", "target_key": "login_node"}, - {"source_key": "Compiler", "target_key": "login_compiler_node"}, - {"source_key": "Slurm Controller", "target_key": "slurm_control_node"}, - {"source_key": "Slurm Worker", "target_key": "slurm_node"} - ] - } - ], - "derived": [ - { - "target_key": "slurm_custom", - "operation": { - "type": "extract_common", - "from_keys": ["login_node", "login_compiler_node", "slurm_control_node", "slurm_node"], - "min_occurrences": 2, - "remove_from_sources": true - } - } - ] - } - } -} diff --git a/build_stream/tests/fixtures/catalogs/catalog_rhel.json b/build_stream/tests/fixtures/catalogs/catalog_rhel.json deleted file mode 100644 index d62b67b73f..0000000000 --- a/build_stream/tests/fixtures/catalogs/catalog_rhel.json +++ /dev/null @@ -1,4220 +0,0 @@ -{ - "Catalog": { - "Name": "Catalog", - "Version": "1.0", - "Identifier": "image-build", - "FunctionalLayer": [ - { - "Name": "login_compiler_node_aarch64", - "FunctionalPackages": [ - "package_id_1", - "package_id_14", - "package_id_2", - "package_id_20", - "package_id_3", - "package_id_4", - "package_id_5", - "package_id_6", - "package_id_7", - "package_id_8", - "package_id_9" - ] - }, - { - "Name": "login_node_x86_64", - "FunctionalPackages": [ - "package_id_1", - "package_id_14", - "package_id_20", - "package_id_3", - "package_id_4", - "package_id_5", - "package_id_7", - "package_id_8", - "package_id_9" - ] - }, - { - "Name": "service_kube_control_plane_x86_64", - "FunctionalPackages": [ - "package_id_21", - "package_id_22", - "package_id_23", - "package_id_24", - "package_id_25", - "package_id_26", - "package_id_27", - "package_id_28", - "package_id_29", - "package_id_3", - "package_id_30", - "package_id_31", - "package_id_32", - "package_id_33", - "package_id_34", - "package_id_35", - "package_id_36", - "package_id_37", - "package_id_38", - "package_id_39", - "package_id_4", - "package_id_40", - "package_id_41", - "package_id_42", - "package_id_43", - "package_id_44", - "package_id_45", - "package_id_46", - "package_id_47", - "package_id_48", - "package_id_49", - "package_id_50", - "package_id_51", - "package_id_52", - "package_id_53", - "package_id_54", - "package_id_55", - "package_id_56", - "package_id_57", - "package_id_58", - "package_id_59", - "package_id_60", - "package_id_61", - "package_id_62", - "package_id_63", - "package_id_64", - "package_id_65", - "package_id_66", - "package_id_67", - "package_id_68", - "package_id_69", - "package_id_70", - "package_id_8", - "package_id_9" - ] - }, - { - "Name": "service_kube_node_x86_64", - "FunctionalPackages": [ - "package_id_21", - "package_id_22", - "package_id_23", - "package_id_24", - "package_id_25", - "package_id_26", - "package_id_27", - "package_id_28", - "package_id_29", - "package_id_3", - "package_id_30", - "package_id_31", - "package_id_32", - "package_id_33", - "package_id_34", - "package_id_35", - "package_id_36", - "package_id_37", - "package_id_38", - "package_id_39", - "package_id_4", - "package_id_40", - "package_id_41", - "package_id_42", - "package_id_43", - "package_id_44", - "package_id_45", - "package_id_46", - "package_id_47", - "package_id_48", - "package_id_49", - "package_id_61", - "package_id_71", - "package_id_72", - "package_id_8", - "package_id_9" - ] - }, - { - "Name": "slurm_control_node_x86_64", - "FunctionalPackages": [ - "package_id_1", - "package_id_10", - "package_id_11", - "package_id_12", - "package_id_13", - "package_id_3", - "package_id_4", - "package_id_5", - "package_id_7", - "package_id_73", - "package_id_74", - "package_id_75", - "package_id_76", - "package_id_8", - "package_id_9" - ] - }, - { - "Name": "slurm_node_aarch64", - "FunctionalPackages": [ - "package_id_1", - "package_id_14", - "package_id_15", - "package_id_16", - "package_id_17", - "package_id_18", - "package_id_19", - "package_id_2", - "package_id_3", - "package_id_4", - "package_id_5", - "package_id_6", - "package_id_7", - "package_id_8", - "package_id_9" - ] - } - ], - "BaseOS": [ - { - "Name": "RHEL", - "Version": "10.0", - "osPackages": [ - "os_package_id_1", - "os_package_id_10", - "os_package_id_11", - "os_package_id_12", - "os_package_id_13", - "os_package_id_14", - "os_package_id_15", - "os_package_id_16", - "os_package_id_17", - "os_package_id_18", - "os_package_id_19", - "os_package_id_2", - "os_package_id_20", - "os_package_id_21", - "os_package_id_22", - "os_package_id_23", - "os_package_id_24", - "os_package_id_25", - "os_package_id_26", - "os_package_id_27", - "os_package_id_28", - "os_package_id_29", - "os_package_id_3", - "os_package_id_30", - "os_package_id_31", - "os_package_id_32", - "os_package_id_33", - "os_package_id_34", - "os_package_id_35", - "os_package_id_36", - "os_package_id_37", - "os_package_id_38", - "os_package_id_39", - "os_package_id_4", - "os_package_id_40", - "os_package_id_41", - "os_package_id_42", - "os_package_id_43", - "os_package_id_44", - "os_package_id_45", - "os_package_id_46", - "os_package_id_47", - "os_package_id_48", - "os_package_id_49", - "os_package_id_5", - "os_package_id_50", - "os_package_id_51", - "os_package_id_52", - "os_package_id_53", - "os_package_id_54", - "os_package_id_55", - "os_package_id_56", - "os_package_id_57", - "os_package_id_58", - "os_package_id_59", - "os_package_id_6", - "os_package_id_60", - "os_package_id_61", - "os_package_id_62", - "os_package_id_63", - "os_package_id_64", - "os_package_id_65", - "os_package_id_66", - "os_package_id_67", - "os_package_id_68", - "os_package_id_69", - "os_package_id_7", - "os_package_id_70", - "os_package_id_71", - "os_package_id_72", - "os_package_id_73", - "os_package_id_74", - "os_package_id_75", - "os_package_id_76", - "os_package_id_77", - "os_package_id_78", - "os_package_id_79", - "os_package_id_8", - "os_package_id_80", - "os_package_id_81", - "os_package_id_82", - "os_package_id_83", - "os_package_id_84", - "os_package_id_85", - "os_package_id_86", - "os_package_id_87", - "os_package_id_88", - "os_package_id_89", - "os_package_id_9", - "os_package_id_90", - "os_package_id_91", - "os_package_id_92", - "os_package_id_93", - "os_package_id_94", - "os_package_id_95", - "os_package_id_96" - ] - } - ], - "Infrastructure": [ - { - "Name": "csi", - "InfrastructurePackages": [ - "infrastructure_package_id_1", - "infrastructure_package_id_10", - "infrastructure_package_id_11", - "infrastructure_package_id_12", - "infrastructure_package_id_13", - "infrastructure_package_id_14", - "infrastructure_package_id_15", - "infrastructure_package_id_16", - "infrastructure_package_id_2", - "infrastructure_package_id_3", - "infrastructure_package_id_4", - "infrastructure_package_id_5", - "infrastructure_package_id_6", - "infrastructure_package_id_7", - "infrastructure_package_id_8", - "infrastructure_package_id_9" - ] - } - ], - "Drivers": [], - "DriverPackages": {}, - "FunctionalPackages": { - "package_id_1": { - "Name": "munge", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "package_id_2": { - "Name": "munge-devel", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_codeready-builder" - } - ] - }, - "package_id_3": { - "Name": "firewalld", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "package_id_4": { - "Name": "python3-firewall", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "package_id_5": { - "Name": "pmix", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "package_id_6": { - "Name": "pmix-devel", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - } - ] - }, - "package_id_7": { - "Name": "nvcr.io/nvidia/hpc-benchmarks", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "image", - "Tag": "25.09", - "Version": "25.09" - }, - "package_id_8": { - "Name": "apptainer", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "epel" - }, - { - "Architecture": "x86_64", - "RepoName": "epel" - } - ] - }, - "package_id_9": { - "Name": "doca-ofed", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm_repo", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "doca" - }, - { - "Architecture": "x86_64", - "RepoName": "doca" - } - ] - }, - "package_id_10": { - "Name": "slurm-slurmctld", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_slurm_custom" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_slurm_custom" - } - ] - }, - "package_id_11": { - "Name": "slurm-slurmdbd", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_slurm_custom" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_slurm_custom" - } - ] - }, - "package_id_12": { - "Name": "python3-PyMySQL", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "package_id_13": { - "Name": "mariadb-server", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "package_id_14": { - "Name": "slurm-slurmd", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_slurm_custom" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_slurm_custom" - } - ] - }, - "package_id_15": { - "Name": "slurm-pam_slurm", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_slurm_custom" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_slurm_custom" - } - ] - }, - "package_id_16": { - "Name": "kernel-devel", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "package_id_17": { - "Name": "kernel-headers", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "package_id_18": { - "Name": "cuda-run", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "iso", - "Sources": [ - { - "Architecture": "aarch64", - "Uri": "https://developer.download.nvidia.com/compute/cuda/13.0.2/local_installers/cuda_13.0.2_580.95.05_linux_sbsa.run" - }, - { - "Architecture": "x86_64", - "Uri": "https://developer.download.nvidia.com/compute/cuda/13.0.2/local_installers/cuda_13.0.2_580.95.05_linux.run" - } - ] - }, - "package_id_19": { - "Name": "nvhpc_2025_2511_Linux_aarch64_cuda_13.0", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64" - ], - "Type": "tarball", - "Sources": [ - { - "Architecture": "aarch64", - "Uri": "https://developer.download.nvidia.com/hpc-sdk/25.11/nvhpc_2025_2511_Linux_aarch64_cuda_13.0.tar.gz" - } - ] - }, - "package_id_20": { - "Name": "slurm", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_slurm_custom" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_slurm_custom" - } - ] - }, - "package_id_21": { - "Name": "docker.io/library/busybox", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "1.36", - "Version": "1.36" - }, - "package_id_22": { - "Name": "git", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "package_id_23": { - "Name": "vim", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "package_id_24": { - "Name": "fuse-overlayfs", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "package_id_25": { - "Name": "podman", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "package_id_26": { - "Name": "kubeadm-1.34.1", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "x86_64", - "RepoName": "kubernetes" - } - ] - }, - "package_id_27": { - "Name": "kubelet-1.34.1", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "x86_64", - "RepoName": "kubernetes" - } - ] - }, - "package_id_28": { - "Name": "container-selinux", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "package_id_29": { - "Name": "cri-o-1.34.1", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "x86_64", - "RepoName": "cri-o" - } - ] - }, - "package_id_30": { - "Name": "docker.io/victoriametrics/victoria-metrics", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "v1.128.0", - "Version": "v1.128.0" - }, - "package_id_31": { - "Name": "docker.io/victoriametrics/vmagent", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "v1.128.0", - "Version": "v1.128.0" - }, - "package_id_32": { - "Name": "docker.io/victoriametrics/vmstorage", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" - }, - "package_id_33": { - "Name": "docker.io/victoriametrics/vminsert", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" - }, - "package_id_34": { - "Name": "docker.io/victoriametrics/vmselect", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" - }, - "package_id_35": { - "Name": "docker.io/alpine/kubectl", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "1.34.1", - "Version": "1.34.1" - }, - "package_id_36": { - "Name": "docker.io/curlimages/curl", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "8.17.0", - "Version": "8.17.0" - }, - "package_id_37": { - "Name": "docker.io/rmohr/activemq", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "5.15.9", - "Version": "5.15.9" - }, - "package_id_38": { - "Name": "docker.io/library/mysql", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "9.3.0", - "Version": "9.3.0" - }, - "package_id_39": { - "Name": "docker.io/dellhpcomniaaisolution/idrac_telemetry_receiver", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "1.0", - "Version": "1.0" - }, - "package_id_40": { - "Name": "docker.io/dellhpcomniaaisolution/kafkapump", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "1.0", - "Version": "1.0" - }, - "package_id_41": { - "Name": "docker.io/dellhpcomniaaisolution/victoriapump", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "1.0", - "Version": "1.0" - }, - "package_id_42": { - "Name": "cryptography==45.0.7", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "pip_module" - }, - "package_id_43": { - "Name": "omsdk==1.2.518", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "pip_module" - }, - "package_id_44": { - "Name": "cffi==1.17.1", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "pip_module" - }, - "package_id_45": { - "Name": "quay.io/strimzi/operator", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "0.48.0", - "Version": "0.48.0" - }, - "package_id_46": { - "Name": "quay.io/strimzi/kafka", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "0.48.0-kafka-4.1.0", - "Version": "0.48.0-kafka-4.1.0" - }, - "package_id_47": { - "Name": "docker.io/dellhpcomniaaisolution/ubuntu-ldms", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "1.0", - "Version": "1.0" - }, - "package_id_48": { - "Name": "strimzi-kafka-operator-helm-3-chart-0.48.0", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "tarball", - "Sources": [ - { - "Architecture": "x86_64", - "Uri": "https://github.com/strimzi/strimzi-kafka-operator/releases/download/0.48.0/strimzi-kafka-operator-helm-3-chart-0.48.0.tgz" - } - ] - }, - "package_id_49": { - "Name": "quay.io/strimzi/kafka-bridge", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "0.33.1", - "Version": "0.33.1" - }, - "package_id_50": { - "Name": "ghcr.io/kube-vip/kube-vip", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "v0.8.9", - "Version": "v0.8.9" - }, - "package_id_51": { - "Name": "registry.k8s.io/kube-apiserver", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "v1.34.1", - "Version": "v1.34.1" - }, - "package_id_52": { - "Name": "registry.k8s.io/kube-controller-manager", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "v1.34.1", - "Version": "v1.34.1" - }, - "package_id_53": { - "Name": "registry.k8s.io/kube-scheduler", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "v1.34.1", - "Version": "v1.34.1" - }, - "package_id_54": { - "Name": "registry.k8s.io/kube-proxy", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "v1.34.1", - "Version": "v1.34.1" - }, - "package_id_55": { - "Name": "registry.k8s.io/coredns/coredns", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "v1.12.1", - "Version": "v1.12.1" - }, - "package_id_56": { - "Name": "registry.k8s.io/pause", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "3.10.1", - "Version": "3.10.1" - }, - "package_id_57": { - "Name": "registry.k8s.io/etcd", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "3.6.4-0", - "Version": "3.6.4-0" - }, - "package_id_58": { - "Name": "docker.io/calico/cni", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "v3.30.3", - "Version": "v3.30.3" - }, - "package_id_59": { - "Name": "docker.io/calico/kube-controllers", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "v3.30.3", - "Version": "v3.30.3" - }, - "package_id_60": { - "Name": "docker.io/calico/node", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "v3.30.3", - "Version": "v3.30.3" - }, - "package_id_61": { - "Name": "quay.io/metallb/speaker", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "v0.15.2", - "Version": "v0.15.2" - }, - "package_id_62": { - "Name": "kubectl-1.34.1", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "x86_64", - "RepoName": "kubernetes" - } - ] - }, - "package_id_63": { - "Name": "prettytable==3.14.0", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "pip_module" - }, - "package_id_64": { - "Name": "python3.12", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "package_id_65": { - "Name": "kubernetes==33.1.0", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "pip_module" - }, - "package_id_66": { - "Name": "PyMySQL==1.1.2", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "pip_module" - }, - "package_id_67": { - "Name": "calico-v3.30.3", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "manifest", - "Sources": [ - { - "Architecture": "x86_64", - "Uri": "https://raw.githubusercontent.com/projectcalico/calico/v3.30.3/manifests/calico.yaml" - } - ] - }, - "package_id_68": { - "Name": "metallb-native-v0.15.2", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "manifest", - "Sources": [ - { - "Architecture": "x86_64", - "Uri": "https://raw.githubusercontent.com/metallb/metallb/v0.15.2/config/manifests/metallb-native.yaml" - } - ] - }, - "package_id_69": { - "Name": "helm-v3.19.0-amd64", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "tarball", - "Sources": [ - { - "Architecture": "x86_64", - "Uri": "https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz" - } - ] - }, - "package_id_70": { - "Name": "nfs-subdir-external-provisioner-4.0.18", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "tarball", - "Sources": [ - { - "Architecture": "x86_64", - "Uri": "https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/releases/download/nfs-subdir-external-provisioner-4.0.18/nfs-subdir-external-provisioner-4.0.18.tgz" - } - ] - }, - "package_id_71": { - "Name": "registry.k8s.io/sig-storage/nfs-subdir-external-provisioner", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "v4.0.2", - "Version": "v4.0.2" - }, - "package_id_72": { - "Name": "quay.io/metallb/controller", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "v0.15.2", - "Version": "v0.15.2" - }, - "package_id_73": { - "Name": "iscsi-initiator-utils", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "package_id_74": { - "Name": "device-mapper-multipath", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "package_id_75": { - "Name": "sg3_utils", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "package_id_76": { - "Name": "lsscsi", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "package_id_77": { - "Name": "nvhpc_2025_2511_Linux_x86_64_cuda_13.0", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "tarball", - "Sources": [ - { - "Architecture": "x86_64", - "Uri": "https://developer.download.nvidia.com/hpc-sdk/25.11/nvhpc_2025_2511_Linux_x86_64_cuda_13.0.tar.gz" - } - ] - } - }, - "OSPackages": { - "os_package_id_1": { - "Name": "which", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_2": { - "Name": "tcpdump", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_3": { - "Name": "traceroute", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_4": { - "Name": "iperf3", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_5": { - "Name": "fping", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "epel" - }, - { - "Architecture": "x86_64", - "RepoName": "epel" - } - ] - }, - "os_package_id_6": { - "Name": "dmidecode", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_7": { - "Name": "hwloc", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_8": { - "Name": "hwloc-libs", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_9": { - "Name": "lshw", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_10": { - "Name": "pciutils", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_11": { - "Name": "vim-enhanced", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_12": { - "Name": "emacs", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_13": { - "Name": "zsh", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_14": { - "Name": "openssh", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_15": { - "Name": "openssh-server", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_16": { - "Name": "openssh-clients", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_17": { - "Name": "rsync", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_18": { - "Name": "file", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_19": { - "Name": "libcurl", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_20": { - "Name": "tar", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_21": { - "Name": "bzip2", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_22": { - "Name": "man-db", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_23": { - "Name": "man-pages", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_24": { - "Name": "strace", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_25": { - "Name": "kexec-tools", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_26": { - "Name": "openssl-devel", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_27": { - "Name": "ipmitool", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_28": { - "Name": "gdb", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_29": { - "Name": "gdb-gdbserver", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_30": { - "Name": "lldb", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_31": { - "Name": "lldb-devel", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_32": { - "Name": "valgrind", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_33": { - "Name": "valgrind-devel", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_34": { - "Name": "ltrace", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_35": { - "Name": "kernel-tools", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_36": { - "Name": "perf", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_37": { - "Name": "papi", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_38": { - "Name": "papi-devel", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_39": { - "Name": "papi-libs", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_40": { - "Name": "cmake", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_41": { - "Name": "make", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_42": { - "Name": "autoconf", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_43": { - "Name": "automake", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_44": { - "Name": "libtool", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_45": { - "Name": "gcc", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_46": { - "Name": "gcc-c++", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_47": { - "Name": "gcc-gfortran", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_48": { - "Name": "binutils", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_49": { - "Name": "binutils-devel", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_50": { - "Name": "clustershell", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "epel" - }, - { - "Architecture": "x86_64", - "RepoName": "epel" - } - ] - }, - "os_package_id_51": { - "Name": "bash-completion", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_52": { - "Name": "systemd", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_53": { - "Name": "systemd-udev", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_54": { - "Name": "kernel", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_55": { - "Name": "dracut", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_56": { - "Name": "dracut-live", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_57": { - "Name": "dracut-network", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_58": { - "Name": "squashfs-tools", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_59": { - "Name": "nfs-utils", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_60": { - "Name": "nfs4-acl-tools", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_61": { - "Name": "NetworkManager", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_62": { - "Name": "nm-connection-editor", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_63": { - "Name": "iproute", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_64": { - "Name": "iputils", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_65": { - "Name": "curl", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_66": { - "Name": "bash", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_67": { - "Name": "coreutils", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_68": { - "Name": "grep", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_69": { - "Name": "sed", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_70": { - "Name": "gawk", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_71": { - "Name": "findutils", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_72": { - "Name": "util-linux", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_73": { - "Name": "kbd", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_74": { - "Name": "lsof", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_75": { - "Name": "cryptsetup", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_76": { - "Name": "lvm2", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_77": { - "Name": "device-mapper", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_78": { - "Name": "rsyslog", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_79": { - "Name": "chrony", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_80": { - "Name": "sudo", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_81": { - "Name": "gzip", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_82": { - "Name": "wget", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_83": { - "Name": "cloud-init", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_84": { - "Name": "glibc-langpack-en", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_85": { - "Name": "gedit", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "epel" - }, - { - "Architecture": "x86_64", - "RepoName": "epel" - } - ] - }, - "os_package_id_86": { - "Name": "docker.io/dellhpcomniaaisolution/image-build-aarch64", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64" - ], - "Type": "image", - "Tag": "1.1", - "Version": "1.1" - }, - "os_package_id_87": { - "Name": "python3-devel", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_88": { - "Name": "python3-cython", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_appstream" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_89": { - "Name": "openssl-libs", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_90": { - "Name": "ovis-ldms", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_ldms" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_ldms" - } - ] - }, - "os_package_id_91": { - "Name": "openldap-clients", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_92": { - "Name": "nss-pam-ldapd", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "epel" - }, - { - "Architecture": "x86_64", - "RepoName": "epel" - } - ] - }, - "os_package_id_93": { - "Name": "sssd", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_94": { - "Name": "oddjob-mkhomedir", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_appstream" - } - ] - }, - "os_package_id_95": { - "Name": "authselect", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "aarch64", - "x86_64" - ], - "Type": "rpm", - "Sources": [ - { - "Architecture": "aarch64", - "RepoName": "aarch64_baseos" - }, - { - "Architecture": "x86_64", - "RepoName": "x86_64_baseos" - } - ] - }, - "os_package_id_96": { - "Name": "docker.io/dellhpcomniaaisolution/image-build-el10", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "1.0", - "Version": "1.0" - } - }, - "Miscellaneous": [], - "InfrastructurePackages": { - "infrastructure_package_id_1": { - "Name": "csi-powerscale", - "Type": "git", - "Version": "v2.15.0", - "SupportedFunctions": [ - { - "Name": "csi" - } - ], - "Architecture": [ - "x86_64" - ], - "Sources": [ - { - "Architecture": "x86_64", - "Uri": "https://github.com/dell/csi-powerscale.git" - } - ] - }, - "infrastructure_package_id_2": { - "Name": "external-snapshotter", - "Type": "git", - "Version": "v8.3.0", - "SupportedFunctions": [ - { - "Name": "csi" - } - ], - "Architecture": [ - "x86_64" - ], - "Sources": [ - { - "Architecture": "x86_64", - "Uri": "https://github.com/kubernetes-csi/external-snapshotter.git" - } - ] - }, - "infrastructure_package_id_3": { - "Name": "helm-charts", - "Type": "git", - "Version": "csi-isilon-2.15.0", - "SupportedFunctions": [ - { - "Name": "csi" - } - ], - "Architecture": [ - "x86_64" - ], - "Sources": [ - { - "Architecture": "x86_64", - "Uri": "https://github.com/dell/helm-charts.git" - } - ] - }, - "infrastructure_package_id_4": { - "Name": "quay.io/dell/container-storage-modules/csi-isilon", - "Type": "image", - "Version": "v2.15.0", - "SupportedFunctions": [ - { - "Name": "csi" - } - ], - "Architecture": [ - "x86_64" - ], - "Tag": "v2.15.0" - }, - "infrastructure_package_id_5": { - "Name": "registry.k8s.io/sig-storage/csi-attacher", - "Type": "image", - "Version": "v4.9.0", - "SupportedFunctions": [ - { - "Name": "csi" - } - ], - "Architecture": [ - "x86_64" - ], - "Tag": "v4.9.0" - }, - "infrastructure_package_id_6": { - "Name": "registry.k8s.io/sig-storage/csi-provisioner", - "Type": "image", - "Version": "v5.3.0", - "SupportedFunctions": [ - { - "Name": "csi" - } - ], - "Architecture": [ - "x86_64" - ], - "Tag": "v5.3.0" - }, - "infrastructure_package_id_7": { - "Name": "registry.k8s.io/sig-storage/csi-snapshotter", - "Type": "image", - "Version": "v8.3.0", - "SupportedFunctions": [ - { - "Name": "csi" - } - ], - "Architecture": [ - "x86_64" - ], - "Tag": "v8.3.0" - }, - "infrastructure_package_id_8": { - "Name": "registry.k8s.io/sig-storage/csi-resizer", - "Type": "image", - "Version": "v1.14.0", - "SupportedFunctions": [ - { - "Name": "csi" - } - ], - "Architecture": [ - "x86_64" - ], - "Tag": "v1.14.0" - }, - "infrastructure_package_id_9": { - "Name": "registry.k8s.io/sig-storage/csi-node-driver-registrar", - "Type": "image", - "Version": "v2.14.0", - "SupportedFunctions": [ - { - "Name": "csi" - } - ], - "Architecture": [ - "x86_64" - ], - "Tag": "v2.14.0" - }, - "infrastructure_package_id_10": { - "Name": "registry.k8s.io/sig-storage/csi-external-health-monitor-controller", - "Type": "image", - "Version": "v0.15.0", - "SupportedFunctions": [ - { - "Name": "csi" - } - ], - "Architecture": [ - "x86_64" - ], - "Tag": "v0.15.0" - }, - "infrastructure_package_id_11": { - "Name": "quay.io/dell/container-storage-modules/dell-csi-replicator", - "Type": "image", - "Version": "v1.13.0", - "SupportedFunctions": [ - { - "Name": "csi" - } - ], - "Architecture": [ - "x86_64" - ], - "Tag": "v1.13.0" - }, - "infrastructure_package_id_12": { - "Name": "quay.io/dell/container-storage-modules/podmon", - "Type": "image", - "Version": "v1.14.0", - "SupportedFunctions": [ - { - "Name": "csi" - } - ], - "Architecture": [ - "x86_64" - ], - "Tag": "v1.14.0" - }, - "infrastructure_package_id_13": { - "Name": "quay.io/dell/container-storage-modules/csm-authorization-sidecar", - "Type": "image", - "Version": "v2.3.0", - "SupportedFunctions": [ - { - "Name": "csi" - } - ], - "Architecture": [ - "x86_64" - ], - "Tag": "v2.3.0" - }, - "infrastructure_package_id_14": { - "Name": "quay.io/dell/container-storage-modules/csi-metadata-retriever", - "Type": "image", - "Version": "v1.12.0", - "SupportedFunctions": [ - { - "Name": "csi" - } - ], - "Architecture": [ - "x86_64" - ], - "Tag": "v1.12.0" - }, - "infrastructure_package_id_15": { - "Name": "registry.k8s.io/sig-storage/snapshot-controller", - "Type": "image", - "Version": "v8.3.0", - "SupportedFunctions": [ - { - "Name": "csi" - } - ], - "Architecture": [ - "x86_64" - ], - "Tag": "v8.3.0" - }, - "infrastructure_package_id_16": { - "Name": "docker.io/dellemc/csm-encryption", - "Type": "image", - "Version": "v0.6.0", - "SupportedFunctions": [ - { - "Name": "csi" - } - ], - "Architecture": [ - "x86_64" - ], - "Tag": "v0.6.0" - } - } - } -} \ No newline at end of file diff --git a/build_stream/tests/fixtures/catalogs/functional_layer.json b/build_stream/tests/fixtures/catalogs/functional_layer.json deleted file mode 100644 index bbe8eb6000..0000000000 --- a/build_stream/tests/fixtures/catalogs/functional_layer.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "Compiler": { - "packages": [ - {"package": "firewalld", "type": "rpm", "repo_name": "x86_64_baseos", "architecture": ["x86_64"]}, - {"package": "munge", "type": "rpm", "repo_name": "x86_64_appstream", "architecture": ["x86_64"]}, - {"package": "python3-firewall", "type": "rpm", "repo_name": "x86_64_baseos", "architecture": ["x86_64"]}, - {"package": "slurm", "type": "rpm", "repo_name": "x86_64_slurm_custom", "architecture": ["x86_64"]}, - {"package": "slurm-slurmd", "type": "rpm", "repo_name": "x86_64_slurm_custom", "architecture": ["x86_64"]} - ] - }, - "K8S Controller": { - "packages": [ - {"package": "PyMySQL==1.1.2", "type": "pip_module", "architecture": ["x86_64"]}, - {"package": "docker.io/bitnamilegacy/kafka", "type": "image", "tag": "3.9.0", "architecture": ["x86_64"]}, - {"package": "docker.io/flannel/flannel", "type": "image", "tag": "v0.22.0", "architecture": ["x86_64"]}, - {"package": "docker.io/flannel/flannel-cni-plugin", "type": "image", "tag": "v1.1.2", "architecture": ["x86_64"]}, - {"package": "docker.io/kubernetesui/dashboard", "type": "image", "tag": "v2.7.0", "architecture": ["x86_64"]}, - {"package": "docker.io/kubernetesui/metrics-scraper", "type": "image", "tag": "v1.0.8", "architecture": ["x86_64"]}, - {"package": "docker.io/library/busybox", "type": "image", "tag": "1.36", "architecture": ["x86_64"]}, - {"package": "docker.io/library/golang", "type": "image", "tag": "1.24", "architecture": ["x86_64"]}, - {"package": "docker.io/library/mysql", "type": "image", "tag": "9.3.0", "architecture": ["x86_64"]}, - {"package": "docker.io/rmohr/activemq", "type": "image", "tag": "5.15.9", "architecture": ["x86_64"]}, - {"package": "calico.archive-v3.29.3", "type": "tarball", "uri": "https://github.com/projectcalico/calico/archive/v3.29.3.tar.gz", "architecture": ["x86_64"]}, - {"package": "firewalld", "type": "rpm", "repo_name": "x86_64_baseos", "architecture": ["x86_64"]}, - {"package": "ghcr.io/k8snetworkplumbingwg/whereabouts", "type": "image", "tag": "latest", "architecture": ["x86_64"]}, - {"package": "ghcr.io/kube-vip/kube-vip", "type": "image", "tag": "v0.8.9", "architecture": ["x86_64"]}, - {"package": "git", "type": "rpm", "repo_name": "x86_64_appstream", "architecture": ["x86_64"]}, - {"package": "helm-v3.16.4-amd64", "type": "tarball", "uri": "https://get.helm.sh/helm-v3.16.4-linux-amd64.tar.gz", "architecture": ["x86_64"]}, - {"package": "kubeadm-v1.31.4-amd64", "type": "tarball", "uri": "https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubeadm", "architecture": ["x86_64"]}, - {"package": "kubectl-v1.31.4-amd64", "type": "tarball", "uri": "https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl", "architecture": ["x86_64"]}, - {"package": "kubelet-v1.31.4-amd64", "type": "tarball", "uri": "https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubelet", "architecture": ["x86_64"]}, - {"package": "kubernetes==33.1.0", "type": "pip_module", "architecture": ["x86_64"]}, - {"package": "calicoctl-v3.29.3-amd64", "type": "tarball", "uri": "https://github.com/projectcalico/calico/releases/download/v3.29.3/calicoctl-linux-amd64", "architecture": ["x86_64"]}, - {"package": "nerdctl-v2.0.5-amd64", "type": "tarball", "uri": "https://github.com/containerd/nerdctl/releases/download/v2.0.5/nerdctl-2.0.5-linux-amd64.tar.gz", "architecture": ["x86_64"]}, - {"package": "nfs-subdir-external-provisioner-4.0.18", "type": "tarball", "uri": "https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/releases/download/nfs-subdir-external-provisioner-4.0.18/nfs-subdir-external-provisioner-4.0.18.tgz", "architecture": ["x86_64"]}, - {"package": "omsdk==1.2.518", "type": "pip_module", "architecture": ["x86_64"]}, - {"package": "prettytable==3.14.0", "type": "pip_module", "architecture": ["x86_64"]}, - {"package": "public.ecr.aws/xilinx_dcg/k8s-device-plugin", "type": "image", "tag": "1.2.0", "architecture": ["x86_64"]}, - {"package": "python3-firewall", "type": "rpm", "repo_name": "x86_64_baseos", "architecture": ["x86_64"]}, - {"package": "cffi==1.17.1", "type": "pip_module", "architecture": ["x86_64"]}, - {"package": "python3.12", "type": "rpm", "repo_name": "x86_64_appstream", "architecture": ["x86_64"]}, - {"package": "quay.io/calico/cni", "type": "image", "tag": "v3.29.3", "architecture": ["x86_64"]}, - {"package": "quay.io/calico/kube-controllers", "type": "image", "tag": "v3.29.3", "architecture": ["x86_64"]}, - {"package": "quay.io/calico/node", "type": "image", "tag": "v3.29.3", "architecture": ["x86_64"]}, - {"package": "quay.io/calico/pod2daemon-flexvol", "type": "image", "tag": "v3.29.1", "architecture": ["x86_64"]}, - {"package": "quay.io/metallb/controller", "type": "image", "tag": "v0.13.9", "architecture": ["x86_64"]}, - {"package": "quay.io/metallb/speaker", "type": "image", "tag": "v0.13.9", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/coredns/coredns", "type": "image", "tag": "v1.11.3", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/cpa/cluster-proportional-autoscaler", "type": "image", "tag": "v1.8.8", "architecture": ["x86_64"]}, - {"package": "cni-v1.4.1-amd64", "type": "tarball", "uri": "https://github.com/containernetworking/plugins/releases/download/v1.4.1/cni-plugins-linux-amd64-v1.4.1.tgz", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/dns/k8s-dns-node-cache", "type": "image", "tag": "1.25.0", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/ingress-nginx/kube-webhook-certgen", "type": "image", "tag": "v1.6.0", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/kube-apiserver", "type": "image", "tag": "v1.31.4", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/kube-controller-manager", "type": "image", "tag": "v1.31.4", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/kube-proxy", "type": "image", "tag": "v1.31.4", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/kube-scheduler", "type": "image", "tag": "v1.31.4", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/kube-state-metrics/kube-state-metrics", "type": "image", "tag": "v2.16.0", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/nfd/node-feature-discovery", "type": "image", "tag": "v0.12.1", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/pause", "type": "image", "tag": "3.10", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/sig-storage/nfs-subdir-external-provisioner", "type": "image", "tag": "v4.0.2", "architecture": ["x86_64"]}, - {"package": "runc.amd64-v1.2.6", "type": "tarball", "uri": "https://github.com/opencontainers/runc/releases/download/v1.2.6/runc.amd64", "architecture": ["x86_64"]}, - {"package": "whereabouts", "type": "git", "uri": "https://github.com/k8snetworkplumbingwg/whereabouts.git", "architecture": ["x86_64"]}, - {"package": "containerd-v2.0.5-amd64", "type": "tarball", "uri": "https://github.com/containerd/containerd/releases/download/v2.0.5/containerd-2.0.5-linux-amd64.tar.gz", "architecture": ["x86_64"]}, - {"package": "crictl-v1.31.1-amd64", "type": "tarball", "uri": "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.31.1/crictl-v1.31.1-linux-amd64.tar.gz", "architecture": ["x86_64"]}, - {"package": "cryptography==45.0.7", "type": "pip_module", "architecture": ["x86_64"]} - ] - }, - "K8S Worker": { - "packages": [ - {"package": "docker.io/bitnamilegacy/kafka", "type": "image", "tag": "3.9.0", "architecture": ["x86_64"]}, - {"package": "docker.io/flannel/flannel", "type": "image", "tag": "v0.22.0", "architecture": ["x86_64"]}, - {"package": "docker.io/flannel/flannel-cni-plugin", "type": "image", "tag": "v1.1.2", "architecture": ["x86_64"]}, - {"package": "docker.io/kubernetesui/dashboard", "type": "image", "tag": "v2.7.0", "architecture": ["x86_64"]}, - {"package": "docker.io/kubernetesui/metrics-scraper", "type": "image", "tag": "v1.0.8", "architecture": ["x86_64"]}, - {"package": "docker.io/library/busybox", "type": "image", "tag": "1.36", "architecture": ["x86_64"]}, - {"package": "docker.io/library/golang", "type": "image", "tag": "1.24", "architecture": ["x86_64"]}, - {"package": "docker.io/library/mysql", "type": "image", "tag": "9.3.0", "architecture": ["x86_64"]}, - {"package": "docker.io/library/nginx", "type": "image", "tag": "1.27.4-alpine", "architecture": ["x86_64"]}, - {"package": "docker.io/rmohr/activemq", "type": "image", "tag": "5.15.9", "architecture": ["x86_64"]}, - {"package": "calico.archive-v3.29.3", "type": "tarball", "uri": "https://github.com/projectcalico/calico/archive/v3.29.3.tar.gz", "architecture": ["x86_64"]}, - {"package": "firewalld", "type": "rpm", "repo_name": "x86_64_baseos", "architecture": ["x86_64"]}, - {"package": "git", "type": "rpm", "repo_name": "x86_64_appstream", "architecture": ["x86_64"]}, - {"package": "kubeadm-v1.31.4-amd64", "type": "tarball", "uri": "https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubeadm", "architecture": ["x86_64"]}, - {"package": "kubelet-v1.31.4-amd64", "type": "tarball", "uri": "https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubelet", "architecture": ["x86_64"]}, - {"package": "calicoctl-v3.29.3-amd64", "type": "tarball", "uri": "https://github.com/projectcalico/calico/releases/download/v3.29.3/calicoctl-linux-amd64", "architecture": ["x86_64"]}, - {"package": "nerdctl-v2.0.5-amd64", "type": "tarball", "uri": "https://github.com/containerd/nerdctl/releases/download/v2.0.5/nerdctl-2.0.5-linux-amd64.tar.gz", "architecture": ["x86_64"]}, - {"package": "nfs-utils", "type": "rpm", "repo_name": "epel", "architecture": ["x86_64"]}, - {"package": "omsdk==1.2.518", "type": "pip_module", "architecture": ["x86_64"]}, - {"package": "python3-firewall", "type": "rpm", "repo_name": "x86_64_baseos", "architecture": ["x86_64"]}, - {"package": "cffi==1.17.1", "type": "pip_module", "architecture": ["x86_64"]}, - {"package": "quay.io/calico/cni", "type": "image", "tag": "v3.29.3", "architecture": ["x86_64"]}, - {"package": "quay.io/calico/kube-controllers", "type": "image", "tag": "v3.29.3", "architecture": ["x86_64"]}, - {"package": "quay.io/calico/node", "type": "image", "tag": "v3.29.3", "architecture": ["x86_64"]}, - {"package": "quay.io/calico/pod2daemon-flexvol", "type": "image", "tag": "v3.29.1", "architecture": ["x86_64"]}, - {"package": "quay.io/metallb/speaker", "type": "image", "tag": "v0.13.9", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/coredns/coredns", "type": "image", "tag": "v1.11.3", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/cpa/cluster-proportional-autoscaler", "type": "image", "tag": "v1.8.8", "architecture": ["x86_64"]}, - {"package": "cni-v1.4.1-amd64", "type": "tarball", "uri": "https://github.com/containernetworking/plugins/releases/download/v1.4.1/cni-plugins-linux-amd64-v1.4.1.tgz", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/dns/k8s-dns-node-cache", "type": "image", "tag": "1.25.0", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/ingress-nginx/kube-webhook-certgen", "type": "image", "tag": "v1.6.0", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/kube-proxy", "type": "image", "tag": "v1.31.4", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/kube-state-metrics/kube-state-metrics", "type": "image", "tag": "v2.16.0", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/nfd/node-feature-discovery", "type": "image", "tag": "v0.12.1", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/pause", "type": "image", "tag": "3.10", "architecture": ["x86_64"]}, - {"package": "registry.k8s.io/sig-storage/nfs-subdir-external-provisioner", "type": "image", "tag": "v4.0.2", "architecture": ["x86_64"]}, - {"package": "conntrack-tools", "type": "rpm", "repo_name": "x86_64_appstream", "architecture": ["x86_64"]}, - {"package": "runc.amd64-v1.2.6", "type": "tarball", "uri": "https://github.com/opencontainers/runc/releases/download/v1.2.6/runc.amd64", "architecture": ["x86_64"]}, - {"package": "tcpdump", "type": "rpm", "repo_name": "epel", "architecture": ["x86_64"]}, - {"package": "traceroute", "type": "rpm", "repo_name": "epel", "architecture": ["x86_64"]}, - {"package": "vim", "type": "rpm", "repo_name": "x86_64_appstream", "architecture": ["x86_64"]}, - {"package": "containerd-v2.0.5-amd64", "type": "tarball", "uri": "https://github.com/containerd/containerd/releases/download/v2.0.5/containerd-2.0.5-linux-amd64.tar.gz", "architecture": ["x86_64"]}, - {"package": "crictl-v1.31.1-amd64", "type": "tarball", "uri": "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.31.1/crictl-v1.31.1-linux-amd64.tar.gz", "architecture": ["x86_64"]}, - {"package": "cryptography==45.0.7", "type": "pip_module", "architecture": ["x86_64"]} - ] - }, - "Login Node": { - "packages": [ - {"package": "firewalld", "type": "rpm", "repo_name": "x86_64_baseos", "architecture": ["x86_64"]}, - {"package": "munge", "type": "rpm", "repo_name": "x86_64_appstream", "architecture": ["x86_64"]}, - {"package": "python3-firewall", "type": "rpm", "repo_name": "x86_64_baseos", "architecture": ["x86_64"]}, - {"package": "slurm", "type": "rpm", "repo_name": "x86_64_slurm_custom", "architecture": ["x86_64"]}, - {"package": "slurm-slurmd", "type": "rpm", "repo_name": "x86_64_slurm_custom", "architecture": ["x86_64"]} - ] - }, - "Slurm Controller": { - "packages": [ - {"package": "firewalld", "type": "rpm", "repo_name": "x86_64_baseos", "architecture": ["x86_64"]}, - {"package": "mariadb-server", "type": "rpm", "repo_name": "x86_64_appstream", "architecture": ["x86_64"]}, - {"package": "munge", "type": "rpm", "repo_name": "x86_64_appstream", "architecture": ["x86_64"]}, - {"package": "python3-PyMySQL", "type": "rpm", "repo_name": "x86_64_appstream", "architecture": ["x86_64"]}, - {"package": "python3-firewall", "type": "rpm", "repo_name": "x86_64_baseos", "architecture": ["x86_64"]}, - {"package": "slurm-slurmctld", "type": "rpm", "repo_name": "x86_64_slurm_custom", "architecture": ["x86_64"]}, - {"package": "slurm-slurmdbd", "type": "rpm", "repo_name": "x86_64_slurm_custom", "architecture": ["x86_64"]} - ] - }, - "Slurm Worker": { - "packages": [ - {"package": "firewalld", "type": "rpm", "repo_name": "x86_64_baseos", "architecture": ["x86_64"]}, - {"package": "munge", "type": "rpm", "repo_name": "x86_64_appstream", "architecture": ["x86_64"]}, - {"package": "python3-firewall", "type": "rpm", "repo_name": "x86_64_baseos", "architecture": ["x86_64"]}, - {"package": "slurm-pam_slurm", "type": "rpm", "repo_name": "x86_64_slurm_custom", "architecture": ["x86_64"]}, - {"package": "slurm-slurmd", "type": "rpm", "repo_name": "x86_64_slurm_custom", "architecture": ["x86_64"]} - ] - } -} diff --git a/build_stream/tests/fixtures/catalogs/sample.py b/build_stream/tests/fixtures/catalogs/sample.py deleted file mode 100644 index 89f2472d6c..0000000000 --- a/build_stream/tests/fixtures/catalogs/sample.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Example script showing programmatic usage of the generator and adapter APIs. - -This script runs the catalog feature-list generator and adapter config generator -directly from Python, configuring logging and handling common errors. -""" - -import logging -import os - -from catalog_parser.generator import generate_root_json_from_catalog, get_functional_layer_roles_from_file, get_package_list -from catalog_parser.adapter import generate_omnia_json_from_catalog -from catalog_parser.adapter_policy import generate_configs_from_policy - -BASE_DIR = os.path.dirname(os.path.dirname(__file__)) -CATALOG_PARSER_DIR = os.path.join(BASE_DIR, "") -CATALOG_PATH = os.path.join(CATALOG_PARSER_DIR, "test_fixtures", "catalog_rhel.json") -SCHEMA_PATH = os.path.join(CATALOG_PARSER_DIR, "resources", "CatalogSchema.json") -FUNCTIONAL_LAYER_PATH = os.path.join(CATALOG_PARSER_DIR, "test_fixtures", "functional_layer.json") -ADAPTER_POLICY_PATH = os.path.join(CATALOG_PARSER_DIR, "resources", "adapter_policy_default.json") -ADAPTER_POLICY_SCHEMA_PATH = os.path.join(CATALOG_PARSER_DIR, "resources", "AdapterPolicySchema.json") - -try: - generate_root_json_from_catalog( - catalog_path=CATALOG_PATH, - schema_path=SCHEMA_PATH, - output_root="out/generator2", - configure_logging=True, - log_file="logs/generator.log", - log_level=logging.INFO, - ) - - generate_omnia_json_from_catalog( - catalog_path=CATALOG_PATH, - schema_path=SCHEMA_PATH, - output_root="out/adapter/config2", - configure_logging=True, - log_file="logs/adapter.log", - log_level=logging.INFO, - ) - - generate_configs_from_policy( - input_dir="out/generator2", - output_dir="out/adapter_policy/config2", - policy_path=ADAPTER_POLICY_PATH, - schema_path=ADAPTER_POLICY_SCHEMA_PATH, - configure_logging=True, - log_file="logs/adapter_policy.log", - log_level=logging.INFO, - ) - - roles = get_functional_layer_roles_from_file(FUNCTIONAL_LAYER_PATH) - print(f"Functional layer roles: {roles}") - - # Get packages for a specific role - result = get_package_list(FUNCTIONAL_LAYER_PATH, role="K8S Controller") - print(f"Packages for role 'K8S Controller': {result}") - - # Get packages for all roles - result = get_package_list(FUNCTIONAL_LAYER_PATH) - print(f"Packages for all roles: {result}") - -except FileNotFoundError as e: - # handle missing catalog/schema - print(f"Missing file: {e}") -except Exception as e: - # handle generic processing errors - print(f"Processing failed: {e}") \ No newline at end of file diff --git a/build_stream/tests/fixtures/catalogs/sample_catalog.py b/build_stream/tests/fixtures/catalogs/sample_catalog.py deleted file mode 100644 index 89f2472d6c..0000000000 --- a/build_stream/tests/fixtures/catalogs/sample_catalog.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Example script showing programmatic usage of the generator and adapter APIs. - -This script runs the catalog feature-list generator and adapter config generator -directly from Python, configuring logging and handling common errors. -""" - -import logging -import os - -from catalog_parser.generator import generate_root_json_from_catalog, get_functional_layer_roles_from_file, get_package_list -from catalog_parser.adapter import generate_omnia_json_from_catalog -from catalog_parser.adapter_policy import generate_configs_from_policy - -BASE_DIR = os.path.dirname(os.path.dirname(__file__)) -CATALOG_PARSER_DIR = os.path.join(BASE_DIR, "") -CATALOG_PATH = os.path.join(CATALOG_PARSER_DIR, "test_fixtures", "catalog_rhel.json") -SCHEMA_PATH = os.path.join(CATALOG_PARSER_DIR, "resources", "CatalogSchema.json") -FUNCTIONAL_LAYER_PATH = os.path.join(CATALOG_PARSER_DIR, "test_fixtures", "functional_layer.json") -ADAPTER_POLICY_PATH = os.path.join(CATALOG_PARSER_DIR, "resources", "adapter_policy_default.json") -ADAPTER_POLICY_SCHEMA_PATH = os.path.join(CATALOG_PARSER_DIR, "resources", "AdapterPolicySchema.json") - -try: - generate_root_json_from_catalog( - catalog_path=CATALOG_PATH, - schema_path=SCHEMA_PATH, - output_root="out/generator2", - configure_logging=True, - log_file="logs/generator.log", - log_level=logging.INFO, - ) - - generate_omnia_json_from_catalog( - catalog_path=CATALOG_PATH, - schema_path=SCHEMA_PATH, - output_root="out/adapter/config2", - configure_logging=True, - log_file="logs/adapter.log", - log_level=logging.INFO, - ) - - generate_configs_from_policy( - input_dir="out/generator2", - output_dir="out/adapter_policy/config2", - policy_path=ADAPTER_POLICY_PATH, - schema_path=ADAPTER_POLICY_SCHEMA_PATH, - configure_logging=True, - log_file="logs/adapter_policy.log", - log_level=logging.INFO, - ) - - roles = get_functional_layer_roles_from_file(FUNCTIONAL_LAYER_PATH) - print(f"Functional layer roles: {roles}") - - # Get packages for a specific role - result = get_package_list(FUNCTIONAL_LAYER_PATH, role="K8S Controller") - print(f"Packages for role 'K8S Controller': {result}") - - # Get packages for all roles - result = get_package_list(FUNCTIONAL_LAYER_PATH) - print(f"Packages for all roles: {result}") - -except FileNotFoundError as e: - # handle missing catalog/schema - print(f"Missing file: {e}") -except Exception as e: - # handle generic processing errors - print(f"Processing failed: {e}") \ No newline at end of file From c1ebd5dfd70bffd7abab308c5c7f475cb88f8987 Mon Sep 17 00:00:00 2001 From: priti-parate <140157516+priti-parate@users.noreply.github.com> Date: Wed, 11 Mar 2026 13:17:07 +0530 Subject: [PATCH 08/11] revert merge conflict changes --- .../common_utils/en_us_validation_msg.py | 3 +- .../schema/build_stream_config.json | 8 ---- .../build_stream_validation.py | 43 ++++++++++++++++++- 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py b/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py index 6773ac0ba4..9e0622dd91 100644 --- a/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py +++ b/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py @@ -446,7 +446,8 @@ def server_spec_network_key_fail_msg(nic_device): BUILD_STREAM_PORT_RANGE_MSG = "build_stream_port must be an integer between 1 and 65535." BUILD_STREAM_PORT_INUSE_MSG = ( "Port {port} is already in use and is not serving build_stream on {host_ip}. Please choose another free port." - +) + BUILD_STREAM_HOST_IP_REQUIRED_MSG = ( "Field 'build_stream_host_ip' is mandatory in build_stream_config.yml. " "Please provide a valid IPv4 address (OIM admin IP or OIM public IP)." diff --git a/common/library/module_utils/input_validation/schema/build_stream_config.json b/common/library/module_utils/input_validation/schema/build_stream_config.json index 7a82de5298..e6108358ce 100644 --- a/common/library/module_utils/input_validation/schema/build_stream_config.json +++ b/common/library/module_utils/input_validation/schema/build_stream_config.json @@ -23,14 +23,6 @@ "pattern": "^$|^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$", "description": "AArch64 inventory host IP for aarch64 builds. Must be in same subnet as OIM admin IP. Accepted values: valid IPv4 address, empty string, or null" }, - "postgres_db_name": { - "type": "string", - "minLength": 1, - "maxLength": 63, - "pattern": "^[a-zA-Z0-9_]+$", - "description": "PostgreSQL database name for build stream. Default: build_stream_db" - - } }, "required": ["enable_build_stream", "build_stream_host_ip", "build_stream_port"], "additionalProperties": false diff --git a/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py b/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py index 1a267fee8d..90fa8e165c 100644 --- a/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py +++ b/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py @@ -184,7 +184,48 @@ def validate_build_stream_config(input_file_path, data, msg.build_stream_host_ip_not_oim_ip_msg(build_stream_host_ip, ethernet_ips) )) - # Validate aarch64_inventory_host_ip (conditional - required if PXE mapping has aarch64 groups) + # Validate aarch64_inventory_host_ip + # Validate build_stream_port availability + build_stream_port = data.get("build_stream_port") + if build_stream_port: + try: + port_int = int(build_stream_port) + if not (1 <= port_int <= 65535): + raise ValueError + except (TypeError, ValueError): + errors.append(create_error_msg( + build_stream_yml, + "build_stream_port", + msg.BUILD_STREAM_PORT_RANGE_MSG, + )) + return errors + + port_in_use = False + try: + with socket.create_connection((build_stream_host_ip, port_int), timeout=2): + port_in_use = True + except (OSError, ValueError): + port_in_use = False + + if port_in_use: + # Port is in use, check if it's build_stream by probing /health + try: + context = ssl._create_unverified_context() + conn = client.HTTPSConnection(build_stream_host_ip, port_int, timeout=2, context=context) + conn.request("GET", "/health") + resp = conn.getresponse() + conn.close() + if resp.status not in [200, 401, 403, 404, 500]: + raise ValueError(f"Unexpected HTTP status {resp.status}") + except Exception as exc: # pylint: disable=broad-except + errors.append(create_error_msg( + build_stream_yml, + "build_stream_port", + msg.BUILD_STREAM_PORT_INUSE_MSG.format(port=port_int, host_ip=build_stream_host_ip, detail=str(exc)), + )) + return errors + + # Validate aarch64_inventory_host_ip aarch64_inventory_host_ip = data.get("aarch64_inventory_host_ip") ### aarch64_inventory_host_ip check From d4caea3ddd73ac2f433f5dd1285366864bab67e7 Mon Sep 17 00:00:00 2001 From: priti-parate <140157516+priti-parate@users.noreply.github.com> Date: Wed, 11 Mar 2026 13:20:03 +0530 Subject: [PATCH 09/11] fix merge conflict issues --- .../input_validation/schema/build_stream_config.json | 2 +- .../validation_flows/build_stream_validation.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/common/library/module_utils/input_validation/schema/build_stream_config.json b/common/library/module_utils/input_validation/schema/build_stream_config.json index e6108358ce..e61cc7a945 100644 --- a/common/library/module_utils/input_validation/schema/build_stream_config.json +++ b/common/library/module_utils/input_validation/schema/build_stream_config.json @@ -22,7 +22,7 @@ "type": ["string", "null"], "pattern": "^$|^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$", "description": "AArch64 inventory host IP for aarch64 builds. Must be in same subnet as OIM admin IP. Accepted values: valid IPv4 address, empty string, or null" - }, + } }, "required": ["enable_build_stream", "build_stream_host_ip", "build_stream_port"], "additionalProperties": false diff --git a/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py b/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py index 90fa8e165c..5dd00bede8 100644 --- a/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py +++ b/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py @@ -281,7 +281,6 @@ def validate_build_stream_config(input_file_path, data, # Check aarch64 host IP reachability using socket (safer than subprocess) try: - import socket # Try to connect to SSH port which is usually open on inventory hosts ssh_port = 22 # SSH reachable = False From 590c4ee66c1987e22a030ed3fa8b918696927a12 Mon Sep 17 00:00:00 2001 From: priti-parate <140157516+priti-parate@users.noreply.github.com> Date: Wed, 11 Mar 2026 13:21:46 +0530 Subject: [PATCH 10/11] update readme doc --- build_stream/README.md | 21 ++-- build_stream/doc/validation.md | 219 +++++++++++++++++++++------------ 2 files changed, 151 insertions(+), 89 deletions(-) diff --git a/build_stream/README.md b/build_stream/README.md index 623171b3b4..a33010b75b 100644 --- a/build_stream/README.md +++ b/build_stream/README.md @@ -1,6 +1,6 @@ # Build Stream -Build Stream is a RESTful API service that orchestrates the creation and management of build jobs for the Omnia infrastructure platform. It provides a centralized interface for managing software catalog parsing, local repository creation, image building, and validation workflows. +**Build Stream** is a **RESTful API** (Representational State Transfer Application Programming Interface) service that orchestrates the creation and management of build jobs for the Omnia infrastructure platform. It provides a centralized interface for managing software catalog parsing, local repository creation, image building, and validation workflows. ## Architecture Overview @@ -14,7 +14,7 @@ Build Stream follows a clean architecture pattern with clear separation of conce ## High-Level Workflow -1. **Authentication**: JWT-based authentication secures all API endpoints +1. **Authentication**: **JWT** (JSON Web Token)-based authentication secures all API endpoints 2. **Job Creation**: Clients submit build requests through the jobs API 3. **Stage Processing**: Jobs are broken into stages (catalog parsing, local repo, build image, validation) 4. **Async Execution**: Stages execute asynchronously with result polling @@ -33,7 +33,7 @@ Key configuration areas: - Database connections (PostgreSQL) - Artifact storage backend (file system or in-memory) - Vault endpoints and authentication -- CORS and server settings +- **CORS** (Cross-Origin Resource Sharing) and server settings ## Getting Started @@ -58,8 +58,8 @@ pip install -r requirements.txt pip install -r requirements-dev.txt # Set environment variables -export HOST=0.0.0.0 -export PORT=8000 +export HOST= +export PORT= # Run development server uvicorn main:app --reload @@ -70,6 +70,8 @@ pytest **API Documentation:** - See Omnia ReadTheDocs for complete API documentation +- Local development docs: `http://localhost:${PORT}/docs` +- Local ReDoc: `http://localhost:${PORT}/redoc` ### Architecture Components @@ -102,13 +104,15 @@ Each major workflow area has dedicated documentation: - **Build Image** - Container image build orchestration - **Validation** - Input validation and output verification +See the `doc/` directory for detailed workflow documentation. + ## Dependencies Build Stream uses FastAPI with the following key dependencies: - FastAPI/Uvicorn for web framework -- SQLAlchemy for database ORM -- Dependency Injector for IoC container -- PyJWT for authentication +- SQLAlchemy for database **ORM** (Object-Relational Mapping) +- Dependency Injector for **IoC** (Inversion of Control) container +- PyJWT for **JWT** (JSON Web Token) authentication - Ansible for infrastructure automation - Vault client for secret management @@ -119,5 +123,4 @@ For troubleshooting and development guidance: 2. Review API logs for error details 3. Consult the audit trail for job execution history 4. Refer to the health check endpoint: `/health` -5. See [tests README](tests/README.md) for pytest guidance and test execution diff --git a/build_stream/doc/validation.md b/build_stream/doc/validation.md index 8ad5780c83..d8ee8cd2e8 100644 --- a/build_stream/doc/validation.md +++ b/build_stream/doc/validation.md @@ -1,29 +1,34 @@ # Validation -The Validation workflow provides comprehensive input and output validation for all Build Stream operations. +The Validation workflow provides comprehensive validation for built images on provided testbeds specified in the PXE mapping file. ## What It Does The Validation workflow provides: -- Input schema validation for all API requests -- Output validation for generated artifacts -- Cross-workflow dependency validation -- Security and compliance checking -- Quality assurance and testing integration +- **validate_image_on_test** - Validates built images on testbeds +- Testbed deployment from PXE mapping file configuration +- Image boot testing and functionality validation +- Network connectivity and service validation +- Performance and resource utilization testing +- Compliance and security validation on target hardware ## Inputs/Outputs **Inputs:** -- API request payloads and parameters -- Generated artifacts and configurations -- Build outputs and container images -- Security scan results and reports +- Built container images from Build Image workflow +- User-specified testbeds from catalog for validation +- PXE mapping file with testbed configurations +- Test validation criteria and test scripts +- Network and hardware specifications +- Expected service configurations **Outputs:** -- Validation reports and results -- Error details and correction suggestions -- Compliance status and recommendations -- Quality metrics and measurements +- Testbed deployment results and status +- Image boot validation reports +- Service functionality test results +- Performance metrics and benchmarks +- Error diagnostics and troubleshooting guides +- Compliance validation reports ## Key Logic Locations @@ -35,101 +40,155 @@ The Validation workflow provides: - `core/validate/services.py` - Validation processing services **Main Components:** -- **ValidateUseCase** - Orchestrates validation processes -- **SchemaValidator** - Handles JSON schema validation -- **SecurityValidator** - Performs security compliance checks -- **QualityValidator** - Assesses output quality +- **ValidateImageOnTestUseCase** - Orchestrates image validation on testbeds +- **PXEMappingParser** - Parses PXE mapping file for testbed configurations +- **TestbedDeployer** - Deploys images to testbeds via PXE +- **ImageBootValidator** - Validates image boot and startup +- **ServiceValidator** - Tests service functionality +- **PerformanceValidator** - Measures performance metrics +- **ComplianceValidator** - Checks compliance on target hardware ## Validation Types -**Input Validation:** -- JSON schema validation for API requests -- Parameter type and range checking -- Authentication and authorization verification -- File format and size validation - -**Output Validation:** -- Generated file structure validation -- Container image security scanning -- Configuration file syntax checking -- Dependency integrity verification - -**Cross-Workflow Validation:** -- Catalog-to-repository dependency validation -- Repository-to-image package validation -- Image-to-deployment compatibility checking -- End-to-end workflow validation +**Image Boot Validation:** +- PXE boot configuration validation +- Image loading and initialization testing +- Kernel and initrd validation +- Boot sequence verification +- Hardware compatibility checking + +**Service Validation:** +- Service startup and registration testing +- API endpoint accessibility validation +- Database connectivity verification +- Network service functionality testing +- Inter-service communication validation + +**Performance Validation:** +- CPU and memory utilization testing +- Disk I/O and network throughput testing +- Response time and latency measurement +- Load testing and stress testing +- Resource optimization validation + +**Compliance Validation:** +- Security policy validation on target hardware +- Regulatory compliance checking +- Configuration standard validation +- Access control verification +- Audit trail validation ## Workflow Flow -1. **Validation Request**: Client submits validation request -2. **Schema Validation**: Input schemas validated against definitions -3. **Security Checking**: Security policies and compliance verified -4. **Quality Assessment**: Output quality metrics evaluated -5. **Dependency Validation**: Cross-component dependencies verified -6. **Report Generation**: Comprehensive validation reports created -7. **Result Storage**: Validation results stored for audit trail -8. **Notification**: Validation status notifications sent - -## Schema Management - -Schema validation includes: -- **JSON Schema** - Standard JSON schema validation -- **Custom Validators** - Business-specific validation rules -- **Version Compatibility** - Schema version compatibility checking -- **Extensible Rules** - Configurable validation policies +1. **Validation Request**: Client submits image validation request with specified testbeds from catalog +2. **PXE Mapping Parsing**: Testbed configurations extracted from PXE mapping file +3. **Testbed Configuration**: User-provided testbeds from catalog are configured for validation +4. **Image Deployment**: Container image deployed to specified testbeds via PXE +5. **Manual PXE Boot**: User runs `set_pxe_boot` utility to boot the images +6. **Boot Validation**: Image boot sequence validated and monitored +7. **Service Testing**: Deployed services tested for functionality +8. **Performance Testing**: Performance metrics collected and analyzed +9. **Compliance Checking**: Security and compliance validation performed +10. **Report Generation**: Comprehensive validation reports created +11. **Result Storage**: Validation results stored for audit trail +12. **Notification**: Validation status notifications sent + +## Manual PXE Boot Step + +After the `validate_image_on_test` API completes image deployment, users must manually run the `set_pxe_boot` utility from `omnia/utils/set_pxe_boot` to initiate the boot process: + +**Required Action:** +```bash +# Run the set_pxe_boot utility from omnia/utils to boot deployed images +omnia/utils/set_pxe_boot --testbed -i +``` + +**Purpose:** +- Configures PXE boot settings for the deployed images +- Initiates the boot sequence on selected testbeds +- Enables monitoring and validation of the boot process +- Provides manual control over boot timing and test execution + +**Parameters:** +- `--testbed`: Target testbed identifier from PXE mapping file +- `-i`: Image name to boot (from validation request) +- Optional: `--timeout`: Boot timeout duration +- Optional: `--debug`: Enable debug logging + +**Integration Notes:** +- Must be run after `validate_image_on_test` API completes successfully +- Prepares testbeds for automated boot validation monitoring +- Enables subsequent boot validation, service testing, and performance measurement + +## PXE Mapping Management + +PXE mapping configuration includes: +- **Testbed Definitions** - Hardware specifications and capabilities +- **Network Configuration** - IP addresses and network settings +- **Boot Parameters** - Kernel parameters and boot options +- **Storage Configuration** - Disk layouts and mount points +- **Validation Criteria** - Test requirements and success criteria ## Security Validation Security checks include: -- **Vulnerability Scanning** - Container image vulnerability analysis -- **Credential Validation** - Secure credential verification -- **Access Control** - Permission and authorization checking -- **Compliance Checking** - Regulatory compliance validation +- **Image Security Scanning** - Container image vulnerability analysis +- **Testbed Security** - Testbed access control and isolation +- **Network Security** - Network segmentation and firewall validation +- **Data Protection** - Sensitive data protection on testbeds +- **Compliance Checking** - Hardware and software compliance validation ## Quality Assurance Quality metrics include: -- **Code Quality** - Generated code style and structure -- **Configuration Validity** - Configuration file correctness -- **Performance Metrics** - Resource usage and efficiency -- **Reliability Checks** - Error handling and robustness +- **Boot Reliability** - Image boot success rate and stability +- **Service Availability** - Service uptime and accessibility +- **Performance Metrics** - Response times and resource utilization +- **Hardware Compatibility** - Hardware driver compatibility and performance +- **Test Coverage** - Validation test completeness and effectiveness ## Integration Points -- Validates inputs for all API endpoints -- Checks outputs from all workflow stages -- Integrates with external security scanning tools -- Connects to compliance and audit systems +- Integrates with Build Image workflow for image validation +- Connects to PXE infrastructure for testbed deployment +- Integrates with monitoring systems for performance metrics +- Connects to testbed management systems for hardware control +- Links to compliance systems for regulatory validation ## Configuration Validation configuration includes: -- Schema definitions and versions -- Security policies and thresholds -- Quality standards and metrics -- Compliance requirements and rules +- PXE mapping file locations and formats +- User-specified testbeds from catalog for validation +- Validation test suites and test scripts +- Performance thresholds and benchmarks +- Compliance rules and security policies ## Error Handling -- Detailed validation error reporting -- Suggested corrections and fixes -- Error categorization and prioritization -- Automated retry for validation failures +- Testbed deployment failure diagnostics +- Image boot error analysis and troubleshooting +- Service failure detection and recovery suggestions +- Performance issue identification and optimization recommendations +- Automated testbed recovery and retry mechanisms ## Reporting Validation reports provide: -- Overall validation status summary -- Detailed error and warning lists -- Security vulnerability assessments -- Quality metrics and trends -- Compliance status and recommendations +- Image validation status summary across testbeds +- Boot performance and reliability metrics +- Service functionality test results +- Performance benchmarks and comparisons +- Hardware compatibility assessment +- Security and compliance validation status +- Troubleshooting guides and recommendations ## Continuous Validation Ongoing validation includes: -- Automated validation in CI/CD pipelines -- Periodic security scanning -- Continuous quality monitoring -- Regular compliance checking +- Automated image testing on new builds +- Periodic testbed health and performance monitoring +- Continuous hardware compatibility validation +- Regular security and compliance checking +- Performance regression testing +- Testbed maintenance and optimization From c8997e2aef91eff36062a01b275bd27f8e7bdcda Mon Sep 17 00:00:00 2001 From: priti-parate <140157516+priti-parate@users.noreply.github.com> Date: Thu, 12 Mar 2026 16:41:07 +0530 Subject: [PATCH 11/11] addressing review comments --- build_stream/README.md | 2 - build_stream/doc/build_image.md | 104 ++++----- build_stream/doc/catalog.md | 23 +- build_stream/doc/developer-guide.md | 319 ---------------------------- build_stream/doc/jobs.md | 48 +++-- 5 files changed, 72 insertions(+), 424 deletions(-) delete mode 100644 build_stream/doc/developer-guide.md diff --git a/build_stream/README.md b/build_stream/README.md index a33010b75b..f69ae6afed 100644 --- a/build_stream/README.md +++ b/build_stream/README.md @@ -70,8 +70,6 @@ pytest **API Documentation:** - See Omnia ReadTheDocs for complete API documentation -- Local development docs: `http://localhost:${PORT}/docs` -- Local ReDoc: `http://localhost:${PORT}/redoc` ### Architecture Components diff --git a/build_stream/doc/build_image.md b/build_stream/doc/build_image.md index 8a159d95a7..fd981c7863 100644 --- a/build_stream/doc/build_image.md +++ b/build_stream/doc/build_image.md @@ -1,114 +1,98 @@ -# Image Building +# OS Image Building -The Image Building workflow orchestrates container image creation for the Omnia platform components. +The OS Image Building workflow orchestrates operating system image creation for functional roles in the Omnia platform. ## What It Does -The Image Building workflow provides: -- Container image build orchestration -- Multi-architecture image support (x86_64, aarch64) -- Docker and Podman integration -- Build context management and optimization -- Image security scanning and validation -- Registry push and distribution management +The OS Image Building workflow provides: +- OS image build orchestration for functional roles +- Multi-architecture OS image support (x86_64, aarch64) +- Package installation and configuration management ## Inputs/Outputs **Inputs:** -- Build specifications and Dockerfiles -- Package lists from local repositories -- Build parameters and configurations -- Security scanning requirements +- Catalog files defining functional roles and packages +- Generated input configuration files +- PXE mapping file for deployment configuration **Outputs:** -- Built container images -- Image metadata and manifests -- Security scan reports -- Registry push confirmations -- Build logs and artifacts +- Built OS images for functional roles +- OS image metadata and manifests +- Package installation logs and validation reports +- OS image deployment configurations ## Key Logic Locations **Primary Files:** -- `api/build_image/routes.py` - HTTP endpoints for build operations -- `orchestrator/build_image/use_cases/` - Build orchestration logic -- `core/build_image/entities.py` - Build domain entities -- `core/build_image/repositories.py` - Build data access -- `core/build_image/services.py` - Build management services +- `api/build_image/routes.py` - HTTP endpoints for OS build operations +- `orchestrator/build_image/use_cases/` - OS build orchestration logic +- `core/build_image/entities.py` - OS build domain entities +- `core/build_image/repositories.py` - OS build data access +- `core/build_image/services.py` - OS build management services **Main Components:** -- **BuildImageUseCase** - Orchestrates image build processes -- **BuildService** - Manages build execution and monitoring -- **MultiArchBuilder** - Handles multi-architecture builds -- **SecurityScanner** - Performs image security analysis +- **BuildOSImageUseCase** - Orchestrates OS image build processes for functional roles +- **OSService** - Manages OS build execution and monitoring +- **MultiArchOSBuilder** - Handles multi-architecture OS builds +- **PackageInstaller** - Manages package installation and configuration ## Workflow Flow -1. **Build Request**: Client submits image build request -2. **Build Context Preparation**: Dockerfiles and dependencies assembled -3. **Multi-Arch Setup**: Build configurations prepared for target architectures -4. **Build Execution**: Container images built using Docker/Podman -5. **Security Scanning**: Built images scanned for vulnerabilities -6. **Manifest Creation**: Multi-architecture manifests generated -7. **Registry Push**: Images pushed to container registries -8. **Validation**: Final image validation and testing +1. **Build Request**: Client submits image build request for functional roles +2. **OS Context Preparation**: Base functional role packages assembled +3. **Multi-Arch Setup**: OS build configurations prepared for target architectures +4. **Package Installation**: Functional role packages installed and configured +5. **OS Customization**: System settings and configurations applied +6. **Image Creation**: OS images built and optimized for deployment ## Architecture Support Supports multiple CPU architectures: - **x86_64** - Standard 64-bit Intel/AMD processors - **aarch64** - 64-bit ARM processors -- **Multi-arch manifests** - Single image supporting multiple architectures + ## Build Optimization Optimizations include: -- **Layer caching** - Reusing unchanged layers across builds +- **Package caching** - Reusing downloaded packages across builds - **Parallel builds** - Concurrent building for multiple architectures -- **Context optimization** - Minimizing build context size -- **Dependency caching** - Caching package downloads +- **Dependency resolution** - Efficient package dependency management ## Security Features Security capabilities include: -- **Vulnerability scanning** - Automated security analysis -- **Base image validation** - Verified base image sources -- **Signature verification** - Package integrity checks -- **Runtime security** - Secure container configurations +- **Package verification** - Automated package integrity validation +- **Base OS validation** - Verified base OS sources and configurations +- **Signature verification** - Package signature and checksum validation + ## Integration Points - Receives packages from local repository workflow - Integrates with validation workflow for quality checks -- Uses Vault for registry credentials -- Connects to container registries for distribution +- Uses Vault for secure credential management +- Connects with deployment systems for functional role provisioning ## Configuration Build configuration includes: -- Build parameters and environment variables -- Registry endpoints and credentials -- Security scanning policies -- Architecture-specific settings +- OS build parameters and environment variables +- Functional role specifications and requirements +- Package installation policies and configurations +- Architecture-specific OS settings ## Error Handling -- Detailed build error reporting +- Detailed OS build error reporting - Step-by-step build progress tracking - Rollback capabilities for failed builds - Automated retry for transient failures ## Monitoring -- Real-time build progress monitoring +- Real-time OS build progress monitoring - Resource usage tracking (CPU, memory, storage) - Build success/failure metrics -- Security scan result tracking - -## Registry Integration - -Supports multiple container registries: -- **Docker Hub** - Public container registry -- **Harbor** - Enterprise container registry -- **Artifactory** - JFrog container registry -- **Custom registries** - Organization-specific registries +- Package installation result tracking diff --git a/build_stream/doc/catalog.md b/build_stream/doc/catalog.md index 8b64f8e4f1..c14ed5cfac 100644 --- a/build_stream/doc/catalog.md +++ b/build_stream/doc/catalog.md @@ -32,9 +32,6 @@ The Catalog workflow provides: - `api/parse_catalog/routes.py` - Catalog parsing endpoints - `orchestrator/catalog/use_cases/parse_catalog.py` - Catalog parsing logic - `orchestrator/catalog/use_cases/generate_input_files.py` - Input file generation -- `core/catalog/entities.py` - Catalog domain entities -- `core/catalog/repositories.py` - Catalog data access -- `generate_catalog.py` - Standalone catalog generation script **Main Components:** - **ParseCatalogUseCase** - Handles catalog parsing and validation @@ -56,6 +53,8 @@ The Catalog workflow provides: ## Package Categorization Packages are categorized into: +- **Base OS Bundles**: Operating system packages (e.g., rhel) +- **Driver Bundles**: Hardware driver packages (e.g., nvidia_gpu_driver) - **Functional Bundles**: Core service packages (service_k8s, slurm_custom, additional_packages) - **Infrastructure Bundles**: CSI and infrastructure packages (csi_driver_powerscale) - **Miscellaneous**: Additional packages that don't fit other categories @@ -71,21 +70,5 @@ Packages are categorized into: Catalog processing is configured through: - Package mapping files -- Role templates -- Platform-specific configurations +- Adapter policy configurations - Validation rules and schemas - -## Error Handling - -- Comprehensive schema validation for catalogs -- Detailed error reporting for invalid packages -- Graceful handling of missing dependencies -- Rollback capabilities for failed processing - -## Standalone Script - -The `generate_catalog.py` script provides: -- Command-line catalog generation -- Batch processing capabilities -- Integration with external CI/CD pipelines -- Detailed logging and error reporting diff --git a/build_stream/doc/developer-guide.md b/build_stream/doc/developer-guide.md deleted file mode 100644 index 2baa9f9a7a..0000000000 --- a/build_stream/doc/developer-guide.md +++ /dev/null @@ -1,319 +0,0 @@ -# Build Stream Developer Guide - -This guide provides developers with comprehensive documentation for understanding and working with the Build Stream codebase. - -## Quick Start - -**For New Developers:** -1. Read the main [README](../README.md) for architecture overview -2. Review this guide for end-to-end workflow understanding -3. Explore specific workflow documentation for detailed implementation -4. Set up development environment using the instructions below - -**Key Concepts:** -- **Jobs**: Orchestrate multi-stage build processes -- **Stages**: Individual workflow steps (catalog, repo, build, validate) -- **Artifacts**: Generated files and outputs from workflows -- **Audit Trail**: Complete logging of all operations - -## Architecture Deep Dive - -### Layer Structure - -``` -build_stream/ -├── api/ # HTTP layer - FastAPI routes and middleware -├── core/ # Domain layer - Business logic and entities -├── orchestrator/ # Application layer - Use cases and workflow coordination -├── infra/ # Infrastructure layer - External integrations -├── common/ # Shared layer - Utilities and configuration -└── doc/ # Documentation layer - Workflow guides -``` - -### Dependency Flow - -``` -HTTP Request → API Routes → Use Cases → Core Services → Repositories - ↓ ↓ ↓ - Authentication Business Logic Data Persistence - ↓ ↓ ↓ - Authorization Validation External Systems -``` - -## Workflow Tracing Guide - -### End-to-End Job Flow - -To trace a complete job from start to finish: - -1. **Job Creation** (`POST /api/v1/jobs`) - - Entry point: `api/jobs/routes.py` - - Use case: `orchestrator/jobs/use_cases/create_job.py` - - Entity: `core/jobs/entities.py` - -2. **Stage Processing** (Async) - - Each stage runs independently - - Result polling handles completion - - Status updates tracked in database - -3. **Artifact Storage** - - Files stored in configured artifact store - - Metadata tracked in database - - Access controlled through permissions - -### Debugging Workflow Issues - -**Common Debugging Steps:** -1. Check job status via `/api/v1/jobs/{job_id}` -2. Review stage-specific error messages -3. Examine audit trail for detailed execution logs -4. Validate inputs using validation workflow -5. Check external system connectivity (Vault, repositories, registries) - -**Key Debug Files:** -- `api/logging_utils.py` - Centralized logging configuration -- `core/exceptions.py` - Domain-specific error definitions -- `infra/` - External integration points - -## Development Workflow - -### Setting Up Development Environment - -```bash -# Clone repository -git clone -cd build_stream - -# Install dependencies -pip install -r requirements.txt -pip install -r requirements-dev.txt - -# Set up environment variables -cp .env.test .env -# Edit .env with your configuration - -# Run database migrations (if using SQL backend) -alembic upgrade head - -# Start development server -uvicorn main:app --reload --host 0.0.0.0 --port 8000 -``` - -### Adding New Features - -**1. Add New API Endpoint:** -- Create route in appropriate `api/` subdirectory -- Add schema definitions for request/response -- Implement authentication/authorization as needed -- Add to `api/router.py` for registration - -**2. Add New Business Logic:** -- Create entities in `core/` domain -- Implement repositories for data access -- Create use cases in `orchestrator/` -- Wire up in dependency injection container - -**3. Add New Workflow Stage:** -- Define stage type in `core/jobs/value_objects.py` -- Implement stage-specific logic -- Add to job creation workflow -- Update documentation - -### Testing Strategy - -**Unit Tests:** -- Test individual components in isolation -- Mock external dependencies -- Focus on business logic validation - -**Integration Tests:** -- Test workflow end-to-end -- Use test database and artifact store -- Validate cross-component interactions - -**API Tests:** -- Test HTTP endpoints thoroughly -- Validate authentication and authorization -- Test error scenarios and edge cases - -## Common Patterns - -### Dependency Injection - -Build Stream uses `dependency-injector` for IoC: - -```python -# In container.py -from dependency_injector import containers, providers - -class Container(containers.DeclarativeContainer): - config = providers.Configuration() - - # Repository providers - job_repository = providers.Singleton( - SqlJobRepository, - session_factory=SessionLocal, - ) - - # Use case providers - create_job_use_case = providers.Factory( - CreateJobUseCase, - job_repository=job_repository, - # ... other dependencies - ) -``` - -### Error Handling - -Consistent error handling across all layers: - -```python -# Domain exceptions -class JobNotFoundError(Exception): - """Raised when a job is not found.""" - -# API error handling -@app.exception_handler(JobNotFoundError) -async def job_not_found_handler(request, exc): - return JSONResponse( - status_code=404, - content={"error": "Job not found", "details": str(exc)} - ) -``` - -### Logging - -Structured logging with correlation IDs: - -```python -from api.logging_utils import log_secure_info - -log_secure_info( - "Job created successfully", - extra={ - "job_id": str(job_id), - "client_id": str(client_id), - "stages": [stage.name for stage in stages] - } -) -``` - -## Security Considerations - -### Authentication Flow - -1. Client authenticates with JWT token -2. Token validated in middleware -3. User context attached to request -4. Authorization checked per endpoint - -### Secure Data Handling - -- Never log sensitive data (passwords, tokens) -- Use Vault for credential storage -- Implement proper access controls -- Audit all sensitive operations - -### Input Validation - -All inputs validated using: -- Pydantic models for type checking -- JSON schemas for structure validation -- Business rules in domain layer -- Security scanning for malicious content - -## Performance Optimization - -### Database Optimization - -- Use connection pooling -- Implement proper indexing -- Batch operations where possible -- Monitor query performance - -### Async Processing - -- Use async/await for I/O operations -- Implement proper error handling -- Monitor async task completion -- Handle resource cleanup - -### Caching Strategies - -- Cache frequently accessed data -- Use appropriate cache invalidation -- Monitor cache hit rates -- Consider distributed caching for scale - -## Troubleshooting Common Issues - -### Job Failures - -**Symptoms:** Jobs stuck in running state or failing unexpectedly - -**Debug Steps:** -1. Check job status and stage details -2. Review error messages in audit trail -3. Verify external system connectivity -4. Check resource availability - -### Performance Issues - -**Symptoms:** Slow API responses or job processing - -**Debug Steps:** -1. Monitor database query performance -2. Check resource utilization -3. Review async task processing -4. Analyze network connectivity - -### Authentication Problems - -**Symptoms:** 401/403 errors on API calls - -**Debug Steps:** -1. Verify JWT token validity -2. Check user permissions -3. Review token expiration -4. Validate token claims - -## External Dependencies - -### Required Services - -- **PostgreSQL** - Primary data storage -- **Vault** - Secure credential storage -- **Pulp** - Repository management (optional) -- **Container Registry** - Image storage - -### Optional Integrations - -- **External CI/CD** - Build pipeline integration -- **Monitoring Systems** - Metrics and alerting -- **Security Scanners** - Vulnerability assessment -- **Compliance Tools** - Regulatory reporting - -## Contributing Guidelines - -### Code Standards - -- Follow PEP 8 for Python code -- Use type hints for all public functions -- Write comprehensive docstrings -- Include unit tests for new features - -### Documentation Requirements - -- Update README for API changes -- Document new workflows in `doc/` -- Update architecture diagrams -- Include examples in docstrings - -### Review Process - -1. Create feature branch from main -2. Implement changes with tests -3. Update documentation -4. Submit pull request -5. Address review feedback -6. Merge after approval diff --git a/build_stream/doc/jobs.md b/build_stream/doc/jobs.md index 58c6a27efe..b4b24186b1 100644 --- a/build_stream/doc/jobs.md +++ b/build_stream/doc/jobs.md @@ -7,9 +7,7 @@ The Jobs workflow manages the complete lifecycle of build jobs in Build Stream, The Jobs workflow provides: - Job creation with idempotency guarantees - Stage-based execution with state management -- Real-time job monitoring and status tracking -- Audit trail for all job operations -- Result polling and notification handling +- Job monitoring and status tracking ## Inputs/Outputs @@ -50,31 +48,35 @@ The Jobs workflow provides: 7. **Result Collection**: Results polled and stored 8. **Audit Logging**: All operations logged for traceability +## Prerequisites + +To run jobs, the following infrastructure components are required: + +- **PostgreSQL Database**: Used for persistent storage of job metadata and status +- **S3-compatible Object Storage**: Utilized for storing build artifacts, such as catalog files and build images +- **Message Queue (e.g., RabbitMQ, Kafka)**: Enables asynchronous communication between job components and facilitates scalable processing +- **Container Runtime (e.g., Docker, containerd)**: Required for building and validating container images + +These components must be properly configured and accessible to the BuildStreaM service for successful job execution. + +## API Documentation + +- See Omnia ReadTheDocs for complete API documentation +- Local development docs: `http://localhost:${PORT}/docs` +- Local ReDoc: `http://localhost:${PORT}/redoc` + ## Stage Types -Jobs support multiple stage types: -- **catalog_roles** - Software catalog processing -- **local_repo** - Local repository creation -- **build_image** - Container image building -- **validate** - Input/output validation +Jobs support multiple stages: +- **parse-catalog** - Software catalog processing +- **generate-input-files** - Input file generation +- **create-local-repository** - Local repository creation +- **build-image-x86_64** - x86_64 OS image building +- **build-image-aarch64** - aarch64 OS image building +- **validate-image-on-test** - Image validation testing ## Error Handling - Invalid state transitions are rejected -- Failed stages can be retried based on configuration - Comprehensive error reporting with context - Audit trail captures all error events - -## Monitoring - -- Job status available via `/api/v1/jobs/{job_id}` endpoint -- List all jobs with filtering options -- Real-time status updates through result polling -- Detailed audit trail for compliance reporting - -## Integration Points - -- Integrates with all other workflows as stages -- Uses Vault for secure credential access -- Stores artifacts in configured artifact store -- Emits events for external system integration