From 080a7c4f3e1e6b1f293aac7e733ecd017e980f2e Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Wed, 21 Jan 2026 22:14:28 +0100 Subject: [PATCH 01/18] Initial commit: monorepo creation From 614b1ebbe1357a65ec0cf9de57daa5cf9b90cee5 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Wed, 21 Jan 2026 22:14:30 +0100 Subject: [PATCH 02/18] Fix Python package paths for monorepo structure Update pyproject.toml files to adjust raw-options root paths from '../..' to '../../..' to account for monorepo subdirectory. --- .gitignore | 27 ++ Makefile | 178 ++++++++ README.md | 172 ++++++++ import_pr.sh | 385 ++++++++++++++++++ .../__templates__/driver/pyproject.toml.tmpl | 2 +- .../packages/jumpstarter-all/pyproject.toml | 2 +- .../jumpstarter-cli-admin/pyproject.toml | 2 +- .../jumpstarter-cli-common/pyproject.toml | 2 +- .../jumpstarter-cli-driver/pyproject.toml | 2 +- .../packages/jumpstarter-cli/pyproject.toml | 2 +- .../jumpstarter-driver-ble/pyproject.toml | 2 +- .../jumpstarter-driver-can/pyproject.toml | 2 +- .../pyproject.toml | 2 +- .../pyproject.toml | 2 +- .../jumpstarter-driver-dutlink/pyproject.toml | 2 +- .../pyproject.toml | 2 +- .../pyproject.toml | 2 +- .../jumpstarter-driver-gpiod/pyproject.toml | 2 +- .../pyproject.toml | 2 +- .../jumpstarter-driver-http/pyproject.toml | 2 +- .../jumpstarter-driver-iscsi/pyproject.toml | 2 +- .../jumpstarter-driver-network/pyproject.toml | 2 +- .../jumpstarter-driver-opendal/pyproject.toml | 2 +- .../jumpstarter-driver-power/pyproject.toml | 2 +- .../pyproject.toml | 2 +- .../pyproject.toml | 2 +- .../jumpstarter-driver-qemu/pyproject.toml | 2 +- .../jumpstarter-driver-ridesx/pyproject.toml | 2 +- .../jumpstarter-driver-sdwire/pyproject.toml | 2 +- .../jumpstarter-driver-shell/pyproject.toml | 2 +- .../jumpstarter-driver-snmp/pyproject.toml | 2 +- .../pyproject.toml | 2 +- .../jumpstarter-driver-ssh/pyproject.toml | 2 +- .../jumpstarter-driver-tasmota/pyproject.toml | 2 +- .../jumpstarter-driver-tftp/pyproject.toml | 2 +- .../jumpstarter-driver-tmt/pyproject.toml | 2 +- .../jumpstarter-driver-uboot/pyproject.toml | 2 +- .../pyproject.toml | 2 +- .../jumpstarter-driver-vnc/pyproject.toml | 2 +- .../jumpstarter-driver-yepkit/pyproject.toml | 2 +- .../jumpstarter-imagehash/pyproject.toml | 2 +- .../jumpstarter-kubernetes/pyproject.toml | 2 +- .../jumpstarter-protocol/pyproject.toml | 2 +- .../jumpstarter-testing/pyproject.toml | 2 +- python/packages/jumpstarter/pyproject.toml | 2 +- typos.toml | 33 ++ 46 files changed, 836 insertions(+), 41 deletions(-) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README.md create mode 100755 import_pr.sh create mode 100644 typos.toml diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..a746eb15e --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +# E2E test artifacts and local configuration +.e2e-setup-complete +.e2e/ +.bats/ +ca.pem +ca-key.pem +ca.csr +server.pem +server-key.pem +server.csr + +# Python +.venv/ +__pycache__/ +*.pyc +*.pyo +*.egg-info/ +dist/ +build/ + +# Editor/IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..a53d5ed3d --- /dev/null +++ b/Makefile @@ -0,0 +1,178 @@ +# Jumpstarter Monorepo Makefile +# +# This Makefile provides common targets that delegate to subdirectory Makefiles. +# + +# Subdirectories containing projects +SUBDIRS := python protocol controller e2e + +# Default target +.PHONY: all +all: build + +# Help target - shows available commands +.PHONY: help +help: + @echo "Jumpstarter Monorepo" + @echo "" + @echo "Available targets:" + @echo " make all - Build all projects (default)" + @echo " make build - Build all projects" + @echo " make test - Run tests in all projects" + @echo " make clean - Clean build artifacts in all projects" + @echo " make lint - Run linters in all projects" + @echo " make fmt - Format code in all projects" + @echo "" + @echo "End-to-end testing:" + @echo " make e2e-setup - Setup e2e test environment (one-time)" + @echo " make e2e-run - Run e2e tests (requires e2e-setup first)" + @echo " make e2e - Same as e2e-run" + @echo " make e2e-full - Full setup + run (for CI or first time)" + @echo " make e2e-clean - Clean up e2e test environment (delete cluster, certs, etc.)" + @echo "" + @echo "Per-project targets:" + @echo " make build- - Build specific project" + @echo " make test- - Test specific project" + @echo " make clean- - Clean specific project" + @echo "" + @echo "Projects: $(SUBDIRS)" + +# Build all projects +.PHONY: build +build: + @for dir in $(SUBDIRS); do \ + if [ -f $$dir/Makefile ]; then \ + echo "Building $$dir..."; \ + $(MAKE) -C $$dir build || true; \ + fi \ + done + +# Test all projects +.PHONY: test +test: + @for dir in $(SUBDIRS); do \ + if [ -f $$dir/Makefile ]; then \ + echo "Testing $$dir..."; \ + $(MAKE) -C $$dir test ; \ + fi \ + done + +# Clean all projects +.PHONY: clean +clean: + @for dir in $(SUBDIRS); do \ + if [ -f $$dir/Makefile ]; then \ + echo "Cleaning $$dir..."; \ + $(MAKE) -C $$dir clean || true; \ + fi \ + done + +# Lint all projects +.PHONY: lint +lint: + @for dir in $(SUBDIRS); do \ + if [ -f $$dir/Makefile ]; then \ + echo "Linting $$dir..."; \ + $(MAKE) -C $$dir lint; \ + fi \ + done + +# Format all projects +.PHONY: fmt +fmt: + @for dir in $(SUBDIRS); do \ + if [ -f $$dir/Makefile ]; then \ + echo "Formatting $$dir..."; \ + $(MAKE) -C $$dir fmt || true; \ + fi \ + done + +# Per-project build targets +.PHONY: build-python build-protocol build-controller build-e2e +build-python: + @if [ -f python/Makefile ]; then $(MAKE) -C python build; fi + +build-protocol: + @if [ -f protocol/Makefile ]; then $(MAKE) -C protocol build; fi + +build-controller: + @if [ -f controller/Makefile ]; then $(MAKE) -C controller build; fi + +build-e2e: + @if [ -f e2e/Makefile ]; then $(MAKE) -C e2e build; fi + +# Per-project test targets +.PHONY: test-python test-protocol test-controller test-e2e +test-python: + @if [ -f python/Makefile ]; then $(MAKE) -C python test; fi + +test-protocol: + @if [ -f protocol/Makefile ]; then $(MAKE) -C protocol test; fi + +test-controller: + @if [ -f controller/Makefile ]; then $(MAKE) -C controller test; fi + +# Setup e2e testing environment (one-time) +.PHONY: e2e-setup +e2e-setup: + @echo "Setting up e2e test environment..." + @bash e2e/setup-e2e.sh + +# Run e2e tests +.PHONY: e2e-run +e2e-run: + @echo "Running e2e tests..." + @bash e2e/run-e2e.sh + +# Convenience alias for running e2e tests +.PHONY: e2e +e2e: e2e-run + +# Full e2e setup + run +.PHONY: e2e-full +e2e-full: + @bash e2e/run-e2e.sh --full + +# Clean up e2e test environment +.PHONY: e2e-clean +e2e-clean: + @echo "Cleaning up e2e test environment..." + @if command -v kind >/dev/null 2>&1; then \ + echo "Deleting jumpstarter kind cluster..."; \ + kind delete cluster --name jumpstarter 2>/dev/null || true; \ + fi + @echo "Removing certificates and setup files..." + @rm -f ca.pem ca-key.pem ca.csr server.pem server-key.pem server.csr + @rm -f .e2e-setup-complete + @echo "Removing local e2e configuration directory..." + @rm -rf .e2e + @echo "Removing virtual environment..." + @rm -rf .venv + @echo "Removing local bats libraries..." + @rm -rf .bats + @if [ -d /etc/jumpstarter/exporters ] && [ -w /etc/jumpstarter/exporters ]; then \ + echo "Removing exporter configs..."; \ + rm -rf /etc/jumpstarter/exporters/* 2>/dev/null || true; \ + fi + @echo "✓ E2E test environment cleaned" + @echo "" + @echo "Note: You may need to manually remove the dex entry from /etc/hosts:" + @echo " sudo sed -i.bak '/dex.dex.svc.cluster.local/d' /etc/hosts" + +# Backward compatibility alias +.PHONY: test-e2e +test-e2e: e2e-run + +# Per-project clean targets +.PHONY: clean-python clean-protocol clean-controller clean-e2e +clean-python: + @if [ -f python/Makefile ]; then $(MAKE) -C python clean; fi + +clean-protocol: + @if [ -f protocol/Makefile ]; then $(MAKE) -C protocol clean; fi + +clean-controller: + @if [ -f controller/Makefile ]; then $(MAKE) -C controller clean; fi + +clean-e2e: + @if [ -f e2e/Makefile ]; then $(MAKE) -C e2e clean; fi diff --git a/README.md b/README.md new file mode 100644 index 000000000..f50fba910 --- /dev/null +++ b/README.md @@ -0,0 +1,172 @@ +# ![bolt](python/assets/bolt.svg) Jumpstarter + +[![Matrix](https://img.shields.io/matrix/jumpstarter%3Amatrix.org?color=blue)](https://matrix.to/#/#jumpstarter:matrix.org) +[![Etherpad](https://img.shields.io/badge/Etherpad-Notes-blue?logo=etherpad)](https://etherpad.jumpstarter.dev/pad-lister) +[![Community Meeting](https://img.shields.io/badge/Weekly%20Meeting-Google%20Meet-blue?logo=google-meet)](https://meet.google.com/gzd-hhbd-hpu) +![GitHub Release](https://img.shields.io/github/v/release/jumpstarter-dev/jumpstarter) +![PyPI - Version](https://img.shields.io/pypi/v/jumpstarter) +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/jumpstarter-dev/jumpstarter) + +A free, open source tool for automated testing on real and virtual hardware with +CI/CD integration. Simplify device automation with consistent rules across local +and distributed environments. + +## Highlights + +- 🧪 **Unified Testing** - One tool for local, virtual, and remote hardware +- 🐍 **Python-Powered** - Leverage Python's testing ecosystem +- 🔌 **Hardware Abstraction** - Simplify complex hardware interfaces with drivers +- 🌐 **Collaborative** - Share test hardware globally +- ⚙️ **CI/CD Ready** - Works with cloud native developer environments and pipelines +- 💻 **Cross-Platform** - Supports Linux and macOS + +## Repository Structure + +This monorepo contains all Jumpstarter components: + +| Directory | Description | +|-----------|-------------| +| [`python/`](python/) | Python client, CLI, drivers, and testing framework | +| [`controller/`](controller/) | Kubernetes controller and operator (Jumpstarter Service) | +| [`protocol/`](protocol/) | gRPC protocol definitions (protobuf) | +| [`e2e/`](e2e/) | End-to-end testing infrastructure | + +## Quick Start + +### Install the CLI + +```shell +pip install --extra-index-url https://pkg.jumpstarter.dev/ jumpstarter-cli +``` + +Or install all Python components: + +```shell +pip install --extra-index-url https://pkg.jumpstarter.dev/ jumpstarter-all +``` + +### Deploy the Service + +To install the Jumpstarter Service in your Kubernetes cluster, see the +[Service Installation](https://jumpstarter.dev/main/getting-started/installation/index.html) +documentation. + +## Components + +### Python Client & Drivers (`python/`) + +The Python implementation provides: +- `jmp` CLI tool for interacting with hardware +- Client libraries for test automation +- Hardware drivers for various devices +- Testing framework integration + +See [`python/README.md`](python/README.md) for details. + +### Jumpstarter Service (`controller/`) + +The Kubernetes-native service that provides: +- Centralized hardware management +- Client and exporter routing +- Authentication and authorization +- Multi-tenant support + +**Prerequisites:** +- Kubernetes v1.11.3+ +- kubectl v1.11.3+ + +See [`controller/README.md`](controller/README.md) for deployment instructions. + +### Protocol (`protocol/`) + +The gRPC-based communication layer that enables: +- Unified interface for virtual and physical hardware +- Secure communication over HTTPS +- Tunneling support for Unix sockets, TCP, and UDP +- Flexible topology with direct or routed connections + +See [`protocol/README.md`](protocol/README.md) for details. + +### End-to-End Tests (`e2e/`) + +Comprehensive testing infrastructure for the entire Jumpstarter stack: +- `setup-e2e.sh` - One-time environment setup (auto-installs bats libraries on macOS) +- `run-e2e.sh` - Quick test runner for iterations +- `action.yml` - GitHub Actions composite action for CI/CD +- Full integration tests covering authentication, exporters, and clients + +Run e2e tests locally: +```shell +# First time setup +make e2e-setup + +# Run tests (repeat as needed) +make e2e # or: make e2e-run + +# Or full setup + run in one command +make e2e-full + +# Clean up e2e environment (delete cluster, certs, etc.) +make e2e-clean +``` + +## Development + +### Prerequisites + +- Python 3.11+ (for Python components) +- Go 1.22+ (for controller) +- Docker/Podman (for container builds) +- kubectl (for Kubernetes deployment) + +### Building + +```shell +# Build all components +make all + +# Build specific components +make python # Python packages +make controller # Controller binary +make protocol # Generate protocol code + +# Run tests +make test + +# Run end-to-end tests +make e2e-setup # First time only +make e2e # Run tests +make e2e-clean # Clean up +``` + +### Running Locally + +```shell +# Start a local development environment +make dev +``` + +## Documentation + +Jumpstarter's documentation is available at [jumpstarter.dev](https://jumpstarter.dev). + +- [Getting Started](https://jumpstarter.dev/main/getting-started/) +- [User Guide](https://jumpstarter.dev/main/introduction/) +- [API Reference](https://jumpstarter.dev/main/api/) +- [Contributing Guide](https://jumpstarter.dev/main/contributing.html) + +## Contributing + +Jumpstarter welcomes contributors of all levels of experience! See the +[contributing guide](https://jumpstarter.dev/main/contributing.html) to get started. + +### Community + +- [Matrix Chat](https://matrix.to/#/#jumpstarter:matrix.org) +- [Weekly Meeting](https://meet.google.com/gzd-hhbd-hpu) +- [Meeting Notes](https://etherpad.jumpstarter.dev/pad-lister) + +## License + +Jumpstarter is licensed under the Apache 2.0 License ([LICENSE](LICENSE) or +[https://www.apache.org/licenses/LICENSE-2.0](https://www.apache.org/licenses/LICENSE-2.0)). diff --git a/import_pr.sh b/import_pr.sh new file mode 100755 index 000000000..c945f37b1 --- /dev/null +++ b/import_pr.sh @@ -0,0 +1,385 @@ +#!/bin/bash +# +# import_pr.sh +# +# Imports a PR from an upstream Jumpstarter repository into the monorepo. +# This script fetches PR commits, generates patches, and applies them with +# the correct directory prefix for the monorepo structure. +# +# Usage: ./import_pr.sh +# +# Arguments: +# repo - One of: python, protocol, controller, e2e +# pr_number - The PR number from the upstream repository +# +# Example: +# ./import_pr.sh python 123 +# ./import_pr.sh controller 45 +# + +set -e + +# Configuration +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TEMP_DIR="${SCRIPT_DIR}/.import-pr-temp" +PATCH_DIR="${TEMP_DIR}/patches" + +# Repository mappings: repo_name -> "github_repo subdir" +declare -A REPO_MAP=( + ["python"]="jumpstarter-dev/jumpstarter python" + ["protocol"]="jumpstarter-dev/jumpstarter-protocol protocol" + ["controller"]="jumpstarter-dev/jumpstarter-controller controller" + ["e2e"]="jumpstarter-dev/jumpstarter-e2e e2e" +) + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +log_step() { + echo -e "${BLUE}[STEP]${NC} $1" +} + +# Cleanup function +cleanup() { + local exit_code=$? + if [ -d "${TEMP_DIR}" ]; then + log_info "Cleaning up temporary directory..." + rm -rf "${TEMP_DIR}" + fi + if [ $exit_code -ne 0 ]; then + log_warn "Script exited with errors. Any partial changes may need to be reverted." + fi +} + +trap cleanup EXIT + +# Print usage +usage() { + echo "Usage: $0 " + echo "" + echo "Import a PR from an upstream repository into the monorepo." + echo "" + echo "Arguments:" + echo " repo - One of: python, protocol, controller, e2e" + echo " pr_number - The PR number from the upstream repository" + echo "" + echo "Examples:" + echo " $0 python 123 # Import PR #123 from jumpstarter repo" + echo " $0 controller 45 # Import PR #45 from controller repo" + echo "" + echo "Repository mappings:" + echo " python -> jumpstarter-dev/jumpstarter" + echo " protocol -> jumpstarter-dev/jumpstarter-protocol" + echo " controller -> jumpstarter-dev/jumpstarter-controller" + echo " e2e -> jumpstarter-dev/jumpstarter-e2e" + exit 1 +} + +# Check dependencies +check_dependencies() { + log_step "Checking dependencies..." + + if ! command -v git &> /dev/null; then + log_error "git is not installed. Please install git first." + exit 1 + fi + + if ! command -v gh &> /dev/null; then + log_error "gh (GitHub CLI) is not installed." + echo "Install it from: https://cli.github.com/" + exit 1 + fi + + # Check if gh is authenticated + if ! gh auth status &> /dev/null; then + log_error "gh is not authenticated. Please run 'gh auth login' first." + exit 1 + fi + + log_info "All dependencies found." +} + +# Validate arguments +validate_args() { + if [ $# -lt 2 ]; then + log_error "Missing arguments." + usage + fi + + local repo="$1" + local pr_number="$2" + + # Validate repo name + if [ -z "${REPO_MAP[$repo]}" ]; then + log_error "Invalid repository name: ${repo}" + echo "Valid options are: python, protocol, controller, e2e" + exit 1 + fi + + # Validate PR number is numeric + if ! [[ "$pr_number" =~ ^[0-9]+$ ]]; then + log_error "PR number must be a positive integer: ${pr_number}" + exit 1 + fi +} + +# Fetch PR information +fetch_pr_info() { + local github_repo="$1" + local pr_number="$2" + + log_step "Fetching PR #${pr_number} info from ${github_repo}..." + + # Get PR details as JSON + local pr_json + pr_json=$(gh pr view "${pr_number}" --repo "${github_repo}" --json title,baseRefName,headRefName,commits,state 2>&1) || { + log_error "Failed to fetch PR #${pr_number} from ${github_repo}" + echo "Make sure the PR exists and you have access to the repository." + exit 1 + } + + # Extract fields + PR_TITLE=$(echo "$pr_json" | jq -r '.title') + PR_BASE_BRANCH=$(echo "$pr_json" | jq -r '.baseRefName') + PR_HEAD_BRANCH=$(echo "$pr_json" | jq -r '.headRefName') + PR_COMMIT_COUNT=$(echo "$pr_json" | jq '.commits | length') + PR_STATE=$(echo "$pr_json" | jq -r '.state') + + log_info "PR Title: ${PR_TITLE}" + log_info "Base Branch: ${PR_BASE_BRANCH}" + log_info "Head Branch: ${PR_HEAD_BRANCH}" + log_info "Commits: ${PR_COMMIT_COUNT}" + log_info "State: ${PR_STATE}" +} + +# Clone repository and checkout PR +clone_and_checkout_pr() { + local github_repo="$1" + local pr_number="$2" + + log_step "Cloning repository and checking out PR..." + + # Create temp directory + mkdir -p "${TEMP_DIR}" + mkdir -p "${PATCH_DIR}" + + local clone_dir="${TEMP_DIR}/repo" + + # Clone the repository + log_info "Cloning ${github_repo}..." + gh repo clone "${github_repo}" "${clone_dir}" -- --depth=1 --no-single-branch 2>/dev/null || { + # If shallow clone fails, try full clone + gh repo clone "${github_repo}" "${clone_dir}" + } + + cd "${clone_dir}" + + # Checkout the PR + log_info "Checking out PR #${pr_number}..." + gh pr checkout "${pr_number}" --repo "${github_repo}" + + # Fetch the base branch to ensure we have it + log_info "Fetching base branch (${PR_BASE_BRANCH})..." + git fetch origin "${PR_BASE_BRANCH}" + + CLONE_DIR="${clone_dir}" +} + +# Generate patches for PR commits +generate_patches() { + log_step "Generating patches..." + + cd "${CLONE_DIR}" + + # Find the merge base between the PR branch and the base branch + local merge_base + merge_base=$(git merge-base "origin/${PR_BASE_BRANCH}" HEAD) + + log_info "Merge base: ${merge_base}" + + # Count commits to be patched + local commit_count + commit_count=$(git rev-list --count "${merge_base}..HEAD") + log_info "Commits to import: ${commit_count}" + + if [ "$commit_count" -eq 0 ]; then + log_error "No commits found between merge base and HEAD." + exit 1 + fi + + # Generate patches + git format-patch -o "${PATCH_DIR}" "${merge_base}..HEAD" + + # Count generated patches + PATCH_COUNT=$(ls -1 "${PATCH_DIR}"/*.patch 2>/dev/null | wc -l | tr -d ' ') + log_info "Generated ${PATCH_COUNT} patch file(s)." +} + +# Apply patches to monorepo +apply_patches() { + local subdir="$1" + local repo_name="$2" + local pr_number="$3" + + log_step "Applying patches to monorepo..." + + cd "${SCRIPT_DIR}" + + # Create branch name + local branch_name="import/${repo_name}-pr-${pr_number}" + + # Check if we're in a git repository + if ! git rev-parse --git-dir &> /dev/null; then + log_error "Not in a git repository. Please run this script from the monorepo root." + exit 1 + fi + + # Check for uncommitted changes + if ! git diff --quiet || ! git diff --cached --quiet; then + log_error "You have uncommitted changes. Please commit or stash them first." + exit 1 + fi + + # Check if branch already exists + if git show-ref --verify --quiet "refs/heads/${branch_name}"; then + log_error "Branch '${branch_name}' already exists." + echo "Delete it with: git branch -D ${branch_name}" + exit 1 + fi + + # Create and checkout new branch + log_info "Creating branch: ${branch_name}" + git checkout -b "${branch_name}" + + # Apply patches with directory prefix + log_info "Applying patches with directory prefix: ${subdir}/" + + local patch_files=("${PATCH_DIR}"/*.patch) + local applied=0 + local failed=0 + + for patch in "${patch_files[@]}"; do + if [ -f "$patch" ]; then + local patch_name + patch_name=$(basename "$patch") + if git am --directory="${subdir}" "$patch" 2>/dev/null; then + log_info "Applied: ${patch_name}" + ((applied++)) + else + log_error "Failed to apply: ${patch_name}" + ((failed++)) + # Abort the am session + git am --abort 2>/dev/null || true + break + fi + fi + done + + if [ "$failed" -gt 0 ]; then + log_error "Failed to apply ${failed} patch(es)." + echo "" + echo "The patch may have conflicts. You can try to resolve them manually:" + echo " 1. git checkout main" + echo " 2. git branch -D ${branch_name}" + echo " 3. Manually apply the changes from the upstream PR" + exit 1 + fi + + APPLIED_COUNT=$applied +} + +# Print success message and next steps +print_success() { + local repo_name="$1" + local pr_number="$2" + local github_repo="$3" + local branch_name="import/${repo_name}-pr-${pr_number}" + + echo "" + echo -e "${GREEN}========================================${NC}" + echo -e "${GREEN} PR Import Successful!${NC}" + echo -e "${GREEN}========================================${NC}" + echo "" + echo "Summary:" + echo " - Source: ${github_repo}#${pr_number}" + echo " - Title: ${PR_TITLE}" + echo " - Branch: ${branch_name}" + echo " - Commits applied: ${APPLIED_COUNT}" + echo "" + echo "Next steps:" + echo " 1. Review the imported commits:" + echo " git log --oneline main..HEAD" + echo "" + echo " 2. Push the branch and create a PR on the monorepo:" + echo " git push -u origin ${branch_name}" + echo " gh pr create --title \"${PR_TITLE}\" --body \"Imported from ${github_repo}#${pr_number}\"" + echo "" + echo " 3. Or if you need to make changes first:" + echo " # Make your changes" + echo " git add -A && git commit --amend" + echo "" +} + +# Main execution +main() { + local repo_name="$1" + local pr_number="$2" + + echo "" + log_info "Starting PR import: ${repo_name} #${pr_number}" + echo "" + + # Validate arguments + validate_args "$@" + + # Check dependencies + check_dependencies + echo "" + + # Parse repo mapping + local repo_info="${REPO_MAP[$repo_name]}" + local github_repo subdir + read -r github_repo subdir <<< "${repo_info}" + + log_info "GitHub Repo: ${github_repo}" + log_info "Monorepo Subdir: ${subdir}/" + echo "" + + # Fetch PR info + fetch_pr_info "${github_repo}" "${pr_number}" + echo "" + + # Clone and checkout PR + clone_and_checkout_pr "${github_repo}" "${pr_number}" + echo "" + + # Generate patches + generate_patches + echo "" + + # Apply patches to monorepo + apply_patches "${subdir}" "${repo_name}" "${pr_number}" + echo "" + + # Print success message + print_success "${repo_name}" "${pr_number}" "${github_repo}" +} + +main "$@" diff --git a/python/__templates__/driver/pyproject.toml.tmpl b/python/__templates__/driver/pyproject.toml.tmpl index e7dbb11ee..71d1643c5 100644 --- a/python/__templates__/driver/pyproject.toml.tmpl +++ b/python/__templates__/driver/pyproject.toml.tmpl @@ -15,7 +15,7 @@ dependencies = [ [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../'} +raw-options = { 'root' = '../../../'} [tool.hatch.metadata.hooks.vcs.urls] Homepage = "https://jumpstarter.dev" diff --git a/python/packages/jumpstarter-all/pyproject.toml b/python/packages/jumpstarter-all/pyproject.toml index a071b28cc..036e4ee46 100644 --- a/python/packages/jumpstarter-all/pyproject.toml +++ b/python/packages/jumpstarter-all/pyproject.toml @@ -52,7 +52,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-cli-admin/pyproject.toml b/python/packages/jumpstarter-cli-admin/pyproject.toml index c8278da56..886d97675 100644 --- a/python/packages/jumpstarter-cli-admin/pyproject.toml +++ b/python/packages/jumpstarter-cli-admin/pyproject.toml @@ -32,7 +32,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-cli-common/pyproject.toml b/python/packages/jumpstarter-cli-common/pyproject.toml index a52e3d0c2..e892545fa 100644 --- a/python/packages/jumpstarter-cli-common/pyproject.toml +++ b/python/packages/jumpstarter-cli-common/pyproject.toml @@ -34,7 +34,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-cli-driver/pyproject.toml b/python/packages/jumpstarter-cli-driver/pyproject.toml index c1856a505..db35f025d 100644 --- a/python/packages/jumpstarter-cli-driver/pyproject.toml +++ b/python/packages/jumpstarter-cli-driver/pyproject.toml @@ -32,7 +32,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-cli/pyproject.toml b/python/packages/jumpstarter-cli/pyproject.toml index 8d8afa4a8..f05a03e02 100644 --- a/python/packages/jumpstarter-cli/pyproject.toml +++ b/python/packages/jumpstarter-cli/pyproject.toml @@ -37,7 +37,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-driver-ble/pyproject.toml b/python/packages/jumpstarter-driver-ble/pyproject.toml index 6f03d3e7f..13d50eebd 100644 --- a/python/packages/jumpstarter-driver-ble/pyproject.toml +++ b/python/packages/jumpstarter-driver-ble/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [tool.hatch.metadata.hooks.vcs.urls] Homepage = "https://jumpstarter.dev" diff --git a/python/packages/jumpstarter-driver-can/pyproject.toml b/python/packages/jumpstarter-driver-can/pyproject.toml index 97b4b463a..c00e80380 100644 --- a/python/packages/jumpstarter-driver-can/pyproject.toml +++ b/python/packages/jumpstarter-driver-can/pyproject.toml @@ -26,7 +26,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-driver-composite/pyproject.toml b/python/packages/jumpstarter-driver-composite/pyproject.toml index bd1e25e58..0fe9e9995 100644 --- a/python/packages/jumpstarter-driver-composite/pyproject.toml +++ b/python/packages/jumpstarter-driver-composite/pyproject.toml @@ -24,7 +24,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-driver-corellium/pyproject.toml b/python/packages/jumpstarter-driver-corellium/pyproject.toml index 05810dcc1..e0bd6f69e 100644 --- a/python/packages/jumpstarter-driver-corellium/pyproject.toml +++ b/python/packages/jumpstarter-driver-corellium/pyproject.toml @@ -27,7 +27,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-driver-dutlink/pyproject.toml b/python/packages/jumpstarter-driver-dutlink/pyproject.toml index acc77f4e8..7e81dc4e3 100644 --- a/python/packages/jumpstarter-driver-dutlink/pyproject.toml +++ b/python/packages/jumpstarter-driver-dutlink/pyproject.toml @@ -37,7 +37,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-driver-energenie/pyproject.toml b/python/packages/jumpstarter-driver-energenie/pyproject.toml index 0e5f22e1c..4aa67ab06 100644 --- a/python/packages/jumpstarter-driver-energenie/pyproject.toml +++ b/python/packages/jumpstarter-driver-energenie/pyproject.toml @@ -30,7 +30,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../'} +raw-options = { 'root' = '../../../'} [build-system] requires = ["hatchling", "hatch-vcs"] diff --git a/python/packages/jumpstarter-driver-flashers/pyproject.toml b/python/packages/jumpstarter-driver-flashers/pyproject.toml index 2127a6fae..26db295b4 100644 --- a/python/packages/jumpstarter-driver-flashers/pyproject.toml +++ b/python/packages/jumpstarter-driver-flashers/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [tool.hatch.metadata.hooks.vcs.urls] Homepage = "https://jumpstarter.dev" diff --git a/python/packages/jumpstarter-driver-gpiod/pyproject.toml b/python/packages/jumpstarter-driver-gpiod/pyproject.toml index 5fb933d80..ff5ca6edc 100644 --- a/python/packages/jumpstarter-driver-gpiod/pyproject.toml +++ b/python/packages/jumpstarter-driver-gpiod/pyproject.toml @@ -26,7 +26,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-driver-http-power/pyproject.toml b/python/packages/jumpstarter-driver-http-power/pyproject.toml index 5791f9a25..39b5f8cbc 100644 --- a/python/packages/jumpstarter-driver-http-power/pyproject.toml +++ b/python/packages/jumpstarter-driver-http-power/pyproject.toml @@ -17,7 +17,7 @@ dependencies = [ [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [tool.hatch.metadata.hooks.vcs.urls] Homepage = "https://jumpstarter.dev" diff --git a/python/packages/jumpstarter-driver-http/pyproject.toml b/python/packages/jumpstarter-driver-http/pyproject.toml index 55d6053f6..06cd3553f 100644 --- a/python/packages/jumpstarter-driver-http/pyproject.toml +++ b/python/packages/jumpstarter-driver-http/pyproject.toml @@ -16,7 +16,7 @@ dependencies = [ [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [tool.hatch.metadata.hooks.vcs.urls] Homepage = "https://jumpstarter.dev" diff --git a/python/packages/jumpstarter-driver-iscsi/pyproject.toml b/python/packages/jumpstarter-driver-iscsi/pyproject.toml index e8ee9cc7e..a591b6a04 100644 --- a/python/packages/jumpstarter-driver-iscsi/pyproject.toml +++ b/python/packages/jumpstarter-driver-iscsi/pyproject.toml @@ -17,7 +17,7 @@ dependencies = [ [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [tool.hatch.metadata.hooks.vcs.urls] Homepage = "https://jumpstarter.dev" diff --git a/python/packages/jumpstarter-driver-network/pyproject.toml b/python/packages/jumpstarter-driver-network/pyproject.toml index fcf695120..b02595109 100644 --- a/python/packages/jumpstarter-driver-network/pyproject.toml +++ b/python/packages/jumpstarter-driver-network/pyproject.toml @@ -46,7 +46,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-driver-opendal/pyproject.toml b/python/packages/jumpstarter-driver-opendal/pyproject.toml index 316e18105..7fe90c9fc 100644 --- a/python/packages/jumpstarter-driver-opendal/pyproject.toml +++ b/python/packages/jumpstarter-driver-opendal/pyproject.toml @@ -23,7 +23,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-driver-power/pyproject.toml b/python/packages/jumpstarter-driver-power/pyproject.toml index 82ae4d2b3..1106dded0 100644 --- a/python/packages/jumpstarter-driver-power/pyproject.toml +++ b/python/packages/jumpstarter-driver-power/pyproject.toml @@ -23,7 +23,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-driver-probe-rs/pyproject.toml b/python/packages/jumpstarter-driver-probe-rs/pyproject.toml index 16243c0bb..56fe83c9b 100644 --- a/python/packages/jumpstarter-driver-probe-rs/pyproject.toml +++ b/python/packages/jumpstarter-driver-probe-rs/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [tool.hatch.metadata.hooks.vcs.urls] Homepage = "https://jumpstarter.dev" diff --git a/python/packages/jumpstarter-driver-pyserial/pyproject.toml b/python/packages/jumpstarter-driver-pyserial/pyproject.toml index 24b8db141..20792b73d 100644 --- a/python/packages/jumpstarter-driver-pyserial/pyproject.toml +++ b/python/packages/jumpstarter-driver-pyserial/pyproject.toml @@ -31,7 +31,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-driver-qemu/pyproject.toml b/python/packages/jumpstarter-driver-qemu/pyproject.toml index c44f78ec8..3c77f05e6 100644 --- a/python/packages/jumpstarter-driver-qemu/pyproject.toml +++ b/python/packages/jumpstarter-driver-qemu/pyproject.toml @@ -30,7 +30,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [tool.uv.sources] jumpstarter-driver-opendal = { workspace = true } diff --git a/python/packages/jumpstarter-driver-ridesx/pyproject.toml b/python/packages/jumpstarter-driver-ridesx/pyproject.toml index 567d25c97..29bd45a22 100644 --- a/python/packages/jumpstarter-driver-ridesx/pyproject.toml +++ b/python/packages/jumpstarter-driver-ridesx/pyproject.toml @@ -16,7 +16,7 @@ dependencies = [ [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [tool.hatch.metadata.hooks.vcs.urls] Homepage = "https://jumpstarter.dev" diff --git a/python/packages/jumpstarter-driver-sdwire/pyproject.toml b/python/packages/jumpstarter-driver-sdwire/pyproject.toml index 9333a68ee..1b50ab474 100644 --- a/python/packages/jumpstarter-driver-sdwire/pyproject.toml +++ b/python/packages/jumpstarter-driver-sdwire/pyproject.toml @@ -25,7 +25,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-driver-shell/pyproject.toml b/python/packages/jumpstarter-driver-shell/pyproject.toml index ca2641272..a866cfc57 100644 --- a/python/packages/jumpstarter-driver-shell/pyproject.toml +++ b/python/packages/jumpstarter-driver-shell/pyproject.toml @@ -27,7 +27,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-driver-snmp/pyproject.toml b/python/packages/jumpstarter-driver-snmp/pyproject.toml index 9cb1a54b5..192edcb16 100644 --- a/python/packages/jumpstarter-driver-snmp/pyproject.toml +++ b/python/packages/jumpstarter-driver-snmp/pyproject.toml @@ -37,7 +37,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-driver-ssh-mitm/pyproject.toml b/python/packages/jumpstarter-driver-ssh-mitm/pyproject.toml index 9d24fd79b..4d4ca9f2b 100644 --- a/python/packages/jumpstarter-driver-ssh-mitm/pyproject.toml +++ b/python/packages/jumpstarter-driver-ssh-mitm/pyproject.toml @@ -20,7 +20,7 @@ ssh_mitm = "jumpstarter_driver_ssh_mitm" [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../'} +raw-options = { 'root' = '../../../'} [tool.hatch.metadata.hooks.vcs.urls] Homepage = "https://jumpstarter.dev" diff --git a/python/packages/jumpstarter-driver-ssh/pyproject.toml b/python/packages/jumpstarter-driver-ssh/pyproject.toml index ee557bc96..e195155a6 100644 --- a/python/packages/jumpstarter-driver-ssh/pyproject.toml +++ b/python/packages/jumpstarter-driver-ssh/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../'} +raw-options = { 'root' = '../../../'} [tool.hatch.metadata.hooks.vcs.urls] Homepage = "https://jumpstarter.dev" diff --git a/python/packages/jumpstarter-driver-tasmota/pyproject.toml b/python/packages/jumpstarter-driver-tasmota/pyproject.toml index ae32a1193..696d957ca 100644 --- a/python/packages/jumpstarter-driver-tasmota/pyproject.toml +++ b/python/packages/jumpstarter-driver-tasmota/pyproject.toml @@ -19,7 +19,7 @@ TasmotaPower = "jumpstarter_driver_tasmota.driver:TasmotaPower" [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [tool.hatch.metadata.hooks.vcs.urls] Homepage = "https://jumpstarter.dev" diff --git a/python/packages/jumpstarter-driver-tftp/pyproject.toml b/python/packages/jumpstarter-driver-tftp/pyproject.toml index bb3429a7b..ddf802dac 100644 --- a/python/packages/jumpstarter-driver-tftp/pyproject.toml +++ b/python/packages/jumpstarter-driver-tftp/pyproject.toml @@ -25,7 +25,7 @@ dev = [ [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [tool.hatch.metadata.hooks.vcs.urls] Homepage = "https://jumpstarter.dev" diff --git a/python/packages/jumpstarter-driver-tmt/pyproject.toml b/python/packages/jumpstarter-driver-tmt/pyproject.toml index 30278746a..6db0f89d1 100644 --- a/python/packages/jumpstarter-driver-tmt/pyproject.toml +++ b/python/packages/jumpstarter-driver-tmt/pyproject.toml @@ -17,7 +17,7 @@ dependencies = [ [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../'} +raw-options = { 'root' = '../../../'} [tool.hatch.metadata.hooks.vcs.urls] Homepage = "https://jumpstarter.dev" diff --git a/python/packages/jumpstarter-driver-uboot/pyproject.toml b/python/packages/jumpstarter-driver-uboot/pyproject.toml index 39f66c289..e08af3621 100644 --- a/python/packages/jumpstarter-driver-uboot/pyproject.toml +++ b/python/packages/jumpstarter-driver-uboot/pyproject.toml @@ -28,7 +28,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [tool.uv.sources] jumpstarter-driver-composite = { workspace = true } diff --git a/python/packages/jumpstarter-driver-ustreamer/pyproject.toml b/python/packages/jumpstarter-driver-ustreamer/pyproject.toml index ed239575f..aeae72844 100644 --- a/python/packages/jumpstarter-driver-ustreamer/pyproject.toml +++ b/python/packages/jumpstarter-driver-ustreamer/pyproject.toml @@ -21,7 +21,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-driver-vnc/pyproject.toml b/python/packages/jumpstarter-driver-vnc/pyproject.toml index 49ce3545f..93aabbcdc 100644 --- a/python/packages/jumpstarter-driver-vnc/pyproject.toml +++ b/python/packages/jumpstarter-driver-vnc/pyproject.toml @@ -21,7 +21,7 @@ vnc = "jumpstarter_driver_vnc.driver:Vnc" [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../'} +raw-options = { 'root' = '../../../'} [tool.hatch.metadata.hooks.vcs.urls] Homepage = "https://jumpstarter.dev" diff --git a/python/packages/jumpstarter-driver-yepkit/pyproject.toml b/python/packages/jumpstarter-driver-yepkit/pyproject.toml index 13607a5d9..a90ae2df7 100644 --- a/python/packages/jumpstarter-driver-yepkit/pyproject.toml +++ b/python/packages/jumpstarter-driver-yepkit/pyproject.toml @@ -19,7 +19,7 @@ Ykush = "jumpstarter_driver_yepkit.driver:Ykush" [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [tool.hatch.metadata.hooks.vcs.urls] Homepage = "https://jumpstarter.dev" diff --git a/python/packages/jumpstarter-imagehash/pyproject.toml b/python/packages/jumpstarter-imagehash/pyproject.toml index 92e13e671..3227bfd07 100644 --- a/python/packages/jumpstarter-imagehash/pyproject.toml +++ b/python/packages/jumpstarter-imagehash/pyproject.toml @@ -21,7 +21,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-kubernetes/pyproject.toml b/python/packages/jumpstarter-kubernetes/pyproject.toml index 39335b9c0..db3a20d62 100644 --- a/python/packages/jumpstarter-kubernetes/pyproject.toml +++ b/python/packages/jumpstarter-kubernetes/pyproject.toml @@ -30,7 +30,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-protocol/pyproject.toml b/python/packages/jumpstarter-protocol/pyproject.toml index 4eaa6fa9d..2cbf7e6a8 100644 --- a/python/packages/jumpstarter-protocol/pyproject.toml +++ b/python/packages/jumpstarter-protocol/pyproject.toml @@ -29,7 +29,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter-testing/pyproject.toml b/python/packages/jumpstarter-testing/pyproject.toml index 6b3f90ace..2b5f84e02 100644 --- a/python/packages/jumpstarter-testing/pyproject.toml +++ b/python/packages/jumpstarter-testing/pyproject.toml @@ -23,7 +23,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/python/packages/jumpstarter/pyproject.toml b/python/packages/jumpstarter/pyproject.toml index 61e54487d..a09ce03f2 100644 --- a/python/packages/jumpstarter/pyproject.toml +++ b/python/packages/jumpstarter/pyproject.toml @@ -44,7 +44,7 @@ source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}. [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../' } +raw-options = { 'root' = '../../../' } [build-system] requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] diff --git a/typos.toml b/typos.toml new file mode 100644 index 000000000..3cc13976e --- /dev/null +++ b/typos.toml @@ -0,0 +1,33 @@ +# Typos configuration for Jumpstarter monorepo +# https://github.com/crate-ci/typos + +[default] +extend-ignore-re = [ + # Ignore hash strings (like 321ba1) + "[a-f0-9]{6,}", +] + +[default.extend-words] +# ANDed/ORed are valid technical terms (combined with AND/OR operations) +ANDed = "ANDed" +Ded = "Ded" # suffix of ANDed in generated CRD docs +ORed = "ORed" + +# mosquitto is the name of an MQTT broker, not a typo of "mosquito" +mosquitto = "mosquitto" + +# ser is short for "serialize" in variable names like ser_json_timedelta +ser = "ser" + +[type.gomod] +# Exclude go.mod and go.sum from spell checking +extend-glob = ["go.mod", "go.sum"] +check-file = false + +[files] +extend-exclude = [ + # Generated files that shouldn't be spell-checked + "*.lock", + # Vendored dependencies + "vendor/", +] From b0623f71633e1656214fdeb812e3f32f632c5be0 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Wed, 21 Jan 2026 22:14:31 +0100 Subject: [PATCH 03/18] Fix multiversion docs script for monorepo Update multiversion.sh to use correct paths with python/ prefix in worktree structure. --- python/docs/multiversion.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/docs/multiversion.sh b/python/docs/multiversion.sh index 02291d373..10c2f11b5 100755 --- a/python/docs/multiversion.sh +++ b/python/docs/multiversion.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -declare -a BRANCHES=("main" "release-0.5" "release-0.6" "release-0.7") +declare -a BRANCHES=("main" "release-0.6" "release-0.7") # https://stackoverflow.com/a/246128 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) @@ -15,10 +15,10 @@ for BRANCH in "${BRANCHES[@]}"; do git worktree add --force "${WORKTREE}" "${BRANCH}" - uv run --project "${WORKTREE}" --isolated --all-packages --group docs \ - make -C "${WORKTREE}/docs" html SPHINXOPTS="-D version=${BRANCH}" + uv run --project "${WORKTREE}/python" --isolated --all-packages --group docs \ + make -C "${WORKTREE}/python/docs" html SPHINXOPTS="-D version=${BRANCH}" - cp -r "${WORKTREE}/docs/build/html" "${OUTPUT_DIR}/${BRANCH}" + cp -r "${WORKTREE}/python/docs/build/html" "${OUTPUT_DIR}/${BRANCH}" git worktree remove --force "${WORKTREE}" done From 3f40eecb6a53d9997e2d5ebd617e8a552315237d Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Wed, 21 Jan 2026 22:14:31 +0100 Subject: [PATCH 04/18] Fix controller and e2e configurations - Update Python container files for monorepo build paths - Copy Kind cluster config with dex nodeport pre-configured - Configure controller and e2e values with certificate placeholder - Patch deploy_with_helm.sh to support EXTRA_VALUES for Helm overlay pattern --- controller/hack/deploy_with_helm.sh | 2 +- controller/hack/kind_cluster.yaml | 17 +++++++++-------- python/.devfile/Containerfile.client | 4 ++-- python/Dockerfile | 4 ++-- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/controller/hack/deploy_with_helm.sh b/controller/hack/deploy_with_helm.sh index abdada4b2..3a54159a0 100755 --- a/controller/hack/deploy_with_helm.sh +++ b/controller/hack/deploy_with_helm.sh @@ -51,7 +51,7 @@ helm ${METHOD} --namespace jumpstarter-lab \ --create-namespace \ ${HELM_SETS} \ --set global.timestamp=$(date +%s) \ - --values ./deploy/helm/jumpstarter/values.kind.yaml jumpstarter \ + --values ./deploy/helm/jumpstarter/values.kind.yaml ${EXTRA_VALUES} jumpstarter \ ./deploy/helm/jumpstarter/ kubectl config set-context --current --namespace=jumpstarter-lab diff --git a/controller/hack/kind_cluster.yaml b/controller/hack/kind_cluster.yaml index 6478b336e..9cdf74eef 100644 --- a/controller/hack/kind_cluster.yaml +++ b/controller/hack/kind_cluster.yaml @@ -17,24 +17,25 @@ nodes: - containerPort: 80 # ingress controller hostPort: 5080 protocol: TCP - - containerPort: 443 - hostPort: 5443 - protocol: TCP - containerPort: 30010 # grpc nodeport hostPort: 8082 protocol: TCP - - containerPort: 30011 # grpc router nodeport (replica 0) + - containerPort: 30011 # grpc router nodeport hostPort: 8083 protocol: TCP - - containerPort: 30012 # grpc router nodeport (replica 1) + - containerPort: 30012 # grpc router nodeport hostPort: 8084 protocol: TCP - - containerPort: 30013 # grpc router nodeport (replica 2) + - containerPort: 30013 # grpc router nodeport hostPort: 8085 protocol: TCP - + - containerPort: 32000 # dex nodeport + hostPort: 5556 + protocol: TCP + - containerPort: 443 + hostPort: 5443 + protocol: TCP # if we needed to mount a hostPath volume into the kind cluster, we can do it like this # extraMounts: # - hostPath: ./bin/e2e-certs # containerPath: /tmp/e2e-certs - diff --git a/python/.devfile/Containerfile.client b/python/.devfile/Containerfile.client index 6dbba09e2..3eabc5df4 100644 --- a/python/.devfile/Containerfile.client +++ b/python/.devfile/Containerfile.client @@ -6,7 +6,7 @@ RUN dnf install -y make git && \ rm -rf /var/cache/dnf COPY --from=uv /uv /uvx /bin/ ADD . /src -RUN make -C /src build +RUN make -C /src/python build FROM quay.io/devfile/base-developer-image:ubi9-latest @@ -28,7 +28,7 @@ RUN dnf -y install make git python3.12 python3.12 libusbx python3-pyusb python3. USER 10001 -RUN --mount=from=builder,source=/src/dist,target=/dist python3.12 -m pip install /dist/*.whl +RUN --mount=from=builder,source=/src/python/dist,target=/dist python3.12 -m pip install /dist/*.whl RUN python3.12 -m pip install pytest diff --git a/python/Dockerfile b/python/Dockerfile index 5b2ffb481..041f8bc1f 100644 --- a/python/Dockerfile +++ b/python/Dockerfile @@ -14,10 +14,10 @@ COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ FROM builder AS wheels ADD . /src -RUN make -C /src build +RUN make -C /src/python build FROM product -RUN --mount=from=wheels,source=/src/dist,target=/dist \ +RUN --mount=from=wheels,source=/src/python/dist,target=/dist \ uv venv /jumpstarter && \ VIRTUAL_ENV=/jumpstarter uv pip install /dist/*.whl ENV PATH="/jumpstarter/bin:$PATH" From d16861e84ae72b3907f9af776bd4b66185c5e956 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Wed, 21 Jan 2026 22:14:31 +0100 Subject: [PATCH 05/18] Configure GitHub Actions and e2e test scripts - Add unified GitHub Actions workflows with path filters - Configure dependabot for all package ecosystems - Remove old .github directories from subdirectories - Install e2e test scripts (setup-e2e.sh, run-e2e.sh, tests.bats) --- .github/dependabot.yml | 34 ++ .../workflows/backport.yaml | 0 .../workflows/build-images.yaml | 75 +++- .../workflows/build-oci-bundle.yaml | 7 +- .../workflows/controller-bundle.yaml | 14 +- .../workflows/controller-kind.yaml | 9 +- .../workflows/controller-tests.yaml | 13 +- .../workflows/documentation.yaml | 11 +- .github/workflows/e2e.yaml | 44 +++ .github/workflows/lint.yaml | 109 ++++++ .../workflows/pr-analytics.yaml | 14 +- .../workflows/python-tests.yaml | 15 +- .../workflows/trigger-packages.yaml | 0 controller/.github/workflows/backport.yaml | 42 -- controller/.github/workflows/e2e.yaml | 28 -- controller/.github/workflows/lint.yaml | 39 -- controller/typos.toml | 6 - e2e/.github/workflows/selftest.yml | 19 - e2e/action.yml | 99 ----- e2e/run-e2e.sh | 170 ++++++++ e2e/setup-e2e.sh | 364 ++++++++++++++++++ e2e/tests.bats | 61 ++- protocol/.github/workflows/lint.yaml | 20 - python/.github/dependabot.yml | 12 - python/.github/workflows/backport.yml | 41 -- python/.github/workflows/build.yaml | 111 ------ python/.github/workflows/e2e.yaml | 24 -- python/.github/workflows/ruff.yaml | 23 -- python/.github/workflows/typos.yaml | 21 - 29 files changed, 896 insertions(+), 529 deletions(-) create mode 100644 .github/dependabot.yml rename e2e/.github/workflows/backport.yml => .github/workflows/backport.yaml (100%) rename controller/.github/workflows/build.yaml => .github/workflows/build-images.yaml (56%) rename python/.github/workflows/build_oci_bundle.yaml => .github/workflows/build-oci-bundle.yaml (73%) rename controller/.github/workflows/check-bundle.yaml => .github/workflows/controller-bundle.yaml (91%) rename controller/.github/workflows/pr-kind.yaml => .github/workflows/controller-kind.yaml (73%) rename controller/.github/workflows/test.yaml => .github/workflows/controller-tests.yaml (59%) rename {python/.github => .github}/workflows/documentation.yaml (92%) create mode 100644 .github/workflows/e2e.yaml create mode 100644 .github/workflows/lint.yaml rename python/.github/workflows/pr_analytics.yaml => .github/workflows/pr-analytics.yaml (59%) rename python/.github/workflows/pytest.yaml => .github/workflows/python-tests.yaml (87%) rename python/.github/workflows/trigger-packages-index.yaml => .github/workflows/trigger-packages.yaml (100%) delete mode 100644 controller/.github/workflows/backport.yaml delete mode 100644 controller/.github/workflows/e2e.yaml delete mode 100644 controller/.github/workflows/lint.yaml delete mode 100644 controller/typos.toml delete mode 100644 e2e/.github/workflows/selftest.yml delete mode 100644 e2e/action.yml create mode 100755 e2e/run-e2e.sh create mode 100755 e2e/setup-e2e.sh delete mode 100644 protocol/.github/workflows/lint.yaml delete mode 100644 python/.github/dependabot.yml delete mode 100644 python/.github/workflows/backport.yml delete mode 100644 python/.github/workflows/build.yaml delete mode 100644 python/.github/workflows/e2e.yaml delete mode 100644 python/.github/workflows/ruff.yaml delete mode 100644 python/.github/workflows/typos.yaml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..e9ff035b8 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,34 @@ +# Dependabot configuration for monorepo +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + # Go modules for controller + - package-ecosystem: "gomod" + directory: "/controller" + schedule: + interval: weekly + + # Go modules for operator + - package-ecosystem: "gomod" + directory: "/controller/deploy/operator" + schedule: + interval: weekly + + # Python dependencies + - package-ecosystem: "pip" + directory: "/python" + schedule: + interval: weekly + + # Devcontainers + - package-ecosystem: "devcontainers" + directory: "/" + schedule: + interval: weekly + + # GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: weekly diff --git a/e2e/.github/workflows/backport.yml b/.github/workflows/backport.yaml similarity index 100% rename from e2e/.github/workflows/backport.yml rename to .github/workflows/backport.yaml diff --git a/controller/.github/workflows/build.yaml b/.github/workflows/build-images.yaml similarity index 56% rename from controller/.github/workflows/build.yaml rename to .github/workflows/build-images.yaml index bf3509f84..7dd9b3614 100644 --- a/controller/.github/workflows/build.yaml +++ b/.github/workflows/build-images.yaml @@ -1,4 +1,5 @@ -name: Build and push container image +name: Build and push container images + on: workflow_dispatch: push: @@ -7,8 +8,10 @@ on: branches: - main - 'release-*' + merge_group: env: + PUSH: ${{ github.repository_owner == 'jumpstarter-dev' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/release-')) }} REGISTRY: quay.io QUAY_ORG: quay.io/jumpstarter-dev @@ -17,18 +20,35 @@ jobs: runs-on: ubuntu-latest permissions: contents: read + packages: write + attestations: write + id-token: write strategy: matrix: include: + # Controller images - image_name: jumpstarter-dev/jumpstarter-controller - dockerfile: Dockerfile - context: . + dockerfile: controller/Dockerfile + context: controller - image_name: jumpstarter-dev/jumpstarter-operator - dockerfile: Dockerfile.operator - context: . + dockerfile: controller/Dockerfile.operator + context: controller - image_name: jumpstarter-dev/jumpstarter-operator-bundle - dockerfile: deploy/operator/bundle.Dockerfile - context: deploy/operator + dockerfile: controller/deploy/operator/bundle.Dockerfile + context: controller/deploy/operator + # Python images (use repo root context for .git access needed by hatch-vcs) + - image_name: jumpstarter-dev/jumpstarter + dockerfile: python/Dockerfile + context: . + - image_name: jumpstarter-dev/jumpstarter-utils + dockerfile: python/Dockerfile.utils + context: python + - image_name: jumpstarter-dev/jumpstarter-dev + dockerfile: python/.devfile/Containerfile + context: python + - image_name: jumpstarter-dev/jumpstarter-devspace + dockerfile: python/.devfile/Containerfile.client + context: . steps: - name: Checkout repository uses: actions/checkout@v4 @@ -41,6 +61,17 @@ jobs: VERSION=${VERSION#v} # remove the leading v prefix for version echo "VERSION=${VERSION}" >> $GITHUB_ENV echo "VERSION=${VERSION}" + + # Convert to PEP 440 compliant version for Python packages + # Format: 0.7.0-1051-g54cd2f08 -> 0.7.0.dev1051+g54cd2f08 + if [[ "$VERSION" =~ ^([0-9]+\.[0-9]+\.[0-9]+)-([0-9]+)-g([a-f0-9]+)$ ]]; then + PEP440_VERSION="${BASH_REMATCH[1]}.dev${BASH_REMATCH[2]}+g${BASH_REMATCH[3]}" + else + # If it's already a clean version (e.g., 0.7.0), use as-is + PEP440_VERSION="$VERSION" + fi + echo "PEP440_VERSION=${PEP440_VERSION}" >> $GITHUB_ENV + echo "PEP440_VERSION=${PEP440_VERSION}" - name: Set build args id: build-args @@ -53,6 +84,7 @@ jobs: echo "BUILD_DATE=${BUILD_DATE}" - name: Set image tags + if: ${{ env.PUSH == 'true' }} id: set-tags run: | TAGS="${{ env.REGISTRY }}/${{ matrix.image_name }}:${{ env.VERSION }}" @@ -61,7 +93,7 @@ jobs: TAGS="$TAGS,${{ env.REGISTRY }}/${{ matrix.image_name }}:latest" fi - if [[ "${{ github.ref }}" == "refs/heads/release-*" ]]; then + if [[ "${{ github.ref }}" == refs/heads/release-* ]]; then RELEASE_BRANCH_NAME=$(basename "${{ github.ref }}") TAGS="$TAGS,${{ env.REGISTRY }}/${{ matrix.image_name }}:${RELEASE_BRANCH_NAME}" fi @@ -76,29 +108,46 @@ jobs: - name: Log in to the Container registry uses: docker/login-action@v3 + if: ${{ env.PUSH == 'true' }} with: registry: ${{ env.REGISTRY }} username: jumpstarter-dev+jumpstarter_ci password: ${{ secrets.QUAY_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ matrix.image_name }} + - name: Build and push Docker image id: push uses: docker/build-push-action@v6 with: context: ${{ matrix.context }} file: ${{ matrix.dockerfile }} - push: true + push: ${{ env.PUSH }} tags: ${{ steps.set-tags.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} platforms: linux/amd64,linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max build-args: | - GIT_VERSION=${{ env.VERSION }} + GIT_VERSION=${{ env.PEP440_VERSION }} GIT_COMMIT=${{ steps.build-args.outputs.git_commit }} BUILD_DATE=${{ steps.build-args.outputs.build_date }} - publish-helm-charts-containers: + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + if: ${{ env.PUSH == 'true' }} + with: + subject-name: ${{ env.REGISTRY }}/${{ matrix.image_name }} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: ${{ env.PUSH }} + + publish-helm-charts: needs: build-and-push-image + if: ${{ github.repository_owner == 'jumpstarter-dev' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/release-')) }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -116,8 +165,8 @@ jobs: run: | echo packaging ${VERSION} # patch the sub-chart app-version, because helm package won't do it - sed -i "s/^appVersion:.*/appVersion: $VERSION/" deploy/helm/jumpstarter/charts/jumpstarter-controller/Chart.yaml - helm package ./deploy/helm/jumpstarter --version "${VERSION}" --app-version "${VERSION}" + sed -i "s/^appVersion:.*/appVersion: $VERSION/" controller/deploy/helm/jumpstarter/charts/jumpstarter-controller/Chart.yaml + helm package ./controller/deploy/helm/jumpstarter --version "${VERSION}" --app-version "${VERSION}" - name: Login helm env: diff --git a/python/.github/workflows/build_oci_bundle.yaml b/.github/workflows/build-oci-bundle.yaml similarity index 73% rename from python/.github/workflows/build_oci_bundle.yaml rename to .github/workflows/build-oci-bundle.yaml index d06d14b78..f130f58ad 100644 --- a/python/.github/workflows/build_oci_bundle.yaml +++ b/.github/workflows/build-oci-bundle.yaml @@ -1,4 +1,5 @@ name: Build and push buildroot-based flasher OCI bundle + on: workflow_dispatch: @@ -14,17 +15,17 @@ jobs: - name: Run build_fits.sh run: | - cd packages/jumpstarter-driver-flashers/oci_bundles/aarch64-itb + cd python/packages/jumpstarter-driver-flashers/oci_bundles/aarch64-itb ./build_fits.sh - name: Upload FIT artifacts uses: actions/upload-artifact@v4 with: name: FIT-images - path: packages/jumpstarter-driver-flashers/oci_bundles/aarch64-itb/data/*.itb + path: python/packages/jumpstarter-driver-flashers/oci_bundles/aarch64-itb/data/*.itb - name: Run build_bundle.sh for aarch64-itb run: | - cd packages/jumpstarter-driver-flashers/oci_bundles && dnf install -y oras + cd python/packages/jumpstarter-driver-flashers/oci_bundles && dnf install -y oras oras login quay.io -u jumpstarter-dev+jumpstarter_ci --password-stdin <<< "${{ secrets.QUAY_TOKEN }}" ./build_bundle.sh quay.io/jumpstarter-dev/jumpstarter-flasher-aarch64-itb:latest aarch64-itb diff --git a/controller/.github/workflows/check-bundle.yaml b/.github/workflows/controller-bundle.yaml similarity index 91% rename from controller/.github/workflows/check-bundle.yaml rename to .github/workflows/controller-bundle.yaml index b988051cc..ab9dc5eb6 100644 --- a/controller/.github/workflows/check-bundle.yaml +++ b/.github/workflows/controller-bundle.yaml @@ -1,9 +1,12 @@ name: Check Bundle + on: pull_request: branches: - main - 'release-*' + paths: + - 'controller/**' jobs: check-bundle: @@ -17,13 +20,13 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.24 + go-version: '1.24' - name: Cache bin directory (deploy/operator) uses: actions/cache@v4 with: - path: deploy/operator/bin/ - key: ${{ runner.os }}-operator-bin-${{ hashFiles('deploy/operator/go.mod') }} + path: controller/deploy/operator/bin/ + key: ${{ runner.os }}-operator-bin-${{ hashFiles('controller/deploy/operator/go.mod') }} restore-keys: | ${{ runner.os }}-operator-bin- @@ -47,7 +50,7 @@ jobs: echo "TAG=${TAG}" - name: Run make bundle - working-directory: deploy/operator + working-directory: controller/deploy/operator run: | make bundle IMG="quay.io/jumpstarter-dev/jumpstarter-operator:${TAG}" @@ -78,7 +81,7 @@ jobs: git checkout -- . || true - name: Run make build-installer - working-directory: deploy/operator + working-directory: controller/deploy/operator run: | make build-installer @@ -92,4 +95,3 @@ jobs: else echo "No uncommitted changes detected. Installer files are up to date." fi - diff --git a/controller/.github/workflows/pr-kind.yaml b/.github/workflows/controller-kind.yaml similarity index 73% rename from controller/.github/workflows/pr-kind.yaml rename to .github/workflows/controller-kind.yaml index 722696f2b..fa152b630 100644 --- a/controller/.github/workflows/pr-kind.yaml +++ b/.github/workflows/controller-kind.yaml @@ -1,10 +1,13 @@ name: Kind based CI + on: workflow_dispatch: pull_request: branches: - main - 'release-*' + paths: + - 'controller/**' jobs: deploy-kind: @@ -16,6 +19,7 @@ jobs: fetch-depth: 0 - name: Run make deploy + working-directory: controller run: make deploy e2e-test-operator: @@ -26,5 +30,6 @@ jobs: with: fetch-depth: 0 - - name: Run make deploy - run: make test-operator-e2e \ No newline at end of file + - name: Run operator e2e test + working-directory: controller + run: make test-operator-e2e diff --git a/controller/.github/workflows/test.yaml b/.github/workflows/controller-tests.yaml similarity index 59% rename from controller/.github/workflows/test.yaml rename to .github/workflows/controller-tests.yaml index 1e0ed26e0..ca6a11a19 100644 --- a/controller/.github/workflows/test.yaml +++ b/.github/workflows/controller-tests.yaml @@ -1,10 +1,14 @@ -name: Unit/Functional tests +name: Controller Unit/Functional tests + on: workflow_dispatch: pull_request: branches: - main - 'release-*' + paths: + - 'controller/**' + - 'protocol/**' jobs: tests: @@ -16,15 +20,16 @@ jobs: fetch-depth: 0 - name: Run controller tests + working-directory: controller run: make test - name: Cache operator bin directory uses: actions/cache@v4 with: - path: deploy/operator/bin/ - key: ${{ runner.os }}-operator-bin-${{ hashFiles('deploy/operator/go.mod') }} + path: controller/deploy/operator/bin/ + key: ${{ runner.os }}-operator-bin-${{ hashFiles('controller/deploy/operator/go.mod') }} restore-keys: | ${{ runner.os }}-operator-bin- - name: Run operator tests - run: make -C deploy/operator test + run: make -C controller/deploy/operator test diff --git a/python/.github/workflows/documentation.yaml b/.github/workflows/documentation.yaml similarity index 92% rename from python/.github/workflows/documentation.yaml rename to .github/workflows/documentation.yaml index 75e9758d2..bb7402d0c 100644 --- a/python/.github/workflows/documentation.yaml +++ b/.github/workflows/documentation.yaml @@ -1,10 +1,16 @@ -name: documentation +name: Documentation on: # Runs on pushes targeting the default branch push: branches: ["main"] + paths: + - 'python/docs/**' + - 'python/packages/**' pull_request: + paths: + - 'python/docs/**' + - 'python/packages/**' merge_group: # Allows you to run this workflow manually from the Actions tab @@ -25,6 +31,7 @@ concurrency: defaults: run: shell: bash + working-directory: python jobs: # Build job @@ -66,7 +73,7 @@ jobs: - name: Upload artifact uses: actions/upload-pages-artifact@v3 with: - path: ./docs/build + path: ./python/docs/build check-warnings: runs-on: ubuntu-latest diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 000000000..d13d6472e --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,44 @@ +name: End-to-end tests + +on: + workflow_dispatch: + pull_request: + branches: + - main + - 'release-*' + merge_group: + +permissions: + contents: read + +jobs: + e2e-tests: + if: github.repository_owner == 'jumpstarter-dev' + strategy: + matrix: + os: + - ubuntu-24.04 + - ubuntu-24.04-arm + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v2 + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: '1.22' + + - name: Setup e2e test environment + run: make e2e-setup + env: + CI: true + + - name: Run e2e tests + run: make e2e-run + env: + CI: true diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 000000000..a08f98cb9 --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,109 @@ +name: Linters + +on: + workflow_dispatch: + push: + branches: + - main + - 'release-*' + pull_request: + branches: + - main + - 'release-*' + merge_group: + +permissions: + contents: read + pull-requests: read + +jobs: + # Detect which paths changed to conditionally run linters + changes: + runs-on: ubuntu-latest + outputs: + controller: ${{ steps.filter.outputs.controller }} + helm: ${{ steps.filter.outputs.helm }} + protocol: ${{ steps.filter.outputs.protocol }} + python: ${{ steps.filter.outputs.python }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + controller: + - 'controller/**' + helm: + - 'controller/deploy/helm/**' + protocol: + - 'protocol/**' + python: + - 'python/**' + + lint-go: + needs: changes + if: needs.changes.outputs.controller == 'true' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.24' + + - name: Run go linter + working-directory: controller + run: make lint + + lint-helm: + needs: changes + if: needs.changes.outputs.helm == 'true' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Run helm linter + working-directory: controller + run: make lint-helm + + lint-protobuf: + needs: changes + if: needs.changes.outputs.protocol == 'true' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Run protobuf linter + working-directory: protocol + run: make lint + + lint-python: + needs: changes + if: needs.changes.outputs.python == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run ruff + uses: astral-sh/ruff-action@84f83ecf9e1e15d26b7984c7ec9cf73d39ffc946 # v3.3.1 + with: + src: './python' + version-file: python/pyproject.toml + + typos: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run typos + uses: crate-ci/typos@0f0ccba9ed1df83948f0c15026e4f5ccfce46109 # v1.32.0 + with: + config: ./typos.toml diff --git a/python/.github/workflows/pr_analytics.yaml b/.github/workflows/pr-analytics.yaml similarity index 59% rename from python/.github/workflows/pr_analytics.yaml rename to .github/workflows/pr-analytics.yaml index 60019b597..401337a1b 100644 --- a/python/.github/workflows/pr_analytics.yaml +++ b/.github/workflows/pr-analytics.yaml @@ -1,4 +1,5 @@ -name: "PR Analytics" +name: PR Analytics + on: workflow_dispatch: inputs: @@ -6,22 +7,23 @@ on: description: "Report date start(d/MM/yyyy)" report_date_end: description: "Report date end(d/MM/yyyy)" + jobs: create-report: - name: "Create report" + name: Create report runs-on: ubuntu-latest permissions: contents: read pull-requests: read issues: write steps: - - name: "Run script for analytics" + - name: Run script for analytics uses: AlexSim93/pull-request-analytics-action@cc57ceb92148c5d5879ca578a2b59f99c3cbe231 # v4.6.1 with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # In the case of a personal access token, it needs to be added to the repository's secrets and used in this field. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REPO_FOR_ISSUE: jumpstarter - GITHUB_OWNER_FOR_ISSUE: jumpstarter-dev - GITHUB_OWNERS_REPOS: jumpstarter-dev/jumpstarter #TODO: check with more repos later, needs PAT: ,jumpstarter-dev/jumpstarter-controller + GITHUB_OWNER_FOR_ISSUE: jumpstarter-dev + GITHUB_OWNERS_REPOS: jumpstarter-dev/jumpstarter USE_CHARTS: true TIMEZONE: "Etc/UTC" REPORT_DATE_START: ${{ inputs.report_date_start }} diff --git a/python/.github/workflows/pytest.yaml b/.github/workflows/python-tests.yaml similarity index 87% rename from python/.github/workflows/pytest.yaml rename to .github/workflows/python-tests.yaml index 81b8ae52e..045d5c782 100644 --- a/python/.github/workflows/pytest.yaml +++ b/.github/workflows/python-tests.yaml @@ -1,16 +1,24 @@ -name: "Run Tests" +name: Python Tests + on: workflow_dispatch: push: branches: - main - release-* + paths: + - 'python/**' + - 'protocol/**' pull_request: + paths: + - 'python/**' + - 'protocol/**' merge_group: permissions: contents: read pull-requests: read + jobs: pytest-matrix: runs-on: ${{ matrix.runs-on }} @@ -60,18 +68,19 @@ jobs: id: cache-fedora-cloud-images uses: actions/cache@v4 with: - path: packages/jumpstarter-driver-qemu/images + path: python/packages/jumpstarter-driver-qemu/images key: fedora-cloud-41-1.4 - name: Download Fedora Cloud images if: steps.cache-fedora-cloud-images.outputs.cache-hit != 'true' run: | for arch in aarch64 x86_64; do - curl -L --output "packages/jumpstarter-driver-qemu/images/Fedora-Cloud-Base-Generic-41-1.4.${arch}.qcow2" \ + curl -L --output "python/packages/jumpstarter-driver-qemu/images/Fedora-Cloud-Base-Generic-41-1.4.${arch}.qcow2" \ "https://download.fedoraproject.org/pub/fedora/linux/releases/41/Cloud/${arch}/images/Fedora-Cloud-Base-Generic-41-1.4.${arch}.qcow2" done - name: Run pytest + working-directory: python run: | make test diff --git a/python/.github/workflows/trigger-packages-index.yaml b/.github/workflows/trigger-packages.yaml similarity index 100% rename from python/.github/workflows/trigger-packages-index.yaml rename to .github/workflows/trigger-packages.yaml diff --git a/controller/.github/workflows/backport.yaml b/controller/.github/workflows/backport.yaml deleted file mode 100644 index bb11015b8..000000000 --- a/controller/.github/workflows/backport.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# WARNING: -# When extending this action, be aware that $GITHUB_TOKEN allows write access to -# the GitHub repository. This means that it should not evaluate user input in a -# way that allows code injection. - -name: Backport - -on: - pull_request_target: - types: [closed, labeled] - -permissions: {} - -jobs: - backport: - name: Backport Pull Request - if: github.repository_owner == 'jumpstarter-dev' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name)) - runs-on: ubuntu-24.04 - steps: - # Use a GitHub App to create the PR so that CI gets triggered - # The App is scoped to Repository > Contents and Pull Requests: write for jumpstarter-dev - - uses: actions/create-github-app-token@3ff1caaa28b64c9cc276ce0a02e2ff584f3900c5 # v2.0.2 - id: app-token - with: - app-id: ${{ secrets.JUMPSTARTER_BACKPORT_BOT_APP_ID }} - private-key: ${{ secrets.JUMPSTARTER_BACKPORT_BOT_PRIVATE_KEY }} - - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - ref: ${{ github.event.pull_request.head.sha }} - token: ${{ steps.app-token.outputs.token }} - - - name: Create backport PRs - uses: korthout/backport-action@436145e922f9561fc5ea157ff406f21af2d6b363 # v3.2.0 - with: - # Config README: https://github.com/korthout/backport-action#backport-action - github_token: ${{ steps.app-token.outputs.token }} - conflict_resolution: draft_commit_conflicts - merge_commits: skip - pull_description: |- - Bot-based backport to `${target_branch}`, triggered by a label in #${pull_number}. - diff --git a/controller/.github/workflows/e2e.yaml b/controller/.github/workflows/e2e.yaml deleted file mode 100644 index f8fbb5a8a..000000000 --- a/controller/.github/workflows/e2e.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: End-to-end tests -on: - workflow_dispatch: - pull_request: - branches: - - main - - 'release-*' -jobs: - e2e-tests: - strategy: - matrix: - os: - - ubuntu-24.04 - - ubuntu-24.04-arm - runs-on: ${{ matrix.os }} - steps: - - uses: jumpstarter-dev/jumpstarter-e2e@main - with: - controller-ref: ${{ github.ref }} - # use the matching branch on the jumpstarter repo - jumpstarter-ref: ${{ github.event.pull_request.base.ref }} - e2e-tests-release-0-7: - runs-on: ubuntu-latest - steps: - - uses: jumpstarter-dev/jumpstarter-e2e@release-0.7 - with: - controller-ref: ${{ github.ref }} - jumpstarter-ref: release-0.7 diff --git a/controller/.github/workflows/lint.yaml b/controller/.github/workflows/lint.yaml deleted file mode 100644 index a0730189d..000000000 --- a/controller/.github/workflows/lint.yaml +++ /dev/null @@ -1,39 +0,0 @@ -name: Linters -on: - workflow_dispatch: - push: - branches: - - main - - 'release-*' - pull_request: - branches: - - main - - 'release-*' - -jobs: - lint-helm: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Run helm linter - run: make lint-helm - - lint-go: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.24 - - - name: Run go linter - run: make lint diff --git a/controller/typos.toml b/controller/typos.toml deleted file mode 100644 index e87413588..000000000 --- a/controller/typos.toml +++ /dev/null @@ -1,6 +0,0 @@ -[default.extend-words] -Ded = "Ded" # from ANDed - -[type.gomod] -extend-glob = ["go.mod", "go.sum"] -check-file = false diff --git a/e2e/.github/workflows/selftest.yml b/e2e/.github/workflows/selftest.yml deleted file mode 100644 index 086e37725..000000000 --- a/e2e/.github/workflows/selftest.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: selftest -on: - - push -jobs: - test: - strategy: - matrix: - os: - - ubuntu-24.04 - - ubuntu-24.04-arm - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - with: - path: e2e - - uses: ./e2e - with: - controller-ref: main - jumpstarter-ref: main diff --git a/e2e/action.yml b/e2e/action.yml deleted file mode 100644 index ab69a87c5..000000000 --- a/e2e/action.yml +++ /dev/null @@ -1,99 +0,0 @@ -name: 'Jumpstarter end-to-end testing' -inputs: - controller-ref: - description: 'jumpstarter-dev/jumpstarter-controller git ref' - required: true - jumpstarter-ref: - description: 'jumpstarter-dev/jumpstarter git ref' - required: true -runs: - using: "composite" - steps: - - name: Install uv - uses: astral-sh/setup-uv@v2 - - name: Install python - shell: bash - run: | - uv python install 3.12 - - name: Install bats - shell: bash - run: | - sudo apt-get update - sudo apt-get install -y bats bats-support bats-assert - - name: Checkout jumpstarter controller - uses: actions/checkout@v4 - with: - repository: jumpstarter-dev/jumpstarter-controller - ref: ${{ inputs.controller-ref }} - path: controller - - name: Checkout jumpstarter - uses: actions/checkout@v4 - with: - repository: jumpstarter-dev/jumpstarter - ref: ${{ inputs.jumpstarter-ref }} - path: jumpstarter - - name: Deploy dex - shell: bash - run: | - go run github.com/cloudflare/cfssl/cmd/cfssl@latest gencert -initca "$GITHUB_ACTION_PATH"/ca-csr.json | \ - go run github.com/cloudflare/cfssl/cmd/cfssljson@latest -bare ca - - go run github.com/cloudflare/cfssl/cmd/cfssl@latest gencert -ca=ca.pem -ca-key=ca-key.pem \ - -config="$GITHUB_ACTION_PATH"/ca-config.json -profile=www "$GITHUB_ACTION_PATH"/dex-csr.json | \ - go run github.com/cloudflare/cfssl/cmd/cfssljson@latest -bare server - - cp "$GITHUB_ACTION_PATH"/kind_cluster.yaml ./controller/hack/kind_cluster.yaml - make -C controller cluster - - kubectl create namespace dex - kubectl -n dex create secret tls dex-tls \ - --cert=server.pem \ - --key=server-key.pem - - go run github.com/mikefarah/yq/v4@latest -i \ - '.jumpstarter-controller.config.authentication.jwt[0].issuer.certificateAuthority = load_str("ca.pem")' \ - "$GITHUB_ACTION_PATH"/values.kind.yaml - - # important! - kubectl create clusterrolebinding oidc-reviewer \ - --clusterrole=system:service-account-issuer-discovery \ - --group=system:unauthenticated - - helm repo add dex https://charts.dexidp.io - helm install --namespace dex --wait -f "$GITHUB_ACTION_PATH"/dex.values.yaml dex dex/dex - - sudo cp ca.pem /usr/local/share/ca-certificates/dex.crt - sudo update-ca-certificates - - echo "127.0.0.1 dex.dex.svc.cluster.local" | sudo tee -a /etc/hosts - - name: Deploy jumpstarter controller - shell: bash - run: | - cp "$GITHUB_ACTION_PATH"/values.kind.yaml ./controller/deploy/helm/jumpstarter/values.kind.yaml - make -C controller deploy - - name: Install jumpstarter - shell: bash - run: | - uv venv - uv pip install \ - ./jumpstarter/packages/jumpstarter-cli \ - ./jumpstarter/packages/jumpstarter-driver-composite \ - ./jumpstarter/packages/jumpstarter-driver-power \ - ./jumpstarter/packages/jumpstarter-driver-opendal - - name: Run jumpstarter - shell: bash - run: | - export ENDPOINT=$(helm get values jumpstarter --output json | jq -r '."jumpstarter-controller".grpc.endpoint') - - sudo mkdir -p /etc/jumpstarter/exporters - sudo chown $USER /etc/jumpstarter/exporters - - export JS_NAMESPACE="jumpstarter-lab" - - . .venv/bin/activate - - export JUMPSTARTER_GRPC_INSECURE=1 - - kubectl create -n "${JS_NAMESPACE}" sa test-client-sa - kubectl create -n "${JS_NAMESPACE}" sa test-exporter-sa - - bats --show-output-of-passing-tests --verbose-run "$GITHUB_ACTION_PATH"/tests.bats diff --git a/e2e/run-e2e.sh b/e2e/run-e2e.sh new file mode 100755 index 000000000..203fad286 --- /dev/null +++ b/e2e/run-e2e.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +# Jumpstarter End-to-End Test Runner +# This script runs the e2e test suite (assumes setup-e2e.sh was run first) + +set -euo pipefail + +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Get the monorepo root (parent of e2e directory) +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +# Color output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +log_info() { + echo -e "${GREEN}[INFO]${NC} $*" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $*" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $*" +} + +# Check if running in CI +is_ci() { + [ -n "${CI:-}" ] || [ -n "${GITHUB_ACTIONS:-}" ] +} + +# Check if setup was completed +check_setup() { + if [ ! -f "$REPO_ROOT/.e2e-setup-complete" ]; then + log_error "Setup not complete! Please run setup-e2e.sh first:" + log_error " bash e2e/setup-e2e.sh" + log_error "" + log_error "Or in CI mode, run the full setup automatically" + return 1 + fi + + # Load setup configuration + source "$REPO_ROOT/.e2e-setup-complete" + + # Export SSL certificate paths for Python + export SSL_CERT_FILE + export REQUESTS_CA_BUNDLE + + # Verify critical components are still running + if ! kubectl get namespace "$JS_NAMESPACE" &> /dev/null; then + log_error "Namespace $JS_NAMESPACE not found. Please run setup-e2e.sh again." + return 1 + fi + + log_info "✓ Setup verified" + return 0 +} + +# Setup environment for bats +setup_bats_env() { + # Always set BATS_LIB_PATH to include local libraries + local LOCAL_BATS_LIB="$REPO_ROOT/.bats/lib" + + if [ -d "$LOCAL_BATS_LIB" ]; then + export BATS_LIB_PATH="$LOCAL_BATS_LIB:${BATS_LIB_PATH:-}" + log_info "Set BATS_LIB_PATH to local libraries: $BATS_LIB_PATH" + else + log_warn "Local bats libraries not found at $LOCAL_BATS_LIB" + log_warn "You may need to run setup-e2e.sh first" + fi +} + +# Run the tests +run_tests() { + log_info "Running jumpstarter e2e tests..." + + cd "$REPO_ROOT" + + # Activate virtual environment + if [ -f .venv/bin/activate ]; then + source .venv/bin/activate + else + log_error "Virtual environment not found. Please run setup-e2e.sh first." + exit 1 + fi + + # Use insecure GRPC for testing + export JUMPSTARTER_GRPC_INSECURE=1 + + # Export variables for bats + export JS_NAMESPACE="${JS_NAMESPACE}" + export ENDPOINT="${ENDPOINT}" + + # Setup bats environment + setup_bats_env + + # Run bats tests + log_info "Running bats tests..." + bats --show-output-of-passing-tests --verbose-run "$SCRIPT_DIR"/tests.bats +} + +# Full setup and run (for CI or first-time use) +full_run() { + log_info "Running full setup + test cycle..." + + if [ -f "$SCRIPT_DIR/setup-e2e.sh" ]; then + bash "$SCRIPT_DIR/setup-e2e.sh" + else + log_error "setup-e2e.sh not found!" + exit 1 + fi + + # After setup, load the configuration + if [ -f "$REPO_ROOT/.e2e-setup-complete" ]; then + source "$REPO_ROOT/.e2e-setup-complete" + # Export SSL certificate paths for Python + export SSL_CERT_FILE + export REQUESTS_CA_BUNDLE + fi + + run_tests +} + +# Main execution +main() { + # Default namespace + export JS_NAMESPACE="${JS_NAMESPACE:-jumpstarter-lab}" + + log_info "=== Jumpstarter E2E Test Runner ===" + log_info "Namespace: $JS_NAMESPACE" + log_info "Repository Root: $REPO_ROOT" + echo "" + + # If --full flag is passed, always run full setup + if [[ "${1:-}" == "--full" ]]; then + full_run + # In CI mode, check if setup was already done + elif is_ci; then + if check_setup 2>/dev/null; then + log_info "Setup already complete, skipping setup and running tests..." + run_tests + else + log_info "Setup not found in CI, running full setup..." + full_run + fi + else + # Local development: require setup to be done first + if check_setup; then + run_tests + else + log_error "" + log_error "Setup is required before running tests." + log_error "" + log_error "Options:" + log_error " 1. Run setup first: bash e2e/setup-e2e.sh" + log_error " 2. Run full cycle: bash e2e/run-e2e.sh --full" + exit 1 + fi + fi + + echo "" + log_info "✓✓✓ All e2e tests completed successfully! ✓✓✓" +} + +# Run main function +main "$@" diff --git a/e2e/setup-e2e.sh b/e2e/setup-e2e.sh new file mode 100755 index 000000000..96a4c7e69 --- /dev/null +++ b/e2e/setup-e2e.sh @@ -0,0 +1,364 @@ +#!/usr/bin/env bash +# Jumpstarter End-to-End Testing Setup Script +# This script performs one-time setup for e2e testing + +set -euo pipefail + +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Get the monorepo root (parent of e2e directory) +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +# Default namespace for tests +export JS_NAMESPACE="${JS_NAMESPACE:-jumpstarter-lab}" + +# Color output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +log_info() { + echo -e "${GREEN}[INFO]${NC} $*" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $*" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $*" +} + +# Check if running in CI +is_ci() { + [ -n "${CI:-}" ] || [ -n "${GITHUB_ACTIONS:-}" ] +} + +# Check if bats libraries are available +check_bats_libraries() { + if ! command -v bats &> /dev/null; then + return 1 + fi + + # Try to load the libraries + if ! bats --version &> /dev/null; then + return 1 + fi + + # Check if libraries can be loaded by testing with a simple script + local test_file=$(mktemp) + cat > "$test_file" <<'EOF' +setup() { + bats_load_library bats-support + bats_load_library bats-assert +} + +@test "dummy" { + run echo "test" + assert_success +} +EOF + + # Run test with current BATS_LIB_PATH + if bats "$test_file" &> /dev/null; then + rm -f "$test_file" + return 0 + else + rm -f "$test_file" + return 1 + fi +} + +# Install bats libraries locally (works on all systems) +install_bats_libraries_local() { + local LIB_DIR="$REPO_ROOT/.bats/lib" + local ORIGINAL_DIR="$PWD" + + log_info "Installing bats helper libraries to $LIB_DIR..." + + mkdir -p "$LIB_DIR" + cd "$LIB_DIR" + + # Install bats-support + if [ ! -d "bats-support" ]; then + log_info "Cloning bats-support..." + git clone --depth 1 https://github.com/bats-core/bats-support.git + else + log_info "bats-support already installed" + fi + + # Install bats-assert + if [ ! -d "bats-assert" ]; then + log_info "Cloning bats-assert..." + git clone --depth 1 https://github.com/bats-core/bats-assert.git + else + log_info "bats-assert already installed" + fi + + # Install bats-file + if [ ! -d "bats-file" ]; then + log_info "Cloning bats-file..." + git clone --depth 1 https://github.com/bats-core/bats-file.git + else + log_info "bats-file already installed" + fi + + cd "$ORIGINAL_DIR" + + # Set BATS_LIB_PATH + export BATS_LIB_PATH="$LIB_DIR:${BATS_LIB_PATH:-}" + + log_info "✓ Bats libraries installed successfully" + log_info "BATS_LIB_PATH set to: $BATS_LIB_PATH" + + # Verify installation worked + if check_bats_libraries; then + log_info "✓ Libraries verified and working" + else + log_error "Libraries installed but verification failed" + log_error "Please check that the following directories exist:" + log_error " $LIB_DIR/bats-support" + log_error " $LIB_DIR/bats-assert" + exit 1 + fi +} + +# Step 1: Install dependencies +install_dependencies() { + log_info "Installing dependencies..." + + # Install uv if not already installed + if ! command -v uv &> /dev/null; then + log_info "Installing uv..." + curl -LsSf https://astral.sh/uv/install.sh | sh + export PATH="$HOME/.cargo/bin:$PATH" + fi + + # Install Python 3.12 + log_info "Installing Python 3.12..." + uv python install 3.12 + + # Install bats if not already installed + if ! command -v bats &> /dev/null; then + log_info "Installing bats..." + if is_ci; then + sudo apt-get update + sudo apt-get install -y bats + elif [[ "$OSTYPE" == "darwin"* ]]; then + log_info "Installing bats-core via Homebrew..." + brew install bats-core + else + log_error "bats not found. Please install it manually:" + log_error " Ubuntu/Debian: sudo apt-get install bats" + log_error " Fedora/RHEL: sudo dnf install bats" + log_error " macOS: brew install bats-core" + exit 1 + fi + fi + + # Always install bats libraries locally for consistency across all systems + # This ensures libraries work regardless of package manager or distribution + if ! check_bats_libraries; then + log_info "Installing bats libraries locally..." + install_bats_libraries_local + else + log_info "✓ Bats libraries are already available" + # Still set BATS_LIB_PATH to include local directory for consistency + export BATS_LIB_PATH="$REPO_ROOT/.bats/lib:${BATS_LIB_PATH:-}" + fi + + log_info "✓ Dependencies installed" +} + +# Step 2: Deploy dex +deploy_dex() { + log_info "Deploying dex..." + + cd "$REPO_ROOT" + + # Generate certificates + log_info "Generating certificates..." + go run github.com/cloudflare/cfssl/cmd/cfssl@latest gencert -initca "$SCRIPT_DIR"/ca-csr.json | \ + go run github.com/cloudflare/cfssl/cmd/cfssljson@latest -bare ca - + go run github.com/cloudflare/cfssl/cmd/cfssl@latest gencert -ca=ca.pem -ca-key=ca-key.pem \ + -config="$SCRIPT_DIR"/ca-config.json -profile=www "$SCRIPT_DIR"/dex-csr.json | \ + go run github.com/cloudflare/cfssl/cmd/cfssljson@latest -bare server + + + make -C controller cluster + + # Create dex namespace and TLS secret + log_info "Creating dex namespace and secrets..." + kubectl create namespace dex + kubectl -n dex create secret tls dex-tls \ + --cert=server.pem \ + --key=server-key.pem + + # Create .e2e directory for configuration files + log_info "Creating .e2e directory for local configuration..." + mkdir -p "$REPO_ROOT/.e2e" + + # Copy values.kind.yaml to .e2e and inject the CA certificate + log_info "Creating values file with CA certificate..." + cp "$SCRIPT_DIR"/values.kind.yaml "$REPO_ROOT/.e2e/values.kind.yaml" + + log_info "Injecting CA certificate into values..." + go run github.com/mikefarah/yq/v4@latest -i \ + '.jumpstarter-controller.config.authentication.jwt[0].issuer.certificateAuthority = load_str("ca.pem")' \ + "$REPO_ROOT/.e2e/values.kind.yaml" + + log_info "✓ Values file with CA certificate created at .e2e/values.kind.yaml" + + # Create OIDC reviewer binding (important!) + log_info "Creating OIDC reviewer cluster role binding..." + kubectl create clusterrolebinding oidc-reviewer \ + --clusterrole=system:service-account-issuer-discovery \ + --group=system:unauthenticated + + # Install dex via helm + log_info "Installing dex via helm..." + helm repo add dex https://charts.dexidp.io + helm install --namespace dex --wait -f "$SCRIPT_DIR"/dex.values.yaml dex dex/dex + + # Install CA certificate + log_info "Installing CA certificate..." + if [[ "$OSTYPE" == "darwin"* ]]; then + # this may be unnecessary, but keeping it here for now + #log_warn "About to add the CA certificate to your macOS login keychain" + #security add-trusted-cert -d -r trustRoot -k ~/Library/Keychains/login.keychain-db ca.pem + #log_info "✓ CA certificate added to macOS login keychain" + true + else + log_warn "About to install the CA certificate system-wide (requires sudo)" + # Detect if this is a RHEL/Fedora system or Debian/Ubuntu system + if [ -d "/etc/pki/ca-trust/source/anchors" ]; then + # RHEL/Fedora/CentOS + sudo cp ca.pem /etc/pki/ca-trust/source/anchors/dex.crt + sudo update-ca-trust + log_info "✓ CA certificate installed system-wide (RHEL/Fedora)" + else + # Debian/Ubuntu + sudo cp ca.pem /usr/local/share/ca-certificates/dex.crt + sudo update-ca-certificates + log_info "✓ CA certificate installed system-wide (Debian/Ubuntu)" + fi + fi + + # Add dex to /etc/hosts if not already present + log_info "Checking /etc/hosts for dex entry..." + if ! grep -q "dex.dex.svc.cluster.local" /etc/hosts 2>/dev/null; then + log_warn "About to add 'dex.dex.svc.cluster.local' to /etc/hosts (requires sudo)" + echo "127.0.0.1 dex.dex.svc.cluster.local" | sudo tee -a /etc/hosts + log_info "✓ Added dex to /etc/hosts" + else + log_info "✓ dex.dex.svc.cluster.local already in /etc/hosts" + fi + + log_info "✓ Dex deployed" +} + +# Step 3: Deploy jumpstarter controller +deploy_controller() { + log_info "Deploying jumpstarter controller..." + + cd "$REPO_ROOT" + + # Deploy with modified values using EXTRA_VALUES environment variable + log_info "Deploying controller with CA certificate..." + EXTRA_VALUES="--values $REPO_ROOT/.e2e/values.kind.yaml" make -C controller deploy + + log_info "✓ Controller deployed" +} + +# Step 4: Install jumpstarter +install_jumpstarter() { + log_info "Installing jumpstarter..." + + cd "$REPO_ROOT" + + # Create virtual environment + uv venv + + # Install jumpstarter packages + uv pip install \ + ./python/packages/jumpstarter-cli \ + ./python/packages/jumpstarter-driver-composite \ + ./python/packages/jumpstarter-driver-power \ + ./python/packages/jumpstarter-driver-opendal + + log_info "✓ Jumpstarter installed" +} + +# Step 5: Setup test environment +setup_test_environment() { + log_info "Setting up test environment..." + + cd "$REPO_ROOT" + + # Get the controller endpoint + export ENDPOINT=$(helm get values jumpstarter --output json | jq -r '."jumpstarter-controller".grpc.endpoint') + log_info "Controller endpoint: $ENDPOINT" + + # Setup exporters directory + echo "Setting up exporters directory in /etc/jumpstarter/exporters..., will need permissions" + sudo mkdir -p /etc/jumpstarter/exporters + sudo chown "$USER" /etc/jumpstarter/exporters + + # Create service accounts + log_info "Creating service accounts..." + kubectl create -n "${JS_NAMESPACE}" sa test-client-sa + kubectl create -n "${JS_NAMESPACE}" sa test-exporter-sa + + # Create a marker file to indicate setup is complete + echo "ENDPOINT=$ENDPOINT" > "$REPO_ROOT/.e2e-setup-complete" + echo "JS_NAMESPACE=$JS_NAMESPACE" >> "$REPO_ROOT/.e2e-setup-complete" + echo "REPO_ROOT=$REPO_ROOT" >> "$REPO_ROOT/.e2e-setup-complete" + echo "SCRIPT_DIR=$SCRIPT_DIR" >> "$REPO_ROOT/.e2e-setup-complete" + + # Set SSL certificate paths for Python to use the generated CA + echo "SSL_CERT_FILE=$REPO_ROOT/ca.pem" >> "$REPO_ROOT/.e2e-setup-complete" + echo "REQUESTS_CA_BUNDLE=$REPO_ROOT/ca.pem" >> "$REPO_ROOT/.e2e-setup-complete" + + # Save BATS_LIB_PATH for test runs + echo "BATS_LIB_PATH=$BATS_LIB_PATH" >> "$REPO_ROOT/.e2e-setup-complete" + + log_info "✓ Test environment ready" +} + +# Main execution +main() { + log_info "=== Jumpstarter E2E Setup ===" + log_info "Namespace: $JS_NAMESPACE" + log_info "Repository Root: $REPO_ROOT" + log_info "Script Directory: $SCRIPT_DIR" + echo "" + + install_dependencies + echo "" + + deploy_dex + echo "" + + deploy_controller + echo "" + + install_jumpstarter + echo "" + + setup_test_environment + echo "" + + log_info "✓✓✓ Setup complete! ✓✓✓" + log_info "" + log_info "To run tests:" + log_info " cd $REPO_ROOT" + log_info " bash e2e/run-e2e.sh" + log_info "" + log_info "Or use the Makefile:" + log_info " make e2e" +} + +# Run main function +main "$@" diff --git a/e2e/tests.bats b/e2e/tests.bats index e84f06304..703e15fc4 100644 --- a/e2e/tests.bats +++ b/e2e/tests.bats @@ -1,5 +1,13 @@ JS_NAMESPACE="${JS_NAMESPACE:-jumpstarter-lab}" +# File to track bash wrapper process PIDs across tests +EXPORTER_PIDS_FILE="${BATS_RUN_TMPDIR:-/tmp}/exporter_pids.txt" + +setup_file() { + # Initialize the PIDs file at the start of all tests + echo "" > "$EXPORTER_PIDS_FILE" +} + setup() { bats_load_library bats-support bats_load_library bats-assert @@ -7,6 +15,47 @@ setup() { bats_require_minimum_version 1.5.0 } +# teardown_file runs once after all tests complete (requires bats-core 1.5.0+) +teardown_file() { + echo "" >&2 + echo "========================================" >&2 + echo "TEARDOWN_FILE RUNNING" >&2 + echo "========================================" >&2 + echo "=== Cleaning up exporter bash processes ===" >&2 + + # Read PIDs from file + if [ -f "$EXPORTER_PIDS_FILE" ]; then + local pids=$(cat "$EXPORTER_PIDS_FILE" | tr '\n' ' ') + echo "Tracked PIDs from file: $pids" >&2 + + while IFS= read -r pid; do + if [ -n "$pid" ]; then + echo "Checking PID $pid..." >&2 + if ps -p "$pid" > /dev/null 2>&1; then + echo " Killing PID $pid" >&2 + kill -9 "$pid" 2>/dev/null || true + else + echo " PID $pid already terminated" >&2 + fi + fi + done < "$EXPORTER_PIDS_FILE" + else + echo "No PIDs file found at $EXPORTER_PIDS_FILE" >&2 + fi + + echo "Checking for orphaned jmp processes..." >&2 + local orphans=$(pgrep -f "jmp run --exporter" 2>/dev/null | wc -l) + echo "Found $orphans orphaned jmp processes" >&2 + + # remove orphaned processes + pkill -9 -f "jmp run --exporter" 2>/dev/null || true + + # Clean up the PIDs file + rm -f "$EXPORTER_PIDS_FILE" + + echo "=== Cleanup complete ===" >&2 +} + wait_for_exporter() { # After a lease operation the exporter is disconnecting from controller and reconnecting. # The disconnect can take a short while so let's avoid catching the pre-disconnect state and early return @@ -69,11 +118,11 @@ wait_for_exporter() { --connector-id kubernetes \ --token $(kubectl create -n "${JS_NAMESPACE}" token test-exporter-sa) - go run github.com/mikefarah/yq/v4@latest -i ". * load(\"$GITHUB_ACTION_PATH/exporter.yaml\")" \ + go run github.com/mikefarah/yq/v4@latest -i ". * load(\"e2e/exporter.yaml\")" \ /etc/jumpstarter/exporters/test-exporter-oidc.yaml - go run github.com/mikefarah/yq/v4@latest -i ". * load(\"$GITHUB_ACTION_PATH/exporter.yaml\")" \ + go run github.com/mikefarah/yq/v4@latest -i ". * load(\"e2e/exporter.yaml\")" \ /etc/jumpstarter/exporters/test-exporter-sa.yaml - go run github.com/mikefarah/yq/v4@latest -i ". * load(\"$GITHUB_ACTION_PATH/exporter.yaml\")" \ + go run github.com/mikefarah/yq/v4@latest -i ". * load(\"e2e/exporter.yaml\")" \ /etc/jumpstarter/exporters/test-exporter-legacy.yaml jmp config client list @@ -86,19 +135,21 @@ while true; do jmp run --exporter test-exporter-oidc done EOF + echo "$!" >> "$EXPORTER_PIDS_FILE" cat <&- & while true; do jmp run --exporter test-exporter-sa done EOF + echo "$!" >> "$EXPORTER_PIDS_FILE" cat <&- & while true; do jmp run --exporter test-exporter-legacy done EOF - + echo "$!" >> "$EXPORTER_PIDS_FILE" wait_for_exporter } @@ -110,7 +161,7 @@ EOF # to verify that the client can operate without a config file JMP_NAMESPACE="${JS_NAMESPACE}" \ JMP_DRIVERS_ALLOW="*" \ - JMP_NAME=test-exporter-legacy \ + JMP_NAME=test-client-legacy \ JMP_ENDPOINT=$(kubectl get clients.jumpstarter.dev -n "${JS_NAMESPACE}" test-client-legacy -o 'jsonpath={.status.endpoint}') \ JMP_TOKEN=$(kubectl get secrets -n "${JS_NAMESPACE}" test-client-legacy-client -o 'jsonpath={.data.token}' | base64 -d) \ jmp shell --selector example.com/board=oidc j power on diff --git a/protocol/.github/workflows/lint.yaml b/protocol/.github/workflows/lint.yaml deleted file mode 100644 index 2492ddece..000000000 --- a/protocol/.github/workflows/lint.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: "Code Quality" -on: - workflow_dispatch: - push: - branches: - - main - pull_request: - -permissions: - contents: read - pull-requests: read -jobs: - lint: - runs-on: "ubuntu-latest" - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Running Linter - run: make lint diff --git a/python/.github/dependabot.yml b/python/.github/dependabot.yml deleted file mode 100644 index f33a02cd1..000000000 --- a/python/.github/dependabot.yml +++ /dev/null @@ -1,12 +0,0 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for more information: -# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates -# https://containers.dev/guide/dependabot - -version: 2 -updates: - - package-ecosystem: "devcontainers" - directory: "/" - schedule: - interval: weekly diff --git a/python/.github/workflows/backport.yml b/python/.github/workflows/backport.yml deleted file mode 100644 index 1c423801a..000000000 --- a/python/.github/workflows/backport.yml +++ /dev/null @@ -1,41 +0,0 @@ -# WARNING: -# When extending this action, be aware that $GITHUB_TOKEN allows write access to -# the GitHub repository. This means that it should not evaluate user input in a -# way that allows code injection. - -name: Backport - -on: - pull_request_target: - types: [closed, labeled] - -permissions: {} - -jobs: - backport: - name: Backport Pull Request - if: github.repository_owner == 'jumpstarter-dev' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name)) - runs-on: ubuntu-24.04 - steps: - # Use a GitHub App to create the PR so that CI gets triggered - # The App is scoped to Repository > Contents and Pull Requests: write for jumpstarter-dev - - uses: actions/create-github-app-token@3ff1caaa28b64c9cc276ce0a02e2ff584f3900c5 # v2.0.2 - id: app-token - with: - app-id: ${{ secrets.JUMPSTARTER_BACKPORT_BOT_APP_ID }} - private-key: ${{ secrets.JUMPSTARTER_BACKPORT_BOT_PRIVATE_KEY }} - - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - ref: ${{ github.event.pull_request.head.sha }} - token: ${{ steps.app-token.outputs.token }} - - - name: Create backport PRs - uses: korthout/backport-action@436145e922f9561fc5ea157ff406f21af2d6b363 # v3.2.0 - with: - # Config README: https://github.com/korthout/backport-action#backport-action - github_token: ${{ steps.app-token.outputs.token }} - conflict_resolution: draft_commit_conflicts - merge_commits: skip - pull_description: |- - Bot-based backport to `${target_branch}`, triggered by a label in #${pull_number}. diff --git a/python/.github/workflows/build.yaml b/python/.github/workflows/build.yaml deleted file mode 100644 index f8a2520b6..000000000 --- a/python/.github/workflows/build.yaml +++ /dev/null @@ -1,111 +0,0 @@ -name: Build and push container image -on: - workflow_dispatch: - push: - branches: - - main - - release-* - tags: - - v* - merge_group: - -env: - PUSH: ${{ github.repository_owner == 'jumpstarter-dev' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/release-')) }} - REGISTRY: quay.io - QUAY_ORG: quay.io/jumpstarter-dev - -jobs: - build-and-push-image: - strategy: - matrix: - image: - - jumpstarter-dev/jumpstarter Dockerfile - - jumpstarter-dev/jumpstarter-utils Dockerfile.utils - - jumpstarter-dev/jumpstarter-dev .devfile/Containerfile - - jumpstarter-dev/jumpstarter-devspace .devfile/Containerfile.client - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - attestations: write - id-token: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Get image name and container file - run: | - IMAGE="${{ matrix.image }}" - IMAGE_NAME=$(echo $IMAGE | awk '{print $1}') - CONTAINERFILE=$(echo $IMAGE | awk '{print $2}') - echo "IMAGE_NAME=${IMAGE_NAME}" >> $GITHUB_ENV - echo "IMAGE_NAME=${IMAGE_NAME}" - echo "CONTAINERFILE=${CONTAINERFILE}" >> $GITHUB_ENV - echo "CONTAINERFILE=${CONTAINERFILE}" - - - name: Get version - if: ${{ env.PUSH == 'true' }} - run: | - VERSION=$(git describe --tags) - VERSION=${VERSION#v} # remove the leading v prefix for version - echo "VERSION=${VERSION}" >> $GITHUB_ENV - echo "VERSION=${VERSION}" - - - name: Set image tags - if: ${{ env.PUSH == 'true' }} - id: set-tags - run: | - TAGS="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }}" - - if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then - TAGS="$TAGS,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" - fi - - if [[ "${{ github.ref }}" == refs/heads/release-* ]]; then - RELEASE_BRANCH_NAME=$(basename "${{ github.ref }}") - TAGS="$TAGS,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${RELEASE_BRANCH_NAME}" - fi - - echo "tags=$TAGS" >> $GITHUB_OUTPUT - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - if: ${{ env.PUSH == 'true' }} - with: - registry: ${{ env.REGISTRY }} - username: jumpstarter-dev+jumpstarter_ci - password: ${{ secrets.QUAY_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - - name: Build and push Docker image - id: push - uses: docker/build-push-action@v6 - with: - context: . - file: ${{ env.CONTAINERFILE }} - push: ${{ env.PUSH }} - tags: ${{ steps.set-tags.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Generate artifact attestation - uses: actions/attest-build-provenance@v1 - if: ${{ env.PUSH == 'true' }} - with: - subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - subject-digest: ${{ steps.push.outputs.digest }} - push-to-registry: ${{ env.PUSH }} diff --git a/python/.github/workflows/e2e.yaml b/python/.github/workflows/e2e.yaml deleted file mode 100644 index 0e79a657d..000000000 --- a/python/.github/workflows/e2e.yaml +++ /dev/null @@ -1,24 +0,0 @@ -name: "Run E2E Tests" -on: - workflow_dispatch: - push: - branches: - - main - - release-* - pull_request: - merge_group: - -permissions: - contents: read - -jobs: - e2e: - if: github.repository_owner == 'jumpstarter-dev' - runs-on: ubuntu-latest - timeout-minutes: 60 - continue-on-error: false - steps: - - uses: jumpstarter-dev/jumpstarter-e2e@main - with: - controller-ref: main - jumpstarter-ref: ${{ github.ref }} diff --git a/python/.github/workflows/ruff.yaml b/python/.github/workflows/ruff.yaml deleted file mode 100644 index 9028ec4b2..000000000 --- a/python/.github/workflows/ruff.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: Lint - -on: - workflow_dispatch: - push: - branches: - - main - - release-* - pull_request: - merge_group: - -permissions: - contents: read - -jobs: - ruff: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Run ruff - uses: astral-sh/ruff-action@84f83ecf9e1e15d26b7984c7ec9cf73d39ffc946 # v3.3.1 - with: - version-file: pyproject.toml diff --git a/python/.github/workflows/typos.yaml b/python/.github/workflows/typos.yaml deleted file mode 100644 index 33a087163..000000000 --- a/python/.github/workflows/typos.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: Spell Check - -on: - workflow_dispatch: - push: - branches: - - main - - release-* - pull_request: - merge_group: - -permissions: - contents: read - -jobs: - typos: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Run typos - uses: crate-ci/typos@0f0ccba9ed1df83948f0c15026e4f5ccfce46109 # v1.32.0 From d2e609cfc6070db4166cd7fe0cabc905007c8e02 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Wed, 21 Jan 2026 22:21:22 +0100 Subject: [PATCH 06/18] Update import_pr.sh with bug fixes --- import_pr.sh | 104 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 76 insertions(+), 28 deletions(-) diff --git a/import_pr.sh b/import_pr.sh index c945f37b1..06853832d 100755 --- a/import_pr.sh +++ b/import_pr.sh @@ -24,13 +24,27 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEMP_DIR="${SCRIPT_DIR}/.import-pr-temp" PATCH_DIR="${TEMP_DIR}/patches" -# Repository mappings: repo_name -> "github_repo subdir" -declare -A REPO_MAP=( - ["python"]="jumpstarter-dev/jumpstarter python" - ["protocol"]="jumpstarter-dev/jumpstarter-protocol protocol" - ["controller"]="jumpstarter-dev/jumpstarter-controller controller" - ["e2e"]="jumpstarter-dev/jumpstarter-e2e e2e" -) +# Repository mapping function (compatible with bash 3.2+) +get_repo_info() { + local repo_name="$1" + case "$repo_name" in + python) + echo "jumpstarter-dev/jumpstarter python" + ;; + protocol) + echo "jumpstarter-dev/jumpstarter-protocol protocol" + ;; + controller) + echo "jumpstarter-dev/jumpstarter-controller controller" + ;; + e2e) + echo "jumpstarter-dev/jumpstarter-e2e e2e" + ;; + *) + echo "" + ;; + esac +} # Colors for output RED='\033[0;31m' @@ -127,7 +141,9 @@ validate_args() { local pr_number="$2" # Validate repo name - if [ -z "${REPO_MAP[$repo]}" ]; then + local repo_info + repo_info=$(get_repo_info "$repo") + if [ -z "$repo_info" ]; then log_error "Invalid repository name: ${repo}" echo "Valid options are: python, protocol, controller, e2e" exit 1 @@ -182,12 +198,9 @@ clone_and_checkout_pr() { local clone_dir="${TEMP_DIR}/repo" - # Clone the repository + # Clone the repository (full clone needed for patch generation) log_info "Cloning ${github_repo}..." - gh repo clone "${github_repo}" "${clone_dir}" -- --depth=1 --no-single-branch 2>/dev/null || { - # If shallow clone fails, try full clone - gh repo clone "${github_repo}" "${clone_dir}" - } + gh repo clone "${github_repo}" "${clone_dir}" cd "${clone_dir}" @@ -195,9 +208,9 @@ clone_and_checkout_pr() { log_info "Checking out PR #${pr_number}..." gh pr checkout "${pr_number}" --repo "${github_repo}" - # Fetch the base branch to ensure we have it - log_info "Fetching base branch (${PR_BASE_BRANCH})..." - git fetch origin "${PR_BASE_BRANCH}" + # Ensure we have the full history of both branches for finding merge base + log_info "Fetching base branch with full history..." + git fetch --unshallow origin "${PR_BASE_BRANCH}" 2>/dev/null || git fetch origin "${PR_BASE_BRANCH}" CLONE_DIR="${clone_dir}" } @@ -206,29 +219,63 @@ clone_and_checkout_pr() { generate_patches() { log_step "Generating patches..." - cd "${CLONE_DIR}" + cd "${CLONE_DIR}" || { + log_error "Failed to cd to ${CLONE_DIR}" + exit 1 + } # Find the merge base between the PR branch and the base branch local merge_base - merge_base=$(git merge-base "origin/${PR_BASE_BRANCH}" HEAD) + if ! merge_base=$(git merge-base "origin/${PR_BASE_BRANCH}" HEAD 2>&1); then + log_error "Failed to find merge base: ${merge_base}" + exit 1 + fi log_info "Merge base: ${merge_base}" - # Count commits to be patched - local commit_count - commit_count=$(git rev-list --count "${merge_base}..HEAD") - log_info "Commits to import: ${commit_count}" + # Count all commits (including merges) + local total_commits + if ! total_commits=$(git rev-list --count "${merge_base}..HEAD" 2>&1); then + log_error "Failed to count commits: ${total_commits}" + exit 1 + fi + + # Count non-merge commits + local non_merge_commits + if ! non_merge_commits=$(git rev-list --count --no-merges "${merge_base}..HEAD" 2>&1); then + log_error "Failed to count non-merge commits: ${non_merge_commits}" + exit 1 + fi + + log_info "Total commits: ${total_commits} (${non_merge_commits} non-merge)" - if [ "$commit_count" -eq 0 ]; then - log_error "No commits found between merge base and HEAD." + if [ "$non_merge_commits" -eq 0 ]; then + log_error "No non-merge commits found between merge base and HEAD." exit 1 fi + + # Check if there are merge commits + local merge_commits=$((total_commits - non_merge_commits)) + if [ "$merge_commits" -gt 0 ]; then + log_warn "PR contains ${merge_commits} merge commit(s) which will be skipped." + log_warn "Only the ${non_merge_commits} non-merge commits will be imported." + fi - # Generate patches - git format-patch -o "${PATCH_DIR}" "${merge_base}..HEAD" + # Generate patches (skip merge commits) + log_info "Generating patches for non-merge commits..." + if ! git format-patch --no-merges -o "${PATCH_DIR}" "${merge_base}..HEAD"; then + log_error "Failed to generate patches." + exit 1 + fi # Count generated patches - PATCH_COUNT=$(ls -1 "${PATCH_DIR}"/*.patch 2>/dev/null | wc -l | tr -d ' ') + PATCH_COUNT=$(find "${PATCH_DIR}" -name "*.patch" 2>/dev/null | wc -l | tr -d ' ') + + if [ "$PATCH_COUNT" -eq 0 ]; then + log_error "No patches were generated." + exit 1 + fi + log_info "Generated ${PATCH_COUNT} patch file(s)." } @@ -354,7 +401,8 @@ main() { echo "" # Parse repo mapping - local repo_info="${REPO_MAP[$repo_name]}" + local repo_info + repo_info=$(get_repo_info "$repo_name") local github_repo subdir read -r github_repo subdir <<< "${repo_info}" From 8de05f5f87b57dba0aee36b4fbb0d6b49c9dd721 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Thu, 22 Jan 2026 09:25:24 +0100 Subject: [PATCH 07/18] update import pr --- import_pr.sh | 52 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/import_pr.sh b/import_pr.sh index 06853832d..cc83c389d 100755 --- a/import_pr.sh +++ b/import_pr.sh @@ -21,8 +21,9 @@ set -e # Configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -TEMP_DIR="${SCRIPT_DIR}/.import-pr-temp" -PATCH_DIR="${TEMP_DIR}/patches" +# TEMP_DIR will be set later with repo name and PR number +TEMP_DIR="" +PATCH_DIR="" # Repository mapping function (compatible with bash 3.2+) get_repo_info() { @@ -70,15 +71,23 @@ log_step() { echo -e "${BLUE}[STEP]${NC} $1" } -# Cleanup function +# Cleanup function - only clean up on success cleanup() { local exit_code=$? - if [ -d "${TEMP_DIR}" ]; then - log_info "Cleaning up temporary directory..." - rm -rf "${TEMP_DIR}" - fi - if [ $exit_code -ne 0 ]; then - log_warn "Script exited with errors. Any partial changes may need to be reverted." + if [ $exit_code -eq 0 ]; then + if [ -n "${TEMP_DIR}" ] && [ -d "${TEMP_DIR}" ]; then + log_info "Cleaning up temporary directory..." + rm -rf "${TEMP_DIR}" + fi + else + if [ -n "${TEMP_DIR}" ] && [ -d "${TEMP_DIR}" ]; then + echo "" + log_warn "Script failed. Temporary directory preserved for debugging:" + log_warn " ${TEMP_DIR}" + echo "" + log_warn "To clean up manually after debugging:" + log_warn " rm -rf ${TEMP_DIR}" + fi fi } @@ -342,10 +351,23 @@ apply_patches() { if [ "$failed" -gt 0 ]; then log_error "Failed to apply ${failed} patch(es)." echo "" - echo "The patch may have conflicts. You can try to resolve them manually:" - echo " 1. git checkout main" - echo " 2. git branch -D ${branch_name}" - echo " 3. Manually apply the changes from the upstream PR" + echo "Debug information:" + echo " - Patches directory: ${PATCH_DIR}" + echo " - Upstream repo clone: ${TEMP_DIR}/repo" + echo " - Current branch: ${branch_name}" + echo "" + echo "To investigate the failure:" + echo " 1. Examine the failed patch:" + echo " cat ${PATCH_DIR}/*.patch | less" + echo "" + echo " 2. Try applying manually to see the conflict:" + echo " git am --directory=${subdir} ${PATCH_DIR}/*.patch" + echo "" + echo " 3. If you want to start over:" + echo " git checkout main" + echo " git branch -D ${branch_name}" + echo " rm -rf ${TEMP_DIR}" + echo "" exit 1 fi @@ -396,6 +418,10 @@ main() { # Validate arguments validate_args "$@" + # Set temp directory with descriptive name + TEMP_DIR="${SCRIPT_DIR}/.import-pr-temp-${repo_name}-${pr_number}" + PATCH_DIR="${TEMP_DIR}/patches" + # Check dependencies check_dependencies echo "" From bd27dae16459d3502f191cfa3da714080ae160cd Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Sun, 7 Dec 2025 22:53:43 +0100 Subject: [PATCH 08/18] sigrok: initial sigrok driver --- .../reference/package-apis/drivers/index.md | 3 + .../reference/package-apis/drivers/sigrok.md | 1 + .../jumpstarter-driver-sigrok/.gitignore | 3 + .../jumpstarter-driver-sigrok/README.md | 110 +++++++ .../examples/exporter.yaml | 21 ++ .../jumpstarter_driver_sigrok/__init__.py | 5 + .../jumpstarter_driver_sigrok/client.py | 37 +++ .../jumpstarter_driver_sigrok/common.py | 49 ++++ .../jumpstarter_driver_sigrok/driver.py | 277 ++++++++++++++++++ .../jumpstarter_driver_sigrok/driver_test.py | 225 ++++++++++++++ .../jumpstarter-driver-sigrok/pyproject.toml | 42 +++ 11 files changed, 773 insertions(+) create mode 120000 python/docs/source/reference/package-apis/drivers/sigrok.md create mode 100644 python/packages/jumpstarter-driver-sigrok/.gitignore create mode 100644 python/packages/jumpstarter-driver-sigrok/README.md create mode 100644 python/packages/jumpstarter-driver-sigrok/examples/exporter.yaml create mode 100644 python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/__init__.py create mode 100644 python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/client.py create mode 100644 python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py create mode 100644 python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py create mode 100644 python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py create mode 100644 python/packages/jumpstarter-driver-sigrok/pyproject.toml diff --git a/python/docs/source/reference/package-apis/drivers/index.md b/python/docs/source/reference/package-apis/drivers/index.md index 2a5e173cc..c1db1e0ea 100644 --- a/python/docs/source/reference/package-apis/drivers/index.md +++ b/python/docs/source/reference/package-apis/drivers/index.md @@ -72,6 +72,8 @@ Drivers for debugging and programming devices: * **[QEMU](qemu.md)** (`jumpstarter-driver-qemu`) - QEMU virtualization platform * **[Corellium](corellium.md)** (`jumpstarter-driver-corellium`) - Corellium virtualization platform +* **[Sigrok](sigrok.md)** (`jumpstarter-driver-sigrok`) - Logic analyzer and + oscilloscope support via sigrok-cli * **[U-Boot](uboot.md)** (`jumpstarter-driver-uboot`) - Universal Bootloader interface * **[RideSX](ridesx.md)** (`jumpstarter-driver-ridesx`) - Flashing and power management for Qualcomm RideSX devices @@ -105,6 +107,7 @@ gpiod.md ridesx.md sdwire.md shell.md +sigrok.md ssh.md snmp.md tasmota.md diff --git a/python/docs/source/reference/package-apis/drivers/sigrok.md b/python/docs/source/reference/package-apis/drivers/sigrok.md new file mode 120000 index 000000000..979eeb03f --- /dev/null +++ b/python/docs/source/reference/package-apis/drivers/sigrok.md @@ -0,0 +1 @@ +../../../../../packages/jumpstarter-driver-sigrok/README.md \ No newline at end of file diff --git a/python/packages/jumpstarter-driver-sigrok/.gitignore b/python/packages/jumpstarter-driver-sigrok/.gitignore new file mode 100644 index 000000000..cbc5d672b --- /dev/null +++ b/python/packages/jumpstarter-driver-sigrok/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +.coverage +coverage.xml diff --git a/python/packages/jumpstarter-driver-sigrok/README.md b/python/packages/jumpstarter-driver-sigrok/README.md new file mode 100644 index 000000000..4c1a75b35 --- /dev/null +++ b/python/packages/jumpstarter-driver-sigrok/README.md @@ -0,0 +1,110 @@ +# Sigrok Driver + +`jumpstarter-driver-sigrok` wraps `sigrok-cli` to provide logic analyzer and oscilloscope capture from Jumpstarter exporters. It supports: +- **Logic analyzers** (digital channels) - with protocol decoding (SPI, I2C, UART, etc.) +- **Oscilloscopes** (analog channels) - voltage waveform capture +- One-shot and streaming capture +- Decoder-friendly channel mappings +- Real-time protocol decoding + +## Installation + +```shell +pip3 install --extra-index-url https://pkg.jumpstarter.dev/simple/ jumpstarter-driver-sigrok +``` + +## Configuration (exporter) + +```yaml +export: + sigrok: + type: jumpstarter_driver_sigrok.driver.Sigrok + driver: demo # sigrok driver (demo, fx2lafw, etc.) + conn: null # optional: USB VID.PID or serial path + executable: null # optional: path to sigrok-cli (auto-detected) + channels: # channel mappings (device_name: semantic_name) + D0: vcc + D1: cs + D2: miso + D3: mosi + D4: clk + D5: sda + D6: scl +``` + +## CaptureConfig (client-side) + +```python +from jumpstarter_driver_sigrok.common import CaptureConfig, DecoderConfig + +config = CaptureConfig( + sample_rate="8MHz", + samples=20000, + pretrigger=5000, + triggers={"cs": "falling"}, + decoders=[ + DecoderConfig( + name="spi", + channels={"clk": "clk", "mosi": "mosi", "miso": "miso", "cs": "cs"}, + annotations=["mosi-data"], + ) + ], +) +``` + +This maps to: +```bash +sigrok-cli -d fx2lafw -c samplerate=8MHz,samples=20000,pretrigger=5000 --triggers D1=falling \ + -P spi:clk=D4:mosi=D3:miso=D2:cs=D1 -A spi=mosi-data +``` + +## Client API + +- `scan()` — list devices for the configured driver +- `capture(config)` — one-shot capture, returns `CaptureResult` with base64 data +- `capture_stream(config)` — streaming capture via `--continuous` +- `get_driver_info()` — driver, conn, channel map +- `get_channel_map()` — device-to-semantic name mappings +- `list_output_formats()` — supported formats (csv, srzip, vcd, binary, bits, ascii) + +## Examples + +### Logic Analyzer (Digital Channels) + +One-shot with trigger: +```bash +sigrok-cli -d fx2lafw -c samplerate=8MHz,samples=20000,pretrigger=5000 --triggers D0=rising -o out.sr +``` + +Real-time decode (SPI): +```bash +sigrok-cli -d fx2lafw -c samplerate=1M --continuous \ + -P spi:clk=D4:mosi=D3:miso=D2:cs=D1 -A spi=mosi-data +``` + +### Oscilloscope (Analog Channels) + +```yaml +export: + oscilloscope: + type: jumpstarter_driver_sigrok.driver.Sigrok + driver: rigol-ds # or demo for testing + conn: usb # or serial path + channels: + A0: CH1 + A1: CH2 +``` + +```python +from jumpstarter_driver_sigrok.common import CaptureConfig + +# Capture analog waveforms +config = CaptureConfig( + sample_rate="1MHz", + samples=10000, + channels=["CH1", "CH2"], # Analog channels + output_format="csv", # or "vcd" for waveform viewers +) +result = client.capture(config) +waveform_data = result.data # bytes with voltage measurements +``` diff --git a/python/packages/jumpstarter-driver-sigrok/examples/exporter.yaml b/python/packages/jumpstarter-driver-sigrok/examples/exporter.yaml new file mode 100644 index 000000000..847a99b29 --- /dev/null +++ b/python/packages/jumpstarter-driver-sigrok/examples/exporter.yaml @@ -0,0 +1,21 @@ +apiVersion: jumpstarter.dev/v1alpha1 +kind: ExporterConfig +metadata: + namespace: default + name: demo +endpoint: grpc.jumpstarter.192.168.0.203.nip.io:8082 +token: "" +export: + sigrok: + type: jumpstarter_driver_sigrok.driver.Sigrok + driver: demo + conn: null + channels: + D0: vcc + D1: cs + D2: miso + D3: mosi + D4: clk + D5: sda + D6: scl + diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/__init__.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/__init__.py new file mode 100644 index 000000000..106d87ae4 --- /dev/null +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/__init__.py @@ -0,0 +1,5 @@ +from jumpstarter_driver_sigrok.common import CaptureConfig, CaptureResult, DecoderConfig +from jumpstarter_driver_sigrok.driver import Sigrok + +__all__ = ["Sigrok", "CaptureConfig", "CaptureResult", "DecoderConfig"] + diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/client.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/client.py new file mode 100644 index 000000000..1e1dc9d20 --- /dev/null +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/client.py @@ -0,0 +1,37 @@ +from dataclasses import dataclass + +from .common import CaptureConfig, CaptureResult +from jumpstarter.client import DriverClient + + +@dataclass(kw_only=True) +class SigrokClient(DriverClient): + """Client methods for the Sigrok driver.""" + + def scan(self) -> str: + return self.call("scan") + + def capture(self, config: CaptureConfig | dict) -> CaptureResult: + return CaptureResult.model_validate(self.call("capture", config)) + + def capture_stream(self, config: CaptureConfig | dict): + """Stream capture data from sigrok-cli. + + Args: + config: CaptureConfig or dict with capture parameters + + Yields: + bytes: Chunks of captured data + """ + for chunk in self.streamingcall("capture_stream", config): + yield chunk + + def get_driver_info(self) -> dict: + return self.call("get_driver_info") + + def get_channel_map(self) -> dict: + return self.call("get_channel_map") + + def list_output_formats(self) -> list[str]: + return self.call("list_output_formats") + diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py new file mode 100644 index 000000000..cc0110e15 --- /dev/null +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, Field + + +class DecoderConfig(BaseModel): + """Protocol decoder configuration (real-time during capture).""" + + name: str + channels: dict[str, str] | None = None + options: dict[str, Any] | None = None + annotations: list[str] | None = None + stack: list["DecoderConfig"] | None = None + + +class CaptureConfig(BaseModel): + sample_rate: str = Field(default="1M", description="e.g., 8MHz, 1M, 24000000") + samples: int | None = Field(default=None, description="number of samples; None for continuous") + pretrigger: int | None = Field(default=None, description="samples before trigger") + triggers: dict[str, str] | None = Field(default=None, description="e.g., {'D0': 'rising'}") + channels: list[str] | None = Field(default=None, description="override default channels by name") + output_format: str = Field( + default="srzip", + description="csv, srzip, vcd, binary, bits, ascii", + ) + decoders: list[DecoderConfig] | None = Field(default=None, description="real-time protocol decoding") + + +class CaptureResult(BaseModel): + """Result from a capture operation. + + Note: data is base64-encoded for reliable JSON transport. Client methods + automatically decode it to bytes for you. + """ + data_b64: str # Base64-encoded binary data + output_format: str + sample_rate: str + channel_map: dict[str, str] + triggers: dict[str, str] | None = None + decoders: list[DecoderConfig] | None = None + + @property + def data(self) -> bytes: + """Get the captured data as bytes (auto-decodes from base64).""" + from base64 import b64decode + return b64decode(self.data_b64) + diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py new file mode 100644 index 000000000..92081b5c6 --- /dev/null +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py @@ -0,0 +1,277 @@ +from __future__ import annotations + +import asyncio +import subprocess +from base64 import b64encode +from dataclasses import dataclass, field +from pathlib import Path +from shutil import which +from tempfile import TemporaryDirectory + +from .common import CaptureConfig, DecoderConfig +from jumpstarter.driver import Driver, export + + +def find_sigrok_cli() -> str: + executable = which("sigrok-cli") + if executable is None: + raise FileNotFoundError("sigrok-cli executable not found in PATH") + return executable + + +def _default_channel_map() -> dict[str, str]: + # Decoder-friendly default names for demo driver + # Maps device channel name -> semantic name + return {"D0": "vcc", "D1": "cs", "D2": "miso", "D3": "mosi", "D4": "clk", "D5": "sda", "D6": "scl"} + + +@dataclass(kw_only=True) +class Sigrok(Driver): + """Sigrok driver wrapping sigrok-cli for logic analyzer and oscilloscope support.""" + + driver: str = "demo" + conn: str | None = None + executable: str = field(default_factory=find_sigrok_cli) + channels: dict[str, str] = field(default_factory=_default_channel_map) + + def __post_init__(self): + if hasattr(super(), "__post_init__"): + super().__post_init__() + + @classmethod + def client(cls) -> str: + return "jumpstarter_driver_sigrok.client.SigrokClient" + + # --- Public API ----------------------------------------------------- + + @export + def scan(self) -> str: + """List devices for the configured driver.""" + cmd = [self.executable, "--driver", self.driver, "--scan"] + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + return result.stdout + + @export + def get_driver_info(self) -> dict: + return { + "driver": self.driver, + "conn": self.conn, + "channels": self.channels, + } + + @export + def get_channel_map(self) -> dict[int, str]: + return self.channels + + @export + def list_output_formats(self) -> list[str]: + return ["csv", "srzip", "vcd", "binary", "bits", "ascii"] + + @export + def capture(self, config: CaptureConfig | dict) -> dict: + """One-shot capture; returns dict with base64-encoded binary data.""" + cfg = CaptureConfig.model_validate(config) + cmd, outfile, tmpdir = self._build_capture_command(cfg) + + try: + self.logger.debug("running sigrok-cli: %s", " ".join(cmd)) + subprocess.run(cmd, check=True) + + data = outfile.read_bytes() + # Return as dict with base64-encoded data (reliable for JSON transport) + return { + "data_b64": b64encode(data).decode("ascii"), + "output_format": cfg.output_format, + "sample_rate": cfg.sample_rate, + "channel_map": self.channels, + "triggers": cfg.triggers, + "decoders": [d.model_dump() for d in cfg.decoders] if cfg.decoders else None, + } + finally: + tmpdir.cleanup() + + @export + async def capture_stream(self, config: CaptureConfig | dict): + """Streaming capture; yields chunks of binary data from sigrok-cli stdout.""" + cfg = CaptureConfig.model_validate(config) + cmd = self._build_stream_command(cfg) + + self.logger.debug("streaming sigrok-cli: %s", " ".join(cmd)) + process = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + try: + if process.stdout is None: + raise RuntimeError("sigrok-cli stdout not available") + + # Stream data in chunks + while True: + chunk = await process.stdout.read(4096) + if not chunk: + break + yield chunk + finally: + process.terminate() + try: + await asyncio.wait_for(process.wait(), timeout=5) + except asyncio.TimeoutError: + process.kill() + + # --- Command builders ----------------------------------------------- + + def _build_capture_command(self, cfg: CaptureConfig) -> tuple[list[str], Path, TemporaryDirectory]: + tmpdir = TemporaryDirectory() + outfile = Path(tmpdir.name) / f"capture.{cfg.output_format}" + + cmd: list[str] = self._base_driver_args() + cmd += self._channel_args(cfg.channels) + cmd += self._config_args(cfg) + cmd += self._trigger_args(cfg) + cmd += self._decoder_args(cfg) + cmd += ["-O", cfg.output_format, "-o", str(outfile)] + + return cmd, outfile, tmpdir + + def _build_stream_command(self, cfg: CaptureConfig) -> list[str]: + cmd: list[str] = self._base_driver_args() + cmd += self._channel_args(cfg.channels) + cmd += self._config_args(cfg, continuous=True) + cmd += self._trigger_args(cfg) + cmd += self._decoder_args(cfg) + cmd += ["-O", cfg.output_format, "-o", "-"] + return cmd + + def _base_driver_args(self) -> list[str]: + if self.conn: + return [self.executable, "-d", f"{self.driver}:conn={self.conn}"] + return [self.executable, "-d", self.driver] + + def _channel_args(self, selected_names: list[str] | None) -> list[str]: + """Build channel selection/renaming args for sigrok-cli. + + Args: + selected_names: Optional list of semantic names to include + + Returns: + List of args like ["-C", "D0=vcc,D1=cs,D2=miso"] + """ + if not self.channels: + return [] + + # Filter channels if specific names requested + if selected_names: + selected_lower = {name.lower() for name in selected_names} + filtered = {dev: user for dev, user in self.channels.items() if user.lower() in selected_lower} + else: + filtered = self.channels + + # Build channel map: device_name=user_name + channel_map = ",".join(f"{dev}={user}" for dev, user in filtered.items()) + return ["-C", channel_map] if channel_map else [] + + def _config_args(self, cfg: CaptureConfig, *, continuous: bool = False) -> list[str]: + parts = [f"samplerate={cfg.sample_rate}"] + if cfg.pretrigger is not None: + parts.append(f"pretrigger={cfg.pretrigger}") + + args: list[str] = [] + if parts: + args += ["-c", ",".join(parts)] + + # sigrok-cli requires one of: --samples, --frames, --time, or --continuous + # If samples is explicitly specified, use that even for streaming + if cfg.samples is not None: + args.extend(["--samples", str(cfg.samples)]) + elif continuous: + args.append("--continuous") + else: + # Default to 1000 samples if not specified + args.extend(["--samples", "1000"]) + + return args + + def _trigger_args(self, cfg: CaptureConfig) -> list[str]: + if not cfg.triggers: + return [] + trigger_parts = [] + for channel, condition in cfg.triggers.items(): + resolved = self._resolve_channel(channel) + trigger_parts.append(f"{resolved}={condition}") + return ["--triggers", ",".join(trigger_parts)] + + def _decoder_args(self, cfg: CaptureConfig) -> list[str]: + if not cfg.decoders: + return [] + + args: list[str] = [] + for decoder in self._flatten_decoders(cfg.decoders): + pin_map = self._resolve_decoder_channels(decoder) + segments = [decoder.name] + + for pin_name, channel_name in pin_map.items(): + segments.append(f"{pin_name}={self._resolve_channel(channel_name)}") + + if decoder.options: + for key, value in decoder.options.items(): + segments.append(f"{key}={value}") + + args += ["-P", ":".join(segments)] + + if decoder.annotations: + args += ["-A", f"{decoder.name}=" + ",".join(decoder.annotations)] + + return args + + def _flatten_decoders(self, decoders: list[DecoderConfig]) -> list[DecoderConfig]: + flat: list[DecoderConfig] = [] + for decoder in decoders: + flat.append(decoder) + if decoder.stack: + flat.extend(self._flatten_decoders(decoder.stack)) + return flat + + def _resolve_decoder_channels(self, decoder: DecoderConfig) -> dict[str, str]: + if decoder.channels: + return decoder.channels + + # Best-effort auto-mapping based on common decoder pin names + defaults = { + "spi": ["clk", "mosi", "miso", "cs"], + "i2c": ["scl", "sda"], + "uart": ["rx", "tx"], + } + pins = defaults.get(decoder.name.lower()) + if not pins: + return {} + + resolved: dict[str, str] = {} + available_lower = {name.lower(): name for name in self.channels.values()} + for pin in pins: + if pin in available_lower: + resolved[pin] = available_lower[pin] + return resolved + + def _resolve_channel(self, name_or_dn: str) -> str: + """Resolve a user-friendly channel name to device channel name. + + Args: + name_or_dn: User-friendly name (e.g., "clk", "mosi") or device name (e.g., "D0") + + Returns: + Device channel name (e.g., "D0", "D1") + """ + candidate = name_or_dn.strip() + + # If already a device channel name, return as-is + if candidate in self.channels: + return candidate + + # Search for user-friendly name in channel values + for dev_name, user_name in self.channels.items(): + if user_name.lower() == candidate.lower(): + return dev_name + + raise ValueError(f"Channel '{name_or_dn}' not found in channel map {self.channels}") diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py new file mode 100644 index 000000000..ea736288f --- /dev/null +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py @@ -0,0 +1,225 @@ +from shutil import which + +import pytest + +from .common import CaptureConfig, CaptureResult +from .driver import Sigrok +from jumpstarter.common.utils import serve + + +@pytest.fixture +def demo_driver_instance(): + """Create a Sigrok driver instance configured for the demo device.""" + # Demo driver has 8 digital channels (D0-D7) and 5 analog (A0-A4) + # Map device channels to decoder-friendly semantic names + return Sigrok( + driver="demo", + channels={ + "D0": "vcc", + "D1": "cs", + "D2": "miso", + "D3": "mosi", + "D4": "clk", + "D5": "sda", + "D6": "scl", + "D7": "gnd", + }, + ) + + +@pytest.fixture +def demo_client(demo_driver_instance): + """Create a client connected to demo driver via serve().""" + with serve(demo_driver_instance) as client: + yield client + + +@pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") +def test_scan_demo_driver(demo_client): + """Test scanning for demo driver via client.""" + result = demo_client.scan() + assert "demo" in result.lower() or "Demo device" in result + + +@pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") +def test_capture_with_demo_driver(demo_client): + """Test one-shot capture with demo driver via client. + + This test verifies client-server serialization through serve() pattern. + """ + cfg = CaptureConfig( + sample_rate="100kHz", + samples=100, + output_format="srzip", + ) + + result = demo_client.capture(cfg) + + # Verify we got a proper CaptureResult Pydantic model, not just a dict + assert isinstance(result, CaptureResult), f"Expected CaptureResult, got {type(result)}" + + # Verify model attributes work correctly - data should be bytes, not base64 string! + assert result.data + assert isinstance(result.data, bytes), f"Expected bytes, got {type(result.data)}" + assert len(result.data) > 0 + assert result.output_format == "srzip" + assert result.sample_rate == "100kHz" + assert isinstance(result.channel_map, dict) + assert len(result.channel_map) > 0 + + +@pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") +def test_capture_csv_format(demo_client): + """Test capture with CSV output format via client.""" + cfg = CaptureConfig( + sample_rate="50kHz", + samples=50, + output_format="csv", + ) + + result = demo_client.capture(cfg) + + # Verify CaptureResult model + assert isinstance(result, CaptureResult) + assert isinstance(result.data, bytes) + + # Decode bytes to string for CSV parsing + csv_text = result.data.decode("utf-8") + + # CSV should have headers and data + assert "vcc" in csv_text or "cs" in csv_text or "clk" in csv_text + + +@pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") +def test_capture_analog_channels(): + """Test capturing analog data from oscilloscope/demo driver. + + Verifies that the API works for analog channels (oscilloscopes) + as well as digital channels (logic analyzers). + """ + # Create driver with analog channel mappings + analog_driver = Sigrok( + driver="demo", + channels={ + "A0": "voltage_in", + "A1": "sine_wave", + "A2": "square_wave", + }, + ) + + with serve(analog_driver) as client: + cfg = CaptureConfig( + sample_rate="100kHz", + samples=20, + channels=["voltage_in", "sine_wave"], # Select specific analog channels + output_format="csv", + ) + + result = client.capture(cfg) + + # Verify we got analog data + assert isinstance(result, CaptureResult) + assert isinstance(result.data, bytes) + + # Parse CSV to check for analog voltage values + csv_text = result.data.decode("utf-8") + + # Should contain voltage values with units (V, mV) + assert "V" in csv_text or "mV" in csv_text + # Should contain our channel names or original analog channel names + assert "voltage_in" in csv_text or "sine_wave" in csv_text or "A0" in csv_text or "A1" in csv_text + + +@pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") +def test_capture_with_dict_config(demo_client): + """Test capture with dict config (not CaptureConfig object). + + Verifies that dict configs are properly validated and serialized. + """ + # Pass config as dict instead of CaptureConfig object + cfg_dict = { + "sample_rate": "100kHz", + "samples": 100, + "output_format": "srzip", + } + + result = demo_client.capture(cfg_dict) + + # Verify we still get a proper CaptureResult model + assert isinstance(result, CaptureResult) + assert result.data + assert isinstance(result.data, bytes) + assert len(result.data) > 0 + assert result.output_format == "srzip" + + +@pytest.mark.skip(reason="sigrok-cli demo driver doesn't support streaming to stdout (-o -)") +def test_capture_stream_with_demo(demo_client): + """Test streaming capture with demo driver via client. + + Note: sigrok-cli has limitations with streaming output to stdout. + The demo driver and most output formats don't produce data when using `-o -`. + This feature works better with real hardware and certain output formats. + """ + cfg = CaptureConfig( + sample_rate="100kHz", + samples=1000, + output_format="binary", + ) + + received_bytes = 0 + chunk_count = 0 + + # Collect all chunks + for chunk in demo_client.capture_stream(cfg): + received_bytes += len(chunk) + chunk_count += 1 + + # Should have received some data + assert received_bytes > 0 + assert chunk_count > 0 + + +def test_get_driver_info(demo_client): + """Test getting driver information via client. + + Verifies dict serialization through client-server boundary. + """ + info = demo_client.get_driver_info() + + # Verify it's a dict (not a custom object) + assert isinstance(info, dict) + assert info["driver"] == "demo" + assert "channels" in info + assert isinstance(info["channels"], dict) + + +def test_get_channel_map(demo_client): + """Test getting channel mappings via client. + + Verifies dict serialization through client-server boundary. + """ + channels = demo_client.get_channel_map() + + # Verify it's a dict with proper string keys/values + assert isinstance(channels, dict) + assert all(isinstance(k, str) and isinstance(v, str) for k, v in channels.items()) + assert channels["D0"] == "vcc" + assert channels["D4"] == "clk" + assert channels["D7"] == "gnd" + + +def test_list_output_formats(demo_client): + """Test listing supported output formats via client. + + Verifies list serialization through client-server boundary. + """ + formats = demo_client.list_output_formats() + + # Verify it's a proper list of strings + assert isinstance(formats, list) + assert all(isinstance(f, str) for f in formats) + assert "csv" in formats + assert "srzip" in formats + assert "vcd" in formats + assert "binary" in formats diff --git a/python/packages/jumpstarter-driver-sigrok/pyproject.toml b/python/packages/jumpstarter-driver-sigrok/pyproject.toml new file mode 100644 index 000000000..f6cd63aa4 --- /dev/null +++ b/python/packages/jumpstarter-driver-sigrok/pyproject.toml @@ -0,0 +1,42 @@ +[project] +name = "jumpstarter-driver-sigrok" +dynamic = ["version", "urls"] +description = "Jumpstarter driver wrapping sigrok-cli for logic analyzer and oscilloscope support" +readme = "README.md" +license = "Apache-2.0" +authors = [ + { name = "Miguel Angel Ajo Pelayo", email = "miguelangel@ajo.es" } +] +requires-python = ">=3.11" +dependencies = [ + "jumpstarter", +] + +[tool.hatch.version] +source = "vcs" +raw-options = { 'root' = '../../'} + +[tool.hatch.metadata.hooks.vcs.urls] +Homepage = "https://jumpstarter.dev" +source_archive = "https://github.com/jumpstarter-dev/repo/archive/{commit_hash}.zip" + +[tool.pytest.ini_options] +addopts = "--cov --cov-report=html --cov-report=xml" +log_cli = true +log_cli_level = "INFO" +testpaths = ["jumpstarter_driver_sigrok"] +asyncio_default_fixture_loop_scope = "function" + +[build-system] +requires = ["hatchling", "hatch-vcs", "hatch-pin-jumpstarter"] +build-backend = "hatchling.build" + +[tool.hatch.build.hooks.pin_jumpstarter] +name = "pin_jumpstarter" + +[dependency-groups] +dev = [ + "pytest-cov>=6.0.0", + "pytest>=8.3.3", + "pytest-asyncio>=0.24.0", +] From c3a5957a489bcce08a33abbc26c5616e905c792c Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Mon, 8 Dec 2025 11:32:57 +0100 Subject: [PATCH 09/18] sigrok: csv and vcd parsing --- .../jumpstarter_driver_sigrok/__init__.py | 10 +- .../jumpstarter_driver_sigrok/common.py | 77 ++++++ .../jumpstarter_driver_sigrok/csv.py | 141 +++++++++++ .../jumpstarter_driver_sigrok/csv_test.py | 132 ++++++++++ .../jumpstarter_driver_sigrok/driver.py | 4 +- .../jumpstarter_driver_sigrok/driver_test.py | 218 ++++++++++++++++ .../jumpstarter_driver_sigrok/vcd.py | 226 +++++++++++++++++ .../jumpstarter_driver_sigrok/vcd_test.py | 232 ++++++++++++++++++ python/uv.lock | 25 ++ 9 files changed, 1061 insertions(+), 4 deletions(-) create mode 100644 python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv.py create mode 100644 python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv_test.py create mode 100644 python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd.py create mode 100644 python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd_test.py diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/__init__.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/__init__.py index 106d87ae4..7b134cb86 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/__init__.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/__init__.py @@ -1,5 +1,11 @@ -from jumpstarter_driver_sigrok.common import CaptureConfig, CaptureResult, DecoderConfig +from jumpstarter_driver_sigrok.common import ( + CaptureConfig, + CaptureResult, + DecoderConfig, + OutputFormat, + Sample, +) from jumpstarter_driver_sigrok.driver import Sigrok -__all__ = ["Sigrok", "CaptureConfig", "CaptureResult", "DecoderConfig"] +__all__ = ["Sigrok", "CaptureConfig", "CaptureResult", "DecoderConfig", "OutputFormat", "Sample"] diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py index cc0110e15..aae5e669d 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py @@ -5,6 +5,27 @@ from pydantic import BaseModel, Field +class OutputFormat: + """Constants for sigrok output formats.""" + CSV = "csv" + BITS = "bits" + ASCII = "ascii" + BINARY = "binary" + SRZIP = "srzip" + VCD = "vcd" + + @classmethod + def all(cls) -> list[str]: + return [cls.CSV, cls.BITS, cls.ASCII, cls.BINARY, cls.SRZIP, cls.VCD] + + +class Sample(BaseModel): + """A single sample with timing information.""" + sample: int # Sample index + time_ns: int # Time in nanoseconds + values: dict[str, int | float] # Channel values (digital: 0/1, analog: voltage) + + class DecoderConfig(BaseModel): """Protocol decoder configuration (real-time during capture).""" @@ -47,3 +68,59 @@ def data(self) -> bytes: from base64 import b64decode return b64decode(self.data_b64) + def decode(self) -> list[Sample] | dict[str, list[int]] | str: + """Parse captured data based on output format. + + Returns: + - CSV format: list[Sample] with timing and all values per sample + - VCD format: list[Sample] with timing and only changed values + - Bits format: dict[str, list[int]] with channel→bit sequences + - ASCII format: str with ASCII art visualization + - Other formats: raises NotImplementedError (use .data for raw bytes) + + Raises: + NotImplementedError: For binary/srzip formats (use .data property) + """ + if self.output_format == OutputFormat.CSV: + from .csv import parse_csv + samples_data = parse_csv(self.data, self.sample_rate) + return [Sample.model_validate(s) for s in samples_data] + elif self.output_format == OutputFormat.VCD: + from .vcd import parse_vcd + samples_data = parse_vcd(self.data, self.sample_rate) + return [Sample.model_validate(s) for s in samples_data] + elif self.output_format == OutputFormat.BITS: + return self._parse_bits() + elif self.output_format == OutputFormat.ASCII: + return self.data.decode("utf-8") + else: + raise NotImplementedError( + f"Parsing not implemented for {self.output_format} format. " + f"Use .data property to get raw bytes." + ) + + def _parse_bits(self) -> dict[str, list[int]]: + """Parse bits format to dict of channel→bit sequences.""" + text = self.data.decode("utf-8") + lines = [line.strip() for line in text.strip().split("\n") if line.strip()] + + # bits format is just columns of 0/1 + # TODO: Need to determine channel mapping from somewhere + # For now, return as generic numbered channels + result: dict[str, list[int]] = {} + + for line in lines: + # Each line might be space/comma separated bits + bits = [int(b) for b in line if b in "01"] + if not result: + # Initialize channels + for i, bit in enumerate(bits): + result[f"CH{i}"] = [bit] + else: + # Append to existing channels + for i, bit in enumerate(bits): + if f"CH{i}" in result: + result[f"CH{i}"].append(bit) + + return result + diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv.py new file mode 100644 index 000000000..1a7626d06 --- /dev/null +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv.py @@ -0,0 +1,141 @@ +"""CSV format parser for sigrok captures.""" + +from __future__ import annotations + +import csv + + +def parse_csv(data: bytes, sample_rate: str) -> list[dict]: + """Parse CSV format to list of samples with timing. + + Args: + data: Raw CSV data as bytes + sample_rate: Sample rate string (e.g., "100kHz", "1MHz") + + Returns: + List of dicts with keys: sample, time_ns, values + """ + text = data.decode("utf-8") + lines = text.strip().split("\n") + + # Parse sample rate for timing calculation + sample_rate_hz = _parse_sample_rate_hz(sample_rate) + time_step_ns = int(1_000_000_000.0 / sample_rate_hz) + + # Skip comment lines and analog preview lines (format: "A0: -10.0000 V DC") + # The actual data starts after a header row with types like "logic,logic,V DC,V DC" + data_lines = _extract_csv_data_lines(lines) + + if not data_lines or len(data_lines) < 2: + return [] + + # Parse the CSV data + reader = csv.reader(data_lines) + + # First row is types (logic, V DC, etc.) - use for channel name inference + types_row = next(reader) + + # Get channel names from types + channel_names = _infer_channel_names(types_row) + + # Parse data rows + samples: list[dict] = [] + for idx, row in enumerate(reader): + values = _parse_csv_row(channel_names, row) + samples.append({ + "sample": idx, + "time_ns": idx * time_step_ns, + "values": values, + }) + + return samples + + +def _parse_sample_rate_hz(sample_rate: str) -> float: + """Parse sample rate string to Hz.""" + rate = sample_rate.strip().upper() + multipliers = {"K": 1e3, "M": 1e6, "G": 1e9} + + for suffix, mult in multipliers.items(): + if rate.endswith(f"{suffix}HZ"): + return float(rate[:-3]) * mult + elif rate.endswith(suffix): + return float(rate[:-1]) * mult + + # Assume Hz if no suffix + return float(rate.rstrip("HZ")) + + +def _extract_csv_data_lines(lines: list[str]) -> list[str]: + """Extract actual CSV data lines, skipping comments and analog preview lines.""" + data_lines = [] + + for _i, line in enumerate(lines): + line = line.strip() + # Skip comment lines + if line.startswith(";"): + continue + # Skip analog preview lines (contain colon, not CSV comma-separated) + if ":" in line and "," not in line: + continue + # This is CSV data + data_lines.append(line) + + return data_lines + + +def _infer_channel_names(types_row: list[str]) -> list[str]: + """Infer channel names from CSV type header row. + + Args: + types_row: List of type strings like ["logic", "logic", "V DC", "V DC"] + + Returns: + List of channel names like ["D0", "D1", "A0", "A1"] + """ + channel_names = [] + digital_count = 0 + analog_count = 0 + + for type_str in types_row: + type_lower = type_str.lower() + if "logic" in type_lower: + channel_names.append(f"D{digital_count}") + digital_count += 1 + elif "v" in type_lower or "dc" in type_lower: + # Analog channel + channel_names.append(f"A{analog_count}") + analog_count += 1 + else: + # Unknown type, use generic name + channel_names.append(f"CH{len(channel_names)}") + + return channel_names + + +def _parse_csv_row(channel_names: list[str], row: list[str]) -> dict[str, int | float]: + """Parse a CSV data row into channel values. + + Args: + channel_names: List of channel names + row: List of value strings + + Returns: + Dict mapping channel name to parsed value + """ + values = {} + + for channel, value in zip(channel_names, row, strict=True): + value = value.strip() + # Try to parse as number (analog) or binary (digital) + try: + if "." in value or "e" in value.lower(): + values[channel] = float(value) + else: + values[channel] = int(value) + except ValueError: + # Keep as string if not a number + values[channel] = value + + return values + diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv_test.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv_test.py new file mode 100644 index 000000000..f0589e962 --- /dev/null +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv_test.py @@ -0,0 +1,132 @@ +"""Tests for CSV format parser.""" + +from shutil import which + +import pytest + +from .client import SigrokClient +from .common import CaptureConfig, CaptureResult, OutputFormat +from .driver import Sigrok +from jumpstarter.common.utils import serve + + +@pytest.fixture +def demo_driver_instance(): + """Create a Sigrok driver instance configured for the demo device.""" + # Demo driver has 8 digital channels (D0-D7) and 5 analog (A0-A4) + # Map device channels to decoder-friendly semantic names + return Sigrok( + driver="demo", + executable="sigrok-cli", + channels={ + "D0": "vcc", + "D1": "cs", + "D2": "miso", + "D3": "mosi", + "D4": "clk", + "D5": "sda", + "D6": "scl", + "D7": "gnd", + }, + ) + + +@pytest.fixture +def demo_client(demo_driver_instance): + """Create a client for the demo Sigrok driver.""" + with serve(demo_driver_instance) as client: + yield client + + +@pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") +def test_csv_format_basic(demo_client: SigrokClient): + """Test CSV format capture with demo driver.""" + cfg = CaptureConfig( + sample_rate="50kHz", + samples=50, + output_format=OutputFormat.CSV, + channels=["vcc", "cs"], # Select specific digital channels + ) + + result = demo_client.capture(cfg) + assert isinstance(result, CaptureResult) + assert isinstance(result.data, bytes) + decoded_data = result.decode() + assert isinstance(decoded_data, list) + assert len(decoded_data) > 0 + # Verify channel names are in the data + first_sample = decoded_data[0] + assert "D0" in first_sample.values or "D1" in first_sample.values + + +@pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") +def test_csv_format_timing(demo_client: SigrokClient): + """Test CSV format timing calculations with integer nanoseconds.""" + cfg = CaptureConfig( + sample_rate="100kHz", + samples=50, + output_format=OutputFormat.CSV, + channels=["D0", "D1", "D2"], # Select specific channels + ) + + result = demo_client.capture(cfg) + assert isinstance(result, CaptureResult) + + # Decode the CSV data + samples = result.decode() + assert isinstance(samples, list) + assert len(samples) > 0 + + # Verify timing progresses correctly + for sample in samples: + assert isinstance(sample.time_ns, int) + # Verify timing progresses (1/100kHz = 10,000ns per sample) + assert sample.time_ns == sample.sample * 10_000 + + +@pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") +def test_csv_format_analog_channels(demo_client: SigrokClient): + """Test CSV capture of analog channels with voltage values.""" + cfg = CaptureConfig( + sample_rate="100kHz", + samples=20, + output_format=OutputFormat.CSV, + channels=["A0", "A1"], # Select specific analog channels + ) + + result = demo_client.capture(cfg) + assert isinstance(result, CaptureResult) + assert isinstance(result.data, bytes) + decoded_data = result.decode() + assert isinstance(decoded_data, list) + assert len(decoded_data) > 0 + + # Check first sample for analog values + first_sample = decoded_data[0] + assert len(first_sample.values) > 0 + + # Analog values should be floats (voltages) + for _channel, value in first_sample.values.items(): + assert isinstance(value, (int, float)) + + +@pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") +def test_csv_format_mixed_channels(demo_client: SigrokClient): + """Test CSV with both digital and analog channels.""" + cfg = CaptureConfig( + sample_rate="100kHz", + samples=30, + output_format=OutputFormat.CSV, + channels=["D0", "D1", "A0"], # Mix of digital and analog + ) + + result = demo_client.capture(cfg) + samples = result.decode() + + assert isinstance(samples, list) + assert len(samples) > 0 + + # Verify we have values for channels + first_sample = samples[0] + assert len(first_sample.values) > 0 + diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py index 92081b5c6..852cd1333 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py @@ -8,7 +8,7 @@ from shutil import which from tempfile import TemporaryDirectory -from .common import CaptureConfig, DecoderConfig +from .common import CaptureConfig, DecoderConfig, OutputFormat from jumpstarter.driver import Driver, export @@ -65,7 +65,7 @@ def get_channel_map(self) -> dict[int, str]: @export def list_output_formats(self) -> list[str]: - return ["csv", "srzip", "vcd", "binary", "bits", "ascii"] + return OutputFormat.all() @export def capture(self, config: CaptureConfig | dict) -> dict: diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py index ea736288f..2766a36c8 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py @@ -223,3 +223,221 @@ def test_list_output_formats(demo_client): assert "srzip" in formats assert "vcd" in formats assert "binary" in formats + + +@pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") +def test_decode_csv_format(demo_client): + """Test decoding CSV format to Sample objects with timing. + + Verifies: + - CSV parsing works through client-server boundary + - Sample objects have timing information + - Values are properly typed (int/float) + """ + from .common import OutputFormat, Sample + + cfg = CaptureConfig( + sample_rate="100kHz", + samples=50, + output_format=OutputFormat.CSV, + channels=["D0", "D1", "D2"], # Select specific channels + ) + + result = demo_client.capture(cfg) + assert isinstance(result, CaptureResult) + + # Decode the CSV data + samples = result.decode() + assert isinstance(samples, list) + assert len(samples) > 0 + + # Verify all samples are Sample objects + for sample in samples: + assert isinstance(sample, Sample) + assert isinstance(sample.sample, int) + assert isinstance(sample.time_ns, int) + assert isinstance(sample.values, dict) + + # Verify timing progresses (1/100kHz = 10,000ns per sample) + assert sample.time_ns == sample.sample * 10_000 + + # Verify values are present + assert len(sample.values) > 0 + + +@pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") +def test_decode_ascii_format(demo_client): + """Test decoding ASCII format returns string visualization. + + Verifies: + - ASCII format decoding works + - Returns string (not bytes) + """ + from .common import OutputFormat + + cfg = CaptureConfig( + sample_rate="50kHz", + samples=20, + output_format=OutputFormat.ASCII, + channels=["D0", "D1"], + ) + + result = demo_client.capture(cfg) + decoded = result.decode() + + # ASCII format should return string + assert isinstance(decoded, str) + assert len(decoded) > 0 + + +@pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") +def test_decode_bits_format(demo_client): + """Test decoding bits format to channel→bit sequences. + + Verifies: + - Bits format decoding works + - Returns dict with bit sequences + """ + from .common import OutputFormat + + cfg = CaptureConfig( + sample_rate="100kHz", + samples=30, + output_format=OutputFormat.BITS, + channels=["D0", "D1", "D2"], + ) + + result = demo_client.capture(cfg) + decoded = result.decode() + + # Bits format should return dict + assert isinstance(decoded, dict) + assert len(decoded) > 0 + + # Each channel should have a list of bits + for channel, bits in decoded.items(): + assert isinstance(channel, str) + assert isinstance(bits, list) + assert all(b in [0, 1] for b in bits) + + +@pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") +def test_decode_vcd_format(demo_client): + """Test decoding VCD format to Sample objects with timing (changes only). + + Verifies: + - VCD parsing works through client-server boundary + - Sample objects have timing information in nanoseconds + - Only changes are recorded (efficient representation) + """ + from .common import OutputFormat, Sample + + cfg = CaptureConfig( + sample_rate="100kHz", + samples=50, + output_format=OutputFormat.VCD, + channels=["D0", "D1", "D2"], # Select specific channels + ) + + result = demo_client.capture(cfg) + assert isinstance(result, CaptureResult) + + # Decode the VCD data + samples = result.decode() + assert isinstance(samples, list) + assert len(samples) > 0 + + # Verify all samples are Sample objects + for sample in samples: + assert isinstance(sample, Sample) + assert isinstance(sample.sample, int) + assert isinstance(sample.time_ns, int) + assert isinstance(sample.values, dict) + + # VCD only records changes, so each sample should have at least one value + assert len(sample.values) > 0 + + # Values should be integers for digital channels + for _channel, value in sample.values.items(): + assert isinstance(value, int) + + +@pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") +def test_decode_vcd_analog_channels(demo_client): + """Test decoding VCD with analog channels. + + Verifies: + - Analog values are parsed correctly in VCD format + - Timing information is in nanoseconds + """ + from .common import OutputFormat, Sample + + cfg = CaptureConfig( + sample_rate="100kHz", + samples=30, + output_format=OutputFormat.VCD, + channels=["A0", "A1"], # Analog channels + ) + + result = demo_client.capture(cfg) + samples = result.decode() + + assert isinstance(samples, list) + assert len(samples) > 0 + + # Check that samples have analog values + first_sample = samples[0] + assert isinstance(first_sample, Sample) + assert isinstance(first_sample.time_ns, int) + assert len(first_sample.values) > 0 + + +@pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") +def test_decode_unsupported_format_raises(demo_client): + """Test that decoding unsupported formats raises NotImplementedError.""" + from .common import OutputFormat + + cfg = CaptureConfig( + sample_rate="100kHz", + samples=10, + output_format=OutputFormat.BINARY, + ) + + result = demo_client.capture(cfg) + + # Binary format should not be decodable + with pytest.raises(NotImplementedError): + result.decode() + + +@pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") +def test_decode_analog_csv(demo_client): + """Test decoding CSV with analog channels (voltage values). + + Verifies: + - Analog values are parsed as floats + - Timing information is included + """ + from .common import OutputFormat, Sample + + cfg = CaptureConfig( + sample_rate="100kHz", + samples=30, + output_format=OutputFormat.CSV, + channels=["A0", "A1"], # Analog channels + ) + + result = demo_client.capture(cfg) + samples = result.decode() + + assert isinstance(samples, list) + assert len(samples) > 0 + + # Check first sample for analog values + first_sample = samples[0] + assert isinstance(first_sample, Sample) + assert len(first_sample.values) > 0 + + # Analog values should be floats (voltages) + for _channel, value in first_sample.values.items(): + assert isinstance(value, (int, float)) diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd.py new file mode 100644 index 000000000..c777e79d0 --- /dev/null +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd.py @@ -0,0 +1,226 @@ +"""VCD (Value Change Dump) format parser for sigrok captures.""" + +from __future__ import annotations + + +def parse_vcd(data: bytes, sample_rate: str) -> list[dict]: + """Parse VCD format to list of samples with timing (changes only). + + VCD format only records when signals change, making it efficient for + sparse data. Each sample represents a time point where one or more + signals changed. + + Args: + data: Raw VCD data as bytes + sample_rate: Sample rate string (not used for VCD as it has its own timescale) + + Returns: + List of dicts with keys: sample, time_ns, values + """ + text = data.decode("utf-8") + lines = text.strip().split("\n") + + # Parse VCD header to extract timescale and channel mapping + timescale_multiplier = 1 # Default: 1 unit = 1 ns + channel_map: dict[str, str] = {} # symbol → channel name + + for line in lines: + line = line.strip() + + # Parse timescale (e.g., "$timescale 1 us $end" means 1 unit = 1000 ns) + if line.startswith("$timescale"): + timescale_multiplier = _parse_timescale(line) + + # Parse variable definitions (e.g., "$var wire 1 ! D0 $end") + if line.startswith("$var"): + parts = line.split() + if len(parts) >= 5: + symbol = parts[3] # e.g., "!" + channel = parts[4] # e.g., "D0" + channel_map[symbol] = channel + + if line == "$enddefinitions $end": + break + + # Parse value changes + samples: list[dict] = [] + sample_idx = 0 + + for line in lines: + line = line.strip() + if not line or line.startswith("$"): + continue + + # Timestamp line (e.g., "#100 1! 0" 1#") + if line.startswith("#"): + sample_data = _parse_vcd_timestamp_line(line, timescale_multiplier, channel_map) + if sample_data is not None: + sample_data["sample"] = sample_idx + samples.append(sample_data) + sample_idx += 1 + + return samples + + +def _parse_timescale(line: str) -> int: + """Parse timescale line and return multiplier to convert to nanoseconds.""" + parts = line.split() + if len(parts) >= 3: + value = parts[1] + unit = parts[2] + # Convert to nanoseconds multiplier + unit_multipliers = {"s": 1e9, "ms": 1e6, "us": 1e3, "ns": 1, "ps": 1e-3} + return int(float(value) * unit_multipliers.get(unit, 1)) + return 1 + + +def _parse_vcd_timestamp_line(line: str, timescale_multiplier: int, channel_map: dict[str, str]) -> dict | None: + """Parse a VCD timestamp line with value changes. + + Args: + line: Line starting with # (e.g., "#100 1! 0" 1#") + timescale_multiplier: Multiplier to convert time units to nanoseconds + channel_map: Mapping from VCD symbols to channel names + + Returns: + Dict with time_ns and values, or None if line is empty + """ + # Split timestamp from values + parts = line.split(maxsplit=1) + time_str = parts[0][1:] # Remove '#' prefix + + # Skip empty time lines + if not time_str: + return None + + time_units = int(time_str) + current_time_ns = time_units * timescale_multiplier + current_values: dict[str, int | float] = {} + + # Parse value changes if present on the same line + if len(parts) > 1: + values_str = parts[1] + _parse_vcd_value_changes(values_str, channel_map, current_values) + + # Return sample data if we have values + if current_values: + return {"time_ns": current_time_ns, "values": current_values} + + return None + + +def _parse_vcd_value_changes(values_str: str, channel_map: dict[str, str], current_values: dict[str, int | float]): + """Parse value change tokens from a VCD line. + + Modifies current_values dict in place. + + Supports: + - Single-bit: "1!", "0abc" + - Binary: "b11110000 abc" + - Real: "r3.14159 xyz", "r-10.5 !", "r1.23e-5 aa" + """ + i = 0 + while i < len(values_str): + char = values_str[i] + + # Single bit change (e.g., "1!", "0abc" for multi-char identifiers) + if char in "01xzXZ": + symbol, new_i = _extract_symbol(values_str, i + 1) + if symbol in channel_map: + channel = channel_map[symbol] + current_values[channel] = 1 if char == "1" else 0 + i = new_i + + # Binary value (e.g., "b1010 !" or "b1010 abc") + elif char == "b": + value, symbol, new_i = _parse_binary_value(values_str, i, channel_map) + if symbol and value is not None: + current_values[channel_map[symbol]] = value + i = new_i + + # Real (analog) value (e.g., "r3.14 !" or "r-10.5 abc") + elif char == "r": + value, symbol, new_i = _parse_real_value(values_str, i, channel_map) + if symbol and value is not None: + current_values[channel_map[symbol]] = value + i = new_i + + # Skip whitespace + elif char == " ": + i += 1 + else: + i += 1 + + +def _extract_symbol(text: str, start: int) -> tuple[str, int]: + """Extract a VCD symbol (can be multi-character) from text. + + Returns: + Tuple of (symbol, next_position) + """ + end = start + while end < len(text) and text[end] != " ": + end += 1 + return text[start:end], end + + +def _parse_binary_value(values_str: str, start: int, channel_map: dict[str, str]) -> tuple[int | None, str | None, int]: + """Parse a binary value like "b1010 abc". + + Returns: + Tuple of (value, symbol, next_position) + """ + # Extract binary value + value_start = start + 1 + value_end = value_start + while value_end < len(values_str) and values_str[value_end] in "01xzXZ": + value_end += 1 + binary_value = values_str[value_start:value_end] + + # Skip whitespace before symbol + while value_end < len(values_str) and values_str[value_end] == " ": + value_end += 1 + + # Extract symbol + symbol, next_pos = _extract_symbol(values_str, value_end) + + if symbol in channel_map: + try: + return int(binary_value, 2), symbol, next_pos + except ValueError: + return 0, symbol, next_pos + + return None, None, next_pos + + +def _parse_real_value(values_str: str, start: int, channel_map: dict[str, str]) -> tuple[float | None, str | None, int]: + """Parse a real (analog) value like "r3.14 abc" or "r-10.5 !". + + Returns: + Tuple of (value, symbol, next_position) + """ + # Extract real value (number with optional sign, decimal, exponent) + value_start = start + 1 + value_end = value_start + while value_end < len(values_str) and values_str[value_end] not in " ": + if values_str[value_end] in "0123456789-.eE+": + value_end += 1 + else: + break + real_value = values_str[value_start:value_end] + + # Skip whitespace before symbol + while value_end < len(values_str) and values_str[value_end] == " ": + value_end += 1 + + # Extract symbol + symbol, next_pos = _extract_symbol(values_str, value_end) + + if symbol in channel_map: + try: + return float(real_value), symbol, next_pos + except ValueError: + return 0.0, symbol, next_pos + + return None, None, next_pos + diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd_test.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd_test.py new file mode 100644 index 000000000..6af6825d6 --- /dev/null +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd_test.py @@ -0,0 +1,232 @@ +"""Tests for VCD (Value Change Dump) format parser.""" + +from base64 import b64encode + +from .common import CaptureResult, OutputFormat, Sample + + +def test_vcd_parser_comprehensive(): + """Test VCD parser with manually constructed VCD data covering all features. + + This test validates: + - Single-character identifiers (!, ", #) + - Multi-character identifiers (aa, ab, abc) + - Timescale parsing (microseconds to nanoseconds) + - Single-bit values (0/1) + - X/Z state handling + - Binary values (vectors) + - Real (analog) values with various formats + """ + # Construct a comprehensive VCD file + vcd_content = """$date Mon Dec 8 2025 $end +$version Test VCD Generator $end +$timescale 1 us $end +$scope module test $end +$var wire 1 ! D0 $end +$var wire 1 " D1 $end +$var wire 1 # D2 $end +$var wire 1 aa CH95 $end +$var wire 1 ab CH96 $end +$var wire 8 abc BUS0 $end +$var real 1 xyz ANALOG0 $end +$upscope $end +$enddefinitions $end +#0 1! 0" 1# 0aa 1ab b00001111 abc r-10.5 xyz +#5 0! 1" x# 1aa +#10 z! 0" 1# b11110000 abc r3.14159 xyz +#25 1! 1" 0# 0aa 0ab b10101010 abc r0.0 xyz +#100 0! 0" 0# r1.23e-5 xyz +""" + + # Create a CaptureResult with this VCD data + result = CaptureResult( + data_b64=b64encode(vcd_content.encode("utf-8")).decode("ascii"), + output_format=OutputFormat.VCD, + sample_rate="1MHz", + channel_map={ + "D0": "d0", + "D1": "d1", + "D2": "d2", + "CH95": "ch95", + "CH96": "ch96", + "BUS0": "bus", + "ANALOG0": "analog", + }, + triggers=None, + decoders=None, + ) + + # Parse the VCD + samples = result.decode() + + # Verify we got the expected number of samples + assert len(samples) == 5 + + # Sample 0 at time 0us = 0ns + s0 = samples[0] + assert s0.time_ns == 0 + assert s0.values["D0"] == 1 + assert s0.values["D1"] == 0 + assert s0.values["D2"] == 1 + assert s0.values["CH95"] == 0 # Multi-char identifier "aa" + assert s0.values["CH96"] == 1 # Multi-char identifier "ab" + assert s0.values["BUS0"] == 0b00001111 # Binary value + assert abs(s0.values["ANALOG0"] - (-10.5)) < 0.001 # Real value + + # Sample 1 at time 5us = 5000ns + s1 = samples[1] + assert s1.time_ns == 5000 + assert s1.values["D0"] == 0 + assert s1.values["D1"] == 1 + assert s1.values["D2"] == 0 # X converted to 0 + assert s1.values["CH95"] == 1 + + # Sample 2 at time 10us = 10000ns + s2 = samples[2] + assert s2.time_ns == 10000 + assert s2.values["D0"] == 0 # Z converted to 0 + assert s2.values["D1"] == 0 + assert s2.values["D2"] == 1 + assert s2.values["BUS0"] == 0b11110000 + assert abs(s2.values["ANALOG0"] - 3.14159) < 0.001 + + # Sample 3 at time 25us = 25000ns + s3 = samples[3] + assert s3.time_ns == 25000 + assert s3.values["D0"] == 1 + assert s3.values["D1"] == 1 + assert s3.values["D2"] == 0 + assert s3.values["CH95"] == 0 + assert s3.values["CH96"] == 0 + assert s3.values["BUS0"] == 0b10101010 + assert abs(s3.values["ANALOG0"] - 0.0) < 0.001 + + # Sample 4 at time 100us = 100000ns + s4 = samples[4] + assert s4.time_ns == 100000 + assert s4.values["D0"] == 0 + assert s4.values["D1"] == 0 + assert s4.values["D2"] == 0 + assert abs(s4.values["ANALOG0"] - 1.23e-5) < 1e-10 # Scientific notation + + +def test_vcd_parser_timescale_variations(): + """Test VCD parser with different timescale values.""" + # Test different timescales + test_cases = [ + ("1 ns", 1, 0), # 1ns timescale, time 0 = 0ns + ("1 us", 1000, 0), # 1us timescale, time 0 = 0ns + ("1 ms", 1000000, 0), # 1ms timescale, time 0 = 0ns + ("10 ns", 10, 100 * 10), # 10ns timescale, time 100 = 1000ns + ("100 ns", 100, 50 * 100), # 100ns timescale, time 50 = 5000ns + ] + + for timescale_str, _multiplier, expected_time_ns in test_cases: + vcd_content = f"""$timescale {timescale_str} $end +$var wire 1 ! D0 $end +$enddefinitions $end +#0 1! +#{100 if expected_time_ns else 0} 0! +""" + result = CaptureResult( + data_b64=b64encode(vcd_content.encode("utf-8")).decode("ascii"), + output_format=OutputFormat.VCD, + sample_rate="1MHz", + channel_map={"D0": "d0"}, + ) + + samples = result.decode() + assert len(samples) >= 1 + # First sample at time 0 + assert samples[0].time_ns == 0 + + +def test_vcd_parser_empty_timestamps(): + """Test VCD parser handles empty timestamp lines correctly.""" + vcd_content = """$timescale 1 ns $end +$var wire 1 ! D0 $end +$enddefinitions $end +#0 1! +#10 0! +# +#20 1! +""" + + result = CaptureResult( + data_b64=b64encode(vcd_content.encode("utf-8")).decode("ascii"), + output_format=OutputFormat.VCD, + sample_rate="1MHz", + channel_map={"D0": "d0"}, + ) + + samples = result.decode() + # Should have 3 samples (empty timestamp line skipped) + assert len(samples) == 3 + assert samples[0].time_ns == 0 + assert samples[1].time_ns == 10 + assert samples[2].time_ns == 20 + + +def test_vcd_parser_large_channel_count(): + """Test VCD parser with large channel counts using multi-char identifiers. + + According to libsigrok vcd_identifier(): + - Channels 0-93: Single char (!, ", ..., ~) + - Channels 94-769: Two lowercase letters (aa, ab, ..., zz) + - Channels 770+: Three lowercase letters (aaa, aab, ...) + """ + # Test identifiers at boundaries + vcd_content = """$timescale 1 ns $end +$var wire 1 ! CH0 $end +$var wire 1 ~ CH93 $end +$var wire 1 aa CH94 $end +$var wire 1 ab CH95 $end +$var wire 1 zz CH769 $end +$var wire 1 aaa CH770 $end +$var wire 1 abc CH800 $end +$enddefinitions $end +#0 1! 0~ 1aa 0ab 1zz 0aaa 1abc +#100 0! 1~ 0aa 1ab 0zz 1aaa 0abc +""" + + result = CaptureResult( + data_b64=b64encode(vcd_content.encode("utf-8")).decode("ascii"), + output_format=OutputFormat.VCD, + sample_rate="1MHz", + channel_map={ + "CH0": "ch0", + "CH93": "ch93", + "CH94": "ch94", + "CH95": "ch95", + "CH769": "ch769", + "CH770": "ch770", + "CH800": "ch800", + }, + ) + + samples = result.decode() + + # Verify first sample + assert len(samples) == 2 + s0 = samples[0] + assert isinstance(s0, Sample) + assert s0.time_ns == 0 + assert s0.values["CH0"] == 1 # Single char: ! + assert s0.values["CH93"] == 0 # Single char: ~ + assert s0.values["CH94"] == 1 # Two char: aa + assert s0.values["CH95"] == 0 # Two char: ab + assert s0.values["CH769"] == 1 # Two char: zz + assert s0.values["CH770"] == 0 # Three char: aaa + assert s0.values["CH800"] == 1 # Three char: abc + + # Verify second sample + s1 = samples[1] + assert s1.time_ns == 100 + assert s1.values["CH0"] == 0 + assert s1.values["CH93"] == 1 + assert s1.values["CH94"] == 0 + assert s1.values["CH95"] == 1 + assert s1.values["CH769"] == 0 + assert s1.values["CH770"] == 1 + assert s1.values["CH800"] == 0 + diff --git a/python/uv.lock b/python/uv.lock index 77b11a509..6087525db 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -31,6 +31,7 @@ members = [ "jumpstarter-driver-ridesx", "jumpstarter-driver-sdwire", "jumpstarter-driver-shell", + "jumpstarter-driver-sigrok", "jumpstarter-driver-snmp", "jumpstarter-driver-ssh", "jumpstarter-driver-ssh-mitm", @@ -2076,6 +2077,30 @@ dev = [ { name = "pytest-cov", specifier = ">=6.0.0" }, ] +[[package]] +name = "jumpstarter-driver-sigrok" +source = { editable = "packages/jumpstarter-driver-sigrok" } +dependencies = [ + { name = "jumpstarter" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-cov" }, +] + +[package.metadata] +requires-dist = [{ name = "jumpstarter", editable = "packages/jumpstarter" }] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=8.3.3" }, + { name = "pytest-asyncio", specifier = ">=0.24.0" }, + { name = "pytest-cov", specifier = ">=6.0.0" }, +] + [[package]] name = "jumpstarter-driver-snmp" source = { editable = "packages/jumpstarter-driver-snmp" } From 9df418252a3b5842f89fd8db75f8f7eff480d2d0 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Mon, 8 Dec 2025 12:07:15 +0100 Subject: [PATCH 10/18] sigrok: improve bit parsing channel mapping --- .../jumpstarter_driver_sigrok/common.py | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py index aae5e669d..a5c2b18e8 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py @@ -100,27 +100,35 @@ def decode(self) -> list[Sample] | dict[str, list[int]] | str: ) def _parse_bits(self) -> dict[str, list[int]]: - """Parse bits format to dict of channel→bit sequences.""" + """Parse bits format to dict of channel→bit sequences. + + Sigrok-cli bits format: "D0:10001\\nD1:01110\\n..." + Each line has format "channel_name:bits" + + Note: For large sample counts, sigrok-cli wraps bits across multiple + lines with repeated channel names. We accumulate all occurrences. + """ text = self.data.decode("utf-8") lines = [line.strip() for line in text.strip().split("\n") if line.strip()] - # bits format is just columns of 0/1 - # TODO: Need to determine channel mapping from somewhere - # For now, return as generic numbered channels result: dict[str, list[int]] = {} for line in lines: - # Each line might be space/comma separated bits - bits = [int(b) for b in line if b in "01"] - if not result: - # Initialize channels - for i, bit in enumerate(bits): - result[f"CH{i}"] = [bit] - else: - # Append to existing channels - for i, bit in enumerate(bits): - if f"CH{i}" in result: - result[f"CH{i}"].append(bit) + # Bits format: "D0:10001" or "A0:10001" + if ":" in line: + channel_device_name, bits_str = line.split(":", 1) + channel_device_name = channel_device_name.strip() + + # Map device name (D0) to user-friendly name (vcc) if available + channel_name = self.channel_map.get(channel_device_name, channel_device_name) + + # Parse bits from this line + bits = [int(b) for b in bits_str if b in "01"] + + # Accumulate bits for this channel (may appear on multiple lines) + if channel_name not in result: + result[channel_name] = [] + result[channel_name].extend(bits) return result From 34a8f976e6e8eca1f958af3b0ebc9889cf790ede Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Mon, 8 Dec 2025 12:07:30 +0100 Subject: [PATCH 11/18] sigrok: improve documentation --- .../jumpstarter-driver-sigrok/README.md | 209 ++++++++++++++---- .../jumpstarter_driver_sigrok/common.py | 5 +- .../jumpstarter_driver_sigrok/driver.py | 8 +- .../jumpstarter_driver_sigrok/driver_test.py | 48 +++- 4 files changed, 214 insertions(+), 56 deletions(-) diff --git a/python/packages/jumpstarter-driver-sigrok/README.md b/python/packages/jumpstarter-driver-sigrok/README.md index 4c1a75b35..4e42a404b 100644 --- a/python/packages/jumpstarter-driver-sigrok/README.md +++ b/python/packages/jumpstarter-driver-sigrok/README.md @@ -1,11 +1,10 @@ # Sigrok Driver -`jumpstarter-driver-sigrok` wraps `sigrok-cli` to provide logic analyzer and oscilloscope capture from Jumpstarter exporters. It supports: -- **Logic analyzers** (digital channels) - with protocol decoding (SPI, I2C, UART, etc.) +`jumpstarter-driver-sigrok` wraps [sigrok-cli](https://sigrok.org/wiki/Sigrok-cli) to provide logic analyzer and oscilloscope capture from Jumpstarter exporters. It supports: +- **Logic analyzers** (digital channels) - **Oscilloscopes** (analog channels) - voltage waveform capture - One-shot and streaming capture -- Decoder-friendly channel mappings -- Real-time protocol decoding +- Multiple output formats with parsing (VCD, CSV, Bits, ASCII) ## Installation @@ -19,44 +18,34 @@ pip3 install --extra-index-url https://pkg.jumpstarter.dev/simple/ jumpstarter-d export: sigrok: type: jumpstarter_driver_sigrok.driver.Sigrok - driver: demo # sigrok driver (demo, fx2lafw, etc.) - conn: null # optional: USB VID.PID or serial path - executable: null # optional: path to sigrok-cli (auto-detected) - channels: # channel mappings (device_name: semantic_name) - D0: vcc - D1: cs + driver: fx2lafw # sigrok driver (demo, fx2lafw, rigol-ds, etc.) + conn: null # optional: USB VID.PID, serial path, or null for auto + channels: # optional: map device channels to friendly names + D0: clk + D1: mosi D2: miso - D3: mosi - D4: clk - D5: sda - D6: scl + D3: cs ``` -## CaptureConfig (client-side) +### Configuration Parameters -```python -from jumpstarter_driver_sigrok.common import CaptureConfig, DecoderConfig +| Parameter | Description | Type | Required | Default | +|-----------|-------------|------|----------|---------| +| `driver` | Sigrok driver name (e.g., `demo`, `fx2lafw`, `rigol-ds`) | str | yes | - | +| `conn` | Connection string (USB VID.PID, serial path, or `null` for auto-detect) | str \| None | no | None | +| `executable` | Path to `sigrok-cli` executable | str | no | Auto-detected from PATH | +| `channels` | Channel mapping from device names (D0, A0) to semantic names (clk, voltage) | dict[str, str] | no | {} (empty) | -config = CaptureConfig( - sample_rate="8MHz", - samples=20000, - pretrigger=5000, - triggers={"cs": "falling"}, - decoders=[ - DecoderConfig( - name="spi", - channels={"clk": "clk", "mosi": "mosi", "miso": "miso", "cs": "cs"}, - annotations=["mosi-data"], - ) - ], -) -``` +## CaptureConfig Parameters (client-side) -This maps to: -```bash -sigrok-cli -d fx2lafw -c samplerate=8MHz,samples=20000,pretrigger=5000 --triggers D1=falling \ - -P spi:clk=D4:mosi=D3:miso=D2:cs=D1 -A spi=mosi-data -``` +| Parameter | Description | Type | Required | Default | +|-----------|-------------|------|----------|---------| +| `sample_rate` | Sampling rate (e.g., `"1M"`, `"8MHz"`, `"24000000"`) | str | no | "1M" | +| `samples` | Number of samples to capture (`None` for continuous) | int \| None | no | None | +| `pretrigger` | Number of samples to capture before trigger | int \| None | no | None | +| `triggers` | Trigger conditions by channel name (e.g., `{"cs": "falling"}`) | dict[str, str] \| None | no | None | +| `channels` | List of channel names to capture (overrides defaults) | list[str] \| None | no | None | +| `output_format` | Output format (vcd, csv, bits, ascii, srzip, binary) | str | no | "vcd" | ## Client API @@ -67,23 +56,107 @@ sigrok-cli -d fx2lafw -c samplerate=8MHz,samples=20000,pretrigger=5000 --trigger - `get_channel_map()` — device-to-semantic name mappings - `list_output_formats()` — supported formats (csv, srzip, vcd, binary, bits, ascii) +## Output Formats + +The driver supports multiple output formats. **VCD (Value Change Dump) is the default** because: +- ✅ **Efficient**: Only records signal changes (not every sample) +- ✅ **Precise timing**: Includes exact timestamps in nanoseconds +- ✅ **Widely supported**: Standard format for signal analysis tools +- ✅ **Mixed signals**: Handles both digital and analog data + +### Available Formats + +| Format | Use Case | Decoded By | +|--------|----------|------------| +| `vcd` (default) | Change-based signals with timing | `result.decode()` → `list[Sample]` | +| `csv` | All samples with timing | `result.decode()` → `list[Sample]` | +| `bits` | Bit sequences by channel | `result.decode()` → `dict[str, list[int]]` | +| `ascii` | ASCII art visualization | `result.decode()` → `str` | +| `srzip` | Raw sigrok session (for PulseView) | `result.data` (raw bytes) | +| `binary` | Raw binary data | `result.data` (raw bytes) | + +### Output Format Constants + +```python +from jumpstarter_driver_sigrok.common import OutputFormat + +config = CaptureConfig( + sample_rate="1MHz", + samples=1000, + output_format=OutputFormat.VCD, # or CSV, BITS, ASCII, SRZIP, BINARY +) +``` + ## Examples -### Logic Analyzer (Digital Channels) +### Example 1: Simple Capture (VCD format - default) + +**Python client code:** +```python +from jumpstarter_driver_sigrok.common import CaptureConfig + +# Capture with default VCD format (efficient, change-based with timing) +config = CaptureConfig( + sample_rate="1MHz", + samples=1000, + channels=["D0", "D1", "D2"], # Use device channel names or mapped names +) +result = client.capture(config) -One-shot with trigger: +# Decode VCD to get samples with timing +samples = result.decode() # list[Sample] +for sample in samples[:5]: + print(f"Time: {sample.time_ns}ns, Values: {sample.values}") +``` + +**Equivalent sigrok-cli command:** ```bash -sigrok-cli -d fx2lafw -c samplerate=8MHz,samples=20000,pretrigger=5000 --triggers D0=rising -o out.sr +sigrok-cli -d fx2lafw -C D0,D1,D2 \ + -c samplerate=1MHz --samples 1000 \ + -O vcd -o /tmp/capture.vcd ``` -Real-time decode (SPI): +--- + +### Example 2: Triggered Capture with Pretrigger + +**Python client code:** +```python +from jumpstarter_driver_sigrok.common import CaptureConfig + +# Capture with trigger and pretrigger buffer (VCD format - default) +config = CaptureConfig( + sample_rate="8MHz", + samples=20000, + pretrigger=5000, # Capture 5000 samples before trigger + triggers={"D0": "rising"}, # Trigger on D0 rising edge + channels=["D0", "D1", "D2", "D3"], + # output_format defaults to VCD (efficient change-based format) +) +result = client.capture(config) + +# Decode to analyze signal changes with precise timing +samples = result.decode() # list[Sample] - only changes recorded +print(f"Captured {len(samples)} signal changes") + +# Access timing and values +for sample in samples[:3]: + print(f"Time: {sample.time_ns}ns, Changed: {sample.values}") +``` + +**Equivalent sigrok-cli command:** ```bash -sigrok-cli -d fx2lafw -c samplerate=1M --continuous \ - -P spi:clk=D4:mosi=D3:miso=D2:cs=D1 -A spi=mosi-data +sigrok-cli -d fx2lafw -C D0,D1,D2,D3 \ + -c samplerate=8MHz,samples=20000,pretrigger=5000 \ + --triggers D0=rising \ + -O vcd -o /tmp/capture.vcd ``` -### Oscilloscope (Analog Channels) +--- +### Example 3: Oscilloscope (Analog Channels) + +**Exporter configuration:** ```yaml export: oscilloscope: @@ -95,16 +168,60 @@ export: A1: CH2 ``` +**Python client code:** ```python -from jumpstarter_driver_sigrok.common import CaptureConfig +from jumpstarter_driver_sigrok.common import CaptureConfig, OutputFormat # Capture analog waveforms config = CaptureConfig( sample_rate="1MHz", samples=10000, channels=["CH1", "CH2"], # Analog channels - output_format="csv", # or "vcd" for waveform viewers + output_format=OutputFormat.CSV, # CSV for voltage values ) result = client.capture(config) -waveform_data = result.data # bytes with voltage measurements + +# Parse voltage data +samples = result.decode() # list[Sample] +for sample in samples[:5]: + print(f"Time: {sample.time_ns}ns") + print(f" CH1: {sample.values.get('A0', 'N/A')}V") + print(f" CH2: {sample.values.get('A1', 'N/A')}V") +``` + +**Equivalent sigrok-cli command:** +```bash +sigrok-cli -d rigol-ds:conn=usb -C A0=CH1,A1=CH2 \ + -c samplerate=1MHz --samples 10000 \ + -O csv -o /tmp/capture.csv +``` + +--- + +### Example 4: Bits Format (Simple Bit Sequences) + +**Python client code:** +```python +from jumpstarter_driver_sigrok.common import CaptureConfig, OutputFormat + +# Capture in bits format (useful for visual inspection) +config = CaptureConfig( + sample_rate="100kHz", + samples=100, + channels=["D0", "D1", "D2"], + output_format=OutputFormat.BITS, +) +result = client.capture(config) + +# Get bit sequences per channel +bits_by_channel = result.decode() # dict[str, list[int]] +for channel, bits in bits_by_channel.items(): + print(f"{channel}: {''.join(map(str, bits[:20]))}") # First 20 bits +``` + +**Equivalent sigrok-cli command:** +```bash +sigrok-cli -d demo -C D0,D1,D2 \ + -c samplerate=100kHz --samples 100 \ + -O bits -o /tmp/capture.bits ``` diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py index a5c2b18e8..10f12662b 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py @@ -43,8 +43,9 @@ class CaptureConfig(BaseModel): triggers: dict[str, str] | None = Field(default=None, description="e.g., {'D0': 'rising'}") channels: list[str] | None = Field(default=None, description="override default channels by name") output_format: str = Field( - default="srzip", - description="csv, srzip, vcd, binary, bits, ascii", + default=OutputFormat.VCD, + description="Output format (default: vcd - efficient change-based format with timing). " + "Options: vcd, csv, srzip, binary, bits, ascii", ) decoders: list[DecoderConfig] | None = Field(default=None, description="real-time protocol decoding") diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py index 852cd1333..eac3f2fbd 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py @@ -19,12 +19,6 @@ def find_sigrok_cli() -> str: return executable -def _default_channel_map() -> dict[str, str]: - # Decoder-friendly default names for demo driver - # Maps device channel name -> semantic name - return {"D0": "vcc", "D1": "cs", "D2": "miso", "D3": "mosi", "D4": "clk", "D5": "sda", "D6": "scl"} - - @dataclass(kw_only=True) class Sigrok(Driver): """Sigrok driver wrapping sigrok-cli for logic analyzer and oscilloscope support.""" @@ -32,7 +26,7 @@ class Sigrok(Driver): driver: str = "demo" conn: str | None = None executable: str = field(default_factory=find_sigrok_cli) - channels: dict[str, str] = field(default_factory=_default_channel_map) + channels: dict[str, str] = field(default_factory=dict) def __post_init__(self): if hasattr(super(), "__post_init__"): diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py index 2766a36c8..1f580634e 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py @@ -2,7 +2,7 @@ import pytest -from .common import CaptureConfig, CaptureResult +from .common import CaptureConfig, CaptureResult, OutputFormat from .driver import Sigrok from jumpstarter.common.utils import serve @@ -68,6 +68,43 @@ def test_capture_with_demo_driver(demo_client): assert len(result.channel_map) > 0 +@pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") +def test_capture_default_format(demo_client): + """Test capture with default output format (VCD). + + VCD is the default because it's the most efficient format: + - Only records changes (not every sample) + - Includes precise timing information + - Widely supported by signal analysis tools + """ + # Don't specify output_format - should default to VCD + cfg = CaptureConfig( + sample_rate="100kHz", + samples=50, + channels=["D0", "D1", "D2"], + ) + + result = demo_client.capture(cfg) + + # Verify we got VCD format by default + assert isinstance(result, CaptureResult) + assert result.output_format == OutputFormat.VCD + assert isinstance(result.data, bytes) + assert len(result.data) > 0 + + # Verify VCD data can be decoded + samples = result.decode() + assert isinstance(samples, list) + assert len(samples) > 0 + + # Verify samples have timing information (VCD feature) + for sample in samples: + assert hasattr(sample, "time_ns") + assert isinstance(sample.time_ns, int) + assert hasattr(sample, "values") + assert isinstance(sample.values, dict) + + @pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") def test_capture_csv_format(demo_client): """Test capture with CSV output format via client.""" @@ -297,6 +334,7 @@ def test_decode_bits_format(demo_client): Verifies: - Bits format decoding works - Returns dict with bit sequences + - Channel names are mapped from device names (D0) to user-friendly names (vcc) """ from .common import OutputFormat @@ -314,11 +352,19 @@ def test_decode_bits_format(demo_client): assert isinstance(decoded, dict) assert len(decoded) > 0 + # Should have user-friendly channel names (vcc, cs, miso) from channel_map + # Not generic names like CH0, CH1 + assert "vcc" in decoded or "D0" in decoded + assert "cs" in decoded or "D1" in decoded + assert "miso" in decoded or "D2" in decoded + # Each channel should have a list of bits for channel, bits in decoded.items(): assert isinstance(channel, str) assert isinstance(bits, list) assert all(b in [0, 1] for b in bits) + # Should have bits (at least some, exact count may vary with demo driver timing) + assert len(bits) > 0 @pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") From edc12ac02924b94c7fa66f541789b83ba77dba56 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Mon, 8 Dec 2025 17:08:23 +0100 Subject: [PATCH 12/18] sigrok: tests should pass without sigrok-cli --- .../jumpstarter_driver_sigrok/driver.py | 27 ++++++++++++++----- .../jumpstarter_driver_sigrok/driver_test.py | 6 +++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py index eac3f2fbd..88bca75a1 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py @@ -12,11 +12,13 @@ from jumpstarter.driver import Driver, export -def find_sigrok_cli() -> str: - executable = which("sigrok-cli") - if executable is None: - raise FileNotFoundError("sigrok-cli executable not found in PATH") - return executable +def find_sigrok_cli() -> str | None: + """Find sigrok-cli executable in PATH. + + Returns: + Path to executable or None if not found + """ + return which("sigrok-cli") @dataclass(kw_only=True) @@ -25,13 +27,21 @@ class Sigrok(Driver): driver: str = "demo" conn: str | None = None - executable: str = field(default_factory=find_sigrok_cli) + executable: str | None = field(default_factory=find_sigrok_cli) channels: dict[str, str] = field(default_factory=dict) def __post_init__(self): if hasattr(super(), "__post_init__"): super().__post_init__() + def _ensure_executable(self): + """Ensure sigrok-cli is available.""" + if self.executable is None: + raise FileNotFoundError( + "sigrok-cli executable not found in PATH. " + "Please install sigrok-cli to use this driver." + ) + @classmethod def client(cls) -> str: return "jumpstarter_driver_sigrok.client.SigrokClient" @@ -41,6 +51,8 @@ def client(cls) -> str: @export def scan(self) -> str: """List devices for the configured driver.""" + self._ensure_executable() + assert self.executable is not None cmd = [self.executable, "--driver", self.driver, "--scan"] result = subprocess.run(cmd, capture_output=True, text=True, check=True) return result.stdout @@ -64,6 +76,7 @@ def list_output_formats(self) -> list[str]: @export def capture(self, config: CaptureConfig | dict) -> dict: """One-shot capture; returns dict with base64-encoded binary data.""" + self._ensure_executable() cfg = CaptureConfig.model_validate(config) cmd, outfile, tmpdir = self._build_capture_command(cfg) @@ -87,6 +100,7 @@ def capture(self, config: CaptureConfig | dict) -> dict: @export async def capture_stream(self, config: CaptureConfig | dict): """Streaming capture; yields chunks of binary data from sigrok-cli stdout.""" + self._ensure_executable() cfg = CaptureConfig.model_validate(config) cmd = self._build_stream_command(cfg) @@ -139,6 +153,7 @@ def _build_stream_command(self, cfg: CaptureConfig) -> list[str]: return cmd def _base_driver_args(self) -> list[str]: + assert self.executable is not None if self.conn: return [self.executable, "-d", f"{self.driver}:conn={self.conn}"] return [self.executable, "-d", self.driver] diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py index 1f580634e..f6db8e066 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py @@ -6,6 +6,12 @@ from .driver import Sigrok from jumpstarter.common.utils import serve +# Skip all integration tests if sigrok-cli is not available +pytestmark = pytest.mark.skipif( + which("sigrok-cli") is None, + reason="sigrok-cli not found in PATH" +) + @pytest.fixture def demo_driver_instance(): From 88d6c53fbb396f3471adcbf80fa756eec5e33bd0 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Mon, 8 Dec 2025 17:10:27 +0100 Subject: [PATCH 13/18] sigrok: add sigrok-cli for testing --- .github/workflows/python-tests.yaml | 11 +++++++++++ python/Dockerfile | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-tests.yaml b/.github/workflows/python-tests.yaml index 045d5c782..634b1f73a 100644 --- a/.github/workflows/python-tests.yaml +++ b/.github/workflows/python-tests.yaml @@ -59,11 +59,22 @@ jobs: sudo apt-get update sudo apt-get install -y libgpiod-dev liblgpio-dev + - name: Install sigrok-cli (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y sigrok-cli + - name: Install Qemu (macOS) if: runner.os == 'macOS' run: | brew install qemu + - name: Install sigrok-cli (macOS) + if: runner.os == 'macOS' + run: | + brew install sigrok-cli + - name: Cache Fedora Cloud images id: cache-fedora-cloud-images uses: actions/cache@v4 diff --git a/python/Dockerfile b/python/Dockerfile index 041f8bc1f..b67061d10 100644 --- a/python/Dockerfile +++ b/python/Dockerfile @@ -7,7 +7,7 @@ RUN dnf install -y make git && \ COPY --from=uv /uv /uvx /bin/ FROM fedora:42 AS product -RUN dnf install -y python3 ustreamer libusb1 android-tools python3-libgpiod && \ +RUN dnf install -y python3 ustreamer libusb1 android-tools python3-libgpiod sigrok-cli && \ dnf clean all && \ rm -rf /var/cache/dnf COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ From 00d07b5c5510428f9579adc8e95aa44672b555c5 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Mon, 8 Dec 2025 21:42:16 +0100 Subject: [PATCH 14/18] sigrok: generate decoded output instead of list --- .../jumpstarter_driver_sigrok/common.py | 12 +++++------ .../jumpstarter_driver_sigrok/csv.py | 20 +++++++++---------- .../jumpstarter_driver_sigrok/csv_test.py | 8 ++++---- .../jumpstarter_driver_sigrok/driver_test.py | 10 +++++----- .../jumpstarter_driver_sigrok/vcd.py | 17 ++++++++-------- .../jumpstarter_driver_sigrok/vcd_test.py | 8 ++++---- 6 files changed, 36 insertions(+), 39 deletions(-) diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py index 10f12662b..9dddd405d 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any +from typing import Any, Iterator from pydantic import BaseModel, Field @@ -69,12 +69,12 @@ def data(self) -> bytes: from base64 import b64decode return b64decode(self.data_b64) - def decode(self) -> list[Sample] | dict[str, list[int]] | str: + def decode(self) -> Iterator[Sample] | dict[str, list[int]] | str: """Parse captured data based on output format. Returns: - - CSV format: list[Sample] with timing and all values per sample - - VCD format: list[Sample] with timing and only changed values + - CSV format: Iterator[Sample] yielding samples with timing and all values per sample + - VCD format: Iterator[Sample] yielding samples with timing and only changed values - Bits format: dict[str, list[int]] with channel→bit sequences - ASCII format: str with ASCII art visualization - Other formats: raises NotImplementedError (use .data for raw bytes) @@ -85,11 +85,11 @@ def decode(self) -> list[Sample] | dict[str, list[int]] | str: if self.output_format == OutputFormat.CSV: from .csv import parse_csv samples_data = parse_csv(self.data, self.sample_rate) - return [Sample.model_validate(s) for s in samples_data] + return (Sample.model_validate(s) for s in samples_data) elif self.output_format == OutputFormat.VCD: from .vcd import parse_vcd samples_data = parse_vcd(self.data, self.sample_rate) - return [Sample.model_validate(s) for s in samples_data] + return (Sample.model_validate(s) for s in samples_data) elif self.output_format == OutputFormat.BITS: return self._parse_bits() elif self.output_format == OutputFormat.ASCII: diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv.py index 1a7626d06..cda197c80 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv.py @@ -3,17 +3,18 @@ from __future__ import annotations import csv +from typing import Iterator -def parse_csv(data: bytes, sample_rate: str) -> list[dict]: - """Parse CSV format to list of samples with timing. +def parse_csv(data: bytes, sample_rate: str) -> Iterator[dict]: + """Parse CSV format to iterator of samples with timing. Args: data: Raw CSV data as bytes sample_rate: Sample rate string (e.g., "100kHz", "1MHz") - Returns: - List of dicts with keys: sample, time_ns, values + Yields: + Dicts with keys: sample, time_ns, values """ text = data.decode("utf-8") lines = text.strip().split("\n") @@ -27,7 +28,7 @@ def parse_csv(data: bytes, sample_rate: str) -> list[dict]: data_lines = _extract_csv_data_lines(lines) if not data_lines or len(data_lines) < 2: - return [] + return # Parse the CSV data reader = csv.reader(data_lines) @@ -38,17 +39,14 @@ def parse_csv(data: bytes, sample_rate: str) -> list[dict]: # Get channel names from types channel_names = _infer_channel_names(types_row) - # Parse data rows - samples: list[dict] = [] + # Parse and yield data rows one by one for idx, row in enumerate(reader): values = _parse_csv_row(channel_names, row) - samples.append({ + yield { "sample": idx, "time_ns": idx * time_step_ns, "values": values, - }) - - return samples + } def _parse_sample_rate_hz(sample_rate: str) -> float: diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv_test.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv_test.py index f0589e962..de61720a0 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv_test.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv_test.py @@ -51,7 +51,7 @@ def test_csv_format_basic(demo_client: SigrokClient): result = demo_client.capture(cfg) assert isinstance(result, CaptureResult) assert isinstance(result.data, bytes) - decoded_data = result.decode() + decoded_data = list(result.decode()) assert isinstance(decoded_data, list) assert len(decoded_data) > 0 # Verify channel names are in the data @@ -73,7 +73,7 @@ def test_csv_format_timing(demo_client: SigrokClient): assert isinstance(result, CaptureResult) # Decode the CSV data - samples = result.decode() + samples = list(result.decode()) assert isinstance(samples, list) assert len(samples) > 0 @@ -97,7 +97,7 @@ def test_csv_format_analog_channels(demo_client: SigrokClient): result = demo_client.capture(cfg) assert isinstance(result, CaptureResult) assert isinstance(result.data, bytes) - decoded_data = result.decode() + decoded_data = list(result.decode()) assert isinstance(decoded_data, list) assert len(decoded_data) > 0 @@ -121,7 +121,7 @@ def test_csv_format_mixed_channels(demo_client: SigrokClient): ) result = demo_client.capture(cfg) - samples = result.decode() + samples = list(result.decode()) assert isinstance(samples, list) assert len(samples) > 0 diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py index f6db8e066..4c25b0f1d 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py @@ -99,7 +99,7 @@ def test_capture_default_format(demo_client): assert len(result.data) > 0 # Verify VCD data can be decoded - samples = result.decode() + samples = list(result.decode()) assert isinstance(samples, list) assert len(samples) > 0 @@ -290,7 +290,7 @@ def test_decode_csv_format(demo_client): assert isinstance(result, CaptureResult) # Decode the CSV data - samples = result.decode() + samples = list(result.decode()) assert isinstance(samples, list) assert len(samples) > 0 @@ -395,7 +395,7 @@ def test_decode_vcd_format(demo_client): assert isinstance(result, CaptureResult) # Decode the VCD data - samples = result.decode() + samples = list(result.decode()) assert isinstance(samples, list) assert len(samples) > 0 @@ -432,7 +432,7 @@ def test_decode_vcd_analog_channels(demo_client): ) result = demo_client.capture(cfg) - samples = result.decode() + samples = list(result.decode()) assert isinstance(samples, list) assert len(samples) > 0 @@ -480,7 +480,7 @@ def test_decode_analog_csv(demo_client): ) result = demo_client.capture(cfg) - samples = result.decode() + samples = list(result.decode()) assert isinstance(samples, list) assert len(samples) > 0 diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd.py index c777e79d0..ed0f6bd70 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd.py @@ -2,9 +2,11 @@ from __future__ import annotations +from typing import Iterator -def parse_vcd(data: bytes, sample_rate: str) -> list[dict]: - """Parse VCD format to list of samples with timing (changes only). + +def parse_vcd(data: bytes, sample_rate: str) -> Iterator[dict]: + """Parse VCD format to iterator of samples with timing (changes only). VCD format only records when signals change, making it efficient for sparse data. Each sample represents a time point where one or more @@ -14,8 +16,8 @@ def parse_vcd(data: bytes, sample_rate: str) -> list[dict]: data: Raw VCD data as bytes sample_rate: Sample rate string (not used for VCD as it has its own timescale) - Returns: - List of dicts with keys: sample, time_ns, values + Yields: + Dicts with keys: sample, time_ns, values """ text = data.decode("utf-8") lines = text.strip().split("\n") @@ -42,8 +44,7 @@ def parse_vcd(data: bytes, sample_rate: str) -> list[dict]: if line == "$enddefinitions $end": break - # Parse value changes - samples: list[dict] = [] + # Parse and yield value changes one by one sample_idx = 0 for line in lines: @@ -56,11 +57,9 @@ def parse_vcd(data: bytes, sample_rate: str) -> list[dict]: sample_data = _parse_vcd_timestamp_line(line, timescale_multiplier, channel_map) if sample_data is not None: sample_data["sample"] = sample_idx - samples.append(sample_data) + yield sample_data sample_idx += 1 - return samples - def _parse_timescale(line: str) -> int: """Parse timescale line and return multiplier to convert to nanoseconds.""" diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd_test.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd_test.py index 6af6825d6..ef4c67fbd 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd_test.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd_test.py @@ -57,7 +57,7 @@ def test_vcd_parser_comprehensive(): ) # Parse the VCD - samples = result.decode() + samples = list(result.decode()) # Verify we got the expected number of samples assert len(samples) == 5 @@ -135,7 +135,7 @@ def test_vcd_parser_timescale_variations(): channel_map={"D0": "d0"}, ) - samples = result.decode() + samples = list(result.decode()) assert len(samples) >= 1 # First sample at time 0 assert samples[0].time_ns == 0 @@ -159,7 +159,7 @@ def test_vcd_parser_empty_timestamps(): channel_map={"D0": "d0"}, ) - samples = result.decode() + samples = list(result.decode()) # Should have 3 samples (empty timestamp line skipped) assert len(samples) == 3 assert samples[0].time_ns == 0 @@ -204,7 +204,7 @@ def test_vcd_parser_large_channel_count(): }, ) - samples = result.decode() + samples = list(result.decode()) # Verify first sample assert len(samples) == 2 From 74c389fce4e7902b3adc07963fb7217a408ff75d Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Mon, 8 Dec 2025 22:08:14 +0100 Subject: [PATCH 15/18] sigrok: channel mapping config fix also, VCD tests should not expect channel mapping when not interacting with sigrok-cli, since sigrok-cli is the one performing mappings. --- .../examples/exporter.yaml | 21 +++++++------- .../jumpstarter_driver_sigrok/common.py | 5 ++++ .../jumpstarter_driver_sigrok/csv_test.py | 3 +- .../jumpstarter_driver_sigrok/driver.py | 4 +-- .../jumpstarter_driver_sigrok/vcd_test.py | 29 +++++-------------- 5 files changed, 27 insertions(+), 35 deletions(-) diff --git a/python/packages/jumpstarter-driver-sigrok/examples/exporter.yaml b/python/packages/jumpstarter-driver-sigrok/examples/exporter.yaml index 847a99b29..5400150af 100644 --- a/python/packages/jumpstarter-driver-sigrok/examples/exporter.yaml +++ b/python/packages/jumpstarter-driver-sigrok/examples/exporter.yaml @@ -8,14 +8,15 @@ token: "" export: sigrok: type: jumpstarter_driver_sigrok.driver.Sigrok - driver: demo - conn: null - channels: - D0: vcc - D1: cs - D2: miso - D3: mosi - D4: clk - D5: sda - D6: scl + config: + driver: demo + conn: null + channels: + D0: vcc + D1: cs + D2: miso + D3: mosi + D4: clk + D5: sda + D6: scl diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py index 9dddd405d..70c24ca96 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py @@ -79,6 +79,11 @@ def decode(self) -> Iterator[Sample] | dict[str, list[int]] | str: - ASCII format: str with ASCII art visualization - Other formats: raises NotImplementedError (use .data for raw bytes) + Note: + Channel names in the output depend on how the data was captured: + - If captured with channel mapping, sigrok-cli outputs mapped names (vcc, cs, etc.) + - If captured without mapping, outputs device names (D0, D1, etc.) + Raises: NotImplementedError: For binary/srzip formats (use .data property) """ diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv_test.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv_test.py index de61720a0..ff944d45a 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv_test.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv_test.py @@ -54,7 +54,8 @@ def test_csv_format_basic(demo_client: SigrokClient): decoded_data = list(result.decode()) assert isinstance(decoded_data, list) assert len(decoded_data) > 0 - # Verify channel names are in the data + # CSV format uses inferred names (D0, D1, etc.) based on column types + # Channel mapping is only preserved in VCD format first_sample = decoded_data[0] assert "D0" in first_sample.values or "D1" in first_sample.values diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py index 88bca75a1..ec1765afd 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py @@ -66,7 +66,7 @@ def get_driver_info(self) -> dict: } @export - def get_channel_map(self) -> dict[int, str]: + def get_channel_map(self) -> dict[str, str]: return self.channels @export @@ -81,7 +81,7 @@ def capture(self, config: CaptureConfig | dict) -> dict: cmd, outfile, tmpdir = self._build_capture_command(cfg) try: - self.logger.debug("running sigrok-cli: %s", " ".join(cmd)) + self.logger.debug("Running sigrok-cli: %s", " ".join(cmd)) subprocess.run(cmd, check=True) data = outfile.read_bytes() diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd_test.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd_test.py index ef4c67fbd..032dfee07 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd_test.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd_test.py @@ -38,20 +38,12 @@ def test_vcd_parser_comprehensive(): #100 0! 0" 0# r1.23e-5 xyz """ - # Create a CaptureResult with this VCD data + # Create a CaptureResult with this VCD data (no channel mapping for parser test) result = CaptureResult( data_b64=b64encode(vcd_content.encode("utf-8")).decode("ascii"), output_format=OutputFormat.VCD, sample_rate="1MHz", - channel_map={ - "D0": "d0", - "D1": "d1", - "D2": "d2", - "CH95": "ch95", - "CH96": "ch96", - "BUS0": "bus", - "ANALOG0": "analog", - }, + channel_map={}, triggers=None, decoders=None, ) @@ -65,6 +57,7 @@ def test_vcd_parser_comprehensive(): # Sample 0 at time 0us = 0ns s0 = samples[0] assert s0.time_ns == 0 + # Channel names come directly from VCD (not mapped) assert s0.values["D0"] == 1 assert s0.values["D1"] == 0 assert s0.values["D2"] == 1 @@ -132,7 +125,7 @@ def test_vcd_parser_timescale_variations(): data_b64=b64encode(vcd_content.encode("utf-8")).decode("ascii"), output_format=OutputFormat.VCD, sample_rate="1MHz", - channel_map={"D0": "d0"}, + channel_map={}, ) samples = list(result.decode()) @@ -156,7 +149,7 @@ def test_vcd_parser_empty_timestamps(): data_b64=b64encode(vcd_content.encode("utf-8")).decode("ascii"), output_format=OutputFormat.VCD, sample_rate="1MHz", - channel_map={"D0": "d0"}, + channel_map={}, ) samples = list(result.decode()) @@ -193,20 +186,12 @@ def test_vcd_parser_large_channel_count(): data_b64=b64encode(vcd_content.encode("utf-8")).decode("ascii"), output_format=OutputFormat.VCD, sample_rate="1MHz", - channel_map={ - "CH0": "ch0", - "CH93": "ch93", - "CH94": "ch94", - "CH95": "ch95", - "CH769": "ch769", - "CH770": "ch770", - "CH800": "ch800", - }, + channel_map={}, ) samples = list(result.decode()) - # Verify first sample + # Verify first sample (channel names come directly from VCD) assert len(samples) == 2 s0 = samples[0] assert isinstance(s0, Sample) From 383861d7db2c6d91390bacc9ca957a1ff038eda9 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Mon, 8 Dec 2025 22:39:21 +0100 Subject: [PATCH 16/18] sigrok: auto instead of null, and better time print --- .../jumpstarter-driver-sigrok/README.md | 10 ++--- .../examples/exporter.yaml | 2 +- .../jumpstarter_driver_sigrok/common.py | 44 ++++++++++++++++++- .../jumpstarter_driver_sigrok/csv.py | 6 +-- .../jumpstarter_driver_sigrok/csv_test.py | 6 +-- .../jumpstarter_driver_sigrok/driver.py | 4 +- .../jumpstarter_driver_sigrok/driver_test.py | 14 +++--- .../jumpstarter_driver_sigrok/vcd.py | 26 +++++------ .../jumpstarter_driver_sigrok/vcd_test.py | 32 +++++++------- 9 files changed, 93 insertions(+), 51 deletions(-) diff --git a/python/packages/jumpstarter-driver-sigrok/README.md b/python/packages/jumpstarter-driver-sigrok/README.md index 4e42a404b..13344e058 100644 --- a/python/packages/jumpstarter-driver-sigrok/README.md +++ b/python/packages/jumpstarter-driver-sigrok/README.md @@ -19,7 +19,7 @@ export: sigrok: type: jumpstarter_driver_sigrok.driver.Sigrok driver: fx2lafw # sigrok driver (demo, fx2lafw, rigol-ds, etc.) - conn: null # optional: USB VID.PID, serial path, or null for auto + conn: auto # optional: USB VID.PID, serial path, or "auto" for auto-detect channels: # optional: map device channels to friendly names D0: clk D1: mosi @@ -32,7 +32,7 @@ export: | Parameter | Description | Type | Required | Default | |-----------|-------------|------|----------|---------| | `driver` | Sigrok driver name (e.g., `demo`, `fx2lafw`, `rigol-ds`) | str | yes | - | -| `conn` | Connection string (USB VID.PID, serial path, or `null` for auto-detect) | str \| None | no | None | +| `conn` | Connection string (USB VID.PID, serial path, or `"auto"` for auto-detect) | str \| None | no | "auto" | | `executable` | Path to `sigrok-cli` executable | str | no | Auto-detected from PATH | | `channels` | Channel mapping from device names (D0, A0) to semantic names (clk, voltage) | dict[str, str] | no | {} (empty) | @@ -106,7 +106,7 @@ result = client.capture(config) # Decode VCD to get samples with timing samples = result.decode() # list[Sample] for sample in samples[:5]: - print(f"Time: {sample.time_ns}ns, Values: {sample.values}") + print(f"Time: {sample.time}s, Values: {sample.values}") ``` **Equivalent sigrok-cli command:** @@ -141,7 +141,7 @@ print(f"Captured {len(samples)} signal changes") # Access timing and values for sample in samples[:3]: - print(f"Time: {sample.time_ns}ns, Changed: {sample.values}") + print(f"Time: {sample.time}s, Changed: {sample.values}") ``` **Equivalent sigrok-cli command:** @@ -184,7 +184,7 @@ result = client.capture(config) # Parse voltage data samples = result.decode() # list[Sample] for sample in samples[:5]: - print(f"Time: {sample.time_ns}ns") + print(f"Time: {sample.time}s") print(f" CH1: {sample.values.get('A0', 'N/A')}V") print(f" CH2: {sample.values.get('A1', 'N/A')}V") ``` diff --git a/python/packages/jumpstarter-driver-sigrok/examples/exporter.yaml b/python/packages/jumpstarter-driver-sigrok/examples/exporter.yaml index 5400150af..7e1029ea9 100644 --- a/python/packages/jumpstarter-driver-sigrok/examples/exporter.yaml +++ b/python/packages/jumpstarter-driver-sigrok/examples/exporter.yaml @@ -10,7 +10,7 @@ export: type: jumpstarter_driver_sigrok.driver.Sigrok config: driver: demo - conn: null + conn: auto channels: D0: vcc D1: cs diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py index 70c24ca96..cc683a694 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py @@ -22,9 +22,51 @@ def all(cls) -> list[str]: class Sample(BaseModel): """A single sample with timing information.""" sample: int # Sample index - time_ns: int # Time in nanoseconds + time: float # Time in seconds (full precision) values: dict[str, int | float] # Channel values (digital: 0/1, analog: voltage) + def __str__(self) -> str: + """Format sample with clean time display using appropriate unit (fs/ps/ns/μs/ms/s).""" + time_str = self._format_time(self.time) + return f"Sample(sample={self.sample}, time={time_str}, values={self.values})" + + @staticmethod + def _format_time(time_s: float) -> str: + """Format time in seconds to the most appropriate unit. + + Args: + time_s: Time in seconds + + Returns: + Formatted string like "1.5ns", "2.3μs", "1.5ms", "2s" + """ + # Special case for zero + if time_s == 0: + return "0s" + + abs_time = abs(time_s) + + # Define units in descending order (seconds to femtoseconds) + units = [ + (1.0, "s"), + (1e-3, "ms"), + (1e-6, "μs"), + (1e-9, "ns"), + (1e-12, "ps"), + (1e-15, "fs"), + ] + + # Find the most appropriate unit + for scale, unit in units: + if abs_time >= scale or scale == 1e-15: # Use fs as minimum + value = time_s / scale + # Format with up to 6 significant digits, remove trailing zeros + formatted = f"{value:.6g}" + return f"{formatted}{unit}" + + # Fallback (should never reach here) + return f"{time_s:.6g}s" + class DecoderConfig(BaseModel): """Protocol decoder configuration (real-time during capture).""" diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv.py index cda197c80..a0cbdde8a 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv.py @@ -14,14 +14,14 @@ def parse_csv(data: bytes, sample_rate: str) -> Iterator[dict]: sample_rate: Sample rate string (e.g., "100kHz", "1MHz") Yields: - Dicts with keys: sample, time_ns, values + Dicts with keys: sample, time (seconds), values """ text = data.decode("utf-8") lines = text.strip().split("\n") # Parse sample rate for timing calculation sample_rate_hz = _parse_sample_rate_hz(sample_rate) - time_step_ns = int(1_000_000_000.0 / sample_rate_hz) + time_step_s = 1.0 / sample_rate_hz # seconds per sample # Skip comment lines and analog preview lines (format: "A0: -10.0000 V DC") # The actual data starts after a header row with types like "logic,logic,V DC,V DC" @@ -44,7 +44,7 @@ def parse_csv(data: bytes, sample_rate: str) -> Iterator[dict]: values = _parse_csv_row(channel_names, row) yield { "sample": idx, - "time_ns": idx * time_step_ns, + "time": idx * time_step_s, "values": values, } diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv_test.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv_test.py index ff944d45a..e15e545a0 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv_test.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/csv_test.py @@ -80,9 +80,9 @@ def test_csv_format_timing(demo_client: SigrokClient): # Verify timing progresses correctly for sample in samples: - assert isinstance(sample.time_ns, int) - # Verify timing progresses (1/100kHz = 10,000ns per sample) - assert sample.time_ns == sample.sample * 10_000 + assert isinstance(sample.time, float) + # Verify timing progresses (1/100kHz = 0.00001s per sample) + assert sample.time == sample.sample * 0.00001 @pytest.mark.skipif(which("sigrok-cli") is None, reason="sigrok-cli not installed") diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py index ec1765afd..35c7b88d0 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver.py @@ -26,7 +26,7 @@ class Sigrok(Driver): """Sigrok driver wrapping sigrok-cli for logic analyzer and oscilloscope support.""" driver: str = "demo" - conn: str | None = None + conn: str | None = "auto" executable: str | None = field(default_factory=find_sigrok_cli) channels: dict[str, str] = field(default_factory=dict) @@ -154,7 +154,7 @@ def _build_stream_command(self, cfg: CaptureConfig) -> list[str]: def _base_driver_args(self) -> list[str]: assert self.executable is not None - if self.conn: + if self.conn and self.conn != "auto": return [self.executable, "-d", f"{self.driver}:conn={self.conn}"] return [self.executable, "-d", self.driver] diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py index 4c25b0f1d..087490e61 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/driver_test.py @@ -105,8 +105,8 @@ def test_capture_default_format(demo_client): # Verify samples have timing information (VCD feature) for sample in samples: - assert hasattr(sample, "time_ns") - assert isinstance(sample.time_ns, int) + assert hasattr(sample, "time") + assert isinstance(sample.time, float) assert hasattr(sample, "values") assert isinstance(sample.values, dict) @@ -298,11 +298,11 @@ def test_decode_csv_format(demo_client): for sample in samples: assert isinstance(sample, Sample) assert isinstance(sample.sample, int) - assert isinstance(sample.time_ns, int) + assert isinstance(sample.time, float) assert isinstance(sample.values, dict) - # Verify timing progresses (1/100kHz = 10,000ns per sample) - assert sample.time_ns == sample.sample * 10_000 + # Verify timing progresses (1/100kHz = 0.00001s per sample) + assert sample.time == sample.sample * 0.00001 # Verify values are present assert len(sample.values) > 0 @@ -403,7 +403,7 @@ def test_decode_vcd_format(demo_client): for sample in samples: assert isinstance(sample, Sample) assert isinstance(sample.sample, int) - assert isinstance(sample.time_ns, int) + assert isinstance(sample.time, float) assert isinstance(sample.values, dict) # VCD only records changes, so each sample should have at least one value @@ -440,7 +440,7 @@ def test_decode_vcd_analog_channels(demo_client): # Check that samples have analog values first_sample = samples[0] assert isinstance(first_sample, Sample) - assert isinstance(first_sample.time_ns, int) + assert isinstance(first_sample.time, float) assert len(first_sample.values) > 0 diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd.py index ed0f6bd70..cd2551906 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd.py @@ -17,13 +17,13 @@ def parse_vcd(data: bytes, sample_rate: str) -> Iterator[dict]: sample_rate: Sample rate string (not used for VCD as it has its own timescale) Yields: - Dicts with keys: sample, time_ns, values + Dicts with keys: sample, time (seconds), values """ text = data.decode("utf-8") lines = text.strip().split("\n") # Parse VCD header to extract timescale and channel mapping - timescale_multiplier = 1 # Default: 1 unit = 1 ns + timescale_multiplier = 1e-9 # Default: 1 unit = 1 ns = 1e-9 seconds channel_map: dict[str, str] = {} # symbol → channel name for line in lines: @@ -61,28 +61,28 @@ def parse_vcd(data: bytes, sample_rate: str) -> Iterator[dict]: sample_idx += 1 -def _parse_timescale(line: str) -> int: - """Parse timescale line and return multiplier to convert to nanoseconds.""" +def _parse_timescale(line: str) -> float: + """Parse timescale line and return multiplier to convert to seconds.""" parts = line.split() if len(parts) >= 3: value = parts[1] unit = parts[2] - # Convert to nanoseconds multiplier - unit_multipliers = {"s": 1e9, "ms": 1e6, "us": 1e3, "ns": 1, "ps": 1e-3} - return int(float(value) * unit_multipliers.get(unit, 1)) - return 1 + # Convert to seconds multiplier + unit_multipliers = {"s": 1.0, "ms": 1e-3, "us": 1e-6, "ns": 1e-9, "ps": 1e-12} + return float(value) * unit_multipliers.get(unit, 1.0) + return 1.0 -def _parse_vcd_timestamp_line(line: str, timescale_multiplier: int, channel_map: dict[str, str]) -> dict | None: +def _parse_vcd_timestamp_line(line: str, timescale_multiplier: float, channel_map: dict[str, str]) -> dict | None: """Parse a VCD timestamp line with value changes. Args: line: Line starting with # (e.g., "#100 1! 0" 1#") - timescale_multiplier: Multiplier to convert time units to nanoseconds + timescale_multiplier: Multiplier to convert time units to seconds channel_map: Mapping from VCD symbols to channel names Returns: - Dict with time_ns and values, or None if line is empty + Dict with time (seconds) and values, or None if line is empty """ # Split timestamp from values parts = line.split(maxsplit=1) @@ -93,7 +93,7 @@ def _parse_vcd_timestamp_line(line: str, timescale_multiplier: int, channel_map: return None time_units = int(time_str) - current_time_ns = time_units * timescale_multiplier + current_time_s = time_units * timescale_multiplier current_values: dict[str, int | float] = {} # Parse value changes if present on the same line @@ -103,7 +103,7 @@ def _parse_vcd_timestamp_line(line: str, timescale_multiplier: int, channel_map: # Return sample data if we have values if current_values: - return {"time_ns": current_time_ns, "values": current_values} + return {"time": current_time_s, "values": current_values} return None diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd_test.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd_test.py index 032dfee07..bf0d829c4 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd_test.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/vcd_test.py @@ -54,9 +54,9 @@ def test_vcd_parser_comprehensive(): # Verify we got the expected number of samples assert len(samples) == 5 - # Sample 0 at time 0us = 0ns + # Sample 0 at time 0us = 0s s0 = samples[0] - assert s0.time_ns == 0 + assert s0.time == 0.0 # Channel names come directly from VCD (not mapped) assert s0.values["D0"] == 1 assert s0.values["D1"] == 0 @@ -66,26 +66,26 @@ def test_vcd_parser_comprehensive(): assert s0.values["BUS0"] == 0b00001111 # Binary value assert abs(s0.values["ANALOG0"] - (-10.5)) < 0.001 # Real value - # Sample 1 at time 5us = 5000ns + # Sample 1 at time 5us = 0.000005s s1 = samples[1] - assert s1.time_ns == 5000 + assert abs(s1.time - 0.000005) < 1e-12 assert s1.values["D0"] == 0 assert s1.values["D1"] == 1 assert s1.values["D2"] == 0 # X converted to 0 assert s1.values["CH95"] == 1 - # Sample 2 at time 10us = 10000ns + # Sample 2 at time 10us = 0.00001s s2 = samples[2] - assert s2.time_ns == 10000 + assert abs(s2.time - 0.00001) < 1e-12 assert s2.values["D0"] == 0 # Z converted to 0 assert s2.values["D1"] == 0 assert s2.values["D2"] == 1 assert s2.values["BUS0"] == 0b11110000 assert abs(s2.values["ANALOG0"] - 3.14159) < 0.001 - # Sample 3 at time 25us = 25000ns + # Sample 3 at time 25us = 0.000025s s3 = samples[3] - assert s3.time_ns == 25000 + assert abs(s3.time - 0.000025) < 1e-12 assert s3.values["D0"] == 1 assert s3.values["D1"] == 1 assert s3.values["D2"] == 0 @@ -94,9 +94,9 @@ def test_vcd_parser_comprehensive(): assert s3.values["BUS0"] == 0b10101010 assert abs(s3.values["ANALOG0"] - 0.0) < 0.001 - # Sample 4 at time 100us = 100000ns + # Sample 4 at time 100us = 0.0001s s4 = samples[4] - assert s4.time_ns == 100000 + assert abs(s4.time - 0.0001) < 1e-12 assert s4.values["D0"] == 0 assert s4.values["D1"] == 0 assert s4.values["D2"] == 0 @@ -131,7 +131,7 @@ def test_vcd_parser_timescale_variations(): samples = list(result.decode()) assert len(samples) >= 1 # First sample at time 0 - assert samples[0].time_ns == 0 + assert samples[0].time == 0.0 def test_vcd_parser_empty_timestamps(): @@ -155,9 +155,9 @@ def test_vcd_parser_empty_timestamps(): samples = list(result.decode()) # Should have 3 samples (empty timestamp line skipped) assert len(samples) == 3 - assert samples[0].time_ns == 0 - assert samples[1].time_ns == 10 - assert samples[2].time_ns == 20 + assert samples[0].time == 0.0 + assert samples[1].time == 1e-8 # 10ns + assert samples[2].time == 2e-8 # 20ns def test_vcd_parser_large_channel_count(): @@ -195,7 +195,7 @@ def test_vcd_parser_large_channel_count(): assert len(samples) == 2 s0 = samples[0] assert isinstance(s0, Sample) - assert s0.time_ns == 0 + assert s0.time == 0.0 assert s0.values["CH0"] == 1 # Single char: ! assert s0.values["CH93"] == 0 # Single char: ~ assert s0.values["CH94"] == 1 # Two char: aa @@ -206,7 +206,7 @@ def test_vcd_parser_large_channel_count(): # Verify second sample s1 = samples[1] - assert s1.time_ns == 100 + assert abs(s1.time - 1e-7) < 1e-15 # 100ns assert s1.values["CH0"] == 0 assert s1.values["CH93"] == 1 assert s1.values["CH94"] == 0 From ad33957737e86dd7f204353b3feb6790462c0b5f Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Mon, 8 Dec 2025 22:48:03 +0100 Subject: [PATCH 17/18] sigrok: better CaptureResult output --- .../jumpstarter_driver_sigrok/common.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py index cc683a694..bffbb97e0 100644 --- a/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py +++ b/python/packages/jumpstarter-driver-sigrok/jumpstarter_driver_sigrok/common.py @@ -105,6 +105,23 @@ class CaptureResult(BaseModel): triggers: dict[str, str] | None = None decoders: list[DecoderConfig] | None = None + def __str__(self) -> str: + """Format CaptureResult with truncated data_b64 field.""" + data_len = len(self.data_b64) + if data_len <= 50: + data_preview = self.data_b64 + else: + # Show first 50 and last 50 chars with ellipsis + data_preview = f"{self.data_b64[:25]}...{self.data_b64[-25:]} ({data_len} chars)" + + return ( + f"CaptureResult(output_format='{self.output_format}', " + f"sample_rate='{self.sample_rate}', " + f"data_size={len(self.data)} bytes, " + f"channels={len(self.channel_map)}, " + f"data_b64='{data_preview}')" + ) + @property def data(self) -> bytes: """Get the captured data as bytes (auto-decodes from base64).""" From 2c92c601c7634691119129ca83b1dc6482d4f625 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Thu, 22 Jan 2026 09:40:26 +0100 Subject: [PATCH 18/18] sigrok: fix vcs path --- python/packages/jumpstarter-driver-sigrok/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/packages/jumpstarter-driver-sigrok/pyproject.toml b/python/packages/jumpstarter-driver-sigrok/pyproject.toml index f6cd63aa4..58518119b 100644 --- a/python/packages/jumpstarter-driver-sigrok/pyproject.toml +++ b/python/packages/jumpstarter-driver-sigrok/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [tool.hatch.version] source = "vcs" -raw-options = { 'root' = '../../'} +raw-options = { 'root' = '../../../'} [tool.hatch.metadata.hooks.vcs.urls] Homepage = "https://jumpstarter.dev"