diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..3b35a29c40 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,67 @@ +name: ci +on: + push: + branches: [main, dev] + paths-ignore: + - '**.md' + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + paths-ignore: + - '**.md' + +permissions: + contents: read + packages: read + +jobs: + checks: + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false + timeout-minutes: 60 + runs-on: [self-hosted, Linux] + container: + image: ghcr.io/dimensionalos/ros-dev:dev + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + ALIBABA_API_KEY: ${{ secrets.ALIBABA_API_KEY }} + + steps: + - uses: actions/checkout@v4 + + - name: Fix permissions + run: | + git config --global --add safe.directory '*' + + - name: Install Python dependencies + run: uv sync --all-extras --no-extra dds --frozen + + - name: Remove pydrake stubs + run: | + find .venv/lib/*/site-packages/pydrake -name '*.pyi' -delete 2>/dev/null || true + + - name: Run tests + if: github.event_name != 'push' + run: | + /entrypoint.sh bash -c "source .venv/bin/activate && pytest --durations=0 -m 'not (tool or mujoco)'" + + - name: Run tests with coverage + if: github.event_name == 'push' + run: | + /entrypoint.sh bash -c "source .venv/bin/activate && _DIMOS_COV=1 coverage run -m pytest --durations=0 -m 'not (tool or mujoco)' && coverage combine && coverage html && coverage report" + + - name: Run mypy + if: ${{ !cancelled() }} + run: | + /entrypoint.sh bash -c "source .venv/bin/activate && MYPYPATH=/opt/ros/humble/lib/python3.10/site-packages mypy dimos" + + - name: Upload coverage report + if: github.event_name == 'push' && !cancelled() + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: htmlcov/ + + - name: Check disk space + if: failure() + run: | + df -h diff --git a/.github/workflows/docker.yml b/.github/workflows/docker-build.yml similarity index 65% rename from .github/workflows/docker.yml rename to .github/workflows/docker-build.yml index 0240df6ff7..a4e25bfcb2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker-build.yml @@ -1,29 +1,27 @@ -name: docker +name: docker-build on: push: - branches: - - main - - dev - paths-ignore: - - '**.md' - pull_request: - types: [opened, synchronize, reopened, ready_for_review] + branches: [main, dev] + paths: + - 'docker/**' + - '.github/workflows/docker-build.yml' + - '.github/workflows/_docker-build-template.yml' + schedule: + - cron: '0 4 * * 1' # Weekly Monday 4am UTC for base image security patches + workflow_dispatch: permissions: contents: read packages: write - pull-requests: read jobs: check-changes: - if: github.event_name != 'pull_request' || github.event.pull_request.draft == false runs-on: [self-hosted, Linux] outputs: - ros: ${{ steps.filter.outputs.ros }} - python: ${{ steps.filter.outputs.python }} - dev: ${{ steps.filter.outputs.dev }} - navigation: ${{ steps.filter.outputs.navigation }} - tests: ${{ steps.filter.outputs.tests }} + ros: ${{ steps.force.outputs.force == 'true' && 'true' || steps.filter.outputs.ros }} + python: ${{ steps.force.outputs.force == 'true' && 'true' || steps.filter.outputs.python }} + dev: ${{ steps.force.outputs.force == 'true' && 'true' || steps.filter.outputs.dev }} + navigation: ${{ steps.force.outputs.force == 'true' && 'true' || steps.filter.outputs.navigation }} branch-tag: ${{ steps.set-tag.outputs.branch_tag }} steps: - name: Fix permissions @@ -31,22 +29,32 @@ jobs: sudo chown -R $USER:$USER ${{ github.workspace }} || true - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check if full rebuild + id: force + run: | + if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "force=true" >> "$GITHUB_OUTPUT" + else + echo "force=false" >> "$GITHUB_OUTPUT" + fi + - id: filter + if: steps.force.outputs.force != 'true' uses: dorny/paths-filter@v3 with: base: ${{ github.event.before }} filters: | - # ros and python are (alternative) root images - # change to root stuff like docker.yml etc triggers rebuild of those - # which cascades into a full rebuild ros: - .github/workflows/_docker-build-template.yml - - .github/workflows/docker.yml + - .github/workflows/docker-build.yml - docker/ros/** python: - .github/workflows/_docker-build-template.yml - - .github/workflows/docker.yml + - .github/workflows/docker-build.yml - docker/python/** dev: @@ -54,14 +62,9 @@ jobs: navigation: - .github/workflows/_docker-build-template.yml - - .github/workflows/docker.yml + - .github/workflows/docker-build.yml - docker/navigation/** - tests: - - dimos/** - - pyproject.toml - - uv.lock - - name: Determine Branch Tag id: set-tag run: | @@ -78,15 +81,6 @@ jobs: echo "branch tag determined: ${branch_tag}" echo branch_tag="${branch_tag}" >> "$GITHUB_OUTPUT" - # just a debugger - inspect-needs: - needs: [check-changes, ros] - runs-on: dimos-runner-ubuntu-2204 - if: always() - steps: - - run: | - echo '${{ toJSON(needs) }}' - ros: needs: [check-changes] if: needs.check-changes.outputs.ros == 'true' @@ -202,48 +196,15 @@ jobs: to-image: ghcr.io/dimensionalos/ros-dev:${{ needs.check-changes.outputs.branch-tag }} dockerfile: dev - run-tests: - needs: [check-changes, ros-dev] - if: ${{ - always() && - needs.check-changes.result == 'success' && - (needs.check-changes.outputs.tests == 'true' || - needs.check-changes.outputs.ros == 'true' || - needs.check-changes.outputs.python == 'true' || - needs.check-changes.outputs.dev == 'true') - }} - uses: ./.github/workflows/tests.yml - secrets: inherit - with: - cmd: "_DIMOS_COV=1 coverage run -m pytest --durations=0 -m 'not (tool or mujoco)' && coverage combine && coverage html && coverage report" - upload-coverage: true - dev-image: ros-dev:${{ (needs.check-changes.outputs.python == 'true' || needs.check-changes.outputs.dev == 'true' || needs.check-changes.outputs.ros == 'true') && needs.ros-dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }} - - run-mypy: - needs: [check-changes, ros-dev] - if: ${{ - always() && - needs.check-changes.result == 'success' && - (needs.check-changes.outputs.tests == 'true' || - needs.check-changes.outputs.ros == 'true' || - needs.check-changes.outputs.python == 'true' || - needs.check-changes.outputs.dev == 'true') - }} - uses: ./.github/workflows/tests.yml - secrets: inherit - with: - cmd: "MYPYPATH=/opt/ros/humble/lib/python3.10/site-packages mypy dimos" - dev-image: ros-dev:${{ (needs.check-changes.outputs.python == 'true' || needs.check-changes.outputs.dev == 'true' || needs.check-changes.outputs.ros == 'true') && needs.ros-dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }} - - ci-complete: - needs: [check-changes, ros, python, ros-python, dev, ros-dev, run-tests, run-mypy] + build-complete: + needs: [check-changes, ros, python, ros-python, dev, ros-dev, navigation] runs-on: [self-hosted, Linux] if: always() steps: - - name: CI gate + - name: Build gate if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} run: | - echo "❌ One or more CI jobs failed or were cancelled" + echo "One or more build jobs failed or were cancelled" exit 1 - - name: CI passed - run: echo "✅ All CI checks passed or were intentionally skipped" + - name: Build passed + run: echo "All build jobs passed or were intentionally skipped" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 14eaaabab9..0000000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: tests - -on: - workflow_call: - inputs: - dev-image: - required: true - type: string - default: "dev:dev" - cmd: - required: true - type: string - upload-coverage: - required: false - type: boolean - default: false - -permissions: - contents: read - packages: read - -jobs: - run-tests: - timeout-minutes: 60 - runs-on: [self-hosted, Linux] - container: - image: ghcr.io/dimensionalos/${{ inputs.dev-image }} - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - ALIBABA_API_KEY: ${{ secrets.ALIBABA_API_KEY }} - - steps: - - uses: actions/checkout@v4 - - - name: Fix permissions - run: | - git config --global --add safe.directory '*' - - - name: Install Python dependencies - run: uv sync --all-extras --no-extra dds --frozen - - - name: Remove pydrake stubs - run: | - find .venv/lib/*/site-packages/pydrake -name '*.pyi' -delete 2>/dev/null || true - - - name: Run tests - run: | - /entrypoint.sh bash -c "source .venv/bin/activate && ${{ inputs.cmd }}" - - - name: Upload coverage report - if: inputs.upload-coverage && !cancelled() - uses: actions/upload-artifact@v4 - with: - name: coverage-report - path: htmlcov/ - - - name: check disk space - if: failure() - run: | - df -h diff --git a/docs/development/docker.md b/docs/development/docker.md index 2f3f4a98ec..c6622ff87f 100644 --- a/docs/development/docker.md +++ b/docs/development/docker.md @@ -100,34 +100,18 @@ Use the helper script: ## CI/CD Pipeline -The workflow in [`.github/workflows/docker.yml`](/.github/workflows/docker.yml) handles: +Images are built by [`.github/workflows/docker-build.yml`](/.github/workflows/docker-build.yml#L4) on merges to `main`/`dev` (when Docker files change) and weekly for base image security patches. -1. **Change detection** - Only rebuilds images when relevant files change -2. **Parallel builds** - ROS and non-ROS tracks build independently -3. **Cascade rebuilds** - Changes to base images trigger downstream rebuilds -4. **Test execution** - Tests run in the freshly built images +Tests and type checking run in [`.github/workflows/ci.yml`](/.github/workflows/ci.yml) using pre-built images. -### Trigger Paths +### Build Trigger Paths | Image | Triggers on changes to | |----------|------------------------------------------------------| | `ros` | `docker/ros/**`, workflow files | -| `python` | `docker/python/**`, `pyproject.toml`, workflow files | +| `python` | `docker/python/**`, workflow files | | `dev` | `docker/dev/**` | -### Test Jobs - -After images build, tests run in parallel: - -| Job | Image | Command | -|-------------------------|---------|---------------------------| -| `run-tests` | dev | `pytest` | -| `run-ros-tests` | ros-dev | `pytest && pytest -m ros` | -| `run-heavy-tests` | dev | `pytest -m heavy` | -| `run-lcm-tests` | dev | `pytest -m lcm` | -| `run-integration-tests` | dev | `pytest -m integration` | -| `run-mypy` | ros-dev | `mypy dimos` | - ## Dockerfile Structure ### Common Patterns