develop -> main

.github/workflows/ci.yml

@@ -39,28 +39,52 @@ jobs:
     name: Unit tests on various OSes and Python versions
     strategy:
       matrix:
-        python-version: ["3.10", "3.11", "3.12"]
+        python-version:
+          - python-version: "3.10"
+            continue-on-error: false
+          - python-version: "3.11"
+            continue-on-error: false
+          - python-version: "3.12"
+            continue-on-error: false
+          # TODO - fix issue with Jaxlib not installing
+          - python-version: "3.13"
+            continue-on-error: true
         os:
-          - macos-latest
-          - windows-latest
-          - ubuntu-latest
-    runs-on: ${{ matrix.os }}
+          - os: macos-latest
+            continue-on-error: false
+          - os: windows-latest
+            continue-on-error: false
+          - os: ubuntu-latest
+            continue-on-error: false
+    runs-on: ${{ matrix.os.os }}
     steps:
       - uses: actions/checkout@v4
       - name: Setup PDM
         uses: pdm-project/setup-pdm@v4
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ matrix.python-version.python-version }}
           cache: true
       - name: Install dependencies
         run: |
           pdm venv create --with-pip --force $PYTHON
           pdm sync --dev -G:all
+        continue-on-error: ${{ matrix.os.continue-on-error || matrix.python-version.continue-on-error }}
       - run: pdm run test-all
+        continue-on-error: ${{ matrix.os.continue-on-error || matrix.python-version.continue-on-error }}
+
+  docker-test:
+    name: Build Docker image and run tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build Docker image
+        run: docker build -t dial:test --target test .
+      - name: Run tests in container
+        run: docker run --rm dial:test pytest tests/
 
   continuous-deployment:
     name: Update deployments automatically
-    needs: ['lint', 'test-all']
+    needs: ['lint', 'test-all', 'docker-test']
     if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop' }}
     runs-on: ubuntu-latest
     steps:

.github/workflows/docker-release.yml

@@ -0,0 +1,56 @@
+name: Release
+
+on:
+  push:
+    branches:
+      - develop
+      - main    
+    tags:
+      - 'v*.*.*'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  publish-image:
+    name: Build and publish Docker image
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=tag
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          build-args: |
+            PRODUCTION=1
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          target: runtime

.github/workflows/helm-release.yml

@@ -0,0 +1,41 @@
+name: Helm
+
+on:
+  push:
+    branches:
+      - "main"
+    paths:
+      - "chart/"
+  workflow_dispatch:
+
+jobs:
+  helm-release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Configure Git
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+
+      - name: Add Helm repo dependencies
+        run: |
+          helm repo add bitnami https://charts.bitnami.com/bitnami
+
+      - name: Run chart-releaser for all applications
+        uses: helm/chart-releaser-action@v1.6.0
+        env:
+          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        with:
+          charts_dir: charts

.gitignore

@@ -8,6 +8,7 @@
 # ignore all configuration files except for our default one
 *.json
 !local-conf.json
+!local-docker-conf.json
 
 # testing reports
 reports/

Dockerfile

@@ -4,7 +4,7 @@ FROM ${REPO}python:3.12-slim AS environment
 
 # set to non-empty string to skip dev dependencies
 ARG PRODUCTION=
-ENV PDM_VERSION=2.19.3 \
+ENV PDM_VERSION=2.21.0 \
   PDM_HOME=/usr/local \
   PDM_CHECK_UPDATE=false \
   PIP_NO_CACHE_DIR=off \

README.md

@@ -40,36 +40,11 @@ To run linter and automatically fix errors:
 
 ## running infrastructure locally
 
-You can use `docker compose up -d` to automatically spin up both a broker instance and a database instance locally.
+You can use `docker compose up -d -f docker-compose-dev.yml` or to automatically spin up both a broker instance and a database instance locally.
 
-To remove the infrastructure containers: `docker compose down -v`; leave off the `-v` flag if you would like to persist the DB data.
-
-Note that if you are also running the Client/Service scripts in Docker, you will need to make sure that you add in appropriate `host` properties inside of `local-conf.json` (the host names are the names of the services in `docker-compose.yml`, instead of `127.0.0.1`):
-
-```json
-{
-  "intersect": {
-    "brokers": [
-      {
-        "username": "intersect_username",
-        "password": "intersect_password",
-        "host": "broker",
-        "port": 1883,
-        "protocol": "mqtt3.1.1"
-      }
-    ]
-  },
-  "dial": {
-    "mongo": {
-      "username": null,
-      "password": null,
-      "host": "mongodb",
-      "port": 27017
-    }
-  }
-}
+If you also want to run DIAL inside the container, you can instead run `docker compose up -d`
 
-```
+To remove the infrastructure containers: `docker compose down -v`; leave off the `-v` flag if you would like to persist the DB data.
 
 ## Running
 
@@ -82,6 +57,7 @@ In a separate terminal, you can run one of the following clients:
 CLI arg `--config` or environment variable `DIAL_CONFIG_FILE` should be a path to a valid JSON configuration. If neither value is set, it will default to `local-conf.json` .
 
 - `local-conf.json` - If you set up the infrastructure locally via `docker compose up`, use this config file.
+- `local-conf-docker.json` - This config file should only be used if you are running DIAL in the Docker image as well.
 
 ## Docker
 
@@ -98,6 +74,43 @@ To run the client, select one of the following:
 - Automatic run: `docker run --rm -it -e DIAL_CONFIG_FILE=/app/config.json -v path-to-your-config.json:/app/config.json dial-image python scripts/automated_client.py`
 - Manual run: `docker run --rm -it -e DIAL_CONFIG_FILE=/app/config.json -v path-to-your-config.json:/app/config.json dial-image python scripts/manual_client.py`
 
+## Kubernetes Deployment (Helm)
+
+A Helm chart is available for deploying Dial to Kubernetes with MongoDB database support. The chart includes:
+
+- Dial service deployment
+- MongoDB subchart from bitnami
+
+### Quick Start
+
+```bash
+cd charts/dial
+helm dependency update
+helm install dial . -n dial --create-namespace
+```
+
+For detailed Helm chart documentation, see:
+- [charts/dial/README.md](charts/dial/README.md) - Comprehensive Helm documentation
+
+### Common Helm Commands
+
+With custom INTERSECT configuration:
+```bash
+helm install dial . -n dial --create-namespace -f values.yaml -f values.config.yaml
+```
+
+With NodePort service:
+```bash
+helm install dial . -n dial --create-namespace -f values.yaml -f values.nodePort.yaml
+```
+
+With external MongoDB:
+```bash
+helm install dial . -n dial --create-namespace \
+  --set mongodb.enabled=false \
+  --set externalMongoDB.connectionString="mongodb://user:pass@host:27017/dial"
+```
+
 ## Testing
 
 You will need `pytest` installed to run the tests, it should be automatically included in your virtual environment if using the PDM workflow.

charts/.gitignore

@@ -0,0 +1 @@
+charts

charts/dial/.gitignore

@@ -0,0 +1,2 @@
+Chart.lock
+charts/

charts/dial/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/dial/Chart.yaml

@@ -0,0 +1,12 @@
+apiVersion: v2
+name: dial
+description: "DIAL (Distributed INTERSECT Active Learning): An INTERSECT service that provides Bayesian optimization and active learning"
+version: 0.1.0
+dependencies:
+  - name: common
+    repository: https://charts.bitnami.com/bitnami
+    version: "^2.9.1"
+  - name: mongodb
+    repository: https://charts.bitnami.com/bitnami
+    version: "^15.0.0"
+    condition: mongodb.enabled