Release 2.0.0

.github/workflows/github-actions.yml

@@ -1,9 +1,7 @@
 on: push
-
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     services:
@@ -18,40 +16,36 @@ jobs:
         # needed because the postgres container does not provide a healthcheck
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
 
-
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.9
-    - name: Install dependencies
-      run: |
-        pip install pip==19.3.1
-        pip install --force-reinstall 'setuptools<58.0.0'
-        pip install -r requirements.txt
-        pip install -r requirements-toil.txt
-        #python manage.py migrate
-    - name: Run migrations
-      run: python manage.py migrate
-    - name: Run test
-      #run: python manage.py test
-      run: |
-        coverage run --source='.' manage.py test
-        coverage report -m --fail-under=75
-    - name: Run flake8
-      uses: suo/flake8-github-action@releases/v1
-      with:
-          checkName: 'build'   # NOTE: this needs to be the same as the job name
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          pip install pip==19.3.1
+          pip install -r requirements.txt
+          pip install -r requirements-toil.txt
+      - name: Run migrations
+        run: python manage.py migrate
+      - name: Run test
+        #run: python manage.py test
+        run: |
+          coverage run --source='.' manage.py test
+          coverage report -m --fail-under=75
+      - name: Run flake8
+        uses: suo/flake8-github-action@releases/v1
+        with:
+          checkName: "build" # NOTE: this needs to be the same as the job name
     env:
-      #database env variables 
+      #database env variables
       RIDGEBACK_DB_NAME: github_actions
       RIDGEBACK_DB_PASSWORD: postgres
       RIDGEBACK_DB_USERNAME: postgres
 
       #lsf env variables
       RIDGEBACK_LSF_WALLTIME: 10:00
-      RIDGEBACK_LSF_SLA: SLA
 
       #celery env variables
       CELERY_LOG_PATH: /sample_path
@@ -69,23 +63,33 @@ jobs:
       RIDGEBACK_RABBITMQ_USERNAME: sample_username
       RIDGEBACK_RABBITMQ_PASSWORD: sample_password
 
-      #pipeline env variables 
+      #pipeline env variables
 
       ARGOS_JOB_STORE_ROOT: /sample_path
       ARGOS_WORK_DIR_ROOT: /sample_path
       ARGOS_TMP_DIR_ROOT: /sample_path
+      ARGOS_PARTITION: sample_SLA
       TEMPO_JOB_STORE_ROOT: /sample_path
       TEMPO_WORK_DIR_ROOT: /sample_path
       TEMPO_TMP_DIR_ROOT: /sample_path
+      TEMPO_PARTITION: sample_SLA
       ACCESS_JOB_STORE_ROOT: /sample_path
       ACCESS_WORK_DIR_ROOT: /sample_path
       ACCESS_TMP_DIR_ROOT: /sample_path
+      ACCESS_PARTITION: sample_SLA
       CMO_CH_JOB_STORE_ROOT: /sample_path
       CMO_CH_WORK_DIR_ROOT: /sample_path
       CMO_CH_TMP_DIR_ROOT: /sample_path
+      CMO_CH_PARTITION: sample_SLA
       ACCESS_HEME_JOB_STORE_ROOT: /sample_path
       ACCESS_HEME_WORK_DIR_ROOT: /sample_path
       ACCESS_HEME_TMP_DIR_ROOT: /sample_path
+      ACCESS_HEME_PARTITION: sample_SLA
+      MICROBIOME_JOB_STORE_ROOT: /sample_path
+      MICROBIOME_WORK_DIR_ROOT: /sample_path
+      MICROBIOME_TMP_DIR_ROOT: /sample_path
+      MICROBIOME_PARTITION: sample_SLA
       DEFAULT_JOB_STORE_ROOT: /sample_path
       DEFAULT_WORK_DIR_ROOT: /sample_path
       DEFAULT_TMP_DIR_ROOT: /sample_path
+      DEFAULT_PARTITION: sample_SLA

.gitignore

@@ -6,3 +6,4 @@ conda/
 toil
 .toil/
 *.sif
+.env

.travis.yml

@@ -12,7 +12,7 @@ addons:
       - postgresql-14
       - postgresql-client-14
 python:
-  - "3.9"
+  - "3.10"
 before_install:
   - sudo apt-get update
   - sudo service postgresql restart 14

Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.10-slim
+
+LABEL org.opencontainers.image.vendor="MSKCC" \
+      org.opencontainers.image.authors="Nikhil Kumar (kumarn1@mskcc.org)" \
+      org.opencontainers.image.created="2025-09-15T16:04:00Z" \
+      org.opencontainers.image.licenses="Apache-2.0" \
+      org.opencontainers.image.version="1.0.0" \
+      org.opencontainers.image.source="https://github.com/mskcc/ridgeback" \
+      org.opencontainers.image.title="Ridgeback" \
+      org.opencontainers.image.description="API for running Toil and Nextflow jobs, supports LSF, SLURM, and singleMachine mode"
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update \
+     # Install dependencies
+        && apt-get -y --no-install-recommends install \
+            wget curl libldap2-dev libsasl2-dev procps libssl-dev libxml2-dev libxslt-dev \
+            libpq-dev gawk nodejs git build-essential openssh-client \
+     # Install alternative ssl library
+        && wget http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb \
+        && dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb \
+      # Install libffi6 for local python commands
+        && wget http://archive.ubuntu.com/ubuntu/pool/main/libf/libffi/libffi6_3.2.1-8_amd64.deb \
+        && dpkg -i libffi6_3.2.1-8_amd64.deb \
+    # Clean up image
+        && rm -rf /var/lib/apt/lists/*
+
+

README.md

@@ -1,6 +1,8 @@
 # Ridgeback
 
-API for Toil LSF
+API for running Toil and Nextflow jobs.
+Supports LSF, SLURM, and singleMachine mode
 
 ## Filebeat
+
 See https://app.gitbook.com/@mskcc-1/s/experimental-dev/filebeat

batch_systems/__init__.py

@@ -0,0 +1,4 @@
+from .lsf_client.lsf_client import LSFClient
+from .slurm_client.slurm_client import SLURMClient
+
+__all__ = ["LSFClient", "SLURMClient"]

batch_systems/batch_system.py

@@ -0,0 +1,140 @@
+from django.conf import settings
+from getpass import getuser
+import logging
+
+
+def get_batch_system(user=getuser()):
+    if settings.BATCH_SYSTEM == "LSF":
+        from batch_systems.lsf_client.lsf_client import LSFClient
+
+        return LSFClient(user)
+    elif settings.BATCH_SYSTEM == "SLURM":
+        from batch_systems.slurm_client.slurm_client import SLURMClient
+
+        return SLURMClient(user)
+    else:
+        raise Exception(f"Batch system {settings.BATCH_SYSTEM} not supported, please use either LSF or SLURM")
+
+
+class BatchClient(object):
+    """
+    Client for a generic Batch system
+
+    Attributes:
+        logger (logging): logging module
+    """
+
+    def __init__(self, user=getuser()):
+        """
+        init function
+        """
+        self.logger = logging.getLogger("BATCH_client")
+        self.logfileName = "batch.log"
+        self.name = "batch"
+        self.user = user
+
+    def submit(self, command, job_args, stdout, job_id, partition, env={}):
+        """
+        Submit command to bath system and store log in stdout
+
+        Args:
+            command (list): command to submit
+            job_args (list): Additional options for leader job
+            stdout (str): log file path
+            job_id (str): job_id
+            partition (str): the batch system partition to use
+            env (dict): Environment variables
+
+        Returns:
+            int: batch job id
+        """
+
+    def terminate(self, job_id):
+        """
+        Kill Batch job
+
+        Args:
+            job_id (str): job_id
+
+        Returns:
+            bool: successful
+        """
+
+    def set_walltime(self, expected_limit, hard_limit):
+        """
+        Set the walltime args of the batch job
+        """
+        walltime_args = []
+        return walltime_args
+
+    def set_memlimit(self, mem_limit, default=None):
+        """
+        Set the memlimit args of the batch job
+        """
+        mem_limit_args = []
+        return mem_limit_args
+
+    def set_num_tasks(self, num_tasks, default=None):
+        """
+        Set the number of tasks for the batch job
+        """
+        num_task_args = []
+        return num_task_args
+
+    def get_env_export_flag(self):
+        """
+        Flag to enable env propagation for the batch jobs
+
+        Returns:
+            str: CLI flag to enable env propagation
+        """
+
+    def set_group(self, group_id):
+        """
+        Set the group args of the batch job
+        """
+        group_id_args = []
+        return group_id_args
+
+    def set_stdout_file(self, stdout_file):
+        """
+        Set the output path of the log file
+        """
+        return []
+
+    def set_service_queue(self, partition):
+        """
+        Set the service queue parameter
+        """
+        service_queue_args = []
+        return service_queue_args
+
+    def status(self, external_job_id):
+        """Parse Batch status
+
+        Args:
+            external_job_id (str): Batch id
+
+        Returns:
+            tuple: (Ridgeback Status int, extra info)
+        """
+        status = None
+        return status
+
+    def suspend(self, job_id):
+        """
+        Suspend Batch job
+        Args:
+            job_id (str): id of job
+        Returns:
+            bool: successful
+        """
+
+    def resume(self, job_id):
+        """
+        Resume Batch job
+        Args:
+            job_id (str): id of job
+        Returns:
+            bool: successful
+        """