Merge/security hardening to main

.env.example

@@ -0,0 +1,26 @@
+# Copy this file to .env and fill in the values before running the server.
+# Never commit .env to git.
+
+# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
+SECRET_KEY=
+
+# Path to the data volume (job files and SQLite DB)
+IMMUNOLYSER_DATA=/pvol
+
+# Set to 'true' if running inside Docker, 'false' on bare server
+IS_DOCKER=false
+
+# Set to 'true' only for local development — NEVER in production
+DEBUG=False
+
+# Sender address for job notification emails (uses Monash SMTP relay, no password needed)
+EMAIL_ADDRESS=noreply@immunolyser.erc.monash.edu
+
+# Public base URL of the server (used in email links)
+BASE_URL=https://immunolyser.erc.monash.edu
+
+# UUID of the demo job shown on the homepage
+DEMO_TASK_ID=
+
+# Days before job files are deleted from the server (default: 30)
+DATA_RETENTION_DAYS=30

.gitignore

@@ -1,6 +1,13 @@
 app/tools
-app/static/images/*
+app/static/images
+.env
 app/__pycache__/*
 __pycache__/
 immunolyser-data/*
-lenv/*
+lenv/*
+.vscode/launch.json
+test.py
+data_files/*
+data_files_2/*
+tools/*
+test.ipynb

Dockerfile

@@ -0,0 +1,173 @@
+FROM python:3.12-slim
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Install build tools, R, tcsh, and other required dependencies
+RUN apt-get update && apt-get install -y \
+    bash \
+    git \
+    tar \
+    build-essential \
+    wget \
+    libssl-dev \
+    libbz2-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    zlib1g-dev \
+    sqlite3 \
+    r-base \
+    man-db \
+    ncompress \
+    tcsh \
+    && \
+    wget https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926/ghostscript-9.26.tar.gz && \
+    tar -xzf ghostscript-9.26.tar.gz && \
+    cd ghostscript-9.26 && \
+    ./configure && make && make install && \
+    pip3 install --no-cache-dir gdown && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Clone the repository and checkout the develop branch in one go
+RUN git clone --branch develop --single-branch https://github.com/prmunday/Immunolyser /app/Immunolyser
+
+# Change to the repository directory
+WORKDIR /app/Immunolyser
+
+# Copy the seq2logo tar.gz file from the local tools folder to the container
+COPY app/tools/seq2logo-2.1.all.tar.gz /app/Immunolyser/app/tools/
+
+# Create a tools directory and extract the tar.gz file there
+RUN mkdir -p /app/Immunolyser/app/tools && \
+    tar -xzf /app/Immunolyser/app/tools/seq2logo-2.1.all.tar.gz -C /app/Immunolyser/app/tools && \
+    rm /app/Immunolyser/app/tools/seq2logo-2.1.all.tar.gz
+
+# Copy the gibbscluster tar.gz file to the container
+COPY app/tools/gibbscluster-2.0f.Linux.tar.gz /app/Immunolyser/app/tools/
+
+# Extract gibbscluster tar.gz
+RUN mkdir -p /app/Immunolyser/app/tools && \
+    tar -xvf /app/Immunolyser/app/tools/gibbscluster-2.0f.Linux.tar.gz -C /app/Immunolyser/app/tools && \
+    rm /app/Immunolyser/app/tools/gibbscluster-2.0f.Linux.tar.gz
+
+# Update GIBBS path in the gibbscluster file
+RUN sed -i 's|setenv\s*GIBBS .*|setenv GIBBS /app/Immunolyser/app/tools/gibbscluster-2.0|' /app/Immunolyser/app/tools/gibbscluster-2.0/gibbscluster
+
+# Comment out the line containing '$resdir .= "/$prefix";' in GibbsCluster-2.0e_SA.pl; command update for seq2logo and gibbs
+RUN sed -i \
+    -e 's|^\(\s*\$resdir .= "/\$prefix";\)|# \1  # Comment or remove this line|' \
+    -e 's|^\(my \$barplot = "\$resdir/images/\$prefix.gibbs.KLDvsCluster.barplot.png";\)|my \$barplot = "\$resdir/images/gibbsKLDvsCluster.barplot.JPG";|' \
+    -e '530s@.*@$cmd .= "$seq2logo -f $corefile -o $logofile -I 2 --format [JPEG] -b $wlc -C 2 -S 2 -t $title \&>/dev/null; ";@' \
+    /app/Immunolyser/app/tools/gibbscluster-2.0/GibbsCluster-2.0e_SA.pl
+
+# Copy the netMHCpan tar.gz file to the container
+COPY app/tools/netMHCpan-4.2c.Linux.tar.gz /app/Immunolyser/app/tools/
+
+# Uncompress and untar the netMHCpan package
+RUN mkdir -p /app/Immunolyser/app/tools && \
+    cat /app/Immunolyser/app/tools/netMHCpan-4.2c.Linux.tar.gz | gunzip | tar xvf - -C /app/Immunolyser/app/tools && \
+    rm /app/Immunolyser/app/tools/netMHCpan-4.2c.Linux.tar.gz && \
+    mkdir -p /app/Immunolyser/app/tools/netMHCpan-4.2/tmp
+
+# Copy the netMHCIIpan tar.gz file to the container
+COPY app/tools/netMHCIIpan-4.3j.Linux.tar.gz /app/Immunolyser/app/tools/
+
+# Uncompress and untar the netMHCIIpan package
+RUN mkdir -p /app/Immunolyser/app/tools && \
+    tar -xvf /app/Immunolyser/app/tools/netMHCIIpan-4.3j.Linux.tar.gz -C /app/Immunolyser/app/tools && \
+    rm /app/Immunolyser/app/tools/netMHCIIpan-4.3j.Linux.tar.gz && \
+    man -d /app/Immunolyser/app/tools/netMHCIIpan-4.3/netMHCIIpan.1 | compress > /app/Immunolyser/app/tools/netMHCIIpan-4.3/netMHCIIpan.Z
+
+# Update netMHCIIpan configuration to use the correct NMHOME path
+RUN sed -i 's|setenv\s*NMHOME\s*/tools/src/netMHCIIpan-4.3|setenv NMHOME ${PWD}/app/tools/netMHCIIpan-4.3|' \
+    /app/Immunolyser/app/tools/netMHCIIpan-4.3/netMHCIIpan
+
+# Update netMHCpan configuration to use the correct NMHOME and TMPDIR paths
+RUN sed -i \
+    -e 's|setenv\s*NMHOME\s*/tools/src/netMHCpan-4.2|setenv NMHOME ${PWD}/app/tools/netMHCpan-4.2|' \
+    -e 's|setenv\s*TMPDIR\s*/tmp|setenv TMPDIR $NMHOME/tmp|' \
+    /app/Immunolyser/app/tools/netMHCpan-4.2/netMHCpan
+
+# Clone MixMHCpred repository
+RUN git clone https://github.com/GfellerLab/MixMHCpred.git /app/Immunolyser/app/tools/MixMHCpred && \
+    chmod +x /app/Immunolyser/app/tools/MixMHCpred/MixMHCpred
+
+RUN wget https://github.com/GfellerLab/MixMHC2pred/releases/download/v2.0.2.2/MixMHC2pred-2.0.zip -P /app/Immunolyser/app/tools && \
+    unzip -o /app/Immunolyser/app/tools/MixMHC2pred-2.0.zip -d /app/Immunolyser/app/tools/MixMHC2pred-2.0 && \
+    rm /app/Immunolyser/app/tools/MixMHC2pred-2.0.zip
+
+# Download Alleles_list_Mouse.txt and put it in PWMdef directory
+RUN wget http://ec2-18-188-210-66.us-east-2.compute.amazonaws.com:4000/data/Alleles_lists/Alleles_list_Mouse.txt -P /app/Immunolyser/app/tools/MixMHC2pred-2.0/PWMdef
+
+# Clone MHC-TP and switch to netmhcpan-data-update-2025 branch
+RUN git clone https://github.com/PurcellLab/MHC-TP.git /app/Immunolyser/app/tools/HLA-PepClust && \
+    cd /app/Immunolyser/app/tools/HLA-PepClust && \
+    git fetch origin netmhcpan-data-update-2025 && \
+    git checkout netmhcpan-data-update-2025
+
+# Set up Python 3.11 virtual environment and install the package
+RUN cd /app/Immunolyser/app/tools/HLA-PepClust && \
+    python3 -m venv hlapepclust-env && \
+    /bin/bash -c "source hlapepclust-env/bin/activate && pip install -e . && deactivate"
+
+# Download the large ref_data zip file and unzip it
+RUN mkdir -p /app/Immunolyser/app/tools/HLA-PepClust/data/ref_data && \
+    cd /app/Immunolyser/app/tools/HLA-PepClust/data/ref_data && \
+    python3 -m gdown 'https://drive.google.com/uc?id=1iAAvir1woMOnURkP46zr_ETqpW2oUgGD' && \
+    unzip Gibbs_motifs_human.zip && \
+    rm Gibbs_motifs_human.zip
+
+# Install mhcflurry
+RUN pip install mhcflurry
+
+# Fetch mhcflurry downloads
+RUN mhcflurry-downloads fetch
+
+# Create a virtual environment for Python 3
+RUN python3 -m venv lenv
+
+# Install dependencies for Python 2 and Python 3
+RUN /bin/bash -c "source lenv/bin/activate && \
+    pip install -r requirements_python2.txt && \
+    pip install -r requirements_python3.txt"
+
+# Install Celery, sqlacheny
+RUN /bin/bash -c "pip install celery"
+RUN /bin/bash -c "pip install SQLAlchemy==2.0.31"
+
+# Run the hotfix script
+RUN /bin/bash -c "python hotfix_package_files.py"
+
+# Download and extract Python 2.7.18
+RUN wget https://www.python.org/ftp/python/2.7.18/Python-2.7.18.tgz && \
+    tar xvf Python-2.7.18.tgz
+
+# Build and install Python 2.7.18
+WORKDIR /app/Immunolyser/Python-2.7.18
+RUN ./configure && \
+    make && \
+    make install
+
+# Install pip for Python 2.7
+RUN wget https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
+    python2 get-pip.py
+
+# Install numpy for Python 2.7
+RUN python2 -m pip install numpy
+
+# Change to the repository directory
+WORKDIR /app/Immunolyser
+
+# Expose Flask and Celery ports
+EXPOSE 5000
+EXPOSE 5555
+
+# Set a default environment variable for IMMUNOLYSER_DATA
+ENV IMMUNOLYSER_DATA=/data
+
+# Copy entrypoint script
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+# Set the entrypoint script
+ENTRYPOINT ["/entrypoint.sh"]

PROD_DEPLOYMENT.sh

@@ -0,0 +1,168 @@
+#!/bin/bash
+# Immunolyser Production Deployment — security/hardening branch
+# Run this on the new prod VM after:
+# 1. New 200GB volume created + attached at /dev/vdb
+# 2. Ubuntu 22.04 LTS fresh install
+# 3. SSH key configured
+
+set -e
+
+echo "=== Step 1: System Dependencies ==="
+sudo apt update && sudo apt install -y \
+    python3-pip python3-venv redis-server \
+    build-essential wget git sqlite3 \
+    r-base tcsh ncompress unzip perl \
+    python2 fonts-urw-base35 gsfonts
+
+# Python2 pip + deps
+python2 -m ensurepip
+python2 -m pip install numpy matplotlib
+
+echo "=== Step 2: Clone Repo ==="
+sudo mkdir -p /var/www/Immunolyser
+sudo chown ubuntu:ubuntu /var/www/Immunolyser
+git clone -b security/hardening https://github.com/PurcellLab/Immunolyser.git /var/www/Immunolyser
+
+echo "=== Step 3: Install Bioinformatics Tools ==="
+cd /var/www/Immunolyser/app/tools
+
+# seq2logo + gibbscluster
+tar -xzf seq2logo-2.1.all.tar.gz
+tar -xzf gibbscluster-2.0f.Linux.tar.gz
+
+# netMHCpan-4.2 (you must have downloaded this from DTU)
+tar -xzf netMHCpan-4.2c.Linux.tar.gz
+mkdir -p netMHCpan-4.2/tmp
+chmod +x netMHCpan-4.2/netMHCpan
+# EDIT: replace setenv NMHOME paths in netMHCpan script:
+sed -i 's|setenv  NMHOME.*|setenv  NMHOME  /var/www/Immunolyser/app/tools/netMHCpan-4.2|g' netMHCpan-4.2/netMHCpan
+
+# netMHCIIpan-4.3 (you must have downloaded this from DTU)
+tar -xzf netMHCIIpan-4.3j.Linux.tar.gz
+mkdir -p netMHCIIpan-4.3/tmp
+chmod +x netMHCIIpan-4.3/netMHCIIpan
+sed -i 's|setenv  NMHOME.*|setenv  NMHOME /var/www/Immunolyser/app/tools/netMHCIIpan-4.3|g' netMHCIIpan-4.3/netMHCIIpan
+
+# MixMHCpred
+wget https://github.com/GfellerLab/MixMHCpred/archive/refs/tags/v3.0.tar.gz -O MixMHCpred.tar.gz
+tar -xzf MixMHCpred.tar.gz
+mv MixMHCpred-3.0 MixMHCpred
+chmod +x MixMHCpred/MixMHCpred
+
+# MixMHC2pred
+wget https://github.com/GfellerLab/MixMHC2pred/archive/refs/tags/v2.0.2.2.tar.gz -O MixMHC2pred.tar.gz
+tar -xzf MixMHC2pred.tar.gz
+mv MixMHC2pred-2.0.2.2 MixMHC2pred-2.0
+chmod +x MixMHC2pred-2.0/MixMHC2pred_unix
+
+# Ghostscript 9.53.3 (not 9.55.0 — font bug)
+wget https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs9533/ghostscript-9.53.3-linux-x86_64.tgz
+tar -xzf ghostscript-9.53.3-linux-x86_64.tgz
+chmod +x ghostscript-9.53.3-linux-x86_64/gs-9533-linux-x86_64
+# Point seq2logo to it:
+sed -i "s|gsPath='gs'|gsPath='/var/www/Immunolyser/app/tools/ghostscript-9.53.3-linux-x86_64/gs-9533-linux-x86_64'|" \
+    /var/www/Immunolyser/app/tools/seq2logo-2.1/Seq2Logo.py
+
+# HLA-PepClust (MUST clone as HLA-PepClust — hardcoded in code)
+git clone https://github.com/PurcellLab/MHC-TP.git HLA-PepClust
+cd HLA-PepClust
+git fetch origin netmhcpan-data-update-2025
+git checkout netmhcpan-data-update-2025
+# Fix scipy version pin bug
+sed -i 's/scipy==1.16.1/scipy>=1.13/' setup.py
+# Create venv + install
+python3 -m venv hlapepclust-env
+source hlapepclust-env/bin/activate
+pip install -e .
+deactivate
+# Download ref_data
+python3 -m pip install gdown
+mkdir -p data/ref_data
+cd data/ref_data
+python3 -m gdown 'https://drive.google.com/uc?id=1iAAvir1woMOnURkP46zr_ETqpW2oUgGD'
+unzip Gibbs_motifs_human.zip
+rm Gibbs_motifs_human.zip
+
+echo "=== Step 4: Python Virtualenv + Dependencies ==="
+cd /var/www/Immunolyser
+python3 -m venv lenv
+source lenv/bin/activate
+pip install -r requirements_python3.txt
+pip install mhcflurry
+mhcflurry-downloads fetch   # ~2GB, takes 10-15 min
+
+echo "=== Step 5: Mount Data Volume ==="
+sudo mkdir -p /pvol
+sudo mount /dev/vdb /pvol
+sudo chown ubuntu:ubuntu /pvol
+# Add to fstab for persistence:
+echo "/dev/vdb /pvol ext4 defaults,nofail 0 2" | sudo tee -a /etc/fstab
+
+echo "=== Step 6: Create Symlink for Job Output ==="
+ln -s /pvol /var/www/Immunolyser/app/static/images
+
+echo "=== Step 7: Environment File ==="
+cd /var/www/Immunolyser
+cp .env.example .env
+# EDIT .env with your values:
+# SECRET_KEY=$(python3 -c "import secrets; print(secrets.token_hex(32))")
+# IMMUNOLYSER_DATA=/pvol
+# IS_DOCKER=false
+# DEBUG=False
+# EMAIL_ADDRESS=noreply@immunolyser.erc.monash.edu
+# BASE_URL=https://immunolyser.erc.monash.edu
+# DEMO_TASK_ID=<uuid from demo job, or leave empty>
+# DATA_RETENTION_DAYS=30
+
+echo "!!! PAUSE: Edit .env manually with SECRET_KEY and other values !!!"
+nano .env
+
+echo "=== Step 8: Initialize Database ==="
+source lenv/bin/activate
+python3 -c "from app.job_registry import init_job_registry; from app.email_registry import init_email_registry; init_job_registry(); init_email_registry(); print('DB initialised')"
+deactivate
+
+echo "=== Step 9: Systemd Services ==="
+
+# Flask app service
+sudo tee /etc/systemd/system/immunolyser.service > /dev/null <<EOF
+[Unit]
+Description=Immunolyser Flask App
+After=network.target redis.service
+
+[Service]
+User=ubuntu
+WorkingDirectory=/var/www/Immunolyser
+EnvironmentFile=/var/www/Immunolyser/.env
+ExecStart=/var/www/Immunolyser/lenv/bin/gunicorn --workers 2 --bind 0.0.0.0:5000 --timeout 120 firstdemo:app
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+# Celery worker service
+sudo tee /etc/systemd/system/immunolyser-celery.service > /dev/null <<EOF
+[Unit]
+Description=Immunolyser Celery Worker
+After=network.target redis.service
+
+[Service]
+User=ubuntu
+WorkingDirectory=/var/www/Immunolyser
+EnvironmentFile=/var/www/Immunolyser/.env
+ExecStart=/var/www/Immunolyser/lenv/bin/celery -A app.celery worker --loglevel=info --concurrency=1
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+sudo systemctl daemon-reload
+sudo systemctl enable immunolyser immunolyser-celery
+sudo systemctl start immunolyser immunolyser-celery
+
+echo "=== Deployment Complete ==="
+echo "Flask app: http://<prod-ip>:5000"
+echo "Check status: sudo systemctl status immunolyser immunolyser-celery"
+echo "Logs: sudo journalctl -u immunolyser -f"