DODAS-TS · dciangot · Aug 12, 2021 · Aug 12, 2021 · Sep 15, 2021 · Sep 15, 2021
diff --git a/.github/workflows/docker-jhub-nb.yml b/.github/workflows/docker-jhub-nb.yml
@@ -46,6 +46,16 @@ jobs:
         file: ./stable/jhub-aas/docker/Dockerfile.base-htc
         platforms: linux/amd64
         build-args: "NOTEBOOK_VERSION=${{ env.RELEASE_VERSION }}"
+    - name: Build container base+HTC+DASK image
+      uses: docker/build-push-action@v2
+      with: 
+        context: ./stable/jhub-aas/docker/
+        outputs: "type=registry,push=true"
+        tags: |
+            ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/notebook:${{ env.RELEASE_VERSION }}-dask
+        file: ./stable/jhub-aas/docker/Dockerfile.dask
+        platforms: linux/amd64
+        build-args: "NOTEBOOK_VERSION=${{ env.RELEASE_VERSION }}"
     - name: Build container base+htc+ROOT image
       uses: docker/build-push-action@v2
       with: 

diff --git a/stable/spark/Chart.yaml → dev/spark/Chart.yaml b/stable/spark/Chart.yaml → dev/spark/Chart.yaml
diff --git a/stable/spark/README.md → dev/spark/README.md b/stable/spark/README.md → dev/spark/README.md
diff --git a/...rk/templates/spark-master-deployment.yaml → ...rk/templates/spark-master-deployment.yaml b/...rk/templates/spark-master-deployment.yaml → ...rk/templates/spark-master-deployment.yaml
diff --git a/stable/spark/values.yaml → dev/spark/values.yaml b/stable/spark/values.yaml → dev/spark/values.yaml
diff --git a/stable/cachingondemand/Chart.yaml b/stable/cachingondemand/Chart.yaml
@@ -3,8 +3,8 @@ annotations:
 apiVersion: v1
 appVersion: "4.11"
 description: A Helm chart for the deployment of a XCache cluster
-name: cachingondemand
-version: 1.0.6
+name: XCache-aaS 
+version: 1.0.7
 icon: https://xrootd.slac.stanford.edu/images/xrootd-logo.png
 maintainers:
   - name: dciangot

diff --git a/stable/fermi/Chart.yaml b/stable/fermi/Chart.yaml
@@ -1,8 +1,8 @@
 annotations:
-  category: stable
+  category: test
 apiVersion: v1
 name: Fermi-experiment
-version: 1.0.4
+version: 1.0.5
 appVersion: 8.9.9
 description: HTCondor pool for Fermi experiment
 keywords:

diff --git a/stable/htcondor-aas/Chart.yaml b/stable/htcondor-aas/Chart.yaml
@@ -4,7 +4,7 @@ apiVersion: v1
 appVersion: 8.9.9
 description: A Helm chart for the deployment of HTCondor cluster
 name: htcondor
-version: 2.1.11
+version: 2.1.13
 icon: https://research.cs.wisc.edu/htcondor/images/HTCondor_wiki_logo_small.png
 maintainers:
   - name: ttedesch

diff --git a/stable/htcondor-aas/templates/deployment_schedd.yaml b/stable/htcondor-aas/templates/deployment_schedd.yaml
@@ -25,6 +25,10 @@ spec:
                 values:
                 - schedd
             topologyKey: kubernetes.io/hostname
+      hostAliases:                                                                                                           
+      - hostnames:                                                                                                           
+        - {{ .Values.schedd.hostname }}
+        ip: {{ .Values.master.publicIP }}
       containers:
       - name: schedd
         resources:

diff --git a/stable/htcondor-aas/templates/deployment_wn.yaml b/stable/htcondor-aas/templates/deployment_wn.yaml
@@ -76,8 +76,9 @@ spec:
         livenessProbe:
           exec:
             command:
-            - ls
-            - /cvmfs/*
+            - bash
+            - -c
+            - "ls /cvmfs/*"
           initialDelaySeconds: 30
           periodSeconds: 30
         {{- end }}

diff --git a/stable/htcondor-aas/templates/external_apps/cvmfs-pod.yaml b/stable/htcondor-aas/templates/external_apps/cvmfs-pod.yaml
@@ -70,8 +70,9 @@ spec:
         livenessProbe:
           exec:
             command:
-            - ls
-            - /cvmfs/*
+            - bash
+            - -c
+            - "ls /cvmfs/*"
           initialDelaySeconds: 30
           periodSeconds: 600
         image: "{{ .Values.cvmfs.image }}:{{ .Values.cvmfs.tag }}"

diff --git a/stable/jhub-aas/Chart.yaml b/stable/jhub-aas/Chart.yaml
@@ -2,7 +2,7 @@ annotations:
   category: stable
 apiVersion: v1
 name: JHUBaaS
-version: 1.0.0
+version: 3.0.3
 appVersion: 1.3.0
 description: JHUBaaS Hub umbrella chart
 keywords:

diff --git a/stable/jhub-aas/docker/Dockerfile b/stable/jhub-aas/docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/dodas-ts/dodas-iam-client-rec:v0.0.5 as REGISTRATION
+FROM ghcr.io/dodas-ts/dodas-iam-client-rec:v0.0.6-pre3 as REGISTRATION
 
 FROM jupyterhub/k8s-hub:0.11.1
 
@@ -17,11 +17,8 @@ RUN echo "jovyan ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \
        rm /bin/sh && \
        ln -s /bin/bash /bin/sh
 
-USER $NB_UID
-
 RUN mkdir -p .init
 
 COPY --from=REGISTRATION /usr/local/bin/dodas-IAMClientRec /srv/.init/dodas-IAMClientRec
 
-
 CMD ["/usr/bin/start.sh", "jupyterhub"]
diff --git a/stable/jhub-aas/docker/Dockerfile.base b/stable/jhub-aas/docker/Dockerfile.base
@@ -17,10 +17,43 @@ RUN chmod +x /usr/local/bin/jupyterhub-singleuser && chmod +x /root/.init/spawn.
 RUN   apt-get update \
     && apt-get install -y  gnupg2 software-properties-common
 
-RUN apt-key adv --keyserver hkp://pgp.surfnet.nl --recv-keys ACDFB08FDC962044D87FF00B512839863D487A87 \
-    && add-apt-repository "deb http://repo.data.kit.edu/ubuntu/bionic ./" 
-
-RUN apt-get install -y oidc-agent
+#RUN apt-key adv --keyserver hkp://pgp.surfnet.nl --recv-keys ACDFB08FDC962044D87FF00B512839863D487A87 \
+#    && add-apt-repository "deb http://repo.data.kit.edu/ubuntu/bionic ./" 
+
+RUN apt-get update && apt-get install -y \
+    software-properties-common \
+    help2man \
+    check \
+    devscripts \
+    make \
+    sudo \
+    ca-certificates \
+    libcurl4-openssl-dev \
+    libsodium-dev \
+    libseccomp-dev \
+    libmicrohttpd-dev \
+    libsecret-1-dev \
+    python3 \
+    python3-pip \ 
+    wget \
+    fuse \
+    git \
+    build-essential \
+    debhelper \
+    pkg-config \
+    perl \
+    sed \
+    libqrencode-dev \
+    fakeroot
+
+
+RUN cd /opt/ && git clone https://github.com/indigo-dc/oidc-agent \
+    && cd oidc-agent \
+    && make \
+    && make install_lib \
+    && make install
+
+#RUN apt-get install -y oidc-agent
 
 COPY ./ca.crt /ca.crt
 

diff --git a/stable/jhub-aas/docker/Dockerfile.base-htc b/stable/jhub-aas/docker/Dockerfile.base-htc
@@ -5,17 +5,13 @@ USER root
 
 WORKDIR /tmp
 
-RUN  wget https://research.cs.wisc.edu/htcondor/tarball/8.9/8.9.9/release/condor-8.9.9-x86_64_Ubuntu20-stripped.tar.gz && \
-    tar -xzvf condor-8.9.9-x86_64_Ubuntu20-stripped.tar.gz && \
-    sudo mv condor-8.9.9-x86_64_Ubuntu20-stripped/bin/* /usr/bin/ && \
-    sudo mkdir /usr/lib/condor && \
-    sudo mv condor-8.9.9-x86_64_Ubuntu20-stripped/lib/condor/* /usr/lib/condor/ && \
-    sudo mv condor-8.9.9-x86_64_Ubuntu20-stripped/lib/python3/* /usr/lib/python3/ && \
-    sudo rm -r condor-8.9.9-x86_64_Ubuntu20-stripped/lib/condor condor-8.9.9-x86_64_Ubuntu20-stripped/lib/python3 && \
-    sudo mv condor-8.9.9-x86_64_Ubuntu20-stripped/lib/* /usr/lib/ && \
-    sudo mkdir -p /usr/libexec/ && sudo mv condor-8.9.9-x86_64_Ubuntu20-stripped/libexec/* /usr/libexec/ && \
-    sudo mkdir /etc/condor
+RUN apt-get install -y curl
 
-COPY ./condor_config /etc/condor/condor_config
+RUN curl -fsSL https://research.cs.wisc.edu/htcondor/repo/keys/HTCondor-Release.gpg.key | apt-key add - \
+    && echo -e "deb [arch=amd64] https://research.cs.wisc.edu/htcondor/repo/ubuntu/8.9 focal main\ndeb-src https://research.cs.wisc.edu/htcondor/repo/ubuntu/8.9 focal main" > /etc/apt/sources.list.d/htcondor.list
+
+RUN apt update && apt-get install -y htcondor
 
+COPY ./condor_config /etc/condor/condor_config
+COPY ./htc-demo  /opt/workspace/htc-example
 WORKDIR /opt/workspace
diff --git a/stable/jhub-aas/docker/Dockerfile.dask b/stable/jhub-aas/docker/Dockerfile.dask
@@ -0,0 +1,70 @@
+ARG  NOTEBOOK_VERSION
+FROM ghcr.io/dodas-ts/notebook:${NOTEBOOK_VERSION}-htc
+USER root
+
+RUN apt-get install -y dpkg-dev cmake g++ gcc binutils libx11-dev libxpm-dev \
+            libxft-dev libxext-dev python libssl-dev \
+            gfortran libpcre3-dev \
+            xlibmesa-glu-dev libglew1.5-dev libftgl-dev \
+            libmysqlclient-dev libfftw3-dev libcfitsio-dev \
+            graphviz-dev libavahi-compat-libdnssd-dev \
+            libldap2-dev python-dev libxml2-dev libkrb5-dev \
+            libgsl0-dev
+
+WORKDIR /usr/local/share/
+RUN mkdir root6build root6
+RUN git clone --branch distrdf-dask https://github.com/vepadulano/root.git root6source
+
+WORKDIR /usr/local/share/root6source
+
+WORKDIR /usr/local/share/root6build
+RUN cmake \
+        -DCMAKE_INSTALL_PREFIX=/usr/local/share/root6 \
+        -DPython3_EXECUTABLE=/opt/conda/bin/python \
+        /usr/local/share/root6source
+
+RUN cmake --build . --target install -- -j4
+
+# Dask and jupyterhub
+RUN python3 -m pip install --upgrade pip
+RUN python3 -m pip install --upgrade setuptools
+RUN python3 -m pip install dask \
+    dask_jobqueue \
+    click==7.1.2 \
+    numpy \
+    bokeh \
+    ipython \
+    jupyterhub \
+    jupyterlab \
+    notebook \
+    jupyter-server-proxy \
+    ipywidgets
+RUN python3 -m pip install "dask[dataframe]"
+
+# Install DASK jobqueue
+RUN pip install dask-remote-jobqueue==0.4.19
+
+WORKDIR /opt/workspace
+# Old approach
+# RUN python3 -m pip install dask-labextension "dask[dataframe]"
+# COPY labextension.yaml /usr/local/lib/python3.6/site-packages/dask_labextension/labextension.yaml
+RUN curl -fsSL https://deb.nodesource.com/setup_14.x | bash -
+RUN DEBIAN_FRONTEND=noninteractive apt install -y nodejs
+RUN npm install npm@latest -g
+RUN npm install -g typescript yarn
+
+RUN python -m pip install jupyter-packaging \
+    && git clone --branch custom_clusters https://github.com/DODAS-TS/dask-labextension.git
+
+WORKDIR /opt/workspace/dask-labextension
+RUN jlpm 
+RUN npm run build && npm install 
+RUN jupyter labextension install .
+
+RUN mkdir -p /root/.ipython
+
+COPY labextension.yaml /opt/conda/lib/python3.9/site-packages/dask_labextension/labextension.yaml
+
+COPY ./jupyterhub-singleuser.sh /usr/local/bin/jupyterhub-singleuser.sh
+RUN chmod +x /usr/local/bin/jupyterhub-singleuser.sh
+WORKDIR /opt/workspace
diff --git a/stable/jhub-aas/docker/htc-demo/simple b/stable/jhub-aas/docker/htc-demo/simple
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+sleep 100
+echo $HOSTNAME
diff --git a/stable/jhub-aas/docker/htc-demo/sub b/stable/jhub-aas/docker/htc-demo/sub
@@ -0,0 +1,7 @@
+universe = vanilla
+executable = simple
+log = simple.log
+output = simple.out
+error = simple.error
++OWNER = "condor"
+queue
diff --git a/stable/jhub-aas/docker/jupyterhub-singleuser b/stable/jhub-aas/docker/jupyterhub-singleuser
@@ -5,7 +5,7 @@
 import re
 import sys
 import time
-from jupyterhub.singleuser import main
+from jupyterlab.labhubapp import main
 from subprocess import Popen, DEVNULL
 
 if __name__ == '__main__':

diff --git a/stable/jhub-aas/docker/jupyterhub-singleuser.sh b/stable/jhub-aas/docker/jupyterhub-singleuser.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+#source /cvmfs/cms.dodas.infn.it/miniconda3/etc/profile.d/conda.sh
+#conda activate cms-dodas
+
+#export PYTHONPATH=/opt/conda/lib/python3.9/site-packages/:$PYTHONPATH
+
+source /usr/local/share/root6/bin/thisroot.sh
+
+#export LD_LIBRARY_PATH=/cvmfs/cms.dodas.infn.it/miniconda3/envs/cms-dodas/lib:$LD_LIBRARY_PATH
+#export JUPYTER_PATH=/opt/conda/etc/jupyter
+#export JUPYTER_CONFIG_DIR=/opt/conda/etc/jupyter
+
+python /usr/local/bin/jupyterhub-singleuser "$@"
diff --git a/stable/jhub-aas/docker/labextension.yaml b/stable/jhub-aas/docker/labextension.yaml
@@ -0,0 +1,14 @@
+labextension:
+  factory:
+    module: 'dask_remote_jobqueue'
+    class: 'RemoteHTCondor'
+    args: []
+    kwargs: {}
+  default:
+    workers: null
+    adapt:
+      null
+      # minimum: 0
+      # maximum: 10
+  initial:
+    []
diff --git a/stable/jhub-aas/docker/spawn.sh b/stable/jhub-aas/docker/spawn.sh
@@ -8,8 +8,17 @@ oidc-gen dodas --issuer $IAM_SERVER \
                --client-secret $IAM_CLIENT_SECRET \
                --rt $REFRESH_TOKEN \
                --confirm-yes \
-               --scope "openid profile email" \
+               --scope "openid profile email wlcg wlcg.groups" \
                --redirect-uri  http://localhost:8843 \
                --pw-cmd "echo \"DUMMY PWD\""
 
 while true; do oidc-token dodas --time 1200 > /tmp/token; sleep 600; done &
+
+if [[ -f "cvmfs/cms.dodas.infn.it/miniconda3/etc/profile.d/conda.sh" ]]; then
+    source /cvmfs/cms.dodas.infn.it/miniconda3/etc/profile.d/conda.sh
+    conda activate cms-dodas
+    export PYTHONPATH=/opt/conda/lib/python3.9/site-packages/:$PYTHONPATH
+    source /cvmfs/cms.dodas.infn.it/miniconda3/envs/cms-dodas/bin/thisroot.sh
+    export JUPYTER_PATH=/opt/conda/etc/jupyter
+    export JUPYTER_CONFIG_DIR=/opt/conda/etc/jupyter
+fi
diff --git a/stable/jhub-aas/templates/ssh-forwarder/deployment-ssh-forwarder.yaml b/stable/jhub-aas/templates/ssh-forwarder/deployment-ssh-forwarder.yaml
@@ -0,0 +1,56 @@
+{{- if .Values.sshForwarder.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ssh-fwd
+  labels:
+    app.kubernetes.io/name: ssh-fwd
+spec:
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: ssh-fwd 
+    spec:
+      priorityClassName: system-node-critical
+      containers:
+      - name: redis 
+        resources:
+          requests:
+            memory: {{ .Values.sshForwarder.redis.requests.memory }}
+            cpu: {{ .Values.sshForwarder.redis.requests.cpu }}
+          limits:
+            memory: {{ .Values.sshForwarder.redis.limits.memory }}
+            cpu: {{ .Values.sshForwarder.redis.limits.cpu }}
+        imagePullPolicy: {{ .Values.sshForwarder.redis.image.pullPolicy }}
+        image: "{{ .Values.sshForwarder.redis.image.name }}:{{ .Values.sshForwarder.redis.image.tag }}"
+        volumeMounts: []
+      - name: ssh-fwd
+        resources:
+          requests:
+            memory: {{ .Values.sshForwarder.sshFWD.requests.memory }}
+            cpu: {{ .Values.sshForwarder.sshFWD.requests.cpu }}
+          limits:
+            memory: {{ .Values.sshForwarder.sshFWD.limits.memory }}
+            cpu: {{ .Values.sshForwarder.sshFWD.limits.cpu }}
+        imagePullPolicy: {{ .Values.sshForwarder.sshFWD.image.pullPolicy }}
+        image: "{{ .Values.sshForwarder.sshFWD.image.name }}:{{ .Values.sshForwarder.sshFWD.image.tag | default .Chart.AppVersion }}"
+      - name: listener 
+        resources:
+          requests:
+            memory: {{ .Values.sshForwarder.sshListener.requests.memory }}
+            cpu: {{ .Values.sshForwarder.sshListener.requests.cpu }}
+          limits:
+            memory: {{ .Values.sshForwarder.sshListener.limits.memory }}
+            cpu: {{ .Values.sshForwarder.sshListener.limits.cpu }}
+        imagePullPolicy: {{ .Values.sshForwarder.sshListener.image.pullPolicy }}
+        image: "{{ .Values.sshForwarder.sshListener.image.name }}:{{ .Values.sshForwarder.sshListener.image.tag | default .Chart.AppVersion }}"
+      volumes:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: ssh-fwd 
+
+
+  replicas: 1
+  strategy:
+    type: Recreate
+{{- end }}