From 9d63c5d3c5493a69cc9efaf91b6f3b62cacace05 Mon Sep 17 00:00:00 2001
From: MattesR <mattes.ruckdeschel@tuhh.de>
Date: Wed, 31 Mar 2021 12:09:14 +0200
Subject: [PATCH 1/9] add nlp-service to hoover deps

---
 liquid_node/configuration.py   |  2 ++
 production-versions.ini        |  1 +
 staging-versions.ini           |  1 +
 templates/hoover-deps.nomad    | 30 ++++++++++++++++++++++++++++++
 templates/hoover-workers.nomad |  2 ++
 templates/hoover.nomad         |  4 ++++
 testing-versions.ini           |  1 +
 7 files changed, 41 insertions(+)

diff --git a/liquid_node/configuration.py b/liquid_node/configuration.py
index 8248518d..76cddc96 100644
--- a/liquid_node/configuration.py
+++ b/liquid_node/configuration.py
@@ -160,6 +160,8 @@ def __init__(self):
         self.tika_count = self.ini.getint('liquid', 'tika_count', fallback=1)
         self.tika_memory_limit = self.ini.getint('liquid', 'tika_memory_limit', fallback=800)
 
+        self.nlp_memory_limit = self.ini.getint('liquid', 'nlp_memory_limit', fallback=800)
+
         self.hypothesis_memory_limit = \
             self.ini.getint('liquid',
                             'hypothesis_memory_limit',
diff --git a/production-versions.ini b/production-versions.ini
index 10454da3..6d783d63 100644
--- a/production-versions.ini
+++ b/production-versions.ini
@@ -11,3 +11,4 @@ liquid-dokuwiki = liquidinvestigations/liquid-dokuwiki:0.1.0
 liquid-nextcloud = liquidinvestigations/liquid-nextcloud:0.2.3
 rocketchat = rocketchat/rocket.chat:3.9.1
 rocketchat-mongo = mongo:4.4
+nlp-service = liquidinvestigations/nlp-service:initial_droned_service
diff --git a/staging-versions.ini b/staging-versions.ini
index d8c5d381..9f54ad98 100644
--- a/staging-versions.ini
+++ b/staging-versions.ini
@@ -11,3 +11,4 @@ liquid-dokuwiki = liquidinvestigations/liquid-dokuwiki:release
 liquid-nextcloud = liquidinvestigations/liquid-nextcloud:0.2.3
 rocketchat = rocketchat/rocket.chat:3.9.1
 rocketchat-mongo = mongo:4.4
+nlp-service = liquidinvestigations/nlp-service:initial_droned_service
diff --git a/templates/hoover-deps.nomad b/templates/hoover-deps.nomad
index 1f6de320..bc174ffe 100644
--- a/templates/hoover-deps.nomad
+++ b/templates/hoover-deps.nomad
@@ -353,6 +353,36 @@ job "hoover-deps" {
     }
   }
 
+  group "nlp-service" {
+    ${ continuous_reschedule() }
+    ${ group_disk() }
+
+    task "nlp" {
+      ${ task_logs() }
+
+      driver = "docker"
+      config {
+        image = "${config.image('nlp-service')}"
+        port_map {
+          nlp = 5000
+        }
+        labels {
+          liquid_task = "hoover-nlp"
+        }
+        memory_hard_limit = ${4 * config.nlp_memory_limit}
+      }
+
+      resources {
+        memory = ${config.nlp_memory_limit}
+        cpu = 1500
+        network {
+          mbits = 1
+          port "nlp" {}
+        }
+      }
+    }
+  }
+
   group "rabbitmq" {
     ${ continuous_reschedule() }
     ${ group_disk() }
diff --git a/templates/hoover-workers.nomad b/templates/hoover-workers.nomad
index 63eeee94..88dc8da9 100644
--- a/templates/hoover-workers.nomad
+++ b/templates/hoover-workers.nomad
@@ -61,6 +61,7 @@ job "hoover-workers" {
           # exec tail -f /dev/null
           if  [ -z "$SNOOP_TIKA_URL" ] \
                   || [ -z "$SNOOP_DB" ] \
+                  || [ -z "$SNOOP_NLP_URL" ] \
                   || [ -z "$SNOOP_ES_URL" ] \
                   || [ -z "$SNOOP_AMQP_URL" ]; then
             echo "incomplete configuration!"
@@ -76,6 +77,7 @@ job "hoover-workers" {
       env {
         SNOOP_ES_URL = "http://{% raw %}${attr.unique.network.ip-address}{% endraw %}:9990/_es"
         SNOOP_TIKA_URL = "http://{% raw %}${attr.unique.network.ip-address}{% endraw %}:9990/_tika/"
+        SNOOP_NLP_URL = "http://{% raw %}${attr.unique.network.ip-address}{% endraw %}:9990/_nlp"
         SNOOP_RABBITMQ_HTTP_URL = "{% raw %}${attr.unique.network.ip-address}{% endraw %}:9990/_rabbit/"
         SNOOP_COLLECTIONS = ${ config.snoop_collections | tojson | tojson }
 
diff --git a/templates/hoover.nomad b/templates/hoover.nomad
index ae578890..e5e1b009 100644
--- a/templates/hoover.nomad
+++ b/templates/hoover.nomad
@@ -188,6 +188,7 @@ job "hoover" {
       env {
         SNOOP_ES_URL = "http://{% raw %}${attr.unique.network.ip-address}{% endraw %}:9990/_es"
         SNOOP_TIKA_URL = "http://{% raw %}${attr.unique.network.ip-address}{% endraw %}:9990/_tika/"
+        SNOOP_NLP_URL = "http://{% raw %}${attr.unique.network.ip-address}{% endraw %}:9990/_nlp"
         SNOOP_RABBITMQ_HTTP_URL = "{% raw %}${attr.unique.network.ip-address}{% endraw %}:9990/_rabbit/"
         SNOOP_COLLECTIONS = ${ config.snoop_collections | tojson | tojson }
       }
@@ -278,6 +279,7 @@ job "hoover" {
           # exec tail -f /dev/null
           if  [ -z "$SNOOP_TIKA_URL" ] \
                   || [ -z "$SNOOP_DB" ] \
+                  || [ -z "$SNOOP_NLP_URL" ] \
                   || [ -z "$SNOOP_ES_URL" ] \
                   || [ -z "$SNOOP_AMQP_URL" ]; then
             echo "incomplete configuration!"
@@ -293,6 +295,7 @@ job "hoover" {
       env {
         SNOOP_ES_URL = "http://{% raw %}${attr.unique.network.ip-address}{% endraw %}:9990/_es"
         SNOOP_TIKA_URL = "http://{% raw %}${attr.unique.network.ip-address}{% endraw %}:9990/_tika/"
+        SNOOP_NLP_URL = "http://{% raw %}${attr.unique.network.ip-address}{% endraw %}:9990/_nlp"
         SNOOP_RABBITMQ_HTTP_URL = "{% raw %}${attr.unique.network.ip-address}{% endraw %}:9990/_rabbit/"
         SNOOP_COLLECTIONS = ${ config.snoop_collections | tojson | tojson }
 
@@ -400,6 +403,7 @@ job "hoover" {
       env {
         SNOOP_ES_URL = "http://{% raw %}${attr.unique.network.ip-address}{% endraw %}:9990/_es"
         SNOOP_TIKA_URL = "http://{% raw %}${attr.unique.network.ip-address}{% endraw %}:9990/_tika/"
+        SNOOP_NLP_URL = "http://{% raw %}${attr.unique.network.ip-address}{% endraw %}:9990/_nlp"
         SNOOP_RABBITMQ_HTTP_URL = "{% raw %}${attr.unique.network.ip-address}{% endraw %}:9990/_rabbit/"
         SNOOP_COLLECTIONS = ${ config.snoop_collections | tojson | tojson }
       }
diff --git a/testing-versions.ini b/testing-versions.ini
index 46132d48..7236dfd1 100644
--- a/testing-versions.ini
+++ b/testing-versions.ini
@@ -11,3 +11,4 @@ liquid-dokuwiki = liquidinvestigations/liquid-dokuwiki:master
 liquid-nextcloud = liquidinvestigations/liquid-nextcloud:0.2.3
 rocketchat = rocketchat/rocket.chat:3.9.1
 rocketchat-mongo = mongo:4.4
+nlp-service = liquidinvestigations/nlp-service:initial_droned_service

From 51e503e35a927793953c32d5ce80082c819ac864 Mon Sep 17 00:00:00 2001
From: MattesR <mattes.ruckdeschel@tuhh.de>
Date: Wed, 31 Mar 2021 12:31:35 +0200
Subject: [PATCH 2/9] change testing version to branch

---
 testing-versions.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testing-versions.ini b/testing-versions.ini
index 7236dfd1..c2840137 100644
--- a/testing-versions.ini
+++ b/testing-versions.ini
@@ -2,7 +2,7 @@
 codimd = liquidinvestigations/codimd-server:master
 h-client = liquidinvestigations/h-client:master
 hoover-search = liquidinvestigations/hoover-search:master
-hoover-snoop2 = liquidinvestigations/hoover-snoop2:master
+hoover-snoop2 = liquidinvestigations/hoover-snoop2:ner-integration
 hoover-ui = liquidinvestigations/hoover-ui:master
 hypothesis-h = liquidinvestigations/hypothesis-h:master
 liquid-authproxy = liquidinvestigations/oauth-proxy:master

From e1c2866be6731757c0db010a6f2b47d71bf64ae7 Mon Sep 17 00:00:00 2001
From: MattesR <mattes.ruckdeschel@tuhh.de>
Date: Wed, 21 Apr 2021 16:25:50 +0200
Subject: [PATCH 3/9] add volume and check to nlp task

---
 templates/hoover-deps.nomad | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/templates/hoover-deps.nomad b/templates/hoover-deps.nomad
index bc174ffe..b2267ee1 100644
--- a/templates/hoover-deps.nomad
+++ b/templates/hoover-deps.nomad
@@ -360,9 +360,17 @@ job "hoover-deps" {
     task "nlp" {
       ${ task_logs() }
 
+      constraint {
+        attribute = "{% raw %}${meta.liquid_volumes}{% endraw %}"
+        operator = "is_set"
+      }
+
       driver = "docker"
       config {
         image = "${config.image('nlp-service')}"
+        volumes = [
+          "{% raw %}${meta.liquid_volumes}{% endraw %}/nlp-service/data:/data",
+        ]
         port_map {
           nlp = 5000
         }
@@ -371,7 +379,9 @@ job "hoover-deps" {
         }
         memory_hard_limit = ${4 * config.nlp_memory_limit}
       }
-
+      env {
+        NLP_SERVICE_PRESET = "full_sm"
+      }
       resources {
         memory = ${config.nlp_memory_limit}
         cpu = 1500
@@ -380,6 +390,19 @@ job "hoover-deps" {
           port "nlp" {}
         }
       }
+      service {
+        name = "hoover-nlp-service"
+        tags = ["fabio-/_nlp strip=/_nlp"]
+        port = "nlp"
+        check {
+          name = "http"
+          initial_status = "critical"
+          type = "http"
+          path = "/config"
+          interval = "${check_interval}"
+          timeout = "600s"
+        }
+      }
     }
   }
 

From 2527a265861cab35e762d2c90d5e2908b48d66fb Mon Sep 17 00:00:00 2001
From: MattesR <mattes.ruckdeschel@tuhh.de>
Date: Fri, 2 Oct 2020 09:54:41 +0200
Subject: [PATCH 4/9] add templates for newsleak and its dependencies

---
 templates/newsleak-deps.nomad | 126 ++++++++++++++++++++++++++++++++++
 templates/newsleak.nomad      |  34 +++++++++
 2 files changed, 160 insertions(+)
 create mode 100644 templates/newsleak-deps.nomad
 create mode 100644 templates/newsleak.nomad

diff --git a/templates/newsleak-deps.nomad b/templates/newsleak-deps.nomad
new file mode 100644
index 00000000..aefd149c
--- /dev/null
+++ b/templates/newsleak-deps.nomad
@@ -0,0 +1,126 @@
+{% from '_lib.hcl' import shutdown_delay, authproxy_group, continuous_reschedule, set_pg_password_template, task_logs, group_disk with context -%} 
+
+job "newsleak-deps" {
+  datacenters = ["dc1"]
+  type = "service"
+  priority = 60
+
+  
+  group "newsleak-pg" {
+    ${ continuous_reschedule() }
+    ${ group_disk() }
+
+    task "newsleak-pg" {
+      ${ task_logs() }
+
+      constraint {
+        attribute = "{% raw %}${meta.liquid_volumes}{% endraw %}"
+        operator = "is_set"
+      }
+
+      affinity {
+        attribute = "{% raw %}${meta.liquid_large_databases}{% endraw %}"
+        value     = "true"
+        weight    = 100
+      }
+
+      driver = "docker"
+
+      ${ shutdown_delay() }
+
+      config {
+        image = "postgres:9.6"
+        volumes = [
+          "{% raw %}${meta.liquid_volumes}{% endraw %}/newsleak/pg/data:/var/lib/postgresql/data",
+        ]
+        labels {
+          liquid_task = "search-pg"
+        }
+        port_map {
+          pg = 5432
+        }
+        # 128MB, the default postgresql shared_memory config
+        shm_size = 134217728
+        memory_hard_limit = 1200
+      }
+      template {
+        data = <<EOF
+          POSTGRES_PASSWORD=newsreader 
+          POSTGRES_USER=newsreader 
+          POSTGRES_DB=newsleak
+        EOF
+        destination = "local/postgres.env"
+        env = true
+      }
+      ${ set_pg_password_template('newsreader') }
+      resources {
+        memory = 350
+        network {
+          mbits = 1
+          port "pg" {}
+        }
+      }
+      service {
+        name = "newsleak-pg"
+        port = "pg"
+        check {
+          name = "tcp"
+          initial_status = "critical"
+          type = "tcp"
+          interval = "${check_interval}"
+          timeout = "${check_timeout}"
+        }
+      }
+    }
+  }
+
+
+  group "newsleak-ner" {
+    ${ continuous_reschedule() }
+    ${ group_disk() }
+
+    task "newsleak-ner" {
+      ${ task_logs() }
+      driver = "docker"
+      config {
+        image = "$uhhlt/newsleak-ner:v1.0"
+        port_map {
+          ner = 5001
+        }
+        labels {
+          liquid_task = "newsleak-ner"
+        }
+      }
+    }
+  }
+
+
+
+  group "newsleak-es" {
+    ${ continuous_reschedule() }
+    ${ group_disk() }
+
+    task "newsleak-es" {
+      ${ task_logs() }
+      driver = "docker"
+        config {
+          image = "elasticsearch:2.4.6"
+          args = ["/bin/sh", "-c", "chown 1000:1000 /usr/share/elasticsearch/data && echo chown done && /usr/local/bin/docker-entrypoint.sh"]
+          volumes = [
+            "{% raw %}${meta.liquid_volumes}{% endraw %}/newsleak/es/data:/usr/share/elasticsearch/data",
+          ]
+          port_map {
+            http = 9200
+            transport = 9300
+          }
+          labels {
+            liquid_task = "newsleak-es"
+          }
+          ulimit {
+            memlock = "-1"
+            nofile = "65536"
+            nproc = "8192"
+          }
+        }
+    }
+  }
\ No newline at end of file
diff --git a/templates/newsleak.nomad b/templates/newsleak.nomad
new file mode 100644
index 00000000..92c5a331
--- /dev/null
+++ b/templates/newsleak.nomad
@@ -0,0 +1,34 @@
+{% from '_lib.hcl' import shutdown_delay, authproxy_group, continuous_reschedule, set_pg_password_template, task_logs, group_disk with context -%} 
+
+job "newsleak" {
+  datacenters = ["dc1"]
+  type = "service"
+  priority = 60
+
+  
+  group "newsleak" {
+    task "nextcloud" {
+      ${ group_disk() }
+
+        constraint {
+          attribute = "{% raw %}${meta.liquid_volumes}{% endraw %}"
+          operator = "is_set"
+        }
+        constraint {
+          attribute = "{% raw %}${meta.liquid_collections}{% endraw %}"
+          operator = "is_set"
+        }
+
+        driver = "docker"
+        config {
+          image = "${config.image('newsleak')}"
+          volumes = [
+            "{% raw %}${meta.liquid_volumes}{% endraw %}/newsleak/ui/themes::/etc/settings/",
+          ]
+          args = ["/bin/sh", "-c", "chown -R 33:33 /var/www/html/ && echo chown done && /entrypoint.sh apache2-foreground"]
+          port_map {
+            http = 80
+          }
+          labels {
+            liquid_task = "nextcloud"
+          }
\ No newline at end of file

From 2d8cf6e054f47c23a2bab4c5990ca475200cf281 Mon Sep 17 00:00:00 2001
From: MattesR <mattes.ruckdeschel@tuhh.de>
Date: Mon, 5 Oct 2020 13:08:23 +0200
Subject: [PATCH 5/9] add templates for newsleak

---
 templates/newsleak-deps.nomad | 114 +++++++++++++++++++++++++---------
 templates/newsleak.nomad      |  77 ++++++++++++++++++++---
 2 files changed, 151 insertions(+), 40 deletions(-)

diff --git a/templates/newsleak-deps.nomad b/templates/newsleak-deps.nomad
index aefd149c..21f425de 100644
--- a/templates/newsleak-deps.nomad
+++ b/templates/newsleak-deps.nomad
@@ -5,7 +5,8 @@ job "newsleak-deps" {
   type = "service"
   priority = 60
 
-  
+  spread { attribute = {% raw %}"${attr.unique.hostname}"{% endraw %} } 
+
   group "newsleak-pg" {
     ${ continuous_reschedule() }
     ${ group_disk() }
@@ -26,15 +27,13 @@ job "newsleak-deps" {
 
       driver = "docker"
 
-      ${ shutdown_delay() }
-
       config {
         image = "postgres:9.6"
         volumes = [
-          "{% raw %}${meta.liquid_volumes}{% endraw %}/newsleak/pg/data:/var/lib/postgresql/data",
+          "{% raw %}${meta.liquid_volumes}{% endraw %}/newsleak/pg/data:/var/lib/postgresql/data"
         ]
         labels {
-          liquid_task = "search-pg"
+          liquid_task = "newsleak-pg"
         }
         port_map {
           pg = 5432
@@ -52,7 +51,9 @@ job "newsleak-deps" {
         destination = "local/postgres.env"
         env = true
       }
+
       ${ set_pg_password_template('newsreader') }
+      
       resources {
         memory = 350
         network {
@@ -74,28 +75,6 @@ job "newsleak-deps" {
     }
   }
 
-
-  group "newsleak-ner" {
-    ${ continuous_reschedule() }
-    ${ group_disk() }
-
-    task "newsleak-ner" {
-      ${ task_logs() }
-      driver = "docker"
-      config {
-        image = "$uhhlt/newsleak-ner:v1.0"
-        port_map {
-          ner = 5001
-        }
-        labels {
-          liquid_task = "newsleak-ner"
-        }
-      }
-    }
-  }
-
-
-
   group "newsleak-es" {
     ${ continuous_reschedule() }
     ${ group_disk() }
@@ -105,9 +84,9 @@ job "newsleak-deps" {
       driver = "docker"
         config {
           image = "elasticsearch:2.4.6"
-          args = ["/bin/sh", "-c", "chown 1000:1000 /usr/share/elasticsearch/data && echo chown done && /usr/local/bin/docker-entrypoint.sh"]
+          #args = ["/bin/sh", "-c", "chown 1000:1000 /usr/share/elasticsearch/data && echo chown done && /docker-entrypoint.sh && echo entrypoint done"]
           volumes = [
-            "{% raw %}${meta.liquid_volumes}{% endraw %}/newsleak/es/data:/usr/share/elasticsearch/data",
+            "{% raw %}${meta.liquid_volumes}{% endraw %}/newsleak/es/data:/usr/share/elasticsearch/data"
           ]
           port_map {
             http = 9200
@@ -122,5 +101,80 @@ job "newsleak-deps" {
             nproc = "8192"
           }
         }
+              service {
+        name = "newsleak-es-http"
+        port = "http"
+        check {
+          name = "http"
+          initial_status = "critical"
+          type = "http"
+          path = "/_cluster/health"
+          interval = "${check_interval}"
+          timeout = "${check_timeout}"
+        }
+      }
+
+      service {
+        name = "newsleak-es-transport"
+        port = "transport"
+        check {
+          name = "transport"
+          initial_status = "critical"
+          type = "tcp"
+          interval = "${check_interval}"
+          timeout = "${check_timeout}"
+        }
+      }
+
+      resources {
+        cpu = 600
+        memory = ${config.elasticsearch_memory_limit}
+        network {
+          mbits = 1
+          port "http" {}
+          port "transport" {}
+        }
+      }
+    }
+  }
+
+  group "newsleak-ner" {
+    ${ continuous_reschedule() }
+    ${ group_disk() }
+
+    task "newsleak-ner" {
+      ${ task_logs() }
+      driver = "docker"
+      config {
+        image = "uhhlt/newsleak-ner:v1.0"
+        port_map {
+          ner = 5001
+        }
+        labels {
+          liquid_task = "newsleak-ner"
+        }
+      }
+      service {
+        name = "newsleak-ner"
+        port = "ner"
+        check {
+          name = "tcp"
+          initial_status = "critical"
+          type = "tcp"
+          interval = "${check_interval}"
+          timeout = "${check_timeout}"
+        }
+      }
+    
+      resources {
+        cpu = 600
+        memory = 350
+        network {
+          mbits = 1
+          port "ner" {}
+        }
+      }
     }
-  }
\ No newline at end of file
+  }
+
+}
\ No newline at end of file
diff --git a/templates/newsleak.nomad b/templates/newsleak.nomad
index 92c5a331..f84addcc 100644
--- a/templates/newsleak.nomad
+++ b/templates/newsleak.nomad
@@ -7,28 +7,85 @@ job "newsleak" {
 
   
   group "newsleak" {
-    task "nextcloud" {
-      ${ group_disk() }
+    ${ group_disk() }
+    task "newsleak-ui" {
+      ${ task_logs() }
 
         constraint {
           attribute = "{% raw %}${meta.liquid_volumes}{% endraw %}"
           operator = "is_set"
         }
-        constraint {
-          attribute = "{% raw %}${meta.liquid_collections}{% endraw %}"
-          operator = "is_set"
-        }
 
         driver = "docker"
         config {
           image = "${config.image('newsleak')}"
           volumes = [
-            "{% raw %}${meta.liquid_volumes}{% endraw %}/newsleak/ui/themes::/etc/settings/",
+            "{% raw %}${meta.liquid_volumes}{% endraw %}/newsleak/ui/themes::/etc/settings/"
           ]
-          args = ["/bin/sh", "-c", "chown -R 33:33 /var/www/html/ && echo chown done && /entrypoint.sh apache2-foreground"]
+          args = ["/bin/sh", "-c", "chown -R 999:999 /etc/settings/ && echo chown done && /entrypoint.sh "]
           port_map {
             http = 80
           }
           labels {
-            liquid_task = "nextcloud"
-          }
\ No newline at end of file
+            liquid_task = "newsleak"
+          }
+        }
+        template {
+          destination = "local/conf"
+          data = <<-EOF
+            ################################
+            ##     NEWSLEAK CONFIG        ##
+            ################################
+
+            # processing configuration
+            processlanguages = eng, deu
+            defaultlanguage = eng
+            paragraphsasdocuments = false
+            paragraphminimumlength = 1500
+            maxdocumentlength = 500000
+            debugMaxDocuments = 0
+            threads = 4
+
+            # ner, dictionaries and pattern extraction
+            nerserviceurl = {{- range service "newsleak-ner" -}}"{{.Address}}:{{.Port}}"{{- end -}}
+            dictionaryfiles = fck.all, deu:fck.deu, eng:fck.eng
+            patternemail = true
+            patternurl = false
+            patternphone = false
+            patternip = false
+
+            # datareader (csv, hoover, or else)
+            datareader = hoover
+
+            # CSV datareader options
+            datadirectory = ./data
+            documentfile = document.csv
+            metadatafile = metadata.csv
+
+            # Hoover datareader options
+            hooverindex = testdata
+            hooverclustername = elasticsearch
+            hooverurl = {{- range service "hoover-es-data" -}}"{{.Address}}"{{- end -}}
+            hooversearchurl = {{- range service "newsleak-ner" -}}"{{.Address}}:{{.Port}}"{{- end -}}
+            hooverport = {{- range service "hoover-es-data" -}}"{{.Port}}"{{- end -}}
+            hoovertmpmetadata = hoover_metadata.csv
+            processlanguages = eng, ron, deu
+            # Newsleak postgres
+            dburl = {{- range service "newsleak-pg" -}}"{{.Address}}:{{.Port}}"{{- end -}}
+            dbname = newsleak
+            dbuser = newsreader
+            dbpass = newsreader
+            dbschema = desc/postgresSchema.sql
+            dbindices = desc/postgresIndices.sql
+
+            # Newsleak elasticsearch index
+            esindex = newsleak
+            esurl = {{- range service "newsleak-es-transport" -}}"{{.Address}}"{{- end -}}
+            esclustername = elasticsearch
+            esport = {{- range service "newsleak-es-transport" -}}"{{.Port}}"{{- end -}}
+          EOF
+      }
+    
+    }
+  }
+}
\ No newline at end of file

From 21c31413d61193d11c76774af5ea97a5baa8be33 Mon Sep 17 00:00:00 2001
From: MattesR <mattes.ruckdeschel@tuhh.de>
Date: Mon, 26 Apr 2021 13:59:42 +0200
Subject: [PATCH 6/9] add proxy

---
 liquid_node/jobs/newsleak.py   |  31 +++++
 templates/newsleak-deps.nomad  |  18 ++-
 templates/newsleak-proxy.nomad |  14 +++
 templates/newsleak.nomad       | 208 +++++++++++++++++++++++++++------
 4 files changed, 226 insertions(+), 45 deletions(-)
 create mode 100644 liquid_node/jobs/newsleak.py
 create mode 100644 templates/newsleak-proxy.nomad

diff --git a/liquid_node/jobs/newsleak.py b/liquid_node/jobs/newsleak.py
new file mode 100644
index 00000000..e172a100
--- /dev/null
+++ b/liquid_node/jobs/newsleak.py
@@ -0,0 +1,31 @@
+from liquid_node import jobs
+
+
+class Newsleak(jobs.Job):
+    name = 'newsleak'
+    template = jobs.TEMPLATES / f'{name}.nomad'
+    app = 'newsleak'
+    stage = 2
+    core_oauth_apps = [
+        {
+            'name': 'newsleak',
+            'vault_path': 'liquid/newsleak/auth.oauth2',
+            'callback': '/oauth2/callback',
+        },
+    ]
+    generate_oauth2_proxy_cookie = True
+
+
+
+class Proxy(jobs.Job):
+    name = 'newsleak-proxy'
+    template = jobs.TEMPLATES / f'{name}.nomad'
+    app = 'newsleak-ui'
+    stage = 4
+
+
+class Deps(jobs.Job):
+    name = 'newsleak-deps'
+    template = jobs.TEMPLATES / f'{name}.nomad'
+    app = 'newsleak-ui'
+    stage = 1
diff --git a/templates/newsleak-deps.nomad b/templates/newsleak-deps.nomad
index 21f425de..05d99a7b 100644
--- a/templates/newsleak-deps.nomad
+++ b/templates/newsleak-deps.nomad
@@ -1,11 +1,11 @@
-{% from '_lib.hcl' import shutdown_delay, authproxy_group, continuous_reschedule, set_pg_password_template, task_logs, group_disk with context -%} 
+{% from '_lib.hcl' import shutdown_delay, authproxy_group, continuous_reschedule, set_pg_password_template, task_logs, group_disk with context -%}
 
 job "newsleak-deps" {
   datacenters = ["dc1"]
   type = "service"
   priority = 60
 
-  spread { attribute = {% raw %}"${attr.unique.hostname}"{% endraw %} } 
+  spread { attribute = {% raw %}"${attr.unique.hostname}"{% endraw %} }
 
   group "newsleak-pg" {
     ${ continuous_reschedule() }
@@ -40,12 +40,12 @@ job "newsleak-deps" {
         }
         # 128MB, the default postgresql shared_memory config
         shm_size = 134217728
-        memory_hard_limit = 1200
+        memory_hard_limit = 1500
       }
       template {
         data = <<EOF
-          POSTGRES_PASSWORD=newsreader 
-          POSTGRES_USER=newsreader 
+          POSTGRES_PASSWORD=newsreader
+          POSTGRES_USER=newsreader
           POSTGRES_DB=newsleak
         EOF
         destination = "local/postgres.env"
@@ -53,7 +53,6 @@ job "newsleak-deps" {
       }
 
       ${ set_pg_password_template('newsreader') }
-      
       resources {
         memory = 350
         network {
@@ -165,10 +164,9 @@ job "newsleak-deps" {
           timeout = "${check_timeout}"
         }
       }
-    
+
       resources {
-        cpu = 600
-        memory = 350
+        memory = 2000
         network {
           mbits = 1
           port "ner" {}
@@ -177,4 +175,4 @@ job "newsleak-deps" {
     }
   }
 
-}
\ No newline at end of file
+}
diff --git a/templates/newsleak-proxy.nomad b/templates/newsleak-proxy.nomad
new file mode 100644
index 00000000..41742be5
--- /dev/null
+++ b/templates/newsleak-proxy.nomad
@@ -0,0 +1,14 @@
+{% from '_lib.hcl' import shutdown_delay, authproxy_group, continuous_reschedule, set_pg_password_template, task_logs, group_disk with context -%}
+
+job "newsleak-proxy" {
+  datacenters = ["dc1"]
+  type = "service"
+  priority = 65
+
+  ${- authproxy_group(
+      'newsleak',
+      host='newsleak.' + liquid_domain,
+      upstream='newsleak-ui',
+    ) }
+}
+
diff --git a/templates/newsleak.nomad b/templates/newsleak.nomad
index f84addcc..ebb9efc8 100644
--- a/templates/newsleak.nomad
+++ b/templates/newsleak.nomad
@@ -1,37 +1,55 @@
-{% from '_lib.hcl' import shutdown_delay, authproxy_group, continuous_reschedule, set_pg_password_template, task_logs, group_disk with context -%} 
+{% from '_lib.hcl' import shutdown_delay, authproxy_group, continuous_reschedule, set_pg_password_template, task_logs, group_disk with context -%}
 
 job "newsleak" {
   datacenters = ["dc1"]
   type = "service"
   priority = 60
 
-  
   group "newsleak" {
     ${ group_disk() }
     task "newsleak-ui" {
       ${ task_logs() }
 
-        constraint {
-          attribute = "{% raw %}${meta.liquid_volumes}{% endraw %}"
-          operator = "is_set"
-        }
+      constraint {
+        attribute = "{% raw %}${meta.liquid_volumes}{% endraw %}"
+        operator = "is_set"
+      }
 
-        driver = "docker"
-        config {
-          image = "${config.image('newsleak')}"
-          volumes = [
-            "{% raw %}${meta.liquid_volumes}{% endraw %}/newsleak/ui/themes::/etc/settings/"
-          ]
-          args = ["/bin/sh", "-c", "chown -R 999:999 /etc/settings/ && echo chown done && /entrypoint.sh "]
-          port_map {
-            http = 80
-          }
-          labels {
-            liquid_task = "newsleak"
-          }
+      driver = "docker"
+      config {
+        image = "${config.image('liquid-newsleak')}"
+        volumes = [
+          "{% raw %}${meta.liquid_volumes}{% endraw %}/newsleak/ui/:/etc/settings/"
+        ]
+        # args = ["/bin/sh", "-c", " cp /local/conf /etc/settings/conf/newsleak.properties && chown 999:999 /etc/settings/conf/newsleak.properties &&echo conf loaded && /opt/newsleak/newsleak-start.sh" ]
+        port_map {
+          ui = 9000
+        }
+        labels {
+          liquid_task = "newsleak"
+        }
+      }
+      service {
+        name = "newsleak-ui"
+        port = "ui"
+        check {
+          name = "tcp"
+          initial_status = "critical"
+          type = "tcp"
+          interval = "${check_interval}"
+          timeout = "${check_timeout}"
+        }
+      }
+      resources {
+        memory = 7000
+        network {
+          mbits = 1
+          port "ui" {}
         }
-        template {
-          destination = "local/conf"
+      }
+      template {
+          destination = "/local/conf"
+          perms = "755"
           data = <<-EOF
             ################################
             ##     NEWSLEAK CONFIG        ##
@@ -47,7 +65,7 @@ job "newsleak" {
             threads = 4
 
             # ner, dictionaries and pattern extraction
-            nerserviceurl = {{- range service "newsleak-ner" -}}"{{.Address}}:{{.Port}}"{{- end -}}
+            nerserviceurl = http://{{- range service "newsleak-ner" -}}{{.Address}}:{{.Port}}{{- end }}
             dictionaryfiles = fck.all, deu:fck.deu, eng:fck.eng
             patternemail = true
             patternurl = false
@@ -58,34 +76,154 @@ job "newsleak" {
             datareader = hoover
 
             # CSV datareader options
-            datadirectory = ./data
+            datadirectory = /opt/newsleak/data
             documentfile = document.csv
             metadatafile = metadata.csv
 
             # Hoover datareader options
-            hooverindex = testdata
-            hooverclustername = elasticsearch
-            hooverurl = {{- range service "hoover-es-data" -}}"{{.Address}}"{{- end -}}
-            hooversearchurl = {{- range service "newsleak-ner" -}}"{{.Address}}:{{.Port}}"{{- end -}}
-            hooverport = {{- range service "hoover-es-data" -}}"{{.Port}}"{{- end -}}
+            hooverindex = ww2
+            hooverclustername = hoover
+            hooverurl = http://{{- range service "hoover-es-master" -}}{{.Address}}{{- end }}
+            hooversearchurl = http://hoover.${config.liquid_domain}:{{- range service "hoover-search" -}}{{.Port}}{{- end }}
+            hooverport = {{- range service "hoover-es-master" -}}{{.Port}}{{- end }}
             hoovertmpmetadata = hoover_metadata.csv
             processlanguages = eng, ron, deu
             # Newsleak postgres
-            dburl = {{- range service "newsleak-pg" -}}"{{.Address}}:{{.Port}}"{{- end -}}
+            dburl = {{- range service "newsleak-pg" -}}{{.Address}}:{{.Port}}{{- end }}
             dbname = newsleak
             dbuser = newsreader
             dbpass = newsreader
-            dbschema = desc/postgresSchema.sql
-            dbindices = desc/postgresIndices.sql
+            dbschema = /opt/newsleak/desc/postgresSchema.sql
+            dbindices = /opt/newsleak/desc/postgresIndices.sql
 
             # Newsleak elasticsearch index
             esindex = newsleak
-            esurl = {{- range service "newsleak-es-transport" -}}"{{.Address}}"{{- end -}}
+            esurl = {{- range service "newsleak-es-transport" -}}{{.Address}}{{- end }}
             esclustername = elasticsearch
-            esport = {{- range service "newsleak-es-transport" -}}"{{.Port}}"{{- end -}}
+            esport = {{- range service "newsleak-es-transport" -}}{{.Port}}{{- end }}
           EOF
       }
-    
+      template {
+        destination = "/local/application.production.conf"
+        perms = "755"
+        data = <<-EOF
+          # This is the main configuration file for the application.
+          # ~~~~~
+
+          # Production mode settings
+          # ~~~~~
+          application.mode=PROD
+
+          # XForwardedSupport="127.0.0.1"
+
+          # play.http.context="/noden/"
+          # play.http.context="/newsleak/"
+
+          # Secret key
+          # ~~~~~
+          # The secret key is used to secure cryptographics functions.
+          # If you deploy your application to several instances be sure to use the same key!
+          play.crypto.secret="CHANGE-THIS-APPLICATION-SECRET-PLEASE_48njXsy34tDJHusy2J"
+
+          # The application languages
+          # ~~~~~
+          play.i18n.langs= ["en"]
+
+          # Global object class
+          # ~~~~~
+          # Define the Global object class for this application.
+          # Default to Global in the root package.
+          # application.global=Global
+
+          # Router
+          # ~~~~~
+          # Define the Router object to use for this application.
+          # This router will be looked up first when the application is starting up,
+          # so make sure this is the entry point.
+          # Furthermore, it's assumed your route file is named properly.
+          # So for an application router like `my.application.Router`,
+          # you may need to define a router file `conf/my.application.routes`.
+          # Default to Routes in the root package (and conf/routes)
+          # application.router=my.application.Routes
+
+
+          # Authorization
+          # ~~~~~
+          #
+          authorization.enabled=false
+          authorization.login="user"
+          authorization.password="test"
+
+
+          # Data Source Configuration
+          # ~~~~~
+          # You can declare as many datasources as you want.
+          # By convention, the default datasource is named `default`
+          #
+
+          # base url for UI links to original sources (e.g. Hoover)
+          source.base.url = "http://hoover.${config.liquid_domain}:{{- range service "hoover-search" -}}{{.Port}}{{- end }}/doc/"
+
+          # Available ES collections (provide db.* configuration for each collection below)
+          es.indices =  [newsleak,newsleak2,newsleak3,newsleak4,newsleak5]
+          # ES connection
+          es.clustername = "elasticsearch"
+          es.address = "{{- range service "newsleak-es-transport" -}}{{.Address}}{{- end }}"
+          es.port = {{- range service "newsleak-es-transport" -}}{{.Port}}{{- end }}
+
+          # Determine the default dataset for the application
+          es.index.default = "newsleak"
+
+          # collection 1
+          db.newsleak.driver=org.postgresql.Driver
+          db.newsleak.url="jdbc:postgresql://{{- range service "newsleak-pg" -}}{{.Address}}:{{.Port}}{{- end }}/newsleak"
+          db.newsleak.username="newsreader"
+          db.newsleak.password="newsreader"
+          es.newsleak.excludeTypes = [Link,Filename,Path,Content-type,SUBJECT,HEADER,Subject,Timezone,sender.id,Recipients.id,Recipients.order]
+
+          # collection 2
+          db.newsleak2.driver=org.postgresql.Driver
+          db.newsleak2.url="jdbc:postgresql://{{- range service "newsleak-pg" -}}{{.Address}}:{{.Port}}{{- end }}/newsleak2"
+          db.newsleak2.username="newsreader"
+          db.newsleak2.password="newsreader"
+          es.newsleak2.excludeTypes = [Link,Filename,Path,Content-type,SUBJECT,HEADER,Subject,Timezone,sender.id,Recipients.id,Recipients.order]
+
+          # collection 3
+          db.newsleak3.driver=org.postgresql.Driver
+          db.newsleak3.url="jdbc:postgresql://{{- range service "newsleak-pg" -}}{{.Address}}:{{.Port}}{{- end }}/newsleak3"
+          db.newsleak3.username="newsreader"
+          db.newsleak3.password="newsreader"
+          es.newsleak3.excludeTypes = [Link,Filename,Path,Content-type,SUBJECT,HEADER,Subject,Timezone,sender.id,Recipients.id,Recipients.order]
+
+          # collection 4
+          db.newsleak4.driver=org.postgresql.Driver
+          db.newsleak4.url="jdbc:postgresql://{{- range service "newsleak-pg" -}}{{.Address}}:{{.Port}}{{- end }}/newsleak4"
+          db.newsleak4.username="newsreader"
+          db.newsleak4.password="newsreader"
+          es.newsleak4.excludeTypes = [Link,Filename,Path,Content-type,SUBJECT,HEADER,Subject,Timezone,sender.id,Recipients.id,Recipients.order]
+
+          # collection 5
+          db.newsleak5.driver=org.postgresql.Driver
+          db.newsleak5.url="jdbc:postgresql://{{- range service "newsleak-pg" -}}{{.Address}}:{{.Port}}{{- end }}/newsleak5"
+          db.newsleak5.username="newsreader"
+          db.newsleak5.password="newsreader"
+          es.newsleak5.excludeTypes = [Link,Filename,Path,Content-type,SUBJECT,HEADER,Subject,Timezone,sender.id,Recipients.id,Recipients.order]
+
+          # --------------------
+          # Other configurations
+          scalikejdbc.global.loggingSQLAndTime.enabled=false
+          scalikejdbc.global.loggingSQLAndTime.singleLineMode=false
+          scalikejdbc.global.loggingSQLAndTime.logLevel=info
+          scalikejdbc.global.loggingSQLAndTime.warningEnabled=false
+          scalikejdbc.global.loggingSQLAndTime.warningThresholdMillis=5
+          scalikejdbc.global.loggingSQLAndTime.warningLogLevel=warn
+
+          play.modules.enabled += "scalikejdbc.PlayModule"
+          # scalikejdbc.PlayModule doesn't depend on Play's DBModule
+          play.modules.disabled += "play.api.db.DBModule"
+        EOF
+      }
     }
   }
-}
\ No newline at end of file
+}
+

From fbd9edfa0483da50a49cd8c75d2f40db4b4591a8 Mon Sep 17 00:00:00 2001
From: MattesR <mattes.ruckdeschel@tuhh.de>
Date: Mon, 26 Apr 2021 14:03:21 +0200
Subject: [PATCH 7/9] stash?

---
 templates/newsleak-deps.nomad | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/templates/newsleak-deps.nomad b/templates/newsleak-deps.nomad
index 05d99a7b..2b80c141 100644
--- a/templates/newsleak-deps.nomad
+++ b/templates/newsleak-deps.nomad
@@ -53,6 +53,10 @@ job "newsleak-deps" {
       }
 
       ${ set_pg_password_template('newsreader') }
+<<<<<<< Updated upstream
+=======
+
+>>>>>>> Stashed changes
       resources {
         memory = 350
         network {
@@ -127,7 +131,11 @@ job "newsleak-deps" {
 
       resources {
         cpu = 600
+<<<<<<< Updated upstream
         memory = ${config.elasticsearch_memory_limit}
+=======
+        memory = 2500
+>>>>>>> Stashed changes
         network {
           mbits = 1
           port "http" {}

From 17e88338e87ccf5e1610cc871750086f07d01e82 Mon Sep 17 00:00:00 2001
From: MattesR <mattes.ruckdeschel@tuhh.de>
Date: Mon, 26 Apr 2021 14:04:30 +0200
Subject: [PATCH 8/9] add last config stuff

---
 liquid_node/configuration.py | 12 ++++++++----
 production-versions.ini      |  1 +
 testing-versions.ini         |  1 +
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/liquid_node/configuration.py b/liquid_node/configuration.py
index 76cddc96..f3458452 100644
--- a/liquid_node/configuration.py
+++ b/liquid_node/configuration.py
@@ -8,7 +8,7 @@
 from .util import import_string
 from .docker import docker
 from liquid_node.jobs import Job, liquid, hoover, dokuwiki, rocketchat, \
-    nextcloud, hypothesis, codimd, ci
+    nextcloud, hypothesis, codimd, ci, newsleak
 
 
 def split_lang_codes(option):
@@ -20,7 +20,7 @@ def split_lang_codes(option):
 
 class Configuration:
     ALL_APPS = ('hoover', 'dokuwiki', 'rocketchat', 'nextcloud',
-                'hypothesis', 'codimd',)
+                'hypothesis', 'codimd', 'newsleak')
     # The core apps can't be turned off.
     CORE_APPS = ('liquid', 'ingress',)
 
@@ -36,7 +36,8 @@ class Configuration:
         'dokuwiki': 'is a wiki system used as a knowledge base for processed information.',
         'codimd': 'is a real-time collaboration pad.',
         'nextcloud': 'has a file share system and a contact list of users.',
-        'rocketchat': 'is the chat app.'
+        'rocketchat': 'is the chat app.',
+        'newsleak' : 'University of Hamburg entity extraction and visualization tool'
     }
 
     ALL_JOBS = [
@@ -68,6 +69,9 @@ class Configuration:
         ci.Drone(),
         ci.Deps(),
         ci.DroneWorkers(),
+        newsleak.Newsleak(),
+        newsleak.Deps(),
+        newsleak.Proxy()
     ]
 
     def __init__(self):
@@ -306,7 +310,7 @@ def tag(name):
             client = tag('h-client')
             return f'h: {h},  client: {client}'
 
-        if name in ['dokuwiki', 'nextcloud']:
+        if name in ['dokuwiki', 'nextcloud', 'newsleak']:
             return tag('liquid-' + name)
 
         return tag(name)
diff --git a/production-versions.ini b/production-versions.ini
index 6d783d63..24f0703b 100644
--- a/production-versions.ini
+++ b/production-versions.ini
@@ -9,6 +9,7 @@ liquid-authproxy = liquidinvestigations/oauth-proxy:0.0.3
 liquid-core = liquidinvestigations/core:0.5.0
 liquid-dokuwiki = liquidinvestigations/liquid-dokuwiki:0.1.0
 liquid-nextcloud = liquidinvestigations/liquid-nextcloud:0.2.3
+liquid-newsleak = liquidinvestigations/newsleak:latest
 rocketchat = rocketchat/rocket.chat:3.9.1
 rocketchat-mongo = mongo:4.4
 nlp-service = liquidinvestigations/nlp-service:initial_droned_service
diff --git a/testing-versions.ini b/testing-versions.ini
index c2840137..37d164ec 100644
--- a/testing-versions.ini
+++ b/testing-versions.ini
@@ -9,6 +9,7 @@ liquid-authproxy = liquidinvestigations/oauth-proxy:master
 liquid-core = liquidinvestigations/core:master
 liquid-dokuwiki = liquidinvestigations/liquid-dokuwiki:master
 liquid-nextcloud = liquidinvestigations/liquid-nextcloud:0.2.3
+liquid-newsleak = liquidinvestigations/newsleak:latest
 rocketchat = rocketchat/rocket.chat:3.9.1
 rocketchat-mongo = mongo:4.4
 nlp-service = liquidinvestigations/nlp-service:initial_droned_service

From f676053a5140d91f1f47cb338e7fd3fd98061535 Mon Sep 17 00:00:00 2001
From: MattesR <mattes.ruckdeschel@tuhh.de>
Date: Mon, 26 Apr 2021 14:16:10 +0200
Subject: [PATCH 9/9] stash

---
 templates/newsleak-deps.nomad | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/templates/newsleak-deps.nomad b/templates/newsleak-deps.nomad
index 2b80c141..6ffd3e9d 100644
--- a/templates/newsleak-deps.nomad
+++ b/templates/newsleak-deps.nomad
@@ -53,10 +53,6 @@ job "newsleak-deps" {
       }
 
       ${ set_pg_password_template('newsreader') }
-<<<<<<< Updated upstream
-=======
-
->>>>>>> Stashed changes
       resources {
         memory = 350
         network {
@@ -131,11 +127,7 @@ job "newsleak-deps" {
 
       resources {
         cpu = 600
-<<<<<<< Updated upstream
-        memory = ${config.elasticsearch_memory_limit}
-=======
         memory = 2500
->>>>>>> Stashed changes
         network {
           mbits = 1
           port "http" {}