wip: add offline local ai support

Author: mvdam

tools/development/README.md

@@ -7,10 +7,14 @@ podman compose up -d
 
 ## Tools
 
-### [LibreTranslate](https://github.com/LibreTranslate/LibreTranslate) - [http://localhost:5000/](http://localhost:5000/)
+### [LibreTranslate](https://github.com/LibreTranslate/LibreTranslate) - [http://localhost:5100/](http://localhost:5100/)
 
 Self-hosted Machine Translation API, used for sensitive content.
 
-### [SpiderFoot](https://github.com/smicallef/spiderfoot) - [http://localhost:5001/](http://localhost:5001/)
+### [SpiderFoot](https://github.com/smicallef/spiderfoot) - [http://localhost:5200/](http://localhost:5200/)
 
 SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
+
+### [Cyberchef](https://github.com/gchq/CyberChef) - [http://localhost:5300/](http://localhost:5300/)
+
+CyberChef is a simple, intuitive web app for carrying out all manner of "cyber" operations within a web browser. These operations include simple encoding like XOR and Base64, more complex encryption like AES, DES and Blowfish, creating binary and hexdumps, compression and decompression of data, calculating hashes and checksums, IPv6 and X.509 parsing, changing character encodings, and much more.

tools/development/docker-compose.yaml

@@ -1,21 +1,61 @@
-version: "3.0"
-
 services:
   libretranslate:
-    container_name: libretranslate
     image: libretranslate/libretranslate
-    restart: always
+    container_name: libretranslate
     ports:
-      - "5000:5000"
+      - "5100:5000"
+    restart: unless-stopped
+    networks:
+      - libretranslate
 
   spiderfoot:
+    image: ctdc/spiderfoot
+    container_name: spiderfoot
     volumes:
       - spiderfoot-data:/var/lib/spiderfoot
-    image: dtagdevsec/spiderfoot
-    container_name: spiderfoot
     ports:
-      - "5001:5001"
+      - "5200:5001"
+    restart: unless-stopped
+    networks:
+      - spiderfoot
+
+  cyberchef:
+    image: ghcr.io/gchq/cyberchef
+    container_name: cyberchef
+    ports:
+      - "5300:80"
+    restart: unless-stopped
+    networks:
+      - cyberchef
+
+  # Vaultwarden
+  vaultwarden:
+    image: vaultwarden/server:latest
+    networks:
+      - vaultwarden
+    container_name: vaultwarden
     restart: unless-stopped
+    environment:
+      DOMAIN: "https://vault.shinyapps.nl"
+      SIGNUPS_ALLOWED: false
+    volumes:
+      - /opt/homelab/vaultwarden/vw-data/:/data/
+    ports:
+      - 4000:80
 
 volumes:
   spiderfoot-data:
+
+networks:
+  libretranslate:
+    name: libretranslate
+    driver: bridge
+  spiderfoot:
+    name: spiderfoot
+    driver: bridge
+  cyberchef:
+    name: cyberchef
+    driver: bridge
+  vaultwarden:
+    name: vaultwarden
+    driver: bridge

tools/development/stopAll.sh

@@ -0,0 +1 @@
+podman stop $(podman ps -a -q) && podman rm $(podman ps -a -q) && podman system prune

tools/llm/README.md

@@ -0,0 +1,37 @@
+## Services
+
+### Ollama
+
+http://localhost:11434
+
+### Open WebUI
+
+http://localhost:3000
+
+### Run the stack
+
+```
+podman compose up -d
+```
+
+## Tips & Tricks
+
+```
+podman exec -ti ollama /bin/bash
+
+ollama pull llama3
+
+podman stats --no-stream
+```
+
+### Podman specifics
+
+```
+podman machine stop
+podman machine set --cpus 4 --memory 8192
+podman machine start
+```
+
+TODO:
+check if we can increase CPU count
+https://www.cpu-monkey.com/en/cpu-apple_m1_pro_10_cpu_16_gpu

tools/llm/docker-compose.yml

@@ -0,0 +1,53 @@
+services:
+  ollama:
+    image: ollama/ollama
+    container_name: ollama
+    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          cpus: 8
+          memory: 8G
+    security_opt:
+      - no-new-privileges
+    cap_drop:
+      - all
+    cap_add:
+      - SYS_NICE
+    ports:
+      - "11434:11434"
+    volumes:
+      - ollama:/root/.ollama
+    networks:
+      - ollama
+
+  openwebui:
+    image: ghcr.io/open-webui/open-webui:main
+    container_name: openwebui
+    ports:
+      - "3000:8080"
+    volumes:
+      - open-webui:/app/backend/data
+    depends_on:
+      - ollama
+    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          cpus: 1
+          memory: 1G
+    networks:
+      - ollama
+
+volumes:
+  ollama:
+  open-webui:
+
+networks:
+  ollama:
+    name: ollama
+    driver: bridge
+  # todo: after install the model, we want to use offline to be sure it does not leak anything
+  offline:
+    name: offline
+    driver: none

tools/llm/start.sh

@@ -0,0 +1,9 @@
+# setup correct limits
+podman machine stop
+podman machine set --cpus 8 --memory 8192
+podman machine start
+
+podman compose up -d
+podman exec -it ollama ollama pull llama3
+
+# todo: stop ollama and start it again offline