docker-compose.yml

services:
  postgres:
    image: postgres:16
    container_name: polaris-postgres
    environment:
      POSTGRES_DB: polaris_db
      POSTGRES_USER: polaris_user
      POSTGRES_PASSWORD: polaris_password
    volumes:
      - postgres_data:/var/lib/postgresql/data
    ports:
      - "5432:5432"
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U polaris_user -d polaris_db"]
      interval: 10s
      timeout: 5s
      retries: 5
    networks:
      - opendata_network
    restart: unless-stopped

  minio:
    image: minio/minio
    ports:
      - "${MINIO_API_PORT:-9000}:9000"
      - "${MINIO_CONSOLE_PORT:-9001}:9001"
    environment:
      MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin}
      MINIO_DOMAIN: minio
    volumes:
      - minio_data:/data
    command: server /data --console-address ":9001"
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:9000/minio/health/live"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 10s
    networks:
      opendata_network:
        aliases:
          - warehouse.minio
          - minio

  mc:
    depends_on:
      minio:
        condition: service_healthy
    image: minio/mc
    volumes:
      - ./minio-config:/config
    networks:
      - opendata_network
    environment:
      MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-miniouser}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-miniouser}
    entrypoint: /bin/sh
    command: /config/setup-minio.sh

  polaris:
    image: apache/polaris:latest
    platform: linux/amd64
    ports:
      - "8181:8181"
      - "8182:8182"
    depends_on:
      minio:
        condition: service_healthy
      postgres:
        condition: service_healthy
    networks:
      - opendata_network
    environment:
      AWS_ACCESS_KEY_ID: minioadmin
      AWS_SECRET_ACCESS_KEY: minioadmin
      AWS_REGION: ${AWS_REGION:-us-east-1}
      AWS_ENDPOINT_URL_S3: http://minio:9000
      AWS_ENDPOINT_URL_STS: http://minio:9000
      POLARIS_BOOTSTRAP_CREDENTIALS: default,admin,password
      polaris.realm-context.realms: default
      polaris.features.DROP_WITH_PURGE_ENABLED: "true"
      POLARIS_PERSISTENCE_TYPE: relational-jdbc
      QUARKUS_DATASOURCE_DB_KIND: postgresql
      QUARKUS_DATASOURCE_JDBC_URL: jdbc:postgresql://postgres:5432/polaris_db
      QUARKUS_DATASOURCE_USERNAME: polaris_user
      QUARKUS_DATASOURCE_PASSWORD: polaris_password
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8182/q/health"]
      interval: 10s
      timeout: 5s
      retries: 20
      start_period: 60s

  polaris-bootstrap:
    image: apache/polaris-admin-tool:latest
    platform: linux/amd64
    depends_on:
      postgres:
        condition: service_healthy
    networks:
      - opendata_network
    environment:
      POLARIS_PERSISTENCE_TYPE: relational-jdbc
      QUARKUS_DATASOURCE_DB_KIND: postgresql
      QUARKUS_DATASOURCE_JDBC_URL: jdbc:postgresql://postgres:5432/polaris_db
      QUARKUS_DATASOURCE_USERNAME: polaris_user
      QUARKUS_DATASOURCE_PASSWORD: polaris_password
      POLARIS_BOOTSTRAP_CREDENTIALS: default,admin,password
    command: ["bootstrap", "--realm", "default", "--credential", "default,admin,password", "--print-credentials"]

  polaris-init:
    image: python:3.11-slim
    depends_on:
      polaris:
        condition: service_healthy
      mc:
        condition: service_completed_successfully
    volumes:
      - ./polaris-config:/config
      - ./dagster-workspace/projects/streamify:/opt/streamify
    networks:
      - opendata_network
    environment:
      POLARIS_HOST: polaris
      MINIO_HOST: minio
    entrypoint: /bin/bash
    command:
      - -c
      - |
        pip install requests > /dev/null 2>&1
        python /config/setup_polaris.py

  zookeeper:
    image: confluentinc/cp-zookeeper:7.5.0
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    networks:
      - opendata_network
    volumes:
      - zookeeper_data:/var/lib/zookeeper/data
      - zookeeper_logs:/var/lib/zookeeper/log

  kafka:
    image: confluentinc/cp-kafka:7.5.0
    depends_on:
      - zookeeper
    ports:
      - "9092:9092"
      - "9093:9093"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:9093
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
    networks:
      - opendata_network
    volumes:
      - kafka_data:/var/lib/kafka/data
    healthcheck:
      test: ["CMD-SHELL", "kafka-broker-api-versions --bootstrap-server localhost:9092"]
      interval: 10s
      timeout: 10s
      retries: 5
      start_period: 30s

  # ---------------------------------------------------------------------------
  # Spark 4.1.1 — Scala 2.13, Java 21
  # Iceberg 1.10.1 (iceberg-spark-runtime-4.0_2.13) — latest stable for Spark 4
  # Hadoop AWS 3.4.1 — matches Spark 4.x bundled Hadoop
  # ---------------------------------------------------------------------------

  spark-master:
    image: apache/spark:4.1.1
    command: /opt/spark/sbin/start-master.sh -h spark-master
    environment:
      SPARK_NO_DAEMONIZE: "true"
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-minioadmin}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-minioadmin}
      AWS_REGION: ${AWS_REGION:-us-east-1}
      AWS_ENDPOINT_URL: http://minio:9000
    ports:
      - "8080:8080"
      - "7077:7077"
    networks:
      - opendata_network
    volumes:
      - ./data:/data
      - ./dagster-workspace/projects/streamify:/opt/streamify

  spark-worker:
    image: apache/spark:4.1.1
    user: root
    command: /opt/spark/sbin/start-worker.sh spark://spark-master:7077
    depends_on:
      - spark-master
    environment:
      SPARK_NO_DAEMONIZE: "true"
      SPARK_WORKER_MEMORY: 4G
      SPARK_WORKER_CORES: 4
      AWS_ACCESS_KEY_ID: minioadmin
      AWS_SECRET_ACCESS_KEY: minioadmin
      AWS_REGION: ${AWS_REGION:-us-east-1}
      AWS_ENDPOINT_URL: http://minio:9000
    networks:
      - opendata_network
    volumes:
      - ./data:/data
      - ./dagster-workspace/projects/streamify:/opt/streamify

  spark-connect:
    image: apache/spark:4.1.1
    user: root
    entrypoint: /bin/bash
    command:
      - -c
      - |
        mkdir -p /root/.ivy2
        chmod -R 777 /root/.ivy2

        # Spark 4.x / Scala 2.13 package coordinates.
        # spark-connect is built into Spark 4 — no separate artifact needed.
        PACKAGES="\
        org.apache.iceberg:iceberg-spark-runtime-4.0_2.13:1.10.1,\
        org.apache.iceberg:iceberg-aws-bundle:1.10.1,\
        org.apache.hadoop:hadoop-aws:3.4.1,\
        org.apache.spark:spark-sql-kafka-0-10_2.13:4.1.1"

        # 1. Pre-download packages via a throwaway SparkPi run so the ivy cache
        #    is warm before the Connect server starts.
        echo "==> Downloading packages into ivy cache..."
        /opt/spark/bin/spark-submit \
          --packages "$$PACKAGES" \
          --conf spark.jars.ivy=/root/.ivy2 \
          --class org.apache.spark.examples.SparkPi \
          /opt/spark/examples/jars/spark-examples_*.jar 1

        # 2. Start Spark Connect server.
        # jars are distributed to executors via Spark's built-in --packages
        # mechanism (uploaded AFTER Spark's own jars, so no Netty/Netty conflict).
        echo "==> Starting Spark Connect server..."
        /opt/spark/sbin/start-connect-server.sh \
          --packages "$$PACKAGES" \
          --conf spark.driver.host=spark-connect \
          --conf spark.driver.memory=2g \
          --conf spark.executor.memory=2g \
          --conf spark.executor.cores=2 \
          --conf spark.jars.ivy=/root/.ivy2 \
          --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \
          --conf spark.sql.catalog.lakehouse=org.apache.iceberg.spark.SparkCatalog \
          --conf spark.sql.catalog.lakehouse.type=rest \
          --conf spark.sql.catalog.lakehouse.uri=http://polaris:8181/api/catalog \
          --conf spark.sql.catalog.lakehouse.warehouse=lakehouse \
          --conf spark.sql.catalog.lakehouse.credential=$${POLARIS_CLIENT_ID}:$${POLARIS_CLIENT_SECRET} \
          --conf spark.sql.catalog.lakehouse.scope=PRINCIPAL_ROLE:ALL \
          --conf spark.sql.catalog.lakehouse.token-refresh-enabled=true \
          --conf spark.sql.catalog.lakehouse.s3.endpoint=http://minio:9000 \
          --conf spark.sql.catalog.lakehouse.s3.access-key-id=minioadmin \
          --conf spark.sql.catalog.lakehouse.s3.secret-access-key=minioadmin \
          --conf spark.sql.catalog.lakehouse.s3.path-style-access=true \
          --conf spark.sql.defaultCatalog=lakehouse \
          --conf spark.hadoop.fs.s3a.access.key=minioadmin \
          --conf spark.hadoop.fs.s3a.secret.key=minioadmin \
          --conf spark.hadoop.fs.s3a.endpoint=http://minio:9000 \
          --conf spark.hadoop.fs.s3a.path.style.access=true \
          --conf spark.hadoop.fs.s3a.connection.ssl.enabled=false \
          --conf spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem \
          --master spark://spark-master:7077
    depends_on:
      - spark-master
    environment:
      SPARK_NO_DAEMONIZE: "true"
      AWS_ACCESS_KEY_ID: minioadmin
      AWS_SECRET_ACCESS_KEY: minioadmin
      AWS_REGION: ${AWS_REGION:-us-east-1}
      AWS_ENDPOINT_URL: http://minio:9000
      POLARIS_CLIENT_ID: ${POLARIS_CLIENT_ID}
      POLARIS_CLIENT_SECRET: ${POLARIS_CLIENT_SECRET}
    ports:
      - "15002:15002"
      - "4041:4040"
    networks:
      - opendata_network
    volumes:
      - ./data:/data
      - ./dagster-workspace/projects/streamify:/opt/streamify
      - spark_ivy_cache:/root/.ivy2

  eventsim:
    build:
      context: ./eventsim
      dockerfile: Dockerfile
    container_name: eventsim
    depends_on:
      kafka:
        condition: service_healthy
    networks:
      - opendata_network
    environment:
      - JAVA_OPTS=-Xmx4G
    entrypoint: ["/bin/bash", "-c"]
    command:
      - |
        /opt/eventsim/eventsim.sh \
          -c /opt/eventsim/examples/example-config.json \
          --from 1 \
          --to 0 \
          --nusers 3000 \
          --growth-rate 0.01 \
          --userid 1 \
          --kafkaBrokerList kafka:9092 \
          --randomseed $$(shuf -i 1-10000 -n 1) \
          --continuous
    restart: unless-stopped
    mem_limit: 6g
    memswap_limit: 8g

  kafdrop:
    image: obsidiandynamics/kafdrop:latest
    ports:
      - "9002:9000"
    environment:
      KAFKA_BROKERCONNECT: kafka:9092
      JVM_OPTS: "-Xms32M -Xmx64M"
      SERVER_SERVLET_CONTEXTPATH: "/"
    networks:
      - opendata_network
    depends_on:
      kafka:
        condition: service_healthy
    restart: unless-stopped

  polaris-console:
    build:
      context: ./polaris-console/console
      dockerfile: docker/Dockerfile
    ports:
      - "3001:80"
    environment:
      VITE_POLARIS_API_URL: http://polaris:8181
      VITE_POLARIS_REALM: default
      VITE_OAUTH_TOKEN_URL: http://polaris:8181/api/catalog/v1/oauth/tokens
      VITE_POLARIS_REALM_HEADER_NAME: realm
    networks:
      - opendata_network
    depends_on:
      polaris:
        condition: service_healthy
    restart: unless-stopped

networks:
  opendata_network:
    driver: bridge

volumes:
  postgres_data:
  minio_data:
  zookeeper_data:
  zookeeper_logs:
  kafka_data:
  # ivy2 cache for spark-connect package downloads (driver-side only)
  spark_ivy_cache: