docker-compose.yml

services:
  spark:
    build:
      context: .
      dockerfile: spark.Dockerfile
    user: root
    environment:
      - SPARK_MODE=master
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
      - SPARK_USER=root
      - PYSPARK_PYTHON=/opt/bitnami/python/bin/python3.12
      - PYSPARK_DRIVER_PYTHON=/opt/bitnami/python/bin/python3.12
      - SPARK_VERSION=3.5.2
      - HADOOP_VERSION=3
    ports:
      - '8080:8080'
      - '7077:7077'
    volumes:
      - .:/main/workspace
      - samba-volume:/main/external
    security_opt:
      - label:disable
      - seccomp:unconfined
      - apparmor:unconfined
    cap_add:
      - ALL
    privileged: true
  spark-worker:
    build:
      context: .
      dockerfile: spark.Dockerfile
    depends_on:
      - spark
    user: root
    deploy:
      replicas: ${N_REPLICAS}
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER=$SPARK_MASTER_URL
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_WORKER_MEMORY=$MEMORY
      - SPARK_EXECUTOR_MEMORY=$MEMORY
      - SPARK_WORKER_CORES=$N_CORES
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
      - SPARK_USER=root
      - PYSPARK_PYTHON=/opt/bitnami/python/bin/python3.12
      - PYSPARK_DRIVER_PYTHON=/opt/bitnami/python/bin/python3.12
      - SPARK_VERSION=3.5.2
      - HADOOP_VERSION=3
    volumes:
      - .:/main/workspace
      - samba-volume:/main/external
    security_opt:
      - label:disable
      - seccomp:unconfined
      - apparmor:unconfined
    cap_add:
      - ALL
    privileged: true
  almond:
    build:
      context: .
      dockerfile: almond.Dockerfile
    user: root
    ports:
      - 8002:8888
    environment:
      - JUPYTER_TOKEN=almond
      - JUPYTERLAB_WORKSPACES_DIR=/main
    volumes:
      - .:/main/workspace
      - samba-volume:/main/external
    security_opt:
      - label:disable
      - seccomp:unconfined
      - apparmor:unconfined
    cap_add:
      - ALL
    privileged: true
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 1
              capabilities: [gpu]
volumes:
  samba-volume:
    driver_opts:
      type: cifs
      o: username=${SMB_USER},password=${SMB_PASSWORD},vers=3.0,rw,file_mode=0777,dir_mode=0777
      device: ${SMB_URL}