docker-stack.yml

version: "3.3"

services:

  traefik:
    image: traefik:1.4.2
    command: --docker --docker.swarmmode --docker.domain=traefik --docker.watch --web --logLevel=DEBUG
    ports:
      - "80:80"
      - "8080:8080"
      - "443:443"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./conf/traefik/traefik.toml:/traefik.toml:ro
      - data_traefik_acme:/etc/traefik/acme
    deploy:
      placement:
        constraints: 
          - node.role == manager

  namenode:
    image: dmitryzagr/hadoop-namenode:1.2.1-hadoop2.8.1-java8
    hostname: namenode
    ports:
      - "50070:50070"
    volumes:
      - namenode_data:/hadoop/dfs/name
    environment:
      - CLUSTER_NAME=test
    env_file:
      - ./hadoop-hive.env
    deploy:
      labels:
        - "traefik.enable=false"
      placement:
        constraints: 
          - node.role == manager

  datanode_1:
    image: dmitryzagr/hadoop-datanode:1.2.1-hadoop2.8.1-java8
    volumes:
      - datanode_data_1:/hadoop/dfs/data
      - datanode_archives:/root/hadoop_data
    env_file:
      - ./hadoop-hive.env
    depends_on:
      - namenode
    deploy:
      mode: global
      labels:
        - "traefik.enable=false"
      placement:
        constraints: 
          - node.role == worker
          - node.labels.disk == ssd

  datanode_2:
    image: dmitryzagr/hadoop-datanode:1.2.1-hadoop2.8.1-java8
    volumes:
      - datanode_data_2:/hadoop/dfs/data
      - datanode_archives:/root/hadoop_data
    env_file:
      - ./hadoop-hive.env
    depends_on:
      - namenode
    deploy:
      mode: global
      labels:
        - "traefik.enable=false"
      placement:
        constraints: 
          - node.role == worker
          - node.labels.disk == ssd

  spark-master:
    image: dmitryzagr/spark-master:2.2.0-hadoop2.8.1-hive-java8
    hostname: spark-master
    ports:
      - "8989:8080"
    env_file:
      - ./hadoop-hive.env
    depends_on:
      - namenode
    deploy:
      labels:
        - "traefik.enable=false"
      placement:
        constraints: 
          - node.role == manager

  spark-worker:
    image: dmitryzagr/spark-worker:2.2.0-hadoop2.8.1-hive-java8
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"
    env_file:
      - ./hadoop-hive.env
      - ./spark.env
    deploy:
      replicas: 2
      labels:
        - "traefik.enable=false"
      placement:
        constraints: 
          - node.role == worker

  hive-server:
    image: dmitryzagr/hadoop-hive:2.2.0-hadoop2.8.1-java8
    hostname: hive-server
    env_file:
      - ./hadoop-hive.env
    environment:
      - "HIVE_CORE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore/metastore"
    depends_on:
      - namenode
      - hive-metastore
    labels:
      - "traefik.enable=false"
    deploy:
      placement:
        constraints: 
          - node.role == manager

  hive-metastore:
    image: dmitryzagr/hadoop-hive:2.2.0-hadoop2.8.1-java8
    hostname: hive-metastore
    env_file:
      - ./hadoop-hive.env
    command: /opt/hive/bin/hive --service metastore
    depends_on:
      - hive-metastore-postgresql
    deploy:
      labels:
        - "traefik.enable=false"
      placement:
        constraints: 
          - node.role == manager

  hive-metastore-postgresql:
    image: dmitryzagr/hive-metastore-postgresql:2.2.0-hadoop2.8.1-java8
    hostname: hive-metastore-postgresql
    volumes:
      - hive-metastore-postgresql_data:/var/lib/postgresql/data
    deploy:
      labels:
        - "traefik.enable=false"
      placement:
        constraints: 
          - node.role == worker
          - node.labels.disk == ssd
          - node.labels.disk.type == hive-metastore

  zeppelin:
    image: apache/zeppelin:0.7.3
    environment:
      - ZEPPELIN_LOG_DIR=/logs
      - ZEPPELIN_NOTEBOOK_DIR=/notebook
      - ZEPPELIN_SERVER_CONTEXT_PATH=/zeppelin
    volumes:
      - ./conf/zeppelin/interpreter.json:/zeppelin/conf/interpreter.json
      - ./conf/zeppelin/conf/shiro.ini:/zeppelin/conf/shiro.ini
      - ./conf/zeppelin/conf/zeppelin-site.xml:/zeppelin/conf/zeppelin-site.xml
      - zeppelin_data-logs:/logs
      - zeppelin_data-notebook:/notebook
    deploy:
      labels:
        - "traefik.backend=zeppelin"
        - "traefik.backend.loadbalancer.sticky=true"
        - "traefik.frontend.rule=PathPrefix:/zeppelin"
        - "traefik.enable=true"
        - "traefik.port=8080"
      placement:
        constraints: 
          - node.role == manager

  visualizer:
    image: dockersamples/visualizer
    ports:
      - "8082:8080"
    stop_grace_period: 1m30s
    volumes:
      - "/var/run/docker.sock:/var/run/docker.sock"
    deploy:
      placement:
        constraints:
          - node.role == manager

volumes:
    namenode_data:
    zeppelin_data-logs:
    zeppelin_data-notebook:
    hive-metastore-postgresql_data:
    datanode_archives: # volume datanode_archives предназначен для загрузки архива
    datanode_data_1:
    datanode_data_2:
    data_traefik_acme:


networks:
  hadoop:
    driver: overlay
    ipam:
      driver: default
      config:
      - subnet: 10.1.0.0/24