docker-compose.yml

version: '3.9'

networks:
    datapipeline:
        driver: bridge
        ipam:
            driver: default
            config:
                - subnet: "172.18.0.0/16"

services:
  zookeeper:
    image: confluentinc/cp-zookeeper:6.2.0
    environment:
      - ZOOKEEPER_CLIENT_PORT=2181
    ports:
      - "2181:2181"
    restart: on-failure
    networks:
      datapipeline:
        ipv4_address: 172.18.0.3

  kafka:
    image: confluentinc/cp-kafka:6.2.0
    environment:
      - KAFKA_BROKER_ID=1
      - KAFKA_ZOOKEEPER_CONNECT=172.18.0.3:2181
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,PLAINTEXT_HOST://172.18.0.4:29092
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
      - KAFKA_LOG4J_LOGGERS=org.apache.zookeeper=ERROR,org.apache.kafka=ERROR,kafka=ERROR,kafka.cluster=ERROR,kafka.controller=ERROR,kafka.coordinator=ERROR,kafka.log=ERROR,kafka.server=ERROR,kafka.zookeeper=ERROR,state.change.logger=ERROR # Remove this line to enable extension logging
      # - KAFKA_LOG4J_ROOT_LOGLEVEL=ERROR
      # - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
    ports:
      - "29092:29092"
    restart: on-failure
    depends_on:
      - zookeeper
    networks:
      datapipeline:
        ipv4_address: 172.18.0.4

  processstream:
    image: btctothemoonz/travisnkento:0.0.8
    command: >
      bash -c "cd ../volume && python bin/processStream.py --model-call-topic call-stream --model-response-topic response-stream --speed 1"
    volumes:
      - '$PWD:/volume'
      - '$PWD/jars:/root/.ivy2/jars'
    ports:
      - '9042:9042'
      - '8080:8080'
      - '4040:4040'
      - '9870:9870'
      - '8088:8088'
      - '8042:8042'
    depends_on:
      - kafka
      - deeplearning
    restart: on-failure
    networks:
      datapipeline:
        ipv4_address: 172.18.0.5
    
  plotstream:
    image: btctothemoonz/travisnkento:0.0.8
    command: >
      bash -c "cd ../volume && python bin/plotData.py --model-call-topic call-stream --speed 1"
    volumes:
      - '$PWD:/volume'
      - '$PWD/jars:/root/.ivy2/jars'
    ports:
      - '5068:5068'
    depends_on:
      #- kafka
      db:
        condition: service_healthy
    restart: on-failure
    networks:
      datapipeline:
        ipv4_address: 172.18.0.9

  sendstream:
    image: btctothemoonz/travisnkento:0.0.8
    command: >
      bash -c "cd ../volume && python bin/sendStream.py --speed 1 --frequency 1"
    volumes:
      - '$PWD:/volume' #so it turns out this line is causing the problem before. 
                      #previously, all the "home" is replaced because of the conflict in name
      - '$PWD/jars:/root/.ivy2/jars'
    #ports:
    #  - '8080:8080'
    depends_on:
      - kafka
      - processstream
    restart: on-failure
    networks:
      datapipeline:
        ipv4_address: 172.18.0.2

  deeplearning:
    image: btctothemoonz/travisnkento:0.0.8
    command: >
      bash -c "cd ../volume && python bin/predictStream.py --model-call-topic call-stream --model-response-topic response-stream --speed 1"
    volumes:
      - '$PWD:/volume'
      - '$PWD/jars:/root/.ivy2/jars'
    ports:
      - '4041:4040'
    depends_on:
    #   - processstream
      db:
        condition: service_healthy
    restart: on-failure
    networks:
      datapipeline:
        ipv4_address: 172.18.0.7

    
  db:
    image: mysql:8.0
    cap_add:
      - SYS_NICE
    environment:
      - MYSQL_DATABASE=mysql # use default; no need to create specify MYSQL_DATABASE
      - MYSQL_ROOT_PASSWORD=mauFJcuf5dhRMQrjj
    ports:
      - '3306:3306'
    volumes:
      # - '$PWD/db:/var/lib/mysql'
      - '$PWD/db:/docker-entrypoint-initdb.d'
      - '$PWD:/volume'
    restart: on-failure
    networks:
      datapipeline:
        ipv4_address: 172.18.0.8
    healthcheck:
      test: ["CMD", 'mysqladmin', 'ping', '-h', 'localhost', '-u', 'root', '-p$$MYSQL_ROOT_PASSWORD' ]
      timeout: 30s

volumes:
  db:
    driver: local

#  To remove and reinit databse:
# -docker compose down -v 
# docker-compose up --force-recreate db