docker-compose.yml

version: '3.2'
services:
    notebook:
          image: jupyter/all-spark-notebook
          hostname: notebook
          depends_on:
            - master
            - worker
            - spark-ui-proxy
          environment:
            - AWS_ACCESS_KEY=${AWS_ACCESS_KEY}
            - AWS_SECRET_KEY=${AWS_SECRET_KEY}
            - GRANT_SUDO=yes
          expose:
              - 8888
              - 4040
          ports:
              - 8888:8888
          volumes:
            - ./:/home/jovyan/work
            - ~/.ivy2:/home/jovyan/.ivy2
            - ~/.jupyter:/home/jovyan/.jupyter
          command: ['/usr/local/bin/start-notebook.sh']
          networks:
            - spark-clstr

    master:
        image: jupyter/all-spark-notebook
        hostname: master
        environment:
            MASTER: spark://master:7077
            SPARK_CONF_DIR: /conf
            SPARK_MASTER_HOST: master
        command: ['/usr/local/spark/bin/spark-class', 'org.apache.spark.deploy.master.Master', '-h', 'master']
        expose:
            - 7001
            - 7002 
            - 7003
            - 7004
            - 7005
            - 7006
            - 7077
            - 6066
        ports:
            - 6066:6066
            - 4040:4040
            - 7077:7077
            - 28080:8080
        networks:
            - spark-clstr
        volumes:
            - ./spark/conf/master:/conf
            - ./spark/data:/tmp/data
            - ~/.ivy2:/home/jovyan/.ivy2

    worker:
        image: jupyter/all-spark-notebook
        command: ['/usr/local/spark/bin/spark-class', 'org.apache.spark.deploy.worker.Worker', 'spark://master:7077']
        depends_on: 
            - master
        environment:
            SPARK_WORKER_DIR: '/tmp/data/worker'
            SPARK_WORKER_PORT: 8881
            SPARK_WORKER_WEBUI_PORT: 8081
        expose:
            - 7012
            - 7013
            - 7014
            - 7015
            - 7016
            - 8881
            - 8081
        volumes:
            - ./spark/conf/worker:/conf
            - ./spark/data:/tmp/data 
            - ~/.ivy2:/home/jovyan/.ivy2
        networks:
            - spark-clstr

    spark-ui-proxy:
        image: yanndegat/spark-ui-proxy
        command: master:8080 9999
        ports:
            - 8080:9999
        networks:
            - spark-clstr


networks:
    spark-clstr: