Corrected working directory for pyspark

nigel.stanger / docker-analytics

Browse code Corrected working directory for pyspark master spark3
1 parent d8b7408 commit 55a239b357b7416a055e6f6b9179b56991abe356 Nigel Stanger authored on 13 May 2019

Patch

Showing 1 changed file

Ignore Space Show notes View docker-compose.yml
version: "3.3" services: spark-master: image: analytics/spark:latest container_name: spark-master hostname: spark-master ports: - "8080:8080" - "7077:7077" networks: - spark-network environment: - "SPARK_LOCAL_IP=spark-master" command: "start-master.sh" volumes: - ${HOME}/tmp/sparkdata:/mnt/sparkdata spark-worker: image: analytics/spark:latest depends_on: - spark-master ports: - 8080 networks: - spark-network environment: - "SPARK_MASTER=spark://spark-master:7077" - "SPARK_WORKER_WEBUI_PORT=8080" - "SPARKMONITOR_UI_PORT=8080" command: "start-worker.sh" volumes: - ${HOME}/tmp/sparkdata:/mnt/sparkdata pyspark: image: analytics/pyspark:latest ports: - "8888:8888" networks: - spark-network environment: - "PYSPARK_PYTHON=/usr/local/bin/python3" working_dir: /home/pyspark/work volumes: - ${HOME}/tmp/sparkdata:/mnt/sparkdata networks: spark-network: driver: bridge ipam: driver: default version: "3.3" services: spark-master: image: analytics/spark:latest container_name: spark-master hostname: spark-master ports: - "8080:8080" - "7077:7077" networks: - spark-network environment: - "SPARK_LOCAL_IP=spark-master" command: "start-master.sh" volumes: - ${HOME}/tmp/sparkdata:/mnt/sparkdata spark-worker: image: analytics/spark:latest depends_on: - spark-master ports: - 8080 networks: - spark-network environment: - "SPARK_MASTER=spark://spark-master:7077" - "SPARK_WORKER_WEBUI_PORT=8080" - "SPARKMONITOR_UI_PORT=8080" command: "start-worker.sh" volumes: - ${HOME}/tmp/sparkdata:/mnt/sparkdata pyspark: image: analytics/pyspark:latest ports: - "8888:8888" networks: - spark-network environment: - "PYSPARK_PYTHON=/usr/local/bin/python3" working_dir: /mnt/sparkdata volumes: - ${HOME}/tmp/sparkdata:/mnt/sparkdata networks: spark-network: driver: bridge ipam: driver: default

Ignore Space Show notes View

docker-compose.yml

version: "3.3"
services:
  spark-master:
    image: analytics/spark:latest
    container_name: spark-master
    hostname: spark-master
    ports:
      - "8080:8080"
      - "7077:7077"
    networks:
      - spark-network
    environment:
      - "SPARK_LOCAL_IP=spark-master"
    command: "start-master.sh"
    volumes:
      - ${HOME}/tmp/sparkdata:/mnt/sparkdata
  spark-worker:
    image: analytics/spark:latest
    depends_on:
      - spark-master
    ports:
      - 8080
    networks:
      - spark-network
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"
      - "SPARK_WORKER_WEBUI_PORT=8080"
      - "SPARKMONITOR_UI_PORT=8080"
    command: "start-worker.sh"
    volumes:
      - ${HOME}/tmp/sparkdata:/mnt/sparkdata
  pyspark:
    image: analytics/pyspark:latest
    ports:
      - "8888:8888"
    networks:
      - spark-network
    environment:
      - "PYSPARK_PYTHON=/usr/local/bin/python3"
    working_dir: /home/pyspark/work
    volumes:
      - ${HOME}/tmp/sparkdata:/mnt/sparkdata
networks:
  spark-network:
    driver: bridge
    ipam:
      driver: default

version: "3.3"
services:
  spark-master:
    image: analytics/spark:latest
    container_name: spark-master
    hostname: spark-master
    ports:
      - "8080:8080"
      - "7077:7077"
    networks:
      - spark-network
    environment:
      - "SPARK_LOCAL_IP=spark-master"
    command: "start-master.sh"
    volumes:
      - ${HOME}/tmp/sparkdata:/mnt/sparkdata
  spark-worker:
    image: analytics/spark:latest
    depends_on:
      - spark-master
    ports:
      - 8080
    networks:
      - spark-network
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"
      - "SPARK_WORKER_WEBUI_PORT=8080"
      - "SPARKMONITOR_UI_PORT=8080"
    command: "start-worker.sh"
    volumes:
      - ${HOME}/tmp/sparkdata:/mnt/sparkdata
  pyspark:
    image: analytics/pyspark:latest
    ports:
      - "8888:8888"
    networks:
      - spark-network
    environment:
      - "PYSPARK_PYTHON=/usr/local/bin/python3"
    working_dir: /mnt/sparkdata
    volumes:
      - ${HOME}/tmp/sparkdata:/mnt/sparkdata
networks:
  spark-network:
    driver: bridge
    ipam:
      driver: default

Show line notes below