Copied in initial files from original COMP 101 demo

nigel.stanger / docker-analytics

Browse code Copied in initial files from original COMP 101 demo master spark3
1 parent 77d770d commit 17d1441da3e14b31aa5063b735c32d7b6f1b1185 Nigel Stanger authored on 10 May 2019

Patch

Showing 6 changed files

Ignore Space Show notes View Makefile 0 → 100644
all: spark pyspark spark: Dockerfile.spark docker build -t comp101/spark:latest -f $< . pyspark: Dockerfile.pyspark docker build -t comp101/pyspark:latest -f $< .

Ignore Space Show notes View docker-compose.yml 0 → 100644
version: "3.3" services: spark-master: image: comp101/spark:latest container_name: spark-master hostname: spark-master ports: - "8080:8080" - "7077:7077" networks: - spark-network environment: - "SPARK_LOCAL_IP=spark-master" - "SPARK_MASTER_PORT=7077" - "SPARK_MASTER_WEBUI_PORT=8080" - "PYSPARK_PYTHON=/usr/local/bin/python3" command: "start-master.sh" volumes: - ${HOME}/tmp/sparkdata:/mnt/sparkdata spark-worker: image: comp101/spark:latest depends_on: - spark-master ports: - 8080 networks: - spark-network environment: - "SPARK_MASTER=spark://spark-master:7077" - "SPARK_WORKER_WEBUI_PORT=8080" - "SPARKMONITOR_UI_PORT=8080" - "PYSPARK_PYTHON=/usr/local/bin/python3" command: "start-worker.sh" volumes: - ${HOME}/tmp/sparkdata:/mnt/sparkdata pyspark: image: comp101/pyspark:latest ports: - "8888:8888" networks: - spark-network environment: - "PYSPARK_PYTHON=/usr/local/bin/python3" working_dir: /mnt/sparkdata volumes: - ${HOME}/tmp/sparkdata:/mnt/sparkdata networks: spark-network: driver: bridge ipam: driver: default

Ignore Space Show notes View

docker-compose.yml 0 → 100644

version: "3.3"
services:
  spark-master:
    image: comp101/spark:latest
    container_name: spark-master
    hostname: spark-master
    ports:
      - "8080:8080"
      - "7077:7077"
    networks:
      - spark-network
    environment:
      - "SPARK_LOCAL_IP=spark-master"
      - "SPARK_MASTER_PORT=7077"
      - "SPARK_MASTER_WEBUI_PORT=8080"
      - "PYSPARK_PYTHON=/usr/local/bin/python3"
    command: "start-master.sh"
    volumes:
      - ${HOME}/tmp/sparkdata:/mnt/sparkdata
  spark-worker:
    image: comp101/spark:latest
    depends_on:
      - spark-master
    ports:
      - 8080
    networks:
      - spark-network
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"
      - "SPARK_WORKER_WEBUI_PORT=8080"
      - "SPARKMONITOR_UI_PORT=8080"
      - "PYSPARK_PYTHON=/usr/local/bin/python3"
    command: "start-worker.sh"
    volumes:
      - ${HOME}/tmp/sparkdata:/mnt/sparkdata
  pyspark:
    image: comp101/pyspark:latest
    ports:
      - "8888:8888"
    networks:
      - spark-network
    environment:
      - "PYSPARK_PYTHON=/usr/local/bin/python3"
    working_dir: /mnt/sparkdata
    volumes:
      - ${HOME}/tmp/sparkdata:/mnt/sparkdata
networks:
  spark-network:
    driver: bridge
    ipam:
      driver: default

Ignore Space Show notes View pyspark/Dockerfile 0 → 100644
FROM comp101/spark ARG NB_USER="pyspark" ARG NB_UID="1000" ARG NB_GID="1000" USER root RUN adduser -D -u $NB_UID $NB_USER RUN apk add --no-cache --virtual .build-deps \ build-base \ zeromq-dev \ python3-dev \ py3-zmq \ && pip install --upgrade pip \ # Tornado 6 breaks sparkmonitor && pip install tornado==5.1 pyspark jupyter sparkmonitor \ && apk del .build-deps ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.7-src.zip # ENV PYSPARK_DRIVER_PYTHON jupyter # ENV PYSPARK_DRIVER_PYTHON_OPTS "notebook --ip=0.0.0.0 --port=8888" # ENV SPARKMONITOR_UI_PORT 8080 USER $NB_UID RUN jupyter nbextension install sparkmonitor --py --user --symlink \ && jupyter nbextension enable sparkmonitor --py --user \ && jupyter serverextension enable --py --user sparkmonitor \ && ipython profile create \ && echo "c.InteractiveShellApp.extensions.append('sparkmonitor.kernelextension')" >> $(ipython profile locate default)/ipython_kernel_config.py COPY pyspark_demo.ipynb /home/$NB_USER/work/pyspark_demo.ipynb # WORKDIR /home/$NB_USER/work WORKDIR /mnt/sparkdata # CMD ["pyspark"] # PySpark doesn't seem to load the sparkmonitor extension, so let's just # go with Jupyter and manually create contexts and sessions as required. CMD ["jupyter", "notebook", "--ip=0.0.0.0", "--port=8888"] # debugging # CMD ["bash"]

Ignore Space Show notes View

pyspark/Dockerfile 0 → 100644

FROM comp101/spark

ARG NB_USER="pyspark"
ARG NB_UID="1000"
ARG NB_GID="1000"

USER root

RUN adduser -D -u $NB_UID $NB_USER

RUN apk add --no-cache --virtual .build-deps \
        build-base \
        zeromq-dev \
        python3-dev \
        py3-zmq \
    && pip install --upgrade pip \
    # Tornado 6 breaks sparkmonitor
    && pip install tornado==5.1 pyspark jupyter sparkmonitor \
    && apk del .build-deps

ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.7-src.zip
# ENV PYSPARK_DRIVER_PYTHON jupyter
# ENV PYSPARK_DRIVER_PYTHON_OPTS "notebook --ip=0.0.0.0 --port=8888"
# ENV SPARKMONITOR_UI_PORT 8080

USER $NB_UID

RUN jupyter nbextension install sparkmonitor --py --user --symlink \
    && jupyter nbextension enable sparkmonitor --py --user \
    && jupyter serverextension enable --py --user sparkmonitor \
    && ipython profile create \
    && echo "c.InteractiveShellApp.extensions.append('sparkmonitor.kernelextension')" >> $(ipython profile locate default)/ipython_kernel_config.py

COPY pyspark_demo.ipynb /home/$NB_USER/work/pyspark_demo.ipynb

# WORKDIR /home/$NB_USER/work
WORKDIR /mnt/sparkdata

# CMD ["pyspark"]
# PySpark doesn't seem to load the sparkmonitor extension, so let's just
# go with Jupyter and manually create contexts and sessions as required.
CMD ["jupyter", "notebook", "--ip=0.0.0.0", "--port=8888"]

# debugging
# CMD ["bash"]

Ignore Space Show notes View spark/Dockerfile 0 → 100644
FROM python:3.6-alpine3.9 COPY spark-2.4.2-bin-hadoop2.7.tgz /usr/local/spark-2.4.2-bin-hadoop2.7.tgz RUN apk add --no-cache \ bash \ openjdk8 \ zeromq \ tini \ && apk add --no-cache --virtual .fetch-deps \ tar \ && cd /usr/local \ && tar -xzf spark-2.4.2-bin-hadoop2.7.tgz \ && mv spark-2.4.2-bin-hadoop2.7 spark \ && rm spark-2.4.2-bin-hadoop2.7.tgz \ && apk del .fetch-deps COPY start-master.sh /usr/local/bin/start-master.sh COPY start-worker.sh /usr/local/bin/start-worker.sh ENV SPARK_HOME /usr/local/spark # Spark doesn't seem to respond directly to SIGTERM as the exit status is # for SIGKILL (137), after a pause. Presumably docker-compose down times out. # Using tini gives immediate exit with status 143 (SIGTERM). ENTRYPOINT ["/sbin/tini", "--"] CMD ["bash"]

Ignore Space Show notes View

spark/Dockerfile 0 → 100644

Ignore Space Show notes View spark/start-master.sh 0 → 100755
#!/bin/sh /usr/local/spark/bin/spark-class org.apache.spark.deploy.master.Master \ --ip $SPARK_LOCAL_IP \ --port $SPARK_MASTER_PORT \ --webui-port $SPARK_MASTER_WEBUI_PORT

Ignore Space Show notes View spark/start-worker.sh 0 → 100755
#!/bin/sh /usr/local/spark/bin/spark-class org.apache.spark.deploy.worker.Worker \ --webui-port $SPARK_WORKER_WEBUI_PORT \ $SPARK_MASTER

Show line notes below