Got PySpark Jupyter kernel working

nigel.stanger / docker-analytics

Browse code Got PySpark Jupyter kernel working • Installed kernel to user directory. • Added IPython initialisation script. • Tweaked environment variables. master spark3
1 parent 37410f3 commit b0d381077ae975f3932f54c84a0d4e3106d71561 Nigel Stanger authored on 13 May 2019

Patch

Showing 6 changed files

Ignore Space Show notes View Makefile
all: spark pyspark spark: spark/Dockerfile spark/start-master.sh spark/start-worker.sh docker build -t analytics/spark:latest -f $< $@ pyspark: pyspark/Dockerfile pyspark/kernel.json pyspark/pyspark-kernel.sh docker build -t analytics/pyspark:latest -f $< $@ kafka: kafka/Dockerfile kafka/start-kafka.sh docker build -t analytics/kafka:latest -f $< $@ all: spark pyspark spark: spark/Dockerfile docker build -t analytics/spark:latest -f $< $@ pyspark: pyspark/Dockerfile docker build -t analytics/pyspark:latest -f $< $@ kafka: kafka/Dockerfile docker build -t analytics/kafka:latest -f $< $@

Ignore Space Show notes View

Makefile

Ignore Space Show notes View docker-compose.yml
version: "3.3" services: spark-master: image: analytics/spark:latest container_name: spark-master hostname: spark-master ports: - "8080:8080" - "7077:7077" networks: - spark-network environment: - "SPARK_LOCAL_IP=spark-master" - "SPARK_MASTER=spark://spark-master:7077" command: "start-master.sh" volumes: - ${HOME}/tmp/sparkdata:/mnt/sparkdata spark-worker: image: analytics/spark:latest depends_on: - spark-master ports: - 8080 networks: - spark-network environment: - "SPARK_MASTER=spark://spark-master:7077" - "SPARK_WORKER_WEBUI_PORT=8080" - "SPARKMONITOR_UI_PORT=8080" command: "start-worker.sh" volumes: - ${HOME}/tmp/sparkdata:/mnt/sparkdata pyspark: image: analytics/pyspark:latest depends_on: - spark-master ports: - "8888:8888" networks: - spark-network environment: - "SPARK_MASTER=spark://spark-master:7077" - "PYSPARK_SUBMIT_ARGS=--master spark://spark-master:7077 pyspark-shell" working_dir: /home/pyspark/work volumes: - ${HOME}/tmp/sparkdata:/mnt/sparkdata kafka: image: analytics/kafka:latest hostname: kafka ports: - 9092 networks: - spark-network volumes: - ${HOME}/tmp/sparkdata:/mnt/sparkdata networks: spark-network: driver: bridge ipam: driver: default version: "3.3" services: spark-master: image: analytics/spark:latest container_name: spark-master hostname: spark-master ports: - "8080:8080" - "7077:7077" networks: - spark-network environment: - "SPARK_LOCAL_IP=spark-master" command: "start-master.sh" volumes: - ${HOME}/tmp/sparkdata:/mnt/sparkdata spark-worker: image: analytics/spark:latest depends_on: - spark-master ports: - 8080 networks: - spark-network environment: - "SPARK_MASTER=spark://spark-master:7077" - "SPARK_WORKER_WEBUI_PORT=8080" - "SPARKMONITOR_UI_PORT=8080" command: "start-worker.sh" volumes: - ${HOME}/tmp/sparkdata:/mnt/sparkdata pyspark: image: analytics/pyspark:latest ports: - "8888:8888" networks: - spark-network environment: - "PYSPARK_PYTHON=/usr/local/bin/python3" working_dir: /home/pyspark/work volumes: - ${HOME}/tmp/sparkdata:/mnt/sparkdata kafka: image: analytics/kafka:latest hostname: kafka ports: - 9092 networks: - spark-network volumes: - ${HOME}/tmp/sparkdata:/mnt/sparkdata networks: spark-network: driver: bridge ipam: driver: default

Ignore Space Show notes View

docker-compose.yml

version: "3.3"
services:
  spark-master:
    image: analytics/spark:latest
    container_name: spark-master
    hostname: spark-master
    ports:
      - "8080:8080"
      - "7077:7077"
    networks:
      - spark-network
    environment:
      - "SPARK_LOCAL_IP=spark-master"
      - "SPARK_MASTER=spark://spark-master:7077"
    command: "start-master.sh"
    volumes:
      - ${HOME}/tmp/sparkdata:/mnt/sparkdata
  spark-worker:
    image: analytics/spark:latest
    depends_on:
      - spark-master
    ports:
      - 8080
    networks:
      - spark-network
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"
      - "SPARK_WORKER_WEBUI_PORT=8080"
      - "SPARKMONITOR_UI_PORT=8080"
    command: "start-worker.sh"
    volumes:
      - ${HOME}/tmp/sparkdata:/mnt/sparkdata
  pyspark:
    image: analytics/pyspark:latest
    depends_on:
      - spark-master
    ports:
      - "8888:8888"
    networks:
      - spark-network
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"
      - "PYSPARK_SUBMIT_ARGS=--master spark://spark-master:7077 pyspark-shell"
    working_dir: /home/pyspark/work
    volumes:
      - ${HOME}/tmp/sparkdata:/mnt/sparkdata
  kafka:
    image: analytics/kafka:latest
    hostname: kafka
    ports:
      - 9092
    networks:
      - spark-network
    volumes:
      - ${HOME}/tmp/sparkdata:/mnt/sparkdata
networks:
  spark-network:
    driver: bridge
    ipam:
      driver: default

version: "3.3"
services:
  spark-master:
    image: analytics/spark:latest
    container_name: spark-master
    hostname: spark-master
    ports:
      - "8080:8080"
      - "7077:7077"
    networks:
      - spark-network
    environment:
      - "SPARK_LOCAL_IP=spark-master"
    command: "start-master.sh"
    volumes:
      - ${HOME}/tmp/sparkdata:/mnt/sparkdata
  spark-worker:
    image: analytics/spark:latest
    depends_on:
      - spark-master
    ports:
      - 8080
    networks:
      - spark-network
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"
      - "SPARK_WORKER_WEBUI_PORT=8080"
      - "SPARKMONITOR_UI_PORT=8080"
    command: "start-worker.sh"
    volumes:
      - ${HOME}/tmp/sparkdata:/mnt/sparkdata
  pyspark:
    image: analytics/pyspark:latest
    ports:
      - "8888:8888"
    networks:
      - spark-network
    environment:
      - "PYSPARK_PYTHON=/usr/local/bin/python3"
    working_dir: /home/pyspark/work
    volumes:
      - ${HOME}/tmp/sparkdata:/mnt/sparkdata
  kafka:
    image: analytics/kafka:latest
    hostname: kafka
    ports:
      - 9092
    networks:
      - spark-network
    volumes:
      - ${HOME}/tmp/sparkdata:/mnt/sparkdata
networks:
  spark-network:
    driver: bridge
    ipam:
      driver: default

Ignore Space Show notes View pyspark/Dockerfile
# Don't use the official jupyter/pyspark-notebook image because it's massive! FROM analytics/spark ARG NB_USER="pyspark" ARG NB_UID="1000" ARG NB_GID="1000" USER root RUN adduser -D -u $NB_UID $NB_USER RUN apk add --no-cache --virtual .build-deps \ build-base \ zeromq-dev \ python3-dev \ py3-zmq RUN pip install --upgrade \ pip \ && pip install \ jupyter \ kafka-python \ pyspark \ sparkmonitor RUN apk del .build-deps ENV PYSPARK_PYTHON="/usr/local/bin/python3" ENV KERNEL_NAME="PySpark" \ KERNELS_TEMPLATE_PATH="/tmp" \ KERNELS_DIR_PATH="/home/$NB_USER/.local/share/jupyter/kernels" \ PYTHONPATH="$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.7-src.zip" \ PYSPARK_DRIVER_PYTHON="$PYSPARK_PYTHON" \ PYSPARK_SUBMIT_ARGS="--master $SPARK_MASTER pyspark-shell" \ PYTHONSTARTUP="$SPARK_HOME/python/pyspark/shell.py" # RUN mkdir -p $KERNELS_DIR_PATH/$KERNEL_NAME \ # && pip install pyhocon \ # && cat $KERNELS_TEMPLATE_PATH/kernel.json \| pyhocon -f json >> $KERNELS_DIR_PATH/$KERNEL_NAME/kernel.json \ # && pip uninstall -y pyhocon pyparsing # ENV SPARKMONITOR_UI_PORT 8080 USER $NB_UID RUN ipython kernel install --user --name $KERNEL_NAME COPY kernel.json pyspark-kernel.sh $KERNELS_DIR_PATH/$KERNEL_NAME/ RUN jupyter nbextension install sparkmonitor --py --user --symlink \ && jupyter nbextension enable sparkmonitor --py --user \ && jupyter serverextension enable --py --user sparkmonitor \ && ipython profile create \ && echo "c.InteractiveShellApp.extensions.append('sparkmonitor.kernelextension')" >> $(ipython profile locate default)/ipython_kernel_config.py RUN mkdir -p /home/$NB_USER/work WORKDIR /home/$NB_USER/work # CMD ["pyspark"] # PySpark doesn't seem to load the sparkmonitor extension, so let's just # go with Jupyter and manually create contexts and sessions as required. CMD ["jupyter", "notebook", "--ip=0.0.0.0", "--port=8888", "--no-browser"] # debugging # CMD ["bash"] # Don't use the official jupyter/pyspark-notebook image because it's massive! FROM analytics/spark ARG NB_USER="pyspark" ARG NB_UID="1000" ARG NB_GID="1000" USER root RUN adduser -D -u $NB_UID $NB_USER RUN apk add --no-cache --virtual .build-deps \ build-base \ zeromq-dev \ python3-dev \ py3-zmq RUN pip install --upgrade \ pip \ && pip install \ pyspark \ jupyter \ kafka-python \ sparkmonitor RUN apk del .build-deps ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.7-src.zip # ENV PYSPARK_DRIVER_PYTHON jupyter # ENV PYSPARK_DRIVER_PYTHON_OPTS "notebook --ip=0.0.0.0 --port=8888" # ENV SPARKMONITOR_UI_PORT 8080 USER $NB_UID RUN jupyter nbextension install sparkmonitor --py --user --symlink \ && jupyter nbextension enable sparkmonitor --py --user \ && jupyter serverextension enable --py --user sparkmonitor \ && ipython profile create \ && echo "c.InteractiveShellApp.extensions.append('sparkmonitor.kernelextension')" >> $(ipython profile locate default)/ipython_kernel_config.py RUN mkdir -p /home/$NB_USER/work WORKDIR /home/$NB_USER/work # CMD ["pyspark"] # PySpark doesn't seem to load the sparkmonitor extension, so let's just # go with Jupyter and manually create contexts and sessions as required. CMD ["jupyter", "notebook", "--ip=0.0.0.0", "--port=8888", "--no-browser"] # debugging # CMD ["bash"]

Ignore Space Show notes View

pyspark/Dockerfile

# Don't use the official jupyter/pyspark-notebook image because it's massive!
FROM analytics/spark

ARG NB_USER="pyspark"
ARG NB_UID="1000"
ARG NB_GID="1000"

USER root

RUN adduser -D -u $NB_UID $NB_USER

RUN apk add --no-cache --virtual .build-deps \
        build-base \
        zeromq-dev \
        python3-dev \
        py3-zmq

RUN pip install --upgrade \
        pip \
    && pip install \
        jupyter \
        kafka-python \
        pyspark \
        sparkmonitor

RUN apk del .build-deps

ENV PYSPARK_PYTHON="/usr/local/bin/python3"

ENV KERNEL_NAME="PySpark" \
    KERNELS_TEMPLATE_PATH="/tmp" \
    KERNELS_DIR_PATH="/home/$NB_USER/.local/share/jupyter/kernels" \
    PYTHONPATH="$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.7-src.zip" \
    PYSPARK_DRIVER_PYTHON="$PYSPARK_PYTHON" \
    PYSPARK_SUBMIT_ARGS="--master $SPARK_MASTER pyspark-shell" \
    PYTHONSTARTUP="$SPARK_HOME/python/pyspark/shell.py"

# RUN mkdir -p $KERNELS_DIR_PATH/$KERNEL_NAME \
#     && pip install pyhocon \
#     && cat $KERNELS_TEMPLATE_PATH/kernel.json | pyhocon -f json >> $KERNELS_DIR_PATH/$KERNEL_NAME/kernel.json \
#     && pip uninstall -y pyhocon pyparsing

# ENV SPARKMONITOR_UI_PORT 8080

USER $NB_UID

RUN ipython kernel install --user --name $KERNEL_NAME
COPY kernel.json pyspark-kernel.sh $KERNELS_DIR_PATH/$KERNEL_NAME/

RUN jupyter nbextension install sparkmonitor --py --user --symlink \
    && jupyter nbextension enable sparkmonitor --py --user \
    && jupyter serverextension enable --py --user sparkmonitor \
    && ipython profile create \
    && echo "c.InteractiveShellApp.extensions.append('sparkmonitor.kernelextension')" >> $(ipython profile locate default)/ipython_kernel_config.py

RUN mkdir -p /home/$NB_USER/work
WORKDIR /home/$NB_USER/work

# CMD ["pyspark"]
# PySpark doesn't seem to load the sparkmonitor extension, so let's just
# go with Jupyter and manually create contexts and sessions as required.
CMD ["jupyter", "notebook", "--ip=0.0.0.0", "--port=8888", "--no-browser"]

# debugging
# CMD ["bash"]

# Don't use the official jupyter/pyspark-notebook image because it's massive!
FROM analytics/spark

ARG NB_USER="pyspark"
ARG NB_UID="1000"
ARG NB_GID="1000"

USER root

RUN adduser -D -u $NB_UID $NB_USER

RUN apk add --no-cache --virtual .build-deps \
        build-base \
        zeromq-dev \
        python3-dev \
        py3-zmq

RUN pip install --upgrade \
        pip \
    && pip install \
        pyspark \
        jupyter \
        kafka-python \
        sparkmonitor

RUN apk del .build-deps

ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.7-src.zip
# ENV PYSPARK_DRIVER_PYTHON jupyter
# ENV PYSPARK_DRIVER_PYTHON_OPTS "notebook --ip=0.0.0.0 --port=8888"
# ENV SPARKMONITOR_UI_PORT 8080

USER $NB_UID

RUN mkdir -p /home/$NB_USER/work
WORKDIR /home/$NB_USER/work

# debugging
# CMD ["bash"]

Ignore Space Show notes View pyspark/kernel.json 0 → 100644
{ "display_name": "PySpark", "language": "python", "argv": [ "~/.localshare/jupyter/kernels/PySpark/pyspark-kernel.sh", "-f", "{connection_file}" ] }

Ignore Space Show notes View pyspark/pyspark-kernel.sh 0 → 100755
#!/bin/sh # module lsst exec /usr/bin/python3 -m ipykernel $@

Ignore Space Show notes View spark/Dockerfile
FROM python:3.6-alpine ENV SPARK_VERSION="2.4.3" \ HADOOP_VERSION="2.7" \ SPARK_INSTALL="/usr/local" RUN apk add --no-cache \ bash \ openjdk8 \ tini \ zeromq RUN apk add --no-cache --virtual .fetch-deps \ curl \ tar RUN curl -s https://www-us.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION.tgz \| tar -xz -C $SPARK_INSTALL && \ cd $SPARK_INSTALL && ln -s spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION spark RUN apk del .fetch-deps COPY start-master.sh /usr/local/bin/start-master.sh COPY start-worker.sh /usr/local/bin/start-worker.sh ENV SPARK_HOME="$SPARK_INSTALL/spark" \ SPARK_LOCAL_IP="localhost" \ SPARK_MASTER_PORT="7077" \ SPARK_MASTER_WEBUI_PORT="8080" ENV SPARK_MASTER="spark://$SPARK_LOCAL_IP:$SPARK_MASTER_PORT" # Spark doesn't seem to respond directly to SIGTERM as the exit status is # for SIGKILL (137), after a pause. Presumably docker-compose down times out. # Using tini gives immediate exit with status 143 (SIGTERM). ENTRYPOINT ["/sbin/tini", "--"] CMD ["/usr/local/bin/start-master.sh"] FROM python:3.6-alpine ENV SPARK_VERSION="2.4.3" \ HADOOP_VERSION="2.7" \ SPARK_INSTALL="/usr/local" RUN apk add --no-cache \ bash \ openjdk8 \ tini \ zeromq RUN apk add --no-cache --virtual .fetch-deps \ curl \ tar RUN curl -s https://www-us.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION.tgz \| tar -xz -C $SPARK_INSTALL && \ cd $SPARK_INSTALL && ln -s spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION spark RUN apk del .fetch-deps COPY start-master.sh /usr/local/bin/start-master.sh COPY start-worker.sh /usr/local/bin/start-worker.sh ENV SPARK_HOME="$SPARK_INSTALL/spark" \ SPARK_LOCAL_IP="localhost" \ SPARK_MASTER_PORT="7077" \ SPARK_MASTER="spark://$SPARK_LOCAL_IP:$SPARK_MASTER_PORT" \ SPARK_MASTER_WEBUI_PORT="8080" \ PYSPARK_PYTHON="/usr/local/bin/python3" # Spark doesn't seem to respond directly to SIGTERM as the exit status is # for SIGKILL (137), after a pause. Presumably docker-compose down times out. # Using tini gives immediate exit with status 143 (SIGTERM). ENTRYPOINT ["/sbin/tini", "--"] CMD ["/usr/local/bin/start-master.sh"]

Ignore Space Show notes View

spark/Dockerfile

FROM python:3.6-alpine

ENV SPARK_VERSION="2.4.3" \
    HADOOP_VERSION="2.7" \
    SPARK_INSTALL="/usr/local"

RUN apk add --no-cache \
        bash \
        openjdk8 \
        tini \
        zeromq

RUN apk add --no-cache --virtual .fetch-deps \
        curl \
        tar

RUN curl -s https://www-us.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION.tgz | tar -xz -C $SPARK_INSTALL && \
    cd $SPARK_INSTALL && ln -s spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION spark

RUN apk del .fetch-deps

COPY start-master.sh /usr/local/bin/start-master.sh
COPY start-worker.sh /usr/local/bin/start-worker.sh

ENV SPARK_HOME="$SPARK_INSTALL/spark" \
    SPARK_LOCAL_IP="localhost" \
    SPARK_MASTER_PORT="7077" \
    SPARK_MASTER_WEBUI_PORT="8080"

ENV SPARK_MASTER="spark://$SPARK_LOCAL_IP:$SPARK_MASTER_PORT"

# Spark doesn't seem to respond directly to SIGTERM as the exit status is
# for SIGKILL (137), after a pause. Presumably docker-compose down times out.
# Using tini gives immediate exit with status 143 (SIGTERM).
ENTRYPOINT ["/sbin/tini", "--"]

CMD ["/usr/local/bin/start-master.sh"]

FROM python:3.6-alpine

ENV SPARK_VERSION="2.4.3" \
    HADOOP_VERSION="2.7" \
    SPARK_INSTALL="/usr/local"

RUN apk add --no-cache \
        bash \
        openjdk8 \
        tini \
        zeromq

RUN apk add --no-cache --virtual .fetch-deps \
        curl \
        tar

RUN apk del .fetch-deps

COPY start-master.sh /usr/local/bin/start-master.sh
COPY start-worker.sh /usr/local/bin/start-worker.sh

ENV SPARK_HOME="$SPARK_INSTALL/spark" \
    SPARK_LOCAL_IP="localhost" \
    SPARK_MASTER_PORT="7077" \
    SPARK_MASTER="spark://$SPARK_LOCAL_IP:$SPARK_MASTER_PORT" \
    SPARK_MASTER_WEBUI_PORT="8080" \
    PYSPARK_PYTHON="/usr/local/bin/python3"

CMD ["/usr/local/bin/start-master.sh"]

Show line notes below