Upgrade Spark to 3.3.1, reduce image file size

Author: luisbelloch

infra/docker/Dockerfile

@@ -1,22 +1,35 @@
-FROM openjdk:11-jdk-slim
+FROM amazoncorretto:11 as corretto-jdk
+RUN $JAVA_HOME/bin/jlink \
+    --verbose \
+    --add-modules ALL-MODULE-PATH \
+    --strip-debug \
+    --no-man-pages \
+    --no-header-files \
+    --compress=2 \
+    --output /opt/jre
+
+FROM debian:stable-slim
 LABEL maintainer="Luis Belloch <docker@luisbelloch.es>"
+ENV JAVA_HOME=/opt/jre
+ENV PATH="${JAVA_HOME}/bin:${PATH}"
+COPY --from=corretto-jdk /opt/jre $JAVA_HOME
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends python3-software-properties python3-numpy curl && \
+    apt-get install -y --no-install-recommends ca-certificates procps python3-software-properties python3-numpy curl && \
     rm -rf /var/lib/apt/lists/*
 
-ARG SPARK_VERSION=3.1.2
+ARG SPARK_VERSION=3.3.1
 ENV SPARK_HOME=/opt/spark
-RUN mkdir -p /opt/spark && curl -s https://downloads.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.2.tgz | tar -xz -C "${SPARK_HOME}" --strip-components=1
-ENV PATH=$SPARK_HOME/bin:$SPARK_HOME/sbin:$PATH
+RUN mkdir -p /opt/spark && curl -s https://downloads.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz | tar -xz -C "${SPARK_HOME}" --strip-components=1
+ENV PATH="${SPARK_HOME}/bin:${SPARK_HOME}/sbin:${PATH}"
 
-RUN cp "${SPARK_HOME}/conf/log4j.properties.template" "${SPARK_HOME}/conf/log4j.properties" && \
-    sed -ibak 's/rootCategory=INFO/rootCategory=ERROR/g' "${SPARK_HOME}/conf/log4j.properties"
+RUN cp "${SPARK_HOME}/conf/log4j2.properties.template" "${SPARK_HOME}/conf/log4j2.properties" && \
+    sed -ibak 's/rootLogger.level = info/rootLogger.level = error/g' "${SPARK_HOME}/conf/log4j2.properties"
 
 ENV SPARK_NO_DAEMONIZE=true
-ENV PYSPARK_PYTHON=python3
+ENV PYSPARK_PYTHON=/usr/bin/python3
+ENV PYSPARK_DRIVER_PYTHON=/usr/bin/python3
 EXPOSE 4040 7077 8080
 
 CMD ["pyspark"]
-

infra/docker/Makefile

@@ -1,5 +1,5 @@
-SPARK_VERSION:=3.1.2
-COURSE_VERSION:=2021.10
+SPARK_VERSION:=3.3.1
+COURSE_VERSION:=2022.12
 IMAGE_NAME:=luisbelloch/spark
 
 .PHONY: help

local_setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 set -euo pipefail
-SPARK_URL=${SPARK_URL:-https://downloads.apache.org/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz}
+SPARK_URL=${SPARK_URL:-https://downloads.apache.org/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz}
 SPARK_PKG=${SPARK_URL##*/}
 SPARK_HOME=${SPARK_HOME:-$(pwd)/.spark}
 
@@ -27,8 +27,8 @@ mkdir -p "${SPARK_HOME}"
 curl -s "${SPARK_URL}" | tar -xz -C "${SPARK_HOME}" --strip-components=1
 
 stderr "${c_step}[2] Reducing log level${c_norm}"
-cp "${SPARK_HOME}"/conf/log4j.properties.template "${SPARK_HOME}"/conf/log4j.properties
-sed -ibak 's/rootCategory=INFO/rootCategory=ERROR/g' "${SPARK_HOME}"/conf/log4j.properties
+cp "${SPARK_HOME}"/conf/log4j2.properties.template "${SPARK_HOME}"/conf/log4j2.properties
+sed -ibak 's/rootLogger.level = info/rootLogger.level = error/g' "${SPARK_HOME}/conf/log4j2.properties"
 
 stderr "${c_step}[3] Testing setup${c_norm}"
 echo 'sc.parallelize(1 to 100).count()' | "${SPARK_HOME}"/bin/spark-shell

playbook.yml

@@ -2,8 +2,8 @@
 - hosts: all
   vars:
     spark_home: /opt/spark
-    spark_pkg_name: spark-3.1.2-bin-hadoop3.2
-    spark_pkg_url: https://downloads.apache.org/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz
+    spark_pkg_name: spark-3.3.1-bin-hadoop3
+    spark_pkg_url: https://downloads.apache.org/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz
 
   tasks:
     - name: Update all packages to the latest version

spark/spark

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -euo pipefail
 
+# Alternative: SPARK_DOCKER_IMAGE=apache/spark-py ./spark-submit script.py
+readonly SPARK_DOCKER_IMAGE=${SPARK_DOCKER_IMAGE:-luisbelloch/spark}
 readonly SPARK_SUBMIT=/opt/spark/bin/spark-submit
 readonly DATA_DIR=/tmp/bigdataupv/data
 readonly WORK_DIR=/tmp/bigdataupv/scripts
@@ -16,7 +18,7 @@ abs_path() {
 }
 
 get_data_volume() {
-    # Probe for source folder first, if it doesn't 
+    # Probe for source folder first, if it doesn't
     # exists then it'll try with current folder
     if [[ -d "${0}" ]]; then
         echo "-v $(abs_path $0):"${DATA_DIR}""
@@ -37,5 +39,5 @@ docker run --rm -ti \
     -w "${WORK_DIR}" \
     -v "${source_folder}":"${WORK_DIR}" \
     $data_volume \
-    luisbelloch/spark "${SPARK_SUBMIT}" "${WORK_DIR}"/$1 ${@:2}
+    ${SPARK_DOCKER_IMAGE} "${SPARK_SUBMIT}" "${WORK_DIR}"/$1 ${@:2}