Spark without minio

technologies/job/spark/metadata.yaml

@@ -81,7 +81,7 @@ contexts:
             dockerInfo:
               image: "saagie/spark"
               baseTag: "2.4-py-3.7"
-              version: "2.4-py-3.7-1.170.0"
+              version: "2.4-py-3.7-1.194.0_sparkWithoutMinio"
   - id: "3.0"
     label: "3.0"
     available: true
@@ -150,7 +150,7 @@ contexts:
             dockerInfo:
               image: "saagie/spark"
               baseTag: "3.0-py-3.7"
-              version: "3.0-py-3.7-1.139.0"
+              version: "3.0-py-3.7-1.194.0_sparkWithoutMinio"
   - id: "3.1"
     label: "3.1"
     available: true
@@ -221,7 +221,7 @@ contexts:
             dockerInfo:
               image: "saagie/spark"
               baseTag: "3.1-py-3.7"
-              version: "3.1-py-3.7-1.139.0"
+              version: "3.1-py-3.7-1.194.0_sparkWithoutMinio"
           - id: "3.8"
             label: "3.8"
             available: true
@@ -230,7 +230,7 @@ contexts:
             dockerInfo:
               image: "saagie/spark"
               baseTag: "3.1-py-3.8"
-              version: "3.1-py-3.8-1.139.0"
+              version: "3.1-py-3.8-1.194.0_sparkWithoutMinio"
           - id: "3.9"
             label: "3.9"
             available: true
@@ -239,7 +239,7 @@ contexts:
             dockerInfo:
               image: "saagie/spark"
               baseTag: "3.1-py-3.9"
-              version: "3.1-py-3.9-1.139.0"
+              version: "3.1-py-3.9-1.194.0_sparkWithoutMinio"
   - id: "3.5"
     label: "3.5"
     available: true
@@ -314,7 +314,7 @@ contexts:
             dockerInfo:
               image: "saagie/spark"
               baseTag: "3.5-py-3.12"
-              version: "3.5-py-3.12-1.190.0"
+              version: "3.5-py-3.12-1.194.0_sparkWithoutMinio"
   - id: 3.1-aws
     label: 3.1 AWS
     available: true
@@ -385,7 +385,7 @@ contexts:
             dockerInfo:
               image: "saagie/spark"
               baseTag: "3.1-aws-py-3.7"
-              version: "3.1-aws-py-3.7-1.139.0"
+              version: "3.1-aws-py-3.7-1.194.0_sparkWithoutMinio"
           - id: "3.8"
             label: "3.8"
             available: true
@@ -394,7 +394,7 @@ contexts:
             dockerInfo:
               image: "saagie/spark"
               baseTag: "3.1-aws-py-3.8"
-              version: "3.1-aws-py-3.8-1.139.0"
+              version: "3.1-aws-py-3.8-1.194.0_sparkWithoutMinio"
           - id: "3.9"
             label: "3.9"
             available: true
@@ -403,4 +403,4 @@ contexts:
             dockerInfo:
               image: "saagie/spark"
               baseTag: "3.1-aws-py-3.9"
-              version: "3.1-aws-py-3.9-1.139.0"
+              version: "3.1-aws-py-3.9-1.194.0_sparkWithoutMinio"

technologies/job/spark/spark-2.4/innerContexts/python/entrypoint.sh

@@ -44,40 +44,19 @@ if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
   SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
 fi
 
+# BEGIN SAAGIE SPECIFIC CODE
 cd /sandbox
- # parse content and if pyfiles extract minio url and inject it
-if [ -f main_script ] && grep -q "\--py-files" main_script;
+if [ -f *.zip ]
 then
-  PYSPARK_FILES="`grep -Po '.*--py-files=\K[^ ]+' main_script`"
-fi;
-
-if [ -n "$PYSPARK_FILES" ]; then
-    PYTHONPATH="$PYTHONPATH:$PYSPARK_FILES"
-    #Copy and unzip pyfiles
-    if [[ $PYSPARK_FILES == *[,]* ]];then
-      echo "PYSPARK_FILES contains comma"
-      pyfiles=$(echo $PYSPARK_FILES | tr "," "\n")
-
-      for file in $pyfiles
-      do
-          echo ">>> [$file]"
-          wget -nv $file
-      done
-    else
-      echo ">>> [$PYSPARK_FILES]"
-      wget -nv $PYSPARK_FILES
-    fi
-    if [ -f *.zip ]
-    then
-      unzip -q *.zip
-    fi
-    if [ -f "requirements.txt" ]
-    then
-      pip install -r requirements.txt
-    fi
-    rm -Rf /opt/spark/work-dir
-    ln -s /sandbox/ /opt/spark/work-dir
+  unzip -q *.zip
+fi
+if [ -f "requirements.txt" ]
+then
+  pip install -r requirements.txt
 fi
+rm -Rf /opt/spark/work-dir
+ln -s /sandbox/ /opt/spark/work-dir
+# END SAAGIE SPECIFIC CODE
 
 PYSPARK_ARGS=""
 if [ -n "$PYSPARK_APP_ARGS" ]; then

technologies/job/spark/spark-2.4/innerContexts/python/spark-2.4-python-3.7/build.me

@@ -1 +1 @@
-12/10/2023
+28/03/2025

technologies/job/spark/spark-2.4/innerContexts/python/spark-2.4-python-3.7/dockerInfo.yaml

@@ -1,4 +1,4 @@
 image: saagie/spark
 baseTag: 2.4-py-3.7
-dynamicVersion: 1.170.0_SDKTECHNO-246
-version: 2.4-py-3.7-1.170.0
+dynamicVersion: 1.194.0_sparkWithoutMinio
+version: 2.4-py-3.7-1.194.0_sparkWithoutMinio

technologies/job/spark/spark-2.4/innerContexts/python/spark-2.4-python-3.7/image_test.yaml

@@ -28,7 +28,6 @@ fileContentTests:
     expectedContents:
       [
         'exec /sbin/tini -s -- ',
-        'wget -nv \$PYSPARK_FILES',
         'unzip -q \*\.zip',
         'ln -s /sandbox/ /opt/spark/work-dir',
         '--py-files=/sandbox/\*'

technologies/job/spark/spark-3.0/innerContexts/python/entrypoint.sh

@@ -38,39 +38,16 @@ fi
 
 # BEGIN SAAGIE SPECIFIC CODE
 cd /sandbox
- # parse content and if pyfiles extract minio url and inject it
-if [ -f main_script ] && grep -q "\--py-files" main_script;
+if [ -f *.zip ]
 then
-  PYSPARK_FILES="`grep -Po '.*--py-files=\K[^ ]+' main_script`"
-fi;
-
-if [ -n "$PYSPARK_FILES" ]; then
-    PYTHONPATH="$PYTHONPATH:$PYSPARK_FILES"
-    #Copy and unzip pyfiles
-    if [[ $PYSPARK_FILES == *[,]* ]];then
-      echo "PYSPARK_FILES contains comma"
-      pyfiles=$(echo $PYSPARK_FILES | tr "," "\n")
-
-      for file in $pyfiles
-      do
-          echo ">>> [$file]"
-          wget -nv $file
-      done
-    else
-      echo ">>> [$PYSPARK_FILES]"
-      wget -nv $PYSPARK_FILES
-    fi
-    if [ -f *.zip ]
-    then
-      unzip -q *.zip
-    fi
-    if [ -f "requirements.txt" ]
-    then
-      pip install -r requirements.txt
-    fi
-    rm -Rf /opt/spark/work-dir
-    ln -s /sandbox/ /opt/spark/work-dir
+  unzip -q *.zip
+fi
+if [ -f "requirements.txt" ]
+then
+  pip install -r requirements.txt
 fi
+rm -Rf /opt/spark/work-dir
+ln -s /sandbox/ /opt/spark/work-dir
 # END SAAGIE SPECIFIC CODE
 
 

technologies/job/spark/spark-3.0/innerContexts/python/spark-3.0-python-3.7/build.me

@@ -1 +1 @@
-04/11/2021
+28/03/2025

technologies/job/spark/spark-3.0/innerContexts/python/spark-3.0-python-3.7/dockerInfo.yaml

@@ -1,4 +1,4 @@
 image: saagie/spark
 baseTag: 3.0-py-3.7
-dynamicVersion: 1.139.0_SDKTECHNO-207
-version: 3.0-py-3.7-1.139.0
+dynamicVersion: 1.194.0_sparkWithoutMinio
+version: 3.0-py-3.7-1.194.0_sparkWithoutMinio

technologies/job/spark/spark-3.0/innerContexts/python/spark-3.0-python-3.7/image_test.yaml

@@ -27,7 +27,6 @@ fileContentTests:
     path: "/opt/entrypoint.sh"
     expectedContents:
       [
-        'wget -nv \$PYSPARK_FILES',
         'unzip -q \*\.zip',
         'ln -s /sandbox/ /opt/spark/work-dir',
         '--py-files=/sandbox/\*',

technologies/job/spark/spark-3.1/innerContexts/python/entrypoint.sh

@@ -38,39 +38,16 @@ fi
 
 # BEGIN SAAGIE SPECIFIC CODE
 cd /sandbox
- # parse content and if pyfiles extract minio url and inject it
-if [ -f main_script ] && grep -q "\--py-files" main_script;
+if [ -f *.zip ]
 then
-  PYSPARK_FILES="`grep -Po '.*--py-files=\K[^ ]+' main_script`"
-fi;
-
-if [ -n "$PYSPARK_FILES" ]; then
-    PYTHONPATH="$PYTHONPATH:$PYSPARK_FILES"
-    #Copy and unzip pyfiles
-    if [[ $PYSPARK_FILES == *[,]* ]];then
-      echo "PYSPARK_FILES contains comma"
-      pyfiles=$(echo $PYSPARK_FILES | tr "," "\n")
-
-      for file in $pyfiles
-      do
-          echo ">>> [$file]"
-          wget -nv $file
-      done
-    else
-      echo ">>> [$PYSPARK_FILES]"
-      wget -nv $PYSPARK_FILES
-    fi
-    if [ -f *.zip ]
-    then
-      unzip -q *.zip
-    fi
-    if [ -f "requirements.txt" ]
-    then
-      pip install -r requirements.txt
-    fi
-    rm -Rf /opt/spark/work-dir
-    ln -s /sandbox/ /opt/spark/work-dir
+  unzip -q *.zip
+fi
+if [ -f "requirements.txt" ]
+then
+  pip install -r requirements.txt
 fi
+rm -Rf /opt/spark/work-dir
+ln -s /sandbox/ /opt/spark/work-dir
 # END SAAGIE SPECIFIC CODE
 
 

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.7/build.me

@@ -1 +1 @@
-12/12/2022
+28/03/2025

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.7/dockerInfo.yaml

@@ -1,4 +1,4 @@
 image: saagie/spark
 baseTag: 3.1-py-3.7
-dynamicVersion: 1.139.0_SDKTECHNO-207
-version: 3.1-py-3.7-1.139.0
+dynamicVersion: 1.194.0_sparkWithoutMinio
+version: 3.1-py-3.7-1.194.0_sparkWithoutMinio

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.7/image_test.yaml

@@ -27,7 +27,6 @@ fileContentTests:
     path: "/opt/entrypoint.sh"
     expectedContents:
       [
-        'wget -nv \$PYSPARK_FILES',
         'unzip -q \*\.zip',
         'ln -s /sandbox/ /opt/spark/work-dir',
         '--py-files=/sandbox/\*',

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.8/build.me

@@ -1 +1 @@
-04/11/2021
+28/03/2025

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.8/dockerInfo.yaml

@@ -1,4 +1,4 @@
 image: saagie/spark
 baseTag: 3.1-py-3.8
-dynamicVersion: 1.139.0_SDKTECHNO-207
-version: 3.1-py-3.8-1.139.0
+dynamicVersion: 1.194.0_sparkWithoutMinio
+version: 3.1-py-3.8-1.194.0_sparkWithoutMinio

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.8/image_test.yaml

@@ -27,7 +27,6 @@ fileContentTests:
     path: "/opt/entrypoint.sh"
     expectedContents:
       [
-        'wget -nv \$PYSPARK_FILES',
         'unzip -q \*\.zip',
         'ln -s /sandbox/ /opt/spark/work-dir',
         '--py-files=/sandbox/\*',

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.9/build.me

@@ -1 +1 @@
-04/11/2021
+28/03/2025

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.9/dockerInfo.yaml

@@ -1,4 +1,4 @@
 image: saagie/spark
 baseTag: 3.1-py-3.9
-dynamicVersion: 1.139.0_SDKTECHNO-207
-version: 3.1-py-3.9-1.139.0
+dynamicVersion: 1.194.0_sparkWithoutMinio
+version: 3.1-py-3.9-1.194.0_sparkWithoutMinio

technologies/job/spark/spark-3.1/innerContexts/python/spark-3.1-python-3.9/image_test.yaml

@@ -27,7 +27,6 @@ fileContentTests:
     path: "/opt/entrypoint.sh"
     expectedContents:
       [
-        'wget -nv \$PYSPARK_FILES',
         'unzip -q \*\.zip',
         'ln -s /sandbox/ /opt/spark/work-dir',
         '--py-files=/sandbox/\*',

technologies/job/spark/spark-3.5/innerContexts/python/entrypoint.sh

@@ -18,6 +18,7 @@
 #
 # Prevent any errors from being silently ignored
 set -eo pipefail
+set -x
 
 attempt_setup_fake_passwd_entry() {
   # Check whether there is a passwd entry for the container UID
@@ -48,39 +49,16 @@ fi
 
 # BEGIN SAAGIE SPECIFIC CODE
 cd /sandbox
- # parse content and if pyfiles extract minio url and inject it
-if [ -f main_script ] && grep -q "\--py-files" main_script;
+if [ -f *.zip ]
 then
-  PYSPARK_FILES="`grep -Po '.*--py-files=\K[^ ]+' main_script`"
-fi;
-
-if [ -n "$PYSPARK_FILES" ]; then
-    PYTHONPATH="$PYTHONPATH:$PYSPARK_FILES"
-    #Copy and unzip pyfiles
-    if [[ $PYSPARK_FILES == *[,]* ]];then
-      echo "PYSPARK_FILES contains comma"
-      pyfiles=$(echo $PYSPARK_FILES | tr "," "\n")
-
-      for file in $pyfiles
-      do
-          echo ">>> [$file]"
-          wget -nv $file
-      done
-    else
-      echo ">>> [$PYSPARK_FILES]"
-      wget -nv $PYSPARK_FILES
-    fi
-    if [ -f *.zip ]
-    then
-      unzip -q *.zip
-    fi
-    if [ -f "requirements.txt" ]
-    then
-      pip install -r requirements.txt
-    fi
-    rm -Rf /opt/spark/work-dir
-    ln -s /sandbox/ /opt/spark/work-dir
+  unzip -q *.zip
+fi
+if [ -f "requirements.txt" ]
+then
+  pip install -r requirements.txt
 fi
+rm -Rf /opt/spark/work-dir
+ln -s /sandbox/ /opt/spark/work-dir
 # END SAAGIE SPECIFIC CODE
 
 SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"

technologies/job/spark/spark-3.5/innerContexts/python/spark-3.5-python-3.12/build.me

@@ -0,0 +1 @@
+true

technologies/job/spark/spark-3.5/innerContexts/python/spark-3.5-python-3.12/dockerInfo.yaml

@@ -1,4 +1,4 @@
 image: saagie/spark
 baseTag: 3.5-py-3.12
-dynamicVersion: 1.190.0_spark-3.5
-version: 3.5-py-3.12-1.190.0
+dynamicVersion: 1.194.0_sparkWithoutMinio
+version: 3.5-py-3.12-1.194.0_sparkWithoutMinio