use spark4

Author: kevinjqliu

dev/spark/Dockerfile

@@ -13,17 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG BASE_IMAGE_SPARK_VERSION=3.5.6
+ARG BASE_IMAGE_SPARK_VERSION=4.0.1
 
 FROM apache/spark:${BASE_IMAGE_SPARK_VERSION}
 
 # Dependency versions - keep these compatible
 ARG ICEBERG_VERSION=1.10.0
-ARG ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12
-ARG SPARK_VERSION=3.5.6
-ARG SCALA_VERSION=2.12
-ARG HADOOP_VERSION=3.3.4
-ARG AWS_SDK_VERSION=1.12.753
+ARG ICEBERG_SPARK_RUNTIME_VERSION=4.0_2.13
+ARG SPARK_VERSION=4.0.1
+ARG HADOOP_VERSION=3.4.1
+ARG SCALA_VERSION=2.13
+ARG AWS_SDK_VERSION=2.34.4
 ARG MAVEN_MIRROR=https://repo.maven.apache.org/maven2
 
 USER root
@@ -47,7 +47,7 @@ ENV JARS_TO_DOWNLOAD="\
     org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \
     org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar \
     org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar \
-    com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_VERSION}/aws-java-sdk-bundle-${AWS_SDK_VERSION}.jar"
+    software/amazon/awssdk/s3-transfer-manager/${AWS_SDK_VERSION}/s3-transfer-manager-${AWS_SDK_VERSION}.jar"
 
 # Download JARs with retry logic
 RUN set -e && \

dev/spark/spark-defaults.conf

@@ -48,3 +48,5 @@ spark.sql.defaultCatalog               rest
 spark.ui.enabled                       true
 spark.eventLog.enabled                 true
 spark.eventLog.dir                     /home/iceberg/spark-events
+
+spark.sql.ansi.enabled                 false