demo with Spark 3.4 (DE-600)

Author: rashtao

demo/README.md

@@ -42,7 +42,7 @@ password `test`.
 Start Spark cluster:
 
 ```shell
-./docker/start_spark_3.3.sh 
+./docker/start_spark_3.4.sh 
 ```
 
 ## Run embedded
@@ -79,9 +79,9 @@ docker run -it --rm \
   -v $(pwd):/demo \
   -v $(pwd)/docker/.ivy2:/opt/bitnami/spark/.ivy2 \
   --network arangodb \
-  docker.io/bitnami/spark:3.3.2 \
+  docker.io/bitnami/spark:3.4.0 \
   ./bin/spark-submit --master spark://spark-master:7077 \
-    --packages="com.arangodb:arangodb-spark-datasource-3.3_2.12:$ARANGO_SPARK_VERSION" \
+    --packages="com.arangodb:arangodb-spark-datasource-3.4_2.12:$ARANGO_SPARK_VERSION" \
     --class Demo /demo/target/demo-$ARANGO_SPARK_VERSION.jar
 ```
 

demo/docker/start_spark_3.3.sh renamed to demo/docker/start_spark_3.4.sh

@@ -9,7 +9,7 @@ docker run -d --network arangodb --ip 172.28.10.1 --name spark-master -h spark-m
   -e SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no \
   -e SPARK_SSL_ENABLED=no \
   -v $(pwd)/docker/import:/import \
-  docker.io/bitnami/spark:3.3.2
+  docker.io/bitnami/spark:3.4.0
 
 docker run -d --network arangodb --ip 172.28.10.11 --name spark-worker-1 -h spark-worker-1 \
   -e SPARK_MODE=worker \
@@ -21,7 +21,7 @@ docker run -d --network arangodb --ip 172.28.10.11 --name spark-worker-1 -h spar
   -e SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no \
   -e SPARK_SSL_ENABLED=no \
   -v $(pwd)/docker/import:/import \
-  docker.io/bitnami/spark:3.3.2
+  docker.io/bitnami/spark:3.4.0
 
 docker run -d --network arangodb --ip 172.28.10.12 --name spark-worker-2 -h spark-worker-2 \
   -e SPARK_MODE=worker \
@@ -33,7 +33,7 @@ docker run -d --network arangodb --ip 172.28.10.12 --name spark-worker-2 -h spar
   -e SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no \
   -e SPARK_SSL_ENABLED=no \
   -v $(pwd)/docker/import:/import \
-  docker.io/bitnami/spark:3.3.2
+  docker.io/bitnami/spark:3.4.0
 
 docker run -d --network arangodb --ip 172.28.10.13 --name spark-worker-3 -h spark-worker-3 \
   -e SPARK_MODE=worker \
@@ -45,4 +45,4 @@ docker run -d --network arangodb --ip 172.28.10.13 --name spark-worker-3 -h spar
   -e SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no \
   -e SPARK_SSL_ENABLED=no \
   -v $(pwd)/docker/import:/import \
-  docker.io/bitnami/spark:3.3.2
+  docker.io/bitnami/spark:3.4.0

demo/pom.xml

@@ -62,16 +62,16 @@
         </profile>
         <profile>
             <id>spark-3.3</id>
-            <activation>
-                <activeByDefault>true</activeByDefault>
-            </activation>
             <properties>
                 <spark.version>3.3.2</spark.version>
                 <spark.compat.version>3.3</spark.compat.version>
             </properties>
         </profile>
         <profile>
             <id>spark-3.4</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
             <properties>
                 <spark.version>3.4.0</spark.version>
                 <spark.compat.version>3.4</spark.compat.version>

demo/python-demo/demo.py

@@ -18,7 +18,7 @@ def create_spark_session() -> SparkSession:
     spark = SparkSession.builder \
         .appName("ArangoDBPySparkDataTypesExample") \
         .master("local[*]") \
-        .config("spark.jars.packages", f"com.arangodb:arangodb-spark-datasource-3.3_2.12:{arango_spark_version}") \
+        .config("spark.jars.packages", f"com.arangodb:arangodb-spark-datasource-3.4_2.12:{arango_spark_version}") \
         .getOrCreate()
 
     return spark

demo/python-demo/requirements.txt

@@ -1 +1 @@
-pyspark[pandas_on_spark]==3.3.2
+pyspark[pandas_on_spark]==3.4.0