added spark propery setting on SparkContext creation

Tale152 · Mar 2, 2022 · 04e14fa · 04e14fa
1 parent a1b1ede
commit 04e14fa
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/src/main/scala/utils/ContextFactory.scala b/src/main/scala/utils/ContextFactory.scala
@@ -14,6 +14,10 @@ object ContextFactory {
      * @return the newly created [[SparkContext]]
      */
     def create(appName: String, masterAddress: String, jarPath: String): SparkContext = new SparkContext(
-        new SparkConf().setAppName(appName).setMaster(masterAddress).setJars(Seq(jarPath))
+        new SparkConf()
+          .setAppName(appName)
+          .setMaster(masterAddress)
+          .setJars(Seq(jarPath))
+          .set("spark.driver.maxResultSize", "2g")
     )
 }