fix: Corrected Spark defaults to fix read/write functionality from Sp…

…ark (#526)
awslabs · Jun 10, 2021 · f96e1bd · f96e1bd
1 parent ea2650f
commit f96e1bd
Showing 1 changed file with 6 additions and 4 deletions.
diff --git a/...e-raas/packages/base-raas-cfn-templates/src/templates/service-catalog/emr-cluster.cfn.yml b/...e-raas/packages/base-raas-cfn-templates/src/templates/service-catalog/emr-cluster.cfn.yml
@@ -266,11 +266,13 @@ Resources:
             yarn.nodemanager.vmem-check-enabled: false
         - Classification: spark-defaults
           ConfigurationProperties:
-            spark.hadoop.io.compression.codecs: 'org.apache.hadoop.io.compress.DefaultCodec,is.hail.io.compress.BGzipCodec,org.apache.hadoop.io.compress.GzipCodec'
+            spark.hadoop.io.compression.codecs: 'org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec'
             spark.serializer: 'org.apache.spark.serializer.KryoSerializer'
-            spark.hadoop.parquet.block.size: '1099511627776'
-            spark.sql.files.maxPartitionBytes: '1099511627776'
-            spark.sql.files.openCostInBytes: '1099511627776'
+            # Defaults picked from spark conf
+            # https://github.com/apache/spark/blob/878527d9fae8945d087ec871bb0a5f49b6341939/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L1322-L1339
+            spark.hadoop.parquet.block.size: '134217728' # 128MB
+            spark.sql.files.maxPartitionBytes: '134217728' # 128 MB
+            spark.sql.files.openCostInBytes: '4194304' # 4 MB
           Configurations: []
       Instances:
         AdditionalMasterSecurityGroups: