This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new be6c8a115 [CELEBORN] VeloxCelebornColumnarBatchSerializer uses the key
and default value of SHUFFLE_COMPRESS to check whether to compress shuffle
output (#6414)
be6c8a115 is described below
commit be6c8a115a43ec717e7072fd78881a071bb108d2
Author: Nicholas Jiang <[email protected]>
AuthorDate: Fri Jul 12 08:07:18 2024 +0700
[CELEBORN] VeloxCelebornColumnarBatchSerializer uses the key and default
value of SHUFFLE_COMPRESS to check whether to compress shuffle output (#6414)
---
.../apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala
b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala
index 1f125a164..19305e95d 100644
---
a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala
+++
b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala
@@ -24,6 +24,7 @@ import org.apache.gluten.vectorized._
import org.apache.spark.SparkEnv
import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config.SHUFFLE_COMPRESS
import org.apache.spark.serializer.{DeserializationStream,
SerializationStream, Serializer, SerializerInstance}
import org.apache.spark.sql.execution.metric.SQLMetric
import org.apache.spark.sql.internal.SQLConf
@@ -76,7 +77,7 @@ private class CelebornColumnarBatchSerializerInstance(
ArrowAbiUtil.exportSchema(allocator, arrowSchema, cSchema)
val conf = SparkEnv.get.conf
val compressionCodec =
- if (conf.getBoolean("spark.shuffle.compress", true)) {
+ if (conf.getBoolean(SHUFFLE_COMPRESS.key,
SHUFFLE_COMPRESS.defaultValue.get)) {
GlutenShuffleUtils.getCompressionCodec(conf)
} else {
null // uncompressed
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
