This is an automated email from the ASF dual-hosted git repository.
marin-ma pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 3a36ef2357 [GLUTEN-11539][VL] Improve error message for unsupported
spark.io.compression.codec (#12360)
3a36ef2357 is described below
commit 3a36ef235727cdb05a20a7fd64bb6b93f66f19b6
Author: BRIJ RAJ KISHORE <[email protected]>
AuthorDate: Mon Jun 29 15:36:15 2026 +0530
[GLUTEN-11539][VL] Improve error message for unsupported
spark.io.compression.codec (#12360)
Emit a unified error for all unsupported codecs that names the
offending codec, hints at spark.shuffle.compress=false to disable
compression entirely, and lists the valid codec choices.
Co-authored-by: Claude Sonnet 4.6 <[email protected]>
---
.../test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala | 7 ++++---
.../main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala | 6 +++---
2 files changed, 7 insertions(+), 6 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
index fcf1fbbbf7..51cb918aa6 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
@@ -2257,16 +2257,17 @@ class MiscOperatorSuite extends
VeloxWholeStageTransformerSuite with AdaptiveSpa
val ex = intercept[IllegalArgumentException] {
GlutenShuffleUtils.getCompressionCodec(conf)
}
- assert(ex.getMessage.contains("snappy is not supported"))
+ assert(ex.getMessage.contains("does not support codec 'snappy'"))
+ assert(ex.getMessage.contains("spark.shuffle.compress=false"))
assert(ex.getMessage.contains(GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key))
}
- test("GLUTEN-11539: spark.io.compression.codec=none throws with actionable
message") {
+ test("GLUTEN-11539: spark.io.compression.codec=none throws pointing to
spark.shuffle.compress") {
val conf =
spark.sparkContext.getConf.clone().set("spark.io.compression.codec", "none")
val ex = intercept[IllegalArgumentException] {
GlutenShuffleUtils.getCompressionCodec(conf)
}
- assert(ex.getMessage.contains("none is not supported"))
+ assert(ex.getMessage.contains("spark.shuffle.compress=false"))
assert(ex.getMessage.contains(GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key))
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala
b/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala
index 4bd89582c1..d7d8d4cbd9 100644
---
a/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala
+++
b/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala
@@ -77,9 +77,9 @@ object GlutenShuffleUtils {
val supportedCodecs =
BackendsApiManager.getSettings.shuffleSupportedCodec()
if (!supportedCodecs.contains(codec)) {
throw new IllegalArgumentException(
- s"Gluten shuffle only supports ${supportedCodecs.mkString(", ")}.
" +
- s"$codec is not supported. " +
- s"You may configure ${GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key} "
+
+ s"Gluten shuffle does not support codec '$codec'. " +
+ s"To disable shuffle compression, set
spark.shuffle.compress=false. " +
+ s"To use a supported codec, set
${GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key} " +
s"to ${supportedCodecs.mkString(" or ")}.")
}
codec
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
