This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 59aaa1cc3 [VL] Quick fix for Uniffle CI error (#5986)
59aaa1cc3 is described below
commit 59aaa1cc36686b34900b44b920679ed335f94302
Author: Hongze Zhang <[email protected]>
AuthorDate: Wed Jun 5 11:41:36 2024 +0800
[VL] Quick fix for Uniffle CI error (#5986)
---
.../org/apache/gluten/integration/BaseMixin.java | 2 +-
.../org/apache/gluten/integration/Suite.scala | 22 ++++++++++++----------
2 files changed, 13 insertions(+), 11 deletions(-)
diff --git
a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java
b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java
index dc1691e50..93c82a6fa 100644
---
a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java
+++
b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java
@@ -75,7 +75,7 @@ public class BaseMixin {
@CommandLine.Option(names = {"--shuffle-partitions"}, description = "Shuffle
partition number", defaultValue = "100")
private int shufflePartitions;
- @CommandLine.Option(names = {"--scan-partitions"}, description = "Scan
partition number. This is an approximate value, so the actual scan partition
number might vary around this value", defaultValue = "100")
+ @CommandLine.Option(names = {"--scan-partitions"}, description = "Scan
partition number. This is an approximate value, so the actual scan partition
number might vary around this value. -1 for letting Spark choose an appropriate
number.", defaultValue = "-1")
private int scanPartitions;
@CommandLine.Option(names = {"--extra-conf"}, description = "Extra Spark
config entries applying to generated Spark session. E.g. --extra-conf=k1=v1
--extra-conf=k2=v2")
diff --git
a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala
b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala
index bb5cb1889..51e1777e2 100644
---
a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala
+++
b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala
@@ -103,16 +103,18 @@ abstract class Suite(
sessionSwitcher.defaultConf().setWarningOnOverriding("spark.sql.codegen.wholeStage",
"false")
}
- // Scan partition number.
- sessionSwitcher
- .defaultConf()
- .setWarningOnOverriding("spark.sql.files.maxPartitionBytes",
s"${ByteUnit.PiB.toBytes(1L)}")
- sessionSwitcher
- .defaultConf()
- .setWarningOnOverriding("spark.sql.files.openCostInBytes", "0")
- sessionSwitcher
- .defaultConf()
- .setWarningOnOverriding("spark.sql.files.minPartitionNum",
s"${(scanPartitions - 1) max 1}")
+ if (scanPartitions != -1) {
+ // Scan partition number.
+ sessionSwitcher
+ .defaultConf()
+ .setWarningOnOverriding("spark.sql.files.maxPartitionBytes",
s"${ByteUnit.PiB.toBytes(1L)}")
+ sessionSwitcher
+ .defaultConf()
+ .setWarningOnOverriding("spark.sql.files.openCostInBytes", "0")
+ sessionSwitcher
+ .defaultConf()
+ .setWarningOnOverriding("spark.sql.files.minPartitionNum",
s"${(scanPartitions - 1) max 1}")
+ }
extraSparkConf.toStream.foreach { kv =>
sessionSwitcher.defaultConf().setWarningOnOverriding(kv._1, kv._2)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
