This is an automated email from the ASF dual-hosted git repository.
chengchengjin pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 622f6b7b0d [VL] Support decimal as double in gluten-it (#9472)
622f6b7b0d is described below
commit 622f6b7b0dbb1d1f82b91c7fdce273c7808eb5e3
Author: Jin Chengcheng <[email protected]>
AuthorDate: Fri May 2 09:29:58 2025 +0100
[VL] Support decimal as double in gluten-it (#9472)
---
.../src/main/java/org/apache/gluten/integration/BaseMixin.java | 9 ++++++---
.../src/main/scala/org/apache/gluten/integration/Constants.scala | 1 -
.../src/main/scala/org/apache/gluten/integration/DataGen.scala | 2 +-
.../src/main/scala/org/apache/gluten/integration/Suite.scala | 8 ++++++--
.../apache/gluten/integration/clickbench/ClickBenchSuite.scala | 2 ++
.../main/scala/org/apache/gluten/integration/ds/TpcdsSuite.scala | 7 ++-----
.../main/scala/org/apache/gluten/integration/h/TpchSuite.scala | 6 ++----
7 files changed, 19 insertions(+), 16 deletions(-)
diff --git
a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java
b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java
index ada777e87e..3473730e07 100644
---
a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java
+++
b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java
@@ -80,6 +80,9 @@ public class BaseMixin {
@CommandLine.Option(names = {"--scan-partitions"}, description = "Scan
partition number. This is an approximate value, so the actual scan partition
number might vary around this value. -1 for letting Spark choose an appropriate
number.", defaultValue = "-1")
private int scanPartitions;
+ @CommandLine.Option(names = {"--decimal-as-double"}, description = "Generate
double values for decimal type column", defaultValue = "false")
+ private boolean decimalAsDouble;
+
@CommandLine.Option(names = {"--extra-conf"}, description = "Extra Spark
config entries applying to generated Spark session. E.g. --extra-conf=k1=v1
--extra-conf=k2=v2")
private Map<String, String> extraSparkConf = Collections.emptyMap();
@@ -126,19 +129,19 @@ public class BaseMixin {
suite = new TpchSuite(runModeEnumeration.getSparkMasterUrl(), actions,
testConf,
baselineConf, extraSparkConfScala, level, errorOnMemLeak, dataDir,
enableUi,
enableHsUi, hsUiPort, disableAqe, disableBhj,
- disableWscg, shufflePartitions, scanPartitions,
baselineMetricMapper, testMetricMapper);
+ disableWscg, shufflePartitions, scanPartitions, decimalAsDouble,
baselineMetricMapper, testMetricMapper);
break;
case "ds":
suite = new TpcdsSuite(runModeEnumeration.getSparkMasterUrl(),
actions, testConf,
baselineConf, extraSparkConfScala, level, errorOnMemLeak, dataDir,
enableUi,
enableHsUi, hsUiPort, disableAqe, disableBhj,
- disableWscg, shufflePartitions, scanPartitions,
baselineMetricMapper, testMetricMapper);
+ disableWscg, shufflePartitions, scanPartitions, decimalAsDouble,
baselineMetricMapper, testMetricMapper);
break;
case "clickbench":
suite = new ClickBenchSuite(runModeEnumeration.getSparkMasterUrl(),
actions, testConf,
baselineConf, extraSparkConfScala, level, errorOnMemLeak, dataDir,
enableUi,
enableHsUi, hsUiPort, disableAqe, disableBhj,
- disableWscg, shufflePartitions, scanPartitions,
baselineMetricMapper, testMetricMapper);
+ disableWscg, shufflePartitions, scanPartitions, decimalAsDouble,
baselineMetricMapper, testMetricMapper);
break;
default:
throw new IllegalArgumentException("TPC benchmark type not found: " +
benchmarkType);
diff --git
a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Constants.scala
b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Constants.scala
index fe4e986cd2..599af7fd77 100644
---
a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Constants.scala
+++
b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Constants.scala
@@ -162,7 +162,6 @@ object Constants {
}
}
- @deprecated
val TYPE_MODIFIER_DECIMAL_AS_DOUBLE: TypeModifier =
new TypeModifier(TypeUtils.decimalAccepts, DoubleType) {
override def modValue(from: Any): Any = {
diff --git
a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/DataGen.scala
b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/DataGen.scala
index 6b2d4ec71b..47aa7d423e 100644
---
a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/DataGen.scala
+++
b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/DataGen.scala
@@ -16,7 +16,7 @@
*/
package org.apache.gluten.integration
-import org.apache.spark.sql.types.{DataType, StructField, StructType}
+import org.apache.spark.sql.types.{DataType, DecimalType, DoubleType,
StructField, StructType}
trait DataGen {
def gen(): Unit
diff --git
a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala
b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala
index 2e5d781bb5..b240855ed8 100644
---
a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala
+++
b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala
@@ -16,15 +16,14 @@
*/
package org.apache.gluten.integration
+import org.apache.gluten.integration.Constants.TYPE_MODIFIER_DECIMAL_AS_DOUBLE
import org.apache.gluten.integration.action.Action
import org.apache.gluten.integration.metrics.MetricMapper
-
import org.apache.spark.SparkConf
import org.apache.spark.deploy.history.HistoryServerHelper
import org.apache.spark.network.util.ByteUnit
import org.apache.spark.sql.ConfUtils.ConfImplicits._
import org.apache.spark.sql.SparkSessionSwitcher
-
import org.apache.log4j.{Level, LogManager}
import java.io.File
@@ -46,6 +45,7 @@ abstract class Suite(
private val disableWscg: Boolean,
private val shufflePartitions: Int,
private val scanPartitions: Int,
+ private val decimalAsDouble: Boolean,
private val baselineMetricMapper: MetricMapper,
private val testMetricMapper: MetricMapper) {
@@ -187,6 +187,10 @@ abstract class Suite(
protected def historyWritePath(): String
+ protected[integration] def typeModifiers(): List[TypeModifier] = {
+ if (decimalAsDouble) List(TYPE_MODIFIER_DECIMAL_AS_DOUBLE) else List()
+ }
+
private[integration] def dataWritePath(scale: Double, genPartitionedData:
Boolean): String
private[integration] def createDataGen(scale: Double, genPartitionedData:
Boolean): DataGen
diff --git
a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchSuite.scala
b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchSuite.scala
index 852c934564..207974d354 100644
---
a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchSuite.scala
+++
b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchSuite.scala
@@ -48,6 +48,7 @@ class ClickBenchSuite(
val disableWscg: Boolean,
val shufflePartitions: Int,
val scanPartitions: Int,
+ val decimalAsDouble: Boolean,
val baselineMetricMapper: MetricMapper,
val testMetricMapper: MetricMapper)
extends Suite(
@@ -66,6 +67,7 @@ class ClickBenchSuite(
disableWscg,
shufflePartitions,
scanPartitions,
+ decimalAsDouble,
baselineMetricMapper,
testMetricMapper
) {
diff --git
a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/ds/TpcdsSuite.scala
b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/ds/TpcdsSuite.scala
index 52a30a2282..cadc844f33 100644
---
a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/ds/TpcdsSuite.scala
+++
b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/ds/TpcdsSuite.scala
@@ -22,7 +22,6 @@ import
org.apache.gluten.integration.ds.TpcdsSuite.{ALL_QUERY_IDS, HISTORY_WRITE
import org.apache.gluten.integration.metrics.MetricMapper
import org.apache.spark.SparkConf
-
import org.apache.log4j.Level
import java.io.File
@@ -44,6 +43,7 @@ class TpcdsSuite(
val disableWscg: Boolean,
val shufflePartitions: Int,
val scanPartitions: Int,
+ val decimalAsDouble: Boolean,
val baselineMetricMapper: MetricMapper,
val testMetricMapper: MetricMapper)
extends Suite(
@@ -62,6 +62,7 @@ class TpcdsSuite(
disableWscg,
shufflePartitions,
scanPartitions,
+ decimalAsDouble,
baselineMetricMapper,
testMetricMapper
) {
@@ -88,10 +89,6 @@ class TpcdsSuite(
"/tpcds-queries"
}
- private def typeModifiers(): List[TypeModifier] = {
- List()
- }
-
override private[integration] def allQueryIds(): Array[String] =
ALL_QUERY_IDS
override private[integration] def desc(): String = "TPC-DS"
diff --git
a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/h/TpchSuite.scala
b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/h/TpchSuite.scala
index e48d52393d..32635620fc 100644
---
a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/h/TpchSuite.scala
+++
b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/h/TpchSuite.scala
@@ -42,6 +42,7 @@ class TpchSuite(
val disableWscg: Boolean,
val shufflePartitions: Int,
val scanPartitions: Int,
+ val decimalAsDouble: Boolean,
val baselineMetricMapper: MetricMapper,
val testMetricMapper: MetricMapper)
extends Suite(
@@ -60,6 +61,7 @@ class TpchSuite(
disableWscg,
shufflePartitions,
scanPartitions,
+ decimalAsDouble,
baselineMetricMapper,
testMetricMapper) {
@@ -84,10 +86,6 @@ class TpchSuite(
"/tpch-queries"
}
- private def typeModifiers(): List[TypeModifier] = {
- List()
- }
-
override private[integration] def allQueryIds(): Array[String] =
TpchSuite.ALL_QUERY_IDS
override private[integration] def desc(): String = "TPC-H"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
