This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 9bacf0fc1 [VL] Add spark monotonically_increasing_id function support
(#4954)
9bacf0fc1 is described below
commit 9bacf0fc16c29dd6df65f8d83a48179dfbc8e088
Author: 高阳阳 <[email protected]>
AuthorDate: Mon Mar 18 08:55:29 2024 +0800
[VL] Add spark monotonically_increasing_id function support (#4954)
---
.../src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala | 1 +
.../io/glutenproject/execution/VeloxFunctionsValidateSuite.scala | 7 +++++++
docs/velox-backend-support-progress.md | 2 +-
.../scala/io/glutenproject/expression/ExpressionMappings.scala | 1 +
.../main/scala/io/glutenproject/expression/ExpressionNames.scala | 1 +
5 files changed, 11 insertions(+), 1 deletion(-)
diff --git
a/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala
b/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala
index 31f2cd533..cb607da4c 100644
---
a/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala
+++
b/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala
@@ -175,6 +175,7 @@ object CHExpressionUtil {
ARRAY_EXCEPT -> DefaultValidator(),
ARRAY_REPEAT -> DefaultValidator(),
DATE_FROM_UNIX_DATE -> DefaultValidator(),
+ MONOTONICALLY_INCREASING_ID -> DefaultValidator(),
SKEWNESS -> DefaultValidator()
)
}
diff --git
a/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala
index 33f5e48ad..b558cc77b 100644
---
a/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala
@@ -444,6 +444,13 @@ class VeloxFunctionsValidateSuite extends
VeloxWholeStageTransformerSuite {
}
}
+ test("Test monotonically_increasing_id function") {
+ runQueryAndCompare("""SELECT monotonically_increasing_id(), l_orderkey
+ | from lineitem limit 1""".stripMargin) {
+ checkOperatorMatch[ProjectExecTransformer]
+ }
+ }
+
test("Test hex function") {
runQueryAndCompare("SELECT hex(l_partkey), hex(l_shipmode) FROM lineitem
limit 1") {
checkOperatorMatch[ProjectExecTransformer]
diff --git a/docs/velox-backend-support-progress.md
b/docs/velox-backend-support-progress.md
index 639211e11..e9ba9d1f2 100644
--- a/docs/velox-backend-support-progress.md
+++ b/docs/velox-backend-support-progress.md
@@ -416,7 +416,7 @@ Gluten supports 199 functions. (Draw to right to see all
data types)
| java_method | |
| | | | | | | |
| | | | | | | | |
| | | |
| least | least | least
| S | | | | | | S | S
| S | S | S | | | | | |
| | | |
| md5 | md5 |
| S | | | S | | | |
| | | | | | | | |
| | | |
-| monotonically_increasing_id | |
| | | | | | | |
| | | | | | | | |
| | | |
+| monotonically_increasing_id | |
| S | | | | | | |
| | | | | | | | |
| | | |
| nanvl | |
| S | | | | | | |
| | | | | | | | |
| | | |
| nvl | |
| | | | | | | |
| | | | | | | | |
| | | |
| nvl2 | |
| | | | | | | |
| | | | | | | | |
| | | |
diff --git
a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionMappings.scala
b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionMappings.scala
index 6fb9d80b0..9f3c2ed81 100644
---
a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionMappings.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionMappings.scala
@@ -242,6 +242,7 @@ object ExpressionMappings {
Sig[CheckOverflow](CHECK_OVERFLOW),
Sig[MakeDecimal](MAKE_DECIMAL),
Sig[PromotePrecision](PROMOTE_PRECISION),
+ Sig[MonotonicallyIncreasingID](MONOTONICALLY_INCREASING_ID),
// Decimal
Sig[UnscaledValue](UNSCALED_VALUE)
) ++ SparkShimLoader.getSparkShims.expressionMappings
diff --git
a/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala
b/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala
index c5a00b51f..7998d7642 100644
---
a/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala
+++
b/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala
@@ -262,6 +262,7 @@ object ExpressionNames {
final val CHECK_OVERFLOW = "check_overflow"
final val MAKE_DECIMAL = "make_decimal"
final val PROMOTE_PRECISION = "promote_precision"
+ final val MONOTONICALLY_INCREASING_ID = "monotonically_increasing_id"
// Directly use child expression transformer
final val KNOWN_FLOATING_POINT_NORMALIZED = "known_floating_point_normalized"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
