This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new f203fd84e [CORE] Enable bit_length Spark function (#5069)
f203fd84e is described below
commit f203fd84e5e4eef153d6b33ae23a065182e97120
Author: PHILO-HE <[email protected]>
AuthorDate: Fri Mar 22 09:49:59 2024 +0800
[CORE] Enable bit_length Spark function (#5069)
---
.../scala/io/glutenproject/utils/CHExpressionUtil.scala | 3 ++-
.../execution/VeloxFunctionsValidateSuite.scala | 16 ++++++++++++----
.../io/glutenproject/expression/ExpressionMappings.scala | 1 +
.../io/glutenproject/expression/ExpressionNames.scala | 1 +
4 files changed, 16 insertions(+), 5 deletions(-)
diff --git
a/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala
b/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala
index 86fd425f4..24555c05c 100644
---
a/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala
+++
b/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala
@@ -177,6 +177,7 @@ object CHExpressionUtil {
DATE_FROM_UNIX_DATE -> DefaultValidator(),
MONOTONICALLY_INCREASING_ID -> DefaultValidator(),
SPARK_PARTITION_ID -> DefaultValidator(),
- SKEWNESS -> DefaultValidator()
+ SKEWNESS -> DefaultValidator(),
+ BIT_LENGTH -> DefaultValidator()
)
}
diff --git
a/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala
index f7f1dc90e..1e9871a6c 100644
---
a/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala
@@ -521,11 +521,11 @@ class VeloxFunctionsValidateSuite extends
VeloxWholeStageTransformerSuite {
test("regexp_replace") {
runQueryAndCompare(
- "SELECT regexp_replace(l_partkey, '\\w', 'something') FROM lineitem
limit 100") {
+ "SELECT regexp_replace(c_comment, '\\w', 'something') FROM customer
limit 50") {
checkOperatorMatch[ProjectExecTransformer]
}
runQueryAndCompare(
- "SELECT regexp_replace(l_partkey, '\\w', 'something', 3) FROM lineitem
limit 100") {
+ "SELECT regexp_replace(c_comment, '\\w', 'something', 3) FROM customer
limit 50") {
checkOperatorMatch[ProjectExecTransformer]
}
}
@@ -533,15 +533,23 @@ class VeloxFunctionsValidateSuite extends
VeloxWholeStageTransformerSuite {
test("lag/lead window function with negative input offset") {
runQueryAndCompare(
"select lag(l_orderkey, -2) over" +
- " (partition by l_suppkey order by l_orderkey) from lineitem ") {
+ " (partition by l_suppkey order by l_orderkey) from lineitem") {
checkOperatorMatch[WindowExecTransformer]
}
runQueryAndCompare(
"select lead(l_orderkey, -2) over" +
- " (partition by l_suppkey order by l_orderkey) from lineitem ") {
+ " (partition by l_suppkey order by l_orderkey) from lineitem") {
checkOperatorMatch[WindowExecTransformer]
}
}
+ test("bit_length") {
+ runQueryAndCompare(
+ "select bit_length(c_comment), bit_length(cast(c_comment as binary))" +
+ " from customer limit 50") {
+ checkOperatorMatch[ProjectExecTransformer]
+ }
+ }
+
}
diff --git
a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionMappings.scala
b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionMappings.scala
index e85f8d4e2..a286ed556 100644
---
a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionMappings.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionMappings.scala
@@ -96,6 +96,7 @@ object ExpressionMappings {
Sig[StringDecode](DECODE),
Sig[Encode](ENCODE),
Sig[Uuid](UUID),
+ Sig[BitLength](BIT_LENGTH),
// URL functions
Sig[ParseUrl](PARSE_URL),
diff --git
a/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala
b/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala
index 5dbb926bf..cb9e1ab71 100644
---
a/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala
+++
b/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala
@@ -113,6 +113,7 @@ object ExpressionNames {
final val DECODE = "decode"
final val ENCODE = "encode"
final val UUID = "uuid"
+ final val BIT_LENGTH = "bit_length"
// URL functions
final val PARSE_URL = "parse_url"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
