This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 71d50f697 [GLUTEN-5203][VL] Support url_encode function (#5204)
71d50f697 is described below
commit 71d50f69773277aa0334f4f982f4ea307e1cc301
Author: Zhen Wang <[email protected]>
AuthorDate: Mon Apr 1 09:55:19 2024 +0800
[GLUTEN-5203][VL] Support url_encode function (#5204)
---
.../scala/io/glutenproject/utils/CHExpressionUtil.scala | 1 +
.../execution/ScalarFunctionsValidateSuite.scala | 15 +++++++++++++++
.../io/glutenproject/expression/ExpressionConverter.scala | 6 ++++++
.../io/glutenproject/expression/ExpressionNames.scala | 1 +
4 files changed, 23 insertions(+)
diff --git
a/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala
b/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala
index d431d0c87..cf90dad61 100644
---
a/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala
+++
b/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala
@@ -178,6 +178,7 @@ object CHExpressionUtil {
MONOTONICALLY_INCREASING_ID -> DefaultValidator(),
SPARK_PARTITION_ID -> DefaultValidator(),
URL_DECODE -> DefaultValidator(),
+ URL_ENCODE -> DefaultValidator(),
SKEWNESS -> DefaultValidator(),
BIT_LENGTH -> DefaultValidator(),
MAKE_YM_INTERVAL -> DefaultValidator(),
diff --git
a/backends-velox/src/test/scala/io/glutenproject/execution/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/io/glutenproject/execution/ScalarFunctionsValidateSuite.scala
index e91d64c6e..44169ac6d 100644
---
a/backends-velox/src/test/scala/io/glutenproject/execution/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/io/glutenproject/execution/ScalarFunctionsValidateSuite.scala
@@ -501,6 +501,21 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateTest {
}
}
+ testWithSpecifiedSparkVersion("Test url_encode function", Some("3.4.2")) {
+ withTempPath {
+ path =>
+ Seq("https://spark.apache.org";)
+ .toDF("a")
+ .write
+ .parquet(path.getCanonicalPath)
+ spark.sparkContext.setLogLevel("info")
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("url_tbl")
+ runQueryAndCompare("select url_encode(a) from url_tbl") {
+ checkOperatorMatch[ProjectExecTransformer]
+ }
+ }
+ }
+
test("Test hex function") {
runQueryAndCompare("SELECT hex(l_partkey), hex(l_shipmode) FROM lineitem
limit 1") {
checkOperatorMatch[ProjectExecTransformer]
diff --git
a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala
b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala
index 4de0ab142..dc86ecb52 100644
---
a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala
@@ -127,6 +127,12 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
child.map(
replaceWithExpressionTransformerInternal(_, attributeSeq,
expressionsMap)),
i)
+ case "encode" =>
+ return GenericExpressionTransformer(
+ ExpressionNames.URL_ENCODE,
+ child.map(
+ replaceWithExpressionTransformerInternal(_, attributeSeq,
expressionsMap)),
+ i)
}
}
case _ =>
diff --git
a/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala
b/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala
index 8717be50d..64b7c5f1f 100644
---
a/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala
+++
b/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala
@@ -120,6 +120,7 @@ object ExpressionNames {
// URL functions
final val PARSE_URL = "parse_url"
final val URL_DECODE = "url_decode"
+ final val URL_ENCODE = "url_encode"
// SparkSQL Math functions
final val ABS = "abs"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
