This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 42c6be838 [VL] Support uuid function (#5014)
42c6be838 is described below
commit 42c6be8387d041f7864171f88bfdf8f0fa92cc76
Author: Zhen Li <[email protected]>
AuthorDate: Thu Mar 21 21:31:30 2024 +0800
[VL] Support uuid function (#5014)
---
.../backendsapi/velox/SparkPlanExecApiImpl.scala | 12 +++++++++++-
.../io/glutenproject/backendsapi/velox/VeloxBackend.scala | 3 ++-
.../execution/VeloxFunctionsValidateSuite.scala | 6 ++++++
docs/velox-backend-support-progress.md | 1 +
.../io/glutenproject/backendsapi/SparkPlanExecApi.scala | 4 ++++
.../io/glutenproject/expression/ExpressionConverter.scala | 2 ++
6 files changed, 26 insertions(+), 2 deletions(-)
diff --git
a/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/SparkPlanExecApiImpl.scala
b/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/SparkPlanExecApiImpl.scala
index 64980209c..e7d2b2225 100644
---
a/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/SparkPlanExecApiImpl.scala
+++
b/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/SparkPlanExecApiImpl.scala
@@ -37,7 +37,7 @@ import
org.apache.spark.sql.catalyst.{AggregateFunctionRewriteRule, FlushableHas
import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder
import org.apache.spark.sql.catalyst.catalog.BucketSpec
import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
-import org.apache.spark.sql.catalyst.expressions.{Alias, Ascending, Attribute,
Cast, CreateNamedStruct, ElementAt, Expression, ExpressionInfo, Generator,
GetArrayItem, GetMapValue, GetStructField, If, IsNaN, Literal, Murmur3Hash,
NamedExpression, NaNvl, PosExplode, Round, SortOrder, StringSplit, StringTrim}
+import org.apache.spark.sql.catalyst.expressions.{Alias, Ascending, Attribute,
Cast, CreateNamedStruct, ElementAt, Expression, ExpressionInfo, Generator,
GetArrayItem, GetMapValue, GetStructField, If, IsNaN, Literal, Murmur3Hash,
NamedExpression, NaNvl, PosExplode, Round, SortOrder, StringSplit, StringTrim,
Uuid}
import
org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression,
HLLAdapter}
import org.apache.spark.sql.catalyst.optimizer.BuildSide
import org.apache.spark.sql.catalyst.plans.JoinType
@@ -129,6 +129,16 @@ class SparkPlanExecApiImpl extends SparkPlanExecApi {
)
}
+ /** Transform Uuid to Substrait. */
+ override def genUuidTransformer(
+ substraitExprName: String,
+ original: Uuid): ExpressionTransformer = {
+ GenericExpressionTransformer(
+ substraitExprName,
+ Seq(LiteralTransformer(Literal(original.randomSeed.get))),
+ original)
+ }
+
/** Transform map_entries to Substrait. */
override def genMapEntriesTransformer(
substraitExprName: String,
diff --git
a/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala
b/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala
index 6201a372d..0ff2bd0d7 100644
---
a/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala
+++
b/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala
@@ -27,7 +27,7 @@ import
io.glutenproject.substrait.rel.LocalFilesNode.ReadFileFormat
import
io.glutenproject.substrait.rel.LocalFilesNode.ReadFileFormat.{DwrfReadFormat,
OrcReadFormat, ParquetReadFormat}
import org.apache.spark.sql.catalyst.catalog.BucketSpec
-import org.apache.spark.sql.catalyst.expressions.{Alias, CumeDist, DenseRank,
Descending, Expression, Lag, Lead, Literal, NamedExpression, NthValue, NTile,
PercentRank, Rand, RangeFrame, Rank, RowNumber, SortOrder,
SpecialFrameBoundary, SpecifiedWindowFrame}
+import org.apache.spark.sql.catalyst.expressions.{Alias, CumeDist, DenseRank,
Descending, Expression, Lag, Lead, Literal, NamedExpression, NthValue, NTile,
PercentRank, Rand, RangeFrame, Rank, RowNumber, SortOrder,
SpecialFrameBoundary, SpecifiedWindowFrame, Uuid}
import
org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression,
Count, Sum}
import org.apache.spark.sql.catalyst.plans.JoinType
import org.apache.spark.sql.catalyst.util.CharVarcharUtils
@@ -388,6 +388,7 @@ object BackendSettings extends BackendSettingsApi {
// Block directly falling back the below functions by
FallbackEmptySchemaRelation.
case alias: Alias => checkExpr(alias.child)
case _: Rand => true
+ case _: Uuid => true
case _ => false
}
}
diff --git
a/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala
index e5bee9761..f7f1dc90e 100644
---
a/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala
@@ -513,6 +513,12 @@ class VeloxFunctionsValidateSuite extends
VeloxWholeStageTransformerSuite {
}
}
+ test("Test uuid function") {
+ runQueryAndCompare("""SELECT uuid() from lineitem limit
100""".stripMargin, false) {
+ checkOperatorMatch[ProjectExecTransformer]
+ }
+ }
+
test("regexp_replace") {
runQueryAndCompare(
"SELECT regexp_replace(l_partkey, '\\w', 'something') FROM lineitem
limit 100") {
diff --git a/docs/velox-backend-support-progress.md
b/docs/velox-backend-support-progress.md
index e9ba9d1f2..56cb95c1b 100644
--- a/docs/velox-backend-support-progress.md
+++ b/docs/velox-backend-support-progress.md
@@ -428,3 +428,4 @@ Gluten supports 199 functions. (Draw to right to see all
data types)
| spark_partition_id | |
| S | | | | | | |
| | | | | | | | |
| | | |
| stack | |
| | | | | | | |
| | | | | | | | |
| | | |
| xxhash64 | xxhash64 | xxhash64
| | | | | | | |
| | | | | | | | |
| | | |
+| uuid | uuid | uuid | S
| | | | | | | |
| | | | | | | | |
| | |
diff --git
a/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala
b/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala
index 4f40a8a23..4166c4a68 100644
---
a/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala
@@ -192,6 +192,10 @@ trait SparkPlanExecApi {
throw new GlutenNotSupportException("NaNvl is not supported")
}
+ def genUuidTransformer(substraitExprName: String, original: Uuid):
ExpressionTransformer = {
+ GenericExpressionTransformer(substraitExprName, Seq(), original)
+ }
+
/** Transform map_entries to Substrait. */
def genMapEntriesTransformer(
substraitExprName: String,
diff --git
a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala
b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala
index 693f69326..f8f3ead05 100644
---
a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala
@@ -554,6 +554,8 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
val childrenTransformers =
e.children.map(replaceWithExpressionTransformerInternal(_,
attributeSeq, expressionsMap))
e.getTransformer(childrenTransformers)
+ case u: Uuid =>
+
BackendsApiManager.getSparkPlanExecApiInstance.genUuidTransformer(substraitExprName,
u)
case expr =>
GenericExpressionTransformer(
substraitExprName,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
