This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new ed3d333c7 [GLUTEN-5936][VL] Add more types in function type validation
and document Cast function (#6963)
ed3d333c7 is described below
commit ed3d333c7c357e95f6736f378c972b714bdfb420
Author: PHILO-HE <[email protected]>
AuthorDate: Thu Aug 22 08:38:12 2024 +0800
[GLUTEN-5936][VL] Add more types in function type validation and document
Cast function (#6963)
---
docs/velox-backend-support-progress.md | 28 ++++++++++++++++++++++
.../sql/GlutenExpressionDataTypesValidation.scala | 6 ++++-
2 files changed, 33 insertions(+), 1 deletion(-)
diff --git a/docs/velox-backend-support-progress.md
b/docs/velox-backend-support-progress.md
index 3676ee4df..aa2e9c999 100644
--- a/docs/velox-backend-support-progress.md
+++ b/docs/velox-backend-support-progress.md
@@ -92,6 +92,34 @@ Gluten supports 28 operators (Drag to right to see all data
types)
Gluten supports 199 functions. (Drag to right to see all data types)
+#### Cast function's support status
+
+ * S: supported.
+ * NS: not supported.
+ * -: not accepted by Spark.
+ * N/A: not applicable case, e.g., from type is as same as to type, where cast
will not actually happen.
+
+| From \ To | BOOLEAN | BYTE | SHORT | INT | LONG | FLOAT | DOUBLE | DECIMAL |
DATE | TIMESTAMP | STRING | BINARY | ARRAY | MAP | STRUCT | NULL |
+|-----------|---------|------|-------|-----|------|-------|--------|---------|------|-----------|--------|--------|-------|-----|--------|------|
+| BOOLEAN | N/A | S | S | S | S | S | S | S |
- | NS | S | - | - | - | - | - |
+| BYTE | S | N/A | S | S | S | S | S | S |
- | NS | S | S | - | - | - | - |
+| SHORT | S | S | N/A | S | S | S | S | S |
- | NS | S | S | - | - | - | - |
+| INT | S | S | S | N/A | S | S | S | S |
- | NS | S | S | - | - | - | - |
+| LONG | S | S | S | S | N/A | S | S | S |
- | NS | S | S | - | - | - | - |
+| FLOAT | S | S | S | S | S | N/A | S | S |
- | NS | S | - | - | - | - | - |
+| DOUBLE | S | S | S | S | S | S | N/A | S |
- | NS | S | - | - | - | - | - |
+| DECIMAL | S | S | S | S | S | S | S | N/A |
- | NS | S | - | - | - | - | - |
+| DATE | NS | NS | NS | NS | NS | NS | NS | NS |
N/A | NS | NS | - | - | - | - | - |
+| TIMESTAMP | NS | NS | NS | NS | NS | NS | NS | NS |
NS | N/A | NS | - | - | - | - | - |
+| STRING | S | S | S | S | S | S | S | S |
NS | NS | N/A | - | - | - | - | - |
+| BINARY | S | S | S | S | S | S | S | S |
NS | NS | S | N/A | - | - | - | - |
+| ARRAY | - | - | - | - | - | - | - | - |
- | - | NS | - | N/A | - | - | - |
+| Map | - | - | - | - | - | - | - | - |
- | - | NS | - | - | N/A | - | - |
+| STRUCT | - | - | - | - | - | - | - | - |
- | - | NS | - | - | - | N/A | - |
+| NULL | S | S | S | S | S | S | S | S |
S | NS | S | S | S | S | S | N/A |
+
+#### Other functions' support status
+
| Spark Functions | Velox/Presto Functions | Velox/Spark
functions | Gluten | Restrictions | BOOLEAN | BYTE | SHORT | INT |
LONG | FLOAT | DOUBLE | DATE | TIMESTAMP | STRING | DECIMAL | NULL | BINARY |
CALENDAR | ARRAY | MAP | STRUCT | UDT |
|------------------------------|------------------------|-----------------------|--------|--------------------------|---------|------|-------|-----|------|-------|--------|------|-----------|--------|---------|------|--------|----------|-------|-----|--------|-----|
| ! | | not
| S | | S | S | S | S | S | S
| S | | | S | | | | |
| | | |
diff --git
a/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenExpressionDataTypesValidation.scala
b/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenExpressionDataTypesValidation.scala
index 170555e5f..d2a961147 100644
---
a/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenExpressionDataTypesValidation.scala
+++
b/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenExpressionDataTypesValidation.scala
@@ -63,6 +63,7 @@ class GlutenExpressionDataTypesValidation extends
WholeStageTransformerSuite {
private val allPrimitiveDataTypes: Seq[DataType] =
Seq(
+ BooleanType,
ByteType,
ShortType,
IntegerType,
@@ -73,7 +74,8 @@ class GlutenExpressionDataTypesValidation extends
WholeStageTransformerSuite {
StringType,
BinaryType,
DateType,
- TimestampType)
+ TimestampType,
+ NullType)
private val allComplexDataTypes: Seq[DataType] = Seq(
// Currently, only check certain inner types, assuming they are
representative
@@ -85,6 +87,7 @@ class GlutenExpressionDataTypesValidation extends
WholeStageTransformerSuite {
def generateChildExpression(t: DataType): Expression = {
t match {
+ case _: BooleanType => Literal(true, t)
case _: IntegralType => Literal(null, t)
case _: FractionalType => Literal(null, t)
case StringType | BinaryType => Literal("123")
@@ -93,6 +96,7 @@ class GlutenExpressionDataTypesValidation extends
WholeStageTransformerSuite {
case ArrayType(_, _) => Literal(null, t)
case MapType(_, _, _) => Literal(null, t)
case StructType(_) => Literal(null, t)
+ case NullType => Literal(null, t)
case _ => throw new UnsupportedOperationException("Not supported type: "
+ t)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
