This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 07277bc20f [CORE] Fix fallback for spark literal unsafe map data as
input (#9908)
07277bc20f is described below
commit 07277bc20f8290c32e1be67eb7d8fe0455d2258d
Author: Zouxxyy <[email protected]>
AuthorDate: Tue Jun 10 13:14:05 2025 +0800
[CORE] Fix fallback for spark literal unsafe map data as input (#9908)
---
.../functions/ScalarFunctionsValidateSuite.scala | 9 ++++
.../substrait/expression/ExpressionBuilder.java | 15 -------
.../substrait/expression/ListLiteralNode.java | 9 ++--
.../substrait/expression/MapLiteralNode.java | 22 ++++++----
.../gluten/substrait/type/BinaryTypeNode.java | 5 +++
.../gluten/substrait/type/BooleanTypeNode.java | 5 +++
.../apache/gluten/substrait/type/DateTypeNode.java | 5 +++
.../gluten/substrait/type/DecimalTypeNode.java | 5 +++
.../apache/gluten/substrait/type/FP32TypeNode.java | 5 +++
.../apache/gluten/substrait/type/FP64TypeNode.java | 5 +++
.../gluten/substrait/type/FixedBinaryTypeNode.java | 5 +++
.../gluten/substrait/type/FixedCharTypeNode.java | 5 +++
.../apache/gluten/substrait/type/I16TypeNode.java | 5 +++
.../apache/gluten/substrait/type/I32TypeNode.java | 5 +++
.../apache/gluten/substrait/type/I64TypeNode.java | 5 +++
.../apache/gluten/substrait/type/I8TypeNode.java | 5 +++
.../substrait/type/IntervalYearTypeNode.java | 5 +++
.../org/apache/gluten/substrait/type/ListNode.java | 5 +++
.../org/apache/gluten/substrait/type/MapNode.java | 5 +++
.../apache/gluten/substrait/type/NothingNode.java | 5 +++
.../gluten/substrait/type/StringTypeNode.java | 5 +++
.../apache/gluten/substrait/type/StructNode.java | 5 +++
.../gluten/substrait/type/TimestampTypeNode.java | 5 +++
.../org/apache/gluten/substrait/type/TypeNode.java | 2 +
.../apache/gluten/expression/ConverterUtils.scala | 50 ++++++++++++++++++++++
25 files changed, 176 insertions(+), 26 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
index 8a307e13bc..58a17bc9d9 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
@@ -511,6 +511,15 @@ abstract class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
+ test("map_from_arrays optimized by Spark constant folding") {
+ withSQLConf(("spark.sql.optimizer.excludedRules", "")) {
+ runQueryAndCompare("""SELECT map_from_arrays(sequence(1, 5),sequence(1,
5)), l_orderkey
+ | from lineitem limit 10""".stripMargin) {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+ }
+
test("raise_error, assert_true") {
runQueryAndCompare("""SELECT assert_true(l_orderkey >= 1), l_orderkey
| from lineitem limit 100""".stripMargin) {
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java
index 07c73452bd..4c456e2142 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java
@@ -23,9 +23,7 @@ import org.apache.gluten.substrait.type.*;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.catalyst.expressions.Attribute;
import org.apache.spark.sql.catalyst.expressions.Expression;
-import org.apache.spark.sql.catalyst.expressions.UnsafeArrayData;
import org.apache.spark.sql.catalyst.util.ArrayData;
-import org.apache.spark.sql.catalyst.util.GenericArrayData;
import org.apache.spark.sql.catalyst.util.MapData;
import org.apache.spark.sql.types.*;
@@ -207,19 +205,6 @@ public class ExpressionBuilder {
public static LiteralNode makeLiteral(Object obj, DataType dataType, Boolean
nullable) {
TypeNode typeNode = ConverterUtils.getTypeNode(dataType, nullable);
- if (obj instanceof UnsafeArrayData) {
- UnsafeArrayData oldObj = (UnsafeArrayData) obj;
- int numElements = oldObj.numElements();
- Object[] elements = new Object[numElements];
- DataType elementType = ((ArrayType) dataType).elementType();
-
- for (int i = 0; i < numElements; i++) {
- elements[i] = oldObj.get(i, elementType);
- }
-
- GenericArrayData newObj = new GenericArrayData(elements);
- return makeListLiteral(newObj, typeNode);
- }
return makeLiteral(obj, typeNode);
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ListLiteralNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ListLiteralNode.java
index 13d0c79483..c5b8f937fd 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ListLiteralNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ListLiteralNode.java
@@ -16,6 +16,7 @@
*/
package org.apache.gluten.substrait.expression;
+import org.apache.gluten.expression.ConverterUtils;
import org.apache.gluten.substrait.type.ListNode;
import org.apache.gluten.substrait.type.TypeNode;
@@ -23,6 +24,7 @@ import io.substrait.proto.Expression;
import io.substrait.proto.Expression.Literal.Builder;
import io.substrait.proto.Type;
import org.apache.spark.sql.catalyst.util.ArrayData;
+import org.apache.spark.sql.types.DataType;
public class ListLiteralNode extends LiteralNodeWithValue<ArrayData> {
public ListLiteralNode(ArrayData array, TypeNode typeNode) {
@@ -31,12 +33,13 @@ public class ListLiteralNode extends
LiteralNodeWithValue<ArrayData> {
@Override
protected void updateLiteralBuilder(Builder literalBuilder, ArrayData array)
{
- Object[] elements = array.array();
TypeNode elementType = ((ListNode) getTypeNode()).getNestedType();
+ DataType elementDataType = ConverterUtils.parseFromTypeNode(elementType);
- if (elements.length > 0) {
+ if (array.numElements() > 0) {
Expression.Literal.List.Builder listBuilder =
Expression.Literal.List.newBuilder();
- for (Object element : elements) {
+ for (int i = 0; i < array.numElements(); i++) {
+ Object element = array.get(i, elementDataType);
LiteralNode elementNode = ExpressionBuilder.makeLiteral(element,
elementType);
Expression.Literal elementExpr = elementNode.getLiteral();
listBuilder.addValues(elementExpr);
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/MapLiteralNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/MapLiteralNode.java
index ea0b8bee3e..211f0f9d1c 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/MapLiteralNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/MapLiteralNode.java
@@ -16,13 +16,16 @@
*/
package org.apache.gluten.substrait.expression;
+import org.apache.gluten.expression.ConverterUtils;
import org.apache.gluten.substrait.type.MapNode;
import org.apache.gluten.substrait.type.TypeNode;
import io.substrait.proto.Expression;
import io.substrait.proto.Expression.Literal.Builder;
import io.substrait.proto.Type;
+import org.apache.spark.sql.catalyst.util.ArrayData;
import org.apache.spark.sql.catalyst.util.MapData;
+import org.apache.spark.sql.types.DataType;
public class MapLiteralNode extends LiteralNodeWithValue<MapData> {
public MapLiteralNode(MapData map, TypeNode typeNode) {
@@ -31,16 +34,19 @@ public class MapLiteralNode extends
LiteralNodeWithValue<MapData> {
@Override
protected void updateLiteralBuilder(Builder literalBuilder, MapData map) {
- Object[] keys = map.keyArray().array();
- Object[] values = map.valueArray().array();
- TypeNode mapType = ((MapNode) getTypeNode()).getKeyType();
+ ArrayData keys = map.keyArray();
+ ArrayData values = map.valueArray();
+ TypeNode keyType = ((MapNode) getTypeNode()).getKeyType();
TypeNode valueType = ((MapNode) getTypeNode()).getValueType();
+ DataType keyDataType = ConverterUtils.parseFromTypeNode(keyType);
+ DataType valueDataType = ConverterUtils.parseFromTypeNode(valueType);
- if (keys.length > 0) {
+ if (keys.numElements() > 0) {
Expression.Literal.Map.Builder mapBuilder =
Expression.Literal.Map.newBuilder();
- for (int i = 0; i < keys.length; ++i) {
- LiteralNode keyNode = ExpressionBuilder.makeLiteral(keys[i], mapType);
- LiteralNode valueNode = ExpressionBuilder.makeLiteral(values[i],
valueType);
+ for (int i = 0; i < keys.numElements(); ++i) {
+ LiteralNode keyNode = ExpressionBuilder.makeLiteral(keys.get(i,
keyDataType), keyType);
+ LiteralNode valueNode =
+ ExpressionBuilder.makeLiteral(values.get(i, valueDataType),
valueType);
Expression.Literal.Map.KeyValue.Builder kvBuilder =
Expression.Literal.Map.KeyValue.newBuilder();
@@ -52,7 +58,7 @@ public class MapLiteralNode extends
LiteralNodeWithValue<MapData> {
literalBuilder.setMap(mapBuilder.build());
} else {
Type.Map.Builder mapTypeBuilder = Type.Map.newBuilder();
- mapTypeBuilder.setKey(mapType.toProtobuf());
+ mapTypeBuilder.setKey(keyType.toProtobuf());
mapTypeBuilder.setValue(valueType.toProtobuf());
literalBuilder.setEmptyMap(mapTypeBuilder.build());
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/BinaryTypeNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/BinaryTypeNode.java
index bbd5cd1541..82770eea88 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/BinaryTypeNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/BinaryTypeNode.java
@@ -40,4 +40,9 @@ public class BinaryTypeNode implements TypeNode, Serializable
{
builder.setBinary(binaryBuilder.build());
return builder.build();
}
+
+ @Override
+ public Boolean nullable() {
+ return nullable;
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/BooleanTypeNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/BooleanTypeNode.java
index 6ae3bd50e0..531da33502 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/BooleanTypeNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/BooleanTypeNode.java
@@ -40,4 +40,9 @@ public class BooleanTypeNode implements TypeNode,
Serializable {
builder.setBool(booleanBuilder.build());
return builder.build();
}
+
+ @Override
+ public Boolean nullable() {
+ return nullable;
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/DateTypeNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/DateTypeNode.java
index 7ae6b9602c..0438abc781 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/DateTypeNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/DateTypeNode.java
@@ -39,4 +39,9 @@ public class DateTypeNode implements TypeNode, Serializable {
builder.setDate(dateBuilder.build());
return builder.build();
}
+
+ @Override
+ public Boolean nullable() {
+ return nullable;
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/DecimalTypeNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/DecimalTypeNode.java
index 57f983fd59..59c810b0ef 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/DecimalTypeNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/DecimalTypeNode.java
@@ -46,4 +46,9 @@ public class DecimalTypeNode implements TypeNode,
Serializable {
builder.setDecimal(decimalBuilder.build());
return builder.build();
}
+
+ @Override
+ public Boolean nullable() {
+ return nullable;
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/FP32TypeNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/FP32TypeNode.java
index c51b410968..76d72ce658 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/FP32TypeNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/FP32TypeNode.java
@@ -40,4 +40,9 @@ public class FP32TypeNode implements TypeNode, Serializable {
builder.setFp32(doubleBuilder.build());
return builder.build();
}
+
+ @Override
+ public Boolean nullable() {
+ return nullable;
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/FP64TypeNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/FP64TypeNode.java
index 39bd7f6e8a..21abe07570 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/FP64TypeNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/FP64TypeNode.java
@@ -40,4 +40,9 @@ public class FP64TypeNode implements TypeNode, Serializable {
builder.setFp64(doubleBuilder.build());
return builder.build();
}
+
+ @Override
+ public Boolean nullable() {
+ return nullable;
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/FixedBinaryTypeNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/FixedBinaryTypeNode.java
index 86889eee3d..7fa6b7ef81 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/FixedBinaryTypeNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/FixedBinaryTypeNode.java
@@ -43,4 +43,9 @@ public class FixedBinaryTypeNode implements TypeNode,
Serializable {
builder.setFixedBinary(fixedBinaryBuilder.build());
return builder.build();
}
+
+ @Override
+ public Boolean nullable() {
+ return nullable;
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/FixedCharTypeNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/FixedCharTypeNode.java
index 1e671651bf..710aa05566 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/FixedCharTypeNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/FixedCharTypeNode.java
@@ -43,4 +43,9 @@ public class FixedCharTypeNode implements TypeNode,
Serializable {
builder.setFixedChar(fixedCharBuilder.build());
return builder.build();
}
+
+ @Override
+ public Boolean nullable() {
+ return nullable;
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/I16TypeNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/I16TypeNode.java
index 7acaf9c6bf..d6a9dc016e 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/I16TypeNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/I16TypeNode.java
@@ -40,4 +40,9 @@ public class I16TypeNode implements TypeNode, Serializable {
builder.setI16(i16Builder.build());
return builder.build();
}
+
+ @Override
+ public Boolean nullable() {
+ return nullable;
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/I32TypeNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/I32TypeNode.java
index d8ae29d539..85ae2ec96c 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/I32TypeNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/I32TypeNode.java
@@ -40,4 +40,9 @@ public class I32TypeNode implements TypeNode, Serializable {
builder.setI32(i32Builder.build());
return builder.build();
}
+
+ @Override
+ public Boolean nullable() {
+ return nullable;
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/I64TypeNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/I64TypeNode.java
index 0042772fd9..57248915a5 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/I64TypeNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/I64TypeNode.java
@@ -40,4 +40,9 @@ public class I64TypeNode implements TypeNode, Serializable {
builder.setI64(i64Builder.build());
return builder.build();
}
+
+ @Override
+ public Boolean nullable() {
+ return nullable;
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/I8TypeNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/I8TypeNode.java
index a3de677ee0..53e69e2ae8 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/I8TypeNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/I8TypeNode.java
@@ -40,4 +40,9 @@ public class I8TypeNode implements TypeNode, Serializable {
builder.setI8(i8Builder.build());
return builder.build();
}
+
+ @Override
+ public Boolean nullable() {
+ return nullable;
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/IntervalYearTypeNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/IntervalYearTypeNode.java
index 384f9e7736..532a2ec3ba 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/IntervalYearTypeNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/IntervalYearTypeNode.java
@@ -40,4 +40,9 @@ public class IntervalYearTypeNode implements TypeNode,
Serializable {
builder.setIntervalYear(intervalYearBuilder.build());
return builder.build();
}
+
+ @Override
+ public Boolean nullable() {
+ return nullable;
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/ListNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/ListNode.java
index 4713e0daa8..4676d69a3b 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/ListNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/ListNode.java
@@ -45,4 +45,9 @@ public class ListNode implements TypeNode, Serializable {
builder.setList(listBuilder.build());
return builder.build();
}
+
+ @Override
+ public Boolean nullable() {
+ return nullable;
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/MapNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/MapNode.java
index bdef11eede..0514c39646 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/MapNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/MapNode.java
@@ -61,4 +61,9 @@ public class MapNode implements TypeNode, Serializable {
builder.setMap(mapBuilder.build());
return builder.build();
}
+
+ @Override
+ public Boolean nullable() {
+ return nullable;
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/NothingNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/NothingNode.java
index 7aa9953cb9..b79994ddae 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/NothingNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/NothingNode.java
@@ -30,4 +30,9 @@ public class NothingNode implements TypeNode, Serializable {
builder.setNothing(nothingBuilder.build());
return builder.build();
}
+
+ @Override
+ public Boolean nullable() {
+ return true;
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/StringTypeNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/StringTypeNode.java
index 83e029e731..68e43c06c6 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/StringTypeNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/StringTypeNode.java
@@ -40,4 +40,9 @@ public class StringTypeNode implements TypeNode, Serializable
{
builder.setString(stringBuilder.build());
return builder.build();
}
+
+ @Override
+ public Boolean nullable() {
+ return nullable;
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/StructNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/StructNode.java
index 6faeb4d02b..c9d69e9463 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/StructNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/StructNode.java
@@ -57,4 +57,9 @@ public class StructNode implements TypeNode, Serializable {
builder.setStruct(structBuilder.build());
return builder.build();
}
+
+ @Override
+ public Boolean nullable() {
+ return nullable;
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/TimestampTypeNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/TimestampTypeNode.java
index 15919fcdee..5c63429cbd 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/TimestampTypeNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/TimestampTypeNode.java
@@ -40,4 +40,9 @@ public class TimestampTypeNode implements TypeNode,
Serializable {
builder.setTimestamp(timestampBuilder.build());
return builder.build();
}
+
+ @Override
+ public Boolean nullable() {
+ return nullable;
+ }
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/TypeNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/TypeNode.java
index a631122e0a..de251fa06a 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/TypeNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/TypeNode.java
@@ -20,4 +20,6 @@ import io.substrait.proto.Type;
public interface TypeNode {
Type toProtobuf();
+
+ Boolean nullable();
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ConverterUtils.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ConverterUtils.scala
index fa59e3be2a..4dce9858d4 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ConverterUtils.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ConverterUtils.scala
@@ -254,6 +254,56 @@ object ConverterUtils extends Logging {
}
}
+ def parseFromTypeNode(typeNode: TypeNode): DataType = {
+ typeNode match {
+ case _: BooleanTypeNode =>
+ BooleanType
+ case _: FP32TypeNode =>
+ FloatType
+ case _: FP64TypeNode =>
+ DoubleType
+ case _: I64TypeNode =>
+ LongType
+ case _: I32TypeNode =>
+ IntegerType
+ case _: I16TypeNode =>
+ ShortType
+ case _: I8TypeNode =>
+ ByteType
+ case _: StringTypeNode =>
+ StringType
+ case _: BinaryTypeNode =>
+ BinaryType
+ case _: DateTypeNode =>
+ DateType
+ case _: IntervalYearTypeNode =>
+ YearMonthIntervalType.DEFAULT
+ case d: DecimalTypeNode =>
+ DecimalType(d.precision, d.scale)
+ case _: TimestampTypeNode =>
+ TimestampType
+ case m: MapNode =>
+ val keyType = parseFromTypeNode(m.getKeyType)
+ val valueType = parseFromTypeNode(m.getValueType)
+ val valueContainsNull = m.getValueType.nullable()
+ MapType(keyType, valueType, valueContainsNull)
+ case a: ListNode =>
+ val elementType = parseFromTypeNode(a.getNestedType)
+ ArrayType(elementType, a.getNestedType.nullable())
+ case s: StructNode =>
+ val fields = s.getFieldTypes.asScala.map({
+ typ =>
+ val field = parseFromTypeNode(typ)
+ StructField("", field, typ.nullable())
+ })
+ StructType(fields.toSeq)
+ case _: NothingNode =>
+ NullType
+ case unknown =>
+ throw new GlutenNotSupportException(s"Type $unknown not supported.")
+ }
+ }
+
def parseFromBytes(bytes: Array[Byte]): ExpressionType = {
val input = CodedInputStream.newInstance(bytes)
val parsed = io.substrait.proto.Type.parseFrom(input)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
