This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 353b91534 [CORE] Introduce GlutenNotSupportException for expected
fallback behavior (#4996)
353b91534 is described below
commit 353b91534d99fa96b7e0def0a8f5d5d9ed009ad9
Author: PHILO-HE <[email protected]>
AuthorDate: Tue Mar 19 13:03:40 2024 +0800
[CORE] Introduce GlutenNotSupportException for expected fallback behavior
(#4996)
---
.../clickhouse/CHSparkPlanExecApi.scala | 13 ++++----
.../execution/CHHashAggregateExecTransformer.scala | 7 ++--
.../expression/CHExpressionTransformer.scala | 16 ++++-----
.../backendsapi/velox/SparkPlanExecApiImpl.scala | 5 +--
.../backendsapi/velox/VeloxBackend.scala | 13 ++++----
.../execution/HashAggregateExecTransformer.scala | 23 +++++++------
.../execution/RowToVeloxColumnarExec.scala | 3 +-
.../execution/VeloxColumnarToRowExec.scala | 3 +-
.../expression/ExpressionTransformer.scala | 7 ++--
.../exception/GlutenNotSupportException.java | 39 ++++++++++++++++++++++
.../main/scala/io/glutenproject/GlutenPlugin.scala | 3 +-
.../backendsapi/SparkPlanExecApi.scala | 14 ++++----
.../BasicPhysicalOperatorTransformer.scala | 5 +--
.../execution/BatchScanExecTransformer.scala | 3 +-
.../HashAggregateExecBaseTransformer.scala | 5 +--
.../execution/ScanTransformerFactory.scala | 5 +--
.../expression/AggregateFunctionsBuilder.scala | 3 +-
.../expression/ArrayExpressionTransformer.scala | 3 +-
.../glutenproject/expression/ConverterUtils.scala | 11 +++---
.../DateTimeExpressionsTransformer.scala | 3 +-
.../expression/DecimalRoundTransformer.scala | 3 +-
.../expression/ExpressionConverter.scala | 15 +++++----
.../expression/LambdaFunctionTransformer.scala | 3 +-
.../expression/MapExpressionTransformer.scala | 7 ++--
.../expression/StringExpressionTransformer.scala | 3 +-
.../expression/UnaryExpressionTransformer.scala | 9 ++---
.../expression/WindowFunctionsBuilder.scala | 3 +-
.../extension/ExpressionExtensionTrait.scala | 3 +-
.../extension/columnar/MiscColumnarRules.scala | 3 +-
.../extension/columnar/TransformHintRule.scala | 8 +++--
.../io/glutenproject/substrait/TypeConverter.scala | 4 ++-
.../utils/DecimalArithmeticUtil.scala | 5 +--
.../glutenproject/utils/PullOutProjectHelper.scala | 4 ++-
33 files changed, 163 insertions(+), 91 deletions(-)
diff --git
a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHSparkPlanExecApi.scala
b/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHSparkPlanExecApi.scala
index 50dd1523f..9bd1f938a 100644
---
a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHSparkPlanExecApi.scala
+++
b/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHSparkPlanExecApi.scala
@@ -18,6 +18,7 @@ package io.glutenproject.backendsapi.clickhouse
import io.glutenproject.GlutenConfig
import io.glutenproject.backendsapi.{BackendsApiManager, SparkPlanExecApi}
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.execution._
import io.glutenproject.expression._
import io.glutenproject.expression.ConverterUtils.FunctionConfig
@@ -321,7 +322,7 @@ class CHSparkPlanExecApi extends SparkPlanExecApi {
left: SparkPlan,
right: SparkPlan,
condition: Option[Expression]): CartesianProductExecTransformer =
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
"CartesianProductExecTransformer is not supported in ch backend.")
override def genBroadcastNestedLoopJoinExecTransformer(
@@ -330,7 +331,7 @@ class CHSparkPlanExecApi extends SparkPlanExecApi {
buildSide: BuildSide,
joinType: JoinType,
condition: Option[Expression]): BroadcastNestedLoopJoinExecTransformer =
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
"BroadcastNestedLoopJoinExecTransformer is not supported in ch backend.")
/** Generate an expression transformer to transform GetMapValue to
Substrait. */
@@ -457,7 +458,7 @@ class CHSparkPlanExecApi extends SparkPlanExecApi {
case union: ColumnarUnionExec =>
wrapChild(union)
case other =>
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
s"Not supported operator ${other.nodeName} for
BroadcastRelation")
}
(newChild, (child.output ++ appendedProjections).map(_.toAttribute),
preProjectionBuildKeys)
@@ -587,7 +588,7 @@ class CHSparkPlanExecApi extends SparkPlanExecApi {
bucketSpec: Option[BucketSpec],
options: Map[String, String],
staticPartitions: TablePartitionSpec): WriteFilesExec = {
- throw new UnsupportedOperationException("ColumnarWriteFilesExec is not
support in ch backend.")
+ throw new GlutenNotSupportException("ColumnarWriteFilesExec is not support
in ch backend.")
}
/**
@@ -633,7 +634,7 @@ class CHSparkPlanExecApi extends SparkPlanExecApi {
val aggregateFunc = aggExpression.aggregateFunction
val substraitAggFuncName =
ExpressionMappings.expressionsMap.get(aggregateFunc.getClass)
if (substraitAggFuncName.isEmpty) {
- throw new UnsupportedOperationException(s"Not currently
supported: $aggregateFunc.")
+ throw new GlutenNotSupportException(s"Not currently supported:
$aggregateFunc.")
}
val childrenNodeList = new JArrayList[ExpressionNode]()
@@ -704,7 +705,7 @@ class CHSparkPlanExecApi extends SparkPlanExecApi {
)
windowExpressionNodes.add(windowFunctionNode)
case _ =>
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
"unsupported window function type: " +
wExpression.windowFunction)
}
diff --git
a/backends-clickhouse/src/main/scala/io/glutenproject/execution/CHHashAggregateExecTransformer.scala
b/backends-clickhouse/src/main/scala/io/glutenproject/execution/CHHashAggregateExecTransformer.scala
index 85b967446..13d02ffa7 100644
---
a/backends-clickhouse/src/main/scala/io/glutenproject/execution/CHHashAggregateExecTransformer.scala
+++
b/backends-clickhouse/src/main/scala/io/glutenproject/execution/CHHashAggregateExecTransformer.scala
@@ -17,6 +17,7 @@
package io.glutenproject.execution
import io.glutenproject.backendsapi.BackendsApiManager
+import io.glutenproject.exception.GlutenNotSupportException
import
io.glutenproject.execution.CHHashAggregateExecTransformer.getAggregateResultAttributes
import io.glutenproject.expression._
import io.glutenproject.substrait.`type`.{TypeBuilder, TypeNode}
@@ -224,7 +225,7 @@ case class CHHashAggregateExecTransformer(
// so far, it only happens in a three-stage count distinct case
// e.g. select sum(a), count(distinct b) from f
if (!child.isInstanceOf[BaseAggregateExec]) {
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
"PartialMerge's child not being
HashAggregateExecBaseTransformer" +
" is unsupported yet")
}
@@ -242,7 +243,7 @@ case class CHHashAggregateExecTransformer(
.replaceWithExpressionTransformer(aggExpr.resultAttribute,
originalInputAttributes)
.doTransform(args))
case other =>
- throw new UnsupportedOperationException(s"$other not supported.")
+ throw new GlutenNotSupportException(s"$other not supported.")
}
for (node <- childrenNodes) {
childrenNodeList.add(node)
@@ -457,7 +458,7 @@ case class CHHashAggregateExecPullOutHelper(
resIndex += 1
resIndex
case other =>
- throw new UnsupportedOperationException(s"Unsupported aggregate
mode: $other.")
+ throw new GlutenNotSupportException(s"Unsupported aggregate mode:
$other.")
}
}
}
diff --git
a/backends-clickhouse/src/main/scala/io/glutenproject/expression/CHExpressionTransformer.scala
b/backends-clickhouse/src/main/scala/io/glutenproject/expression/CHExpressionTransformer.scala
index c213ddd12..bb8b704d9 100644
---
a/backends-clickhouse/src/main/scala/io/glutenproject/expression/CHExpressionTransformer.scala
+++
b/backends-clickhouse/src/main/scala/io/glutenproject/expression/CHExpressionTransformer.scala
@@ -17,6 +17,7 @@
package io.glutenproject.expression
import io.glutenproject.backendsapi.clickhouse.CHBackendSettings
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.expression.ConverterUtils.FunctionConfig
import io.glutenproject.substrait.expression._
@@ -55,13 +56,12 @@ case class CHTruncTimestampTransformer(
override def doTransform(args: java.lang.Object): ExpressionNode = {
// The format must be constant string in the function date_trunc of ch.
if (!original.format.foldable) {
- throw new UnsupportedOperationException(
- s"The format ${original.format} must be constant string.")
+ throw new GlutenNotSupportException(s"The format ${original.format} must
be constant string.")
}
val formatStr = original.format.eval().asInstanceOf[UTF8String]
if (formatStr == null) {
- throw new UnsupportedOperationException("The format is null.")
+ throw new GlutenNotSupportException("The format is null.")
}
val (newFormatStr, timeZoneIgnore) =
formatStr.toString.toLowerCase(Locale.ROOT) match {
@@ -76,7 +76,7 @@ case class CHTruncTimestampTransformer(
// Can not support now.
// case "microsecond" => "microsecond"
// case "millisecond" => "millisecond"
- case _ => throw new UnsupportedOperationException(s"The format
$formatStr is invalidate.")
+ case _ => throw new GlutenNotSupportException(s"The format $formatStr is
invalidate.")
}
// Currently, data_trunc function can not support to set the specified
timezone,
@@ -88,7 +88,7 @@ case class CHTruncTimestampTransformer(
s"${CHBackendSettings.getBackendConfigPrefix}.runtime_config.timezone")
)
) {
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
s"It doesn't support trunc the format $newFormatStr with the specified
timezone " +
s"${timeZoneId.get}.")
}
@@ -136,13 +136,13 @@ case class CHStringTranslateTransformer(
!matchingNode.isInstanceOf[StringLiteralNode] ||
!replaceNode.isInstanceOf[StringLiteralNode]
) {
- throw new UnsupportedOperationException(s"$original not supported yet.")
+ throw new GlutenNotSupportException(s"$original not supported yet.")
}
val matchingLiteral = matchingNode.asInstanceOf[StringLiteralNode].getValue
val replaceLiteral = replaceNode.asInstanceOf[StringLiteralNode].getValue
if (matchingLiteral.length() != replaceLiteral.length()) {
- throw new UnsupportedOperationException(s"$original not supported yet.")
+ throw new GlutenNotSupportException(s"$original not supported yet.")
}
GenericExpressionTransformer(
@@ -193,7 +193,7 @@ case class CHPosExplodeTransformer(
Lists.newArrayList(childNode),
ConverterUtils.getTypeNode(structType, false))
case _ =>
- throw new UnsupportedOperationException(s"posexplode($childType) not
supported yet.")
+ throw new GlutenNotSupportException(s"posexplode($childType) not
supported yet.")
}
}
}
diff --git
a/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/SparkPlanExecApiImpl.scala
b/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/SparkPlanExecApiImpl.scala
index b8da55ea9..e14ff795b 100644
---
a/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/SparkPlanExecApiImpl.scala
+++
b/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/SparkPlanExecApiImpl.scala
@@ -18,6 +18,7 @@ package io.glutenproject.backendsapi.velox
import io.glutenproject.GlutenConfig
import io.glutenproject.backendsapi.SparkPlanExecApi
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.execution._
import io.glutenproject.expression._
import io.glutenproject.expression.ConverterUtils.FunctionConfig
@@ -81,7 +82,7 @@ class SparkPlanExecApiImpl extends SparkPlanExecApi {
val decimalType = original.dataType.asInstanceOf[DecimalType]
val precision = decimalType.precision
if (precision > 18) {
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
"GetArrayItem not support decimal precision more than 18")
}
}
@@ -482,7 +483,7 @@ class SparkPlanExecApiImpl extends SparkPlanExecApi {
SQLConf.get.getConf(SQLConf.MAP_KEY_DEDUP_POLICY)
!= SQLConf.MapKeyDedupPolicy.EXCEPTION.toString
) {
- throw new UnsupportedOperationException("Only EXCEPTION policy is
supported!")
+ throw new GlutenNotSupportException("Only EXCEPTION policy is
supported!")
}
GenericExpressionTransformer(substraitExprName, children, expr)
}
diff --git
a/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala
b/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala
index 12a76a3d2..6201a372d 100644
---
a/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala
+++
b/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala
@@ -18,6 +18,7 @@ package io.glutenproject.backendsapi.velox
import io.glutenproject.{GlutenConfig, GlutenPlugin, VELOX_BRANCH,
VELOX_REVISION, VELOX_REVISION_TIME}
import io.glutenproject.backendsapi._
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.execution.WriteFilesExecTransformer
import io.glutenproject.expression.WindowFunctionsBuilder
import io.glutenproject.extension.ValidationResult
@@ -271,7 +272,7 @@ object BackendSettings extends BackendSettingsApi {
func => {
val windowExpression = func match {
case alias: Alias =>
WindowFunctionsBuilder.extractWindowExpression(alias.child)
- case _ => throw new UnsupportedOperationException(s"$func is not
supported.")
+ case _ => throw new GlutenNotSupportException(s"$func is not
supported.")
}
// Block the offloading by checking Velox's current limitations
@@ -285,7 +286,7 @@ object BackendSettings extends BackendSettingsApi {
order =>
order.direction match {
case Descending =>
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
"DESC order is not supported when" +
" literal bound type is used!")
case _ =>
@@ -295,17 +296,15 @@ object BackendSettings extends BackendSettingsApi {
order.dataType match {
case ByteType | ShortType | IntegerType | LongType |
DateType =>
case _ =>
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
"Only integral type & date type are" +
" supported for sort key when literal bound type
is used!")
})
val rawValue = e.eval().toString.toLong
if (isUpperBound && rawValue < 0) {
- throw new UnsupportedOperationException(
- "Negative upper bound is not supported!")
+ throw new GlutenNotSupportException("Negative upper bound
is not supported!")
} else if (!isUpperBound && rawValue > 0) {
- throw new UnsupportedOperationException(
- "Positive lower bound is not supported!")
+ throw new GlutenNotSupportException("Positive lower bound
is not supported!")
}
case _ =>
}
diff --git
a/backends-velox/src/main/scala/io/glutenproject/execution/HashAggregateExecTransformer.scala
b/backends-velox/src/main/scala/io/glutenproject/execution/HashAggregateExecTransformer.scala
index 565edd9f6..cbe0beab2 100644
---
a/backends-velox/src/main/scala/io/glutenproject/execution/HashAggregateExecTransformer.scala
+++
b/backends-velox/src/main/scala/io/glutenproject/execution/HashAggregateExecTransformer.scala
@@ -17,6 +17,7 @@
package io.glutenproject.execution
import io.glutenproject.backendsapi.BackendsApiManager
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.expression._
import io.glutenproject.expression.ConverterUtils.FunctionConfig
import io.glutenproject.extension.columnar.RewriteTypedImperativeAggregate
@@ -142,7 +143,7 @@ abstract class HashAggregateExecTransformer(
expr.mode match {
case Partial | PartialMerge =>
case _ =>
- throw new UnsupportedOperationException(s"${expr.mode} not
supported.")
+ throw new GlutenNotSupportException(s"${expr.mode} not supported.")
}
val aggFunc = expr.aggregateFunction
aggFunc match {
@@ -234,7 +235,7 @@ abstract class HashAggregateExecTransformer(
)
aggregateNodeList.add(aggFunctionNode)
case other =>
- throw new UnsupportedOperationException(s"$other is not supported.")
+ throw new GlutenNotSupportException(s"$other is not supported.")
}
}
@@ -262,7 +263,7 @@ abstract class HashAggregateExecTransformer(
)
aggregateNodeList.add(aggFunctionNode)
case other =>
- throw new UnsupportedOperationException(s"$other is not
supported.")
+ throw new GlutenNotSupportException(s"$other is not supported.")
}
case _ if aggregateFunction.aggBufferAttributes.size > 1 =>
generateMergeCompanionNode()
@@ -301,7 +302,7 @@ abstract class HashAggregateExecTransformer(
ConverterUtils
.getTypeNode(aggregateFunction.dataType,
aggregateFunction.nullable))
case other =>
- throw new UnsupportedOperationException(s"$other is not
supported.")
+ throw new GlutenNotSupportException(s"$other is not
supported.")
}
case _ =>
typeNodeList.add(
@@ -369,7 +370,7 @@ abstract class HashAggregateExecTransformer(
exprNodes.addAll(childNodes)
case _: HyperLogLogPlusPlus if aggFunc.aggBufferAttributes.size != 1 =>
- throw new UnsupportedOperationException("Only one input attribute is
expected.")
+ throw new GlutenNotSupportException("Only one input attribute is
expected.")
case _ @VeloxIntermediateData.Type(veloxTypes: Seq[DataType]) =>
val rewrittenInputAttributes =
@@ -421,7 +422,7 @@ abstract class HashAggregateExecTransformer(
}
exprNodes.add(getRowConstructNode(args, childNodes,
newInputAttributes, aggFunc))
case other =>
- throw new UnsupportedOperationException(s"$other is not
supported.")
+ throw new GlutenNotSupportException(s"$other is not supported.")
}
case _ =>
@@ -473,7 +474,7 @@ abstract class HashAggregateExecTransformer(
aggregateExpressions.foreach(
aggExpr => {
if (aggExpr.filter.isDefined) {
- throw new UnsupportedOperationException("Filter in final aggregation
is not supported.")
+ throw new GlutenNotSupportException("Filter in final aggregation is
not supported.")
} else {
// The number of filters should be aligned with that of aggregate
functions.
aggFilterList.add(null)
@@ -494,7 +495,7 @@ abstract class HashAggregateExecTransformer(
colIdx += 1
}
case _ =>
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
s"$aggFunc of ${aggExpr.mode.toString} is not supported.")
}
addFunctionNode(args, aggFunc, childrenNodes, aggExpr.mode,
aggregateFunctionList)
@@ -637,7 +638,7 @@ abstract class HashAggregateExecTransformer(
.doTransform(args)
}
case other =>
- throw new UnsupportedOperationException(s"$other not supported.")
+ throw new GlutenNotSupportException(s"$other not supported.")
}
addFunctionNode(
args,
@@ -706,7 +707,7 @@ object VeloxAggregateFunctionsBuilder {
var sigName = ExpressionMappings.expressionsMap.get(aggregateFunc.getClass)
if (sigName.isEmpty) {
- throw new UnsupportedOperationException(s"not currently supported:
$aggregateFunc.")
+ throw new GlutenNotSupportException(s"not currently supported:
$aggregateFunc.")
}
aggregateFunc match {
@@ -814,7 +815,7 @@ case class HashAggregateExecPullOutHelper(
case Final =>
Seq(aggregateAttributes(index))
case other =>
- throw new UnsupportedOperationException(s"Unsupported aggregate
mode: $other.")
+ throw new GlutenNotSupportException(s"Unsupported aggregate
mode: $other.")
})
}.toList
}
diff --git
a/backends-velox/src/main/scala/io/glutenproject/execution/RowToVeloxColumnarExec.scala
b/backends-velox/src/main/scala/io/glutenproject/execution/RowToVeloxColumnarExec.scala
index 832fbb18a..df8299e02 100644
---
a/backends-velox/src/main/scala/io/glutenproject/execution/RowToVeloxColumnarExec.scala
+++
b/backends-velox/src/main/scala/io/glutenproject/execution/RowToVeloxColumnarExec.scala
@@ -18,6 +18,7 @@ package io.glutenproject.execution
import io.glutenproject.backendsapi.BackendsApiManager
import io.glutenproject.columnarbatch.ColumnarBatches
+import io.glutenproject.exception.GlutenException
import io.glutenproject.exec.Runtimes
import io.glutenproject.memory.arrowalloc.ArrowBufferAllocators
import io.glutenproject.memory.nmm.NativeMemoryManagers
@@ -47,7 +48,7 @@ case class RowToVeloxColumnarExec(child: SparkPlan) extends
RowToColumnarExecBas
override def doExecuteColumnarInternal(): RDD[ColumnarBatch] = {
BackendsApiManager.getValidatorApiInstance.doSchemaValidate(schema).foreach {
reason =>
- throw new UnsupportedOperationException(
+ throw new GlutenException(
s"Input schema contains unsupported type when convert row to
columnar for $schema " +
s"due to $reason")
}
diff --git
a/backends-velox/src/main/scala/io/glutenproject/execution/VeloxColumnarToRowExec.scala
b/backends-velox/src/main/scala/io/glutenproject/execution/VeloxColumnarToRowExec.scala
index e0de3d476..67bb4ba8c 100644
---
a/backends-velox/src/main/scala/io/glutenproject/execution/VeloxColumnarToRowExec.scala
+++
b/backends-velox/src/main/scala/io/glutenproject/execution/VeloxColumnarToRowExec.scala
@@ -17,6 +17,7 @@
package io.glutenproject.execution
import io.glutenproject.columnarbatch.ColumnarBatches
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.extension.ValidationResult
import io.glutenproject.memory.nmm.NativeMemoryManagers
import io.glutenproject.utils.Iterators
@@ -59,7 +60,7 @@ case class VeloxColumnarToRowExec(child: SparkPlan) extends
ColumnarToRowExecBas
case _: StructType =>
case _: NullType =>
case _ =>
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
s"${field.dataType} is unsupported in " +
s"VeloxColumnarToRowExec.")
}
diff --git
a/backends-velox/src/main/scala/io/glutenproject/expression/ExpressionTransformer.scala
b/backends-velox/src/main/scala/io/glutenproject/expression/ExpressionTransformer.scala
index 34db62283..cf0445d7c 100644
---
a/backends-velox/src/main/scala/io/glutenproject/expression/ExpressionTransformer.scala
+++
b/backends-velox/src/main/scala/io/glutenproject/expression/ExpressionTransformer.scala
@@ -16,6 +16,7 @@
*/
package io.glutenproject.expression
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.expression.ConverterUtils.FunctionConfig
import
io.glutenproject.expression.ExpressionConverter.replaceWithExpressionTransformer
import io.glutenproject.substrait.expression._
@@ -76,7 +77,7 @@ case class VeloxGetStructFieldTransformer(
// Append the nested index to selection node.
node.addNestedChildIdx(JInteger.valueOf(ordinal))
case other =>
- throw new UnsupportedOperationException(s"$other is not supported.")
+ throw new GlutenNotSupportException(s"$other is not supported.")
}
}
}
@@ -127,14 +128,14 @@ case class VeloxStringSplitTransformer(
!regexExpr.isInstanceOf[LiteralTransformer] ||
!limitExpr.isInstanceOf[LiteralTransformer]
) {
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
"Gluten only supports literal input as limit/regex for split
function.")
}
val limit =
limitExpr.doTransform(args).asInstanceOf[IntLiteralNode].getValue
val regex =
regexExpr.doTransform(args).asInstanceOf[StringLiteralNode].getValue
if (limit > 0 || regex.length > 1) {
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
s"$original supported single-length regex and negative limit, but
given $limit and $regex")
}
diff --git
a/gluten-core/src/main/java/io/glutenproject/exception/GlutenNotSupportException.java
b/gluten-core/src/main/java/io/glutenproject/exception/GlutenNotSupportException.java
new file mode 100644
index 000000000..7b5067aff
--- /dev/null
+++
b/gluten-core/src/main/java/io/glutenproject/exception/GlutenNotSupportException.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.glutenproject.exception;
+
+/**
+ * Generally, this exception should be thrown if Gluten doesn't support an
operation, but vanilla
+ * Spark supports it. We expect Gluten makes such operation or its relevant
operations fall back to
+ * vanilla Spark when handling this exception.
+ */
+public class GlutenNotSupportException extends GlutenException {
+
+ public GlutenNotSupportException() {}
+
+ public GlutenNotSupportException(String message) {
+ super(message);
+ }
+
+ public GlutenNotSupportException(Throwable t) {
+ super(t);
+ }
+
+ public GlutenNotSupportException(String message, Throwable t) {
+ super(message, t);
+ }
+}
diff --git a/gluten-core/src/main/scala/io/glutenproject/GlutenPlugin.scala
b/gluten-core/src/main/scala/io/glutenproject/GlutenPlugin.scala
index 4aa56f57e..670c9411d 100644
--- a/gluten-core/src/main/scala/io/glutenproject/GlutenPlugin.scala
+++ b/gluten-core/src/main/scala/io/glutenproject/GlutenPlugin.scala
@@ -20,6 +20,7 @@ import
io.glutenproject.GlutenConfig.GLUTEN_DEFAULT_SESSION_TIMEZONE_KEY
import io.glutenproject.GlutenPlugin.{GLUTEN_SESSION_EXTENSION_NAME,
SPARK_SESSION_EXTS_KEY}
import io.glutenproject.backendsapi.BackendsApiManager
import io.glutenproject.events.GlutenBuildInfoEvent
+import io.glutenproject.exception.GlutenException
import io.glutenproject.expression.ExpressionMappings
import io.glutenproject.extension.{ColumnarOverrides,
OthersExtensionOverrides, QueryStagePrepOverrides, StrategyOverrides}
import io.glutenproject.test.TestStats
@@ -144,7 +145,7 @@ private[glutenproject] class GlutenDriverPlugin extends
DriverPlugin with Loggin
// off-heap bytes
if (!conf.contains(GlutenConfig.GLUTEN_OFFHEAP_SIZE_KEY)) {
- throw new
UnsupportedOperationException(s"${GlutenConfig.GLUTEN_OFFHEAP_SIZE_KEY} is not
set")
+ throw new GlutenException(s"${GlutenConfig.GLUTEN_OFFHEAP_SIZE_KEY} is
not set")
}
// Session's local time zone must be set. If not explicitly set by user,
its default
// value (detected for the platform) is used, consistent with spark.
diff --git
a/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala
b/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala
index 5788a8b97..b4466cf20 100644
---
a/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala
@@ -16,6 +16,7 @@
*/
package io.glutenproject.backendsapi
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.execution._
import io.glutenproject.expression._
import io.glutenproject.substrait.expression.{ExpressionBuilder,
ExpressionNode, WindowFunctionNode}
@@ -196,7 +197,7 @@ trait SparkPlanExecApi {
left: ExpressionTransformer,
right: ExpressionTransformer,
original: NaNvl): ExpressionTransformer = {
- throw new UnsupportedOperationException("NaNvl is not supported")
+ throw new GlutenNotSupportException("NaNvl is not supported")
}
/** Transform map_entries to Substrait. */
@@ -204,7 +205,7 @@ trait SparkPlanExecApi {
substraitExprName: String,
child: ExpressionTransformer,
expr: Expression): ExpressionTransformer = {
- throw new UnsupportedOperationException("map_entries is not supported")
+ throw new GlutenNotSupportException("map_entries is not supported")
}
/** Transform inline to Substrait. */
@@ -212,7 +213,7 @@ trait SparkPlanExecApi {
substraitExprName: String,
child: ExpressionTransformer,
expr: Expression): ExpressionTransformer = {
- throw new UnsupportedOperationException("map_entries is not supported")
+ throw new GlutenNotSupportException("map_entries is not supported")
}
/**
@@ -488,7 +489,7 @@ trait SparkPlanExecApi {
val aggregateFunc = aggExpression.aggregateFunction
val substraitAggFuncName =
ExpressionMappings.expressionsMap.get(aggregateFunc.getClass)
if (substraitAggFuncName.isEmpty) {
- throw new UnsupportedOperationException(s"Not currently
supported: $aggregateFunc.")
+ throw new GlutenNotSupportException(s"Not currently supported:
$aggregateFunc.")
}
val childrenNodeList = aggregateFunc.children
@@ -605,7 +606,7 @@ trait SparkPlanExecApi {
)
windowExpressionNodes.add(windowFunctionNode)
case _ =>
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
"unsupported window function type: " +
wExpression.windowFunction)
}
@@ -646,8 +647,7 @@ trait SparkPlanExecApi {
extraFilters
}
case _ =>
- throw new UnsupportedOperationException(
- s"${sparkExecNode.getClass.toString} is not supported.")
+ throw new
GlutenNotSupportException(s"${sparkExecNode.getClass.toString} is not
supported.")
}
}
diff --git
a/gluten-core/src/main/scala/io/glutenproject/execution/BasicPhysicalOperatorTransformer.scala
b/gluten-core/src/main/scala/io/glutenproject/execution/BasicPhysicalOperatorTransformer.scala
index 96867389d..75da80cb5 100644
---
a/gluten-core/src/main/scala/io/glutenproject/execution/BasicPhysicalOperatorTransformer.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/execution/BasicPhysicalOperatorTransformer.scala
@@ -17,6 +17,7 @@
package io.glutenproject.execution
import io.glutenproject.backendsapi.BackendsApiManager
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.expression.{ConverterUtils, ExpressionConverter,
ExpressionTransformer}
import io.glutenproject.extension.{GlutenPlan, ValidationResult}
import io.glutenproject.metrics.MetricsUpdater
@@ -373,7 +374,7 @@ object FilterHandler extends PredicateHelper {
case scan: FileScan =>
scan.dataFilters
case _ =>
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
s"${batchScan.scan.getClass.toString} is not supported")
}
case _ =>
@@ -415,6 +416,6 @@ object FilterHandler extends PredicateHelper {
batchScan,
allPushDownFilters = Some(pushDownFilters))
case other =>
- throw new UnsupportedOperationException(s"${other.getClass.toString}
is not supported.")
+ throw new GlutenNotSupportException(s"${other.getClass.toString} is
not supported.")
}
}
diff --git
a/gluten-core/src/main/scala/io/glutenproject/execution/BatchScanExecTransformer.scala
b/gluten-core/src/main/scala/io/glutenproject/execution/BatchScanExecTransformer.scala
index dfb448f13..fb88b2076 100644
---
a/gluten-core/src/main/scala/io/glutenproject/execution/BatchScanExecTransformer.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/execution/BatchScanExecTransformer.scala
@@ -17,6 +17,7 @@
package io.glutenproject.execution
import io.glutenproject.backendsapi.BackendsApiManager
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.extension.ValidationResult
import io.glutenproject.metrics.MetricsUpdater
import io.glutenproject.substrait.rel.LocalFilesNode.ReadFileFormat
@@ -105,7 +106,7 @@ abstract class BatchScanExecTransformerBase(
case fileScan: FileScan =>
pushdownFilters.getOrElse(fileScan.dataFilters)
case _ =>
- throw new UnsupportedOperationException(s"${scan.getClass.toString} is
not supported")
+ throw new GlutenNotSupportException(s"${scan.getClass.toString} is not
supported")
}
override def getMetadataColumns(): Seq[AttributeReference] = Seq.empty
diff --git
a/gluten-core/src/main/scala/io/glutenproject/execution/HashAggregateExecBaseTransformer.scala
b/gluten-core/src/main/scala/io/glutenproject/execution/HashAggregateExecBaseTransformer.scala
index dc094cbc1..e18823391 100644
---
a/gluten-core/src/main/scala/io/glutenproject/execution/HashAggregateExecBaseTransformer.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/execution/HashAggregateExecBaseTransformer.scala
@@ -18,6 +18,7 @@ package io.glutenproject.execution
import io.glutenproject.GlutenConfig
import io.glutenproject.backendsapi.BackendsApiManager
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.expression._
import io.glutenproject.extension.ValidationResult
import io.glutenproject.metrics.MetricsUpdater
@@ -128,7 +129,7 @@ abstract class HashAggregateExecBaseTransformer(
aggregateExpressions.foreach {
expr =>
if (!checkAggFuncModeSupport(expr.aggregateFunction, expr.mode)) {
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
s"Unsupported aggregate mode: ${expr.mode} for
${expr.aggregateFunction.prettyName}")
}
}
@@ -159,7 +160,7 @@ abstract class HashAggregateExecBaseTransformer(
case Complete => "COMPLETE"
case Final => "FINAL"
case other =>
- throw new UnsupportedOperationException(s"not currently supported:
$other.")
+ throw new GlutenNotSupportException(s"not currently supported:
$other.")
}
}
diff --git
a/gluten-core/src/main/scala/io/glutenproject/execution/ScanTransformerFactory.scala
b/gluten-core/src/main/scala/io/glutenproject/execution/ScanTransformerFactory.scala
index 363391490..c0122186a 100644
---
a/gluten-core/src/main/scala/io/glutenproject/execution/ScanTransformerFactory.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/execution/ScanTransformerFactory.scala
@@ -16,6 +16,7 @@
*/
package io.glutenproject.execution
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.expression.ExpressionConverter
import io.glutenproject.extension.columnar.TransformHints
import io.glutenproject.sql.shims.SparkShimLoader
@@ -88,7 +89,7 @@ object ScanTransformerFactory {
table =
SparkShimLoader.getSparkShims.getBatchScanExecTable(batchScanExec)
)
case _ =>
- throw new UnsupportedOperationException(s"Unsupported scan $scan")
+ throw new GlutenNotSupportException(s"Unsupported scan $scan")
}
}
}
@@ -122,7 +123,7 @@ object ScanTransformerFactory {
}
} else {
if (validation) {
- throw new UnsupportedOperationException(s"Unsupported scan
${batchScan.scan}")
+ throw new GlutenNotSupportException(s"Unsupported scan
${batchScan.scan}")
}
// If filter expressions aren't empty, we need to transform the inner
operators,
// and fallback the BatchScanExec itself.
diff --git
a/gluten-core/src/main/scala/io/glutenproject/expression/AggregateFunctionsBuilder.scala
b/gluten-core/src/main/scala/io/glutenproject/expression/AggregateFunctionsBuilder.scala
index ab6c13832..fd9d341f3 100644
---
a/gluten-core/src/main/scala/io/glutenproject/expression/AggregateFunctionsBuilder.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/expression/AggregateFunctionsBuilder.scala
@@ -17,6 +17,7 @@
package io.glutenproject.expression
import io.glutenproject.backendsapi.BackendsApiManager
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.expression.ConverterUtils.FunctionConfig
import io.glutenproject.substrait.expression.ExpressionBuilder
@@ -44,7 +45,7 @@ object AggregateFunctionsBuilder {
substraitAggFuncName.get,
aggregateFunc)
) {
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
s"Aggregate function not supported for $aggregateFunc.")
}
diff --git
a/gluten-core/src/main/scala/io/glutenproject/expression/ArrayExpressionTransformer.scala
b/gluten-core/src/main/scala/io/glutenproject/expression/ArrayExpressionTransformer.scala
index a5ddad257..b79ba85d3 100644
---
a/gluten-core/src/main/scala/io/glutenproject/expression/ArrayExpressionTransformer.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/expression/ArrayExpressionTransformer.scala
@@ -17,6 +17,7 @@
package io.glutenproject.expression
import io.glutenproject.backendsapi.BackendsApiManager
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.expression.ConverterUtils.FunctionConfig
import io.glutenproject.substrait.expression.{ExpressionBuilder,
ExpressionNode}
@@ -39,7 +40,7 @@ case class CreateArrayTransformer(
// transformation is only supported when useStringTypeWhenEmpty is false
// because ClickHouse and Velox currently doesn't support this config.
if (useStringTypeWhenEmpty && children.isEmpty) {
- throw new UnsupportedOperationException(s"$original not supported yet.")
+ throw new GlutenNotSupportException(s"$original not supported yet.")
}
val childNodes = children.map(_.doTransform(args)).asJava
diff --git
a/gluten-core/src/main/scala/io/glutenproject/expression/ConverterUtils.scala
b/gluten-core/src/main/scala/io/glutenproject/expression/ConverterUtils.scala
index ec03c920a..95f6b2861 100644
---
a/gluten-core/src/main/scala/io/glutenproject/expression/ConverterUtils.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/expression/ConverterUtils.scala
@@ -17,6 +17,7 @@
package io.glutenproject.expression
import io.glutenproject.backendsapi.BackendsApiManager
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.substrait.`type`._
import io.glutenproject.utils.SubstraitPlanPrinterUtil
@@ -208,7 +209,7 @@ object ConverterUtils extends Logging {
case Type.KindCase.NOTHING =>
(NullType, true)
case unsupported =>
- throw new UnsupportedOperationException(s"Type $unsupported not
supported.")
+ throw new GlutenNotSupportException(s"Type $unsupported not
supported.")
}
}
@@ -259,7 +260,7 @@ object ConverterUtils extends Logging {
case _: NullType =>
TypeBuilder.makeNothing()
case unknown =>
- throw new UnsupportedOperationException(s"Type $unknown not
supported.")
+ throw new GlutenNotSupportException(s"Type $unknown not supported.")
}
}
@@ -385,7 +386,7 @@ object ConverterUtils extends Logging {
case NullType =>
"nothing"
case other =>
- throw new UnsupportedOperationException(s"Type $other not supported.")
+ throw new GlutenNotSupportException(s"Type $other not supported.")
}
}
@@ -405,7 +406,7 @@ object ConverterUtils extends Logging {
case FunctionConfig.NON =>
funcName.concat(":")
case other =>
- throw new UnsupportedOperationException(s"$other is not supported.")
+ throw new GlutenNotSupportException(s"$other is not supported.")
}
for (idx <- datatypes.indices) {
@@ -431,7 +432,7 @@ object ConverterUtils extends Logging {
case LeftAnti =>
"Anti"
case other =>
- throw new UnsupportedOperationException(s"Unsupported join type:
$other")
+ throw new GlutenNotSupportException(s"Unsupported join type: $other")
}
}
diff --git
a/gluten-core/src/main/scala/io/glutenproject/expression/DateTimeExpressionsTransformer.scala
b/gluten-core/src/main/scala/io/glutenproject/expression/DateTimeExpressionsTransformer.scala
index 938a97a14..213df7f25 100644
---
a/gluten-core/src/main/scala/io/glutenproject/expression/DateTimeExpressionsTransformer.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/expression/DateTimeExpressionsTransformer.scala
@@ -17,6 +17,7 @@
package io.glutenproject.expression
import io.glutenproject.backendsapi.BackendsApiManager
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.expression.ConverterUtils.FunctionConfig
import io.glutenproject.substrait.expression.{ExpressionBuilder,
ExpressionNode}
@@ -49,7 +50,7 @@ case class ExtractDateTransformer(
val dateFieldName =
DateTimeExpressionsTransformer.EXTRACT_DATE_FIELD_MAPPING.get(original.getClass)
if (dateFieldName.isEmpty) {
- throw new UnsupportedOperationException(s"$original not supported yet.")
+ throw new GlutenNotSupportException(s"$original not supported yet.")
}
val fieldNode = ExpressionBuilder.makeStringLiteral(dateFieldName.get)
val expressNodes = Lists.newArrayList(fieldNode, childNode)
diff --git
a/gluten-core/src/main/scala/io/glutenproject/expression/DecimalRoundTransformer.scala
b/gluten-core/src/main/scala/io/glutenproject/expression/DecimalRoundTransformer.scala
index a188589ea..8d326f177 100644
---
a/gluten-core/src/main/scala/io/glutenproject/expression/DecimalRoundTransformer.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/expression/DecimalRoundTransformer.scala
@@ -16,6 +16,7 @@
*/
package io.glutenproject.expression
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.expression.ConverterUtils.FunctionConfig
import io.glutenproject.substrait.expression.{ExpressionBuilder,
ExpressionNode}
@@ -52,7 +53,7 @@ case class DecimalRoundTransformer(
DecimalType(math.min(integralLeastNumDigits + newScale, 38), newScale)
}
case _ =>
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
s"Decimal type is expected but received
${original.child.dataType.typeName}.")
}
diff --git
a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala
b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala
index 5c994bdc0..b13786a11 100644
---
a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala
@@ -18,13 +18,14 @@ package io.glutenproject.expression
import io.glutenproject.GlutenConfig
import io.glutenproject.backendsapi.BackendsApiManager
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.execution.{ColumnarToRowExecBase,
WholeStageTransformer}
import io.glutenproject.test.TestStats
import io.glutenproject.utils.{DecimalArithmeticUtil, PlanUtil}
import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.{InternalRow, SQLConfHelper}
-import org.apache.spark.sql.catalyst.expressions.{BinaryArithmetic, _}
+import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext,
ExprCode}
import org.apache.spark.sql.catalyst.optimizer.NormalizeNaNAndZero
import org.apache.spark.sql.execution.{ScalarSubquery, _}
@@ -74,7 +75,7 @@ object ExpressionConverter extends SQLConfHelper with Logging
{
replaceWithExpressionTransformerInternal(_, attributeSeq,
expressionsMap)),
udf)
case _ =>
- throw new UnsupportedOperationException(s"Not supported python udf:
$udf.")
+ throw new GlutenNotSupportException(s"Not supported python udf: $udf.")
}
}
@@ -91,7 +92,7 @@ object ExpressionConverter extends SQLConfHelper with Logging
{
replaceWithExpressionTransformerInternal(_, attributeSeq,
expressionsMap)),
udf)
case _ =>
- throw new UnsupportedOperationException(s"Not supported scala udf:
$udf.")
+ throw new GlutenNotSupportException(s"Not supported scala udf: $udf.")
}
}
@@ -117,7 +118,7 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
// Check whether Gluten supports this expression
val substraitExprNameOpt = expressionsMap.get(expr.getClass)
if (substraitExprNameOpt.isEmpty) {
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
s"Not supported to map spark function name" +
s" to substrait function name: $expr, class name:
${expr.getClass.getSimpleName}.")
}
@@ -125,7 +126,7 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
// Check whether each backend supports this expression
if
(!BackendsApiManager.getValidatorApiInstance.doExprValidate(substraitExprName,
expr)) {
- throw new UnsupportedOperationException(s"Not supported: $expr.")
+ throw new GlutenNotSupportException(s"Not supported: $expr.")
}
expr match {
case extendedExpr
@@ -281,7 +282,7 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
)
case i: In =>
if (i.list.exists(!_.foldable)) {
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
s"In list option does not support non-foldable expression,
${i.list.map(_.sql)}")
}
InTransformer(
@@ -484,7 +485,7 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
// PrecisionLoss=false: velox not support / ch support
// TODO ch support PrecisionLoss=true
if (!BackendsApiManager.getSettings.allowDecimalArithmetic) {
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
s"Not support ${SQLConf.DECIMAL_OPERATIONS_ALLOW_PREC_LOSS.key} " +
s"${conf.decimalOperationsAllowPrecisionLoss} mode")
}
diff --git
a/gluten-core/src/main/scala/io/glutenproject/expression/LambdaFunctionTransformer.scala
b/gluten-core/src/main/scala/io/glutenproject/expression/LambdaFunctionTransformer.scala
index 05ba33025..f93af3180 100644
---
a/gluten-core/src/main/scala/io/glutenproject/expression/LambdaFunctionTransformer.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/expression/LambdaFunctionTransformer.scala
@@ -16,6 +16,7 @@
*/
package io.glutenproject.expression
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.substrait.expression.{ExpressionBuilder,
ExpressionNode}
import org.apache.spark.sql.catalyst.expressions.LambdaFunction
@@ -31,7 +32,7 @@ case class LambdaFunctionTransformer(
override def doTransform(args: Object): ExpressionNode = {
// Need to fallback when hidden be true as it's not supported in Velox
if (hidden) {
- throw new UnsupportedOperationException(s"Unsupported LambdaFunction
with hidden be true.")
+ throw new GlutenNotSupportException(s"Unsupported LambdaFunction with
hidden be true.")
}
val functionMap = args.asInstanceOf[java.util.HashMap[String,
java.lang.Long]]
val functionId = ExpressionBuilder.newScalarFunction(
diff --git
a/gluten-core/src/main/scala/io/glutenproject/expression/MapExpressionTransformer.scala
b/gluten-core/src/main/scala/io/glutenproject/expression/MapExpressionTransformer.scala
index c2e4f631a..1b4f5c206 100644
---
a/gluten-core/src/main/scala/io/glutenproject/expression/MapExpressionTransformer.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/expression/MapExpressionTransformer.scala
@@ -17,6 +17,7 @@
package io.glutenproject.expression
import io.glutenproject.backendsapi.BackendsApiManager
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.expression.ConverterUtils.FunctionConfig
import io.glutenproject.substrait.expression.{ExpressionBuilder,
ExpressionNode}
@@ -36,7 +37,7 @@ case class CreateMapTransformer(
// transformation is only supported when useStringTypeWhenEmpty is false
// because ClickHouse and Velox currently doesn't support this config.
if (children.isEmpty && useStringTypeWhenEmpty) {
- throw new UnsupportedOperationException(s"$original not supported yet.")
+ throw new GlutenNotSupportException(s"$original not supported yet.")
}
val childNodes = new java.util.ArrayList[ExpressionNode]()
@@ -67,11 +68,11 @@ case class GetMapValueTransformer(
override def doTransform(args: java.lang.Object): ExpressionNode = {
if (BackendsApiManager.getSettings.alwaysFailOnMapExpression()) {
- throw new UnsupportedOperationException(s"$original not supported yet.")
+ throw new GlutenNotSupportException(s"$original not supported yet.")
}
if (failOnError) {
- throw new UnsupportedOperationException(s"$original not supported yet.")
+ throw new GlutenNotSupportException(s"$original not supported yet.")
}
val childNode = child.doTransform(args)
diff --git
a/gluten-core/src/main/scala/io/glutenproject/expression/StringExpressionTransformer.scala
b/gluten-core/src/main/scala/io/glutenproject/expression/StringExpressionTransformer.scala
index 93d1a7bed..b1ac0ddae 100644
---
a/gluten-core/src/main/scala/io/glutenproject/expression/StringExpressionTransformer.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/expression/StringExpressionTransformer.scala
@@ -16,6 +16,7 @@
*/
package io.glutenproject.expression
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.expression.ConverterUtils.FunctionConfig
import io.glutenproject.substrait.expression._
@@ -65,7 +66,7 @@ case class RegExpReplaceTransformer(
!posNode.isInstanceOf[IntLiteralNode] ||
posNode.asInstanceOf[IntLiteralNode].getValue != 1
) {
- throw new UnsupportedOperationException(s"$original not supported yet.")
+ throw new GlutenNotSupportException(s"$original not supported yet.")
}
GenericExpressionTransformer(substraitExprName, Seq(subject, regexp, rep),
original)
diff --git
a/gluten-core/src/main/scala/io/glutenproject/expression/UnaryExpressionTransformer.scala
b/gluten-core/src/main/scala/io/glutenproject/expression/UnaryExpressionTransformer.scala
index 6ba9f39ac..1e82b9c82 100644
---
a/gluten-core/src/main/scala/io/glutenproject/expression/UnaryExpressionTransformer.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/expression/UnaryExpressionTransformer.scala
@@ -17,6 +17,7 @@
package io.glutenproject.expression
import io.glutenproject.backendsapi.BackendsApiManager
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.expression.ConverterUtils.FunctionConfig
import io.glutenproject.substrait.`type`.ListNode
import io.glutenproject.substrait.`type`.MapNode
@@ -68,7 +69,7 @@ case class ExplodeTransformer(
case m: MapNode =>
ExpressionBuilder.makeScalarFunction(functionId, expressionNodes,
m.getNestedType)
case _ =>
- throw new UnsupportedOperationException(s"explode($childTypeNode) not
supported yet.")
+ throw new GlutenNotSupportException(s"explode($childTypeNode) not
supported yet.")
}
}
}
@@ -105,7 +106,7 @@ case class PosExplodeTransformer(
val (valType, valContainsNull) = original.child.dataType match {
case a: ArrayType => (a.elementType, a.containsNull)
case _ =>
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
s"posexplode(${original.child.dataType}) not supported yet.")
}
val outputType = MapType(keyType, valType, valContainsNull)
@@ -143,7 +144,7 @@ case class PosExplodeTransformer(
Lists.newArrayList(mapFromArraysExprNode),
ConverterUtils.getTypeNode(structType, false))
case _ =>
- throw new UnsupportedOperationException(s"posexplode($childType) not
supported yet.")
+ throw new GlutenNotSupportException(s"posexplode($childType) not
supported yet.")
}
}
}
@@ -204,7 +205,7 @@ case class RandTransformer(
override def doTransform(args: java.lang.Object): ExpressionNode = {
if (!original.hideSeed) {
// TODO: for user-specified seed, we need to pass partition index to
native engine.
- throw new UnsupportedOperationException("User-specified seed is not
supported.")
+ throw new GlutenNotSupportException("User-specified seed is not
supported.")
}
val functionMap = args.asInstanceOf[java.util.HashMap[String,
java.lang.Long]]
val functionId = ExpressionBuilder.newScalarFunction(
diff --git
a/gluten-core/src/main/scala/io/glutenproject/expression/WindowFunctionsBuilder.scala
b/gluten-core/src/main/scala/io/glutenproject/expression/WindowFunctionsBuilder.scala
index 41a988c36..07ad3ae23 100644
---
a/gluten-core/src/main/scala/io/glutenproject/expression/WindowFunctionsBuilder.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/expression/WindowFunctionsBuilder.scala
@@ -16,6 +16,7 @@
*/
package io.glutenproject.expression
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.expression.ConverterUtils.FunctionConfig
import io.glutenproject.substrait.expression.ExpressionBuilder
@@ -28,7 +29,7 @@ object WindowFunctionsBuilder {
val functionMap = args.asInstanceOf[java.util.HashMap[String,
java.lang.Long]]
val substraitFunc =
ExpressionMappings.expressionsMap.get(windowFunc.getClass)
if (substraitFunc.isEmpty) {
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
s"not currently supported: ${windowFunc.getClass.getName}.")
}
diff --git
a/gluten-core/src/main/scala/io/glutenproject/extension/ExpressionExtensionTrait.scala
b/gluten-core/src/main/scala/io/glutenproject/extension/ExpressionExtensionTrait.scala
index 8542a7fdb..1e8e20c21 100644
---
a/gluten-core/src/main/scala/io/glutenproject/extension/ExpressionExtensionTrait.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/extension/ExpressionExtensionTrait.scala
@@ -16,6 +16,7 @@
*/
package io.glutenproject.extension
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.expression.{ExpressionTransformer, Sig}
import org.apache.spark.internal.Logging
@@ -57,7 +58,7 @@ trait ExpressionExtensionTrait {
/** Get the custom agg function substrait name and the input types of the
child */
def buildCustomAggregateFunction(
aggregateFunc: AggregateFunction): (Option[String], Seq[DataType]) = {
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
s"Aggregate function ${aggregateFunc.getClass} is not supported.")
}
}
diff --git
a/gluten-core/src/main/scala/io/glutenproject/extension/columnar/MiscColumnarRules.scala
b/gluten-core/src/main/scala/io/glutenproject/extension/columnar/MiscColumnarRules.scala
index 79fd37f3f..4fbadb0b5 100644
---
a/gluten-core/src/main/scala/io/glutenproject/extension/columnar/MiscColumnarRules.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/extension/columnar/MiscColumnarRules.scala
@@ -18,6 +18,7 @@ package io.glutenproject.extension.columnar
import io.glutenproject.GlutenConfig
import io.glutenproject.backendsapi.BackendsApiManager
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.execution._
import io.glutenproject.expression.ExpressionConverter
import io.glutenproject.extension.{ColumnarToRowLike, GlutenPlan}
@@ -479,7 +480,7 @@ object MiscColumnarRules {
TransformHints.tagNotTransformable(newSource,
validateResult.reason.get)
newSource
case other =>
- throw new UnsupportedOperationException(s"${other.getClass.toString}
is not supported.")
+ throw new GlutenNotSupportException(s"${other.getClass.toString} is
not supported.")
}
}
}
diff --git
a/gluten-core/src/main/scala/io/glutenproject/extension/columnar/TransformHintRule.scala
b/gluten-core/src/main/scala/io/glutenproject/extension/columnar/TransformHintRule.scala
index 67ee36887..a8ed0b886 100644
---
a/gluten-core/src/main/scala/io/glutenproject/extension/columnar/TransformHintRule.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/extension/columnar/TransformHintRule.scala
@@ -18,6 +18,7 @@ package io.glutenproject.extension.columnar
import io.glutenproject.GlutenConfig
import io.glutenproject.backendsapi.BackendsApiManager
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.execution._
import io.glutenproject.expression.ExpressionUtils.getExpressionTreeDepth
import io.glutenproject.extension.{GlutenPlan, ValidationResult}
@@ -108,7 +109,7 @@ object TransformHints {
case TRANSFORM_UNSUPPORTED(None, _) =>
originalHint
case _ =>
- throw new UnsupportedOperationException(
+ throw new GlutenNotSupportException(
"Plan was already tagged as non-transformable, " +
s"cannot mark it as transformable after
that:\n${plan.toString()}")
}
@@ -737,11 +738,14 @@ case class AddTransformHintRule() extends Rule[SparkPlan]
{
// Currently we assume a plan to be transformable by default.
}
} catch {
- case e: UnsupportedOperationException =>
+ case e @ (_: GlutenNotSupportException | _:
UnsupportedOperationException) =>
TransformHints.tagNotTransformable(
plan,
s"${e.getMessage}, original Spark plan is " +
s"${plan.getClass}(${plan.children.toList.map(_.getClass)})")
+ if (!e.isInstanceOf[GlutenNotSupportException]) {
+ logDebug("This exception may need to be fixed: " + e.getMessage)
+ }
}
}
}
diff --git
a/gluten-core/src/main/scala/io/glutenproject/substrait/TypeConverter.scala
b/gluten-core/src/main/scala/io/glutenproject/substrait/TypeConverter.scala
index 0ae1ccbfe..6f46ab79b 100644
--- a/gluten-core/src/main/scala/io/glutenproject/substrait/TypeConverter.scala
+++ b/gluten-core/src/main/scala/io/glutenproject/substrait/TypeConverter.scala
@@ -16,6 +16,8 @@
*/
package io.glutenproject.substrait
+import io.glutenproject.exception.GlutenNotSupportException
+
import org.apache.spark.sql.types.{BinaryType, BooleanType, ByteType,
DataType, DoubleType, FloatType, IntegerType, LongType, ShortType, StringType,
TimestampType}
import com.google.protobuf.CodedInputStream
@@ -50,7 +52,7 @@ object TypeConverter {
case BINARY => ExpressionType(BinaryType,
isNullable(t.getBinary.getNullability))
case TIMESTAMP => ExpressionType(TimestampType,
isNullable(t.getTimestamp.getNullability))
case t =>
- throw new UnsupportedOperationException(s"Conversion from substrait
type not supported: $t")
+ throw new GlutenNotSupportException(s"Conversion from substrait type
not supported: $t")
}
}
}
diff --git
a/gluten-core/src/main/scala/io/glutenproject/utils/DecimalArithmeticUtil.scala
b/gluten-core/src/main/scala/io/glutenproject/utils/DecimalArithmeticUtil.scala
index da1feab1e..28b9f8a03 100644
---
a/gluten-core/src/main/scala/io/glutenproject/utils/DecimalArithmeticUtil.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/utils/DecimalArithmeticUtil.scala
@@ -17,6 +17,7 @@
package io.glutenproject.utils
import io.glutenproject.backendsapi.BackendsApiManager
+import io.glutenproject.exception.GlutenNotSupportException
import io.glutenproject.expression.{CheckOverflowTransformer,
ChildTransformer, DecimalArithmeticExpressionTransformer, ExpressionTransformer}
import org.apache.spark.sql.catalyst.analysis.DecimalPrecision
@@ -60,7 +61,7 @@ object DecimalArithmeticUtil {
resultPrecision =
Math.min(type1.precision - type1.scale, type2.precision -
type2.scale + resultScale)
case other =>
- throw new UnsupportedOperationException(s"$other is not supported.")
+ throw new GlutenNotSupportException(s"$other is not supported.")
}
adjustScaleIfNeeded(resultPrecision, resultScale)
}
@@ -101,7 +102,7 @@ object DecimalArithmeticUtil {
case _: Multiply => OperationType.MULTIPLY
case _: Divide => OperationType.DIVIDE
case other =>
- throw new UnsupportedOperationException(s"$other is not supported.")
+ throw new GlutenNotSupportException(s"$other is not supported.")
}
}
diff --git
a/gluten-core/src/main/scala/io/glutenproject/utils/PullOutProjectHelper.scala
b/gluten-core/src/main/scala/io/glutenproject/utils/PullOutProjectHelper.scala
index 31acc9698..a519772fc 100644
---
a/gluten-core/src/main/scala/io/glutenproject/utils/PullOutProjectHelper.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/utils/PullOutProjectHelper.scala
@@ -16,6 +16,8 @@
*/
package io.glutenproject.utils
+import io.glutenproject.exception.GlutenNotSupportException
+
import org.apache.spark.sql.catalyst.expressions._
import
org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression,
AggregateFunction}
import org.apache.spark.sql.execution.aggregate._
@@ -108,7 +110,7 @@ trait PullOutProjectHelper {
resultExpressions = newResultExpressions
)
case _ =>
- throw new UnsupportedOperationException(s"Unsupported agg $agg")
+ throw new GlutenNotSupportException(s"Unsupported agg $agg")
}
protected def rewriteAggregateExpression(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
