yma11 commented on code in PR #6712:
URL: https://github.com/apache/incubator-gluten/pull/6712#discussion_r1716853629
##########
backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala:
##########
@@ -87,56 +87,8 @@ object VeloxBackendSettings extends BackendSettingsApi {
}
}
- val parquetTypeValidatorWithComplexTypeFallback:
PartialFunction[StructField, String] = {
- case StructField(_, arrayType: ArrayType, _, _) =>
- arrayType.simpleString + " is forced to fallback."
- case StructField(_, mapType: MapType, _, _) =>
- mapType.simpleString + " is forced to fallback."
- case StructField(_, structType: StructType, _, _) =>
- structType.simpleString + " is forced to fallback."
- case StructField(_, timestampType: TimestampType, _, _)
- if GlutenConfig.getConf.forceParquetTimestampTypeScanFallbackEnabled
=>
- timestampType.simpleString + " is forced to fallback."
- }
- val orcTypeValidatorWithComplexTypeFallback: PartialFunction[StructField,
String] = {
- case StructField(_, arrayType: ArrayType, _, _) =>
- arrayType.simpleString + " is forced to fallback."
- case StructField(_, mapType: MapType, _, _) =>
- mapType.simpleString + " is forced to fallback."
- case StructField(_, structType: StructType, _, _) =>
- structType.simpleString + " is forced to fallback."
- case StructField(_, stringType: StringType, _, metadata)
- if isCharType(stringType, metadata) =>
- CharVarcharUtils.getRawTypeString(metadata) + " not support"
- case StructField(_, TimestampType, _, _) => "TimestampType not support"
- }
format match {
- case ParquetReadFormat =>
- val typeValidator: PartialFunction[StructField, String] = {
- // Parquet scan of nested array with struct/array as element type is
unsupported in Velox.
- case StructField(_, arrayType: ArrayType, _, _)
- if arrayType.elementType.isInstanceOf[StructType] =>
- "StructType as element in ArrayType"
- case StructField(_, arrayType: ArrayType, _, _)
- if arrayType.elementType.isInstanceOf[ArrayType] =>
- "ArrayType as element in ArrayType"
- // Parquet scan of nested map with struct as key type,
- // or array type as value type is not supported in Velox.
- case StructField(_, mapType: MapType, _, _) if
mapType.keyType.isInstanceOf[StructType] =>
- "StructType as Key in MapType"
- case StructField(_, mapType: MapType, _, _)
- if mapType.valueType.isInstanceOf[ArrayType] =>
- "ArrayType as Value in MapType"
- case StructField(_, TimestampType, _, _)
- if
GlutenConfig.getConf.forceParquetTimestampTypeScanFallbackEnabled =>
Review Comment:
Yes. I keep this check and we may have a try by setting
`forceParquetTimestampTypeScanFallbackEnabled` to false once the related
support merged.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
