This is an automated email from the ASF dual-hosted git repository.
mingliang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 68ec418601 [GLUTEN-10309][CORE] Improve the implementation of
NativeWritePostRule (#10310)
68ec418601 is described below
commit 68ec4186010a63b96a956346a1e4466b1be70511
Author: Jiaan Geng <[email protected]>
AuthorDate: Wed Aug 6 11:46:46 2025 +0800
[GLUTEN-10309][CORE] Improve the implementation of NativeWritePostRule
(#10310)
---
.../org/apache/gluten/backendsapi/velox/VeloxBackend.scala | 11 ++++-------
.../sql/execution/datasources/GlutenWriterColumnarRules.scala | 10 ++++------
2 files changed, 8 insertions(+), 13 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
index b6b259d6bd..bb6b7597bc 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
@@ -369,14 +369,11 @@ object VeloxBackendSettings extends BackendSettingsApi {
}
override def supportNativeWrite(fields: Array[StructField]): Boolean = {
- fields.map {
- field =>
- field.dataType match {
- case _: StructType | _: ArrayType | _: MapType => return false
- case _ =>
- }
+ def isNotSupported(dataType: DataType): Boolean = dataType match {
+ case _: StructType | _: ArrayType | _: MapType => true
+ case _ => false
}
- true
+ !fields.exists(field => isNotSupported(field.dataType))
}
override def supportExpandExec(): Boolean = true
diff --git
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala
index 59360712a0..606fa377b8 100644
---
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala
+++
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala
@@ -39,10 +39,6 @@ object GlutenWriterColumnarRules {
"org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat" ->
"parquet"
)
private def getNativeFormat(cmd: DataWritingCommand): Option[String] = {
- if (!BackendsApiManager.getSettings.enableNativeWriteFiles()) {
- return None
- }
-
cmd match {
case command: CreateDataSourceTableAsSelectCommand
if !BackendsApiManager.getSettings.skipNativeCtas(command) =>
@@ -104,9 +100,11 @@ object GlutenWriterColumnarRules {
override def apply(p: SparkPlan): SparkPlan = p match {
case rc @ DataWritingCommandExec(cmd, child) =>
// The same thread can set these properties in the last query
submission.
- val fields = child.output.toStructType.fields
val format =
- if (BackendsApiManager.getSettings.supportNativeWrite(fields)) {
+ if (
+
BackendsApiManager.getSettings.supportNativeWrite(child.schema.fields) &&
+ BackendsApiManager.getSettings.enableNativeWriteFiles()
+ ) {
getNativeFormat(cmd)
} else {
None
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
