This is an automated email from the ASF dual-hosted git repository.
zhli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 493a0abe45 [VL] Code clean for hasUnsupportedColumns function (#7477)
493a0abe45 is described below
commit 493a0abe459018e92e85d314a7e7467c694cbfcd
Author: Zhen Li <[email protected]>
AuthorDate: Sat Oct 12 16:26:54 2024 +0800
[VL] Code clean for hasUnsupportedColumns function (#7477)
[VL] Code clean for hasUnsupportedColumns function.
---
.../org/apache/spark/sql/execution/FileSourceScanExecShim.scala | 5 +----
.../spark/sql/execution/datasources/v2/BatchScanExecShim.scala | 5 +----
.../org/apache/spark/sql/execution/FileSourceScanExecShim.scala | 3 +--
.../spark/sql/execution/datasources/v2/BatchScanExecShim.scala | 3 +--
.../org/apache/spark/sql/execution/FileSourceScanExecShim.scala | 4 +---
.../spark/sql/execution/datasources/v2/BatchScanExecShim.scala | 3 +--
.../org/apache/spark/sql/execution/FileSourceScanExecShim.scala | 3 +--
.../spark/sql/execution/datasources/v2/BatchScanExecShim.scala | 3 +--
8 files changed, 8 insertions(+), 21 deletions(-)
diff --git
a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala
b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala
index 77b3b4809b..5439d43755 100644
---
a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala
+++
b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala
@@ -58,10 +58,7 @@ abstract class FileSourceScanExecShim(
def metadataColumns: Seq[AttributeReference] = Seq.empty
- def hasUnsupportedColumns: Boolean = {
- // Below name has special meaning in Velox.
- output.exists(a => a.name == "$path" || a.name == "$bucket")
- }
+ def hasUnsupportedColumns: Boolean = false
def isMetadataColumn(attr: Attribute): Boolean = false
diff --git
a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala
b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala
index e445dd33a5..dcfb1c1447 100644
---
a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala
+++
b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala
@@ -44,10 +44,7 @@ abstract class BatchScanExecShim(
def metadataColumns: Seq[AttributeReference] = Seq.empty
- def hasUnsupportedColumns: Boolean = {
- // Below name has special meaning in Velox.
- output.exists(a => a.name == "$path" || a.name == "$bucket")
- }
+ def hasUnsupportedColumns: Boolean = false
override def doExecuteColumnar(): RDD[ColumnarBatch] = {
throw new UnsupportedOperationException("Need to implement this method")
diff --git
a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala
b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala
index ffc8684f52..7815e93827 100644
---
a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala
+++
b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala
@@ -66,8 +66,7 @@ abstract class FileSourceScanExecShim(
val metadataColumnsNames = metadataColumns.map(_.name)
output
.filterNot(metadataColumns.toSet)
- .exists(v => metadataColumnsNames.contains(v.name)) ||
- output.exists(a => a.name == "$path" || a.name == "$bucket")
+ .exists(v => metadataColumnsNames.contains(v.name))
}
def isMetadataColumn(attr: Attribute): Boolean =
metadataColumns.contains(attr)
diff --git
a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala
b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala
index 06eb69a359..ec5c097d97 100644
---
a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala
+++
b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala
@@ -56,8 +56,7 @@ abstract class BatchScanExecShim(
val metadataColumnsNames = metadataColumns.map(_.name)
output
.filterNot(metadataColumns.toSet)
- .exists(v => metadataColumnsNames.contains(v.name)) ||
- output.exists(a => a.name == "$path" || a.name == "$bucket")
+ .exists(v => metadataColumnsNames.contains(v.name))
}
override def doExecuteColumnar(): RDD[ColumnarBatch] = {
diff --git
a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala
b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala
index 356438d3cf..0343942bf3 100644
---
a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala
+++
b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala
@@ -65,9 +65,7 @@ abstract class FileSourceScanExecShim(
val metadataColumnsNames = metadataColumns.map(_.name)
output
.filterNot(metadataColumns.toSet)
- .exists(v => metadataColumnsNames.contains(v.name)) ||
- // Below name has special meaning in Velox.
- output.exists(a => a.name == "$path" || a.name == "$bucket")
+ .exists(v => metadataColumnsNames.contains(v.name))
}
def isMetadataColumn(attr: Attribute): Boolean =
metadataColumns.contains(attr)
diff --git
a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala
b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala
index 5fdc3b04d5..32698d2ad8 100644
---
a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala
+++
b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala
@@ -65,8 +65,7 @@ abstract class BatchScanExecShim(
val metadataColumnsNames = metadataColumns.map(_.name)
output
.filterNot(metadataColumns.toSet)
- .exists(v => metadataColumnsNames.contains(v.name)) ||
- output.exists(a => a.name == "$path" || a.name == "$bucket")
+ .exists(v => metadataColumnsNames.contains(v.name))
}
override def doExecuteColumnar(): RDD[ColumnarBatch] = {
diff --git
a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala
b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala
index 5ec4499ec6..463afbbca4 100644
---
a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala
+++
b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala
@@ -69,8 +69,7 @@ abstract class FileSourceScanExecShim(
val metadataColumnsNames = metadataColumns.map(_.name)
output
.filterNot(metadataColumns.toSet)
- .exists(v => metadataColumnsNames.contains(v.name)) ||
- output.exists(a => a.name == "$path" || a.name == "$bucket")
+ .exists(v => metadataColumnsNames.contains(v.name))
}
def isMetadataColumn(attr: Attribute): Boolean =
metadataColumns.contains(attr)
diff --git
a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala
b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala
index fb3078d397..905a838a11 100644
---
a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala
+++
b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala
@@ -67,8 +67,7 @@ abstract class BatchScanExecShim(
val metadataColumnsNames = metadataColumns.map(_.name)
output
.filterNot(metadataColumns.toSet)
- .exists(v => metadataColumnsNames.contains(v.name)) ||
- output.exists(a => a.name == "$path" || a.name == "$bucket")
+ .exists(v => metadataColumnsNames.contains(v.name))
}
override def doExecuteColumnar(): RDD[ColumnarBatch] = {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
