This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 7a1357514f [VL] Remove unused logic in
ColumnarCachedBatchSerializer#supportsColumnarInput (#9413)
7a1357514f is described below
commit 7a1357514f26f2be74bd95601859aa11aa13ce5d
Author: Hongze Zhang <[email protected]>
AuthorDate: Thu Apr 24 16:48:45 2025 +0100
[VL] Remove unused logic in
ColumnarCachedBatchSerializer#supportsColumnarInput (#9413)
---
.../spark/sql/execution/ColumnarCachedBatchSerializer.scala | 13 +------------
1 file changed, 1 insertion(+), 12 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala
b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala
index e86d49ccef..1d52e57ab2 100644
---
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala
+++
b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala
@@ -101,18 +101,7 @@ class ColumnarCachedBatchSerializer extends
CachedBatchSerializer with Logging {
}
override def supportsColumnarInput(schema: Seq[Attribute]): Boolean = {
- // Note, there is a issue that, if gluten columnar scan is disabled and
vanilla Spark
- // columnar is enabled, then the following plan would fail.
- // InMemoryTableScan
- // InMemoryRelation
- // (vanilla Spark columnar Scan) Parquet
- // The reason is that, Spark will remove the top level `ColumnarToRow` and
call
- // `convertColumnarBatchToCachedBatch`, but the inside ColumnarBatch is
not arrow-based.
- // See: `InMemoryRelation.apply()`.
- // So we should disallow columnar input if using vanilla Spark columnar
scan.
- val noVanillaSparkColumnarScan = glutenConf.enableColumnarFileScan ||
- !glutenConf.getConf(GlutenConfig.VANILLA_VECTORIZED_READERS_ENABLED)
- glutenConf.enableGluten && validateSchema(schema) &&
noVanillaSparkColumnarScan
+ glutenConf.enableGluten && validateSchema(schema)
}
override def supportsColumnarOutput(schema: StructType): Boolean = {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
