This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 72fc04e20d [CORE] Fix performance issue when allScanPartitions size is
very large (#8126)
72fc04e20d is described below
commit 72fc04e20d3dce09365bc5d5e40476f341c4620b
Author: WangGuangxin <[email protected]>
AuthorDate: Mon Dec 9 20:51:53 2024 +0800
[CORE] Fix performance issue when allScanPartitions size is very large
(#8126)
---
.../main/scala/org/apache/gluten/execution/WholeStageTransformer.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala
index dbfc11c136..a752f33ef6 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala
@@ -373,7 +373,7 @@ case class WholeStageTransformer(child: SparkPlan,
materializeInput: Boolean = f
* care of SCAN there won't be any other RDD for SCAN. As a result,
genFirstStageIterator
* rather than genFinalStageIterator will be invoked
*/
- val allScanPartitions = basicScanExecTransformers.map(_.getPartitions)
+ val allScanPartitions =
basicScanExecTransformers.map(_.getPartitions.toIndexedSeq)
val allScanSplitInfos =
getSplitInfosFromPartitions(basicScanExecTransformers,
allScanPartitions)
if (GlutenConfig.getConf.enableHdfsViewfs) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
