This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 64941e696 [GLUTEN-6990][VL] Remove config
`a.g.s.c.extended.columnar.transform.rules` and
`a.g.s.c.extended.columnar.post.rules` from Velox backend (#6991)
64941e696 is described below
commit 64941e6967c7181726144b14365c80daa0b0a539
Author: Hongze Zhang <[email protected]>
AuthorDate: Fri Aug 23 16:01:01 2024 +0800
[GLUTEN-6990][VL] Remove config `a.g.s.c.extended.columnar.transform.rules`
and `a.g.s.c.extended.columnar.post.rules` from Velox backend (#6991)
Closes #6990
---
.../org/apache/gluten/backendsapi/velox/VeloxRuleApi.scala | 11 +----------
.../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 13 ++++++-------
.../src/main/scala/org/apache/gluten/GlutenConfig.scala | 2 ++
3 files changed, 9 insertions(+), 17 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxRuleApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxRuleApi.scala
index f152da885..abb39c5bb 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxRuleApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxRuleApi.scala
@@ -18,7 +18,7 @@ package org.apache.gluten.backendsapi.velox
import org.apache.gluten.backendsapi.RuleApi
import org.apache.gluten.datasource.ArrowConvertorRule
-import org.apache.gluten.extension.{ArrowScanReplaceRule,
BloomFilterMightContainJointRewriteRule, CollectRewriteRule,
FlushableHashAggregateRule, HLLRewriteRule}
+import org.apache.gluten.extension._
import
org.apache.gluten.extension.EmptySchemaWorkaround.{FallbackEmptySchemaRelation,
PlanOneRowRelation}
import org.apache.gluten.extension.columnar._
import
org.apache.gluten.extension.columnar.MiscColumnarRules.{RemoveGlutenTableCacheColumnarToRow,
RemoveTopmostColumnarToRow, RewriteSubqueryBroadcast, TransformPreOverrides}
@@ -31,7 +31,6 @@ import org.apache.gluten.sql.shims.SparkShimLoader
import org.apache.spark.sql.execution.{ColumnarCollapseTransformStages,
GlutenFallbackReporter}
import org.apache.spark.sql.expression.UDFResolver
-import org.apache.spark.util.SparkPlanRules
class VeloxRuleApi extends RuleApi {
import VeloxRuleApi._
@@ -71,8 +70,6 @@ private object VeloxRuleApi {
injector.injectTransform(_ => EliminateLocalSort)
injector.injectTransform(_ => CollapseProjectExecTransformer)
injector.injectTransform(c => FlushableHashAggregateRule.apply(c.session))
- injector.injectTransform(
- c =>
SparkPlanRules.extendedColumnarRule(c.conf.extendedColumnarTransformRules)(c.session))
injector.injectTransform(c => InsertTransitions(c.outputsColumnar))
// Gluten columnar: Fallback policies.
@@ -85,8 +82,6 @@ private object VeloxRuleApi {
.getExtendedColumnarPostRules()
.foreach(each => injector.injectPost(c => each(c.session)))
injector.injectPost(c => ColumnarCollapseTransformStages(c.conf))
- injector.injectTransform(
- c =>
SparkPlanRules.extendedColumnarRule(c.conf.extendedColumnarPostRules)(c.session))
// Gluten columnar: Final rules.
injector.injectFinal(c => RemoveGlutenTableCacheColumnarToRow(c.session))
@@ -115,16 +110,12 @@ private object VeloxRuleApi {
injector.inject(_ => EliminateLocalSort)
injector.inject(_ => CollapseProjectExecTransformer)
injector.inject(c => FlushableHashAggregateRule.apply(c.session))
- injector.inject(
- c =>
SparkPlanRules.extendedColumnarRule(c.conf.extendedColumnarTransformRules)(c.session))
injector.inject(c => InsertTransitions(c.outputsColumnar))
injector.inject(c => RemoveTopmostColumnarToRow(c.session,
c.ac.isAdaptiveContext()))
SparkShimLoader.getSparkShims
.getExtendedColumnarPostRules()
.foreach(each => injector.inject(c => each(c.session)))
injector.inject(c => ColumnarCollapseTransformStages(c.conf))
- injector.inject(
- c =>
SparkPlanRules.extendedColumnarRule(c.conf.extendedColumnarPostRules)(c.session))
injector.inject(c => RemoveGlutenTableCacheColumnarToRow(c.session))
injector.inject(c => GlutenFallbackReporter(c.conf, c.session))
injector.inject(_ => RemoveFallbackTagRule())
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index f8a075ec0..c4799366d 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -21,22 +21,22 @@ import org.apache.gluten.utils.{BackendTestSettings,
SQLQueryTestSettings}
import org.apache.spark.GlutenSortShuffleSuite
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.expressions._
-import
org.apache.spark.sql.connector.{GlutenDataSourceV2DataFrameSessionCatalogSuite,
GlutenDataSourceV2DataFrameSuite, GlutenDataSourceV2FunctionSuite,
GlutenDataSourceV2SQLSessionCatalogSuite, GlutenDataSourceV2SQLSuite,
GlutenDataSourceV2Suite, GlutenFileDataSourceV2FallBackSuite,
GlutenLocalScanSuite, GlutenSupportsCatalogOptionsSuite,
GlutenTableCapabilityCheckSuite, GlutenWriteDistributionAndOrderingSuite}
+import org.apache.spark.sql.connector._
import org.apache.spark.sql.execution._
import
org.apache.spark.sql.execution.adaptive.velox.VeloxAdaptiveQueryExecSuite
-import org.apache.spark.sql.execution.datasources.{GlutenBucketingUtilsSuite,
GlutenCSVReadSchemaSuite, GlutenDataSourceStrategySuite, GlutenDataSourceSuite,
GlutenFileFormatWriterSuite, GlutenFileIndexSuite,
GlutenFileSourceStrategySuite, GlutenHadoopFileLinesReaderSuite,
GlutenHeaderCSVReadSchemaSuite, GlutenJsonReadSchemaSuite,
GlutenMergedOrcReadSchemaSuite, GlutenMergedParquetReadSchemaSuite,
GlutenOrcCodecSuite, GlutenOrcReadSchemaSuite, GlutenParquetCodecSuite,
GlutenParquetReadSc [...]
+import org.apache.spark.sql.execution.datasources._
import
org.apache.spark.sql.execution.datasources.binaryfile.GlutenBinaryFileFormatSuite
import
org.apache.spark.sql.execution.datasources.csv.{GlutenCSVLegacyTimeParserSuite,
GlutenCSVv1Suite, GlutenCSVv2Suite}
import
org.apache.spark.sql.execution.datasources.json.{GlutenJsonLegacyTimeParserSuite,
GlutenJsonV1Suite, GlutenJsonV2Suite}
-import
org.apache.spark.sql.execution.datasources.orc.{GlutenOrcColumnarBatchReaderSuite,
GlutenOrcFilterSuite, GlutenOrcPartitionDiscoverySuite, GlutenOrcSourceSuite,
GlutenOrcV1FilterSuite, GlutenOrcV1PartitionDiscoverySuite,
GlutenOrcV1QuerySuite, GlutenOrcV1SchemaPruningSuite, GlutenOrcV2QuerySuite,
GlutenOrcV2SchemaPruningSuite}
-import
org.apache.spark.sql.execution.datasources.parquet.{GlutenParquetColumnIndexSuite,
GlutenParquetCompressionCodecPrecedenceSuite, GlutenParquetEncodingSuite,
GlutenParquetFileFormatV1Suite, GlutenParquetFileFormatV2Suite,
GlutenParquetInteroperabilitySuite, GlutenParquetIOSuite,
GlutenParquetProtobufCompatibilitySuite, GlutenParquetRebaseDatetimeV1Suite,
GlutenParquetRebaseDatetimeV2Suite, GlutenParquetSchemaInferenceSuite,
GlutenParquetSchemaSuite, GlutenParquetThriftCompatibility [...]
+import org.apache.spark.sql.execution.datasources.orc._
+import org.apache.spark.sql.execution.datasources.parquet._
import org.apache.spark.sql.execution.datasources.text.{GlutenTextV1Suite,
GlutenTextV2Suite}
import org.apache.spark.sql.execution.datasources.v2.GlutenFileTableSuite
import org.apache.spark.sql.execution.exchange.GlutenEnsureRequirementsSuite
import org.apache.spark.sql.execution.joins.{GlutenBroadcastJoinSuite,
GlutenExistenceJoinSuite, GlutenInnerJoinSuite, GlutenOuterJoinSuite}
-import
org.apache.spark.sql.extension.{GlutenCollapseProjectExecTransformerSuite,
GlutenCustomerExpressionTransformerSuite, GlutenCustomerExtensionSuite,
GlutenSessionExtensionSuite}
+import
org.apache.spark.sql.extension.{GlutenCollapseProjectExecTransformerSuite,
GlutenCustomerExpressionTransformerSuite, GlutenSessionExtensionSuite}
import org.apache.spark.sql.hive.execution.GlutenHiveSQLQuerySuite
-import
org.apache.spark.sql.sources.{GlutenBucketedReadWithoutHiveSupportSuite,
GlutenBucketedWriteWithoutHiveSupportSuite, GlutenCreateTableAsSelectSuite,
GlutenDDLSourceLoadSuite,
GlutenDisableUnnecessaryBucketedScanWithoutHiveSupportSuite,
GlutenDisableUnnecessaryBucketedScanWithoutHiveSupportSuiteAE,
GlutenExternalCommandRunnerSuite, GlutenFilteredScanSuite, GlutenFiltersSuite,
GlutenInsertSuite, GlutenPartitionedWriteSuite, GlutenPathOptionSuite,
GlutenPrunedScanSuite, GlutenResolve [...]
+import org.apache.spark.sql.sources._
// Some settings' line length exceeds 100
// scalastyle:off line.size.limit
@@ -44,7 +44,6 @@ import
org.apache.spark.sql.sources.{GlutenBucketedReadWithoutHiveSupportSuite,
class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenSessionExtensionSuite]
- enableSuite[GlutenCustomerExtensionSuite]
enableSuite[GlutenCustomerExpressionTransformerSuite]
enableSuite[GlutenDataFrameAggregateSuite]
diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
index fa78060da..9e5161fea 100644
--- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
+++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
@@ -1657,6 +1657,7 @@ object GlutenConfig {
.booleanConf
.createWithDefault(false)
+ // FIXME: This only works with CH backend.
val EXTENDED_COLUMNAR_TRANSFORM_RULES =
buildConf("spark.gluten.sql.columnar.extended.columnar.transform.rules")
.withAlternative("spark.gluten.sql.columnar.extended.columnar.pre.rules")
@@ -1664,6 +1665,7 @@ object GlutenConfig {
.stringConf
.createWithDefaultString("")
+ // FIXME: This only works with CH backend.
val EXTENDED_COLUMNAR_POST_RULES =
buildConf("spark.gluten.sql.columnar.extended.columnar.post.rules")
.doc("A comma-separated list of classes for the extended columnar post
rules.")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
