This is an automated email from the ASF dual-hosted git repository.
rui pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new c78032d553 [Substrait] Remove unused enable_row_group_maxmin_index
option (#11277)
c78032d553 is described below
commit c78032d553c5e3b82eb2a6f338ad7c4eacf7b51c
Author: David Sisson <[email protected]>
AuthorDate: Thu Dec 18 00:54:28 2025 -0800
[Substrait] Remove unused enable_row_group_maxmin_index option (#11277)
---
.../execution/iceberg/ClickHouseIcebergHiveTableSupport.scala | 1 -
.../execution/hive/GlutenClickHouseHiveTableSuite.scala | 1 -
.../apache/gluten/substrait/rel/IcebergLocalFilesNode.java | 11 ++---------
.../java/org/apache/gluten/substrait/rel/LocalFilesNode.java | 5 +----
.../main/resources/substrait/proto/substrait/algebra.proto | 4 +---
.../main/scala/org/apache/gluten/config/GlutenConfig.scala | 10 ----------
6 files changed, 4 insertions(+), 28 deletions(-)
diff --git
a/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergHiveTableSupport.scala
b/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergHiveTableSupport.scala
index efd26a8572..63b391ca7d 100644
---
a/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergHiveTableSupport.scala
+++
b/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergHiveTableSupport.scala
@@ -50,7 +50,6 @@ class ClickHouseIcebergHiveTableSupport {
.set("spark.gluten.sql.columnar.iterator", "true")
.set("spark.gluten.sql.columnar.hashagg.enablefinal", "true")
.set(GlutenConfig.NATIVE_VALIDATION_ENABLED.key, "false")
- .set(GlutenConfig.ENABLE_PARQUET_ROW_GROUP_MAX_MIN_INDEX.key, "true")
.set("spark.hive.exec.dynamic.partition.mode", "nonstrict")
.set(GlutenConfig.GLUTEN_SUPPORTED_HIVE_UDFS.key, "my_add")
.set("spark.shuffle.manager",
"org.apache.spark.shuffle.sort.ColumnarShuffleManager")
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala
index 7dfc1553e1..a95cc940f3 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala
@@ -56,7 +56,6 @@ class GlutenClickHouseHiveTableSuite
.set("spark.gluten.sql.columnar.iterator", "true")
.set("spark.gluten.sql.columnar.hashagg.enablefinal", "true")
.set(GlutenConfig.NATIVE_VALIDATION_ENABLED.key, "false")
- .set(GlutenConfig.ENABLE_PARQUET_ROW_GROUP_MAX_MIN_INDEX.key, "true")
.set(
"spark.sql.warehouse.dir",
this.getClass.getResource("/").getPath +
"tests-working-home/spark-warehouse")
diff --git
a/gluten-iceberg/src/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java
b/gluten-iceberg/src/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java
index 503cd84a78..0cc15d97d8 100644
---
a/gluten-iceberg/src/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java
+++
b/gluten-iceberg/src/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java
@@ -16,8 +16,6 @@
*/
package org.apache.gluten.substrait.rel;
-import org.apache.gluten.config.GlutenConfig;
-
import io.substrait.proto.ReadRel;
import org.apache.iceberg.DeleteFile;
@@ -63,9 +61,7 @@ public class IcebergLocalFilesNode extends LocalFilesNode {
switch (fileFormat) {
case ParquetReadFormat:
ReadRel.LocalFiles.FileOrFiles.ParquetReadOptions parquetReadOptions =
- ReadRel.LocalFiles.FileOrFiles.ParquetReadOptions.newBuilder()
-
.setEnableRowGroupMaxminIndex(GlutenConfig.get().enableParquetRowGroupMaxMinIndex())
- .build();
+
ReadRel.LocalFiles.FileOrFiles.ParquetReadOptions.newBuilder().build();
icebergBuilder.setParquet(parquetReadOptions);
break;
case OrcReadFormat:
@@ -102,10 +98,7 @@ public class IcebergLocalFilesNode extends LocalFilesNode {
switch (delete.format()) {
case PARQUET:
ReadRel.LocalFiles.FileOrFiles.ParquetReadOptions parquetReadOptions
=
- ReadRel.LocalFiles.FileOrFiles.ParquetReadOptions.newBuilder()
- .setEnableRowGroupMaxminIndex(
- GlutenConfig.get().enableParquetRowGroupMaxMinIndex())
- .build();
+
ReadRel.LocalFiles.FileOrFiles.ParquetReadOptions.newBuilder().build();
deleteFileBuilder.setParquet(parquetReadOptions);
break;
case ORC:
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/rel/LocalFilesNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/rel/LocalFilesNode.java
index 5a1b106db8..bc42dd302e 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/rel/LocalFilesNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/rel/LocalFilesNode.java
@@ -216,10 +216,7 @@ public class LocalFilesNode implements SplitInfo {
switch (fileFormat) {
case ParquetReadFormat:
ReadRel.LocalFiles.FileOrFiles.ParquetReadOptions parquetReadOptions
=
- ReadRel.LocalFiles.FileOrFiles.ParquetReadOptions.newBuilder()
- .setEnableRowGroupMaxminIndex(
- GlutenConfig.get().enableParquetRowGroupMaxMinIndex())
- .build();
+
ReadRel.LocalFiles.FileOrFiles.ParquetReadOptions.newBuilder().build();
fileBuilder.setParquet(parquetReadOptions);
break;
case OrcReadFormat:
diff --git
a/gluten-substrait/src/main/resources/substrait/proto/substrait/algebra.proto
b/gluten-substrait/src/main/resources/substrait/proto/substrait/algebra.proto
index 176a4e4c25..7d72332baa 100644
---
a/gluten-substrait/src/main/resources/substrait/proto/substrait/algebra.proto
+++
b/gluten-substrait/src/main/resources/substrait/proto/substrait/algebra.proto
@@ -146,9 +146,7 @@ message ReadRel {
// The length in byte to read from this item
uint64 length = 8;
- message ParquetReadOptions {
- bool enable_row_group_maxmin_index = 1;
- }
+ message ParquetReadOptions {}
message ArrowReadOptions {}
message OrcReadOptions {}
message DwrfReadOptions {}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
index 40d668091b..aa18973328 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
@@ -331,9 +331,6 @@ class GlutenConfig(conf: SQLConf) extends
GlutenCoreConfig(conf) {
def textIputEmptyAsDefault: Boolean = getConf(TEXT_INPUT_EMPTY_AS_DEFAULT)
- def enableParquetRowGroupMaxMinIndex: Boolean =
- getConf(ENABLE_PARQUET_ROW_GROUP_MAX_MIN_INDEX)
-
// Please use `BackendsApiManager.getSettings.enableNativeWriteFiles()`
instead
def enableNativeWriter: Option[Boolean] = getConf(NATIVE_WRITER_ENABLED)
@@ -1356,13 +1353,6 @@ object GlutenConfig extends ConfigRegistry {
.booleanConf
.createWithDefault(false);
- val ENABLE_PARQUET_ROW_GROUP_MAX_MIN_INDEX =
- buildConf("spark.gluten.sql.parquet.maxmin.index")
- .internal()
- .doc("Enable row group max min index for parquet file scan")
- .booleanConf
- .createWithDefault(false)
-
val ENABLE_REWRITE_DATE_TIMESTAMP_COMPARISON =
buildConf("spark.gluten.sql.rewrite.dateTimestampComparison")
.doc(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
