This is an automated email from the ASF dual-hosted git repository.
kazuyukitanimura pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new e2f5df6d04 chor: remove legacy use.lazyMaterialization (#4998)
e2f5df6d04 is described below
commit e2f5df6d04608e6968d837346c90738a6d066a10
Author: KAZUYUKI TANIMURA <[email protected]>
AuthorDate: Wed Jul 22 11:10:30 2026 -0700
chor: remove legacy use.lazyMaterialization (#4998)
## Which issue does this PR close?
## Rationale for this change
`spark.comet.use.lazyMaterialization` is not a real option anymore, only
used in tests without actual lazy materialization code.
## What changes are included in this PR?
Removed the option
## How are these changes tested?
Existing tests
---
spark/src/main/scala/org/apache/comet/CometConf.scala | 11 -----------
.../org/apache/comet/parquet/ParquetReadSuite.scala | 19 ++++++++-----------
2 files changed, 8 insertions(+), 22 deletions(-)
diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala
b/spark/src/main/scala/org/apache/comet/CometConf.scala
index 1b71486f09..958a22cc2b 100644
--- a/spark/src/main/scala/org/apache/comet/CometConf.scala
+++ b/spark/src/main/scala/org/apache/comet/CometConf.scala
@@ -704,17 +704,6 @@ object CometConf extends ShimCometConf {
.booleanConf
.createWithDefault(false)
- val COMET_USE_LAZY_MATERIALIZATION: ConfigEntry[Boolean] = conf(
- "spark.comet.use.lazyMaterialization")
- .internal()
- .category(CATEGORY_PARQUET)
- .doc(
- "Whether to enable lazy materialization for Comet. When this is turned
on, Comet will " +
- "read Parquet data source lazily for string and binary columns. For
filter operations, " +
- "lazy materialization will improve read performance by skipping unused
pages.")
- .booleanConf
- .createWithDefault(true)
-
val COMET_ENABLE_PARTIAL_HASH_AGGREGATE: ConfigEntry[Boolean] =
conf("spark.comet.testing.aggregate.partialMode.enabled")
.internal()
diff --git
a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala
b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala
index c294837d10..688578de37 100644
--- a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala
@@ -1290,17 +1290,14 @@ abstract class ParquetReadSuite extends CometTestBase {
.option("parquet.enable.dictionary", enableDictionary.toString)
.parquet(file.getCanonicalPath)
- Seq(true, false).foreach { useLazyMaterialization =>
- Seq(true, false).foreach { enableCometExec =>
- Seq(4, 13, 4049).foreach { batchSize =>
- withSQLConf(
- CometConf.COMET_BATCH_SIZE.key -> batchSize.toString,
- CometConf.COMET_EXEC_ENABLED.key -> enableCometExec.toString,
- CometConf.COMET_USE_LAZY_MATERIALIZATION.key ->
useLazyMaterialization.toString) {
- readParquetFile(file.getCanonicalPath) { parquetDf =>
- actions.foreach { action =>
- checkAnswer(action(parquetDf), action(df))
- }
+ Seq(true, false).foreach { enableCometExec =>
+ Seq(4, 13, 4049).foreach { batchSize =>
+ withSQLConf(
+ CometConf.COMET_BATCH_SIZE.key -> batchSize.toString,
+ CometConf.COMET_EXEC_ENABLED.key -> enableCometExec.toString) {
+ readParquetFile(file.getCanonicalPath) { parquetDf =>
+ actions.foreach { action =>
+ checkAnswer(action(parquetDf), action(df))
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
