This is an automated email from the ASF dual-hosted git repository.
peter-toth pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
from 6d020ca9295e [SPARK-59683][SQL][HIVE] Support first-class CHAR/VARCHAR
in script TRANSFORM output
add 0c14861e3231 [SPARK-59620][SQL] Late-materialization storage-filter
pushdown in the vectorized Parquet reader
No new revisions were added by this update.
Summary of changes:
.../org/apache/spark/sql/internal/SQLConf.scala | 62 +
...torageFilterPushdownBenchmark-jdk21-results.txt | 564 +++
...torageFilterPushdownBenchmark-jdk25-results.txt | 564 +++
.../StorageFilterPushdownBenchmark-results.txt | 564 +++
.../LateMaterializationParquetRecordReader.java | 1032 ++++++
.../parquet/ParquetBlockAccounting.java | 184 +
.../datasources/parquet/ParquetReadState.java | 200 +-
.../parquet/SpecificParquetRecordReaderBase.java | 28 +-
.../execution/datasources/parquet/ValueCopier.java | 135 +
.../parquet/VectorizedColumnReader.java | 25 +-
.../parquet/VectorizedParquetRecordReader.java | 110 +-
.../spark/sql/execution/DataSourceScanExec.scala | 77 +-
.../sql/execution/datasources/FileFormat.scala | 78 +
.../execution/datasources/FileSourceStrategy.scala | 65 +-
.../datasources/parquet/ParquetFileFormat.scala | 121 +-
.../datasources/parquet/ParquetRowIndexUtil.scala | 5 +
.../datasources/parquet/ParquetStorageFilter.scala | 355 ++
.../org/apache/spark/sql/DataFrameJoinSuite.scala | 3 +-
.../scala/org/apache/spark/sql/SubquerySuite.scala | 2 +-
.../benchmark/StorageFilterPushdownBenchmark.scala | 302 ++
.../parquet/ParquetStorageFilterSuite.scala | 3698 ++++++++++++++++++++
.../datasources/parquet/ParquetTestAccess.scala | 134 +-
.../parquet/VectorizedRleValuesReaderSuite.scala | 48 +-
23 files changed, 8212 insertions(+), 144 deletions(-)
create mode 100644
sql/core/benchmarks/StorageFilterPushdownBenchmark-jdk21-results.txt
create mode 100644
sql/core/benchmarks/StorageFilterPushdownBenchmark-jdk25-results.txt
create mode 100644
sql/core/benchmarks/StorageFilterPushdownBenchmark-results.txt
create mode 100644
sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/LateMaterializationParquetRecordReader.java
create mode 100644
sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/ParquetBlockAccounting.java
create mode 100644
sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/ValueCopier.java
create mode 100644
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetStorageFilter.scala
create mode 100644
sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/StorageFilterPushdownBenchmark.scala
create mode 100644
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetStorageFilterSuite.scala
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
