This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
from 4812fd2b21 [spark] Fallback to spark except query if increment query
with rescale bucket (#4989)
add 9d6fe5f76b [parquet] Refactory parquet reader using spark code. (#4982)
No new revisions were added by this update.
Summary of changes:
LICENSE | 14 +
.../converter/Arrow2PaimonVectorConverter.java | 11 +-
.../paimon/arrow/writer/ArrowFieldWriters.java | 4 +-
.../apache/paimon/data/columnar/ColumnVector.java | 9 +
.../paimon/data/columnar/ColumnarRowIterator.java | 21 +-
.../paimon/data/columnar/MapColumnVector.java | 4 -
.../data/columnar/VectorizedRowIterator.java | 2 +-
.../columnar/heap/AbstractArrayBasedVector.java | 80 ++
.../data/columnar/heap/AbstractHeapVector.java | 33 +-
.../data/columnar/heap/AbstractStructVector.java | 35 +-
.../columnar/heap/CastedArrayColumnVector.java | 51 +-
.../data/columnar/heap/CastedMapColumnVector.java | 61 ++
...apRowVector.java => CastedRowColumnVector.java} | 49 +-
.../data/columnar/heap/ElementCountable.java | 2 +-
.../paimon/data/columnar/heap/HeapArrayVector.java | 54 +-
.../data/columnar/heap/HeapBooleanVector.java | 37 +
.../paimon/data/columnar/heap/HeapByteVector.java | 17 +
.../paimon/data/columnar/heap/HeapBytesVector.java | 77 +-
.../data/columnar/heap/HeapDoubleVector.java | 17 +
.../paimon/data/columnar/heap/HeapFloatVector.java | 17 +
.../paimon/data/columnar/heap/HeapIntVector.java | 37 +
.../paimon/data/columnar/heap/HeapLongVector.java | 17 +
.../paimon/data/columnar/heap/HeapMapVector.java | 50 +-
.../paimon/data/columnar/heap/HeapRowVector.java | 33 +-
.../paimon/data/columnar/heap/HeapShortVector.java | 17 +
.../data/columnar/heap/HeapTimestampVector.java | 27 +-
.../columnar/writable/AbstractWritableVector.java | 72 +-
.../columnar/writable/WritableBooleanVector.java | 6 +
.../columnar/writable/WritableBytesVector.java | 2 +-
.../columnar/writable/WritableColumnVector.java | 16 +
.../data/columnar/writable/WritableIntVector.java | 4 +
.../apache/paimon/utils/VectorMappingUtils.java | 24 -
.../data/calumnar/heap/RowColumnVectorTest.java | 69 ++
.../apache/paimon/format/FormatReadWriteTest.java | 99 +-
.../apache/paimon/flink/PreAggregationITCase.java | 135 ---
.../format/orc/reader/OrcMapColumnVector.java | 9 +-
.../format/parquet/ParquetReaderFactory.java | 39 +-
.../format/parquet/newreader/ColumnarBatch.java | 55 +
.../parquet/newreader/ParquetColumnVector.java | 378 +++++++
.../{reader => newreader}/ParquetReadState.java | 96 +-
.../parquet/newreader/ParquetVectorUpdater.java | 98 ++
.../newreader/ParquetVectorUpdaterFactory.java | 811 +++++++++++++++
.../parquet/newreader/RowIndexGenerator.java | 66 ++
.../parquet/newreader/VectorizedColumnReader.java | 369 +++++++
.../VectorizedDeltaBinaryPackedReader.java | 299 ++++++
.../newreader/VectorizedDeltaByteArrayReader.java | 153 +++
.../VectorizedDeltaLengthByteArrayReader.java | 95 ++
.../newreader/VectorizedParquetRecordReader.java | 349 +++++++
.../newreader/VectorizedPlainValuesReader.java | 319 ++++++
.../parquet/newreader/VectorizedReaderBase.java | 147 +++
.../newreader/VectorizedRleValuesReader.java | 1046 ++++++++++++++++++++
.../parquet/newreader/VectorizedValuesReader.java | 122 +++
.../format/parquet/reader/BytesColumnReader.java | 6 +-
.../reader/FixedLenBytesBinaryColumnReader.java | 4 +-
.../reader/FixedLenBytesDecimalColumnReader.java | 4 +-
.../format/parquet/reader/NestedColumnReader.java | 13 +-
.../reader/NestedPrimitiveColumnReader.java | 10 +-
.../parquet/reader/ParquetDecimalVector.java | 94 +-
.../parquet/reader/ParquetSplitReaderUtil.java | 39 +-
.../format/parquet/reader/RowColumnReader.java | 2 +-
.../paimon/format/parquet/type/ParquetField.java | 15 +-
.../format/parquet/type/ParquetGroupField.java | 10 +-
.../format/parquet/type/ParquetPrimitiveField.java | 10 +-
.../format/parquet/ParquetColumnVectorTest.java | 466 ++-------
.../format/parquet/ParquetReadWriteTest.java | 8 +-
.../spark/sql/InsertOverwriteTableTestBase.scala | 12 +-
66 files changed, 5389 insertions(+), 958 deletions(-)
create mode 100644
paimon-common/src/main/java/org/apache/paimon/data/columnar/heap/AbstractArrayBasedVector.java
copy
paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldFirstNonNullValueAgg.java
=>
paimon-common/src/main/java/org/apache/paimon/data/columnar/heap/AbstractStructVector.java
(53%)
copy
paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcArrayColumnVector.java
=>
paimon-common/src/main/java/org/apache/paimon/data/columnar/heap/CastedArrayColumnVector.java
(51%)
create mode 100644
paimon-common/src/main/java/org/apache/paimon/data/columnar/heap/CastedMapColumnVector.java
copy
paimon-common/src/main/java/org/apache/paimon/data/columnar/heap/{HeapRowVector.java
=> CastedRowColumnVector.java} (53%)
create mode 100644
paimon-common/src/test/java/org/apache/paimon/data/calumnar/heap/RowColumnVectorTest.java
create mode 100644
paimon-format/src/main/java/org/apache/paimon/format/parquet/newreader/ColumnarBatch.java
create mode 100644
paimon-format/src/main/java/org/apache/paimon/format/parquet/newreader/ParquetColumnVector.java
copy paimon-format/src/main/java/org/apache/paimon/format/parquet/{reader =>
newreader}/ParquetReadState.java (57%)
create mode 100644
paimon-format/src/main/java/org/apache/paimon/format/parquet/newreader/ParquetVectorUpdater.java
create mode 100644
paimon-format/src/main/java/org/apache/paimon/format/parquet/newreader/ParquetVectorUpdaterFactory.java
create mode 100644
paimon-format/src/main/java/org/apache/paimon/format/parquet/newreader/RowIndexGenerator.java
create mode 100644
paimon-format/src/main/java/org/apache/paimon/format/parquet/newreader/VectorizedColumnReader.java
create mode 100644
paimon-format/src/main/java/org/apache/paimon/format/parquet/newreader/VectorizedDeltaBinaryPackedReader.java
create mode 100644
paimon-format/src/main/java/org/apache/paimon/format/parquet/newreader/VectorizedDeltaByteArrayReader.java
create mode 100644
paimon-format/src/main/java/org/apache/paimon/format/parquet/newreader/VectorizedDeltaLengthByteArrayReader.java
create mode 100644
paimon-format/src/main/java/org/apache/paimon/format/parquet/newreader/VectorizedParquetRecordReader.java
create mode 100644
paimon-format/src/main/java/org/apache/paimon/format/parquet/newreader/VectorizedPlainValuesReader.java
create mode 100644
paimon-format/src/main/java/org/apache/paimon/format/parquet/newreader/VectorizedReaderBase.java
create mode 100644
paimon-format/src/main/java/org/apache/paimon/format/parquet/newreader/VectorizedRleValuesReader.java
create mode 100644
paimon-format/src/main/java/org/apache/paimon/format/parquet/newreader/VectorizedValuesReader.java
