This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a change to branch release-0.8
in repository https://gitbox.apache.org/repos/asf/paimon.git
from aa92afaec [core] Add listDirectories to FileIO (#3205)
add a5d1b3c77 [flink] Add merge-engine check when executing row level
batch update and delete (#3181)
add 9bb755dd8 [test] By removing all default exclusions the nested classes
get executed (#3211)
add 8ff2537b2 [core] Introduce RawFileSplitRead to accelerate batch read
for primary key table (#3209)
add ffb903287 [api] Publish DataStream Sink & Source API (#3192)
add b3eeea91e [core] Introduce file index read/write framework. (#3177)
add 1ae041bde [doc] Update directory-maven-plugin to 1.0 version (#3213)
add c46c40b9d [core] Unify delete&update&mergeinto for merge engines
(#3208)
add 815dc698a [flink] Fixed the issue where deletes could not be performed
when the table is in dynamic bucket mode. (#3214)
add 5e10f732b [doc] Add file-creation-time-millis example in flink-api
add eb4086bec [core] Introduce TTL for tag
add 0bd955cb3 [core] Adjust codes TTL for tag
add c05cbcade [core] Expose columnar data in
RowColumnVector/ArrayColumnVector/MapColumnVector (#3207)
add e27ceb464 Default parallelism also considers numShufflePartitions
(#3218)
add 85bd8a35c [ci] Use hugo v0.124.1 to build documents (#3239)
add 3686cce72 [test] Fix unstable test case
CatalogITCaseBase#testTagsTable (#3238)
add 9c2e86def [spark] Change SQLConfUtils to scala (#3242)
add 47174b99d [flink] Check parallelism of committer operator while
runtime. (#3237)
add 80cf3fcb4 [core] fix memory leak bug of bulkLoader (#3243)
add b1b0acd79 [core] Increase default sorted run stop trigger (#3220)
No new revisions were added by this update.
Summary of changes:
.github/workflows/docs-tests.yml | 2 +-
docs/content/flink/action-jars.md | 4 +-
docs/content/maintenance/manage-tags.md | 7 +
docs/content/maintenance/write-performance.md | 2 +-
docs/content/program-api/flink-api.md | 116 ++++-----
.../shortcodes/generated/core_configuration.html | 22 +-
.../main/java/org/apache/paimon/CoreOptions.java | 120 +++++++++-
.../paimon/data/columnar/ArrayColumnVector.java | 2 +
.../paimon/data/columnar/MapColumnVector.java | 4 +
.../paimon/data/columnar/RowColumnVector.java | 2 +
.../paimon/data/columnar/heap/HeapArrayVector.java | 5 +
.../paimon/data/columnar/heap/HeapMapVector.java | 10 +
.../paimon/data/columnar/heap/HeapRowVector.java | 5 +
.../apache/paimon/fileindex/FileIndexFormat.java | 17 +-
.../apache/paimon/fileindex/FileIndexOptions.java | 69 ++++++
.../bloomfilter/BloomFilterFileIndex.java | 22 +-
.../src/main/java/org/apache/paimon/fs/FileIO.java | 8 +-
.../org/apache/paimon/utils/BloomFilter64.java | 42 +++-
.../java/org/apache/paimon/utils/LazyField.java | 40 ++--
.../java/org/apache/paimon/utils/MathUtils.java | 9 +
.../apache/paimon/utils/VectorMappingUtils.java | 36 +++
.../java/org/apache/paimon/AbstractFileStore.java | 7 +-
.../org/apache/paimon/AppendOnlyFileStore.java | 12 +-
.../src/main/java/org/apache/paimon/FileStore.java | 9 +-
.../java/org/apache/paimon/KeyValueFileStore.java | 18 +-
.../src/main/java/org/apache/paimon/Snapshot.java | 42 ++--
.../org/apache/paimon/append/AppendOnlyWriter.java | 9 +-
.../deletionvectors/ApplyDeletionVectorReader.java | 13 -
.../java/org/apache/paimon/io/DataFileMeta.java | 54 ++++-
.../apache/paimon/io/DataFileMetaSerializer.java | 6 +-
.../org/apache/paimon/io/DataFilePathFactory.java | 6 +
.../apache/paimon/io/FileIndexRecordReader.java | 90 +++++++
.../java/org/apache/paimon/io/FileIndexWriter.java | 194 +++++++++++++++
.../org/apache/paimon/io/FileRecordReader.java | 36 +--
.../apache/paimon/io/KeyValueDataFileWriter.java | 4 +-
.../org/apache/paimon/io/RowDataFileWriter.java | 31 ++-
.../apache/paimon/io/RowDataRollingFileWriter.java | 7 +-
.../java/org/apache/paimon/lookup/BulkLoader.java | 2 +
.../apache/paimon/mergetree/MergeTreeWriter.java | 16 +-
.../paimon/operation/AppendOnlyFileStoreRead.java | 192 ---------------
.../paimon/operation/AppendOnlyFileStoreScan.java | 45 +++-
.../paimon/operation/AppendOnlyFileStoreWrite.java | 13 +-
.../paimon/operation/FileStoreCommitImpl.java | 2 +-
...eFileStoreRead.java => MergeFileSplitRead.java} | 25 +-
.../apache/paimon/operation/RawFileSplitRead.java | 266 +++++++++++++++++++++
.../{FileStoreRead.java => SplitRead.java} | 4 +-
.../org/apache/paimon/schema/SchemaValidation.java | 54 +++--
.../apache/paimon/stats/FieldStatsConverters.java | 15 ++
.../paimon/table/AbstractFileStoreTable.java | 30 ++-
.../paimon/table/AppendOnlyFileStoreTable.java | 12 +-
.../paimon/table/PrimaryKeyFileStoreTable.java | 23 +-
.../org/apache/paimon/table/ReadonlyTable.java | 17 ++
.../main/java/org/apache/paimon/table/Table.java | 7 +
.../apache/paimon/table/sink/TableCommitImpl.java | 20 +-
.../paimon/table/source/AbstractDataTableRead.java | 10 +-
.../paimon/table/source/KeyValueTableRead.java | 123 +++++++---
.../org/apache/paimon/table/source/TableRead.java | 4 +-
.../org/apache/paimon/table/system/TagsTable.java | 45 ++--
.../apache/paimon/{Changelog.java => tag/Tag.java} | 159 +++++++++---
.../org/apache/paimon/tag/TagAutoCreation.java | 23 +-
.../java/org/apache/paimon/tag/TagAutoManager.java | 68 ++++++
.../java/org/apache/paimon/tag/TagTimeExpire.java | 82 +++++++
.../org/apache/paimon/utils/JsonSerdeUtil.java | 2 +
.../org/apache/paimon/utils/SnapshotManager.java | 5 +-
.../java/org/apache/paimon/utils/TagManager.java | 54 ++++-
.../test/java/org/apache/paimon/TestFileStore.java | 4 +-
.../AppendOnlyTableCompactionCoordinatorTest.java | 3 +-
.../apache/paimon/append/AppendOnlyWriterTest.java | 4 +-
.../paimon/crosspartition/IndexBootstrapTest.java | 3 +-
.../apache/paimon/format/FileFormatSuffixTest.java | 4 +-
.../paimon/io/DataFileTestDataGenerator.java | 3 +-
.../org/apache/paimon/io/DataFileTestUtils.java | 9 +-
.../apache/paimon/io/RollingFileWriterTest.java | 4 +-
.../ManifestCommittableSerializerTest.java | 3 +-
.../paimon/manifest/ManifestFileMetaTestBase.java | 6 +-
.../org/apache/paimon/mergetree/LevelsTest.java | 14 +-
.../apache/paimon/mergetree/MergeTreeTestBase.java | 21 +-
.../mergetree/SortBufferWriteBufferTestBase.java | 2 +-
.../mergetree/compact/IntervalPartitionTest.java | 3 +-
.../mergetree/compact/UniversalCompactionTest.java | 2 +-
.../paimon/operation/ExpireSnapshotsTest.java | 9 +-
.../apache/paimon/operation/FileDeletionTest.java | 19 +-
...reReadTest.java => MergeFileSplitReadTest.java} | 6 +-
.../org/apache/paimon/schema/TableSchemaTest.java | 19 ++
.../paimon/table/AppendOnlyFileStoreTableTest.java | 149 ++++++++++++
.../paimon/table/source/SplitGeneratorTest.java | 3 +-
.../apache/paimon/table/system/TagsTableTest.java | 38 +--
...toCreationTest.java => TagAutoManagerTest.java} | 103 +++++++-
.../test/java/org/apache/paimon/tag/TagTest.java | 102 ++++++++
.../org/apache/paimon/utils/TagManagerTest.java | 197 +++++++++++++++
.../flink/sink/cdc/FlinkCdcMultiTableSink.java | 1 +
.../paimon/flink/sink/cdc/RichCdcSinkBuilder.java | 70 +++++-
.../paimon/flink/action/CreateTagAction.java | 14 +-
.../flink/action/CreateTagActionFactory.java | 17 +-
.../apache/paimon/flink/action/DeleteAction.java | 12 +
.../paimon/flink/action/MergeIntoAction.java | 46 +++-
.../flink/action/MergeIntoActionFactory.java | 34 +--
.../paimon/flink/action/SortCompactAction.java | 28 +--
.../paimon/flink/action/TableActionBase.java | 2 +-
.../paimon/flink/procedure/CreateTagProcedure.java | 44 +++-
.../paimon/flink/procedure/MergeIntoProcedure.java | 3 +-
.../sink/AutoTagForSavepointCommitterOperator.java | 9 +-
.../flink/sink/BatchWriteGeneratorTagOperator.java | 6 +-
.../paimon/flink/sink/CommitterOperator.java | 11 +
.../org/apache/paimon/flink/sink/FlinkSink.java | 21 +-
.../apache/paimon/flink/sink/FlinkSinkBuilder.java | 88 ++++---
.../apache/paimon/flink/sink/FlinkTableSink.java | 7 +-
.../paimon/flink/sink/FlinkTableSinkBase.java | 28 ++-
...veParallelism.java => LogFlinkSinkBuilder.java} | 18 +-
.../flink/sink/MultiTablesCompactorSink.java | 1 +
.../paimon/flink/sink/SortCompactSinkBuilder.java | 22 +-
.../SupportsRowLevelOperationFlinkTableSink.java | 93 ++++---
.../paimon/flink/source/DataTableSource.java | 21 +-
.../paimon/flink/source/FlinkSourceBuilder.java | 144 ++++++-----
.../apache/paimon/flink/CatalogTableITCase.java | 4 +-
.../paimon/flink/DynamicBucketTableITCase.java | 6 +-
.../org/apache/paimon/flink/FileStoreITCase.java | 113 ++++++---
.../apache/paimon/flink/ReadWriteTableITCase.java | 193 ++++-----------
.../flink/UnawareBucketAppendOnlyTableITCase.java | 12 +-
.../paimon/flink/action/BranchActionITCase.java | 4 +-
.../paimon/flink/action/MergeIntoActionITCase.java | 53 ----
.../paimon/flink/action/TagActionITCase.java | 2 +-
.../AutoTagForSavepointCommitterOperatorTest.java | 6 +-
.../paimon/flink/sink/CommitterOperatorTest.java | 33 ++-
.../sink/CompactionTaskSimpleSerializerTest.java | 3 +-
.../paimon/flink/sink/StoreMultiCommitterTest.java | 2 +
.../source/FileStoreSourceSplitGeneratorTest.java | 3 +-
.../source/FileStoreSourceSplitSerializerTest.java | 3 +-
.../flink/source/TestChangelogDataReadWrite.java | 47 ++--
.../paimon/flink/util/ReadWriteTableTestUtil.java | 22 +-
.../format/orc/reader/OrcArrayColumnVector.java | 5 +
.../format/orc/reader/OrcMapColumnVector.java | 10 +
.../format/orc/reader/OrcRowColumnVector.java | 5 +
.../apache/paimon/spark/util/SQLConfUtils.scala} | 12 +-
.../apache/paimon/spark/utils/SQLConfUtils.java | 29 ---
.../apache/paimon/spark/util/SQLConfUtils.scala} | 12 +-
.../apache/paimon/spark/utils/SQLConfUtils.java | 29 ---
.../apache/paimon/spark/util/SQLConfUtils.scala} | 12 +-
.../apache/paimon/spark/utils/SQLConfUtils.java | 29 ---
.../apache/paimon/spark/SparkGenericCatalog.java | 2 +-
.../paimon/spark/procedure/CreateTagProcedure.java | 13 +-
.../paimon/spark/commands/PaimonSparkWriter.scala | 6 +-
.../apache/paimon/spark/util/SQLConfUtils.scala | 9 +-
.../CreateAndDeleteTagProcedureTest.scala | 3 +-
pom.xml | 6 +-
145 files changed, 3272 insertions(+), 1286 deletions(-)
create mode 100644
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexOptions.java
copy
paimon-core/src/main/java/org/apache/paimon/utils/IOExceptionSupplier.java =>
paimon-common/src/main/java/org/apache/paimon/utils/LazyField.java (62%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/io/FileIndexRecordReader.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/io/FileIndexWriter.java
delete mode 100644
paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreRead.java
rename
paimon-core/src/main/java/org/apache/paimon/operation/{KeyValueFileStoreRead.java
=> MergeFileSplitRead.java} (94%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/operation/RawFileSplitRead.java
rename
paimon-core/src/main/java/org/apache/paimon/operation/{FileStoreRead.java =>
SplitRead.java} (93%)
copy paimon-core/src/main/java/org/apache/paimon/{Changelog.java =>
tag/Tag.java} (51%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/tag/TagAutoManager.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/tag/TagTimeExpire.java
rename
paimon-core/src/test/java/org/apache/paimon/operation/{KeyValueFileStoreReadTest.java
=> MergeFileSplitReadTest.java} (99%)
rename
paimon-core/src/test/java/org/apache/paimon/tag/{TagAutoCreationTest.java =>
TagAutoManagerTest.java} (78%)
create mode 100644 paimon-core/src/test/java/org/apache/paimon/tag/TagTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/utils/TagManagerTest.java
copy
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/{AdaptiveParallelism.java
=> LogFlinkSinkBuilder.java} (66%)
copy paimon-common/src/main/java/org/apache/paimon/utils/MapBuilder.java =>
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactSinkBuilder.java
(69%)
copy
paimon-spark/{paimon-spark-common/src/main/java/org/apache/paimon/spark/utils/SQLConfUtils.java
=>
paimon-spark-3.1/src/main/scala/org/apache/paimon/spark/util/SQLConfUtils.scala}
(79%)
delete mode 100644
paimon-spark/paimon-spark-3.1/src/main/scala/org/apache/paimon/spark/utils/SQLConfUtils.java
copy
paimon-spark/{paimon-spark-common/src/main/java/org/apache/paimon/spark/utils/SQLConfUtils.java
=>
paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/util/SQLConfUtils.scala}
(79%)
delete mode 100644
paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/utils/SQLConfUtils.java
rename
paimon-spark/{paimon-spark-common/src/main/java/org/apache/paimon/spark/utils/SQLConfUtils.java
=>
paimon-spark-3.3/src/main/scala/org/apache/paimon/spark/util/SQLConfUtils.scala}
(79%)
delete mode 100644
paimon-spark/paimon-spark-3.3/src/main/scala/org/apache/paimon/spark/utils/SQLConfUtils.java
copy
paimon-common/src/main/java/org/apache/paimon/codegen/codesplit/CodeRewriter.java
=>
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/SQLConfUtils.scala
(80%)
