This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a change to branch release-0.8
in repository https://gitbox.apache.org/repos/asf/paimon.git
from b1b0acd79 [core] Increase default sorted run stop trigger (#3220)
add bf24f8bd2 [core] Expose reader and dv from ApplyDeletionVectorReader
(#3244)
add 3d28d8f17 [core] Fix Flink batch reads the paimon partition table
jobManager oom (#3245)
add c2c0d752b [core] Remove useless OrcReaderFactory.computeProjectionMask
add 8b1f91ee7 [doc] Dedicated compaction enables Asynchronous Compaction
add 303342eb1 [doc] Recommended Engines
add b5960cd75 [spark] Bump Spark version 3.4.3 (#3247)
add 10ef1d246 [dep] Move netty dep from bundle to flink (#3248)
add 9969b3726 [doc] Trino starts version 420
add 4b878fe5d [Feature] Support create a empty branch and create a branch
based on snapshotId (#2938)
add 9f9e46aed [spark] Add the fields in reservedFilters into the
estimation of stats (#3255)
add a68385cf2 [core] Introduce ApplyDeletionFileRecordIterator to know
underly iterator and dv (#3257)
add 6c51e7580 [core] Add log to partition expire
add 42ceb8580 [core] Fix bug of DataFileMeta's to_string() (#3259)
add f0c43a88f [core] Move the hilbert dependencies to paimon-common module
(#3251)
add 63ef2fa7d [flink] Skip listPartitions scan for
FlinkRecomputeStatisticsProgram (#3261)
add 3812847f6 [core] Optimize partitions table to avoid OOM (#3258)
add 873db32ea [core] optimize the first row merge engine for high level
record (#3262)
add a74f49d2a [core] Enable file index for map type with map-keys (#3221)
add 01f0764ae [core] Expose `IndexFile` in `Split`. (#3226)
add 9b0699581 [core] Refactor DataSplit field order
add 6521678ec [core] Increase serialVersionUID for DataSplit
add 68355b564 [spark] drop partition when all the conditions is
partition-related (#3263)
add 77cfb9a83 [docs] Update lowest trino version to 420 (#3266)
add e2573f8b0 [core][spark] Use list partitions with predicate (#3267)
add 820eca53d [cdc] Use CdcSourceRecord as source type instead of raw
String (#2783)
add 4e63f552c [core] Expose path from IndexFile (#3268)
add 61160a52e [core] Fix test or dev with NoSuchMethodException in IDE in
OrcConf (#3279)
add fa73f493b [spark] fix when the delete condition does not match any
partitions (#3270)
add 82f5a1dfb [spark] Support delete multiple tags (#3278)
add 7d4efd431 [doc] Fix call procedure miss sys namespace (#3282)
add 251355c91 [doc] Document delete multiple tags
add c796439b6 [core] Use readFully in DeletionVectorsIndexFile and
explicitly specify dvRanges' type as LinkedHashMap (#3283)
add bc26bb927 [core] Introduce privilege system for catalog based on
FileSystem (#2789)
add a987128fd [core] Remove catalog limitation in
InitFileBasedPrivilegeProcedure
add e4f5b31d3 [core] Fix partition type batch write with DATE and TIME
(#3284)
add 0ff663242 [hive] Reuse table StorageDescriptor info (#3269)
No new revisions were added by this update.
Summary of changes:
docs/content/engines/overview.md | 60 ++-
docs/content/engines/trino.md | 8 +-
docs/content/maintenance/dedicated-compaction.md | 5 +
docs/content/maintenance/manage-privileges.md | 246 +++++++++++
docs/content/maintenance/manage-tags.md | 10 +-
docs/content/project/download.md | 2 +-
docs/content/spark/procedures.md | 2 +-
paimon-bundle/pom.xml | 7 -
paimon-common/pom.xml | 16 +
.../main/java/org/apache/paimon/CoreOptions.java | 70 +---
.../{FileIndexer.java => FileIndexCommon.java} | 30 +-
.../apache/paimon/fileindex/FileIndexFormat.java | 100 +++--
.../apache/paimon/fileindex/FileIndexOptions.java | 196 ++++++++-
.../paimon/fileindex/FileIndexPredicate.java | 6 +-
.../apache/paimon/fileindex/FileIndexReader.java | 28 +-
.../apache/paimon/fileindex/FileIndexWriter.java | 19 +-
.../bloomfilter/BloomFilterFileIndex.java | 8 +-
.../empty/EmptyFileIndexReader.java} | 63 +--
.../java/org/apache/paimon/options/Options.java | 7 +
.../apache/paimon/reader/EmptyRecordReader.java | 25 +-
.../apache/paimon/sort/hilbert/HilbertIndexer.java | 0
.../org/apache/paimon/utils/BinaryStringUtils.java | 14 +-
.../org/apache/paimon/utils/ConvertBinaryUtil.java | 0
.../java/org/apache/paimon/utils/StringUtils.java | 13 +
paimon-common/src/main/resources/META-INF/NOTICE | 2 +
.../fileindex/FileIndexFormatFormatTest.java | 35 +-
paimon-core/pom.xml | 6 -
.../java/org/apache/paimon/AbstractFileStore.java | 4 +-
.../java/org/apache/paimon/branch/TableBranch.java | 14 +
.../org/apache/paimon/catalog/AbstractCatalog.java | 25 +-
.../java/org/apache/paimon/catalog/Catalog.java | 9 +
.../paimon/catalog/FileSystemCatalogFactory.java | 18 +-
...r.java => ApplyDeletionFileRecordIterator.java} | 59 +--
.../deletionvectors/ApplyDeletionVectorReader.java | 13 +-
.../paimon/deletionvectors/DeletionVector.java | 4 +-
.../deletionvectors/DeletionVectorsIndexFile.java | 43 +-
.../org/apache/paimon/index/IndexFileHandler.java | 6 +-
.../org/apache/paimon/index/IndexFileMeta.java | 13 +-
.../paimon/index/IndexFileMetaSerializer.java | 8 +-
.../java/org/apache/paimon/io/DataFileMeta.java | 19 +-
...ndexRecordReader.java => FileIndexSkipper.java} | 32 +-
.../java/org/apache/paimon/io/FileIndexWriter.java | 172 ++++++--
.../org/apache/paimon/manifest/PartitionEntry.java | 136 ++++++
.../mergetree/compact/FirstRowMergeFunction.java | 5 +
.../compact/FirstRowMergeFunctionWrapper.java | 5 +
.../paimon/operation/AbstractFileStoreScan.java | 29 ++
.../paimon/operation/AppendOnlyFileStoreWrite.java | 5 +
.../org/apache/paimon/operation/FileStoreScan.java | 9 +
.../apache/paimon/operation/PartitionExpire.java | 18 +-
.../apache/paimon/operation/RawFileSplitRead.java | 63 ++-
.../privilege/AllGrantedPrivilegeChecker.java | 58 +--
.../Metric.java => privilege/EntityType.java} | 16 +-
.../privilege/FileBasedPrivilegeManager.java | 457 +++++++++++++++++++++
.../NoPrivilegeException.java} | 23 +-
.../apache/paimon/privilege/PrivilegeChecker.java | 34 +-
.../paimon/privilege/PrivilegeCheckerImpl.java | 154 +++++++
.../apache/paimon/privilege/PrivilegeManager.java | 89 ++++
.../org/apache/paimon/privilege/PrivilegeType.java | 65 +++
.../apache/paimon/privilege/PrivilegedCatalog.java | 256 ++++++++++++
.../paimon/privilege/PrivilegedFileStore.java | 200 +++++++++
.../paimon/privilege/PrivilegedFileStoreTable.java | 308 ++++++++++++++
.../paimon/table/AbstractFileStoreTable.java | 10 +
.../org/apache/paimon/table/ReadonlyTable.java | 16 +
.../main/java/org/apache/paimon/table/Table.java | 8 +
.../org/apache/paimon/table/source/DataSplit.java | 126 ++++--
.../org/apache/paimon/table/source/IndexFile.java | 45 +-
.../java/org/apache/paimon/table/source/Split.java | 10 +
.../org/apache/paimon/table/source/TableScan.java | 2 +-
.../snapshot/IncrementalStartingScanner.java | 87 +++-
.../table/source/snapshot/SnapshotReader.java | 7 +-
.../table/source/snapshot/SnapshotReaderImpl.java | 67 +--
.../apache/paimon/table/system/AuditLogTable.java | 12 +
.../paimon/table/system/PartitionsTable.java | 206 ++--------
.../org/apache/paimon/utils/BranchManager.java | 90 +++-
.../org/apache/paimon/utils/JsonSerdeUtil.java | 16 +-
.../apache/paimon/utils/ScanParallelExecutor.java | 14 +-
.../test/java/org/apache/paimon/TestFileStore.java | 2 +
.../paimon/crosspartition/IndexBootstrapTest.java | 1 +
.../DeletionVectorsIndexFileTest.java | 85 +++-
.../paimon/index/IndexFileMetaSerializerTest.java | 4 +-
.../paimon/operation/MergeFileSplitReadTest.java | 1 +
.../privilege/FileBasedPrivilegeManagerTest.java | 170 ++++++++
.../org/apache/paimon/stats/StatsTableTest.java | 4 +-
.../paimon/table/AppendOnlyFileStoreTableTest.java | 161 +++++++-
.../paimon/table/FileStoreTableTestBase.java | 4 +-
.../org/apache/paimon/table/source/SplitTest.java | 1 +
.../table/source/snapshot/SnapshotReaderTest.java | 87 ++++
paimon-e2e-tests/pom.xml | 2 +-
.../paimon/flink/procedure/CompactProcedure.java | 5 +-
paimon-flink/paimon-flink-cdc/pom.xml | 4 +
.../cdc/CdcSourceRecord.java} | 57 +--
.../flink/action/cdc/MessageQueueSchemaUtils.java | 2 +-
.../flink/action/cdc/SyncDatabaseActionBase.java | 2 +-
.../paimon/flink/action/cdc/SyncJobHandler.java | 4 +-
.../flink/action/cdc/SyncTableActionBase.java | 2 +-
.../action/cdc/SynchronizationActionBase.java | 10 +-
.../flink/action/cdc/format/RecordParser.java | 28 +-
.../cdc/format/debezium/DebeziumRecordParser.java | 5 +-
.../flink/action/cdc/kafka/KafkaActionUtils.java | 48 ++-
.../action/cdc/mongodb/MongoDBActionUtils.java | 12 +-
.../action/cdc/mongodb/MongoDBRecordParser.java | 11 +-
.../cdc/mongodb/MongoDBSyncDatabaseAction.java | 3 +-
.../action/cdc/mongodb/MongoDBSyncTableAction.java | 3 +-
.../flink/action/cdc/mysql/MySqlActionUtils.java | 11 +-
.../flink/action/cdc/mysql/MySqlRecordParser.java | 8 +-
.../action/cdc/mysql/MySqlSyncDatabaseAction.java | 3 +-
.../action/cdc/mysql/MySqlSyncTableAction.java | 3 +-
.../action/cdc/postgres/PostgresActionUtils.java | 11 +-
.../action/cdc/postgres/PostgresRecordParser.java | 9 +-
.../cdc/postgres/PostgresSyncTableAction.java | 3 +-
.../flink/action/cdc/pulsar/PulsarActionUtils.java | 31 +-
.../CdcDebeziumDeserializationSchema.java | 97 +++++
.../CdcJsonDeserializationSchema.java | 74 ++++
.../watermark/CdcTimestampExtractorFactory.java | 17 +-
.../action/cdc/watermark/CdcWatermarkStrategy.java | 9 +-
.../cdc/kafka/KafkaSyncTableActionITCase.java | 6 +-
paimon-flink/paimon-flink-common/pom.xml | 9 +
.../java/org/apache/paimon/flink/FlinkCatalog.java | 42 +-
.../flink/procedure/CompactDatabaseProcedure.java | 5 +-
.../paimon/flink/procedure/CompactProcedure.java | 5 +-
.../flink/procedure/CreateBranchProcedure.java | 24 +-
.../paimon/flink/procedure/MergeIntoProcedure.java | 5 +-
.../CreatePrivilegedUserProcedure.java} | 24 +-
.../DropPrivilegedUserProcedure.java} | 24 +-
.../privilege/GrantPrivilegeToUserProcedure.java | 79 ++++
.../privilege/InitFileBasedPrivilegeProcedure.java | 63 +++
.../privilege/PrivilegeProcedureBase.java | 32 +-
.../RevokePrivilegeFromUserProcedure.java | 87 ++++
.../flink/source/FileStoreSourceSplitReader.java | 2 +-
.../flink/source/align/PlaceholderSplit.java | 4 +
.../paimon/flink/source/operator/ReadOperator.java | 2 +-
.../services/org.apache.paimon.factories.Factory | 5 +
.../apache/paimon/flink/CatalogTableITCase.java | 119 ++++--
.../paimon/flink/action/BranchActionITCase.java | 80 ++++
.../SortCompactActionForUnawareBucketITCase.java | 2 +
.../privilege/PrivilegeProcedureITCase.java | 323 +++++++++++++++
.../source/ContinuousFileSplitEnumeratorTest.java | 2 +
.../flink/source/FileSplitEnumeratorTestBase.java | 1 +
.../source/FileStoreSourceSplitGeneratorTest.java | 2 +
.../source/FileStoreSourceSplitSerializerTest.java | 2 +
.../java/org/apache/paimon/format/OrcOptions.java | 12 +-
.../apache/paimon/format/orc/OrcReaderFactory.java | 20 -
.../java/org/apache/paimon/hive/HiveCatalog.java | 33 +-
.../org/apache/paimon/hive/HiveCatalogFactory.java | 5 -
.../apache/paimon/hive/HiveCatalogITCaseBase.java | 73 +++-
.../paimon/hive/mapred/PaimonInputSplitTest.java | 2 +
paimon-spark/paimon-spark-3.4/pom.xml | 2 +-
.../sql/CreateAndDeleteTagProcedureTest.scala | 28 ++
paimon-spark/paimon-spark-common/pom.xml | 4 -
.../paimon/spark/procedure/DeleteTagProcedure.java | 7 +-
.../org/apache/paimon/spark/PaimonBaseScan.scala | 5 +
.../org/apache/paimon/spark/PaimonStatistics.scala | 2 +-
.../scala/org/apache/paimon/spark/ScanHelper.scala | 14 +-
.../commands/DeleteFromPaimonTableCommand.scala | 46 ++-
.../paimon/spark/commands/SparkDataFileMeta.scala | 7 +-
.../spark/commands/UpdatePaimonTableCommand.scala | 6 +-
.../spark/statistics/StatisticsHelperBase.scala | 13 +-
.../org/apache/paimon/spark/ScanHelperTest.scala | 6 +-
.../paimon/spark/sql/AnalyzeTableTestBase.scala | 7 +
.../paimon/spark/sql/DeleteFromTableTest.scala | 6 +
.../paimon/spark/sql/InsertOverwriteTest.scala | 17 +
161 files changed, 5124 insertions(+), 1197 deletions(-)
create mode 100644 docs/content/maintenance/manage-privileges.md
copy paimon-common/src/main/java/org/apache/paimon/fileindex/{FileIndexer.java
=> FileIndexCommon.java} (54%)
copy
paimon-common/src/main/java/org/apache/paimon/{predicate/OnlyPartitionKeyEqualVisitor.java
=> fileindex/empty/EmptyFileIndexReader.java} (54%)
copy
paimon-core/src/main/java/org/apache/paimon/mergetree/compact/MergeFunctionWrapper.java
=> paimon-common/src/main/java/org/apache/paimon/reader/EmptyRecordReader.java
(72%)
rename {paimon-core =>
paimon-common}/src/main/java/org/apache/paimon/sort/hilbert/HilbertIndexer.java
(100%)
rename {paimon-core =>
paimon-common}/src/main/java/org/apache/paimon/utils/ConvertBinaryUtil.java
(100%)
copy
paimon-core/src/main/java/org/apache/paimon/deletionvectors/{ApplyDeletionVectorReader.java
=> ApplyDeletionFileRecordIterator.java} (50%)
rename
paimon-core/src/main/java/org/apache/paimon/io/{FileIndexRecordReader.java =>
FileIndexSkipper.java} (73%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/manifest/PartitionEntry.java
copy paimon-common/src/main/java/org/apache/paimon/predicate/LeafFunction.java
=>
paimon-core/src/main/java/org/apache/paimon/privilege/AllGrantedPrivilegeChecker.java
(52%)
copy paimon-core/src/main/java/org/apache/paimon/{metrics/Metric.java =>
privilege/EntityType.java} (74%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/privilege/FileBasedPrivilegeManager.java
copy
paimon-core/src/main/java/org/apache/paimon/{table/source/OutOfRangeException.java
=> privilege/NoPrivilegeException.java} (54%)
copy
paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/schema/JdbcTableInfo.java
=> paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegeChecker.java
(54%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegeCheckerImpl.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegeManager.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegeType.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedCatalog.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStore.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStoreTable.java
copy paimon-common/src/main/java/org/apache/paimon/utils/Reference.java =>
paimon-core/src/main/java/org/apache/paimon/table/source/IndexFile.java (56%)
create mode 100644
paimon-core/src/test/java/org/apache/paimon/privilege/FileBasedPrivilegeManagerTest.java
copy
paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/{sink/cdc/CdcRecord.java
=> action/cdc/CdcSourceRecord.java} (50%)
create mode 100644
paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/serialization/CdcDebeziumDeserializationSchema.java
create mode 100644
paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/serialization/CdcJsonDeserializationSchema.java
copy
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/{DeleteTagProcedure.java
=> privilege/CreatePrivilegedUserProcedure.java} (62%)
copy
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/{DeleteTagProcedure.java
=> privilege/DropPrivilegedUserProcedure.java} (64%)
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/privilege/GrantPrivilegeToUserProcedure.java
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/privilege/InitFileBasedPrivilegeProcedure.java
copy paimon-common/src/test/resources/codesplit/groups/code/IfInWhile.txt =>
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/privilege/PrivilegeProcedureBase.java
(59%)
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/privilege/RevokePrivilegeFromUserProcedure.java
create mode 100644
paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/privilege/PrivilegeProcedureITCase.java
