This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a change to branch release-0.8
in repository https://gitbox.apache.org/repos/asf/paimon.git
from 0ff663242 [hive] Reuse table StorageDescriptor info (#3269)
add 212c7c2fa [doc] Update ververica flink cdc to flink website
add e9d5ebceb [core] Fix the merge order in lookup compaction (#3286)
add 8f4f818b9 [core] Add Nullable to userDefinedSeqComparator in
LookupChangelogMergeFunctionWrapper
add 7b378dcf7 [cdc] When the cdc source modifies field comments, all field
comments in the existing paimon table will be set to empty. (#3088)
add b53c3680d [core] TagsTable create time should be nullable (#3289)
add 183c81df0 [core] TagsTable time_retained should be nullable
add 075d77455 [cdc] Fix Multibyte Character Encoding in
MultiTableCommittable Serialization
add 2ee67f38a [cdc] Optimize MultiTableCommittableSerializer
add 0325d4215 [flink] Add serialVersionUID to PrepareCommitOperator
add b8802e3a5 [flink][spark] Delete in flink should produce changelog no
matter what (#2594)
add 2a64fa43f [flink] Delete in flink should only pushdown partition or
table dropping
add 93666c23f [core] Protect tagManager.createTag in TagAutoCreation
add 3112c9768 [cdc] Combine mode should throw better exception to dynamic
bucket table (#3291)
add 6137fd0d5 [core] incremental-between-scan-mode should respect
changelog-producer (#3290)
add eb6695f57 [core] Fix SST files written using SstFileWriter don't work
with TtlDB (#3292)
add fcb7c1ddd [core] Introduce record-level expire time (#3293)
add db3a6939f [spark] Delete supports all merge engines (#3294)
new 08ec5800d [doc] Update to released
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/config.toml | 16 +--
docs/content/flink/sql-query.md | 3 +
docs/content/primary-key-table/overview.md | 8 ++
docs/content/spark/sql-query.md | 3 +
.../shortcodes/generated/core_configuration.html | 22 ++-
.../generated/mongodb_sync_database.html | 2 +-
.../shortcodes/generated/mongodb_sync_table.html | 2 +-
.../shortcodes/generated/mysql_sync_database.html | 4 +-
.../shortcodes/generated/mysql_sync_table.html | 4 +-
.../shortcodes/generated/postgres_sync_table.html | 4 +-
.../main/java/org/apache/paimon/CoreOptions.java | 48 ++++++-
.../predicate/AllPrimaryKeyEqualVisitor.java | 110 ---------------
.../predicate/DeletePushDownVisitorTest.java | 148 ---------------------
.../org/apache/paimon/io/FileReaderFactory.java | 14 +-
.../paimon/io/KeyValueFileReaderFactory.java | 7 +-
.../org/apache/paimon/io/RecordLevelExpire.java | 92 +++++++++++++
.../java/org/apache/paimon/lookup/BulkLoader.java | 19 +++
.../apache/paimon/mergetree/MergeTreeReaders.java | 16 +--
.../compact/ChangelogMergeTreeRewriter.java | 4 +-
.../FullChangelogMergeTreeCompactRewriter.java | 4 +-
.../LookupChangelogMergeFunctionWrapper.java | 48 ++++++-
.../compact/LookupMergeTreeCompactRewriter.java | 13 +-
.../compact/MergeTreeCompactRewriter.java | 6 +-
.../paimon/operation/KeyValueFileStoreWrite.java | 20 ++-
.../java/org/apache/paimon/table/TableUtils.java | 77 -----------
.../table/source/AbstractInnerTableScan.java | 6 +
.../org/apache/paimon/table/system/TagsTable.java | 19 ++-
.../org/apache/paimon/tag/TagAutoCreation.java | 4 +-
.../paimon/catalog/PrimaryKeyTableTestBase.java | 47 +++++++
.../crosspartition/GlobalIndexAssignerTest.java | 23 ++++
.../LookupChangelogMergeFunctionWrapperTest.java | 87 ++++++++++++
.../apache/paimon/table/RecordLevelExpireTest.java | 70 ++++++++++
.../snapshot/IncrementalStartingScannerTest.java | 34 ++---
.../apache/paimon/table/system/TagsTableTest.java | 15 +--
.../flink/action/cdc/CdcActionCommonUtils.java | 20 +++
.../flink/action/cdc/format/RecordParser.java | 55 ++++----
.../action/cdc/format/canal/CanalRecordParser.java | 19 ++-
.../cdc/format/debezium/DebeziumRecordParser.java | 11 +-
.../mongodb/strategy/Mongo4VersionStrategy.java | 18 +--
.../cdc/mongodb/strategy/MongoVersionStrategy.java | 27 ++--
.../flink/action/cdc/mysql/MySqlRecordParser.java | 20 ++-
.../action/cdc/postgres/PostgresRecordParser.java | 18 +--
.../cdc/CdcDynamicTableParsingProcessFunction.java | 2 +-
.../cdc/CdcMultiplexRecordChannelComputer.java | 9 ++
.../sink/cdc/CdcRecordStoreMultiWriteOperator.java | 4 +-
.../flink/sink/cdc/NewTableSchemaBuilder.java | 9 +-
.../flink/sink/cdc/RichCdcMultiplexRecord.java | 23 ++--
.../cdc/RichCdcMultiplexRecordEventParser.java | 2 +-
.../paimon/flink/sink/cdc/RichCdcRecord.java | 40 ++++--
.../paimon/flink/sink/cdc/RichEventParser.java | 15 +--
.../cdc/UpdatedDataFieldsProcessFunctionBase.java | 8 +-
.../kafka/KafkaCanalSyncDatabaseActionITCase.java | 2 +-
.../cdc/mysql/MySqlSyncTableActionITCase.java | 40 ++++++
.../src/test/resources/mysql/sync_table_setup.sql | 7 +
.../sink/MultiTableCommittableSerializer.java | 26 ++--
.../paimon/flink/sink/PrepareCommitOperator.java | 2 +
.../SupportsRowLevelOperationFlinkTableSink.java | 24 +---
.../apache/paimon/flink/BatchFileStoreITCase.java | 27 ++++
.../apache/paimon/flink/ReadWriteTableITCase.java | 2 +-
.../sink/MultiTableCommittableSerializerTest.java | 75 +++++++++--
.../paimon/spark/procedure/CompactProcedure.java | 8 +-
.../spark/catalyst/analysis/RowLevelOp.scala | 6 +-
.../commands/DeleteFromPaimonTableCommand.scala | 28 ++--
.../paimon/spark/commands/PaimonCommand.scala | 10 ++
.../paimon/spark/commands/PaimonSparkWriter.scala | 8 ++
.../paimon/spark/sql/DeleteFromTableTest.scala | 63 +++++++--
66 files changed, 984 insertions(+), 643 deletions(-)
delete mode 100644
paimon-common/src/main/java/org/apache/paimon/predicate/AllPrimaryKeyEqualVisitor.java
copy paimon-common/src/main/java/org/apache/paimon/utils/KeyValueIterator.java
=> paimon-core/src/main/java/org/apache/paimon/io/FileReaderFactory.java (78%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/io/RecordLevelExpire.java
delete mode 100644
paimon-core/src/main/java/org/apache/paimon/table/TableUtils.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireTest.java
