This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a change to branch release-0.5
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
from e06c3429d [doc] Minor document optimization (#1864)
add 5c31cadfe Modify the flink documentation link to point to the flink
stable version. (#1866)
add 313c051b6 [flink][kafka-cdc] Kafka cdc doesn't need to check table's
primary keys (#1867)
add fb4aa7f56 [doc] Add configuration instructions for Apache Atlas (#1875)
add 2746a2393 [flink] Refactor Flink sink metrics by introducing
CommitterMetrics (#1874)
add b429d688a [Refactor] Add Kafka cdc computed column test and reduce
unnecessary e2e tests (#1872)
add de65d531a [hotfix] Fix that MySQL sync-database action parses table
prefix incorrectly (#1868)
add 4baefcdf1 [flink][mysql-cdc] Map MySQL BINARY to Paimon VARBINARY and
fix parse of MySQL BIT(N) value (#1844)
add 27b155f45 [flink] Supported ogg format and optimize format scalability
(#1827)
add 0dbcff69f [core] support z-order range sort action (#1846)
add 6f72239dc [hotfix] Fix code style for zorder
No new revisions were added by this update.
Summary of changes:
LICENSE | 2 +
docs/content/api/flink-api.md | 2 +-
docs/content/concepts/append-only-table.md | 23 +-
docs/content/engines/flink.md | 2 +-
docs/content/filesystems/hdfs.md | 2 +-
docs/content/filesystems/oss.md | 2 +-
docs/content/filesystems/s3.md | 2 +-
docs/content/how-to/cdc-ingestion.md | 16 +-
docs/content/how-to/creating-catalogs.md | 4 +-
docs/content/how-to/lookup-joins.md | 2 +-
docs/content/how-to/writing-tables.md | 2 +-
docs/content/maintenance/manage-tags.md | 4 +-
docs/content/maintenance/rescale-bucket.md | 6 +-
.../shortcodes/generated/kafka_sync_database.html | 2 +-
.../shortcodes/generated/kafka_sync_table.html | 2 +-
.../layouts/shortcodes/generated/sort-compact.html | 55 ++
.../java/org/apache/paimon/utils/StringUtils.java | 15 +
.../java/org/apache/paimon/utils/TypeUtils.java | 16 +-
.../org/apache/paimon/sort/zorder/ZIndexer.java | 397 ++++++++++++++
.../org/apache/paimon/utils/ZOrderByteUtils.java | 243 +++++++++
.../apache/paimon/sort/zorder/ZIndexerTest.java | 70 +++
.../apache/paimon/utils/TestZOrderByteUtil.java | 426 +++++++++++++++
.../paimon/tests/cdc/MySqlCdcE2eTestBase.java | 143 -----
.../org/apache/paimon/flink/action/ActionBase.java | 46 ++
.../apache/paimon/flink/action/CompactAction.java | 4 +
.../paimon/flink/action/CompactActionFactory.java | 38 +-
.../paimon/flink/action/SortCompactAction.java | 132 +++++
.../paimon/flink/action/TableActionBase.java | 49 +-
.../paimon/flink/action/cdc/kafka/KafkaSchema.java | 74 ++-
.../action/cdc/kafka/KafkaSyncDatabaseAction.java | 36 +-
.../cdc/kafka/KafkaSyncDatabaseActionFactory.java | 2 +-
.../action/cdc/kafka/KafkaSyncTableAction.java | 23 +-
.../cdc/kafka/KafkaSyncTableActionFactory.java | 2 +-
.../flink/action/cdc/kafka/formats/DataFormat.java | 82 +++
.../action/cdc/kafka/formats/RecordParser.java | 105 ++++
.../cdc/kafka/formats/RecordParserFactory.java | 49 ++
.../{ => formats}/canal/CanalFieldParser.java | 2 +-
.../{ => formats}/canal/CanalRecordParser.java | 180 +++---
.../cdc/kafka/formats/ogg/OggRecordParser.java | 222 ++++++++
.../cdc/mysql/MySqlDebeziumJsonEventParser.java | 25 +-
.../cdc/mysql/MySqlSyncDatabaseActionFactory.java | 2 +-
.../flink/action/cdc/mysql/MySqlTypeUtils.java | 6 +-
.../apache/paimon/flink/shuffle/RangeShuffle.java | 444 +++++++++++++++
.../org/apache/paimon/flink/sink/Committer.java | 11 +-
.../apache/paimon/flink/sink/CommitterMetrics.java | 67 +++
.../paimon/flink/sink/CommitterOperator.java | 10 +-
.../apache/paimon/flink/sink/CompactorSink.java | 8 +-
.../org/apache/paimon/flink/sink/FlinkSink.java | 5 +-
.../apache/paimon/flink/sink/FlinkWriteSink.java | 10 +-
.../apache/paimon/flink/sink/StoreCommitter.java | 36 +-
.../paimon/flink/sink/StoreMultiCommitter.java | 19 +-
.../flink/sink/UnawareBucketCompactionSink.java | 8 +-
.../paimon/flink/sink/cdc/CdcRecordUtils.java | 4 +-
.../flink/sink/cdc/FlinkCdcMultiTableSink.java | 8 +-
.../cdc/RichCdcMultiplexRecordEventParser.java | 13 +-
.../apache/paimon/flink/sorter/SortOperator.java | 124 +++++
.../apache/paimon/flink/sorter/TableSorter.java | 117 ++++
.../apache/paimon/flink/sorter/ZorderSorter.java | 63 +++
.../paimon/flink/sorter/ZorderSorterUtils.java | 156 ++++++
.../flink/action/OrderRewriteActionITCase.java | 229 ++++++++
.../action/cdc/kafka/KafkaActionITCaseBase.java | 21 +
.../kafka/KafkaCanalSyncDatabaseActionITCase.java | 42 +-
.../cdc/kafka/KafkaCanalSyncTableActionITCase.java | 101 +++-
....java => KafkaOggSyncDatabaseActionITCase.java} | 364 +++++--------
.../cdc/kafka/KafkaOggSyncTableActionITCase.java | 603 +++++++++++++++++++++
.../flink/action/cdc/kafka/KafkaSchemaITCase.java | 2 +-
.../cdc/mysql/MySqlCdcTypeMappingITCase.java | 8 +-
.../cdc/mysql/MySqlSyncTableActionITCase.java | 8 +-
...rTest.java => KafkaLogStoreRegisterITCase.java} | 2 +-
.../paimon/flink/sink/CommitterOperatorTest.java | 34 +-
.../paimon/flink/sink/StoreMultiCommitterTest.java | 12 +-
.../table/computedcolumn/canal-data-1.txt | 12 +-
.../database/include/topic0/canal-data-1.txt | 0
.../database/prefixsuffix/topic0/canal-data-1.txt | 0
.../database/prefixsuffix/topic0/canal-data-2.txt | 0
.../database/prefixsuffix/topic0/canal-data-3.txt | 0
.../database/prefixsuffix/topic1/canal-data-1.txt | 0
.../database/prefixsuffix/topic1/canal-data-2.txt | 0
.../database/prefixsuffix/topic1/canal-data-3.txt | 0
.../schemaevolution/topic0/canal-data-1.txt | 0
.../schemaevolution/topic0/canal-data-2.txt | 0
.../schemaevolution/topic0/canal-data-3.txt | 0
.../schemaevolution/topic1/canal-data-1.txt | 0
.../schemaevolution/topic1/canal-data-2.txt | 0
.../schemaevolution/topic1/canal-data-3.txt | 0
.../schemaevolution/topic2/canal-data-1.txt | 0
.../schemaevolution/topic2/canal-data-2.txt | 0
.../schemaevolution/topic2/canal-data-3.txt | 0
.../canal}/table/alltype/canal-data.txt | 0
.../canal}/table/nononddldata/canal-data-1.txt | 0
.../canal}/table/schemaevolution/canal-data-1.txt | 0
.../canal}/table/schemaevolution/canal-data-2.txt | 0
.../canal}/table/schemaevolution/canal-data-3.txt | 0
.../canal}/table/schemaevolution/canal-data-4.txt | 0
.../canal}/table/schemaevolution/canal-data-5.txt | 0
.../schemaevolutionmissingddl/canal-data-1.txt | 0
.../schemaevolutionmissingddl/canal-data-2.txt | 0
.../schemaevolutionmissingddl/canal-data-3.txt | 0
.../schemaevolutionmissingddl/canal-data-4.txt | 0
.../schemaevolutionmissingddl/canal-data-5.txt | 0
.../table/schemaevolutionmultiple/canal-data-1.txt | 0
.../table/schemaevolutionmultiple/canal-data-2.txt | 0
.../canal}/table/startupmode/canal-data-1.txt | 0
.../canal}/table/startupmode/canal-data-2.txt | 0
.../ogg/database/include/topic0/ogg-data-1.txt | 22 +
.../database/prefixsuffix/topic0/ogg-data-1.txt | 13 +-
.../database/prefixsuffix/topic0/ogg-data-2.txt | 13 +-
.../database/prefixsuffix/topic1/ogg-data-1.txt | 13 +-
.../database/prefixsuffix/topic1/ogg-data-2.txt | 13 +-
.../database/schemaevolution/topic0/ogg-data-1.txt | 13 +-
.../database/schemaevolution/topic0/ogg-data-2.txt | 13 +-
.../database/schemaevolution/topic1/ogg-data-1.txt | 13 +-
.../database/schemaevolution/topic1/ogg-data-2.txt | 13 +-
.../kafka/ogg/table/schemaevolution/ogg-data-1.txt | 13 +-
.../kafka/ogg/table/schemaevolution/ogg-data-2.txt | 13 +-
.../kafka/ogg/table/schemaevolution/ogg-data-3.txt | 23 +
.../kafka/ogg/table/startupmode/ogg-data-1.txt | 13 +-
.../kafka/ogg/table/startupmode/ogg-data-2.txt | 13 +-
.../resources/mysql/type_mapping_test_setup.sql | 6 +-
119 files changed, 4362 insertions(+), 903 deletions(-)
create mode 100644 docs/layouts/shortcodes/generated/sort-compact.html
create mode 100644
paimon-core/src/main/java/org/apache/paimon/sort/zorder/ZIndexer.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/utils/ZOrderByteUtils.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/sort/zorder/ZIndexerTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/utils/TestZOrderByteUtil.java
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/SortCompactAction.java
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/kafka/formats/DataFormat.java
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/kafka/formats/RecordParser.java
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/kafka/formats/RecordParserFactory.java
rename
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/kafka/{
=> formats}/canal/CanalFieldParser.java (98%)
rename
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/kafka/{
=> formats}/canal/CanalRecordParser.java (77%)
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/kafka/formats/ogg/OggRecordParser.java
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/shuffle/RangeShuffle.java
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CommitterMetrics.java
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sorter/SortOperator.java
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sorter/TableSorter.java
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sorter/ZorderSorter.java
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sorter/ZorderSorterUtils.java
create mode 100644
paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/OrderRewriteActionITCase.java
copy
paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/kafka/{KafkaCanalSyncDatabaseActionITCase.java
=> KafkaOggSyncDatabaseActionITCase.java} (56%)
create mode 100644
paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaOggSyncTableActionITCase.java
rename
paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/kafka/{KafkaLogStoreRegisterTest.java
=> KafkaLogStoreRegisterITCase.java} (99%)
copy paimon-common/src/main/java/org/apache/paimon/annotation/ConfigGroup.java
=>
paimon-flink/paimon-flink-common/src/test/resources/kafka.canal/table/computedcolumn/canal-data-1.txt
(73%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/database/include/topic0/canal-data-1.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/database/prefixsuffix/topic0/canal-data-1.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/database/prefixsuffix/topic0/canal-data-2.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/database/prefixsuffix/topic0/canal-data-3.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/database/prefixsuffix/topic1/canal-data-1.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/database/prefixsuffix/topic1/canal-data-2.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/database/prefixsuffix/topic1/canal-data-3.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/database/schemaevolution/topic0/canal-data-1.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/database/schemaevolution/topic0/canal-data-2.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/database/schemaevolution/topic0/canal-data-3.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/database/schemaevolution/topic1/canal-data-1.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/database/schemaevolution/topic1/canal-data-2.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/database/schemaevolution/topic1/canal-data-3.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/database/schemaevolution/topic2/canal-data-1.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/database/schemaevolution/topic2/canal-data-2.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/database/schemaevolution/topic2/canal-data-3.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/table/alltype/canal-data.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/table/nononddldata/canal-data-1.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/table/schemaevolution/canal-data-1.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/table/schemaevolution/canal-data-2.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/table/schemaevolution/canal-data-3.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/table/schemaevolution/canal-data-4.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/table/schemaevolution/canal-data-5.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/table/schemaevolutionmissingddl/canal-data-1.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/table/schemaevolutionmissingddl/canal-data-2.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/table/schemaevolutionmissingddl/canal-data-3.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/table/schemaevolutionmissingddl/canal-data-4.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/table/schemaevolutionmissingddl/canal-data-5.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/table/schemaevolutionmultiple/canal-data-1.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/table/schemaevolutionmultiple/canal-data-2.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/table/startupmode/canal-data-1.txt (100%)
rename paimon-flink/paimon-flink-common/src/test/resources/{kafka.canal =>
kafka/canal}/table/startupmode/canal-data-2.txt (100%)
create mode 100644
paimon-flink/paimon-flink-common/src/test/resources/kafka/ogg/database/include/topic0/ogg-data-1.txt
copy paimon-common/src/main/java/org/apache/paimon/annotation/ConfigGroup.java
=>
paimon-flink/paimon-flink-common/src/test/resources/kafka/ogg/database/prefixsuffix/topic0/ogg-data-1.txt
(60%)
copy paimon-common/src/main/java/org/apache/paimon/annotation/ConfigGroup.java
=>
paimon-flink/paimon-flink-common/src/test/resources/kafka/ogg/database/prefixsuffix/topic0/ogg-data-2.txt
(58%)
copy paimon-common/src/main/java/org/apache/paimon/annotation/ConfigGroup.java
=>
paimon-flink/paimon-flink-common/src/test/resources/kafka/ogg/database/prefixsuffix/topic1/ogg-data-1.txt
(60%)
copy paimon-common/src/main/java/org/apache/paimon/annotation/ConfigGroup.java
=>
paimon-flink/paimon-flink-common/src/test/resources/kafka/ogg/database/prefixsuffix/topic1/ogg-data-2.txt
(60%)
copy paimon-common/src/main/java/org/apache/paimon/annotation/ConfigGroup.java
=>
paimon-flink/paimon-flink-common/src/test/resources/kafka/ogg/database/schemaevolution/topic0/ogg-data-1.txt
(60%)
copy paimon-common/src/main/java/org/apache/paimon/annotation/ConfigGroup.java
=>
paimon-flink/paimon-flink-common/src/test/resources/kafka/ogg/database/schemaevolution/topic0/ogg-data-2.txt
(60%)
copy paimon-common/src/main/java/org/apache/paimon/annotation/ConfigGroup.java
=>
paimon-flink/paimon-flink-common/src/test/resources/kafka/ogg/database/schemaevolution/topic1/ogg-data-1.txt
(60%)
copy paimon-common/src/main/java/org/apache/paimon/annotation/ConfigGroup.java
=>
paimon-flink/paimon-flink-common/src/test/resources/kafka/ogg/database/schemaevolution/topic1/ogg-data-2.txt
(59%)
copy paimon-common/src/main/java/org/apache/paimon/annotation/ConfigGroup.java
=>
paimon-flink/paimon-flink-common/src/test/resources/kafka/ogg/table/schemaevolution/ogg-data-1.txt
(61%)
copy paimon-common/src/main/java/org/apache/paimon/annotation/ConfigGroup.java
=>
paimon-flink/paimon-flink-common/src/test/resources/kafka/ogg/table/schemaevolution/ogg-data-2.txt
(61%)
create mode 100644
paimon-flink/paimon-flink-common/src/test/resources/kafka/ogg/table/schemaevolution/ogg-data-3.txt
copy paimon-common/src/main/java/org/apache/paimon/annotation/ConfigGroup.java
=>
paimon-flink/paimon-flink-common/src/test/resources/kafka/ogg/table/startupmode/ogg-data-1.txt
(61%)
copy paimon-common/src/main/java/org/apache/paimon/annotation/ConfigGroup.java
=>
paimon-flink/paimon-flink-common/src/test/resources/kafka/ogg/table/startupmode/ogg-data-2.txt
(62%)
