This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a change to branch release-0.8
in repository https://gitbox.apache.org/repos/asf/paimon.git
from 276d1ab51 [flink] Introduce
ManifestCommittableSerializerCompatibilityTest
new 499e9c5db [minor] Add Nullable to logOffsets in Snapshot
new ad371d5ca [minor] Fix exception message in FileStoreCommitImpl
new 64e031343 [doc] add some descriptions about data writing in
basic-concepts (#3301)
new 580208828 [cdc] Fixed when the order of the same field differs, it is
considered a schema change. (#3314)
new d83a06255 [doc] Reorg Append table pages
new 45be92ac2 [hive] fix db-location conflict with location-in-properties
(#3309)
new a177de33a [doc] Update version and link of Doris and Trino (#3322)
new 3c75ca773 [spark] Fix drop database cascade when database is not empy
with SparkGenericCatalog (#3324)
new 92e2f00f8 [core] FileStoreWrite.State should get max sequence number
from RecordWriter instead of from data files (#3327)
new 508b260fd [spark] Add check for provider when creating table with
SparkCatalog (#3326)
new 636b9fa58 [cdc] Add more checker for the timestamp type. (#3312)
new b8b6e8664 [doc] Optimize useless message in dedicated-compaction
new 51103e043 [core] Add JsonIgnoreProperties to all JSON class
new a088a044c [doc] Add Specification Page
new 688d12adf [core] Avoid potential NPE in FileStoreCommitImpl and
TagAutoCreation
new ee3cd40a1 [bundle] add source code to the source jar for
paimon-bundle (#3333)
new b1944274e [flink] Respect waitCompaction in FlinkCdcMultiTableSink
(#3337)
new de32dae41 [core] Incremental-between tags should deduplicate records
(#3338)
new 9e63ea6e2 [spark][core] fix when a table have no snapshot or no
statistics (#3341)
new 0b63cb8a9 [spark] Fix drop database cascade when database with view
with SparkGenericCatalog (#3347)
new 3881d6a88 [core] Revert branchs in TagsTable (#3350)
new e3e4303ab [core] Fix that ignore-delete hasn't handle rowkind.field
(#3365)
new 78a850937 [flink] Adjust the setMaxParallelism api call in
QueryService (#3377)
new f91a22029 [spark] Fix HiveMigrator hive api version compatible issue
(#3381)
new 6355d13a0 [flink] Cleanup QueryService file after job is down (#3382)
new 638e89a4a [doc] Add doc for StarRocks (#3392)
new 67db033e8 [doc] Remove outdate roadmap page
new 42299afdc [doc] Fix typo in mongo-cdc.md (#3413)
new f3e67b7b3 [doc] Update managed memory guide for write buffer managing
(#3417)
new d78658a18 [spark] SparkHilbertUDF hilbertCurvePosBytes classnotfound
fix (#3415)
new cfcc72d7d [hive] Fix flink not synchronizing to hive metastore when
deleting table partitions using hive catalog (#3411)
The 31 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../{append-queue-table.md => append-queue.md} | 48 ++--
docs/content/append-table/append-scalable-table.md | 114 ----------
docs/content/append-table/append-table.md | 64 ++++++
docs/content/concepts/basic-concepts.md | 2 +-
docs/content/concepts/specification.md | 253 +++++++++++++++++++++
docs/content/engines/overview.md | 9 +-
docs/content/engines/starrocks.md | 186 +++++++++++++++
docs/content/flink/cdc-ingestion/mongo-cdc.md | 4 +-
docs/content/learn-paimon/understand-files.md | 2 +-
docs/content/maintenance/dedicated-compaction.md | 10 +-
docs/content/maintenance/system-tables.md | 2 +-
docs/content/maintenance/write-performance.md | 4 +
docs/content/migration/migration-from-hive.md | 2 +-
docs/content/project/roadmap.md | 48 ----
docs/static/img/for-scalable.png | Bin 1331757 -> 0 bytes
paimon-bundle/pom.xml | 66 ++++++
.../src/assembly/src-assembly.xml | 37 +--
.../apache/paimon/sort/hilbert/HilbertIndexer.java | 2 +-
.../src/main/java/org/apache/paimon/Changelog.java | 2 +
.../src/main/java/org/apache/paimon/Snapshot.java | 3 +-
.../org/apache/paimon/append/AppendOnlyWriter.java | 5 +
.../java/org/apache/paimon/consumer/Consumer.java | 2 +
.../org/apache/paimon/mergetree/MergeSorter.java | 4 +
.../apache/paimon/mergetree/MergeTreeWriter.java | 5 +
.../paimon/operation/AbstractFileStoreWrite.java | 6 +
.../paimon/operation/AppendOnlyFileStoreWrite.java | 6 +-
.../paimon/operation/FileStoreCommitImpl.java | 7 +-
.../apache/paimon/operation/FileStoreWrite.java | 6 +-
.../paimon/operation/KeyValueFileStoreWrite.java | 4 +-
.../paimon/operation/MergeFileSplitRead.java | 109 ++++-----
.../apache/paimon/operation/RawFileSplitRead.java | 12 +
.../org/apache/paimon/operation/SplitRead.java | 46 +++-
.../java/org/apache/paimon/stats/ColStats.java | 2 +
.../java/org/apache/paimon/stats/Statistics.java | 2 +
.../paimon/table/AbstractFileStoreTable.java | 5 +
.../paimon/table/AppendOnlyFileStoreTable.java | 7 +-
.../paimon/table/PrimaryKeyFileStoreTable.java | 21 +-
.../apache/paimon/table/PrimaryKeyTableUtils.java | 8 +-
.../apache/paimon/table/sink/RowKindGenerator.java | 4 +
.../apache/paimon/table/sink/TableWriteImpl.java | 16 +-
.../paimon/table/source/KeyValueTableRead.java | 79 +++----
.../IncrementalChangelogReadProvider.java | 96 ++++++++
.../splitread/IncrementalDiffReadProvider.java | 65 ++++++
.../splitread/IncrementalDiffSplitRead.java} | 95 +++++++-
.../splitread/MergeFileSplitReadProvider.java | 68 ++++++
.../source/splitread/RawFileSplitReadProvider.java | 60 +++++
.../table/source/splitread/SplitReadProvider.java | 14 +-
.../apache/paimon/table/system/StatisticTable.java | 26 ++-
.../org/apache/paimon/table/system/TagsTable.java | 35 +--
.../src/main/java/org/apache/paimon/tag/Tag.java | 2 +
.../org/apache/paimon/tag/TagAutoCreation.java | 8 +-
.../java/org/apache/paimon/utils/RecordWriter.java | 3 +
.../paimon/operation/MergeFileSplitReadTest.java | 2 +-
.../apache/paimon/table/IncrementalTableTest.java | 18 +-
.../apache/paimon/table/system/TagsTableTest.java | 35 +--
.../cdc/format/debezium/CdcDebeziumSchema.java | 91 --------
.../action/cdc/mysql/format/DebeziumEvent.java | 2 +
.../flink/sink/cdc/FlinkCdcMultiTableSink.java | 4 +-
.../paimon/flink/sink/cdc/RichEventParser.java | 17 +-
.../cdc/UpdatedDataFieldsProcessFunctionBase.java | 11 +
.../cdc/mysql/MySqlSyncTableActionITCase.java | 42 ++++
.../UpdatedDataFieldsProcessFunctionBaseTest.java | 19 ++
.../src/test/resources/mysql/sync_table_setup.sql | 11 +
.../paimon/flink/service/QueryAddressRegister.java | 14 +-
.../paimon/flink/service/QueryFileMonitor.java | 5 +
.../apache/paimon/flink/service/QueryService.java | 19 +-
.../paimon/flink/sink/LocalMergeOperator.java | 3 +-
.../apache/paimon/flink/BatchFileStoreITCase.java | 16 ++
.../paimon/flink/RemoteLookupJoinITCase.java | 31 +++
.../java/org/apache/paimon/hive/HiveCatalog.java | 20 ++
.../paimon/hive/TBPropertiesLocationHelper.java | 1 +
.../apache/paimon/hive/migrate/HiveMigrator.java | 9 +-
.../apache/paimon/hive/HiveCatalogITCaseBase.java | 52 +++++
paimon-spark/paimon-spark-common/pom.xml | 5 -
.../java/org/apache/paimon/spark/SparkCatalog.java | 17 +-
.../apache/paimon/spark/SparkGenericCatalog.java | 16 +-
.../paimon/spark/catalog/SparkBaseCatalog.java | 5 +
.../apache/paimon/spark/sort/SparkHilbertUDF.java | 14 +-
.../commands/PaimonAnalyzeTableColumnCommand.scala | 11 +-
.../apache/paimon/spark/PaimonHiveTestBase.scala | 10 +-
.../apache/paimon/spark/PaimonSparkTestBase.scala | 13 +-
.../paimon/spark/sql/AnalyzeTableTestBase.scala | 8 +
.../org/apache/paimon/spark/sql/DDLTestBase.scala | 13 ++
.../spark/sql/DDLWithHiveCatalogTestBase.scala | 31 ++-
.../paimon/spark/sql/SparkVersionSupport.scala} | 7 +-
85 files changed, 1646 insertions(+), 612 deletions(-)
rename docs/content/append-table/{append-queue-table.md => append-queue.md}
(92%)
delete mode 100644 docs/content/append-table/append-scalable-table.md
create mode 100644 docs/content/append-table/append-table.md
create mode 100644 docs/content/concepts/specification.md
create mode 100644 docs/content/engines/starrocks.md
delete mode 100644 docs/content/project/roadmap.md
delete mode 100644 docs/static/img/for-scalable.png
rename docs/content/append-table/overview.md =>
paimon-bundle/src/assembly/src-assembly.xml (51%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/source/splitread/IncrementalChangelogReadProvider.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/source/splitread/IncrementalDiffReadProvider.java
rename paimon-core/src/main/java/org/apache/paimon/{operation/DiffReader.java
=> table/source/splitread/IncrementalDiffSplitRead.java} (55%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/source/splitread/MergeFileSplitReadProvider.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/source/splitread/RawFileSplitReadProvider.java
copy paimon-common/src/main/java/org/apache/paimon/utils/FieldsComparator.java
=>
paimon-core/src/main/java/org/apache/paimon/table/source/splitread/SplitReadProvider.java
(70%)
delete mode 100644
paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/debezium/CdcDebeziumSchema.java
copy
paimon-spark/{paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTest.scala
=>
paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/SparkVersionSupport.scala}
(83%)
