This is an automated email from the ASF dual-hosted git repository.
jark pushed a change to branch release-0.9
in repository https://gitbox.apache.org/repos/asf/fluss.git
from 417c2207a [paimon] Fix union read paimon dv table issue (#2326)
new 55e57d3d7 [docs] Update Quickstart to Use S3 (via RustFS) Instead of
Local File System (#2569)
new b3d1103b6 Fix list rebalance procedure throw ClassCastException (#2637)
new e81c8c316 [hotfix][server] KvSnapshotLeaseMetadataJsonSerde serialize
kv snapshot path use toString() instead of getPath() (#2633)
new 294395128 [kv] Validate column data types for aggregation function
during table creation (#2323)
new d8c18fdc5 [docs] Improve aggregation.md with complete SQL examples for
rbm32/rbm64 (#2643)
new 3015f6b56 [client] Thrown exception if ServerType mismatch. (#2606)
new f404ca361 [docs] Fix null value syntax in aggregation merge engine
examples (#2645)
new 09951e76b [flink] Add exactly-once sink support with undo recovery for
aggregation tables (#2613)
new 9ed4d34c1 [hotfix][spark] correct shaded artifact name for
fluss-spark-common in fluss-spark-3.x (#2641)
new 9a6e4c846 [lake/iceberg] support RowType conversion in
FlussArrayAsIcebergList (#2649)
new 0580ceee1 [test] Fix flaky testUndoRecoveryMultipleKeys by
checkpointing keyIndex in FailingCountingSource
new 657521922 [hotfix] Fix flaky testRescaleUp by fixing race condition in
ProducerOffsetsStore.deleteSnapshot
new 732d4bc3c [license] Keep original license headers for lightproto
source code (#2174)
new 4454a5a00 [docs] add introduction for auto-increment column (#2650)
new 0f22f788a [lake/paimon] Skip readable snapshot recomputation when
latest compacted snapshot already in ZK (#2640)
new 176758598 [docs] Update quickstart-flink doc to use
'kv.snapshot.interval' = 0s to avoid client obtain token issue (#2660)
new 86f704f47 [lake/flink] Fix lake tiering doesn't work in flink 2.2
(#2657)
new 4bdc5d61d [kv] Introduce TabletState to support persistence of
auto-inc buffer and row count (#2651)
new d6fd1f1f6 [kv] Support COUNT(*) pushdown for primary key tables and
introduce GetTableStats RPC (#2651)
The 19 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
LICENSE | 30 +-
docker/quickstart-flink/prepare_build.sh | 4 +-
.../java/org/apache/fluss/client/admin/Admin.java | 8 +
.../org/apache/fluss/client/admin/FlussAdmin.java | 110 ++
.../fluss/client/metadata/MetadataUpdater.java | 5 +-
.../fluss/client/utils/ClientRpcMessageUtils.java | 27 +
.../fluss/client/write/IdempotenceManager.java | 10 +-
.../apache/fluss/client/write/WriterClient.java | 6 +-
.../fluss/client/admin/FlussAdminITCase.java | 200 ++++
.../fluss/client/table/FlussTableITCase.java | 49 +-
.../fluss/client/write/RecordAccumulatorTest.java | 3 +-
.../org/apache/fluss/client/write/SenderTest.java | 6 +-
.../java/org/apache/fluss/cluster/ServerType.java | 27 +-
...eption.java => InvalidServerTypeException.java} | 11 +-
.../org/apache/fluss/metadata/AggFunction.java | 14 +-
.../org/apache/fluss/metadata/AggFunctionType.java | 89 +-
.../org/apache/fluss/metadata/AggFunctions.java | 3 +
.../AlignedArray.java => metadata/TableStats.java} | 46 +-
.../org/apache/fluss/utils/ProtoCodecUtils.java | 13 +-
.../apache/fluss/metadata/TableDescriptorTest.java | 45 +-
.../fluss/flink/adapter/RuntimeContextAdapter.java | 24 +-
.../flink/adapter/TypeInformationAdapter.java | 64 ++
.../flink/sink/Flink118UndoRecoveryITCase.java} | 4 +-
.../fluss/flink/tiering/Flink18TieringITCase.java | 14 +-
.../flink/sink/Flink119UndoRecoveryITCase.java} | 4 +-
.../fluss/flink/tiering/Flink19TieringITCase.java | 14 +-
.../flink/sink/Flink120UndoRecoveryITCase.java} | 4 +-
.../fluss/flink/tiering/Flink20TieringITCase.java | 14 +-
.../flink/adapter/TypeInformationAdapter.java | 64 ++
.../flink/sink/Flink22UndoRecoveryITCase.java} | 4 +-
.../fluss/flink/tiering/Flink22TieringITCase.java | 14 +-
.../apache/fluss/flink/FlinkConnectorOptions.java | 9 +
.../fluss/flink/adapter/RuntimeContextAdapter.java | 18 +
.../flink/adapter/TypeInformationAdapter.java | 73 ++
.../fluss/flink/catalog/FlinkTableFactory.java | 19 +-
.../procedure/ListRebalanceProcessProcedure.java | 2 +-
.../apache/fluss/flink/sink/ChannelComputer.java | 24 +
.../flink/sink/FlinkRowDataChannelComputer.java | 12 +-
.../org/apache/fluss/flink/sink/FlinkSink.java | 100 +-
.../apache/fluss/flink/sink/FlinkTableSink.java | 15 +-
.../org/apache/fluss/flink/sink/FlussSink.java | 1 +
.../apache/fluss/flink/sink/FlussSinkBuilder.java | 31 +-
.../shuffle/StatisticsOrRecordTypeInformation.java | 20 +-
.../apache/fluss/flink/sink/state/WriterState.java | 110 ++
.../flink/sink/state/WriterStateSerializer.java | 194 ++++
.../{writer => }/undo/BucketRecoveryContext.java | 13 +-
.../flink/sink/undo/ProducerOffsetReporter.java | 56 +
.../flink/sink/undo/RecoveryOffsetManager.java | 618 +++++++++++
.../flink/sink/{writer => }/undo/UndoComputer.java | 2 +-
.../{writer => }/undo/UndoRecoveryExecutor.java | 2 +-
.../{writer => }/undo/UndoRecoveryManager.java | 2 +-
.../flink/sink/undo/UndoRecoveryOperator.java | 695 +++++++++++++
.../sink/undo/UndoRecoveryOperatorFactory.java | 417 ++++++++
.../fluss/flink/sink/writer/UpsertSinkWriter.java | 55 +-
.../fluss/flink/source/FlinkTableSource.java | 3 +-
.../committer/CommittableMessageTypeInfo.java | 8 +-
.../source/TableBucketWriteResultTypeInfo.java | 8 +-
.../fluss/flink/utils/FlinkAggFunctionParser.java | 15 +-
.../flink/utils/FlinkConnectorOptionsUtils.java | 34 +
.../apache/fluss/flink/utils/FlinkConversions.java | 2 +-
.../apache/fluss/flink/utils/PushdownUtils.java | 78 +-
.../flink/procedure/FlinkProcedureITCase.java | 2 +-
.../fluss/flink/sink/UndoRecoveryITCase.java | 1098 ++++++++++++++++++++
.../sink/state/WriterStateSerializerTest.java | 174 ++++
.../fluss/flink/sink/testutils/CountingSource.java | 393 +++++++
.../sink/testutils/FailingCountingSource.java | 620 +++++++++++
.../flink/sink/testutils/TestAdminAdapter.java | 313 ++++++
.../flink/sink/undo/RecoveryOffsetManagerTest.java | 850 +++++++++++++++
.../sink/{writer => }/undo/UndoComputerTest.java | 2 +-
.../undo/UndoRecoveryExecutorTest.java | 73 +-
.../undo/UndoRecoveryManagerITCase.java | 16 +-
.../flink/source/FlinkTableSourceBatchITCase.java | 55 +-
.../apache/fluss/flink/tiering/TieringITCase.java | 2 +-
.../flink/utils/FlinkAggFunctionParserTest.java | 38 +-
.../apache/fluss/flink/utils/TestLogScanner.java | 10 -
.../iceberg/source/FlussArrayAsIcebergList.java | 13 +
.../source/FlussRowAsIcebergRecordTest.java | 46 +-
.../utils/DvTableReadableSnapshotRetriever.java | 92 +-
.../DvTableReadableSnapshotRetrieverTest.java | 98 +-
.../generator/generator/NameUtilities.java | 13 +-
.../generator/generator/ProtoCodeGenerator.java | 13 +-
.../protogen/generator/generator/ProtoGen.java | 13 +-
.../protogen/generator/generator/ProtoGenUtil.java | 13 +-
.../generator/ProtobufAbstractRepeated.java | 13 +-
.../generator/generator/ProtobufBooleanField.java | 13 +-
.../generator/generator/ProtobufBytesField.java | 13 +-
.../protogen/generator/generator/ProtobufEnum.java | 13 +-
.../generator/generator/ProtobufEnumField.java | 13 +-
.../generator/generator/ProtobufField.java | 13 +-
.../generator/generator/ProtobufMessage.java | 13 +-
.../generator/generator/ProtobufMessageField.java | 13 +-
.../generator/generator/ProtobufNumberField.java | 13 +-
.../generator/generator/ProtobufRecordsField.java | 13 +-
.../generator/ProtobufRepeatedBytesField.java | 13 +-
.../generator/ProtobufRepeatedEnumField.java | 13 +-
.../generator/ProtobufRepeatedMessageField.java | 13 +-
.../generator/ProtobufRepeatedNumberField.java | 13 +-
.../generator/ProtobufRepeatedStringField.java | 13 +-
.../generator/generator/ProtobufStringField.java | 13 +-
.../generator/generator/RecordsFieldFinder.java | 13 +-
.../fluss/protogen/maven/plugin/ProtoGenMojo.java | 13 +-
.../rpc/entity/TableStatsResultForBucket.java | 35 +-
.../fluss/rpc/gateway/TabletServerGateway.java | 10 +
.../fluss/rpc/netty/client/ServerConnection.java | 24 +
.../org/apache/fluss/rpc/protocol/ApiKeys.java | 3 +-
fluss-rpc/src/main/proto/FlussApi.proto | 47 +
.../apache/fluss/rpc/TestingGatewayService.java | 1 +
.../fluss/rpc/TestingTabletGatewayService.java | 10 +-
.../rpc/netty/authenticate/AuthenticationTest.java | 4 +-
.../authenticate/SaslAuthenticationITCase.java | 2 +-
.../rpc/netty/client/ServerConnectionTest.java | 55 +
.../org/apache/fluss/server/RpcServiceBase.java | 1 +
.../coordinator/producer/ProducerOffsetsStore.java | 9 +-
.../apache/fluss/server/kv/KvRecoverHelper.java | 254 ++++-
.../java/org/apache/fluss/server/kv/KvTablet.java | 75 +-
.../fluss/server/kv/autoinc/AutoIncIDRange.java | 84 ++
.../server/kv/autoinc/AutoIncrementManager.java | 28 +-
.../autoinc/BoundedSegmentSequenceGenerator.java | 21 +-
.../fluss/server/kv/autoinc/SequenceGenerator.java | 14 +
.../kv/autoinc/ZkSequenceGeneratorFactory.java | 33 +-
.../fluss/server/kv/prewrite/KvPreWriteBuffer.java | 69 +-
.../aggregate/factory/FieldListaggAggFactory.java | 2 +-
.../aggregate/factory/FieldStringAggFactory.java | 2 +-
.../server/kv/snapshot/CompletedSnapshot.java | 52 +-
.../kv/snapshot/CompletedSnapshotJsonSerde.java | 60 +-
.../server/kv/snapshot/KvTabletSnapshotTarget.java | 26 +-
.../kv/snapshot/RocksIncrementalSnapshot.java | 12 +-
.../fluss/server/kv/snapshot/SnapshotResult.java | 12 +-
.../fluss/server/kv/snapshot/SnapshotRunner.java | 4 +-
.../fluss/server/kv/snapshot/TabletState.java | 74 ++
.../org/apache/fluss/server/log/LogTablet.java | 4 +
.../org/apache/fluss/server/replica/Replica.java | 43 +-
.../fluss/server/replica/ReplicaManager.java | 18 +
.../apache/fluss/server/tablet/TabletService.java | 15 +
.../fluss/server/utils/ServerRpcMessageUtils.java | 44 +
.../server/utils/TableDescriptorValidation.java | 12 +-
.../lease/KvSnapshotLeaseMetadataJsonSerde.java | 2 +-
.../org/apache/fluss/server/kv/KvTabletTest.java | 143 +++
.../kv/autoinc/SegmentSequenceGeneratorTest.java | 38 +
.../autoinc/TestingSequenceGeneratorFactory.java | 1 +
.../server/kv/prewrite/KvPreWriteBufferTest.java | 77 +-
.../snapshot/CompletedSnapshotJsonSerdeTest.java | 12 +-
.../kv/snapshot/KvTabletSnapshotTargetTest.java | 13 +-
.../kv/snapshot/PeriodicSnapshotManagerTest.java | 4 +-
.../kv/snapshot/RocksIncrementalSnapshotTest.java | 6 +-
.../server/replica/KvReplicaRestoreITCase.java | 64 +-
.../server/tablet/TestTabletServerGateway.java | 7 +
.../apache/fluss/server/testutils/KvTestUtils.java | 4 +-
.../KvSnapshotLeaseMetadataJsonSerdeTest.java | 8 +-
fluss-spark/fluss-spark-3.4/pom.xml | 2 +-
fluss-spark/fluss-spark-3.5/pom.xml | 2 +-
pom.xml | 2 +-
website/docs/quickstart/flink.md | 72 +-
website/docs/quickstart/lakehouse.md | 305 ++++--
.../docs/table-design/merge-engines/aggregation.md | 142 ++-
website/docs/table-design/table-types/pk-table.md | 73 ++
156 files changed, 9161 insertions(+), 722 deletions(-)
copy
fluss-common/src/main/java/org/apache/fluss/exception/{InvalidConfigException.java
=> InvalidServerTypeException.java} (81%)
copy fluss-common/src/main/java/org/apache/fluss/{row/array/AlignedArray.java
=> metadata/TableStats.java} (52%)
create mode 100644
fluss-flink/fluss-flink-1.18/src/main/java/org/apache/fluss/flink/adapter/TypeInformationAdapter.java
copy
fluss-flink/{fluss-flink-1.20/src/test/java/org/apache/fluss/flink/sink/Flink120TableSinkITCase.java
=>
fluss-flink-1.18/src/test/java/org/apache/fluss/flink/sink/Flink118UndoRecoveryITCase.java}
(86%)
copy
fluss-common/src/main/java/org/apache/fluss/exception/SequenceOverflowException.java
=>
fluss-flink/fluss-flink-1.18/src/test/java/org/apache/fluss/flink/tiering/Flink18TieringITCase.java
(76%)
copy
fluss-flink/{fluss-flink-1.20/src/test/java/org/apache/fluss/flink/sink/Flink120TableSinkITCase.java
=>
fluss-flink-1.19/src/test/java/org/apache/fluss/flink/sink/Flink119UndoRecoveryITCase.java}
(86%)
copy
fluss-common/src/main/java/org/apache/fluss/exception/SequenceOverflowException.java
=>
fluss-flink/fluss-flink-1.19/src/test/java/org/apache/fluss/flink/tiering/Flink19TieringITCase.java
(76%)
copy
fluss-flink/{fluss-flink-2.2/src/test/java/org/apache/fluss/flink/sink/Flink22TableSinkITCase.java
=>
fluss-flink-1.20/src/test/java/org/apache/fluss/flink/sink/Flink120UndoRecoveryITCase.java}
(87%)
copy
fluss-common/src/main/java/org/apache/fluss/exception/SequenceOverflowException.java
=>
fluss-flink/fluss-flink-1.20/src/test/java/org/apache/fluss/flink/tiering/Flink20TieringITCase.java
(76%)
create mode 100644
fluss-flink/fluss-flink-2.2/src/main/java/org/apache/fluss/flink/adapter/TypeInformationAdapter.java
copy
fluss-flink/{fluss-flink-1.20/src/test/java/org/apache/fluss/flink/sink/Flink120TableSinkITCase.java
=>
fluss-flink-2.2/src/test/java/org/apache/fluss/flink/sink/Flink22UndoRecoveryITCase.java}
(87%)
copy
fluss-common/src/main/java/org/apache/fluss/exception/SequenceOverflowException.java
=>
fluss-flink/fluss-flink-2.2/src/test/java/org/apache/fluss/flink/tiering/Flink22TieringITCase.java
(76%)
create mode 100644
fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/adapter/TypeInformationAdapter.java
create mode 100644
fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/state/WriterState.java
create mode 100644
fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/state/WriterStateSerializer.java
rename
fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/{writer
=> }/undo/BucketRecoveryContext.java (85%)
create mode 100644
fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/undo/ProducerOffsetReporter.java
create mode 100644
fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/undo/RecoveryOffsetManager.java
rename
fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/{writer
=> }/undo/UndoComputer.java (99%)
rename
fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/{writer
=> }/undo/UndoRecoveryExecutor.java (99%)
rename
fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/{writer
=> }/undo/UndoRecoveryManager.java (99%)
create mode 100644
fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/undo/UndoRecoveryOperator.java
create mode 100644
fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/undo/UndoRecoveryOperatorFactory.java
create mode 100644
fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/UndoRecoveryITCase.java
create mode 100644
fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/state/WriterStateSerializerTest.java
create mode 100644
fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/testutils/CountingSource.java
create mode 100644
fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/testutils/FailingCountingSource.java
create mode 100644
fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/testutils/TestAdminAdapter.java
create mode 100644
fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/undo/RecoveryOffsetManagerTest.java
rename
fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/{writer
=> }/undo/UndoComputerTest.java (99%)
rename
fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/{writer
=> }/undo/UndoRecoveryExecutorTest.java (74%)
rename
fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/{writer
=> }/undo/UndoRecoveryManagerITCase.java (98%)
copy fluss-common/src/main/java/org/apache/fluss/row/map/AlignedMap.java =>
fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/TableStatsResultForBucket.java
(52%)
create mode 100644
fluss-server/src/main/java/org/apache/fluss/server/kv/autoinc/AutoIncIDRange.java
create mode 100644
fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/TabletState.java
