This is an automated email from the ASF dual-hosted git repository.
jark pushed a change to branch release-0.9
in repository https://gitbox.apache.org/repos/asf/fluss.git
from 5e04dcc47 [client] Implement Lookup with insert-if-not-exists on the
client side (#2573)
new 7bd28226c [flink] Support "lookup.insert-if-not-exists" option for
Flink Lookup Join (#2601)
new 6543da48d [flink] Fix skipping partition pushdown for primary key
lakehouse table in batch mode. (#2585)
new ceef4535c [server] Support alter table config
`table.log.tiered.local-segments` (#2508)
new 6e7fc2669 [kv] Support kv snapshot lease (#2179)
new 1fd153034 [kv] Add undo recovery support for aggregation tables (#2545)
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/fluss/client/admin/Admin.java | 9 +
.../org/apache/fluss/client/admin/FlussAdmin.java | 5 +
.../apache/fluss/client/admin/KvSnapshotLease.java | 73 ++
.../fluss/client/admin/KvSnapshotLeaseImpl.java | 90 ++
...shot.java => AcquireKvSnapshotLeaseResult.java} | 47 +-
.../apache/fluss/client/metadata/KvSnapshots.java | 8 +
.../fluss/client/table/scanner/log/LogScanner.java | 10 +
.../client/table/scanner/log/LogScannerImpl.java | 17 +
.../fluss/client/utils/ClientRpcMessageUtils.java | 77 ++
.../client/admin/CustomFlussClusterITCase.java | 125 +++
.../fluss/client/admin/FlussAdminITCase.java | 74 ++
.../client/metadata/TestingClientSchemaGetter.java | 6 +-
.../security/acl/FlussAuthorizationITCase.java | 132 +++
.../batch/KvSnapshotBatchScannerITCase.java | 114 +++
.../client/table/scanner/log/LogFetcherTest.java | 5 +-
.../org/apache/fluss/config/ConfigOptions.java | 10 +-
.../org/apache/fluss/config/FlussConfigUtils.java | 3 +-
...BucketReplica.java => TableBucketSnapshot.java} | 36 +-
.../java/org/apache/fluss/metrics/MetricNames.java | 5 +
.../org/apache/fluss/utils/ByteArrayWrapper.java | 38 +-
.../java/org/apache/fluss/utils/FlussPaths.java | 37 +
.../java/org/apache/fluss/record/TestData.java | 14 +
.../apache/fluss/utils/ByteArrayWrapperTest.java | 99 ++
.../apache/fluss/flink/FlinkConnectorOptions.java | 29 +
.../fluss/flink/catalog/FlinkTableFactory.java | 9 +-
...dure.java => DropKvSnapshotLeaseProcedure.java} | 28 +-
.../fluss/flink/procedure/ProcedureManager.java | 3 +-
.../sink/writer/undo/BucketRecoveryContext.java | 137 +++
.../fluss/flink/sink/writer/undo/UndoComputer.java | 150 ++++
.../sink/writer/undo/UndoRecoveryExecutor.java | 287 ++++++
.../sink/writer/undo/UndoRecoveryManager.java | 248 +++++
.../org/apache/fluss/flink/source/FlinkSource.java | 20 +-
.../fluss/flink/source/FlinkTableSource.java | 45 +-
.../org/apache/fluss/flink/source/FlussSource.java | 4 +-
.../source/enumerator/FlinkSourceEnumerator.java | 189 +++-
...nt.java => FinishedKvSnapshotConsumeEvent.java} | 40 +-
.../source/lookup/FlinkAsyncLookupFunction.java | 7 +-
.../flink/source/lookup/FlinkLookupFunction.java | 7 +-
.../flink/source/reader/FlinkSourceReader.java | 53 ++
.../fluss/flink/source/reader/LeaseContext.java | 91 ++
.../FlussSourceEnumeratorStateSerializer.java | 45 +-
.../flink/source/state/SourceEnumeratorState.java | 16 +-
.../apache/fluss/flink/utils/PushdownUtils.java | 3 +-
.../flink/procedure/FlinkProcedureITCase.java | 43 +-
.../flink/sink/writer/undo/UndoComputerTest.java | 199 ++++
.../sink/writer/undo/UndoRecoveryExecutorTest.java | 234 +++++
.../writer/undo/UndoRecoveryManagerITCase.java | 999 +++++++++++++++++++++
.../fluss/flink/source/FlinkTableSourceITCase.java | 129 +++
.../enumerator/FlinkSourceEnumeratorTest.java | 29 +-
.../source/lookup/FlinkLookupFunctionTest.java | 12 +-
.../state/SourceEnumeratorStateSerializerTest.java | 23 +-
.../source/testutils/FlinkRowAssertionsUtils.java | 9 +
.../apache/fluss/flink/utils/TestLogScanner.java | 159 ++++
.../apache/fluss/flink/utils/TestUpsertWriter.java | 118 +++
.../flink/FlinkUnionReadPrimaryKeyTableITCase.java | 79 +-
.../org/apache/fluss/rpc/gateway/AdminGateway.java | 18 +
.../org/apache/fluss/rpc/protocol/ApiKeys.java | 5 +-
fluss-rpc/src/main/proto/FlussApi.proto | 34 +
.../coordinator/CompletedSnapshotStoreManager.java | 15 +-
.../coordinator/CoordinatorEventProcessor.java | 9 +-
.../server/coordinator/CoordinatorServer.java | 35 +-
.../server/coordinator/CoordinatorService.java | 121 ++-
.../coordinator/lease/KvSnapshotLeaseHandler.java | 233 +++++
.../coordinator/lease/KvSnapshotLeaseManager.java | 489 ++++++++++
.../lease/KvSnapshotLeaseMetadataManager.java | 281 ++++++
.../server/kv/snapshot/CompletedSnapshotStore.java | 128 ++-
.../server/kv/snapshot/SharedKvFileRegistry.java | 57 +-
.../org/apache/fluss/server/log/LogTablet.java | 10 +-
.../org/apache/fluss/server/replica/Replica.java | 21 +
.../fluss/server/replica/ReplicaManager.java | 20 +-
.../fluss/server/utils/ServerRpcMessageUtils.java | 68 ++
.../apache/fluss/server/zk/ZooKeeperClient.java | 33 +
.../org/apache/fluss/server/zk/data/ZkData.java | 36 +
.../zk/data/lease/KvSnapshotLeaseMetadata.java | 71 ++
.../lease/KvSnapshotLeaseMetadataJsonSerde.java | 84 ++
.../server/zk/data/lease/KvSnapshotTableLease.java | 190 ++++
.../data/lease/KvSnapshotTableLeaseJsonSerde.java | 116 +++
.../CompletedSnapshotStoreManagerTest.java | 6 +-
.../coordinator/CoordinatorEventProcessorTest.java | 18 +-
.../server/coordinator/TestCoordinatorGateway.java | 24 +
.../lease/KvSnapshotLeaseHandlerTest.java | 287 ++++++
.../lease/KvSnapshotLeaseManagerTest.java | 612 +++++++++++++
.../lease/KvSnapshotLeaseMetadataManagerTest.java | 281 ++++++
.../rebalance/RebalanceManagerTest.java | 18 +-
.../statemachine/TableBucketStateMachineTest.java | 18 +-
.../kv/snapshot/CompletedSnapshotStoreTest.java | 250 +++++-
.../kv/snapshot/KvTabletSnapshotTargetTest.java | 3 +-
.../kv/snapshot/SharedKvFileRegistryTest.java | 90 ++
.../ZooKeeperCompletedSnapshotStoreTest.java | 3 +-
.../server/log/remote/RemoteLogManagerTest.java | 59 ++
.../server/testutils/FlussClusterExtension.java | 2 +-
.../fluss/server/zk/ZooKeeperClientTest.java | 32 +
.../KvSnapshotLeaseMetadataJsonSerdeTest.java | 57 ++
.../lease/KvSnapshotTableLeaseJsonSerdeTest.java | 54 ++
fluss-test-coverage/pom.xml | 3 +
website/docs/_configs/_partial_config.mdx | 16 +-
website/docs/engine-flink/options.md | 59 +-
website/docs/engine-flink/procedures.md | 34 +
website/docs/maintenance/configuration.md | 3 +-
.../maintenance/observability/monitor-metrics.md | 12 +-
100 files changed, 8216 insertions(+), 277 deletions(-)
create mode 100644
fluss-client/src/main/java/org/apache/fluss/client/admin/KvSnapshotLease.java
create mode 100644
fluss-client/src/main/java/org/apache/fluss/client/admin/KvSnapshotLeaseImpl.java
copy
fluss-client/src/main/java/org/apache/fluss/client/metadata/{LakeSnapshot.java
=> AcquireKvSnapshotLeaseResult.java} (51%)
copy
fluss-common/src/main/java/org/apache/fluss/metadata/{TableBucketReplica.java
=> TableBucketSnapshot.java} (66%)
copy
fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/event/TieringReachMaxDurationEvent.java
=> fluss-common/src/main/java/org/apache/fluss/utils/ByteArrayWrapper.java
(55%)
create mode 100644
fluss-common/src/test/java/org/apache/fluss/utils/ByteArrayWrapperTest.java
copy
fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/procedure/{CancelRebalanceProcedure.java
=> DropKvSnapshotLeaseProcedure.java} (60%)
create mode 100644
fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/undo/BucketRecoveryContext.java
create mode 100644
fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/undo/UndoComputer.java
create mode 100644
fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/undo/UndoRecoveryExecutor.java
create mode 100644
fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/undo/UndoRecoveryManager.java
copy
fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/event/{PartitionBucketsUnsubscribedEvent.java
=> FinishedKvSnapshotConsumeEvent.java} (55%)
create mode 100644
fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/LeaseContext.java
create mode 100644
fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/writer/undo/UndoComputerTest.java
create mode 100644
fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/writer/undo/UndoRecoveryExecutorTest.java
create mode 100644
fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/writer/undo/UndoRecoveryManagerITCase.java
create mode 100644
fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/TestLogScanner.java
create mode 100644
fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/TestUpsertWriter.java
create mode 100644
fluss-server/src/main/java/org/apache/fluss/server/coordinator/lease/KvSnapshotLeaseHandler.java
create mode 100644
fluss-server/src/main/java/org/apache/fluss/server/coordinator/lease/KvSnapshotLeaseManager.java
create mode 100644
fluss-server/src/main/java/org/apache/fluss/server/coordinator/lease/KvSnapshotLeaseMetadataManager.java
create mode 100644
fluss-server/src/main/java/org/apache/fluss/server/zk/data/lease/KvSnapshotLeaseMetadata.java
create mode 100644
fluss-server/src/main/java/org/apache/fluss/server/zk/data/lease/KvSnapshotLeaseMetadataJsonSerde.java
create mode 100644
fluss-server/src/main/java/org/apache/fluss/server/zk/data/lease/KvSnapshotTableLease.java
create mode 100644
fluss-server/src/main/java/org/apache/fluss/server/zk/data/lease/KvSnapshotTableLeaseJsonSerde.java
create mode 100644
fluss-server/src/test/java/org/apache/fluss/server/coordinator/lease/KvSnapshotLeaseHandlerTest.java
create mode 100644
fluss-server/src/test/java/org/apache/fluss/server/coordinator/lease/KvSnapshotLeaseManagerTest.java
create mode 100644
fluss-server/src/test/java/org/apache/fluss/server/coordinator/lease/KvSnapshotLeaseMetadataManagerTest.java
create mode 100644
fluss-server/src/test/java/org/apache/fluss/server/zk/data/lease/KvSnapshotLeaseMetadataJsonSerdeTest.java
create mode 100644
fluss-server/src/test/java/org/apache/fluss/server/zk/data/lease/KvSnapshotTableLeaseJsonSerdeTest.java
