This is an automated email from the ASF dual-hosted git repository.
roman pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 8e63767 [FLINK-23460] Use global flag for enabling final checkpoints
in SubtaskCheckpointCoordinator
add 13bfa2b [hotfix][state/changelog] Drop unused handle constructor and
reader interface
add e653487 [hotfix][tests] Introduce TestTaskStateManagerBuilder
add 717a552 [FLINK-21353][state] Add DFS-based StateChangelog
add 7461096 [FLINK-21353][state/changelog] Implement batching
add aec46c1 [FLINK-21353][state/changelog] Implement retries
add 47446cd [FLINK-21353][tests] Use FS store in ChangelogBackend tests
No new revisions were added by this update.
Summary of changes:
.../flink-dstl-dfs}/pom.xml | 49 +--
.../changelog/fs/BatchingStateChangeUploader.java | 221 ++++++++++++
.../changelog/fs/FsStateChangelogOptions.java | 132 +++++++
.../changelog/fs/FsStateChangelogStorage.java | 95 +++++
.../fs/FsStateChangelogStorageFactory.java | 37 ++
.../flink/changelog/fs/FsStateChangelogWriter.java | 393 +++++++++++++++++++++
.../flink/changelog/fs/OutputStreamWithPos.java | 62 ++++
.../org/apache/flink/changelog/fs/RetryPolicy.java | 117 ++++++
.../flink/changelog/fs/RetryingExecutor.java | 170 +++++++++
.../flink/changelog/fs/SchedulerFactory.java | 61 ++++
.../flink/changelog/fs/StateChangeFormat.java | 158 +++++++++
.../flink/changelog/fs/StateChangeFsUploader.java | 145 ++++++++
.../apache/flink/changelog/fs/StateChangeSet.java | 71 ++++
.../flink/changelog/fs/StateChangeUploader.java | 137 +++++++
.../apache/flink/changelog/fs/UploadResult.java | 73 ++++
...me.state.changelog.StateChangelogStorageFactory | 16 +
.../fs/BatchingStateChangeUploaderTest.java | 221 ++++++++++++
.../changelog/fs/FsStateChangelogStorageTest.java | 44 +++
.../fs/FsStateChangelogWriterSqnTest.java | 159 +++++++++
.../changelog/fs/FsStateChangelogWriterTest.java | 243 +++++++++++++
.../flink/changelog/fs/RetryingExecutorTest.java | 180 ++++++++++
.../changelog/fs/TestingStateChangeUploader.java | 89 +++++
.../src/test/resources/log4j2.properties | 28 ++
flink-dstl/pom.xml | 42 +++
.../TaskExecutorStateChangelogStoragesManager.java | 3 +-
.../changelog/ChangelogStateHandleStreamImpl.java | 17 -
.../state/changelog/SequenceNumberRange.java | 91 +++++
.../StateChangelogHandleStreamHandleReader.java | 4 +-
.../changelog/StateChangelogStorageFactory.java | 4 +-
.../changelog/StateChangelogStorageLoader.java | 3 +-
.../state/changelog/StateChangelogWriter.java | 2 +-
.../flink/runtime/taskexecutor/TaskExecutor.java | 11 +-
.../flink/runtime/state/StateBackendTestBase.java | 11 +-
...kExecutorStateChangelogStoragesManagerTest.java | 9 +-
.../flink/runtime/state/TestTaskStateManager.java | 69 ++--
.../runtime/state/TestTaskStateManagerBuilder.java | 110 ++++++
.../inmemory/StateChangelogStorageLoaderTest.java | 7 +-
.../inmemory/StateChangelogStorageTest.java | 2 +-
.../flink-statebackend-changelog/pom.xml | 7 +
...logDelegateEmbeddedRocksDBStateBackendTest.java | 10 +
.../ChangelogDelegateFileStateBackendTest.java | 13 +
.../changelog/ChangelogDelegateHashMapTest.java | 12 +
.../ChangelogDelegateMemoryStateBackendTest.java | 13 +
.../changelog/ChangelogStateBackendTestUtils.java | 14 +
.../streaming/state/RocksDBAsyncSnapshotTest.java | 8 +-
.../api/operators/async/AsyncWaitOperatorTest.java | 2 -
.../tasks/InterruptSensitiveRestoreTest.java | 13 +-
.../runtime/tasks/OneInputStreamTaskTest.java | 6 +-
.../runtime/tasks/RestoreStreamTaskTest.java | 3 -
.../tasks/SourceOperatorStreamTaskTest.java | 6 +-
.../tasks/StreamTaskFinalCheckpointsTest.java | 6 +-
.../tasks/StreamTaskMailboxTestHarnessBuilder.java | 13 +-
.../state/StatefulOperatorChainedTaskTest.java | 4 +-
pom.xml | 1 +
54 files changed, 3284 insertions(+), 133 deletions(-)
copy {flink-state-backends/flink-statebackend-changelog =>
flink-dstl/flink-dstl-dfs}/pom.xml (65%)
create mode 100644
flink-dstl/flink-dstl-dfs/src/main/java/org/apache/flink/changelog/fs/BatchingStateChangeUploader.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/main/java/org/apache/flink/changelog/fs/FsStateChangelogOptions.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/main/java/org/apache/flink/changelog/fs/FsStateChangelogStorage.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/main/java/org/apache/flink/changelog/fs/FsStateChangelogStorageFactory.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/main/java/org/apache/flink/changelog/fs/FsStateChangelogWriter.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/main/java/org/apache/flink/changelog/fs/OutputStreamWithPos.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/main/java/org/apache/flink/changelog/fs/RetryPolicy.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/main/java/org/apache/flink/changelog/fs/RetryingExecutor.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/main/java/org/apache/flink/changelog/fs/SchedulerFactory.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/main/java/org/apache/flink/changelog/fs/StateChangeFormat.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/main/java/org/apache/flink/changelog/fs/StateChangeFsUploader.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/main/java/org/apache/flink/changelog/fs/StateChangeSet.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/main/java/org/apache/flink/changelog/fs/StateChangeUploader.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/main/java/org/apache/flink/changelog/fs/UploadResult.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/main/resources/META-INF/services/org.apache.flink.runtime.state.changelog.StateChangelogStorageFactory
create mode 100644
flink-dstl/flink-dstl-dfs/src/test/java/org/apache/flink/changelog/fs/BatchingStateChangeUploaderTest.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/test/java/org/apache/flink/changelog/fs/FsStateChangelogStorageTest.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/test/java/org/apache/flink/changelog/fs/FsStateChangelogWriterSqnTest.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/test/java/org/apache/flink/changelog/fs/FsStateChangelogWriterTest.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/test/java/org/apache/flink/changelog/fs/RetryingExecutorTest.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/test/java/org/apache/flink/changelog/fs/TestingStateChangeUploader.java
create mode 100644
flink-dstl/flink-dstl-dfs/src/test/resources/log4j2.properties
create mode 100644 flink-dstl/pom.xml
create mode 100644
flink-runtime/src/main/java/org/apache/flink/runtime/state/changelog/SequenceNumberRange.java
create mode 100644
flink-runtime/src/test/java/org/apache/flink/runtime/state/TestTaskStateManagerBuilder.java
