This is an automated email from the ASF dual-hosted git repository. pmaheshwari pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/samza.git
commit f40402948f37d84c000f1b4340180e538ac75c82 Merge: bad9ee8 c95d828 Author: Prateek Maheshwari <[email protected]> AuthorDate: Tue Aug 10 10:57:32 2021 -0700 Merge pull request #1514 from apache/state-backend-async-commit All changes for SEP-28: https://cwiki.apache.org/confluence/display/SAMZA/SEP-28%3A+Samza+State+Backend+Interface+and+Checkpointing+Improvements SEP-29: https://cwiki.apache.org/confluence/display/SAMZA/SEP-29%3A+Blob+Store+Based+State+Backup+And+Restore build.gradle | 1 + gradle/dependency-versions.gradle | 2 +- .../org/apache/samza/checkpoint/Checkpoint.java | 57 +- .../org/apache/samza/checkpoint/CheckpointId.java | 51 +- .../{Checkpoint.java => CheckpointV1.java} | 37 +- .../org/apache/samza/checkpoint/CheckpointV2.java | 123 +++ .../java/org/apache/samza/job/model/JobModel.java | 4 + .../samza/storage/BlobStoreAdminFactory.java | 30 +- .../samza/storage/KafkaChangelogRestoreParams.java | 77 ++ .../apache/samza/storage/StateBackendAdmin.java | 32 +- .../apache/samza/storage/StateBackendFactory.java | 63 ++ .../org/apache/samza/storage/StorageEngine.java | 8 + .../org/apache/samza/storage/StoreProperties.java | 23 +- .../apache/samza/storage/TaskBackupManager.java | 92 ++ .../apache/samza/storage/TaskRestoreManager.java | 14 +- .../samza/storage/blobstore/BlobStoreManager.java | 83 ++ .../storage/blobstore/BlobStoreManagerFactory.java | 24 +- .../apache/samza/storage/blobstore/Metadata.java | 114 +++ .../blobstore/exceptions/DeletedException.java | 33 +- .../blobstore/exceptions/RetriableException.java | 25 +- .../org/apache/samza/storage/kv/KeyValueStore.java | 13 + .../apache/samza/system/ChangelogSSPIterator.java | 2 +- .../org/apache/samza/system/SystemFactory.java | 2 +- .../apache/samza/checkpoint/TestCheckpointId.java | 53 ++ .../checkpoint/azure/AzureCheckpointManager.java | 10 +- .../azure/ITestAzureCheckpointManager.java | 15 +- .../checkpoint/kafka/KafkaChangelogSSPOffset.java | 40 +- .../kafka/KafkaStateCheckpointMarker.java | 145 ++++ .../clustermanager/ClusterBasedJobCoordinator.java | 15 + .../org/apache/samza/config/BlobStoreConfig.java} | 30 +- .../java/org/apache/samza/config/JobConfig.java | 28 + .../org/apache/samza/config/StorageConfig.java | 124 ++- .../java/org/apache/samza/config/TaskConfig.java | 45 + .../samza/coordinator/MetadataResourceUtil.java | 2 +- .../samza/serializers/CheckpointV2Serde.java | 57 ++ .../apache/samza/serializers/JsonCheckpoint.java | 57 ++ .../serializers/model/JsonCheckpointV2Mixin.java | 46 + .../model/KafkaStateCheckpointMarkerMixin.java | 48 ++ .../samza/serializers/model/SamzaObjectMapper.java | 23 + .../standalone/PassthroughJobCoordinator.java | 14 +- .../apache/samza/startpoint/StartpointManager.java | 2 +- .../storage/KafkaChangelogStateBackendFactory.java | 236 ++++++ .../NonTransactionalStateTaskRestoreManager.java | 81 +- .../apache/samza/storage/StorageManagerUtil.java | 63 +- .../org/apache/samza/storage/StorageRecovery.java | 16 +- .../samza/storage/TaskRestoreManagerFactory.java | 82 -- .../samza/storage/TaskSideInputStorageManager.java | 18 +- .../samza/storage/TaskStorageCommitManager.java | 366 ++++++++ .../TransactionalStateTaskRestoreManager.java | 130 ++- .../storage/blobstore/BlobStoreBackupManager.java | 350 ++++++++ .../storage/blobstore/BlobStoreRestoreManager.java | 367 ++++++++ .../blobstore/BlobStoreStateBackendFactory.java | 100 +++ .../samza/storage/blobstore/diff/DirDiff.java | 244 ++++++ .../samza/storage/blobstore/index/DirIndex.java | 176 ++++ .../samza/storage/blobstore/index/FileBlob.java | 85 ++ .../samza/storage/blobstore/index/FileIndex.java | 114 +++ .../storage/blobstore/index/FileMetadata.java | 138 +++ .../storage/blobstore/index/SnapshotIndex.java | 124 +++ .../storage/blobstore/index/SnapshotMetadata.java | 115 +++ .../blobstore/index/serde/JsonDirIndexMixin.java | 58 ++ .../blobstore/index/serde/JsonFileBlobMixin.java} | 34 +- .../blobstore/index/serde/JsonFileIndexMixin.java | 53 ++ .../index/serde/JsonFileMetadataMixin.java | 55 ++ .../index/serde/JsonSnapshotIndexMixin.java | 53 ++ .../index/serde/JsonSnapshotMetadataMixin.java | 53 ++ .../blobstore/index/serde/SnapshotIndexSerde.java | 74 ++ .../metrics/BlobStoreBackupManagerMetrics.java | 136 +++ .../metrics/BlobStoreRestoreManagerMetrics.java | 94 +++ .../storage/blobstore/util/BlobStoreUtil.java | 622 ++++++++++++++ .../samza/storage/blobstore/util/DirDiffUtil.java | 403 +++++++++ .../java/org/apache/samza/system/SystemAdmins.java | 8 + .../samza/system/inmemory/InMemoryManager.java | 2 +- .../samza/table/retry/AsyncRetriableTable.java | 11 +- .../apache/samza/table/retry/FailsafeAdapter.java | 43 +- .../java/org/apache/samza/util/FutureUtil.java | 191 +++++ .../apache/samza/checkpoint/CheckpointTool.scala | 29 +- .../apache/samza/checkpoint/OffsetManager.scala | 30 +- .../file/FileSystemCheckpointManager.scala | 13 +- .../apache/samza/container/SamzaContainer.scala | 161 ++-- .../org/apache/samza/container/TaskInstance.scala | 324 +++++-- .../samza/container/TaskInstanceMetrics.scala | 12 +- .../apache/samza/job/local/ProcessJobFactory.scala | 15 +- ...eckpointSerde.scala => CheckpointV1Serde.scala} | 69 +- .../samza/storage/ContainerStorageManager.java | 448 ++++++---- ...fkaNonTransactionalStateTaskBackupManager.scala | 91 ++ .../KafkaTransactionalStateTaskBackupManager.scala | 97 +++ .../NonTransactionalStateTaskStorageManager.scala | 144 ---- .../samza/storage/TaskStorageManagerFactory.java | 46 - .../TransactionalStateTaskStorageManager.scala | 201 ----- .../scala/org/apache/samza/util/FileUtil.scala | 9 +- .../kafka/TestKafkaChangelogSSPOffset.java | 73 ++ .../kafka/TestKafkaStateCheckpointMarker.java | 114 +++ .../org/apache/samza/config/TestStorageConfig.java | 259 +++++- .../org/apache/samza/job/model/TestJobModel.java | 2 +- .../samza/serializers/TestCheckpointV2Serde.java | 96 +++ .../TestKafkaChangelogStateBackendFactory.java | 80 ++ .../storage/TestTaskStorageCommitManager.java | 916 ++++++++++++++++++++ .../TestTransactionalStateTaskRestoreManager.java | 252 +++--- .../blobstore/TestBlobStoreBackupManager.java | 542 ++++++++++++ .../blobstore/TestBlobStoreRestoreManager.java | 362 ++++++++ .../blobstore/serde/TestSnapshotIndexSerde.java | 56 ++ .../storage/blobstore/util/BlobStoreTestUtil.java | 296 +++++++ .../storage/blobstore/util/TestBlobStoreUtil.java | 936 +++++++++++++++++++++ .../storage/blobstore/util/TestDirDiffUtil.java | 127 +++ .../blobstore/util/TestDirDiffUtilMisc.java | 71 ++ .../java/org/apache/samza/util/TestFutureUtil.java | 222 +++++ .../samza/checkpoint/TestCheckpointTool.scala | 66 +- .../samza/checkpoint/TestOffsetManager.scala | 30 +- .../file/TestFileSystemCheckpointManager.scala | 10 +- .../samza/container/TestSamzaContainer.scala | 34 - .../apache/samza/container/TestTaskInstance.scala | 742 ++++++++++++++-- ...ointSerde.scala => TestCheckpointV1Serde.scala} | 26 +- .../samza/storage/TestContainerStorageManager.java | 266 +++++- .../TestTransactionalStateTaskStorageManager.java | 534 ------------ .../scala/org/apache/samza/util/TestFileUtil.scala | 49 +- .../checkpoint/kafka/KafkaCheckpointLogKey.java | 9 +- .../kafka/KafkaCheckpointLogKeySerde.java | 7 +- .../samza/system/kafka/KafkaConsumerProxy.java | 5 +- .../checkpoint/kafka/KafkaCheckpointManager.scala | 123 ++- .../kafka/TestKafkaCheckpointLogKeySerde.java | 14 +- .../kafka/TestKafkaCheckpointManagerJava.java | 29 +- .../samza/storage/TestTaskStorageManager.scala | 215 ++--- .../TestTransactionalStateTaskBackupManager.java | 276 ++++++ .../kafka/TestKafkaCheckpointManager.scala | 230 ++++- .../InMemoryKeyValueStorageEngineFactory.java | 2 - .../samza/storage/kv/RocksDbOptionsHelper.java | 124 ++- .../kv/RocksDbKeyValueStorageEngineFactory.scala | 10 +- .../samza/storage/kv/RocksDbKeyValueStore.scala | 19 +- .../storage/kv/TestRocksDbKeyValueStore.scala | 20 +- .../kv/BaseKeyValueStorageEngineFactory.java | 9 +- .../kv/MockKeyValueStorageEngineFactory.java | 4 +- .../kv/TestBaseKeyValueStorageEngineFactory.java | 37 +- .../CheckpointVersionIntegrationTest.java | 149 ++++ .../samza/storage/MyStatefulApplication.java | 157 ++++ .../kv/TransactionalStateIntegrationTest.java | 135 +-- ...ransactionalStateMultiStoreIntegrationTest.java | 120 +-- .../processor/TestZkLocalApplicationRunner.java | 4 +- .../NonTransactionalStateIntegrationTest.scala | 5 +- .../test/integration/StreamTaskTestUtil.scala | 8 +- 139 files changed, 13422 insertions(+), 2498 deletions(-)
