This is an automated email from the ASF dual-hosted git repository.
pmaheshwari pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/samza.git.
from bad9ee8 [SAMZA-2666] Updating tests to use TestRunner framework
(#1510)
add aa58950 SAMZA-2591: Async Commit [1/3]: Checkpoint v2 migration
(#1489)
add c117a68 SAMZA-2591: Async Commit [2/3]: Task Commit api changes and
async commit (#1490)
add c85aade SAMZA-2591: Async Commit [3/3]: Container restore lifecycle
(#1491)
add 7cc4eaa SAMZA-2657: Blob Store as backend for Samza State backup and
restore (#1501)
add c2b77f0 Fix StorageConfig bug from newly introduced BlobStoreRestore
factory config (#1503)
add 354620f SnapshotIndexSerde serialization fixes
add 48a8785 Checkpoint-tool changes for checkpoint v2
add 2063155 HOT FIX: KafkaConsumerProxy without any registered
TopicPartitions
add 271ef64 Fix async cleanup TaskMetric
add 7d61a7d Added lastCommitNs and commitTimedOut metrics, updated
default timeout to 30 mins
add 7f5ff3a Updated log line in BlobStoreUtil
add 35c0dbc Checkpoint version config precedence list
add 7676da9 Per store restore manager config precendence
add 5379f97 Job level state backend configs
add 0697457 Fixed case where there are no stores to backup for CSM
add e74c241 Added tests for CSM for non configured stores
add ad408d5 Cleanup of TODOs
add 785d506 checkstype cleanup
add 3c8a462 checkstyle for samza-api
add 65b5df6 Test fix for failing job: only backup persistent stores
add 48699c4 Merge pull request #1512 from
dxichen/checkpoint-restore-precendence-cleanup
add e975b86 Merge branch 'master' into state-backend-async-commit
add c95d828 Merge pull request #1513 from
dxichen/merge-state-backend-async-commit
new f404029 Merge pull request #1514 from
apache/state-backend-async-commit
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
build.gradle | 1 +
gradle/dependency-versions.gradle | 2 +-
.../org/apache/samza/checkpoint/Checkpoint.java | 57 +-
.../org/apache/samza/checkpoint/CheckpointId.java | 51 +-
.../{Checkpoint.java => CheckpointV1.java} | 37 +-
.../org/apache/samza/checkpoint/CheckpointV2.java | 123 +++
.../java/org/apache/samza/job/model/JobModel.java | 4 +
.../samza/storage/BlobStoreAdminFactory.java | 18 +-
.../samza/storage/KafkaChangelogRestoreParams.java | 77 ++
.../StateBackendAdmin.java} | 18 +-
.../apache/samza/storage/StateBackendFactory.java | 63 ++
.../org/apache/samza/storage/StorageEngine.java | 8 +
.../org/apache/samza/storage/StoreProperties.java | 23 +-
.../apache/samza/storage/TaskBackupManager.java | 92 ++
.../apache/samza/storage/TaskRestoreManager.java | 14 +-
.../samza/storage/blobstore/BlobStoreManager.java | 83 ++
.../storage/blobstore/BlobStoreManagerFactory.java | 13 +-
.../apache/samza/storage/blobstore/Metadata.java | 114 +++
.../blobstore/exceptions/DeletedException.java | 23 +-
.../blobstore/exceptions/RetriableException.java | 23 +-
.../org/apache/samza/storage/kv/KeyValueStore.java | 13 +
.../apache/samza/system/ChangelogSSPIterator.java | 2 +-
.../org/apache/samza/system/SystemFactory.java | 2 +-
.../apache/samza/checkpoint/TestCheckpointId.java | 53 ++
.../checkpoint/azure/AzureCheckpointManager.java | 10 +-
.../azure/ITestAzureCheckpointManager.java | 15 +-
.../checkpoint/kafka/KafkaChangelogSSPOffset.java | 40 +-
.../kafka/KafkaStateCheckpointMarker.java | 145 ++++
.../clustermanager/ClusterBasedJobCoordinator.java | 15 +
...ointManagerConfig.java => BlobStoreConfig.java} | 24 +-
.../java/org/apache/samza/config/JobConfig.java | 28 +
.../org/apache/samza/config/StorageConfig.java | 124 ++-
.../java/org/apache/samza/config/TaskConfig.java | 45 +
.../samza/coordinator/MetadataResourceUtil.java | 2 +-
.../samza/serializers/CheckpointV2Serde.java | 57 ++
.../apache/samza/serializers/JsonCheckpoint.java | 57 ++
...kModelMixIn.java => JsonCheckpointV2Mixin.java} | 31 +-
...n.java => KafkaStateCheckpointMarkerMixin.java} | 28 +-
.../samza/serializers/model/SamzaObjectMapper.java | 23 +
.../standalone/PassthroughJobCoordinator.java | 14 +-
.../apache/samza/startpoint/StartpointManager.java | 2 +-
.../storage/KafkaChangelogStateBackendFactory.java | 236 ++++++
.../NonTransactionalStateTaskRestoreManager.java | 81 +-
.../apache/samza/storage/StorageManagerUtil.java | 63 +-
.../org/apache/samza/storage/StorageRecovery.java | 16 +-
.../samza/storage/TaskRestoreManagerFactory.java | 82 --
.../samza/storage/TaskSideInputStorageManager.java | 18 +-
.../samza/storage/TaskStorageCommitManager.java | 366 ++++++++
.../TransactionalStateTaskRestoreManager.java | 130 ++-
.../storage/blobstore/BlobStoreBackupManager.java | 350 ++++++++
.../storage/blobstore/BlobStoreRestoreManager.java | 367 ++++++++
.../blobstore/BlobStoreStateBackendFactory.java | 100 +++
.../samza/storage/blobstore/diff/DirDiff.java | 244 ++++++
.../samza/storage/blobstore/index/DirIndex.java | 176 ++++
.../samza/storage/blobstore/index/FileBlob.java | 85 ++
.../samza/storage/blobstore/index/FileIndex.java | 114 +++
.../storage/blobstore/index/FileMetadata.java | 138 +++
.../storage/blobstore/index/SnapshotIndex.java | 124 +++
.../storage/blobstore/index/SnapshotMetadata.java | 115 +++
.../blobstore/index/serde/JsonDirIndexMixin.java} | 38 +-
.../blobstore/index/serde/JsonFileBlobMixin.java} | 19 +-
.../blobstore/index/serde/JsonFileIndexMixin.java} | 33 +-
.../index/serde/JsonFileMetadataMixin.java} | 37 +-
.../index/serde/JsonSnapshotIndexMixin.java} | 33 +-
.../index/serde/JsonSnapshotMetadataMixin.java} | 38 +-
.../blobstore/index/serde/SnapshotIndexSerde.java | 74 ++
.../metrics/BlobStoreBackupManagerMetrics.java | 136 +++
.../metrics/BlobStoreRestoreManagerMetrics.java | 94 +++
.../storage/blobstore/util/BlobStoreUtil.java | 622 ++++++++++++++
.../samza/storage/blobstore/util/DirDiffUtil.java | 403 +++++++++
.../java/org/apache/samza/system/SystemAdmins.java | 8 +
.../samza/system/inmemory/InMemoryManager.java | 2 +-
.../samza/table/retry/AsyncRetriableTable.java | 11 +-
.../apache/samza/table/retry/FailsafeAdapter.java | 43 +-
.../java/org/apache/samza/util/FutureUtil.java | 191 +++++
.../apache/samza/checkpoint/CheckpointTool.scala | 29 +-
.../apache/samza/checkpoint/OffsetManager.scala | 30 +-
.../file/FileSystemCheckpointManager.scala | 13 +-
.../apache/samza/container/SamzaContainer.scala | 161 ++--
.../org/apache/samza/container/TaskInstance.scala | 324 +++++--
.../samza/container/TaskInstanceMetrics.scala | 12 +-
.../apache/samza/job/local/ProcessJobFactory.scala | 15 +-
...eckpointSerde.scala => CheckpointV1Serde.scala} | 69 +-
.../samza/storage/ContainerStorageManager.java | 448 ++++++----
...fkaNonTransactionalStateTaskBackupManager.scala | 91 ++
.../KafkaTransactionalStateTaskBackupManager.scala | 97 +++
.../NonTransactionalStateTaskStorageManager.scala | 144 ----
.../apache/samza/storage/TaskStorageManager.scala | 37 -
.../samza/storage/TaskStorageManagerFactory.java | 46 -
.../TransactionalStateTaskStorageManager.scala | 201 -----
.../scala/org/apache/samza/util/FileUtil.scala | 9 +-
.../kafka/TestKafkaChangelogSSPOffset.java | 73 ++
.../kafka/TestKafkaStateCheckpointMarker.java | 114 +++
.../org/apache/samza/config/TestStorageConfig.java | 259 +++++-
.../org/apache/samza/job/model/TestJobModel.java | 2 +-
.../samza/serializers/TestCheckpointV2Serde.java | 96 +++
.../TestKafkaChangelogStateBackendFactory.java | 80 ++
.../storage/TestTaskStorageCommitManager.java | 916 ++++++++++++++++++++
.../TestTransactionalStateTaskRestoreManager.java | 252 +++---
.../blobstore/TestBlobStoreBackupManager.java | 542 ++++++++++++
.../blobstore/TestBlobStoreRestoreManager.java | 362 ++++++++
.../blobstore/serde/TestSnapshotIndexSerde.java | 56 ++
.../storage/blobstore/util/BlobStoreTestUtil.java | 296 +++++++
.../storage/blobstore/util/TestBlobStoreUtil.java | 936 +++++++++++++++++++++
.../storage/blobstore/util/TestDirDiffUtil.java | 127 +++
.../blobstore/util/TestDirDiffUtilMisc.java | 71 ++
.../java/org/apache/samza/util/TestFutureUtil.java | 222 +++++
.../samza/checkpoint/TestCheckpointTool.scala | 66 +-
.../samza/checkpoint/TestOffsetManager.scala | 30 +-
.../file/TestFileSystemCheckpointManager.scala | 10 +-
.../samza/container/TestSamzaContainer.scala | 34 -
.../apache/samza/container/TestTaskInstance.scala | 742 ++++++++++++++--
...ointSerde.scala => TestCheckpointV1Serde.scala} | 26 +-
.../samza/storage/TestContainerStorageManager.java | 266 +++++-
.../TestTransactionalStateTaskStorageManager.java | 534 ------------
.../scala/org/apache/samza/util/TestFileUtil.scala | 49 +-
.../checkpoint/kafka/KafkaCheckpointLogKey.java | 9 +-
.../kafka/KafkaCheckpointLogKeySerde.java | 7 +-
.../samza/system/kafka/KafkaConsumerProxy.java | 5 +-
.../checkpoint/kafka/KafkaCheckpointManager.scala | 123 ++-
.../kafka/TestKafkaCheckpointLogKeySerde.java | 14 +-
.../kafka/TestKafkaCheckpointManagerJava.java | 29 +-
.../samza/storage/TestTaskStorageManager.scala | 215 ++---
.../TestTransactionalStateTaskBackupManager.java | 276 ++++++
.../kafka/TestKafkaCheckpointManager.scala | 230 ++++-
.../InMemoryKeyValueStorageEngineFactory.java | 2 -
.../samza/storage/kv/RocksDbOptionsHelper.java | 124 ++-
.../kv/RocksDbKeyValueStorageEngineFactory.scala | 10 +-
.../samza/storage/kv/RocksDbKeyValueStore.scala | 19 +-
.../storage/kv/TestRocksDbKeyValueStore.scala | 20 +-
.../kv/BaseKeyValueStorageEngineFactory.java | 9 +-
.../kv/MockKeyValueStorageEngineFactory.java | 4 +-
.../kv/TestBaseKeyValueStorageEngineFactory.java | 37 +-
.../CheckpointVersionIntegrationTest.java | 149 ++++
.../samza/storage/MyStatefulApplication.java | 157 ++++
.../kv/TransactionalStateIntegrationTest.java | 135 +--
...ransactionalStateMultiStoreIntegrationTest.java | 120 +--
.../processor/TestZkLocalApplicationRunner.java | 4 +-
.../NonTransactionalStateIntegrationTest.scala | 5 +-
.../test/integration/StreamTaskTestUtil.scala | 8 +-
140 files changed, 13153 insertions(+), 2606 deletions(-)
copy samza-api/src/main/java/org/apache/samza/checkpoint/{Checkpoint.java =>
CheckpointV1.java} (69%)
create mode 100644
samza-api/src/main/java/org/apache/samza/checkpoint/CheckpointV2.java
copy
samza-core/src/main/java/org/apache/samza/container/disk/DiskQuotaPolicyFactory.java
=> samza-api/src/main/java/org/apache/samza/storage/BlobStoreAdminFactory.java
(66%)
create mode 100644
samza-api/src/main/java/org/apache/samza/storage/KafkaChangelogRestoreParams.java
copy samza-api/src/main/java/org/apache/samza/{config/ConfigFactory.java =>
storage/StateBackendAdmin.java} (70%)
create mode 100644
samza-api/src/main/java/org/apache/samza/storage/StateBackendFactory.java
create mode 100644
samza-api/src/main/java/org/apache/samza/storage/TaskBackupManager.java
rename {samza-core =>
samza-api}/src/main/java/org/apache/samza/storage/TaskRestoreManager.java (86%)
create mode 100644
samza-api/src/main/java/org/apache/samza/storage/blobstore/BlobStoreManager.java
copy
samza-core/src/main/java/org/apache/samza/clustermanager/FaultDomainManagerFactory.java
=>
samza-api/src/main/java/org/apache/samza/storage/blobstore/BlobStoreManagerFactory.java
(74%)
create mode 100644
samza-api/src/main/java/org/apache/samza/storage/blobstore/Metadata.java
copy
samza-sql-shell/src/main/java/org/apache/samza/sql/client/exceptions/CliException.java
=>
samza-api/src/main/java/org/apache/samza/storage/blobstore/exceptions/DeletedException.java
(59%)
mode change 100755 => 100644
copy
samza-sql-shell/src/main/java/org/apache/samza/sql/client/exceptions/CliException.java
=>
samza-api/src/main/java/org/apache/samza/storage/blobstore/exceptions/RetriableException.java
(71%)
mode change 100755 => 100644
create mode 100644
samza-api/src/test/java/org/apache/samza/checkpoint/TestCheckpointId.java
rename
samza-api/src/main/java/org/apache/samza/checkpoint/CheckpointedChangelogOffset.java
=>
samza-core/src/main/java/org/apache/samza/checkpoint/kafka/KafkaChangelogSSPOffset.java
(57%)
create mode 100644
samza-core/src/main/java/org/apache/samza/checkpoint/kafka/KafkaStateCheckpointMarker.java
copy
samza-core/src/main/java/org/apache/samza/config/{FileSystemCheckpointManagerConfig.java
=> BlobStoreConfig.java} (63%)
create mode 100644
samza-core/src/main/java/org/apache/samza/serializers/CheckpointV2Serde.java
create mode 100644
samza-core/src/main/java/org/apache/samza/serializers/JsonCheckpoint.java
copy
samza-core/src/main/java/org/apache/samza/serializers/model/{JsonTaskModelMixIn.java
=> JsonCheckpointV2Mixin.java} (61%)
copy
samza-core/src/main/java/org/apache/samza/serializers/model/{JsonProcessorLocalityMixIn.java
=> KafkaStateCheckpointMarkerMixin.java} (61%)
create mode 100644
samza-core/src/main/java/org/apache/samza/storage/KafkaChangelogStateBackendFactory.java
delete mode 100644
samza-core/src/main/java/org/apache/samza/storage/TaskRestoreManagerFactory.java
create mode 100644
samza-core/src/main/java/org/apache/samza/storage/TaskStorageCommitManager.java
create mode 100644
samza-core/src/main/java/org/apache/samza/storage/blobstore/BlobStoreBackupManager.java
create mode 100644
samza-core/src/main/java/org/apache/samza/storage/blobstore/BlobStoreRestoreManager.java
create mode 100644
samza-core/src/main/java/org/apache/samza/storage/blobstore/BlobStoreStateBackendFactory.java
create mode 100644
samza-core/src/main/java/org/apache/samza/storage/blobstore/diff/DirDiff.java
create mode 100644
samza-core/src/main/java/org/apache/samza/storage/blobstore/index/DirIndex.java
create mode 100644
samza-core/src/main/java/org/apache/samza/storage/blobstore/index/FileBlob.java
create mode 100644
samza-core/src/main/java/org/apache/samza/storage/blobstore/index/FileIndex.java
create mode 100644
samza-core/src/main/java/org/apache/samza/storage/blobstore/index/FileMetadata.java
create mode 100644
samza-core/src/main/java/org/apache/samza/storage/blobstore/index/SnapshotIndex.java
create mode 100644
samza-core/src/main/java/org/apache/samza/storage/blobstore/index/SnapshotMetadata.java
copy
samza-core/src/main/java/org/apache/samza/{serializers/model/JsonProcessorLocalityMixIn.java
=> storage/blobstore/index/serde/JsonDirIndexMixin.java} (50%)
copy
samza-core/src/main/java/org/apache/samza/{serializers/model/JsonLocalityModelMixIn.java
=> storage/blobstore/index/serde/JsonFileBlobMixin.java} (68%)
copy
samza-core/src/main/java/org/apache/samza/{serializers/model/JsonProcessorLocalityMixIn.java
=> storage/blobstore/index/serde/JsonFileIndexMixin.java} (57%)
copy
samza-core/src/main/java/org/apache/samza/{serializers/model/JsonProcessorLocalityMixIn.java
=> storage/blobstore/index/serde/JsonFileMetadataMixin.java} (53%)
copy
samza-core/src/main/java/org/apache/samza/{serializers/model/JsonProcessorLocalityMixIn.java
=> storage/blobstore/index/serde/JsonSnapshotIndexMixin.java} (51%)
copy
samza-core/src/main/java/org/apache/samza/{serializers/model/JsonTaskModelMixIn.java
=> storage/blobstore/index/serde/JsonSnapshotMetadataMixin.java} (57%)
create mode 100644
samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/SnapshotIndexSerde.java
create mode 100644
samza-core/src/main/java/org/apache/samza/storage/blobstore/metrics/BlobStoreBackupManagerMetrics.java
create mode 100644
samza-core/src/main/java/org/apache/samza/storage/blobstore/metrics/BlobStoreRestoreManagerMetrics.java
create mode 100644
samza-core/src/main/java/org/apache/samza/storage/blobstore/util/BlobStoreUtil.java
create mode 100644
samza-core/src/main/java/org/apache/samza/storage/blobstore/util/DirDiffUtil.java
create mode 100644
samza-core/src/main/java/org/apache/samza/util/FutureUtil.java
rename
samza-core/src/main/scala/org/apache/samza/serializers/{CheckpointSerde.scala
=> CheckpointV1Serde.scala} (54%)
create mode 100644
samza-core/src/main/scala/org/apache/samza/storage/KafkaNonTransactionalStateTaskBackupManager.scala
create mode 100644
samza-core/src/main/scala/org/apache/samza/storage/KafkaTransactionalStateTaskBackupManager.scala
delete mode 100644
samza-core/src/main/scala/org/apache/samza/storage/NonTransactionalStateTaskStorageManager.scala
delete mode 100644
samza-core/src/main/scala/org/apache/samza/storage/TaskStorageManager.scala
delete mode 100644
samza-core/src/main/scala/org/apache/samza/storage/TaskStorageManagerFactory.java
delete mode 100644
samza-core/src/main/scala/org/apache/samza/storage/TransactionalStateTaskStorageManager.scala
create mode 100644
samza-core/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaChangelogSSPOffset.java
create mode 100644
samza-core/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaStateCheckpointMarker.java
create mode 100644
samza-core/src/test/java/org/apache/samza/serializers/TestCheckpointV2Serde.java
create mode 100644
samza-core/src/test/java/org/apache/samza/storage/TestKafkaChangelogStateBackendFactory.java
create mode 100644
samza-core/src/test/java/org/apache/samza/storage/TestTaskStorageCommitManager.java
create mode 100644
samza-core/src/test/java/org/apache/samza/storage/blobstore/TestBlobStoreBackupManager.java
create mode 100644
samza-core/src/test/java/org/apache/samza/storage/blobstore/TestBlobStoreRestoreManager.java
create mode 100644
samza-core/src/test/java/org/apache/samza/storage/blobstore/serde/TestSnapshotIndexSerde.java
create mode 100644
samza-core/src/test/java/org/apache/samza/storage/blobstore/util/BlobStoreTestUtil.java
create mode 100644
samza-core/src/test/java/org/apache/samza/storage/blobstore/util/TestBlobStoreUtil.java
create mode 100644
samza-core/src/test/java/org/apache/samza/storage/blobstore/util/TestDirDiffUtil.java
create mode 100644
samza-core/src/test/java/org/apache/samza/storage/blobstore/util/TestDirDiffUtilMisc.java
create mode 100644
samza-core/src/test/java/org/apache/samza/util/TestFutureUtil.java
rename
samza-core/src/test/scala/org/apache/samza/serializers/{TestCheckpointSerde.scala
=> TestCheckpointV1Serde.scala} (69%)
delete mode 100644
samza-core/src/test/scala/org/apache/samza/storage/TestTransactionalStateTaskStorageManager.java
rename {samza-core/src/test/scala =>
samza-kafka/src/test/java}/org/apache/samza/storage/TestTaskStorageManager.scala
(83%)
create mode 100644
samza-kafka/src/test/java/org/apache/samza/storage/TestTransactionalStateTaskBackupManager.java
create mode 100644
samza-test/src/test/java/org/apache/samza/checkpoint/CheckpointVersionIntegrationTest.java
create mode 100644
samza-test/src/test/java/org/apache/samza/storage/MyStatefulApplication.java
