zoltar9264 commented on code in PR #22669:
URL: https://github.com/apache/flink/pull/22669#discussion_r1246040024
##########
flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksIncrementalSnapshotStrategy.java:
##########
@@ -334,53 +325,61 @@ public SnapshotResult<KeyedStateHandle>
get(CloseableRegistry snapshotCloseableR
}
}
- private void uploadSstFiles(
- @Nonnull Map<StateHandleID, StreamStateHandle> sstFiles,
- @Nonnull Map<StateHandleID, StreamStateHandle> miscFiles,
+ /** upload files and return total uploaded size. */
+ private long uploadSnapshotFiles(
+ @Nonnull List<HandleAndLocalPath> sstFiles,
+ @Nonnull List<HandleAndLocalPath> miscFiles,
@Nonnull CloseableRegistry snapshotCloseableRegistry,
@Nonnull CloseableRegistry tmpResourcesRegistry)
throws Exception {
// write state data
Preconditions.checkState(localBackupDirectory.exists());
- Map<StateHandleID, Path> sstFilePaths = new HashMap<>();
- Map<StateHandleID, Path> miscFilePaths = new HashMap<>();
-
Path[] files = localBackupDirectory.listDirectory();
+ long uploadedSize = 0;
if (files != null) {
+ List<Path> sstFilePaths = new ArrayList<>(files.length);
+ List<Path> miscFilePaths = new ArrayList<>(files.length);
+
createUploadFilePaths(files, sstFiles, sstFilePaths,
miscFilePaths);
final CheckpointedStateScope stateScope =
sharingFilesStrategy ==
SnapshotType.SharingFilesStrategy.NO_SHARING
? CheckpointedStateScope.EXCLUSIVE
: CheckpointedStateScope.SHARED;
- sstFiles.putAll(
+
+ List<HandleAndLocalPath> uploadedSstFiles =
stateUploader.uploadFilesToCheckpointFs(
sstFilePaths,
checkpointStreamFactory,
stateScope,
snapshotCloseableRegistry,
- tmpResourcesRegistry));
- miscFiles.putAll(
+ tmpResourcesRegistry);
+ uploadedSize +=
+ uploadedSstFiles.stream()
+ .mapToLong(e -> e.getHandle().getStateSize())
+ .sum();
+ sstFiles.addAll(uploadedSstFiles);
+
+ List<HandleAndLocalPath> uploadedMiscFiles =
stateUploader.uploadFilesToCheckpointFs(
miscFilePaths,
checkpointStreamFactory,
stateScope,
snapshotCloseableRegistry,
- tmpResourcesRegistry));
-
- synchronized (uploadedStateIDs) {
+ tmpResourcesRegistry);
+ uploadedSize +=
+ uploadedMiscFiles.stream()
+ .mapToLong(e -> e.getHandle().getStateSize())
+ .sum();
+ miscFiles.addAll(uploadedMiscFiles);
+
+ synchronized (uploadedSstFilesMap) {
switch (sharingFilesStrategy) {
case FORWARD_BACKWARD:
case FORWARD:
- uploadedStateIDs.put(
- checkpointId,
- sstFiles.entrySet().stream()
- .collect(
- Collectors.toMap(
- Map.Entry::getKey,
- t ->
t.getValue().getStateSize())));
+ uploadedSstFilesMap.put(checkpointId, sstFiles);
Review Comment:
I had the same concern before, so I checked the context and found no code
that would modify `sstFiles`, so I didn't add this protection. But now since
you have the same concern, and more importantly to prevent a future change from
causing the problem, I think it makes sense to add protection. Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
