qinjunjerry commented on a change in pull request #13309:
URL: https://github.com/apache/flink/pull/13309#discussion_r482453290
##########
File path:
flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/WritableSavepoint.java
##########
@@ -88,40 +106,119 @@ public final void write(String path) {
List<BootstrapTransformationWithID<?>>
newOperatorTransformations = metadata.getNewOperators();
DataSet<OperatorState> newOperatorStates =
writeOperatorStates(newOperatorTransformations, savepointPath);
- List<OperatorState> existingOperators =
metadata.getExistingOperators();
+ List<OperatorState> existingOperatorStateList =
metadata.getExistingOperators();
- DataSet<OperatorState> finalOperatorStates =
unionOperatorStates(newOperatorStates, existingOperators);
+ DataSet<OperatorState> finalOperatorStates;
+ if (existingOperatorStateList.isEmpty()) {
+ finalOperatorStates = newOperatorStates;
+ } else {
+ DataSet<OperatorState> existingOperatorStates =
newOperatorStates.getExecutionEnvironment()
+ .fromCollection(existingOperatorStateList);
+
+ existingOperatorStates
+ .map(new OperatorState2PathSetMapFunction())
+ .flatMap(new
FlatMapFunction<Set<Optional<Path>>, Optional<Path>>() {
+ @Override
+ public void flatMap(Set<Optional<Path>>
value, Collector<Optional<Path>> out) throws Exception {
+ for (Optional<Path> path :
value) {
+ out.collect(path);
+ }
+ }
+ })
+ .filter(Optional<Path>::isPresent)
+ .map(Optional<Path>::get)
+ .returns(TypeInformation.of(new
TypeHint<Path>(){}))
+ .map(new MapFunction<Path, Path>() {
+ @Override
+ public Path map(Path existingStateFile)
throws IOException {
+
savepointPath.getFileSystem().mkdirs(savepointPath);
+ Files.copy(
+
Paths.get(existingStateFile.getPath()), // source file
+ Paths.get(path + "/" +
existingStateFile.getName()) // destination file
+ );
+ return existingStateFile;
+ }
+ })
+ .output(new DiscardingOutputFormat<>());
+
+ finalOperatorStates =
newOperatorStates.union(existingOperatorStates);
+ }
finalOperatorStates
.reduceGroup(new
MergeOperatorStates(metadata.getMasterStates()))
.name("reduce(OperatorState)")
.output(new SavepointOutputFormat(savepointPath))
.name(path);
}
- private DataSet<OperatorState>
unionOperatorStates(DataSet<OperatorState> newOperatorStates,
List<OperatorState> existingOperators) {
- DataSet<OperatorState> finalOperatorStates;
- if (existingOperators.isEmpty()) {
- finalOperatorStates = newOperatorStates;
- } else {
- DataSet<OperatorState> wrappedCollection =
newOperatorStates
- .getExecutionEnvironment()
- .fromCollection(existingOperators);
-
- finalOperatorStates =
newOperatorStates.union(wrappedCollection);
- }
- return finalOperatorStates;
- }
-
private DataSet<OperatorState> writeOperatorStates(
- List<BootstrapTransformationWithID<?>>
newOperatorStates,
- Path savepointWritePath) {
+ List<BootstrapTransformationWithID<?>> newOperatorStates,
+ Path savepointWritePath) {
return newOperatorStates
.stream()
.map(newOperatorState -> newOperatorState
.getBootstrapTransformation()
.writeOperatorState(newOperatorState.getOperatorID(), stateBackend,
metadata.getMaxParallelism(), savepointWritePath))
.reduce(DataSet::union)
- .orElseThrow(() -> new
IllegalStateException("Savepoint's must contain at least one operator"));
+ .orElseThrow(() -> new IllegalStateException("Savepoint
must contain at least one operator"));
+ }
+
+ /**
+ * Map an OperatorState to a set of Paths which represent the state
files.
+ */
+ static class OperatorState2PathSetMapFunction implements
MapFunction<OperatorState, Set<Optional<Path>>> {
Review comment:
Thanks for the hint. Done.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
