EricJoy2048 commented on code in PR #2485:
URL:
https://github.com/apache/incubator-seatunnel/pull/2485#discussion_r953249998
##########
seatunnel-engine/seatunnel-engine-storage/checkpoint-storage-api/src/main/java/org/apache/seatunnel/engine/checkpoint/storage/api/AbstractCheckpointStorage.java:
##########
@@ -79,4 +96,73 @@ public byte[] serializeCheckPointData(PipelineState state)
throws IOException {
public PipelineState deserializeCheckPointData(byte[] data) throws
IOException {
return serializer.deserialize(data, PipelineState.class);
}
+
+ public void setStorageNameSpace(String storageNameSpace) {
+ if (storageNameSpace != null) {
+ this.storageNameSpace = storageNameSpace;
+ }
+ }
+
+ public Set<String> getLatestPipelineNames(List<String> fileNames) {
+ Map<String, String> latestPipelineMap = new HashMap<>();
+ fileNames.forEach(fileName -> {
+ String filePipelineId =
fileName.split(FILE_NAME_SPLIT)[FILE_NAME_PIPELINE_ID_INDEX];
+ int fileVersion =
Integer.parseInt(fileName.split(FILE_NAME_SPLIT)[FILE_SORT_ID_INDEX]);
+ if (latestPipelineMap.containsKey(filePipelineId)) {
+ int oldVersion =
Integer.parseInt(latestPipelineMap.get(filePipelineId).split(FILE_NAME_SPLIT)[FILE_SORT_ID_INDEX]);
+ if (fileVersion > oldVersion) {
+ latestPipelineMap.put(filePipelineId, fileName);
+ }
+ } else {
+ latestPipelineMap.put(filePipelineId, fileName);
+ }
+ });
+ Set<String> latestPipelines = new HashSet<>(latestPipelineMap.size());
+ latestPipelineMap.forEach((pipelineId, fileName) ->
latestPipelines.add(fileName));
+ return latestPipelines;
+ }
+
+ /**
+ * get latest checkpoint file name
+ *
+ * @param fileNames file names
+ * @return latest checkpoint file name
+ */
+ public String getLatestCheckpointFileNameByJobIdAndPipelineId(List<String>
fileNames, String pipelineId) {
+ AtomicReference<String> latestFileName = new AtomicReference<>();
+ AtomicInteger latestVersion = new AtomicInteger();
+ fileNames.forEach(fileName -> {
+ int fileVersion =
Integer.parseInt(fileName.split(FILE_NAME_SPLIT)[FILE_SORT_ID_INDEX]);
+ String filePipelineId =
fileName.split(FILE_NAME_SPLIT)[FILE_NAME_PIPELINE_ID_INDEX];
+ if (pipelineId.equals(filePipelineId) && fileVersion >
latestVersion.get()) {
+ latestVersion.set(fileVersion);
+ latestFileName.set(fileName);
+ }
+ });
+ return latestFileName.get();
+ }
+
+ /**
+ * get the latest checkpoint file name
+ *
+ * @param fileName file names. note: file name cannot contain parent path
+ * @return latest checkpoint file name
+ */
+ public String getPipelineIdByFileName(String fileName) {
+ return fileName.split(FILE_NAME_SPLIT)[FILE_NAME_PIPELINE_ID_INDEX];
+ }
+
+ @Override
+ public void asyncStoreCheckPoint(PipelineState state) {
+ if (null == this.executor || this.executor.isShutdown()) {
+ this.executor =
Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2 +
1, new StorageThreadFactory());
Review Comment:
> We currently do not have a unified Config acquisition. We use
`System.getProperties` get,WDYT?
Actually, we already have config , look at `EngineConfig` class
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
