Caideyipi commented on code in PR #15599:
URL: https://github.com/apache/iotdb/pull/15599#discussion_r2133259053
##########
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeTaskMeta.java:
##########
@@ -58,18 +88,105 @@ public class PipeTaskMeta {
private final Set<PipeRuntimeException> exceptionMessages =
Collections.newSetFromMap(new ConcurrentHashMap<>());
- public PipeTaskMeta(/* @NotNull */ final ProgressIndex progressIndex, final
int leaderNodeId) {
+ public PipeTaskMeta(
+ /* @NotNull */ final ProgressIndex progressIndex,
+ final int leaderNodeId,
+ final int taskIndex,
+ final boolean needPersistProgressIndex) {
this.progressIndex.set(progressIndex);
this.leaderNodeId.set(leaderNodeId);
+ this.taskIndex = taskIndex;
+ // PipeTaskMeta created in configNode doesn't need to persist progress
index.
+ this.needPersistProgressIndex = needPersistProgressIndex;
+ this.progressIndexPersistFile =
+ new File(
+ IoTDBConstant.DN_DEFAULT_DATA_DIR
+ + File.separator
+ + IoTDBConstant.SYSTEM_FOLDER_NAME
+ + File.separator
+ + PipeConfig.getInstance().getPipeHardlinkBaseDirName()
+ + File.separator
+ +
PipeConfig.getInstance().getPipeProgressIndexPersistDirName(),
+ PREFIX + taskIndex);
}
public ProgressIndex getProgressIndex() {
return progressIndex.get();
}
public ProgressIndex updateProgressIndex(final ProgressIndex updateIndex) {
- return progressIndex.updateAndGet(
+ // only pipeTaskMeta that need to updateProgressIndex will persist
progress index
+ // isRegisterPersistTask is used to avoid multiple threads registering
persist task concurrently
+ if (needPersistProgressIndex
+ && !isRegisterPersistTask.getAndSet(true)
+ && this.persistProgressIndexFuture == null) {
+ this.persistProgressIndexFuture =
+ PipePeriodicalJobExecutor.submitBackgroundJob(
+ this::persistProgressIndex, 0, TimeUnit.SECONDS.toMillis(20));
Review Comment:
That shall be 20s?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
