This is an automated email from the ASF dual-hosted git repository.
heiming pushed a commit to branch tiered_storage
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/tiered_storage by this push:
new 8af7d625318 add cleanup method in the migration task
8af7d625318 is described below
commit 8af7d6253186f7c4bb6eb18939e3623699a4292c
Author: HeimingZ <[email protected]>
AuthorDate: Thu May 25 16:57:29 2023 +0800
add cleanup method in the migration task
---
.../db/engine/migration/LocalMigrationTask.java | 35 ++++++++++++++--------
.../iotdb/db/engine/migration/MigrationTask.java | 21 ++++++++++++-
.../db/engine/migration/RemoteMigrationTask.java | 34 +++++++++++----------
3 files changed, 61 insertions(+), 29 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/migration/LocalMigrationTask.java
b/server/src/main/java/org/apache/iotdb/db/engine/migration/LocalMigrationTask.java
index 99b0a78c8d7..5401ff61a42 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/migration/LocalMigrationTask.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/migration/LocalMigrationTask.java
@@ -25,6 +25,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.Arrays;
public class LocalMigrationTask extends MigrationTask {
private static final Logger logger =
LoggerFactory.getLogger(LocalMigrationTask.class);
@@ -36,43 +37,51 @@ public class LocalMigrationTask extends MigrationTask {
@Override
public void migrate() {
+ // dest tsfile may exist if the last same migration task hasn't completed
when the system
+ // shutdown.
+ filesShouldDelete.addAll(Arrays.asList(destTsFile, destResourceFile,
destModsFile));
+ cleanup();
+
// copy TsFile and resource file
tsFileResource.readLock();
try {
destTsFile.getParentFile().mkdirs();
- fsFactory.copyFile(srcFile, destTsFile);
- fsFactory.copyFile(srcResourceFile, destResourceFile);
+ migrateFile(srcFile, destTsFile);
+ migrateFile(srcResourceFile, destResourceFile);
} catch (Exception e) {
- logger.error("Fail to copy TsFile from local {} to local {}", srcFile,
srcResourceFile);
- destTsFile.delete();
- destResourceFile.delete();
+ if (!tsFileResource.isDeleted()) {
+ logger.error("Fail to copy TsFile from local {} to local {}", srcFile,
srcResourceFile);
+ }
+ cleanup();
return;
} finally {
tsFileResource.readUnlock();
}
+
// close mods file and replace TsFile path
tsFileResource.writeLock();
try {
tsFileResource.resetModFile();
// migrate MOD file only when it exists
if (srcModsFile.exists()) {
- fsFactory.copyFile(srcModsFile, destModsFile);
+ migrateFile(srcModsFile, destModsFile);
}
tsFileResource.setFile(destTsFile);
tsFileResource.increaseTierLevel();
tsFileResource.setStatus(TsFileResourceStatus.NORMAL);
} catch (Exception e) {
- logger.error("Fail to copy mods file from local {} to local {}",
srcModsFile, destModsFile);
- destTsFile.delete();
- destResourceFile.delete();
- destModsFile.delete();
+ if (!tsFileResource.isDeleted()) {
+ logger.error("Fail to copy mods file from local {} to local {}",
srcModsFile, destModsFile);
+ }
+ cleanup();
return;
} finally {
tsFileResource.writeUnlock();
}
+
// clear src files
- srcFile.delete();
- srcResourceFile.delete();
- srcModsFile.delete();
+ filesShouldDelete.clear();
+ filesShouldDelete.addAll(Arrays.asList(srcFile, srcResourceFile,
srcModsFile));
+ cleanup();
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/migration/MigrationTask.java
b/server/src/main/java/org/apache/iotdb/db/engine/migration/MigrationTask.java
index 711bd4048c8..f6dfef0338d 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/migration/MigrationTask.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/migration/MigrationTask.java
@@ -27,6 +27,8 @@ import org.apache.iotdb.tsfile.utils.FSUtils;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
public abstract class MigrationTask implements Runnable {
protected static final FSFactory fsFactory =
FSFactoryProducer.getFSFactory();
@@ -34,7 +36,6 @@ public abstract class MigrationTask implements Runnable {
protected final MigrationCause cause;
protected final TsFileResource tsFileResource;
protected final String targetDir;
-
protected final File srcFile;
protected final File destTsFile;
protected final File srcResourceFile;
@@ -42,6 +43,8 @@ public abstract class MigrationTask implements Runnable {
protected final File srcModsFile;
protected final File destModsFile;
+ protected final List<File> filesShouldDelete = new ArrayList<>();
+
protected MigrationTask(MigrationCause cause, TsFileResource tsFileResource,
String targetDir)
throws IOException {
this.cause = cause;
@@ -86,4 +89,20 @@ public abstract class MigrationTask implements Runnable {
}
public abstract void migrate();
+
+ protected void migrateFile(File src, File dest) throws IOException {
+ fsFactory.copyFile(src, dest);
+ filesShouldDelete.add(dest);
+ }
+
+ protected void cleanup() {
+ filesShouldDelete.forEach(this::deleteIfExist);
+ filesShouldDelete.clear();
+ }
+
+ protected void deleteIfExist(File file) {
+ if (file.exists()) {
+ file.delete();
+ }
+ }
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/migration/RemoteMigrationTask.java
b/server/src/main/java/org/apache/iotdb/db/engine/migration/RemoteMigrationTask.java
index ee209234019..f567d04636a 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/migration/RemoteMigrationTask.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/migration/RemoteMigrationTask.java
@@ -25,6 +25,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.Arrays;
public class RemoteMigrationTask extends MigrationTask {
private static final Logger logger =
LoggerFactory.getLogger(RemoteMigrationTask.class);
@@ -36,35 +37,38 @@ public class RemoteMigrationTask extends MigrationTask {
@Override
public void migrate() {
- // tsfile may exist on the remote if the last same migration task hasn't
completed when the
- // system shutdown.
- if (destTsFile.exists()) {
- destTsFile.delete();
- }
- if (destResourceFile.exists()) {
- destResourceFile.delete();
- }
+ // dest tsfile may exist if the last same migration task hasn't completed
when the system
+ // shutdown.
+ filesShouldDelete.addAll(Arrays.asList(destTsFile, destResourceFile));
+ cleanup();
+
// copy TsFile and resource file
tsFileResource.readLock();
try {
- fsFactory.copyFile(srcFile, destTsFile);
- fsFactory.copyFile(srcResourceFile, destResourceFile);
+ migrateFile(srcFile, destTsFile);
+ migrateFile(srcResourceFile, destResourceFile);
} catch (Exception e) {
- logger.error("Fail to copy TsFile from local {} to remote {}", srcFile,
srcResourceFile);
- destTsFile.delete();
- destResourceFile.delete();
+ if (!tsFileResource.isDeleted()) {
+ logger.error("Fail to copy TsFile from local {} to remote {}",
srcFile, srcResourceFile);
+ }
+ cleanup();
return;
} finally {
tsFileResource.readUnlock();
}
+
// clear src files
tsFileResource.writeLock();
try {
- srcFile.delete();
+ filesShouldDelete.clear();
+ filesShouldDelete.add(srcFile);
+ cleanup();
tsFileResource.increaseTierLevel();
tsFileResource.setStatus(TsFileResourceStatus.NORMAL_ON_REMOTE);
} catch (Exception e) {
- logger.error("Fail to delete local TsFile {}", srcFile);
+ if (!tsFileResource.isDeleted()) {
+ logger.error("Fail to delete local TsFile {}", srcFile);
+ }
} finally {
tsFileResource.writeUnlock();
}
