This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.12 by this push:
new adb5131 [IOTDB-2138] [To rel/0.12] Fix data loss after IoTDB recover
(#4563)
adb5131 is described below
commit adb513100d24deed2f10291cb3e896582a8d046c
Author: liuxuxin <[email protected]>
AuthorDate: Tue Dec 14 13:56:17 2021 +0800
[IOTDB-2138] [To rel/0.12] Fix data loss after IoTDB recover (#4563)
---
.../level/LevelCompactionTsFileManagement.java | 13 +++++--------
.../engine/storagegroup/StorageGroupProcessor.java | 2 ++
.../db/engine/storagegroup/TsFileResource.java | 22 ++++++++++++++++++----
3 files changed, 25 insertions(+), 12 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
index aabb674..0e1c12a 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
@@ -421,27 +421,24 @@ public class LevelCompactionTsFileManagement extends
TsFileManagement {
CompactionFileInfo targetFileInfo = logAnalyzer.getTargetFileInfo();
String[] dataDirs =
IoTDBDescriptor.getInstance().getConfig().getDataDirs();
File targetFile = null;
- boolean isSeq = logAnalyzer.isSeq();
if (targetFileInfo == null || sourceFileInfo.isEmpty()) {
return;
}
// get tsfile resource from list, as they have been recovered in
StorageGroupProcessor
TsFileResource targetResource = null;
+ File targetResourceFile = null;
for (String dataDir : dataDirs) {
if ((targetFile = targetFileInfo.getFile(dataDir)).exists()) {
targetResource = new TsFileResource(targetFile);
+ targetResourceFile = new File(targetFile.getPath() +
TsFileResource.RESOURCE_SUFFIX);
}
}
if (targetResource != null) {
- RestorableTsFileIOWriter writer = new
RestorableTsFileIOWriter(targetFile);
- if (writer.hasCrashed()) {
- // target tsfile is not compeleted
- writer.close();
- logger.info(
- "[Compaction][Recover] target file {} is not complete, remove
it", targetResource);
+ if (!targetResourceFile.exists()) {
+ // target file resource has not been generated yet
+ // delete target file if exists
targetResource.remove();
} else {
- writer.close();
// complete compaction, delete source files
logger.info(
"[Compaction][Recover] target file {} is compeleted, remove
resource file",
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 7d103e7..8da66de 100755
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -673,6 +673,8 @@ public class StorageGroupProcessor {
try {
// this tsfile is not zero level, no need to perform redo wal
if (TsFileResource.getMergeLevel(tsFileResource.getTsFile().getName())
> 0) {
+ writer =
+ recoverPerformer.recover(false, this::getWalDirectByteBuffer,
this::releaseWalBuffer);
tsFileResource.setClosed(true);
tsFileManagement.add(tsFileResource, isSeq);
continue;
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index ab33b23..ee9cdc3 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -32,7 +32,8 @@ import org.apache.iotdb.db.query.filter.TsFileFilter;
import org.apache.iotdb.db.service.UpgradeSevice;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
-import org.apache.iotdb.tsfile.file.metadata.*;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
@@ -51,9 +52,21 @@ import java.io.OutputStream;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Paths;
-import java.util.*;
-
-import static org.apache.iotdb.db.conf.IoTDBConstant.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Random;
+import java.util.Set;
+
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SEPARATOR;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_INDEX;
+import static
org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_MERGECNT_INDEX;
+import static
org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_SEPARATOR;
+import static
org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_TIME_INDEX;
+import static
org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_UNSEQMERGECNT_INDEX;
+import static
org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_VERSION_INDEX;
import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
@SuppressWarnings("java:S1135") // ignore todos
@@ -491,6 +504,7 @@ public class TsFileResource {
public void removeResourceFile() {
try {
+ fsFactory.deleteIfExists(fsFactory.getFile(file.getPath() +
RESOURCE_SUFFIX + TEMP_SUFFIX));
fsFactory.deleteIfExists(fsFactory.getFile(file.getPath() +
RESOURCE_SUFFIX));
} catch (IOException e) {
logger.error("TsFileResource {} cannot be deleted: {}", file,
e.getMessage());
