This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch tsfile_name
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/tsfile_name by this push:
new 35086c870d Update filename when upgrading from old version (#6603)
35086c870d is described below
commit 35086c870d660b1a3f03630163468989d4e86a17
Author: Alan Choo <[email protected]>
AuthorDate: Wed Jul 6 19:06:59 2022 +0800
Update filename when upgrading from old version (#6603)
---
.../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 7 ++
.../compaction/inner/InnerSpaceCompactionTask.java | 2 +-
.../impl/ReadPointCompactionPerformer.java | 10 ++-
.../iotdb/db/engine/storagegroup/DataRegion.java | 79 ++++++++++++++++------
.../db/engine/storagegroup/TsFileManager.java | 18 -----
5 files changed, 71 insertions(+), 45 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java
index ec23692ef3..76cb550da2 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java
@@ -114,6 +114,8 @@ public class IoTDBStartCheck {
private static final String DATA_REGION_CONSENSUS_PROTOCOL =
"data_region_consensus_protocol";
private static final String IOTDB_VERSION_STRING = "iotdb_version";
+ private boolean upgradeFromOldVersion = false;
+
public static IoTDBStartCheck getInstance() {
return IoTDBConfigCheckHolder.INSTANCE;
}
@@ -237,6 +239,7 @@ public class IoTDBStartCheck {
logger.error("IoTDB version is too old, please upgrade to 0.12
firstly.");
System.exit(-1);
} else if (versionString.startsWith("0.12") ||
versionString.startsWith("0.13")) {
+ upgradeFromOldVersion = true;
checkWALNotExists();
upgradePropertiesFile();
MetadataUpgrader.upgrade();
@@ -474,4 +477,8 @@ public class IoTDBStartCheck {
logger.error("Unexpected consensus group type");
return false;
}
+
+ public boolean isUpgradeFromOldVersion() {
+ return upgradeFromOldVersion;
+ }
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java
index 7895307a39..319286f99e 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java
@@ -111,7 +111,7 @@ public class InnerSpaceCompactionTask extends
AbstractCompactionTask {
targetTsFileResource =
TsFileNameGenerator.getInnerCompactionTargetFileResource(
selectedTsFileResourceList, sequence);
- } catch (IOException e) {
+ } catch (Exception e) {
LOGGER.error("Failed to get target file for {}",
selectedTsFileResourceList, e);
return;
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadPointCompactionPerformer.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadPointCompactionPerformer.java
index eecb159196..6dcf802127 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadPointCompactionPerformer.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadPointCompactionPerformer.java
@@ -34,7 +34,7 @@ import
org.apache.iotdb.db.engine.compaction.writer.AbstractCompactionWriter;
import org.apache.iotdb.db.engine.compaction.writer.CrossSpaceCompactionWriter;
import org.apache.iotdb.db.engine.compaction.writer.InnerSpaceCompactionWriter;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator;
+import org.apache.iotdb.db.engine.storagegroup.TsFileName;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.metadata.path.AlignedPath;
@@ -253,12 +253,10 @@ public class ReadPointCompactionPerformer
allResources.sort(
(o1, o2) -> {
try {
- TsFileNameGenerator.TsFileName n1 =
- TsFileNameGenerator.getTsFileName(o1.getTsFile().getName());
- TsFileNameGenerator.TsFileName n2 =
- TsFileNameGenerator.getTsFileName(o2.getTsFile().getName());
+ TsFileName n1 = TsFileName.parse(o1.getTsFile().getName());
+ TsFileName n2 = TsFileName.parse(o2.getTsFile().getName());
return (int) (n2.getVersion() - n1.getVersion());
- } catch (IOException e) {
+ } catch (Exception e) {
return 0;
}
});
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
index c8ceecd187..1568ad04aa 100755
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
@@ -31,6 +31,7 @@ import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.IoTDBStartCheck;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.StorageEngineV2;
@@ -118,6 +119,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
@@ -426,6 +428,12 @@ public class DataRegion {
}
try {
+ // check whether upgrading from v0.12, v.13
+ // Notice: delete this when release v0.15
+ if (IoTDBStartCheck.getInstance().isUpgradeFromOldVersion()) {
+ upgradeTsFileName();
+ }
+
// collect candidate TsFiles from sequential and unsequential data
directory
Pair<List<TsFileResource>, List<TsFileResource>> seqTsFilesPair =
getAllFiles(DirectoryManager.getInstance().getAllSequenceFileFolders());
@@ -561,6 +569,46 @@ public class DataRegion {
}
}
+ private void upgradeTsFileName() {
+ List<File> tsFiles = new ArrayList<>();
+ for (String baseDir : DirectoryManager.getInstance().getAllFilesFolders())
{
+ File fileFolder =
+ fsFactory.getFile(baseDir + File.separator +
logicalStorageGroupName, dataRegionId);
+ if (!fileFolder.exists()) {
+ continue;
+ }
+ File[] subFiles = fileFolder.listFiles();
+ if (subFiles != null) {
+ for (File partitionFolder : subFiles) {
+ if (!partitionFolder.isDirectory()) {
+ logger.warn("{} is not a directory.",
partitionFolder.getAbsolutePath());
+ } else if
(!partitionFolder.getName().equals(IoTDBConstant.UPGRADE_FOLDER_NAME)) {
+ // some TsFileResource may be being persisted when the system
crashed, try recovering
+ // such resources
+ continueFailedRenames(partitionFolder, TEMP_SUFFIX);
+ Collections.addAll(
+ tsFiles,
+ fsFactory.listFilesBySuffix(partitionFolder.getAbsolutePath(),
TSFILE_SUFFIX));
+ }
+ }
+ }
+ }
+
+ tsFiles.sort(Comparator.comparingLong(file ->
TsFileName.parseVersion(file.getName())));
+ int curr = 0, next = 1;
+ while (next < tsFiles.size()) {
+ TsFileName currFileName = TsFileName.parse(tsFiles.get(curr).getName());
+ TsFileName nextFileName = TsFileName.parse(tsFiles.get(next).getName());
+ if (currFileName.getTime() > nextFileName.getTime()) {
+ currFileName.setTime(nextFileName.getTime());
+ TsFileResource tsFileResource = new TsFileResource(tsFiles.get(curr));
+ tsFileResource.renameTo(currFileName.toFileName());
+ }
+ curr = next;
+ next++;
+ }
+ }
+
private void initCompaction() {
if (!config.isEnableSeqSpaceCompaction()
&& !config.isEnableUnseqSpaceCompaction()
@@ -617,7 +665,6 @@ public class DataRegion {
*/
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
private void updateLatestFlushedTime() throws IOException {
-
VersionController versionController =
new SimpleFileVersionController(storageGroupSysDir.getPath());
long currentVersion = versionController.currVersion();
@@ -656,7 +703,7 @@ public class DataRegion {
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
private Pair<List<TsFileResource>, List<TsFileResource>>
getAllFiles(List<String> folders)
- throws IOException, DataRegionException {
+ throws IOException {
List<File> tsFiles = new ArrayList<>();
List<File> upgradeFiles = new ArrayList<>();
for (String baseDir : folders) {
@@ -732,20 +779,19 @@ public class DataRegion {
}
/** check if the tsfile's time is smaller than system current time */
- private void checkTsFileTime(File tsFile) throws DataRegionException {
+ private void checkTsFileTime(File tsFile) {
String[] items = tsFile.getName().replace(TSFILE_SUFFIX,
"").split(FILE_NAME_SEPARATOR);
long fileTime = Long.parseLong(items[0]);
long currentTime = System.currentTimeMillis();
if (fileTime > currentTime) {
- throw new DataRegionException(
- String.format(
- "data region %s[%s] is down, because the time of tsfile %s is
larger than system current time, "
- + "file time is %d while system current time is %d, please
check it.",
- logicalStorageGroupName,
- dataRegionId,
- tsFile.getAbsolutePath(),
- fileTime,
- currentTime));
+ logger.error(
+ "data region {}[{}] is down, because the time of tsfile {} is larger
than system current time, file time is {} while system current time is {},
please check it.",
+ logicalStorageGroupName,
+ dataRegionId,
+ tsFile.getAbsolutePath(),
+ fileTime,
+ currentTime);
+ System.exit(-1);
}
}
@@ -1538,14 +1584,7 @@ public class DataRegion {
String filePath =
TsFileNameGenerator.generateNewTsFilePathWithMkdir(
- sequence,
- logicalStorageGroupName,
- dataRegionId,
- timePartitionId,
- createdTime,
- version, // TODO: change version to 0L when field
partitionMaxFileVersions is removed
- 0,
- 0);
+ sequence, logicalStorageGroupName, dataRegionId, timePartitionId,
createdTime, 0, 0, 0);
return getTsFileProcessor(sequence, filePath, timePartitionId);
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileManager.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileManager.java
index c563ddd58b..35954b848d 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileManager.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileManager.java
@@ -556,24 +556,6 @@ public class TsFileManager {
}
}
- //
({systemTime}-{versionNum}-{innerCompactionNum}-{crossCompactionNum}.tsfile)
- public static int compareFileName(File o1, File o2) {
- String[] items1 = o1.getName().replace(TSFILE_SUFFIX,
"").split(FILE_NAME_SEPARATOR);
- String[] items2 = o2.getName().replace(TSFILE_SUFFIX,
"").split(FILE_NAME_SEPARATOR);
- long ver1 = Long.parseLong(items1[0]);
- long ver2 = Long.parseLong(items2[0]);
- int cmp = Long.compare(ver1, ver2);
- if (cmp == 0) {
- int cmpVersion = Long.compare(Long.parseLong(items1[1]),
Long.parseLong(items2[1]));
- if (cmpVersion == 0) {
- return Long.compare(Long.parseLong(items1[2]),
Long.parseLong(items2[2]));
- }
- return cmpVersion;
- } else {
- return cmp;
- }
- }
-
public long getNextCompactionTaskId() {
return currentCompactionTaskSerialId.getAndIncrement();
}
