This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 59a7ec1 optimize upgrade tool (#1234)
59a7ec1 is described below
commit 59a7ec16b551b59123fd362c3e89eab7fef1ee8b
Author: Haonan <[email protected]>
AuthorDate: Sat May 23 14:37:30 2020 +0800
optimize upgrade tool (#1234)
* Fix inserting unsequence data after upgrading
---
docs/UserGuide/0-Get Started/1-QuickStart.md | 2 +-
.../5-Operation Manual/4-SQL Reference.md | 2 +-
.../5-Operation Manual/4-SQL Reference.md | 2 +-
.../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 44 +++++++++++++++++++++-
.../engine/storagegroup/StorageGroupProcessor.java | 18 +++++++++
.../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 6 +--
6 files changed, 67 insertions(+), 7 deletions(-)
diff --git a/docs/UserGuide/0-Get Started/1-QuickStart.md
b/docs/UserGuide/0-Get Started/1-QuickStart.md
index c7603ad..94e168d 100755
--- a/docs/UserGuide/0-Get Started/1-QuickStart.md
+++ b/docs/UserGuide/0-Get Started/1-QuickStart.md
@@ -258,4 +258,4 @@ Under the root path of incubator-iotdb:
> mvn clean package -pl cli -am -DskipTests
```
-After build, the IoTDB client will be at the folder
"client/target/iotdb-client-{project.version}".
\ No newline at end of file
+After build, the IoTDB cli will be at the folder
"cli/target/iotdb-cli-{project.version}".
\ No newline at end of file
diff --git a/docs/UserGuide/5-Operation Manual/4-SQL Reference.md
b/docs/UserGuide/5-Operation Manual/4-SQL Reference.md
index 3fb209f..0483a19 100644
--- a/docs/UserGuide/5-Operation Manual/4-SQL Reference.md
+++ b/docs/UserGuide/5-Operation Manual/4-SQL Reference.md
@@ -797,7 +797,7 @@ Eg. SELECT FIRST_VALUE (status), FIRST_VALUE (temperature)
FROM root.ln.wf01.wt0
Note: the statement needs to satisfy this constraint: <PrefixPath> + <Path> =
<Timeseries>
```
-* LAST_VALUE(Rename from `LAST` at `V0.10.0`)
+* LAST_VALUE
The LAST_VALUE function returns the last point value of the choosen
timeseries(one or more).
diff --git a/docs/zh/UserGuide/5-Operation Manual/4-SQL Reference.md
b/docs/zh/UserGuide/5-Operation Manual/4-SQL Reference.md
index 7e404ed..818ae2c 100644
--- a/docs/zh/UserGuide/5-Operation Manual/4-SQL Reference.md
+++ b/docs/zh/UserGuide/5-Operation Manual/4-SQL Reference.md
@@ -783,7 +783,7 @@ Note: the statement needs to satisfy this constraint:
<PrefixPath> + <Path> = <T
```
* LAST_VALUE
-原有的 `LAST` 方法在 `v0.10.0` 版本更名为 `LAST_VALUE`。
+
```
SELECT LAST_VALUE (Path) (COMMA LAST_VALUE (Path))* FROM <FromClause> [WHERE
<WhereClause>]?
Eg. SELECT LAST_VALUE (status), LAST_VALUE (temperature) FROM
root.ln.wf01.wt01 WHERE root.ln.wf01.wt01.temperature < 24
diff --git
a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
index 7e45ce6..eacca25 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
@@ -19,15 +19,20 @@
package org.apache.iotdb.db.conf;
import org.apache.commons.io.FileUtils;
+import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.metadata.MLogWriter;
import org.apache.iotdb.db.metadata.MetadataConstant;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
+import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.nio.file.Files;
+import java.util.List;
import java.util.Properties;
public class IoTDBConfigCheck {
@@ -37,6 +42,8 @@ public class IoTDBConfigCheck {
public static final String PROPERTIES_FILE_NAME = "system.properties";
public static final String SCHEMA_DIR =
IoTDBDescriptor.getInstance().getConfig().getSchemaDir();
+ public static final String WAL_DIR =
+ IoTDBDescriptor.getInstance().getConfig().getWalFolder();
private static final Logger logger =
LoggerFactory.getLogger(IoTDBDescriptor.class);
// this is a initial parameter.
private static String timestampPrecision = "ms";
@@ -70,10 +77,10 @@ public class IoTDBConfigCheck {
}
tsfileFileSystem =
IoTDBDescriptor.getInstance().getConfig().getTsFileStorageFs().toString();
-
createDir(SCHEMA_DIR);
checkFile(SCHEMA_DIR);
logger.info("System configuration is ok.");
+
}
private void createDir(String filepath) {
@@ -121,6 +128,7 @@ public class IoTDBConfigCheck {
properties.load(new InputStreamReader(inputStream,
TSFileConfig.STRING_CHARSET));
// need to upgrade
if (!properties.containsKey("iotdb_version")) {
+ checkUnClosedTsFileV1();
upgradeMlog();
} else {
checkProperties();
@@ -193,6 +201,40 @@ public class IoTDBConfigCheck {
}
}
+ private void checkUnClosedTsFileV1() {
+ if (SystemFileFactory.INSTANCE.getFile(WAL_DIR).isDirectory()
+ && SystemFileFactory.INSTANCE.getFile(WAL_DIR).list().length != 0) {
+ logger.error("Unclosed Version-1 TsFile detected, please run 'flush' on
V0.9 IoTDB"
+ + " before upgrading to V0.10");
+ System.exit(-1);
+ }
+
checkUnClosedTsFileV1Helper(DirectoryManager.getInstance().getAllSequenceFileFolders());
+
checkUnClosedTsFileV1Helper(DirectoryManager.getInstance().getAllUnSequenceFileFolders());
+ }
+
+ private void checkUnClosedTsFileV1Helper(List<String> folders) {
+ for (String baseDir : folders) {
+ File fileFolder = FSFactoryProducer.getFSFactory().getFile(baseDir);
+ if (!fileFolder.isDirectory()) {
+ continue;
+ }
+ for (File storageGroup : fileFolder.listFiles()) {
+ if (!storageGroup.isDirectory()) {
+ continue;
+ }
+ File[] tsfiles = FSFactoryProducer.getFSFactory()
+ .listFilesBySuffix(storageGroup.toString(),
TsFileConstant.TSFILE_SUFFIX);
+ File[] resources = FSFactoryProducer.getFSFactory()
+ .listFilesBySuffix(storageGroup.toString(),
TsFileResource.RESOURCE_SUFFIX);
+ if (tsfiles.length != resources.length) {
+ logger.error("Unclosed Version-1 TsFile detected, please run 'flush'
on V0.9 IoTDB"
+ + " before upgrading to V0.10");
+ System.exit(-1);
+ }
+ }
+ }
+ }
+
private static class IoTDBConfigCheckHolder {
private static final IoTDBConfigCheck INSTANCE = new IoTDBConfigCheck();
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index b250a60..58a7810 100755
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -1509,6 +1509,24 @@ public class StorageGroupProcessor {
}
mergeLock.writeLock().unlock();
insertLock.writeLock().unlock();
+
+ // after upgrade complete, update partitionLatestFlushedTimeForEachDevice
+ if (countUpgradeFiles() == 0) {
+ for (Entry<Long, Map<String, Long>> entry :
newlyFlushedPartitionLatestFlushedTimeForEachDevice
+ .entrySet()) {
+ long timePartitionId = entry.getKey();
+ Map<String, Long> latestFlushTimeForPartition =
partitionLatestFlushedTimeForEachDevice
+ .getOrDefault(timePartitionId, new HashMap<>());
+ for (Entry<String, Long> endTimeMap : entry.getValue().entrySet()) {
+ String device = endTimeMap.getKey();
+ long endTime = endTimeMap.getValue();
+ if (latestFlushTimeForPartition.getOrDefault(device, Long.MIN_VALUE)
< endTime) {
+ partitionLatestFlushedTimeForEachDevice
+ .computeIfAbsent(timePartitionId, id -> new
HashMap<>()).put(device, endTime);
+ }
+ }
+ }
+ }
}
public void merge(boolean fullMerge) {
diff --git
a/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java
b/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java
index 3b61a32..9fc72d2 100644
---
a/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java
+++
b/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java
@@ -41,7 +41,6 @@ import org.apache.iotdb.tsfile.file.MetaMarker;
import org.apache.iotdb.tsfile.file.footer.ChunkGroupFooter;
import org.apache.iotdb.tsfile.file.header.ChunkHeader;
import org.apache.iotdb.tsfile.file.header.PageHeader;
-import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetadata;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -339,7 +338,7 @@ public class TsFileOnlineUpgradeTool implements
AutoCloseable {
long startOffsetOfChunkGroup = 0;
boolean newChunkGroup = true;
long versionOfChunkGroup = 0;
- List<ChunkGroupMetadata> newMetaData = new ArrayList<>();
+ int chunkGroupCount = 0;
List<List<PageHeader>> pageHeadersInChunkGroup = new ArrayList<>();
List<List<ByteBuffer>> pageDataInChunkGroup = new ArrayList<>();
List<List<Boolean>> pagePartitionInfoInChunkGroup = new ArrayList<>();
@@ -389,6 +388,7 @@ public class TsFileOnlineUpgradeTool implements
AutoCloseable {
measurementSchemaList.clear();
pagePartitionInfoInChunkGroup.clear();
newChunkGroup = true;
+ chunkGroupCount++;
break;
default:
@@ -403,7 +403,7 @@ public class TsFileOnlineUpgradeTool implements
AutoCloseable {
}
} catch (IOException e2) {
logger.info("TsFile upgrade process cannot proceed at position {} after
{} chunk groups "
- + "recovered, because : {}", this.position(), newMetaData.size(),
e2.getMessage());
+ + "recovered, because : {}", this.position(), chunkGroupCount,
e2.getMessage());
} finally {
if (tsFileInput != null) {
tsFileInput.close();
