This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 8f3edd7 [IOTDB-672] Fix delete storage group bug during restart
(#1310)
8f3edd7 is described below
commit 8f3edd731d7fbe5f091efa1dfb4ccafd6ac82784
Author: Haonan <[email protected]>
AuthorDate: Thu Jun 4 16:24:54 2020 +0800
[IOTDB-672] Fix delete storage group bug during restart (#1310)
* fix delete all timeseries in storage group and recover bug
---
docs/SystemDesign/SchemaManager/SchemaManager.md | 4 +-
.../zh/SystemDesign/SchemaManager/SchemaManager.md | 4 +-
.../org/apache/iotdb/db/metadata/MManager.java | 48 +++++++++-------------
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 11 ++---
4 files changed, 27 insertions(+), 40 deletions(-)
diff --git a/docs/SystemDesign/SchemaManager/SchemaManager.md
b/docs/SystemDesign/SchemaManager/SchemaManager.md
index cec2c86..c8b6616 100644
--- a/docs/SystemDesign/SchemaManager/SchemaManager.md
+++ b/docs/SystemDesign/SchemaManager/SchemaManager.md
@@ -55,11 +55,11 @@ In the process of initializing, MManager will replay the
mlog to load the metada
* if succeed
* delete the LeafMNode
* read tlog using offset in the LeafMNode, update tag
inverted index
- * if the storage group becomes empty after deleting,
return the name of it
+ * if the storage group becomes empty after deleting,
record its name
* if failed
* return the full path of failed timeseries
- * iterate the returned empty storage group list, and delete them
* if not restart
+ * delete the recorded empty storage group
* persist log into mlog
* currently, we won't delete the tag/attribute info of that
timeseries in tlog
diff --git a/docs/zh/SystemDesign/SchemaManager/SchemaManager.md
b/docs/zh/SystemDesign/SchemaManager/SchemaManager.md
index 8a4cba2..afda6c1 100644
--- a/docs/zh/SystemDesign/SchemaManager/SchemaManager.md
+++ b/docs/zh/SystemDesign/SchemaManager/SchemaManager.md
@@ -54,11 +54,11 @@ IoTDB 的元数据统一由 MManger 管理，包括以下几个部分：
* 若获取成功
* 删除MTree中对应的LeafMNode
* 根据LeafMNode中的offset，读取标签文件，更新 tagIndex 索引
- * 若发现某一个存储组为空，则返回为空的存储组名
+ * 若发现某一个存储组为空，则记录下该空的存储组名
* 若获取失败
* 返回此未删除成功的时间序列
- * 遍历上一步返回的为空的存储组，依次删除
* 如果非重启（需要记录日志）
+ * 删除记录下的空的存储组
* 将所删除的时间序列信息记录到 mlog 中
* 目前并不会删除 tlog 中关于此时间序列的标签/属性信息。
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 8bfd480..c754c76 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -72,7 +72,7 @@ public class MManager {
private MTree mtree;
private MLogWriter logWriter;
private TagLogFile tagLogFile;
- private boolean writeToLog;
+ private boolean isRecovering;
// device -> DeviceMNode
private RandomDeleteCache<String, MNode> mNodeCache;
@@ -108,7 +108,7 @@ public class MManager {
logFilePath = schemaDir + File.separator + MetadataConstant.METADATA_LOG;
// do not write log when recover
- writeToLog = false;
+ isRecovering = true;
int cacheSize = config.getmManagerCacheSize();
mNodeCache =
@@ -156,7 +156,7 @@ public class MManager {
}
logWriter = new MLogWriter(config.getSchemaDir(),
MetadataConstant.METADATA_LOG);
- writeToLog = true;
+ isRecovering = false;
} catch (IOException | MetadataException e) {
mtree = new MTree();
logger.error("Cannot read MTree from file, using an empty new one", e);
@@ -243,12 +243,9 @@ public class MManager {
createTimeseries(plan, offset);
break;
case MetadataOperationType.DELETE_TIMESERIES:
- Pair<Set<String>, String> pair = deleteTimeseries(args[1]);
- for (String deleteStorageGroup : pair.left) {
-
StorageEngine.getInstance().deleteAllDataFilesInOneStorageGroup(deleteStorageGroup);
- }
- if (!pair.right.isEmpty()) {
- throw new DeleteFailedException(pair.right);
+ String failedTimeseries = deleteTimeseries(args[1]);
+ if (!failedTimeseries.isEmpty()) {
+ throw new DeleteFailedException(failedTimeseries);
}
break;
case MetadataOperationType.SET_STORAGE_GROUP:
@@ -319,7 +316,7 @@ public class MManager {
}
// write log
- if (writeToLog) {
+ if (!isRecovering) {
// either tags or attributes is not empty
if ((plan.getTags() != null && !plan.getTags().isEmpty())
|| (plan.getAttributes() != null &&
!plan.getAttributes().isEmpty())) {
@@ -357,11 +354,9 @@ public class MManager {
* Delete all timeseries under the given path, may cross different storage
group
*
* @param prefixPath path to be deleted, could be root or a prefix path or a
full path
- * @return 1. The Set contains StorageGroups that contain no more timeseries
after this deletion
- * files of such StorageGroups should be deleted to reclaim disk space. 2.
The String is the
- * deletion failed Timeseries
+ * @return The String is the deletion failed Timeseries
*/
- public Pair<Set<String>, String> deleteTimeseries(String prefixPath) throws
MetadataException {
+ public String deleteTimeseries(String prefixPath) throws MetadataException {
lock.writeLock().lock();
if (isStorageGroup(prefixPath)) {
@@ -378,8 +373,6 @@ public class MManager {
mNodeCache.clear();
}
try {
- Set<String> emptyStorageGroups = new HashSet<>();
-
List<String> allTimeseries = mtree.getAllTimeseriesName(prefixPath);
// Monitor storage group seriesPath is not allowed to be deleted
allTimeseries.removeIf(p ->
p.startsWith(MonitorConstants.STAT_STORAGE_GROUP_PREFIX));
@@ -387,15 +380,18 @@ public class MManager {
Set<String> failedNames = new HashSet<>();
for (String p : allTimeseries) {
try {
- String emptyStorageGroup =
deleteOneTimeseriesAndUpdateStatisticsAndLog(p);
- if (emptyStorageGroup != null) {
- emptyStorageGroups.add(emptyStorageGroup);
+ String emptyStorageGroup = deleteOneTimeseriesAndUpdateStatistics(p);
+ if (!isRecovering) {
+ if (emptyStorageGroup != null) {
+
StorageEngine.getInstance().deleteAllDataFilesInOneStorageGroup(emptyStorageGroup);
+ }
+ logWriter.deleteTimeseries(p);
}
} catch (DeleteFailedException e) {
failedNames.add(e.getName());
}
}
- return new Pair<>(emptyStorageGroups, String.join(",", failedNames));
+ return String.join(",", failedNames);
} catch (IOException e) {
throw new MetadataException(e.getMessage());
} finally {
@@ -432,7 +428,7 @@ public class MManager {
* @param path full path from root to leaf node
* @return after delete if the storage group is empty, return its name,
otherwise return null
*/
- private String deleteOneTimeseriesAndUpdateStatisticsAndLog(String path)
+ private String deleteOneTimeseriesAndUpdateStatistics(String path)
throws MetadataException, IOException {
lock.writeLock().lock();
try {
@@ -457,10 +453,6 @@ public class MManager {
.ifPresent(val -> maxSeriesNumberAmongStorageGroup = val);
}
}
-
- if (writeToLog) {
- logWriter.deleteTimeseries(path);
- }
return storageGroupName;
} finally {
lock.writeLock().unlock();
@@ -482,7 +474,7 @@ public class MManager {
ActiveTimeSeriesCounter.getInstance().init(storageGroup);
seriesNumberInStorageGroups.put(storageGroup, 0);
}
- if (writeToLog) {
+ if (!isRecovering) {
logWriter.setStorageGroup(storageGroup);
}
} catch (IOException e) {
@@ -523,7 +515,7 @@ public class MManager {
}
}
// if success
- if (writeToLog) {
+ if (!isRecovering) {
logWriter.deleteStorageGroup(storageGroup);
}
}
@@ -978,7 +970,7 @@ public class MManager {
lock.writeLock().lock();
try {
getStorageGroupNode(storageGroup).setDataTTL(dataTTL);
- if (writeToLog) {
+ if (!isRecovering) {
logWriter.setTTL(storageGroup, dataTTL);
}
} finally {
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index 5d6a268..48697ee 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -1143,18 +1143,13 @@ public class PlanExecutor implements IPlanExecutor {
List<Path> deletePathList = deleteTimeSeriesPlan.getPaths();
try {
deleteDataOfTimeSeries(deletePathList);
- Set<String> emptyStorageGroups = new HashSet<>();
List<String> failedNames = new LinkedList<>();
for (Path path : deletePathList) {
- Pair<Set<String>, String> pair =
mManager.deleteTimeseries(path.toString());
- emptyStorageGroups.addAll(pair.left);
- if (!pair.right.isEmpty()) {
- failedNames.add(pair.right);
+ String failedTimeseries = mManager.deleteTimeseries(path.toString());
+ if (!failedTimeseries.isEmpty()) {
+ failedNames.add(failedTimeseries);
}
}
- for (String deleteStorageGroup : emptyStorageGroups) {
-
StorageEngine.getInstance().deleteAllDataFilesInOneStorageGroup(deleteStorageGroup);
- }
if (!failedNames.isEmpty()) {
throw new DeleteFailedException(String.join(",", failedNames));
}
