This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch feature/pathid
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/feature/pathid by this push:
new 4c94eb0 temporary commit
4c94eb0 is described below
commit 4c94eb0a817d3c597289aa1a384456141dee7ea6
Author: xiangdong huang <[email protected]>
AuthorDate: Fri May 29 18:40:49 2020 +0800
temporary commit
---
.../db/conf/adapter/ActiveTimeSeriesCounter.java | 18 +--
.../db/conf/adapter/IActiveTimeSeriesCounter.java | 10 +-
.../org/apache/iotdb/db/metadata/MLogWriter.java | 4 +-
.../org/apache/iotdb/db/metadata/MManager.java | 168 ++++++++++-----------
.../java/org/apache/iotdb/db/metadata/MTree.java | 150 +++++++++---------
.../iotdb/db/metadata/id/ID2NodeManager.java} | 30 ++--
.../iotdb/db/metadata/{ => id}/IDManager.java | 21 +--
.../iotdb/db/metadata/mnode/InternalMNode.java | 1 +
.../apache/iotdb/db/metadata/mnode/LeafMNode.java | 11 ++
.../org/apache/iotdb/db/metadata/mnode/MNode.java | 6 +
.../iotdb/db/metadata/mnode/StorageGroupMNode.java | 10 ++
.../apache/iotdb/db/metadata/IDManagerTest.java | 1 +
12 files changed, 238 insertions(+), 192 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/conf/adapter/ActiveTimeSeriesCounter.java
b/server/src/main/java/org/apache/iotdb/db/conf/adapter/ActiveTimeSeriesCounter.java
index 7251924..dde6256 100644
---
a/server/src/main/java/org/apache/iotdb/db/conf/adapter/ActiveTimeSeriesCounter.java
+++
b/server/src/main/java/org/apache/iotdb/db/conf/adapter/ActiveTimeSeriesCounter.java
@@ -32,17 +32,17 @@ public class ActiveTimeSeriesCounter implements
IActiveTimeSeriesCounter {
/**
* Map[StorageGroup, HyperLogLogCounter]
*/
- private static Map<String, HyperLogLog> storageGroupHllMap = new
ConcurrentHashMap<>();
+ private static Map<Integer, HyperLogLog> storageGroupHllMap = new
ConcurrentHashMap<>();
/**
* Map[StorageGroup, ActiveTimeSeriesRatio]
*/
- private static Map<String, Double> activeRatioMap = new
ConcurrentHashMap<>();
+ private static Map<Integer, Double> activeRatioMap = new
ConcurrentHashMap<>();
/**
* Map[StorageGroup, ActiveTimeSeriesNumber]
*/
- private static Map<String, Long> activeTimeSeriesNumMap = new
ConcurrentHashMap<>();
+ private static Map<Integer, Long> activeTimeSeriesNumMap = new
ConcurrentHashMap<>();
/**
* LOG2M decide the precision of the HyperLogLog algorithm
@@ -52,14 +52,14 @@ public class ActiveTimeSeriesCounter implements
IActiveTimeSeriesCounter {
private ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
@Override
- public void init(String storageGroup) {
+ public void init(int storageGroup) {
storageGroupHllMap.put(storageGroup, new HyperLogLog(LOG2M));
activeRatioMap.put(storageGroup, 0D);
activeTimeSeriesNumMap.put(storageGroup, 0L);
}
@Override
- public void offer(String storageGroup, String device, String measurement) {
+ public void offer(int storageGroup, String device, String measurement) {
String path = device + IoTDBConstant.PATH_SEPARATOR + measurement;
try {
HyperLogLog log = storageGroupHllMap.get(storageGroup);
@@ -78,7 +78,7 @@ public class ActiveTimeSeriesCounter implements
IActiveTimeSeriesCounter {
}
@Override
- public void updateActiveRatio(String storageGroup) {
+ public void updateActiveRatio(int storageGroup) {
lock.writeLock().lock();
try {
HyperLogLog log = storageGroupHllMap.get(storageGroup);
@@ -98,7 +98,7 @@ public class ActiveTimeSeriesCounter implements
IActiveTimeSeriesCounter {
for (double number : activeTimeSeriesNumMap.values()) {
totalActiveTsNum += number;
}
- for (Map.Entry<String, Long> entry :
activeTimeSeriesNumMap.entrySet()) {
+ for (Map.Entry<Integer, Long> entry :
activeTimeSeriesNumMap.entrySet()) {
double activeRatio = 0;
if (totalActiveTsNum > 0) {
activeRatio = entry.getValue() / totalActiveTsNum;
@@ -119,7 +119,7 @@ public class ActiveTimeSeriesCounter implements
IActiveTimeSeriesCounter {
}
@Override
- public double getActiveRatio(String storageGroup) {
+ public double getActiveRatio(int storageGroup) {
lock.writeLock().lock();
double ratio;
try {
@@ -134,7 +134,7 @@ public class ActiveTimeSeriesCounter implements
IActiveTimeSeriesCounter {
}
@Override
- public void delete(String storageGroup) {
+ public void delete(int storageGroup) {
storageGroupHllMap.remove(storageGroup);
activeRatioMap.remove(storageGroup);
activeTimeSeriesNumMap.remove(storageGroup);
diff --git
a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IActiveTimeSeriesCounter.java
b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IActiveTimeSeriesCounter.java
index 43bc10c..9b5b77c 100644
---
a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IActiveTimeSeriesCounter.java
+++
b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IActiveTimeSeriesCounter.java
@@ -25,7 +25,7 @@ public interface IActiveTimeSeriesCounter {
*
* @param storageGroup the given storage group to be initialized
*/
- void init(String storageGroup);
+ void init(int storageGroup);
/**
* Register a time series to the active time series counter
@@ -34,14 +34,14 @@ public interface IActiveTimeSeriesCounter {
* @param device the device name of the time series
* @param measurement the sensor name of the time series
*/
- void offer(String storageGroup, String device, String measurement);
+ void offer(int storageGroup, String device, String measurement);
/**
* Update the ActiveRatioMap
*
* @param storageGroup whose counter will be refreshed after the update
*/
- void updateActiveRatio(String storageGroup);
+ void updateActiveRatio(int storageGroup);
/**
* Get the active time series number proportion of the given storage group
@@ -49,13 +49,13 @@ public interface IActiveTimeSeriesCounter {
* @param storageGroup the storage group to be calculated
* @return the active time series number proportion of the given storage
group
*/
- double getActiveRatio(String storageGroup);
+ double getActiveRatio(int storageGroup);
/**
* Delete the counter for the given storage group
*
* @param storageGroup whose counter will be removed
*/
- void delete(String storageGroup);
+ void delete(int storageGroup);
}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java
b/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java
index 1956605..aed8fed 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java
@@ -119,8 +119,8 @@ public class MLogWriter {
writer.flush();
}
- public void setStorageGroup(String storageGroup) throws IOException {
- writer.write(MetadataOperationType.SET_STORAGE_GROUP + "," + storageGroup);
+ public void setStorageGroup(String storageGroup, int id) throws IOException {
+ writer.write(String.format("%d,%s,%d",
MetadataOperationType.SET_STORAGE_GROUP, storageGroup, id));
writer.newLine();
writer.flush();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 0642f31..e065907 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -52,16 +52,16 @@ import
org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.metadata.PathNotExistException;
import org.apache.iotdb.db.exception.metadata.StorageGroupAlreadySetException;
import org.apache.iotdb.db.exception.metadata.StorageGroupNotSetException;
+import org.apache.iotdb.db.metadata.id.ID2NodeManager;
+import org.apache.iotdb.db.metadata.id.IDManager;
import org.apache.iotdb.db.metadata.mnode.InternalMNode;
import org.apache.iotdb.db.metadata.mnode.LeafMNode;
import org.apache.iotdb.db.metadata.mnode.MNode;
import org.apache.iotdb.db.metadata.mnode.StorageGroupMNode;
-import org.apache.iotdb.db.monitor.MonitorConstants;
import org.apache.iotdb.db.qp.constant.SQLConstant;
import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan;
import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult;
-import org.apache.iotdb.db.utils.RandomDeleteCache;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.tsfile.common.cache.LRUCache;
import org.apache.iotdb.tsfile.exception.cache.CacheException;
@@ -84,6 +84,13 @@ public class MManager {
private static final Logger logger = LoggerFactory.getLogger(MManager.class);
private static final String TIME_SERIES_TREE_HEADER = "=== Timeseries Tree
===\n\n";
+ private ID2NodeManager id2StorageGroups = new ID2NodeManager();
+ //only the string without the storage group name.
+ private ID2NodeManager id2Devices = new ID2NodeManager();
+ //only the measurement name
+ private ID2NodeManager id2Measurement = new ID2NodeManager();
+
+
// the lock for read/insert
private ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
// the log file seriesPath
@@ -92,8 +99,7 @@ public class MManager {
private MLogWriter logWriter;
private TagLogFile tagLogFile;
private boolean writeToLog;
- // device -> DeviceMNode
- private RandomDeleteCache<String, MNode> mNodeCache;
+
// currently, if a key is not existed in the mRemoteSchemaCache, an
IOException will be thrown
private LRUCache<String, MeasurementSchema> mRemoteSchemaCache;
@@ -101,11 +107,14 @@ public class MManager {
private Map<String, Map<String, Set<LeafMNode>>> tagIndex = new HashMap<>();
// storage group name -> the series number
- private Map<String, Integer> seriesNumberInStorageGroups = new HashMap<>();
+ private Map<Integer, Integer> seriesNumberInStorageGroups = new HashMap<>();
private long maxSeriesNumberAmongStorageGroup;
private boolean initialized;
private IoTDBConfig config;
+ //TODO init this one.
+ private StorageGroupMNode statStorageGroup;
+
private static class MManagerHolder {
private MManagerHolder() {
@@ -132,20 +141,7 @@ public class MManager {
writeToLog = false;
int cacheSize = config.getmManagerCacheSize();
- mNodeCache = new RandomDeleteCache<String, MNode>(cacheSize) {
- @Override
- public MNode loadObjectByKey(String key) throws CacheException {
- lock.readLock().lock();
- try {
- return mtree.getNodeByPathWithStorageGroupCheck(key);
- } catch (MetadataException e) {
- throw new CacheException(e);
- } finally {
- lock.readLock().unlock();
- }
- }
- };
int remoteCacheSize = config.getmRemoteSchemaCacheSize();
mRemoteSchemaCache = new LRUCache<String,
MeasurementSchema>(remoteCacheSize) {
@@ -159,6 +155,7 @@ public class MManager {
cache.keySet().removeIf(s -> s.startsWith(key));
}
};
+
}
public static MManager getInstance() {
@@ -179,10 +176,10 @@ public class MManager {
initFromLog(logFile);
if (config.isEnableParameterAdapter()) {
- List<String> storageGroups = mtree.getAllStorageGroupNames();
- for (String sg : storageGroups) {
- MNode node = mtree.getNodeByPath(sg);
- seriesNumberInStorageGroups.put(sg, node.getLeafCount());
+ List<StorageGroupMNode> storageGroups =
mtree.getAllStorageGroupNodes();
+ for (StorageGroupMNode sg : storageGroups) {
+ seriesNumberInStorageGroups.put(sg.getId(), sg.getLeafCount());
+ id2StorageGroups.put(sg.getId(), sg);
}
maxSeriesNumberAmongStorageGroup =
seriesNumberInStorageGroups.values().stream().max(Integer::compareTo).orElse(0);
@@ -190,7 +187,7 @@ public class MManager {
logWriter = new MLogWriter(config.getSchemaDir(),
MetadataConstant.METADATA_LOG);
writeToLog = true;
- } catch (IOException | MetadataException e) {
+ } catch (IOException e) {
mtree = new MTree();
logger.error("Cannot read MTree from file, using an empty new one", e);
}
@@ -222,7 +219,6 @@ public class MManager {
lock.writeLock().lock();
try {
this.mtree = new MTree();
- this.mNodeCache.clear();
this.tagIndex.clear();
this.seriesNumberInStorageGroups.clear();
this.maxSeriesNumberAmongStorageGroup = 0;
@@ -276,9 +272,9 @@ public class MManager {
createTimeseries(plan, offset);
break;
case MetadataOperationType.DELETE_TIMESERIES:
- Pair<Set<String>, String> pair = deleteTimeseries(args[1]);
- for (String deleteStorageGroup : pair.left) {
-
StorageEngine.getInstance().deleteAllDataFilesInOneStorageGroup(deleteStorageGroup);
+ Pair<Set<StorageGroupMNode>, String> pair = deleteTimeseries(args[1]);
+ for (StorageGroupMNode deleteStorageGroup : pair.left) {
+
StorageEngine.getInstance().deleteAllDataFilesInOneStorageGroup(deleteStorageGroup.getFullPath());
}
if (!pair.right.isEmpty()) {
throw new DeleteFailedException(pair.right);
@@ -313,16 +309,16 @@ public class MManager {
/*
* get the storage group with auto create schema
*/
- String storageGroupName;
+ StorageGroupMNode storageGroup;
try {
- storageGroupName = mtree.getStorageGroupName(path);
+ storageGroup = mtree.findStorageGroupNode(path);
} catch (StorageGroupNotSetException e) {
if (!config.isAutoCreateSchemaEnabled()) {
throw e;
}
- storageGroupName =
+ String storageGroupName =
MetaUtils.getStorageGroupNameByLevel(path,
config.getDefaultStorageGroupLevel());
- setStorageGroup(storageGroupName);
+ storageGroup = setStorageGroup(storageGroupName);
}
// check memory
@@ -344,8 +340,8 @@ public class MManager {
// update statistics
if (config.isEnableParameterAdapter()) {
- int size = seriesNumberInStorageGroups.get(storageGroupName);
- seriesNumberInStorageGroups.put(storageGroupName, size + 1);
+ int size = seriesNumberInStorageGroups.get(storageGroup.getId());
+ seriesNumberInStorageGroups.put(storageGroup.getId(), size + 1);
if (size + 1 > maxSeriesNumberAmongStorageGroup) {
maxSeriesNumberAmongStorageGroup = size + 1;
}
@@ -394,37 +390,28 @@ public class MManager {
* files of such StorageGroups should be deleted to reclaim disk space. 2.
The String is the
* deletion failed Timeseries
*/
- public Pair<Set<String>, String> deleteTimeseries(String prefixPath) throws
MetadataException {
+ public Pair<Set<StorageGroupMNode>, String> deleteTimeseries(String
prefixPath) throws MetadataException {
lock.writeLock().lock();
// clear cached schema
mRemoteSchemaCache.removeItem(prefixPath);
- if (isStorageGroup(prefixPath)) {
+ MNode node = mtree.getNodeByPath(prefixPath);
- if (config.isEnableParameterAdapter()) {
- int size = seriesNumberInStorageGroups.get(prefixPath);
- seriesNumberInStorageGroups.put(prefixPath, 0);
- if (size == maxSeriesNumberAmongStorageGroup) {
- seriesNumberInStorageGroups.values().stream()
- .max(Integer::compareTo)
- .ifPresent(val -> maxSeriesNumberAmongStorageGroup = val);
- }
- }
-
- mNodeCache.clear();
- }
try {
- Set<String> emptyStorageGroups = new HashSet<>();
+ Set<StorageGroupMNode> emptyStorageGroups = new HashSet<>();
- List<String> allTimeseries = mtree.getAllTimeseriesName(prefixPath);
+ List<LeafMNode> allTimeseries = mtree.getAllLeafNodes(node);
// Monitor storage group seriesPath is not allowed to be deleted
- allTimeseries.removeIf(p ->
p.startsWith(MonitorConstants.STAT_STORAGE_GROUP_PREFIX));
+ if (statStorageGroup != null) {
+ int statSGId = statStorageGroup.getId();
+ allTimeseries.removeIf(p -> IDManager.getStorageGroupID(p.getId()) ==
statSGId);
+ }
Set<String> failedNames = new HashSet<>();
- for (String p : allTimeseries) {
+ for (LeafMNode p : allTimeseries) {
try {
- String emptyStorageGroup =
deleteOneTimeseriesAndUpdateStatisticsAndLog(p);
+ StorageGroupMNode emptyStorageGroup =
deleteOneTimeseriesAndUpdateStatisticsAndLog(p);
if (emptyStorageGroup != null) {
emptyStorageGroups.add(emptyStorageGroup);
}
@@ -466,66 +453,76 @@ public class MManager {
}
/**
- * @param path full path from root to leaf node
- * @return after delete if the storage group is empty, return its name,
otherwise return null
+ * @param timeSeries full path from root to leaf node
+ * @return after delete if the storage group is empty, return the node,
otherwise return null
*/
- private String deleteOneTimeseriesAndUpdateStatisticsAndLog(String path)
+ private StorageGroupMNode
deleteOneTimeseriesAndUpdateStatisticsAndLog(LeafMNode timeSeries)
throws MetadataException, IOException {
lock.writeLock().lock();
try {
- Pair<String, LeafMNode> pair =
mtree.deleteTimeseriesAndReturnEmptyStorageGroup(path);
- removeFromTagInvertedIndex(pair.right);
- String storageGroupName = pair.left;
+ StorageGroupMNode emptySG =
mtree.deleteTimeseriesAndReturnEmptyStorageGroup(timeSeries);
+ removeFromTagInvertedIndex(timeSeries);
+
+
decreaseSeriesNumberStaticstics(IDManager.getStorageGroupID(timeSeries.getId()),
1);
+
+ if (writeToLog) {
+ logWriter.deleteTimeseries(timeSeries.getFullPath());
+ }
- // TODO: delete the path node and all its ancestors
- mNodeCache.clear();
try {
IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(-1);
} catch (ConfigAdjusterException e) {
throw new MetadataException(e);
}
- if (config.isEnableParameterAdapter()) {
- String storageGroup = getStorageGroupName(path);
- int size = seriesNumberInStorageGroups.get(storageGroup);
- seriesNumberInStorageGroups.put(storageGroup, size - 1);
- if (size == maxSeriesNumberAmongStorageGroup) {
- seriesNumberInStorageGroups.values().stream().max(Integer::compareTo)
- .ifPresent(val -> maxSeriesNumberAmongStorageGroup = val);
- }
- }
- if (writeToLog) {
- logWriter.deleteTimeseries(path);
- }
- return storageGroupName;
+ return emptySG;
} finally {
lock.writeLock().unlock();
}
}
+ private void decreaseSeriesNumberStaticstics(int sgId, int delta) {
+ if (config.isEnableParameterAdapter()) {
+ int size = seriesNumberInStorageGroups.get(sgId);
+ seriesNumberInStorageGroups.put(sgId, size - delta);
+ if (size == maxSeriesNumberAmongStorageGroup) {
+ seriesNumberInStorageGroups.values().stream().max(Integer::compareTo)
+ .ifPresent(val -> maxSeriesNumberAmongStorageGroup = val);
+ }
+ }
+ }
+
+ private void resetSeriesNumberStaticsticsForASG(int sgId) {
+ decreaseSeriesNumberStaticstics(sgId,
seriesNumberInStorageGroups.get(sgId));
+ }
+
+
+
/**
* Set storage group of the given path to MTree. Check
*
* @param storageGroup root.node.(node)*
*/
- public void setStorageGroup(String storageGroup) throws MetadataException {
+ public StorageGroupMNode setStorageGroup(String storageGroup) throws
MetadataException {
lock.writeLock().lock();
+ StorageGroupMNode node = null;
try {
- mtree.setStorageGroup(storageGroup);
+ node = mtree.setStorageGroup(storageGroup);
IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(1);
if (config.isEnableParameterAdapter()) {
- ActiveTimeSeriesCounter.getInstance().init(storageGroup);
- seriesNumberInStorageGroups.put(storageGroup, 0);
+ ActiveTimeSeriesCounter.getInstance().init(node.getId());
+ seriesNumberInStorageGroups.put(node.getId(), 0);
}
if (writeToLog) {
- logWriter.setStorageGroup(storageGroup);
+ logWriter.setStorageGroup(storageGroup, node.getId());
}
+ return node;
} catch (IOException e) {
throw new MetadataException(e.getMessage());
} catch (ConfigAdjusterException e) {
- mtree.deleteStorageGroup(storageGroup);
+ mtree.deleteStorageGroup(node);
throw new MetadataException(e);
} finally {
lock.writeLock().unlock();
@@ -545,17 +542,16 @@ public class MManager {
mRemoteSchemaCache.removeItem(storageGroup);
// try to delete storage group
- List<LeafMNode> leafMNodes = mtree.deleteStorageGroup(storageGroup);
+ StorageGroupMNode sgNode = mtree.getStorageGroupNode(storageGroup);
+ List<LeafMNode> leafMNodes = mtree.deleteStorageGroup(sgNode);
for (LeafMNode leafMNode : leafMNodes) {
removeFromTagInvertedIndex(leafMNode);
}
- mNodeCache.clear();
-
if (config.isEnableParameterAdapter()) {
IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(-1);
- int size = seriesNumberInStorageGroups.get(storageGroup);
+ int size = seriesNumberInStorageGroups.get(sgNode.getId());
IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(size *
-1);
- ActiveTimeSeriesCounter.getInstance().delete(storageGroup);
+ ActiveTimeSeriesCounter.getInstance().delete(sgNode.getId());
seriesNumberInStorageGroups.remove(storageGroup);
if (size == maxSeriesNumberAmongStorageGroup) {
maxSeriesNumberAmongStorageGroup =
@@ -941,10 +937,10 @@ public class MManager {
* Get storage group node by path. If storage group is not set,
StorageGroupNotSetException will
* be thrown
*/
- public StorageGroupMNode getStorageGroupNode(String path) throws
MetadataException {
+ public StorageGroupMNode getStorageGroupNode(String storageGroupPath) throws
MetadataException {
lock.readLock().lock();
try {
- return mtree.getStorageGroupNode(path);
+ return mtree.getStorageGroupNode(storageGroupPath);
} finally {
lock.readLock().unlock();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
index bded1d9..ae604a2 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
@@ -36,17 +36,20 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Queue;
import java.util.Set;
+import java.util.Stack;
import java.util.TreeSet;
import java.util.regex.Pattern;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.metadata.AliasAlreadyExistException;
+import org.apache.iotdb.db.exception.metadata.DeleteFailedException;
import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException;
import org.apache.iotdb.db.exception.metadata.PathNotExistException;
import org.apache.iotdb.db.exception.metadata.StorageGroupAlreadySetException;
import org.apache.iotdb.db.exception.metadata.StorageGroupNotSetException;
+import org.apache.iotdb.db.metadata.id.IDManager;
import org.apache.iotdb.db.metadata.mnode.InternalMNode;
import org.apache.iotdb.db.metadata.mnode.LeafMNode;
import org.apache.iotdb.db.metadata.mnode.MNode;
@@ -86,14 +89,8 @@ public class MTree implements Serializable {
* @param props props
* @param alias alias of measurement
*/
- LeafMNode createTimeseries(
- String path,
- TSDataType dataType,
- TSEncoding encoding,
- CompressionType compressor,
- Map<String, String> props,
- String alias)
- throws MetadataException {
+ LeafMNode createTimeseries(String path, TSDataType dataType, TSEncoding
encoding,
+ CompressionType compressor, Map<String, String> props, String alias)
throws MetadataException {
String[] nodeNames = MetaUtils.getNodeNames(path);
if (nodeNames.length <= 2 || !nodeNames[0].equals(root.getName())) {
throw new IllegalPathException(path);
@@ -180,7 +177,7 @@ public class MTree implements Serializable {
*
* @param path path
*/
- void setStorageGroup(String path) throws MetadataException {
+ StorageGroupMNode setStorageGroup(String path) throws MetadataException {
String[] nodeNames = MetaUtils.getNodeNames(path);
MNode cur = root;
if (nodeNames.length <= 1 || !nodeNames[0].equals(root.getName())) {
@@ -206,41 +203,30 @@ public class MTree implements Serializable {
StorageGroupMNode storageGroupMNode =
new StorageGroupMNode(
cur, nodeNames[i], path,
IoTDBDescriptor.getInstance().getConfig().getDefaultTTL());
+ //acquire an ID
+ storageGroupMNode.setId(IDManager.newSGNumber());
cur.addChild(nodeNames[i], storageGroupMNode);
+ return storageGroupMNode;
}
}
/** Delete a storage group */
- List<LeafMNode> deleteStorageGroup(String path) throws MetadataException {
- MNode cur = getNodeByPath(path);
- if (!(cur instanceof StorageGroupMNode)) {
- throw new StorageGroupNotSetException(path);
- }
+ List<LeafMNode> deleteStorageGroup(StorageGroupMNode storageGroupNode)
+ throws DeleteFailedException {
+
// Suppose current system has root.a.b.sg1, root.a.sg2, and delete
root.a.b.sg1
// delete the storage group node sg1
- cur.getParent().deleteChild(cur.getName());
+ storageGroupNode.getParent().deleteChild(storageGroupNode.getName());
- // collect all the LeafMNode in this storage group
- List<LeafMNode> leafMNodes = new LinkedList<>();
- Queue<MNode> queue = new LinkedList<>();
- queue.add(cur);
- while (!queue.isEmpty()) {
- MNode node = queue.poll();
- for (MNode child : node.getChildren().values()) {
- if (child instanceof LeafMNode) {
- leafMNodes.add((LeafMNode) child);
- } else {
- queue.add(child);
- }
- }
- }
- cur = cur.getParent();
- // delete node b while retain root.a.sg2
- while (!IoTDBConstant.PATH_ROOT.equals(cur.getName()) &&
cur.getChildren().size() == 0) {
- cur.getParent().deleteChild(cur.getName());
- cur = cur.getParent();
+ // collect all the LeafMNode in this storage group
+ List<LeafMNode> leafMNodes = getAllLeafNodes(storageGroupNode);
+ //release the reference of these leaves parent node.
+ for (LeafMNode node : leafMNodes) {
+ node.setParent(null);
}
+ removeEmptyParents(storageGroupNode.getParent());
+ storageGroupNode.setParent(null);
return leafMNodes;
}
@@ -274,38 +260,42 @@ public class MTree implements Serializable {
/**
* Delete path. The path should be a full path from root to leaf node
*
- * @param path Format: root.node(.node)+
+ * @param timeSeries the leaf node
+ * @return the storage group node if there is no more time series under the
sg after deletion,
+ * null otherwise
+ * @throws DeleteFailedException if some node is used.
*/
- Pair<String, LeafMNode> deleteTimeseriesAndReturnEmptyStorageGroup(String
path)
- throws MetadataException {
- MNode curNode = getNodeByPath(path);
- if (!(curNode instanceof LeafMNode)) {
- throw new PathNotExistException(path);
- }
- String[] nodes = MetaUtils.getNodeNames(path);
- if (nodes.length == 0 || !IoTDBConstant.PATH_ROOT.equals(nodes[0])) {
- throw new IllegalPathException(path);
- }
- // delete the last node of path
- curNode.getParent().deleteChild(curNode.getName());
- LeafMNode deletedNode = (LeafMNode) curNode;
- if (deletedNode.getAlias() != null) {
- curNode.getParent().deleteAliasChild(((LeafMNode) curNode).getAlias());
+ StorageGroupMNode deleteTimeseriesAndReturnEmptyStorageGroup(LeafMNode
timeSeries)
+ throws DeleteFailedException {
+ timeSeries.getParent().deleteChild(timeSeries.getName());
+ if (timeSeries.getAlias() != null) {
+ timeSeries.getParent().deleteAliasChild(timeSeries.getAlias());
}
- curNode = curNode.getParent();
+ MNode node = timeSeries.getParent();
+ return removeEmptyParents(node);
+ }
+
+ /**
+ * remove this node if it has no child, and remove its parenets if they have
no children any more.
+ * @param node
+ * @return StorageGroupMNode if there is a sg node and it is removed,
otherwise null
+ * @throws DeleteFailedException
+ */
+ private StorageGroupMNode removeEmptyParents(MNode node) throws
DeleteFailedException{
+ StorageGroupMNode result = null;
// delete all empty ancestors except storage group
- while (!IoTDBConstant.PATH_ROOT.equals(curNode.getName())
- && curNode.getChildren().size() == 0) {
- // if current storage group has no time series, return the storage group
name
- if (curNode instanceof StorageGroupMNode) {
- return new Pair<>(curNode.getFullPath(), deletedNode);
+ while (node.getParent() != null && node.getChildren().isEmpty()) {
+ //not the root node.
+ node.getParent().deleteChild(node.getName());
+ if (node instanceof StorageGroupMNode) {
+ result = (StorageGroupMNode) node;
}
- curNode.getParent().deleteChild(curNode.getName());
- curNode = curNode.getParent();
}
- return new Pair<>(null, deletedNode);
+ return result;
}
+
+
/**
* Get measurement schema for a given path. Path must be a complete Path
from root to leaf node.
*/
@@ -347,12 +337,12 @@ public class MTree implements Serializable {
}
/** Get storage group node, if the give path is not a storage group, throw
exception */
- StorageGroupMNode getStorageGroupNode(String path) throws MetadataException {
- MNode node = getNodeByPath(path);
+ StorageGroupMNode getStorageGroupNode(String storageGroupPath) throws
MetadataException {
+ MNode node = getNodeByPath(storageGroupPath);
if (node instanceof StorageGroupMNode) {
return (StorageGroupMNode) node;
} else {
- throw new StorageGroupNotSetException(path);
+ throw new StorageGroupNotSetException(storageGroupPath);
}
}
@@ -366,7 +356,7 @@ public class MTree implements Serializable {
*
* @return last node in given seriesPath
*/
- MNode getNodeByPath(String path) throws MetadataException {
+ MNode getNodeByPath(String path) throws IllegalPathException,
PathNotExistException {
String[] nodes = MetaUtils.getNodeNames(path);
if (nodes.length == 0 || !nodes[0].equals(root.getName())) {
throw new IllegalPathException(path);
@@ -463,19 +453,19 @@ public class MTree implements Serializable {
}
/**
- * Get storage group name by path
+ * Get storage group node by path
*
* <p>e.g., root.sg1 is storage group, path is root.sg1.d1, return root.sg1
*
* @return storage group in the given path
*/
- String getStorageGroupName(String path) throws StorageGroupNotSetException {
+ StorageGroupMNode findStorageGroupNode(String path) throws
StorageGroupNotSetException {
String[] nodes = MetaUtils.getNodeNames(path);
MNode cur = root;
for (int i = 1; i < nodes.length; i++) {
cur = cur.getChild(nodes[i]);
if (cur instanceof StorageGroupMNode) {
- return cur.getFullPath();
+ return (StorageGroupMNode) cur;
} else if (cur == null) {
throw new StorageGroupNotSetException(path);
}
@@ -483,6 +473,9 @@ public class MTree implements Serializable {
throw new StorageGroupNotSetException(path);
}
+
+
+
/** Check whether the given path contains a storage group */
boolean checkStorageGroupByPath(String path) {
String[] nodes = MetaUtils.getNodeNames(path);
@@ -589,7 +582,7 @@ public class MTree implements Serializable {
tsRow[0] = nodePath;
tsRow[1] = ((LeafMNode) node).getAlias();
MeasurementSchema measurementSchema = ((LeafMNode) node).getSchema();
- tsRow[2] = getStorageGroupName(nodePath);
+ tsRow[2] = findStorageGroupNode(nodePath).getFullPath();
tsRow[3] = measurementSchema.getType().toString();
tsRow[4] = measurementSchema.getEncodingType().toString();
tsRow[5] = measurementSchema.getCompressor().toString();
@@ -903,4 +896,25 @@ public class MTree implements Serializable {
}
}
}
-}
+
+ public List<LeafMNode> getAllLeafNodes(MNode start) {
+ List<LeafMNode> leaves = new ArrayList<>();
+ Stack<MNode> nodes = new Stack<>();
+ if (start instanceof LeafMNode) {
+ leaves.add((LeafMNode) start);
+ return leaves;
+ }
+ nodes.push(start);
+ while (!nodes.isEmpty()) {
+ MNode node = nodes.pop();
+ for (MNode child : node.getChildren().values()) {
+ if (child instanceof LeafMNode) {
+ leaves.add((LeafMNode) child);
+ } else {
+ nodes.push(child);
+ }
+ }
+ }
+ return leaves;
+ }
+}
\ No newline at end of file
diff --git
a/server/src/test/java/org/apache/iotdb/db/metadata/IDManagerTest.java
b/server/src/main/java/org/apache/iotdb/db/metadata/id/ID2NodeManager.java
similarity index 59%
copy from server/src/test/java/org/apache/iotdb/db/metadata/IDManagerTest.java
copy to server/src/main/java/org/apache/iotdb/db/metadata/id/ID2NodeManager.java
index 8bba51d..ae6dc6b 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/IDManagerTest.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/id/ID2NodeManager.java
@@ -16,21 +16,27 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.db.metadata;
-import org.apache.iotdb.db.exception.metadata.MetadataException;
-import org.junit.Assert;
-import org.junit.Test;
+package org.apache.iotdb.db.metadata.id;
-public class IDManagerTest {
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.iotdb.db.metadata.mnode.MNode;
- @Test
- public void testNewGenerator() throws MetadataException {
- Assert.assertEquals(1, IDManager.newSGNumber());
- Assert.assertEquals(1, IDManager.newDeviceNumber());
- Assert.assertEquals(1, IDManager.newMeasurementNumber());
- //0000 0000 0000 0010 0000 0000 0000 0000 0000 0000 0000 0010 0000 0000
0000 0010
- Assert.assertEquals(0x0002000000020002L, IDManager.newID());
+public class ID2NodeManager {
+ List<MNode> id2strings = new ArrayList<>();
+
+
+ /**
+ * when calling this method, you must guarantee that all ids that less than
id has been put.
+ * @param id
+ * @param node
+ */
+ public void put(int id, MNode node) {
+ id2strings.set(id, node);
}
+ public MNode get(int id) {
+ return id2strings.get(id);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/IDManager.java
b/server/src/main/java/org/apache/iotdb/db/metadata/id/IDManager.java
similarity index 79%
rename from server/src/main/java/org/apache/iotdb/db/metadata/IDManager.java
rename to server/src/main/java/org/apache/iotdb/db/metadata/id/IDManager.java
index 187c925..5848ae0 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/IDManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/id/IDManager.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.db.metadata;
+package org.apache.iotdb.db.metadata.id;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.iotdb.db.exception.metadata.MetadataException;
@@ -26,9 +26,9 @@ import
org.apache.iotdb.db.exception.metadata.MetadataException;
*/
public class IDManager {
- private static int storageGroupIDLength = 16; //16 bits, must be not greater
than 32
- private static int deviceIDLength = 32; //32 bits, must be not greater than
32
- private static int measurementLength = 16; //16 bits, must be not greater
than 32
+ private static int storageGroupIDLength = 12; //must be not greater than 32
+ private static int deviceIDLength = 32; //must be not greater than 32
+ private static int measurementLength = 20; //must be not greater than 32
private static int maxSGID;
private static int maxDeivceID;
@@ -57,6 +57,7 @@ public class IDManager {
private static AtomicInteger deviceGenerator = new AtomicInteger(0);
private static AtomicInteger measurementGenerator = new AtomicInteger(0);
+
public static int newSGNumber() throws MetadataException {
if (sgGenerator.get() == maxSGID) {
throw new MetadataException("too many storage groups: {}",
sgGenerator.get());
@@ -82,16 +83,16 @@ public class IDManager {
return (0L | ((long) newSGNumber()) << (deviceIDLength +
measurementLength) | ((long) newDeviceNumber()) << measurementLength |
newMeasurementNumber());
}
- public int getStorageGroupID(long id) {
- return (int)(id >>> (deviceIDLength + measurementLength));
+ public static int getStorageGroupID(long fullId) {
+ return (int)(fullId >>> (deviceIDLength + measurementLength));
}
- public int getDeviceID(long id) {
- return (int)((id << storageGroupIDLength) >>> (storageGroupIDLength +
measurementLength));
+ public static int getDeviceID(long fullId) {
+ return (int)((fullId << storageGroupIDLength) >>> (storageGroupIDLength +
measurementLength));
}
- public int getMeasurementID(long id) {
- return (int)((id << storageGroupIDLength + deviceIDLength) >>>
(storageGroupIDLength + deviceIDLength));
+ public static int getMeasurementID(long fullId) {
+ return (int)((fullId << storageGroupIDLength + deviceIDLength) >>>
(storageGroupIDLength + deviceIDLength));
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/mnode/InternalMNode.java
b/server/src/main/java/org/apache/iotdb/db/metadata/mnode/InternalMNode.java
index 2d199cb..01fa329 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mnode/InternalMNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mnode/InternalMNode.java
@@ -56,6 +56,7 @@ public class InternalMNode extends MNode {
/**
* If delete a leafMNode, lock its parent, if delete an InternalNode, lock
itself
+ * @throws DeleteFailedException if the method can not get the lock (e.g.,
someone is writing data.)
*/
@Override
public void deleteChild(String name) throws DeleteFailedException {
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/mnode/LeafMNode.java
b/server/src/main/java/org/apache/iotdb/db/metadata/mnode/LeafMNode.java
index bb64a10..403be60 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mnode/LeafMNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mnode/LeafMNode.java
@@ -41,6 +41,8 @@ public class LeafMNode extends MNode {
private TimeValuePair cachedLastValuePair = null;
+ private int id;
+
/**
* @param alias alias of measurementName
*/
@@ -140,4 +142,13 @@ public class LeafMNode extends MNode {
public void setAlias(String alias) {
this.alias = alias;
}
+
+
+ public int getId() {
+ return id;
+ }
+
+ public void setId(int id) {
+ this.id = id;
+ }
}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mnode/MNode.java
b/server/src/main/java/org/apache/iotdb/db/metadata/mnode/MNode.java
index 181c309..b4a2a6c 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mnode/MNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mnode/MNode.java
@@ -65,11 +65,13 @@ public abstract class MNode implements Serializable {
/**
* delete a child
+ * @throws DeleteFailedException if the child is used.
*/
public abstract void deleteChild(String name) throws DeleteFailedException;
/**
* delete the alias of a child
+ * @throws DeleteFailedException if the child is used.
*/
public abstract void deleteAliasChild(String alias) throws
DeleteFailedException;
@@ -118,6 +120,10 @@ public abstract class MNode implements Serializable {
return parent;
}
+ public void setParent(MNode node) {
+ this.parent = node;
+ }
+
public abstract Map<String, MNode> getChildren();
public String getName() {
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/mnode/StorageGroupMNode.java
b/server/src/main/java/org/apache/iotdb/db/metadata/mnode/StorageGroupMNode.java
index a122d95..e7492e3 100644
---
a/server/src/main/java/org/apache/iotdb/db/metadata/mnode/StorageGroupMNode.java
+++
b/server/src/main/java/org/apache/iotdb/db/metadata/mnode/StorageGroupMNode.java
@@ -28,6 +28,8 @@ public class StorageGroupMNode extends InternalMNode {
*/
private long dataTTL;
+ private int id;
+
public StorageGroupMNode(MNode parent, String name, String fullPath, long
dataTTL) {
super(parent, name);
@@ -43,4 +45,12 @@ public class StorageGroupMNode extends InternalMNode {
this.dataTTL = dataTTL;
}
+ public int getId() {
+ return id;
+ }
+
+ public void setId(int id) {
+ this.id = id;
+ }
+
}
\ No newline at end of file
diff --git
a/server/src/test/java/org/apache/iotdb/db/metadata/IDManagerTest.java
b/server/src/test/java/org/apache/iotdb/db/metadata/IDManagerTest.java
index 8bba51d..ac94cc6 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/IDManagerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/IDManagerTest.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.metadata;
import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.db.metadata.id.IDManager;
import org.junit.Assert;
import org.junit.Test;
