This is an automated email from the ASF dual-hosted git repository.
geniuspig pushed a commit to branch optimize_path
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/optimize_path by this push:
new 0e5b4f9 add method
0e5b4f9 is described below
commit 0e5b4f90a46db708b9aa6bdc8a712160c33fc2af
Author: zhutianci <[email protected]>
AuthorDate: Tue Jul 14 16:41:43 2020 +0800
add method
---
.../org/apache/iotdb/db/metadata/MManager.java | 94 ++++++++++-----
.../java/org/apache/iotdb/db/metadata/MTree.java | 132 +++++++++++++++++++++
.../org/apache/iotdb/db/metadata/MetaUtils.java | 24 ++++
.../iotdb/db/qp/physical/crud/InsertRowPlan.java | 7 +-
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 2 +-
.../apache/iotdb/db/utils/RandomDeleteCache.java | 17 ++-
6 files changed, 232 insertions(+), 44 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 75cf379..5232485 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -143,20 +143,7 @@ public class MManager {
isRecovering = true;
int cacheSize = config.getmManagerCacheSize();
- mNodeCache = new RandomDeleteCache<String, MNode>(cacheSize) {
-
- @Override
- public MNode loadObjectByKey(String key) throws CacheException {
- lock.readLock().lock();
- try {
- return mtree.getNodeByPathWithStorageGroupCheck(key);
- } catch (MetadataException e) {
- throw new CacheException(e);
- } finally {
- lock.readLock().unlock();
- }
- }
- };
+ mNodeCache = new RandomDeleteCache<String, MNode>(cacheSize) {};
int remoteCacheSize = config.getmRemoteSchemaCacheSize();
mRemoteSchemaCache = new LRUCache<String,
MeasurementSchema>(remoteCacheSize) {
@@ -1084,48 +1071,86 @@ public class MManager {
* @param path path
*/
public MNode getDeviceNodeWithAutoCreateAndReadLock(
- String path, boolean autoCreateSchema, int sgLevel) throws
MetadataException {
+ String path, List<String> nodes, boolean autoCreateSchema, int sgLevel)
throws MetadataException {
lock.readLock().lock();
MNode node = null;
- boolean shouldSetStorageGroup;
try {
node = mNodeCache.get(path);
+ if (node == null) {
+ node = mtree.getNodeByNodesWithStorageGroupCheck(nodes);
+ if (path == null) {
+ mNodeCache.put(node.getFullPath(), node);
+ } else {
+ mNodeCache.put(path, node);
+ }
+ }
return node;
- } catch (CacheException e) {
+ } catch (Exception e) {
if (!autoCreateSchema) {
throw new PathNotExistException(path);
}
+ if (e.getCause() instanceof StorageGroupNotSetException) {
+ String storageGroupName = MetaUtils.getStorageGroupNameByLevel(path,
sgLevel);
+ setStorageGroup(storageGroupName);
+ node = mtree.getDeviceNodeWithAutoCreating(path, sgLevel);
+ return node;
+ }
+ if (e.getCause() instanceof StorageGroupAlreadySetException) {
+ node = mtree.getDeviceNodeWithAutoCreating(path, sgLevel);
+ return node;
+ }
+ return node;
} finally {
if (node != null) {
node.readLock();
}
lock.readLock().unlock();
}
+ }
- lock.writeLock().lock();
+ /**
+ * get device node, if the storage group is not set, create it when
autoCreateSchema is true <p>
+ * (we develop this method as we need to get the node's lock after we get
the lock.writeLock())
+ *
+ * <p>!!!!!!Attention!!!!! must call the return node's readUnlock() if you
call this method.
+ *
+ * @param path path
+ */
+ public MNode getDeviceNodeWithAutoCreateAndReadLock(
+ String path, boolean autoCreateSchema, int sgLevel) throws
MetadataException {
+ lock.readLock().lock();
+ MNode node = null;
try {
- try {
- node = mNodeCache.get(path);
- return node;
- } catch (CacheException e) {
- shouldSetStorageGroup = e.getCause() instanceof
StorageGroupNotSetException;
+ node = mNodeCache.get(path);
+ if (node == null) {
+ node = mtree.getNodeByPathWithStorageGroupCheck(path);
+ if (path == null) {
+ mNodeCache.put(node.getFullPath(), node);
+ } else {
+ mNodeCache.put(path, node);
+ }
}
-
- if (shouldSetStorageGroup) {
+ return node;
+ } catch (Exception e) {
+ if (!autoCreateSchema) {
+ throw new PathNotExistException(path);
+ }
+ if (e.getCause() instanceof StorageGroupNotSetException) {
String storageGroupName = MetaUtils.getStorageGroupNameByLevel(path,
sgLevel);
setStorageGroup(storageGroupName);
+ node = mtree.getDeviceNodeWithAutoCreating(path, sgLevel);
+ return node;
+ }
+ if (e.getCause() instanceof StorageGroupAlreadySetException) {
+ node = mtree.getDeviceNodeWithAutoCreating(path, sgLevel);
+ return node;
}
- node = mtree.getDeviceNodeWithAutoCreating(path, sgLevel);
- return node;
- } catch (StorageGroupAlreadySetException e) {
- // ignore set storage group concurrently
- node = mtree.getDeviceNodeWithAutoCreating(path, sgLevel);
return node;
} finally {
if (node != null) {
node.readLock();
}
- lock.writeLock().unlock();
+ lock.readLock().unlock();
}
}
@@ -1137,13 +1162,18 @@ public class MManager {
path, config.isAutoCreateSchemaEnabled(),
config.getDefaultStorageGroupLevel());
}
+ public MNode getDeviceNodeWithAutoCreateAndReadLock(String path,
List<String> nodes) throws MetadataException {
+ return getDeviceNodeWithAutoCreateAndReadLock(
+ path, nodes, config.isAutoCreateSchemaEnabled(),
config.getDefaultStorageGroupLevel());
+ }
+
public MNode getDeviceNode(String path) throws MetadataException {
lock.readLock().lock();
MNode node;
try {
node = mNodeCache.get(path);
return node;
- } catch (CacheException e) {
+ } catch (Exception e) {
throw new PathNotExistException(path);
} finally {
lock.readLock().unlock();
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
index d3921f7..e132c07 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
@@ -237,6 +237,40 @@ public class MTree implements Serializable {
}
/**
+ * Set storage group. Make sure check seriesPath before setting storage group
+ *
+ * @param nodeNames nodeNames
+ */
+ void setStorageGroup(List<String> nodeNames) throws MetadataException {
+ MNode cur = root;
+ if (nodeNames.size() <= 1 || !nodeNames.get(0).equals(root.getName())) {
+ throw new IllegalPathException(nodeNames.toString());
+ }
+ int i = 1;
+ // e.g., path = root.a.b.sg, create internal nodes for a, b
+ while (i < nodeNames.size() - 1) {
+ MNode temp = cur.getChild(nodeNames.get(i));
+ if (temp == null) {
+ cur.addChild(nodeNames.get(i), new MNode(cur, nodeNames.get(i)));
+ } else if (temp instanceof StorageGroupMNode) {
+ // before set storage group, check whether the exists or not
+ throw new StorageGroupAlreadySetException(temp.getFullPath());
+ }
+ cur = cur.getChild(nodeNames.get(i));
+ i++;
+ }
+ if (cur.hasChild(nodeNames.get(i))) {
+ // node b has child sg
+ throw new StorageGroupAlreadySetException(nodeNames.toString());
+ } else {
+ StorageGroupMNode storageGroupMNode =
+ new StorageGroupMNode(
+ cur, nodeNames.get(i),
IoTDBDescriptor.getInstance().getConfig().getDefaultTTL());
+ cur.addChild(nodeNames.get(i), storageGroupMNode);
+ }
+ }
+
+ /**
* Delete a storage group
*/
List<MeasurementMNode> deleteStorageGroup(String path) throws
MetadataException {
@@ -273,6 +307,42 @@ public class MTree implements Serializable {
}
/**
+ * Delete a storage group
+ */
+ List<MeasurementMNode> deleteStorageGroup(List<String> nodes) throws
MetadataException {
+ MNode cur = getNodeByNodes(nodes);
+ if (!(cur instanceof StorageGroupMNode)) {
+ throw new StorageGroupNotSetException(nodes.toString());
+ }
+ // Suppose current system has root.a.b.sg1, root.a.sg2, and delete
root.a.b.sg1
+ // delete the storage group node sg1
+ cur.getParent().deleteChild(cur.getName());
+
+ // collect all the LeafMNode in this storage group
+ List<MeasurementMNode> leafMNodes = new LinkedList<>();
+ Queue<MNode> queue = new LinkedList<>();
+ queue.add(cur);
+ while (!queue.isEmpty()) {
+ MNode node = queue.poll();
+ for (MNode child : node.getChildren().values()) {
+ if (child instanceof MeasurementMNode) {
+ leafMNodes.add((MeasurementMNode) child);
+ } else {
+ queue.add(child);
+ }
+ }
+ }
+
+ cur = cur.getParent();
+ // delete node b while retain root.a.sg2
+ while (!IoTDBConstant.PATH_ROOT.equals(cur.getName()) &&
cur.getChildren().size() == 0) {
+ cur.getParent().deleteChild(cur.getName());
+ cur = cur.getParent();
+ }
+ return leafMNodes;
+ }
+
+ /**
* Check whether path is storage group or not
*
* <p>e.g., path = root.a.b.sg. if nor a and b is StorageGroupMNode and sg
is a StorageGroupMNode
@@ -375,6 +445,37 @@ public class MTree implements Serializable {
}
/**
+ * Get node by path with storage group check If storage group is not set,
+ * StorageGroupNotSetException will be thrown
+ */
+ MNode getNodeByNodesWithStorageGroupCheck(List<String> nodes) throws
MetadataException {
+ boolean storageGroupChecked = false;
+ if (nodes.isEmpty() || !nodes.get(0).equals(root.getName())) {
+ throw new IllegalPathException(nodes.toString());
+ }
+
+ MNode cur = root;
+ for (int i = 1; i < nodes.size(); i++) {
+ if (!cur.hasChild(nodes.get(i))) {
+ if (!storageGroupChecked) {
+ throw new StorageGroupNotSetException(nodes.toString());
+ }
+ throw new PathNotExistException(nodes.toString());
+ }
+ cur = cur.getChild(nodes.get(i));
+
+ if (cur instanceof StorageGroupMNode) {
+ storageGroupChecked = true;
+ }
+ }
+
+ if (!storageGroupChecked) {
+ throw new StorageGroupNotSetException(nodes.toString());
+ }
+ return cur;
+ }
+
+ /**
* Get storage group node, if the give path is not a storage group, throw
exception
*/
StorageGroupMNode getStorageGroupNode(String path) throws MetadataException {
@@ -387,6 +488,18 @@ public class MTree implements Serializable {
}
/**
+ * Get storage group node, if the give path is not a storage group, throw
exception
+ */
+ StorageGroupMNode getStorageGroupNode(List<String> nodes) throws
MetadataException {
+ MNode node = getNodeByNodes(nodes);
+ if (node instanceof StorageGroupMNode) {
+ return (StorageGroupMNode) node;
+ } else {
+ throw new StorageGroupNotSetException(nodes.toString());
+ }
+ }
+
+ /**
* Get node by the path
*
* @return last node in given seriesPath
@@ -407,6 +520,25 @@ public class MTree implements Serializable {
}
/**
+ * Get node by the path
+ *
+ * @return last node in given seriesPath
+ */
+ MNode getNodeByNodes(List<String> nodes) throws MetadataException {
+ if (nodes.isEmpty() || !nodes.get(0).equals(root.getName())) {
+ throw new IllegalPathException(nodes.toString());
+ }
+ MNode cur = root;
+ for (int i = 1; i < nodes.size(); i++) {
+ if (!cur.hasChild(nodes.get(i))) {
+ throw new PathNotExistException(nodes.toString());
+ }
+ cur = cur.getChild(nodes.get(i));
+ }
+ return cur;
+ }
+
+ /**
* Get all storage groups under the given path
*
* @return storage group list
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MetaUtils.java
b/server/src/main/java/org/apache/iotdb/db/metadata/MetaUtils.java
index 05f7374..7108285 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MetaUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MetaUtils.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.metadata;
import static org.apache.iotdb.db.conf.IoTDBConstant.PATH_WILDCARD;
import java.util.Arrays;
+import java.util.List;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
@@ -60,6 +61,10 @@ public class MetaUtils {
return idx >= nodes.length ? PATH_WILDCARD : nodes[idx];
}
+ public static List<String> getDeviceNodeNames(String path) {
+ return Arrays.asList(path.split(PATH_SEPARATOR));
+ }
+
/**
* Get storage group name when creating schema automatically is enable
*
@@ -79,4 +84,23 @@ public class MetaUtils {
}
return storageGroupName.toString();
}
+
+ /**
+ * Get storage group name when creating schema automatically is enable
+ *
+ * e.g., nodes = [root, a, b, c] and level = 1, return [root, a]
+ *
+ * @param nodeNames nodeNames
+ * @param level level
+ */
+ public static List<String> getStorageGroupNodesByLevel(List<String>
nodeNames, int level) throws MetadataException {
+ if (nodeNames.size() <= level ||
!nodeNames.get(0).equals(IoTDBConstant.PATH_ROOT)) {
+ throw new IllegalPathException(nodeNames.toString());
+ }
+ for(int i = level + 1; nodeNames.size() > level; i--) {
+ nodeNames.remove(i);
+ }
+ return nodeNames;
+ }
+
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowPlan.java
b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowPlan.java
index 0bef4f8..1d1e584 100644
---
a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowPlan.java
+++
b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowPlan.java
@@ -51,6 +51,7 @@ public class InsertRowPlan extends InsertPlan {
private long time;
private Object[] values;
+ private List<String> deviceNodes;
// if isNeedInferType is true, the values must be String[], so we could
infer types from them
// if values is object[], we could use the raw type of them, and we should
set this to false
@@ -111,20 +112,22 @@ public class InsertRowPlan extends InsertPlan {
}
}
- public InsertRowPlan(String deviceId, long insertTime, String[]
measurementList,
+ public InsertRowPlan(List<String> deviceNodes, String deviceId, long
insertTime, String[] measurementList,
TSDataType[] dataTypes, Object[] insertValues) {
super(Operator.OperatorType.INSERT);
this.time = insertTime;
+ this.deviceNodes = deviceNodes;
this.deviceId = deviceId;
this.measurements = measurementList;
this.dataTypes = dataTypes;
this.values = insertValues;
}
- public InsertRowPlan(String deviceId, long insertTime, String[]
measurementList,
+ public InsertRowPlan(List<String> deviceNodes, String deviceId, long
insertTime, String[] measurementList,
String[] insertValues) {
super(Operator.OperatorType.INSERT);
this.time = insertTime;
+ this.deviceNodes = deviceNodes;
this.deviceId = deviceId;
this.measurements = measurementList;
this.dataTypes = new TSDataType[measurements.length];
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
index efce1a3..7c9272c 100644
---
a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
+++
b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
@@ -177,7 +177,7 @@ public class PhysicalGenerator {
"For Insert command, cannot specified more than one seriesPath:
" + paths);
}
- return new InsertRowPlan(paths.get(0).getFullPath(), insert.getTime(),
+ return new InsertRowPlan(paths.get(0).getNodes(), insert.getTime(),
insert.getMeasurementList(), insert.getValueList());
case MERGE:
if (operator.getTokenIntType() == SQLConstant.TOK_FULL_MERGE) {
diff --git
a/server/src/main/java/org/apache/iotdb/db/utils/RandomDeleteCache.java
b/server/src/main/java/org/apache/iotdb/db/utils/RandomDeleteCache.java
index 90921ce..7ae70d6 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/RandomDeleteCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/RandomDeleteCache.java
@@ -28,20 +28,21 @@ public abstract class RandomDeleteCache<K, V> implements
Cache<K, V> {
private int cacheSize;
private Map<K, V> cache;
- public RandomDeleteCache(int cacheSize) {
+ protected RandomDeleteCache(int cacheSize) {
this.cacheSize = cacheSize;
this.cache = new ConcurrentHashMap<>();
}
@Override
- public V get(K key) throws CacheException {
- V v = cache.get(key);
- if (v == null) {
+ public V get(K key) {
+ return cache.get(key);
+ }
+
+ public void put(K key, V value) {
+ cache.put(key, value);
+ if(cache.get(key) == null) {
randomRemoveObjectIfCacheIsFull();
- cache.put(key, loadObjectByKey(key));
- v = cache.get(key);
}
- return v;
}
private void randomRemoveObjectIfCacheIsFull() {
@@ -58,8 +59,6 @@ public abstract class RandomDeleteCache<K, V> implements
Cache<K, V> {
cache.remove(key);
}
- public abstract V loadObjectByKey(K key) throws CacheException;
-
public void removeObject(K key) {
cache.remove(key);
}
