This is an automated email from the ASF dual-hosted git repository. sunzesong pushed a commit to branch jira_444 in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 447a648093e16888a68908857d743b6f2a4e650d Author: samperson1997 <[email protected]> AuthorDate: Tue Feb 25 19:32:22 2020 +0800 [IOTDB-444] Enhanced wildcard query on device path --- .../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 6 +- .../org/apache/iotdb/db/metadata/MManager.java | 2 +- .../java/org/apache/iotdb/db/metadata/MTree.java | 71 ++++++++++++++++++---- .../apache/iotdb/db/qp/executor/PlanExecutor.java | 3 +- .../iotdb/db/qp/strategy/PhysicalGenerator.java | 7 +-- .../iotdb/db/integration/IoTDBAlignByDeviceIT.java | 4 +- .../iotdb/db/metadata/MManagerAdvancedTest.java | 14 +++-- .../iotdb/db/metadata/MManagerBasicTest.java | 57 ++++++++++++++++- 8 files changed, 133 insertions(+), 31 deletions(-) diff --git a/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4 b/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4 index 1a8558f..6a92ea8 100644 --- a/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4 +++ b/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4 @@ -31,11 +31,6 @@ statement | DELETE FROM prefixPath (COMMA prefixPath)* (whereClause)? #deleteStatement | SET STORAGE GROUP TO prefixPath #setStorageGroup | DELETE STORAGE GROUP prefixPath (COMMA prefixPath)* #deleteStorageGroup - | CREATE PROPERTY ID #createProperty - | ADD LABEL label=ID TO PROPERTY propertyName=ID #addLabel - | DELETE LABEL label=ID FROM PROPERTY propertyName=ID #deleteLabel - | LINK prefixPath TO propertyLabelPair #linkPath - | UNLINK prefixPath FROM propertyLabelPair #unlinkPath | SHOW METADATA #showMetadata // not support yet | DESCRIBE prefixPath #describePath // not support yet | CREATE INDEX ON timeseriesPath USING function=ID indexWithClause? whereClause? #createIndex //not support yet @@ -292,6 +287,7 @@ nodeName : ID | INT | STAR + | ID STAR | STRING_LITERAL ; diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java index 649a267..0ef29eb 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java @@ -558,7 +558,7 @@ public class MManager { * wildcard can only match one level, otherwise it can match to the tail. * @return A HashSet instance which stores devices names with given prefixPath. */ - public List<String> getDevices(String prefixPath) throws MetadataException { + public Set<String> getDevices(String prefixPath) throws MetadataException { lock.readLock().lock(); try { return mtree.getDevices(prefixPath); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java index 9be1476..a56e52c 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java @@ -33,6 +33,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.metadata.IllegalPathException; @@ -503,13 +504,20 @@ public class MTree implements Serializable { return; } String nodeReg = MetaUtils.getNodeRegByIdx(idx, nodes); - if (!(PATH_WILDCARD).equals(nodeReg)) { + if (!nodeReg.contains(PATH_WILDCARD)) { if (node.hasChild(nodeReg)) { findPath(node.getChild(nodeReg), nodes, idx + 1, parent + node.getName() + PATH_SEPARATOR, timeseriesSchemaList); } } else { for (MNode child : node.getChildren().values()) { + String prefix = ""; + if (!nodeReg.equals(PATH_WILDCARD)) { + prefix = nodeReg.split("\\*")[0]; + } + if (!child.getName().startsWith(prefix)) { + return; + } findPath(child, nodes, idx + 1, parent + node.getName() + PATH_SEPARATOR, timeseriesSchemaList); } @@ -525,17 +533,56 @@ public class MTree implements Serializable { * @return All child nodes' seriesPath(s) of given seriesPath. */ Set<String> getChildNodePathInNextLevel(String path) throws MetadataException { - MNode node = getNodeByPath(path); - - if (node instanceof LeafMNode) { - return new HashSet<>(); + String[] nodes = MetaUtils.getNodeNames(path); + if (nodes.length == 0 || !nodes[0].equals(root.getName())) { + throw new IllegalPathException(path); } + Set<String> childNodePaths = new TreeSet<>(); + findChildNodePathInNextLevel(root, nodes, 1, "", childNodePaths, nodes.length + 1); + return childNodePaths; + } - Set<String> childPaths = new HashSet<>(); - for (MNode child : node.getChildren().values()) { - childPaths.add(path + PATH_SEPARATOR + child.getName()); + /** + * Traverse the MTree to match all child node path in next level + * + * @param node the current traversing node + * @param nodes split the prefix path with '.' + * @param idx the current index of array nodes + * @param parent store the node string having traversed + * @param res store all matched device names + * @param length expected length of path + */ + private void findChildNodePathInNextLevel(MNode node, String[] nodes, int idx, String parent, + Set<String> res, int length) { + String nodeReg = MetaUtils.getNodeRegByIdx(idx, nodes); + if (!nodeReg.contains(PATH_WILDCARD)) { + if (idx == length) { + res.add(parent + node.getName()); + } else { + findChildNodePathInNextLevel(node.getChild(nodeReg), nodes, idx + 1, + parent + node.getName() + PATH_SEPARATOR, res, length); + } + } else { + if (node instanceof InternalMNode) { + for (MNode child : node.getChildren().values()) { + String prefix = ""; + if (!nodeReg.equals(PATH_WILDCARD)) { + prefix = nodeReg.split("\\*")[0]; + } + if (!child.getName().startsWith(prefix)) { + return; + } + if (idx == length) { + res.add(parent + node.getName()); + } else { + findChildNodePathInNextLevel(child, nodes, idx + 1, + parent + node.getName() + PATH_SEPARATOR, res, length); + } + } + } else if (idx == length) { + res.add(parent + node.getName()); + } } - return childPaths; } /** @@ -543,12 +590,12 @@ public class MTree implements Serializable { * * @return a list contains all distinct devices names */ - List<String> getDevices(String prefixPath) throws MetadataException { + Set<String> getDevices(String prefixPath) throws MetadataException { String[] nodes = MetaUtils.getNodeNames(prefixPath); if (nodes.length == 0 || !nodes[0].equals(root.getName())) { throw new IllegalPathException(prefixPath); } - List<String> devices = new ArrayList<>(); + Set<String> devices = new TreeSet<>(); findDevices(root, nodes, 1, "", devices); return devices; } @@ -562,7 +609,7 @@ public class MTree implements Serializable { * @param parent store the node string having traversed * @param res store all matched device names */ - private void findDevices(MNode node, String[] nodes, int idx, String parent, List<String> res) { + private void findDevices(MNode node, String[] nodes, int idx, String parent, Set<String> res) { String nodeReg = MetaUtils.getNodeRegByIdx(idx, nodes); if (!(PATH_WILDCARD).equals(nodeReg)) { if (node.hasChild(nodeReg)) { diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java index 2cf135d..8956d4c 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java @@ -333,8 +333,7 @@ public class PlanExecutor implements IPlanExecutor { throws MetadataException { ListDataSet listDataSet = new ListDataSet(Collections.singletonList(new Path(COLUMN_DEVICES)), Collections.singletonList(TSDataType.TEXT)); - List<String> devices; - devices = MManager.getInstance().getDevices(showDevicesPlan.getPath().toString()); + Set<String> devices = MManager.getInstance().getDevices(showDevicesPlan.getPath().toString()); for (String s : devices) { RowRecord record = new RowRecord(0); Field field = new Field(TSDataType.TEXT); diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java index ee44e56..ea1897a 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java @@ -281,8 +281,9 @@ public class PhysicalGenerator { for (String device : devices) { // per device in FROM after deduplication Path fullPath = Path.addPrefixPath(suffixPath, device); try { + // remove stars in SELECT to get actual paths List<String> actualPaths = MManager.getInstance() - .getAllTimeseriesName(fullPath.getFullPath()); // remove stars in SELECT to get actual paths + .getAllTimeseriesName(fullPath.getFullPath()); // for actual non exist path if (actualPaths.isEmpty() && originAggregations.isEmpty()) { @@ -426,9 +427,7 @@ public class PhysicalGenerator { Set<String> deviceSet = new LinkedHashSet<>(); try { for (Path path : paths) { - List<String> tempDS; - tempDS = MManager.getInstance().getDevices(path.getFullPath()); - + Set<String> tempDS = MManager.getInstance().getDevices(path.getFullPath()); deviceSet.addAll(tempDS); } retDevices = new ArrayList<>(deviceSet); diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAlignByDeviceIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAlignByDeviceIT.java index fb7c396..86aadaf 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAlignByDeviceIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAlignByDeviceIT.java @@ -595,9 +595,9 @@ public class IoTDBAlignByDeviceIT { @Test public void unusualCaseTest1() throws ClassNotFoundException { String[] retArray = new String[]{ + "root.other.d1,1,", "root.vehicle.d0,11,", - "root.vehicle.d1,2,", - "root.other.d1,1," + "root.vehicle.d1,2," }; Class.forName(Config.JDBC_DRIVER_NAME); diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerAdvancedTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerAdvancedTest.java index 33962ae..3b90be8 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerAdvancedTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerAdvancedTest.java @@ -84,10 +84,16 @@ public class MManagerAdvancedTest { assertEquals("root.vehicle.d0", mmanager.getStorageGroupName("root.vehicle.d0.s1")); List<String> pathList = mmanager.getAllTimeseriesName("root.vehicle.d1.*"); assertEquals(6, pathList.size()); - List<String> paths = mmanager.getAllTimeseriesName("root.vehicle.d0"); - assertEquals(6, paths.size()); - paths = mmanager.getAllTimeseriesName("root.vehicle.d2"); - assertEquals(0, paths.size()); + pathList = mmanager.getAllTimeseriesName("root.vehicle.d0"); + assertEquals(6, pathList.size()); + pathList = mmanager.getAllTimeseriesName("root.vehicle.d*"); + assertEquals(12, pathList.size()); + pathList = mmanager.getAllTimeseriesName("root.ve*.*"); + assertEquals(12, pathList.size()); + pathList = mmanager.getAllTimeseriesName("root.vehicle*.d*.s1"); + assertEquals(2, pathList.size()); + pathList = mmanager.getAllTimeseriesName("root.vehicle.d2"); + assertEquals(0, pathList.size()); } catch (MetadataException e) { e.printStackTrace(); fail(e.getMessage()); diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java index 819e0c5..da87dfa 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java @@ -26,6 +26,8 @@ import static org.junit.Assert.fail; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Set; +import java.util.TreeSet; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.metadata.MetadataException; import org.apache.iotdb.db.exception.query.PathException; @@ -303,7 +305,7 @@ public class MManagerBasicTest { CompressionType.GZIP, null); manager.createTimeseries("root.laptop.d2.s1", TSDataType.INT32, TSEncoding.PLAIN, CompressionType.GZIP, null); - List<String> devices = new ArrayList<>(); + Set<String> devices = new TreeSet<>(); devices.add("root.laptop.d1"); devices.add("root.laptop.d2"); // usual condition @@ -319,4 +321,57 @@ public class MManagerBasicTest { fail(e.getMessage()); } } + + @Test + public void testGetChildNodePathInNextLevel() { + MManager manager = MManager.getInstance(); + String[] res = new String[]{ + "[root.laptop, root.vehicle]", + "[root.laptop.b1, root.laptop.b2]", + "[root.laptop.b1.d1, root.laptop.b1.d2]", + "[root.laptop.b1, root.laptop.b2, root.vehicle.b1, root.vehicle.b2]", + "[root.laptop.b1.d1, root.laptop.b1.d2, root.vehicle.b1.d0, root.vehicle.b1.d2, root.vehicle.b1.d3]", + "[root.laptop.b1.d1, root.laptop.b1.d2]", + "[root.laptop.b1.d1, root.laptop.b1.d2, root.laptop.b2.d1, root.laptop.b2.d2]", + "[root.laptop.b1.d1.s0, root.laptop.b1.d1.s1, root.laptop.b1.d2.s0, root.laptop.b2.d1.s1, root.laptop.b2.d1.s3, root.laptop.b2.d2.s2]" + }; + + try { + manager.setStorageGroup("root.laptop"); + manager.setStorageGroup("root.vehicle"); + + manager.createTimeseries("root.laptop.b1.d1.s0", TSDataType.INT32, TSEncoding.PLAIN, + CompressionType.GZIP, null); + manager.createTimeseries("root.laptop.b1.d1.s1", TSDataType.INT32, TSEncoding.PLAIN, + CompressionType.GZIP, null); + manager.createTimeseries("root.laptop.b1.d2.s0", TSDataType.INT32, TSEncoding.PLAIN, + CompressionType.GZIP, null); + manager.createTimeseries("root.laptop.b2.d1.s1", TSDataType.INT32, TSEncoding.PLAIN, + CompressionType.GZIP, null); + manager.createTimeseries("root.laptop.b2.d1.s3", TSDataType.INT32, TSEncoding.PLAIN, + CompressionType.GZIP, null); + manager.createTimeseries("root.laptop.b2.d2.s2", TSDataType.INT32, TSEncoding.PLAIN, + CompressionType.GZIP, null); + manager.createTimeseries("root.vehicle.b1.d0.s0", TSDataType.INT32, TSEncoding.PLAIN, + CompressionType.GZIP, null); + manager.createTimeseries("root.vehicle.b2.d0.s1", TSDataType.INT32, TSEncoding.PLAIN, + CompressionType.GZIP, null); + manager.createTimeseries("root.vehicle.b1.d2.s2", TSDataType.INT32, TSEncoding.PLAIN, + CompressionType.GZIP, null); + manager.createTimeseries("root.vehicle.b1.d3.s3", TSDataType.INT32, TSEncoding.PLAIN, + CompressionType.GZIP, null); + + assertEquals(res[0], manager.getChildNodePathInNextLevel("root").toString()); + assertEquals(res[1], manager.getChildNodePathInNextLevel("root.laptop").toString()); + assertEquals(res[2], manager.getChildNodePathInNextLevel("root.laptop.b1").toString()); + assertEquals(res[3], manager.getChildNodePathInNextLevel("root.*").toString()); + assertEquals(res[4], manager.getChildNodePathInNextLevel("root.*.b1").toString()); + assertEquals(res[5], manager.getChildNodePathInNextLevel("root.l*.b1").toString()); + assertEquals(res[6], manager.getChildNodePathInNextLevel("root.l*.*").toString()); + assertEquals(res[7], manager.getChildNodePathInNextLevel("root.l*.b*.*").toString()); + } catch (MetadataException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } }
