This is an automated email from the ASF dual-hosted git repository.
chenyz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 5ad07424552 Optimize fetch template schema (#11566)
5ad07424552 is described below
commit 5ad07424552d5e8df80f72b763f71f7c155142ae
Author: Chen YZ <[email protected]>
AuthorDate: Sat Nov 18 22:05:07 2023 +0800
Optimize fetch template schema (#11566)
Optimize fetch template schema
---
.../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 22 ++++----------
.../mnode/iterator/AbstractTraverserIterator.java | 11 ++++++-
.../impl/mem/mnode/iterator/MNodeIterator.java | 5 ++++
.../mtree/impl/pbtree/CachedMTreeStore.java | 5 ++++
.../mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 22 ++++----------
.../schemaregion/mtree/traverser/Traverser.java | 25 +++++++++++++++-
.../commons/schema/node/utils/IMNodeIterator.java | 2 ++
.../commons/schema/tree/AbstractTreeVisitor.java | 35 ++++++++++++++++++++++
8 files changed, 93 insertions(+), 34 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
index 04e0a5753a4..b561dbf058f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
@@ -704,18 +704,13 @@ public class MTreeBelowSGMemoryImpl {
try (MeasurementCollector<Void, IMemMNode> collector =
new MeasurementCollector<Void, IMemMNode>(
rootNode, pathPattern, store, false,
SchemaConstant.ALL_MATCH_SCOPE) {
- private final Set<PartialPath> visitedTemplateDevice = new
HashSet<>();
-
protected Void collectMeasurement(IMeasurementMNode<IMemMNode> node)
{
IDeviceMNode<IMemMNode> deviceMNode =
getParentOfNextMatchedNode().getAsDeviceMNode();
int templateId = deviceMNode.getSchemaTemplateIdWithState();
- PartialPath devicePath = deviceMNode.getPartialPath();
if (templateId >= 0) {
- if (!visitedTemplateDevice.contains(devicePath)) {
- schemaTree.appendTemplateDevice(
- deviceMNode.getPartialPath(), deviceMNode.isAligned(),
templateId, null);
- visitedTemplateDevice.add(devicePath);
- }
+ schemaTree.appendTemplateDevice(
+ deviceMNode.getPartialPath(), deviceMNode.isAligned(),
templateId, null);
+ skipTemplateChildren(deviceMNode);
} else {
MeasurementPath path = getCurrentMeasurementPathInTraverse(node);
if (nodes[nodes.length - 1].equals(node.getAlias())) {
@@ -744,18 +739,13 @@ public class MTreeBelowSGMemoryImpl {
try (MeasurementCollector<Void, IMemMNode> collector =
new MeasurementCollector<Void, IMemMNode>(
rootNode, patternTree, store, SchemaConstant.ALL_MATCH_SCOPE) {
- private final Set<PartialPath> visitedTemplateDevice = new
HashSet<>();
-
protected Void collectMeasurement(IMeasurementMNode<IMemMNode> node)
{
IDeviceMNode<IMemMNode> deviceMNode =
getParentOfNextMatchedNode().getAsDeviceMNode();
int templateId = deviceMNode.getSchemaTemplateIdWithState();
- PartialPath devicePath = deviceMNode.getPartialPath();
if (templateId >= 0) {
- if (!visitedTemplateDevice.contains(devicePath)) {
- schemaTree.appendTemplateDevice(
- deviceMNode.getPartialPath(), deviceMNode.isAligned(),
templateId, null);
- visitedTemplateDevice.add(devicePath);
- }
+ schemaTree.appendTemplateDevice(
+ deviceMNode.getPartialPath(), deviceMNode.isAligned(),
templateId, null);
+ skipTemplateChildren(deviceMNode);
} else {
MeasurementPath path = getCurrentMeasurementPathInTraverse(node);
path.setMeasurementAlias(node.getAlias());
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/AbstractTraverserIterator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/AbstractTraverserIterator.java
index 024bec82a31..f7f6d7d1ae6 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/AbstractTraverserIterator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/AbstractTraverserIterator.java
@@ -46,6 +46,8 @@ public abstract class AbstractTraverserIterator<N extends
IMNode<N>> implements
// if true, the pre deleted measurement or pre deactivated template won't be
processed
private boolean skipPreDeletedSchema = false;
+ private boolean skipTemplateChildren = false;
+
protected AbstractTraverserIterator(
IMTreeStore<N> store,
IDeviceMNode<N> parent,
@@ -65,6 +67,11 @@ public abstract class AbstractTraverserIterator<N extends
IMNode<N>> implements
this.skipPreDeletedSchema = skipPreDeletedSchema;
}
+ @Override
+ public void skipTemplateChildren() {
+ skipTemplateChildren = true;
+ }
+
private Template getActivatedSchemaTemplate(
IDeviceMNode<N> node, Map<Integer, Template> templateMap) {
// new cluster, the used template is directly recorded as template id in
device mnode
@@ -89,7 +96,9 @@ public abstract class AbstractTraverserIterator<N extends
IMNode<N>> implements
if (directChildrenIterator.hasNext()) {
nextMatchedNode = directChildrenIterator.next();
usingDirectChildrenIterator = true;
- } else if (templateChildrenIterator != null &&
templateChildrenIterator.hasNext()) {
+ } else if (!skipTemplateChildren
+ && templateChildrenIterator != null
+ && templateChildrenIterator.hasNext()) {
nextMatchedNode = templateChildrenIterator.next();
usingDirectChildrenIterator = false;
} else {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/MNodeIterator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/MNodeIterator.java
index 569580f5cc1..55005d3d024 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/MNodeIterator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/MNodeIterator.java
@@ -42,6 +42,11 @@ public class MNodeIterator<N extends IMNode<?>> implements
IMNodeIterator<N> {
return iterator.next();
}
+ @Override
+ public void skipTemplateChildren() {
+ // do nothing
+ }
+
@Override
public void close() {
iterator = null;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/CachedMTreeStore.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/CachedMTreeStore.java
index d557777bfe2..2a0b2de01c5 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/CachedMTreeStore.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/CachedMTreeStore.java
@@ -736,6 +736,11 @@ public class CachedMTreeStore implements
IMTreeStore<ICachedMNode> {
isIteratingDisk = false;
}
+ @Override
+ public void skipTemplateChildren() {
+ // do nothing
+ }
+
@Override
public void close() {
try {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java
index 30d650960f4..4347292e3e4 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java
@@ -818,19 +818,14 @@ public class MTreeBelowSGCachedImpl {
try (MeasurementCollector<Void, ICachedMNode> collector =
new MeasurementCollector<Void, ICachedMNode>(
rootNode, pathPattern, store, false,
SchemaConstant.ALL_MATCH_SCOPE) {
- private final Set<PartialPath> visitedTemplateDevice = new
HashSet<>();
-
protected Void collectMeasurement(IMeasurementMNode<ICachedMNode>
node) {
IDeviceMNode<ICachedMNode> deviceMNode =
getParentOfNextMatchedNode().getAsDeviceMNode();
int templateId = deviceMNode.getSchemaTemplateIdWithState();
- PartialPath devicePath = deviceMNode.getPartialPath();
if (templateId >= 0) {
- if (!visitedTemplateDevice.contains(devicePath)) {
- schemaTree.appendTemplateDevice(
- deviceMNode.getPartialPath(), deviceMNode.isAligned(),
templateId, null);
- visitedTemplateDevice.add(devicePath);
- }
+ schemaTree.appendTemplateDevice(
+ deviceMNode.getPartialPath(), deviceMNode.isAligned(),
templateId, null);
+ skipTemplateChildren(deviceMNode);
} else {
MeasurementPath path = getCurrentMeasurementPathInTraverse(node);
if (nodes[nodes.length - 1].equals(node.getAlias())) {
@@ -859,19 +854,14 @@ public class MTreeBelowSGCachedImpl {
try (MeasurementCollector<Void, ICachedMNode> collector =
new MeasurementCollector<Void, ICachedMNode>(
rootNode, patternTree, store, SchemaConstant.ALL_MATCH_SCOPE) {
- private final Set<PartialPath> visitedTemplateDevice = new
HashSet<>();
-
protected Void collectMeasurement(IMeasurementMNode<ICachedMNode>
node) {
IDeviceMNode<ICachedMNode> deviceMNode =
getParentOfNextMatchedNode().getAsDeviceMNode();
int templateId = deviceMNode.getSchemaTemplateIdWithState();
- PartialPath devicePath = deviceMNode.getPartialPath();
if (templateId >= 0) {
- if (!visitedTemplateDevice.contains(devicePath)) {
- schemaTree.appendTemplateDevice(
- deviceMNode.getPartialPath(), deviceMNode.isAligned(),
templateId, null);
- visitedTemplateDevice.add(devicePath);
- }
+ schemaTree.appendTemplateDevice(
+ deviceMNode.getPartialPath(), deviceMNode.isAligned(),
templateId, null);
+ skipTemplateChildren(deviceMNode);
} else {
MeasurementPath path = getCurrentMeasurementPathInTraverse(node);
path.setMeasurementAlias(node.getAlias());
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java
index d1a2c5d7b5b..b73624f2a1b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.commons.path.PathPatternTree;
import org.apache.iotdb.commons.path.fa.IFAState;
import org.apache.iotdb.commons.path.fa.IFATransition;
import org.apache.iotdb.commons.schema.node.IMNode;
+import org.apache.iotdb.commons.schema.node.role.IDeviceMNode;
import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory;
import org.apache.iotdb.commons.schema.node.utils.IMNodeIterator;
import org.apache.iotdb.commons.schema.tree.AbstractTreeVisitor;
@@ -72,6 +73,7 @@ public abstract class Traverser<R, N extends IMNode<N>>
extends AbstractTreeVisi
// default false means fullPath pattern match
protected boolean isPrefixMatch = false;
+ private IDeviceMNode<N> skipTemplateDevice;
protected Traverser() {}
@@ -137,11 +139,17 @@ public abstract class Traverser<R, N extends IMNode<N>>
extends AbstractTreeVisi
@Override
protected N getChild(N parent, String childName) throws MetadataException {
+ return getChild(parent, childName, parent == skipTemplateDevice);
+ }
+
+ private N getChild(N parent, String childName, boolean skipTemplateChildren)
+ throws MetadataException {
N child = null;
if (parent.isAboveDatabase()) {
child = parent.getChild(childName);
} else {
if (templateMap != null
+ && !skipTemplateChildren
&& !templateMap.isEmpty() // this task will cover some timeseries
represented by template
&& (parent.isDevice()
&& parent.getAsDeviceMNode().getSchemaTemplateId()
@@ -179,13 +187,14 @@ public abstract class Traverser<R, N extends IMNode<N>>
extends AbstractTreeVisi
throws Exception {
return new IMNodeIterator<N>() {
private N next = null;
+ private boolean skipTemplateChildren = false;
@Override
public boolean hasNext() {
if (next == null) {
while (next == null && childrenName.hasNext()) {
try {
- next = getChild(parent, childrenName.next());
+ next = getChild(parent, childrenName.next(),
skipTemplateChildren);
} catch (Throwable e) {
logger.warn(e.getMessage(), e);
throw new RuntimeException(e);
@@ -205,6 +214,11 @@ public abstract class Traverser<R, N extends IMNode<N>>
extends AbstractTreeVisi
return result;
}
+ @Override
+ public void skipTemplateChildren() {
+ skipTemplateChildren = true;
+ }
+
@Override
public void close() {
if (next != null) {
@@ -217,6 +231,7 @@ public abstract class Traverser<R, N extends IMNode<N>>
extends AbstractTreeVisi
@Override
protected Iterator<N> getChildrenIterator(N parent) throws MetadataException
{
+ IMNodeIterator<N> currentChildrenIterator;
if (parent.isAboveDatabase()) {
return new MNodeIterator<>(parent.getChildren().values().iterator());
} else {
@@ -307,4 +322,12 @@ public abstract class Traverser<R, N extends IMNode<N>>
extends AbstractTreeVisi
}
return null;
}
+
+ protected void skipTemplateChildren(IDeviceMNode<N> deviceMNode) {
+ skipTemplateDevice = deviceMNode;
+ Iterator<N> iterator = getCurrentChildrenIterator();
+ if (iterator instanceof IMNodeIterator) {
+ ((IMNodeIterator<N>) iterator).skipTemplateChildren();
+ }
+ }
}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/utils/IMNodeIterator.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/utils/IMNodeIterator.java
index de1c710b228..75fd9d40c67 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/utils/IMNodeIterator.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/utils/IMNodeIterator.java
@@ -24,5 +24,7 @@ import java.util.Iterator;
public interface IMNodeIterator<N extends IMNode<?>> extends Iterator<N> {
+ void skipTemplateChildren();
+
void close();
}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java
index 8583a0d5101..caa8019ef87 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java
@@ -400,6 +400,19 @@ public abstract class AbstractTreeVisitor<N extends
ITreeNode, R> implements Sch
protected abstract Iterator<N> getChildrenIterator(N parent,
Iterator<String> childrenName)
throws Exception;
+ /**
+ * Get current children iterator generated by {@link
AbstractTreeVisitor#getChildrenIterator}
+ *
+ * @return null if there is no current children iterator
+ */
+ protected Iterator<N> getCurrentChildrenIterator() {
+ if (visitorStack.isEmpty()) {
+ return null;
+ } else {
+ return visitorStack.peek().iterator.getIterator();
+ }
+ }
+
// Release a child node.
protected void releaseNode(N node) {}
@@ -520,6 +533,8 @@ public abstract class AbstractTreeVisitor<N extends
ITreeNode, R> implements Sch
}
}
+ public abstract Iterator<N> getIterator();
+
protected void close() {
if (nextMatchedChild != null) {
releaseNode(nextMatchedChild);
@@ -560,6 +575,11 @@ public abstract class AbstractTreeVisitor<N extends
ITreeNode, R> implements Sch
return;
}
}
+
+ @Override
+ public Iterator<N> getIterator() {
+ return null;
+ }
}
// only one fuzzy transition which may match batch children, need to iterate
and check all
@@ -599,6 +619,11 @@ public abstract class AbstractTreeVisitor<N extends
ITreeNode, R> implements Sch
}
}
+ @Override
+ public Iterator<N> getIterator() {
+ return childrenIterator;
+ }
+
@Override
protected void close() {
super.close();
@@ -695,6 +720,11 @@ public abstract class AbstractTreeVisitor<N extends
ITreeNode, R> implements Sch
}
}
+ @Override
+ public Iterator<N> getIterator() {
+ return iterator;
+ }
+
@Override
protected void close() {
super.close();
@@ -780,6 +810,11 @@ public abstract class AbstractTreeVisitor<N extends
ITreeNode, R> implements Sch
}
}
+ @Override
+ public Iterator<N> getIterator() {
+ return iterator;
+ }
+
/**
* Try to get next matched state from sourceState and add it into
currentStateMatchInfo
*
