This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.12 by this push:
new 8343f22 [To rel/0.12][IOTDB-1738] Cache paths list in batched insert
plan (#4034) (#4053)
8343f22 is described below
commit 8343f225c04e637a0478982cd4e01024b45e5bae
Author: BaiJian <[email protected]>
AuthorDate: Tue Sep 28 15:09:35 2021 +0800
[To rel/0.12][IOTDB-1738] Cache paths list in batched insert plan (#4034)
(#4053)
---
checkstyle.xml | 2 +-
.../apache/iotdb/cluster/metadata/CMManager.java | 24 +++++++++-------
pom.xml | 2 +-
.../db/qp/physical/crud/InsertMultiTabletPlan.java | 17 +++++++++---
.../iotdb/db/qp/physical/crud/InsertRowPlan.java | 10 +++++--
.../physical/crud/InsertRowsOfOneDevicePlan.java | 21 +++++++++++---
.../iotdb/db/qp/physical/crud/InsertRowsPlan.java | 32 ++++++++++++++++------
.../qp/physical/sys/CreateMultiTimeSeriesPlan.java | 15 ++++++++--
8 files changed, 90 insertions(+), 33 deletions(-)
diff --git a/checkstyle.xml b/checkstyle.xml
index 76fe6e1..0a9d199 100644
--- a/checkstyle.xml
+++ b/checkstyle.xml
@@ -227,4 +227,4 @@
</module>
<module name="CommentsIndentation"/>
</module>
-</module>
\ No newline at end of file
+</module>
diff --git
a/cluster/src/main/java/org/apache/iotdb/cluster/metadata/CMManager.java
b/cluster/src/main/java/org/apache/iotdb/cluster/metadata/CMManager.java
index 75ff989..4ef2e12 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/metadata/CMManager.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/metadata/CMManager.java
@@ -51,10 +51,10 @@ import org.apache.iotdb.db.metadata.template.Template;
import org.apache.iotdb.db.qp.constant.SQLConstant;
import org.apache.iotdb.db.qp.physical.BatchPlan;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
-import org.apache.iotdb.db.qp.physical.crud.*;
import org.apache.iotdb.db.qp.physical.crud.InsertMultiTabletPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
+import org.apache.iotdb.db.qp.physical.crud.InsertRowsOfOneDevicePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertRowsPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan;
import org.apache.iotdb.db.qp.physical.crud.SetDeviceTemplatePlan;
@@ -502,7 +502,7 @@ public class CMManager extends MManager {
// for CreateMultiTimeSeriesPlan, use getPaths() to get all timeseries to
get related storage
// groups.
if (plan instanceof BatchPlan) {
- storageGroups.addAll(getStorageGroups(getValidStorageGroups(plan)));
+ storageGroups.addAll(getStorageGroups(getValidStorageGroups((BatchPlan)
plan)));
} else if (plan instanceof InsertRowPlan || plan instanceof
InsertTabletPlan) {
storageGroups.addAll(
getStorageGroups(Collections.singletonList(((InsertPlan)
plan).getDeviceId())));
@@ -530,12 +530,13 @@ public class CMManager extends MManager {
}
}
- private List<PartialPath> getValidStorageGroups(PhysicalPlan plan) {
+ private List<PartialPath> getValidStorageGroups(BatchPlan plan) {
List<PartialPath> paths = new ArrayList<>();
- for (int i = 0; i < plan.getPaths().size(); i++) {
+ List<PartialPath> originalPaths = plan.getPrefixPaths();
+ for (int i = 0; i < originalPaths.size(); i++) {
// has permission to create sg
- if (!((BatchPlan) plan).getResults().containsKey(i)) {
- paths.add(plan.getPaths().get(i));
+ if (!plan.getResults().containsKey(i)) {
+ paths.add(originalPaths.get(i));
}
}
return paths;
@@ -861,14 +862,17 @@ public class CMManager extends MManager {
*/
public void pullTimeSeriesSchemas(List<PartialPath> prefixPaths, Node
ignoredGroup)
throws MetadataException {
+ // Remove duplicated prefix paths to optimize
+ Set<PartialPath> prefixPathSet = new HashSet<>(prefixPaths);
+ List<PartialPath> uniquePrefixPaths = new ArrayList<>(prefixPathSet);
logger.debug(
"{}: Pulling timeseries schemas of {}, ignored group {}",
metaGroupMember.getName(),
- prefixPaths,
+ uniquePrefixPaths,
ignoredGroup);
// split the paths by the data groups that should hold them
Map<PartitionGroup, List<String>> partitionGroupPathMap = new HashMap<>();
- for (PartialPath prefixPath : prefixPaths) {
+ for (PartialPath prefixPath : uniquePrefixPaths) {
if
(SQLConstant.RESERVED_TIME.equalsIgnoreCase(prefixPath.getFullPath())) {
continue;
}
@@ -886,8 +890,8 @@ public class CMManager extends MManager {
logger.debug(
"{}: pulling schemas of {} and other {} paths from {} groups",
metaGroupMember.getName(),
- prefixPaths.get(0),
- prefixPaths.size() - 1,
+ uniquePrefixPaths.get(0),
+ uniquePrefixPaths.size() - 1,
partitionGroupPathMap.size());
}
for (Entry<PartitionGroup, List<String>> partitionGroupListEntry :
diff --git a/pom.xml b/pom.xml
index 2cd9ca0..81cc46d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1369,4 +1369,4 @@
</properties>
</profile>
</profiles>
-</project>
\ No newline at end of file
+</project>
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertMultiTabletPlan.java
b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertMultiTabletPlan.java
index 953fb78..2ac9504 100644
---
a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertMultiTabletPlan.java
+++
b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertMultiTabletPlan.java
@@ -29,7 +29,11 @@ import org.apache.iotdb.service.rpc.thrift.TSStatus;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.TreeMap;
/**
* Mainly used in the distributed version, when multiple InsertTabletPlans
belong to a raft
@@ -88,6 +92,8 @@ public class InsertMultiTabletPlan extends InsertPlan
implements BatchPlan {
/** record the result of creation of time series */
private Map<Integer, TSStatus> results = new TreeMap<>();
+ private List<PartialPath> prefixPaths;
+
boolean[] isExecuted;
public InsertMultiTabletPlan() {
@@ -124,11 +130,14 @@ public class InsertMultiTabletPlan extends InsertPlan
implements BatchPlan {
}
public List<PartialPath> getPrefixPaths() {
- Set<PartialPath> result = new HashSet<>();
+ if (prefixPaths != null) {
+ return prefixPaths;
+ }
+ prefixPaths = new ArrayList<>(insertTabletPlanList.size());
for (InsertTabletPlan insertTabletPlan : insertTabletPlanList) {
- result.add(insertTabletPlan.getDeviceId());
+ prefixPaths.add(insertTabletPlan.getDeviceId());
}
- return new ArrayList<>(result);
+ return prefixPaths;
}
@Override
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowPlan.java
b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowPlan.java
index e4b7eb1..a2f19bc 100644
---
a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowPlan.java
+++
b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowPlan.java
@@ -62,6 +62,7 @@ public class InsertRowPlan extends InsertPlan {
private boolean isNeedInferType = false;
private List<Object> failedValues;
+ private List<PartialPath> paths;
public InsertRowPlan() {
super(OperatorType.INSERT);
@@ -253,12 +254,15 @@ public class InsertRowPlan extends InsertPlan {
@Override
public List<PartialPath> getPaths() {
- List<PartialPath> ret = new ArrayList<>();
+ if (paths != null) {
+ return paths;
+ }
+ paths = new ArrayList<>(measurements.length);
for (String m : measurements) {
PartialPath fullPath = deviceId.concatNode(m);
- ret.add(fullPath);
+ paths.add(fullPath);
}
- return ret;
+ return paths;
}
public Object[] getValues() {
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowsOfOneDevicePlan.java
b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowsOfOneDevicePlan.java
index acf41a8..5f6a45e 100644
---
a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowsOfOneDevicePlan.java
+++
b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowsOfOneDevicePlan.java
@@ -28,7 +28,14 @@ import org.apache.iotdb.service.rpc.thrift.TSStatus;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
public class InsertRowsOfOneDevicePlan extends InsertPlan implements BatchPlan
{
@@ -50,6 +57,8 @@ public class InsertRowsOfOneDevicePlan extends InsertPlan
implements BatchPlan {
/** record the result of insert rows */
private Map<Integer, TSStatus> results = new HashMap<>();
+ private List<PartialPath> paths;
+
public InsertRowsOfOneDevicePlan() {
super(OperatorType.BATCH_INSERT_ONE_DEVICE);
}
@@ -105,11 +114,15 @@ public class InsertRowsOfOneDevicePlan extends InsertPlan
implements BatchPlan {
@Override
public List<PartialPath> getPaths() {
- Set<PartialPath> paths = new HashSet<>();
+ if (paths != null) {
+ return paths;
+ }
+ Set<PartialPath> pathSet = new HashSet<>();
for (InsertRowPlan plan : rowPlans) {
- paths.addAll(plan.getPaths());
+ pathSet.addAll(plan.getPaths());
}
- return new ArrayList<>(paths);
+ paths = new ArrayList<>(pathSet);
+ return paths;
}
@Override
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowsPlan.java
b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowsPlan.java
index 5c10901..b0e5550 100644
---
a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowsPlan.java
+++
b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowsPlan.java
@@ -29,7 +29,13 @@ import org.apache.iotdb.service.rpc.thrift.TSStatus;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
public class InsertRowsPlan extends InsertPlan implements BatchPlan {
@@ -53,6 +59,9 @@ public class InsertRowsPlan extends InsertPlan implements
BatchPlan {
/** record the result of insert rows */
private Map<Integer, TSStatus> results = new HashMap<>();
+ private List<PartialPath> paths;
+ private List<PartialPath> prefixPaths;
+
public InsertRowsPlan() {
super(OperatorType.BATCH_INSERT_ROWS);
insertRowPlanList = new ArrayList<>();
@@ -72,20 +81,27 @@ public class InsertRowsPlan extends InsertPlan implements
BatchPlan {
@Override
public List<PartialPath> getPaths() {
- List<PartialPath> result = new ArrayList<>();
- for (InsertRowPlan insertRowPlan : insertRowPlanList) {
- result.addAll(insertRowPlan.getPaths());
+ if (paths != null) {
+ return paths;
}
- return result;
+ Set<PartialPath> pathSet = new HashSet<>();
+ for (InsertRowPlan plan : insertRowPlanList) {
+ pathSet.addAll(plan.getPaths());
+ }
+ paths = new ArrayList<>(pathSet);
+ return paths;
}
@Override
public List<PartialPath> getPrefixPaths() {
- Set<PartialPath> result = new HashSet<>();
+ if (prefixPaths != null) {
+ return prefixPaths;
+ }
+ prefixPaths = new ArrayList<>(insertRowPlanList.size());
for (InsertRowPlan insertRowPlan : insertRowPlanList) {
- result.add(insertRowPlan.getDeviceId());
+ prefixPaths.add(insertRowPlan.getDeviceId());
}
- return new ArrayList<>(result);
+ return prefixPaths;
}
@Override
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CreateMultiTimeSeriesPlan.java
b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CreateMultiTimeSeriesPlan.java
index 2897334..b98761f 100644
---
a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CreateMultiTimeSeriesPlan.java
+++
b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CreateMultiTimeSeriesPlan.java
@@ -34,7 +34,13 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
/**
* create multiple timeSeries, could be split to several sub Plans to execute
in different DataGroup
@@ -42,6 +48,7 @@ import java.util.*;
public class CreateMultiTimeSeriesPlan extends PhysicalPlan implements
BatchPlan {
private List<PartialPath> paths;
+ private List<PartialPath> prefixPaths;
private List<TSDataType> dataTypes;
private List<TSEncoding> encodings;
private List<CompressionType> compressors;
@@ -141,7 +148,11 @@ public class CreateMultiTimeSeriesPlan extends
PhysicalPlan implements BatchPlan
@Override
public List<PartialPath> getPrefixPaths() {
- return Collections.emptyList();
+ if (prefixPaths != null) {
+ return prefixPaths;
+ }
+ prefixPaths =
paths.stream().map(PartialPath::getDevicePath).collect(Collectors.toList());
+ return prefixPaths;
}
public TSStatus[] getFailingStatus() {
