This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new dc3d9f1 [IOTDB-749] Handle select * from root OOM (#1884)
dc3d9f1 is described below
commit dc3d9f1555b94dd1a38514c3de0682241063fde3
Author: Steve Yurong Su <[email protected]>
AuthorDate: Thu Oct 29 11:42:59 2020 +0800
[IOTDB-749] Handle select * from root OOM (#1884)
* fix slimit logical errors in ConcatPathOptimizer
* add ITs
* fix tests
* fix tests
* set default MaxQueryDeduplicatedPathNum value back when
IoTDBQueryMemoryControlIT is over
---
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 110 +++++------
.../db/integration/IoTDBQueryMemoryControlIT.java | 219 +++++++++++++++++++++
2 files changed, 269 insertions(+), 60 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
index 5d089a6..eb41392 100644
---
a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
+++
b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
@@ -50,7 +50,6 @@ public class ConcatPathOptimizer implements ILogicalOptimizer
{
private static final String WARNING_NO_SUFFIX_PATHS = "given SFWOperator
doesn't have suffix paths, cannot concat seriesPath";
private static final String WARNING_NO_PREFIX_PATHS = "given SFWOperator
doesn't have prefix paths, cannot concat seriesPath";
-
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
@Override
public Operator transform(Operator operator, int maxDeduplicatedPathNum)
@@ -89,24 +88,12 @@ public class ConcatPathOptimizer implements
ILogicalOptimizer {
boolean isAlignByDevice = false;
if (operator instanceof QueryOperator) {
- if (!((QueryOperator) operator).isAlignByDevice() || ((QueryOperator)
operator)
- .isLastQuery()) {
-
- // concat and remove star
- if (((QueryOperator) operator).hasSlimit()) {
- int seriesLimit = ((QueryOperator) operator).getSeriesLimit();
- int seriesOffset = ((QueryOperator) operator).getSeriesOffset();
- if (seriesLimit > maxDeduplicatedPathNum) {
- throw new PathNumOverLimitException(maxDeduplicatedPathNum,
seriesLimit);
- }
- concatSelect(prefixPaths, select, seriesLimit, seriesOffset);
- slimitTrim(select, seriesOffset);
- } else {
- concatSelect(prefixPaths, select, maxDeduplicatedPathNum + 1, 0);
- if (select.getSuffixPaths().size() > maxDeduplicatedPathNum) {
- throw new PathNumOverLimitException(maxDeduplicatedPathNum);
- }
- }
+ if (!((QueryOperator) operator).isAlignByDevice()
+ || ((QueryOperator) operator).isLastQuery()) {
+ // concat paths and remove stars
+ int seriesLimit = ((QueryOperator) operator).getSeriesLimit();
+ int seriesOffset = ((QueryOperator) operator).getSeriesOffset();
+ concatSelect(prefixPaths, select, seriesLimit, seriesOffset,
maxDeduplicatedPathNum);
} else {
isAlignByDevice = true;
for (PartialPath path : initialSuffixPaths) {
@@ -171,8 +158,9 @@ public class ConcatPathOptimizer implements
ILogicalOptimizer {
* Extract paths from select&from cql, expand them into complete versions,
and reassign them to
* selectOperator's suffixPathList. Treat aggregations similarly.
*/
- private void concatSelect(List<PartialPath> fromPaths, SelectOperator
selectOperator, int limit, int offset)
- throws LogicalOptimizeException {
+ private void concatSelect(List<PartialPath> fromPaths, SelectOperator
selectOperator, int limit,
+ int offset, int maxDeduplicatedPathNum)
+ throws LogicalOptimizeException, PathNumOverLimitException {
List<PartialPath> suffixPaths = judgeSelectOperator(selectOperator);
List<PartialPath> allPaths = new ArrayList<>();
@@ -192,38 +180,12 @@ public class ConcatPathOptimizer implements
ILogicalOptimizer {
}
}
- removeStarsInPath(allPaths, afterConcatAggregations, selectOperator,
limit, offset);
- }
-
- /**
- * Make 'SOFFSET' take effect by trimming the suffixList and aggregations of
the
- * selectOperator.
- *
- * @param seriesOffset is ensured to be non-negative integer
- */
- private void slimitTrim(SelectOperator select, int seriesOffset)
- throws LogicalOptimizeException {
- List<PartialPath> suffixList = select.getSuffixPaths();
- List<String> aggregations = select.getAggregations();
- int size = suffixList.size();
-
- // check parameter range
- if (size == 0) {
- throw new LogicalOptimizeException("SOFFSET <SOFFSETValue>: SOFFSETValue
exceeds the range.");
- }
- int endPosition = seriesOffset + size;
-
- // trim aggregations if exists
- if (aggregations != null && !aggregations.isEmpty()) {
- List<String> trimedAggregations = new ArrayList<>(
- aggregations.subList(seriesOffset, endPosition));
- select.setAggregations(trimedAggregations);
- }
+ removeStarsInPath(allPaths, afterConcatAggregations, selectOperator,
limit, offset,
+ maxDeduplicatedPathNum);
}
private FilterOperator concatFilter(List<PartialPath> fromPaths,
FilterOperator operator,
- Set<PartialPath> filterPaths)
- throws LogicalOptimizeException {
+ Set<PartialPath> filterPaths) throws LogicalOptimizeException {
if (!operator.isLeaf()) {
List<FilterOperator> newFilterList = new ArrayList<>();
for (FilterOperator child : operator.getChildren()) {
@@ -235,7 +197,8 @@ public class ConcatPathOptimizer implements
ILogicalOptimizer {
FunctionOperator functionOperator = (FunctionOperator) operator;
PartialPath filterPath = functionOperator.getSinglePath();
// do nothing in the cases of "where time > 5" or "where root.d1.s1 > 5"
- if (SQLConstant.isReservedPath(filterPath) ||
filterPath.getFirstNode().startsWith(SQLConstant.ROOT)) {
+ if (SQLConstant.isReservedPath(filterPath) || filterPath.getFirstNode()
+ .startsWith(SQLConstant.ROOT)) {
filterPaths.add(filterPath);
return operator;
}
@@ -259,8 +222,7 @@ public class ConcatPathOptimizer implements
ILogicalOptimizer {
}
private FilterOperator constructBinaryFilterTreeWithAnd(List<PartialPath>
noStarPaths,
- FilterOperator operator)
- throws LogicalOptimizeException {
+ FilterOperator operator) throws LogicalOptimizeException {
FilterOperator filterBinaryTree = new FilterOperator(SQLConstant.KW_AND);
FilterOperator currentNode = filterBinaryTree;
for (int i = 0; i < noStarPaths.size(); i++) {
@@ -286,7 +248,8 @@ public class ConcatPathOptimizer implements
ILogicalOptimizer {
* @param paths list of paths which may contain stars
* @return a unique seriesPath list
*/
- private List<PartialPath> removeStarsInPathWithUnique(List<PartialPath>
paths) throws LogicalOptimizeException {
+ private List<PartialPath> removeStarsInPathWithUnique(List<PartialPath>
paths)
+ throws LogicalOptimizeException {
List<PartialPath> retPaths = new ArrayList<>();
HashSet<PartialPath> pathSet = new HashSet<>();
try {
@@ -305,38 +268,65 @@ public class ConcatPathOptimizer implements
ILogicalOptimizer {
return retPaths;
}
+ @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
private void removeStarsInPath(List<PartialPath> paths, List<String>
afterConcatAggregations,
- SelectOperator selectOperator, int limit, int offset) throws
LogicalOptimizeException {
+ SelectOperator selectOperator, int limit, int offset, int
maxDeduplicatedPathNum)
+ throws LogicalOptimizeException, PathNumOverLimitException {
+ limit = limit == 0 || maxDeduplicatedPathNum < limit ?
maxDeduplicatedPathNum + 1 : limit;
+ boolean hasOffset = offset != 0;
+ int consumed = 0;
List<PartialPath> retPaths = new ArrayList<>();
List<String> newAggregations = new ArrayList<>();
+
for (int i = 0; i < paths.size(); i++) {
try {
Pair<List<PartialPath>, Integer> pair = removeWildcard(paths.get(i),
limit, offset);
+
List<PartialPath> actualPaths = pair.left;
- offset = offset - pair.right;
if (paths.get(i).getTsAlias() != null) {
if (actualPaths.size() == 1) {
actualPaths.get(0).setTsAlias(paths.get(i).getTsAlias());
} else if (actualPaths.size() >= 2) {
throw new LogicalOptimizeException(
- "alias '" + paths.get(i).getTsAlias() + "' can only be matched
with one time series");
+ "alias '" + paths.get(i).getTsAlias()
+ + "' can only be matched with one time series");
}
}
for (PartialPath actualPath : actualPaths) {
retPaths.add(actualPath);
- if (afterConcatAggregations != null &&
!afterConcatAggregations.isEmpty()) {
- newAggregations.add(afterConcatAggregations.get(i));
+ extendListSafely(afterConcatAggregations, i, newAggregations);
+ }
+
+ consumed += pair.right;
+ if (offset != 0) {
+ int delta = offset - pair.right;
+ offset = Math.max(delta, 0);
+ if (delta < 0) {
+ limit += delta;
}
+ } else {
+ limit -= pair.right;
+ }
+ if (limit == 0) {
+ if (retPaths.size() == maxDeduplicatedPathNum + 1) {
+ throw new PathNumOverLimitException(maxDeduplicatedPathNum);
+ }
+ break;
}
} catch (MetadataException e) {
throw new LogicalOptimizeException("error when remove star: " +
e.getMessage());
}
}
+
+ if (consumed == 0 ? hasOffset : retPaths.isEmpty()) {
+ throw new LogicalOptimizeException("SOFFSET <SOFFSETValue>: SOFFSETValue
exceeds the range.");
+ }
selectOperator.setSuffixPathList(retPaths);
selectOperator.setAggregations(newAggregations);
}
- protected Pair<List<PartialPath>, Integer> removeWildcard(PartialPath path,
int limit, int offset) throws MetadataException {
+ protected Pair<List<PartialPath>, Integer> removeWildcard(PartialPath path,
int limit, int offset)
+ throws MetadataException {
return IoTDB.metaManager.getAllTimeseriesPathWithAlias(path, limit,
offset);
}
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBQueryMemoryControlIT.java
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBQueryMemoryControlIT.java
new file mode 100644
index 0000000..8a0f3ba
--- /dev/null
+++
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBQueryMemoryControlIT.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.integration;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSetMetaData;
+import java.sql.Statement;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.jdbc.Config;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class IoTDBQueryMemoryControlIT {
+
+ private int defaultMaxQueryDeduplicatedPathNum;
+
+ private static final String[] sqls = new String[]{
+ "set storage group to root.ln",
+
+ "create timeseries root.ln.wf01.wt01 with
datatype=BOOLEAN,encoding=PLAIN",
+ "create timeseries root.ln.wf01.wt02 with
datatype=BOOLEAN,encoding=PLAIN",
+ "create timeseries root.ln.wf01.wt03 with
datatype=BOOLEAN,encoding=PLAIN",
+ "create timeseries root.ln.wf01.wt04 with
datatype=BOOLEAN,encoding=PLAIN",
+ "create timeseries root.ln.wf01.wt05 with
datatype=BOOLEAN,encoding=PLAIN",
+
+ "create timeseries root.ln.wf02.wt01 with datatype=FLOAT,encoding=RLE",
+ "create timeseries root.ln.wf02.wt02 with datatype=FLOAT,encoding=RLE",
+ "create timeseries root.ln.wf02.wt03 with datatype=FLOAT,encoding=RLE",
+ "create timeseries root.ln.wf02.wt04 with datatype=FLOAT,encoding=RLE",
+ "create timeseries root.ln.wf02.wt05 with datatype=FLOAT,encoding=RLE",
+
+ "create timeseries root.ln.wf03.wt01 with datatype=TEXT,encoding=PLAIN",
+ "create timeseries root.ln.wf03.wt02 with datatype=TEXT,encoding=PLAIN",
+ "create timeseries root.ln.wf03.wt03 with datatype=TEXT,encoding=PLAIN",
+ "create timeseries root.ln.wf03.wt04 with datatype=TEXT,encoding=PLAIN",
+ "create timeseries root.ln.wf03.wt05 with datatype=TEXT,encoding=PLAIN",
+ };
+
+ @Before
+ public void setUp() throws Exception {
+ defaultMaxQueryDeduplicatedPathNum =
IoTDBDescriptor.getInstance().getConfig()
+ .getMaxQueryDeduplicatedPathNum();
+
IoTDBDescriptor.getInstance().getConfig().setMaxQueryDeduplicatedPathNum(10);
+ EnvironmentUtils.envSetUp();
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ createTimeSeries();
+ }
+
+ private static void createTimeSeries() {
+ try (Statement statement = DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
"root").createStatement()) {
+ for (String sql : sqls) {
+ statement.execute(sql);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ EnvironmentUtils.cleanEnv();
+ IoTDBDescriptor.getInstance().getConfig()
+ .setMaxQueryDeduplicatedPathNum(defaultMaxQueryDeduplicatedPathNum);
+ }
+
+ @Test
+ public void selectWildcard() {
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
"root");
+ Statement statement = connection.createStatement()) {
+ statement.execute("select * from root");
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("Too many paths in one query!"));
+ }
+ }
+
+ @Test
+ public void selectWildcardSlimit10() {
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
"root");
+ Statement statement = connection.createStatement()) {
+ statement.execute("select * from root slimit 10");
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void selectWildcardSlimit11() {
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
"root");
+ Statement statement = connection.createStatement()) {
+ statement.execute("select * from root slimit 11");
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("Too many paths in one query!"));
+ }
+ }
+
+ @Test
+ public void selectWildcardWildcardWildcardSlimit5Soffset7() {
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
"root");
+ Statement statement = connection.createStatement()) {
+ statement.execute("select wf01.*, wf02.*, wf03.* from root.ln slimit 5
soffset 7");
+ ResultSetMetaData resultSetMetaData =
statement.getResultSet().getMetaData();
+ assertEquals(1 + 5, resultSetMetaData.getColumnCount());
+ for (int i = 2; i < 3 + 2; ++i) {
+ System.out.println(resultSetMetaData.getColumnName(i));
+
assertTrue(resultSetMetaData.getColumnName(i).contains("root.ln.wf02.wt0"));
+ }
+ for (int i = 3 + 2; i < 5 + 2; ++i) {
+ System.out.println(resultSetMetaData.getColumnName(i));
+
assertTrue(resultSetMetaData.getColumnName(i).contains("root.ln.wf03.wt0"));
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void selectWildcardWildcardWildcardSlimit5Soffset5() {
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
"root");
+ Statement statement = connection.createStatement()) {
+ statement.execute("select wf01.*, wf02.*, wf03.* from root.ln slimit 5
soffset 5");
+ ResultSetMetaData resultSetMetaData =
statement.getResultSet().getMetaData();
+ assertEquals(1 + 5, resultSetMetaData.getColumnCount());
+ for (int i = 2; i < 5 + 2; ++i) {
+ System.out.println(resultSetMetaData.getColumnName(i));
+
assertTrue(resultSetMetaData.getColumnName(i).contains("root.ln.wf02.wt0"));
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void selectWildcardWildcardWildcardSlimit15Soffset5() {
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
"root");
+ Statement statement = connection.createStatement()) {
+ statement.execute("select wf01.*, wf03.*, wf02.* from root.ln slimit 15
soffset 5");
+ ResultSetMetaData resultSetMetaData =
statement.getResultSet().getMetaData();
+ assertEquals(1 + 10, resultSetMetaData.getColumnCount());
+ for (int i = 2; i < 5 + 2; ++i) {
+ System.out.println(resultSetMetaData.getColumnName(i));
+
assertTrue(resultSetMetaData.getColumnName(i).contains("root.ln.wf03.wt0"));
+ }
+ for (int i = 5 + 2; i < 10 + 2; ++i) {
+ System.out.println(resultSetMetaData.getColumnName(i));
+
assertTrue(resultSetMetaData.getColumnName(i).contains("root.ln.wf02.wt0"));
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void selectWildcardWildcardWildcardSlimit15Soffset4() {
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
"root");
+ Statement statement = connection.createStatement()) {
+ statement.execute("select wf01.*, wf02.*, wf03.* from root.ln slimit 15
soffset 4");
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("Too many paths in one query!"));
+ }
+ }
+
+ @Test
+ public void selectWildcardWildcardWildcardSlimit3Soffset4() {
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
"root");
+ Statement statement = connection.createStatement()) {
+ statement.execute("select wf01.*, wf02.*, wf03.* from root.ln slimit 3
soffset 4");
+ ResultSetMetaData resultSetMetaData =
statement.getResultSet().getMetaData();
+ assertEquals(1 + 3, resultSetMetaData.getColumnCount());
+ for (int i = 2; i < 1 + 2; ++i) {
+ System.out.println(resultSetMetaData.getColumnName(i));
+
assertTrue(resultSetMetaData.getColumnName(i).contains("root.ln.wf01.wt0"));
+ }
+ for (int i = 1 + 2; i < 3 + 2; ++i) {
+ System.out.println(resultSetMetaData.getColumnName(i));
+
assertTrue(resultSetMetaData.getColumnName(i).contains("root.ln.wf02.wt0"));
+ }
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("Too many paths in one query!"));
+ }
+ }
+}
