This is an automated email from the ASF dual-hosted git repository. xiangweiwei pushed a commit to branch RequeryV2 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ef45eb5eceebfff0275591b51cbc69746ec11018 Merge: dbe29eb aca9faa Author: Alima777 <[email protected]> AuthorDate: Tue May 25 21:57:06 2021 +0800 Fix conclicts .github/workflows/sonar-coveralls.yml | 8 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4 | 2 +- .../cluster/query/ClusterConcatPathOptimizer.java | 37 --- .../apache/iotdb/cluster/query/ClusterPlanner.java | 6 - .../iotdb/cluster/query/LocalQueryExecutor.java | 58 +++-- .../cluster/query/aggregate/ClusterAggregator.java | 10 +- cluster/src/test/resources/logback.xml | 4 - cluster/src/test/resources/node1conf/logback.xml | 282 +++++++++++++++++++++ cluster/src/test/resources/node2conf/logback.xml | 282 +++++++++++++++++++++ cluster/src/test/resources/node3conf/logback.xml | 282 +++++++++++++++++++++ docs/UserGuide/Advanced-Features/Triggers.md | 6 +- docs/UserGuide/Cluster/Cluster-Setup-Example.md | 37 ++- docs/UserGuide/Cluster/Cluster-Setup.md | 2 - docs/zh/UserGuide/Advanced-Features/Triggers.md | 7 +- docs/zh/UserGuide/Cluster/Cluster-Setup-Example.md | 37 ++- docs/zh/UserGuide/Cluster/Cluster-Setup.md | 2 - .../java/org/apache/iotdb/rabbitmq/Constant.java | 2 +- .../iotdb/rabbitmq/RabbitMQChannelUtils.java | 1 + .../apache/iotdb/rabbitmq/RabbitMQConsumer.java | 2 +- .../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 2 +- .../iotdb/jdbc/IoTDBPreparedStatementTest.java | 42 ++- .../resources/conf/iotdb-engine.properties | 8 + server/src/assembly/resources/conf/iotdb-env.bat | 12 +- .../org/apache/iotdb/db/auth/AuthorityChecker.java | 3 +- .../db/auth/authorizer/LocalFileAuthorizer.java | 3 +- .../iotdb/db/auth/user/BasicUserManager.java | 10 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 22 +- .../org/apache/iotdb/db/conf/IoTDBConstant.java | 2 - .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 30 +-- .../db/engine/compaction/TsFileManagement.java | 4 +- .../level/LevelCompactionTsFileManagement.java | 6 +- .../iotdb/db/engine/merge/task/MergeFileTask.java | 8 +- .../db/engine/merge/task/MergeMultiChunkTask.java | 13 +- .../db/engine/storagegroup/TsFileProcessor.java | 4 +- .../iotdb/db/metrics/server/ServerArgument.java | 12 +- .../main/java/org/apache/iotdb/db/qp/Planner.java | 6 +- .../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 67 ++--- .../qp/strategy/optimizer/ConcatPathOptimizer.java | 69 ++--- .../apache/iotdb/db/qp/utils/WildcardsRemover.java | 68 +++-- .../db/query/dataset/AlignByDeviceDataSet.java | 3 + .../dataset/groupby/LocalGroupByExecutor.java | 3 + .../iotdb/db/query/expression/Expression.java | 22 +- .../iotdb/db/query/expression/ResultColumn.java | 46 ++++ .../query/expression/binary/BinaryExpression.java | 8 +- .../query/expression/unary/FunctionExpression.java | 34 ++- .../db/query/expression/unary/MinusExpression.java | 2 +- .../query/expression/unary/TimeSeriesOperand.java | 2 +- .../apache/iotdb/db/service/MetricsService.java | 3 + .../org/apache/iotdb/db/service/StartupChecks.java | 2 +- .../apache/iotdb/db/utils/QueryDataSetUtils.java | 2 + .../auth/authorizer/LocalFileAuthorizerTest.java | 4 +- .../db/auth/user/LocalFileUserManagerTest.java | 4 +- .../iotdb/db/engine/merge/MergeTaskTest.java | 7 +- .../iotdb/db/integration/IoTDBGroupByUnseqIT.java | 136 ++++++++++ .../db/integration/IoTDBWithoutAllNullIT.java | 72 ++++++ .../iotdb/db/metadata/MManagerBasicTest.java | 1 + .../org/apache/iotdb/db/metadata/MTreeTest.java | 1 + .../iotdb/db/qp/logical/IndexLogicalPlanTest.java | 6 + .../apache/iotdb/session/IoTDBSessionSimpleIT.java | 81 ++++++ .../java/org/apache/iotdb/session/SessionUT.java | 2 +- site/src/main/.vuepress/config.js | 4 +- .../test/java/org/apache/iotdb/db/sql/Cases.java | 78 +++++- .../java/org/apache/iotdb/db/sql/ClusterIT.java | 3 + .../java/org/apache/iotdb/db/sql/SingleNodeIT.java | 3 + .../apache/iotdb/tsfile/read/common/RowRecord.java | 5 + .../tsfile/read/query/dataset/QueryDataSet.java | 4 + .../org/apache/iotdb/tsfile/utils/PublicBAOS.java | 1 + .../zeppelin/iotdb/IoTDBInterpreterTest.java | 27 +- 68 files changed, 1704 insertions(+), 320 deletions(-) diff --cc server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java index 74ecb18,132fdba..097b877 --- a/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java @@@ -1020,9 -1003,10 +1020,8 @@@ public class IoTDBSqlVisitor extends Sq return queryOp; } - @Override - public Operator visitSelectClause(SelectClauseContext ctx) { - SelectOperator selectOp = new SelectOperator(SQLConstant.TOK_SELECT, zoneId); - + public void parseSelectClause(SelectClauseContext ctx) { + SelectComponent selectComponent = new SelectComponent(zoneId); - if (ctx.topClause() != null) { // TODO: parse info of top clause into selectOp visitTopClause(ctx.topClause()); @@@ -1360,12 -1305,46 +1359,46 @@@ List<TypeClauseContext> list = ctx.typeClause(); Map<TSDataType, IFill> fillTypes = new EnumMap<>(TSDataType.class); for (TypeClauseContext typeClause : list) { - parseTypeClause(typeClause, fillTypes); + if (typeClause.ALL() != null) { + if (typeClause.linearClause() != null) { + throw new SQLParserException("fill all doesn't support linear fill"); + } + parseAllTypeClause(typeClause, fillTypes); + break; + } else { + parsePrimitiveTypeClause(typeClause, fillTypes); + } } - queryOp.setFill(true); - queryOp.setFillTypes(fillTypes); + fillClauseComponent.setFillTypes(fillTypes); + queryOp.setSpecialClauseComponent(fillClauseComponent); } + private void parseAllTypeClause(TypeClauseContext ctx, Map<TSDataType, IFill> fillTypes) { + IFill fill; + long preRange; + if (ctx.previousUntilLastClause() != null) { + if (ctx.previousUntilLastClause().DURATION() != null) { + preRange = + DatetimeUtils.convertDurationStrToLong( + ctx.previousUntilLastClause().DURATION().getText()); + } else { + preRange = IoTDBDescriptor.getInstance().getConfig().getDefaultFillInterval(); + } + fill = new PreviousFill(preRange, true); + } else { + if (ctx.previousClause().DURATION() != null) { + preRange = + DatetimeUtils.convertDurationStrToLong(ctx.previousClause().DURATION().getText()); + } else { + preRange = IoTDBDescriptor.getInstance().getConfig().getDefaultFillInterval(); + } + fill = new PreviousFill(preRange); + } + for (TSDataType tsDataType : TSDataType.values()) { + fillTypes.put(tsDataType, fill.copy()); + } + } + private void parseLimitClause(LimitClauseContext ctx, Operator operator) { int limit; try { @@@ -1505,41 -1468,17 +1538,17 @@@ } // all type use the same fill way if (typeClause.ALL() != null) { - IFill fill; - if (typeClause.previousUntilLastClause() != null) { - long preRange; - if (typeClause.previousUntilLastClause().DURATION() != null) { - preRange = - DatetimeUtils.convertDurationStrToLong( - typeClause.previousUntilLastClause().DURATION().getText()); - } else { - preRange = IoTDBDescriptor.getInstance().getConfig().getDefaultFillInterval(); - } - fill = new PreviousFill(preRange, true); - } else { - long preRange; - if (typeClause.previousClause().DURATION() != null) { - preRange = - DatetimeUtils.convertDurationStrToLong( - typeClause.previousClause().DURATION().getText()); - } else { - preRange = IoTDBDescriptor.getInstance().getConfig().getDefaultFillInterval(); - } - fill = new PreviousFill(preRange); - } - for (TSDataType tsDataType : TSDataType.values()) { - fillTypes.put(tsDataType, fill.copy()); - } + parseAllTypeClause(typeClause, fillTypes); break; } else { - parseTypeClause(typeClause, fillTypes); + parsePrimitiveTypeClause(typeClause, fillTypes); } } - queryOp.setFill(true); - queryOp.setFillTypes(fillTypes); + groupByFillClauseComponent.setFillTypes(fillTypes); + queryOp.setSpecialClauseComponent(groupByFillClauseComponent); } - private void parseTypeClause(TypeClauseContext ctx, Map<TSDataType, IFill> fillTypes) { + private void parsePrimitiveTypeClause(TypeClauseContext ctx, Map<TSDataType, IFill> fillTypes) { TSDataType dataType = parseType(ctx.dataType().getText()); if (ctx.linearClause() != null && dataType == TSDataType.TEXT) { throw new SQLParserException( diff --cc server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java index f8b3eb8,69c6d94..33982d6 --- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java @@@ -27,11 -27,10 +27,10 @@@ import org.apache.iotdb.db.qp.constant. import org.apache.iotdb.db.qp.logical.Operator; import org.apache.iotdb.db.qp.logical.crud.BasicFunctionOperator; import org.apache.iotdb.db.qp.logical.crud.FilterOperator; -import org.apache.iotdb.db.qp.logical.crud.FromOperator; +import org.apache.iotdb.db.qp.logical.crud.FromComponent; import org.apache.iotdb.db.qp.logical.crud.FunctionOperator; - import org.apache.iotdb.db.qp.logical.crud.LastQueryOperator; import org.apache.iotdb.db.qp.logical.crud.QueryOperator; -import org.apache.iotdb.db.qp.logical.crud.SelectOperator; +import org.apache.iotdb.db.qp.logical.crud.SelectComponent; import org.apache.iotdb.db.qp.utils.WildcardsRemover; import org.apache.iotdb.db.query.expression.ResultColumn; import org.apache.iotdb.db.service.IoTDB; @@@ -81,20 -77,22 +77,22 @@@ public class ConcatPathOptimizer implem return false; } - FromOperator from = queryOperator.getFromOperator(); ++ FromComponent from = queryOperator.getFromComponent(); + if (from == null || from.getPrefixPaths().isEmpty()) { + LOGGER.warn(WARNING_NO_PREFIX_PATHS); + return false; + } + return true; } private void concatSelect(QueryOperator queryOperator) throws LogicalOptimizeException { - if (queryOperator.isAlignByDevice() && !(queryOperator instanceof LastQueryOperator)) { - return; - } - - List<PartialPath> prefixPaths = queryOperator.getFromOperator().getPrefixPaths(); + List<PartialPath> prefixPaths = queryOperator.getFromComponent().getPrefixPaths(); List<ResultColumn> resultColumns = new ArrayList<>(); - for (ResultColumn suffixColumn : queryOperator.getSelectOperator().getResultColumns()) { + for (ResultColumn suffixColumn : queryOperator.getSelectComponent().getResultColumns()) { suffixColumn.concat(prefixPaths, resultColumns); } - queryOperator.getSelectOperator().setResultColumns(resultColumns); + queryOperator.getSelectComponent().setResultColumns(resultColumns); } private void removeWildcardsInSelectPaths(QueryOperator queryOperator, int fetchSize) @@@ -104,31 -101,29 +101,29 @@@ return; } - WildcardsRemover wildcardsRemover = new WildcardsRemover(this, queryOperator, fetchSize); + WildcardsRemover wildcardsRemover = new WildcardsRemover(queryOperator, fetchSize); List<ResultColumn> resultColumns = new ArrayList<>(); - for (ResultColumn resultColumn : queryOperator.getSelectOperator().getResultColumns()) { + for (ResultColumn resultColumn : queryOperator.getSelectComponent().getResultColumns()) { resultColumn.removeWildcards(wildcardsRemover, resultColumns); if (wildcardsRemover.checkIfPathNumberIsOverLimit(resultColumns)) { break; } } wildcardsRemover.checkIfSoffsetIsExceeded(resultColumns); - queryOperator.getSelectOperator().setResultColumns(resultColumns); + queryOperator.getSelectComponent().setResultColumns(resultColumns); } - private void concatFilter(QueryOperator queryOperator) throws LogicalOptimizeException { + private void concatFilterAndRemoveWildcards(QueryOperator queryOperator) + throws LogicalOptimizeException { FilterOperator filter = queryOperator.getFilterOperator(); if (filter == null) { return; } Set<PartialPath> filterPaths = new HashSet<>(); - if (!queryOperator.isAlignByDevice() || queryOperator instanceof LastQueryOperator) { - // GROUP_BY_DEVICE leaves the concatFilter to PhysicalGenerator to optimize filter without - // prefix first - queryOperator.setFilterOperator( - concatFilter(queryOperator.getFromComponent().getPrefixPaths(), filter, filterPaths)); - } + queryOperator.setFilterOperator( + concatFilterAndRemoveWildcards( - queryOperator.getFromOperator().getPrefixPaths(), filter, filterPaths)); ++ queryOperator.getFromComponent().getPrefixPaths(), filter, filterPaths)); queryOperator.getFilterOperator().setPathSet(filterPaths); } diff --cc server/src/main/java/org/apache/iotdb/db/qp/utils/WildcardsRemover.java index 6e85e8e,eb3e02b..2e261f4 --- a/server/src/main/java/org/apache/iotdb/db/qp/utils/WildcardsRemover.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/utils/WildcardsRemover.java @@@ -35,41 -35,42 +36,36 @@@ import java.util.ArrayList import java.util.Collections; import java.util.List; -/** Removes wildcards (applying memory control and slimit/soffset control) */ public class WildcardsRemover { - private final ConcatPathOptimizer concatPathOptimizer; - private final int maxDeduplicatedPathNum; - private final int soffset; + private int soffset = 0; - private int offset = 0; - private int limit = Integer.MAX_VALUE; - private int currentOffset; - private int currentLimit; - - /** Records the path number that the MManager totally returned. */ - private int consumed; ++ private int currentOffset = 0; ++ private int currentLimit = Integer.MAX_VALUE; + private int consumed = 0; - public WildcardsRemover( - ConcatPathOptimizer concatPathOptimizer, QueryOperator queryOperator, int fetchSize) { - this.concatPathOptimizer = concatPathOptimizer; - + public WildcardsRemover(QueryOperator queryOperator, int fetchSize) { // Dataset of last query actually has only three columns, so we shouldn't limit the path num // while constructing logical plan // To avoid overflowing because logicalOptimize function may do maxDeduplicatedPathNum + 1, we // set it to Integer.MAX_VALUE - 1 maxDeduplicatedPathNum = - queryOperator.isLastQuery() + queryOperator instanceof LastQueryOperator ? Integer.MAX_VALUE - 1 : QueryResourceManager.getInstance().getMaxDeduplicatedPathNum(fetchSize); - soffset = queryOperator.getSeriesOffset(); - currentOffset = soffset; - - final int slimit = queryOperator.getSeriesLimit(); - currentLimit = - slimit == 0 || maxDeduplicatedPathNum < slimit ? maxDeduplicatedPathNum + 1 : slimit; + if (queryOperator.getSpecialClauseComponent() != null) { + soffset = queryOperator.getSpecialClauseComponent().getSeriesOffset(); - offset = soffset; ++ currentOffset = soffset; - consumed = 0; + final int slimit = queryOperator.getSpecialClauseComponent().getSeriesLimit(); - limit = slimit == 0 || maxDeduplicatedPathNum < slimit ? maxDeduplicatedPathNum + 1 : slimit; ++ currentLimit = ++ slimit == 0 || maxDeduplicatedPathNum < slimit ? maxDeduplicatedPathNum + 1 : slimit; + } } - public WildcardsRemover(ConcatPathOptimizer concatPathOptimizer) { - this.concatPathOptimizer = concatPathOptimizer; + public WildcardsRemover() { maxDeduplicatedPathNum = Integer.MAX_VALUE - 1; - soffset = 0; - currentLimit = maxDeduplicatedPathNum + 1; - consumed = 0; } public List<PartialPath> removeWildcardFrom(PartialPath path) throws LogicalOptimizeException { diff --cc server/src/test/java/org/apache/iotdb/db/qp/logical/IndexLogicalPlanTest.java index c8515d2,38593cb..14ec78d --- a/server/src/test/java/org/apache/iotdb/db/qp/logical/IndexLogicalPlanTest.java +++ b/server/src/test/java/org/apache/iotdb/db/qp/logical/IndexLogicalPlanTest.java @@@ -108,9 -108,9 +108,12 @@@ public class IndexLogicalPlanTest Assert.assertEquals(OperatorType.QUERY, queryOperator.getType()); Assert.assertEquals( "Glu", - queryOperator.getSelectOperator().getResultColumns().get(0).getExpression().toString()); + queryOperator.getSelectComponent().getResultColumns().get(0).getExpression().toString()); Assert.assertEquals( - "root.Ery.*", queryOperator.getFromOperator().getPrefixPaths().get(0).getFullPath()); ++ "Glu", ++ queryOperator.getSelectComponent().getResultColumns().get(0).getExpression().toString()); ++ Assert.assertEquals( + "root.Ery.*", queryOperator.getFromComponent().getPrefixPaths().get(0).getFullPath()); Assert.assertEquals(IndexType.RTREE_PAA, queryOperator.getIndexType()); Assert.assertEquals(2, queryOperator.getProps().size()); Assert.assertEquals(2, (int) queryOperator.getProps().get(TOP_K)); @@@ -132,9 -132,9 +135,12 @@@ Assert.assertEquals(OperatorType.QUERY, queryOperator.getType()); Assert.assertEquals( "Speed", - queryOperator.getSelectOperator().getResultColumns().get(0).getExpression().toString()); + queryOperator.getSelectComponent().getResultColumns().get(0).getExpression().toString()); + Assert.assertEquals( ++ "Speed", ++ queryOperator.getSelectComponent().getResultColumns().get(0).getExpression().toString()); + Assert.assertEquals( - "root.Wind.AZQ02", queryOperator.getFromOperator().getPrefixPaths().get(0).getFullPath()); + "root.Wind.AZQ02", queryOperator.getFromComponent().getPrefixPaths().get(0).getFullPath()); Assert.assertEquals(IndexType.ELB_INDEX, queryOperator.getIndexType()); Assert.assertEquals(2, queryOperator.getProps().size()); Assert.assertEquals("[1.0, 2.0, 1.0]", queryOperator.getProps().get(THRESHOLD).toString());
