This is an automated email from the ASF dual-hosted git repository. sunzesong pushed a commit to branch jira_775 in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 801c8f532ab076834c26c815dac269fb65be7e3e Author: samperson1997 <[email protected]> AuthorDate: Tue Jun 23 23:17:14 2020 +0800 [IOTDB-775] Add SQL of creating snapshot manually --- .../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 13 ++++++++ .../org/apache/iotdb/db/metadata/MManager.java | 39 ++++++++++++---------- .../main/java/org/apache/iotdb/db/qp/Planner.java | 1 + .../apache/iotdb/db/qp/constant/SQLConstant.java | 2 ++ .../apache/iotdb/db/qp/executor/PlanExecutor.java | 16 ++++++--- .../org/apache/iotdb/db/qp/logical/Operator.java | 2 +- .../db/qp/logical/sys/CreateSnapshotOperator.java | 30 +++++++++++++++++ .../db/qp/physical/sys/CreateSnapshotPlan.java | 37 ++++++++++++++++++++ .../iotdb/db/qp/strategy/LogicalGenerator.java | 12 ++++--- .../iotdb/db/qp/strategy/PhysicalGenerator.java | 3 ++ 10 files changed, 129 insertions(+), 26 deletions(-) diff --git a/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4 b/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4 index 8170130..f78e854 100644 --- a/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4 +++ b/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4 @@ -80,6 +80,7 @@ statement | LOAD STRING_LITERAL autoCreateSchema? #loadFiles | REMOVE STRING_LITERAL #removeFile | MOVE STRING_LITERAL STRING_LITERAL #moveFile + | CREATE SNAPSHOT FOR SCHEMA #createSnapshot | SELECT INDEX func=ID //not support yet LR_BRACKET p1=fullPath COMMA p2=fullPath COMMA n1=timeValue COMMA n2=timeValue COMMA @@ -883,6 +884,18 @@ LATEST : L A T E S T ; +SNAPSHOT + : S N A P S H O T + ; + +FOR + : F O R + ; + +SCHEMA + : S C H E M A + ; + //============================ // End of the keywords list //============================ diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java index 16847ca..1506d2b 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java @@ -1780,24 +1780,29 @@ public class MManager { "MTree snapshot need not be created. Current mlog line number: {}, last snapshot line number: {}", logWriter.getLineNumber(), lastSnapshotLogLineNumber); } else { - lock.readLock().lock(); - logger.info("Start creating MTree snapshot. This may take a while..."); - try { - mtree.serializeTo(mtreeSnapshotTmpPath, logWriter.getLineNumber()); - lastSnapshotLogLineNumber = logWriter.getLineNumber(); - File tmpFile = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotTmpPath); - File snapshotFile = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotPath); - if (snapshotFile.exists()) { - Files.delete(snapshotFile.toPath()); - } - if (tmpFile.renameTo(snapshotFile)) { - logger.info("Finish creating MTree snapshot to {}.", mtreeSnapshotPath); - } - } catch (IOException e) { - logger.warn("Failed to create MTree snapshot to {}", mtreeSnapshotPath, e); - } finally { - lock.readLock().unlock(); + createMTreeSnapshot(); + } + } + + public void createMTreeSnapshot() { + lock.readLock().lock(); + logger.info("Start creating MTree snapshot. This may take a while..."); + try { + mtree.serializeTo(mtreeSnapshotTmpPath, logWriter.getLineNumber()); + lastSnapshotLogLineNumber = logWriter.getLineNumber(); + File tmpFile = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotTmpPath); + File snapshotFile = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotPath); + if (snapshotFile.exists()) { + Files.delete(snapshotFile.toPath()); + } + if (tmpFile.renameTo(snapshotFile)) { + logger.info("Finish creating MTree snapshot to {}, to mlog line number: {}", + mtreeSnapshotPath, lastSnapshotLogLineNumber); } + } catch (IOException e) { + logger.warn("Failed to create MTree snapshot to {}", mtreeSnapshotPath, e); + } finally { + lock.readLock().unlock(); } } } diff --git a/server/src/main/java/org/apache/iotdb/db/qp/Planner.java b/server/src/main/java/org/apache/iotdb/db/qp/Planner.java index 3b31bbe..74f6aa8 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/Planner.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/Planner.java @@ -100,6 +100,7 @@ public class Planner { case CLEAR_CACHE: case NULL: case SHOW_MERGE_STATUS: + case CREATE_SNAPSHOT: return operator; case QUERY: case UPDATE: diff --git a/server/src/main/java/org/apache/iotdb/db/qp/constant/SQLConstant.java b/server/src/main/java/org/apache/iotdb/db/qp/constant/SQLConstant.java index 1acf1a7..ca075a6 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/constant/SQLConstant.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/constant/SQLConstant.java @@ -152,6 +152,8 @@ public class SQLConstant { public static final int TOK_SHOW_MERGE_STATUS = 87; + public static final int TOK_CREATE_SNAPSHOT = 88; + public static final Map<Integer, String> tokenSymbol = new HashMap<>(); public static final Map<Integer, String> tokenNames = new HashMap<>(); public static final Map<Integer, Integer> reverseWords = new HashMap<>(); diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java index 3cf570e..3e11619 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java @@ -106,6 +106,7 @@ import org.apache.iotdb.db.qp.physical.sys.AlterTimeSeriesPlan; import org.apache.iotdb.db.qp.physical.sys.AuthorPlan; import org.apache.iotdb.db.qp.physical.sys.ClearCachePlan; import org.apache.iotdb.db.qp.physical.sys.CountPlan; +import org.apache.iotdb.db.qp.physical.sys.CreateSnapshotPlan; import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; import org.apache.iotdb.db.qp.physical.sys.DataAuthPlan; import org.apache.iotdb.db.qp.physical.sys.DeleteStorageGroupPlan; @@ -261,6 +262,9 @@ public class PlanExecutor implements IPlanExecutor { case CLEAR_CACHE: operateClearCache((ClearCachePlan) plan); return true; + case CREATE_SNAPSHOT: + operateCreateSnapshot((CreateSnapshotPlan) plan); + return true; default: throw new UnsupportedOperationException( String.format("operation %s is not supported", plan.getOperatorType())); @@ -282,6 +286,10 @@ public class PlanExecutor implements IPlanExecutor { TimeSeriesMetadataCache.getInstance().clear(); } + private void operateCreateSnapshot(CreateSnapshotPlan plan) { + mManager.createMTreeSnapshot(); + } + private void operateFlush(FlushPlan plan) throws StorageGroupNotSetException { if (plan.getPaths() == null) { StorageEngine.getInstance().syncCloseAllProcessor(); @@ -1124,9 +1132,9 @@ public class PlanExecutor implements IPlanExecutor { if (measurementNode.getSchema().getType() != insertTabletPlan.getDataTypes()[i]) { if (!enablePartialInsert) { throw new QueryProcessException(String.format( - "Datatype mismatch, Insert measurement %s type %s, metadata tree type %s", - measurement, insertTabletPlan.getDataTypes()[i], - measurementNode.getSchema().getType())); + "Datatype mismatch, Insert measurement %s type %s, metadata tree type %s", + measurement, insertTabletPlan.getDataTypes()[i], + measurementNode.getSchema().getType())); } else { insertTabletPlan.markMeasurementInsertionFailed(i); continue; @@ -1140,7 +1148,7 @@ public class PlanExecutor implements IPlanExecutor { StorageEngine.getInstance().insertTablet(insertTabletPlan); if (insertTabletPlan.getFailedMeasurements() != null) { throw new StorageEngineException( - "failed to insert measurements " + insertTabletPlan.getFailedMeasurements()); + "failed to insert measurements " + insertTabletPlan.getFailedMeasurements()); } } catch (StorageEngineException | MetadataException e) { throw new QueryProcessException(e); diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java index 746326c..ead36f6 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java @@ -76,6 +76,6 @@ public abstract class Operator { GRANT_WATERMARK_EMBEDDING, REVOKE_WATERMARK_EMBEDDING, TTL, DELETE_STORAGE_GROUP, LOAD_CONFIGURATION, SHOW, LOAD_FILES, REMOVE_FILE, MOVE_FILE, LAST, GROUP_BY_FILL, ALTER_TIMESERIES, FLUSH, MERGE, FULL_MERGE, CLEAR_CACHE, - SHOW_MERGE_STATUS + SHOW_MERGE_STATUS, CREATE_SNAPSHOT } } diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/CreateSnapshotOperator.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/CreateSnapshotOperator.java new file mode 100644 index 0000000..341eb03 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/CreateSnapshotOperator.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.qp.logical.sys; + +import org.apache.iotdb.db.qp.logical.RootOperator; + +public class CreateSnapshotOperator extends RootOperator { + + public CreateSnapshotOperator(int tokenIntType) { + super(tokenIntType); + operatorType = OperatorType.CREATE_SNAPSHOT; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CreateSnapshotPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CreateSnapshotPlan.java new file mode 100644 index 0000000..3e46a54 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CreateSnapshotPlan.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.qp.physical.sys; + +import java.util.List; +import org.apache.iotdb.db.qp.logical.Operator.OperatorType; +import org.apache.iotdb.db.qp.physical.PhysicalPlan; +import org.apache.iotdb.tsfile.read.common.Path; + +public class CreateSnapshotPlan extends PhysicalPlan { + + public CreateSnapshotPlan() { + super(false, OperatorType.CREATE_SNAPSHOT); + } + + @Override + public List<Path> getPaths() { + return null; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java index ebd41f0..f27e02f 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java @@ -48,6 +48,7 @@ import org.apache.iotdb.db.qp.logical.sys.AuthorOperator; import org.apache.iotdb.db.qp.logical.sys.AuthorOperator.AuthorType; import org.apache.iotdb.db.qp.logical.sys.ClearCacheOperator; import org.apache.iotdb.db.qp.logical.sys.CountOperator; +import org.apache.iotdb.db.qp.logical.sys.CreateSnapshotOperator; import org.apache.iotdb.db.qp.logical.sys.CreateTimeSeriesOperator; import org.apache.iotdb.db.qp.logical.sys.DataAuthOperator; import org.apache.iotdb.db.qp.logical.sys.DeleteStorageGroupOperator; @@ -78,6 +79,7 @@ import org.apache.iotdb.db.qp.strategy.SqlBaseParser.ConstantContext; import org.apache.iotdb.db.qp.strategy.SqlBaseParser.CountNodesContext; import org.apache.iotdb.db.qp.strategy.SqlBaseParser.CountTimeseriesContext; import org.apache.iotdb.db.qp.strategy.SqlBaseParser.CreateRoleContext; +import org.apache.iotdb.db.qp.strategy.SqlBaseParser.CreateSnapshotContext; import org.apache.iotdb.db.qp.strategy.SqlBaseParser.CreateTimeseriesContext; import org.apache.iotdb.db.qp.strategy.SqlBaseParser.CreateUserContext; import org.apache.iotdb.db.qp.strategy.SqlBaseParser.DateExpressionContext; @@ -1481,10 +1483,7 @@ public class LogicalGenerator extends SqlBaseBaseListener { /** * parse time expression, which is addition and subtraction expression of duration time, now() or - * DataTimeFormat time. - * <p> - * eg. now() + 1d - 2h - * </p> + * DataTimeFormat time. <p> eg. now() + 1d - 2h </p> */ private Long parseDateExpression(DateExpressionContext ctx) { long time; @@ -1622,4 +1621,9 @@ public class LogicalGenerator extends SqlBaseBaseListener { super.enterShowMergeStatus(ctx); initializedOperator = new ShowMergeStatusOperator(SQLConstant.TOK_SHOW_MERGE_STATUS); } + + public void enterCreateSnapshot(CreateSnapshotContext ctx) { + super.enterCreateSnapshot(ctx); + initializedOperator = new CreateSnapshotOperator(SQLConstant.TOK_CREATE_SNAPSHOT); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java index d050e5d..2ba9a20 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java @@ -76,6 +76,7 @@ import org.apache.iotdb.db.qp.physical.sys.AlterTimeSeriesPlan; import org.apache.iotdb.db.qp.physical.sys.AuthorPlan; import org.apache.iotdb.db.qp.physical.sys.ClearCachePlan; import org.apache.iotdb.db.qp.physical.sys.CountPlan; +import org.apache.iotdb.db.qp.physical.sys.CreateSnapshotPlan; import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; import org.apache.iotdb.db.qp.physical.sys.DataAuthPlan; import org.apache.iotdb.db.qp.physical.sys.DeleteStorageGroupPlan; @@ -258,6 +259,8 @@ public class PhysicalGenerator { return new ClearCachePlan(); case SHOW_MERGE_STATUS: return new ShowMergeStatusPlan(); + case CREATE_SNAPSHOT: + return new CreateSnapshotPlan(); default: throw new LogicalOperatorException(operator.getType().toString(), ""); }
