samperson1997 commented on a change in pull request #2024:
URL: https://github.com/apache/iotdb/pull/2024#discussion_r522788691
##########
File path:
server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
##########
@@ -1199,6 +1217,36 @@ public void
enterAttributeClauses(AttributeClausesContext ctx) {
initializedOperator = createTimeSeriesOperator;
}
+ @Override
+ public void enterIndexWithClause(SqlBaseParser.IndexWithClauseContext ctx) {
+ super.enterIndexWithClause(ctx);
+ IndexType indexType;
+ try {
+ indexType = IndexType.getIndexType(ctx.indexName.getText());
+ } catch (UnsupportedIndexTypeException e) {
+ throw new SQLParserException(
+ String.format("index type %s is not supported.",
ctx.indexName.getText()));
Review comment:
```suggestion
String.format(ctx.indexName.getText()));
```
##########
File path:
server/src/main/java/org/apache/iotdb/db/exception/index/UnsupportedIndexTypeException.java
##########
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.iotdb.db.exception.index;
+
+
+import org.apache.iotdb.rpc.TSStatusCode;
+
+public class UnsupportedIndexTypeException extends QueryIndexException {
+
+ private static final long serialVersionUID = -7091830159338197925L;
+
+ public UnsupportedIndexTypeException(String message) {
+ super(message, TSStatusCode.UNSUPPORTED_INDEX_TYPE_ERROR.getStatusCode());
+ }
Review comment:
Now we put the public used message in the exception class, so that we
don't need to repeat the message every time the exception is thrown : )
```suggestion
public UnsupportedIndexTypeException(String indexType) {
super("Unsupported index type: " + indexType,
TSStatusCode.UNSUPPORTED_INDEX_TYPE_ERROR.getStatusCode());
}
```
##########
File path: server/src/main/java/org/apache/iotdb/db/index/common/IndexType.java
##########
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.index.common;
+
+import org.apache.iotdb.db.exception.index.UnsupportedIndexTypeException;
+import org.apache.iotdb.tsfile.exception.NotImplementedException;
+
+public enum IndexType {
+
+ NO_INDEX,
+ RTREE_PAA,
+ ELB_INDEX,
+ KV_INDEX
+ ;
+
+ /**
+ * judge the index type.
+ *
+ * @param i an integer used to determine index type
+ * @return index type
+ */
+ public static IndexType deserialize(short i) {
+ switch (i) {
+ case 0:
+ return NO_INDEX;
+ case 1:
+ return RTREE_PAA;
+ case 2:
+ return ELB_INDEX;
+ case 3:
+ return KV_INDEX;
+ default:
+ throw new NotImplementedException("Given index is not implemented");
+ }
+ }
+
+ public static int getSerializedSize() {
+ return Short.BYTES;
+ }
+
+ /**
+ * judge the index deserialize type.
+ *
+ * @return the integer used to determine index type
+ */
+ public short serialize() {
+ switch (this) {
+ case NO_INDEX:
+ return 0;
+ case RTREE_PAA:
+ return 1;
+ case ELB_INDEX:
+ return 2;
+ case KV_INDEX:
+ return 3;
+ default:
+ throw new NotImplementedException("Given index is not implemented");
+ }
+ }
+
+ public static IndexType getIndexType(String indexTypeString)
+ throws UnsupportedIndexTypeException {
+ String normalized = indexTypeString.toUpperCase();
+ try {
+ return IndexType.valueOf(normalized);
+ } catch (IllegalArgumentException e) {
+ throw new UnsupportedIndexTypeException("unsupported index type:" +
indexTypeString);
Review comment:
```suggestion
throw new UnsupportedIndexTypeException(indexTypeString);
```
##########
File path:
server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
##########
@@ -1712,4 +1779,162 @@ public void enterCreateSnapshot(CreateSnapshotContext
ctx) {
super.enterCreateSnapshot(ctx);
initializedOperator = new
CreateSnapshotOperator(SQLConstant.TOK_CREATE_SCHEMA_SNAPSHOT);
}
+
+ /**
+ * for create index command, time should only have an end time.
+ *
+ * @param operator create index plan
+ */
+ private long parseCreateIndexFilter(CreateIndexOperator operator) {
+ FilterOperator filterOperator = operator.getFilterOperator();
+ if (filterOperator.getTokenIntType() != SQLConstant.GREATERTHAN
+ && filterOperator.getTokenIntType() !=
SQLConstant.GREATERTHANOREQUALTO) {
+ throw new SQLParserException(
+ "For create index command, where clause must be like : time > XXX or
time >= XXX");
+ }
+ long time = Long.parseLong(((BasicFunctionOperator)
filterOperator).getValue());
+ if (filterOperator.getTokenIntType() == SQLConstant.LESSTHAN) {
+ time = time - 1;
+ }
+ return time;
+ }
+
+ /**
+ * For parsing CreateIndex
+ *
+ * <p>The default implementation does nothing.</p>
+ */
+ @Override
+ public void enterCreateIndex(SqlBaseParser.CreateIndexContext ctx) {
+ super.enterCreateIndex(ctx);
+ createIndexOp = new CreateIndexOperator(SQLConstant.TOK_CREATE_INDEX);
+ selectOp = new SelectOperator(SQLConstant.TOK_SELECT);
+ List<PrefixPathContext> prefixPaths =
Collections.singletonList(ctx.prefixPath());
+ for (PrefixPathContext prefixPath : prefixPaths) {
+ PartialPath path = parsePrefixPath(prefixPath);
+ selectOp.addSelectPath(path);
+ }
+ createIndexOp.setSelectOperator(selectOp);
+ initializedOperator = createIndexOp;
+ operatorType = SQLConstant.TOK_CREATE_INDEX;
+ }
+
+ @Override
+ public void enterDropIndex(SqlBaseParser.DropIndexContext ctx) {
+ super.enterDropIndex(ctx);
+ DropIndexOperator dropIndexOperator = new
DropIndexOperator(SQLConstant.TOK_DROP_INDEX);
+ selectOp = new SelectOperator(SQLConstant.TOK_SELECT);
+ List<PrefixPathContext> prefixPaths =
Collections.singletonList(ctx.prefixPath());
+ for (PrefixPathContext prefixPath : prefixPaths) {
+ PartialPath path = parsePrefixPath(prefixPath);
+ selectOp.addSelectPath(path);
+ }
+ dropIndexOperator.setSelectOperator(selectOp);
+ try {
+
dropIndexOperator.setIndexType(IndexType.getIndexType(ctx.indexName.getText()));
+ } catch (UnsupportedIndexTypeException e) {
+ throw new SQLParserException(
+ String.format("index type %s is not supported.",
ctx.indexName.getText()));
Review comment:
```suggestion
String.format(ctx.indexName.getText()));
```
##########
File path:
server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
##########
@@ -28,6 +28,7 @@
import org.apache.iotdb.db.rescon.PrimitiveArrayManager;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.common.BatchData;
Review comment:
Remove unused import
##########
File path: server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
##########
@@ -271,6 +271,38 @@
*/
private boolean enableMemControl = true;
+ /**
+ * Is the write ahead log enable.
+ */
+ private boolean enableIndex = false;
+
+ /**
+ * How many threads can concurrently build index. When <= 0, use CPU core
number.
+ */
+ private int concurrentIndexBuildThread =
Runtime.getRuntime().availableProcessors();
+
+ /**
+ * If we enable the memory-control mechanism during index building , {@code
indexBufferSize}
+ * refers to the byte-size of memory buffer threshold. For each index
processor, all indexes in
+ * one {@linkplain org.apache.iotdb.db.index.IndexFileProcessor
IndexFileProcessor} share a total
+ * common buffer size. With the memory-control mechanism, the occupied
memory of all raw data and
+ * index structures will be counted. If the memory buffer size reaches this
threshold, the indexes
+ * will be flushed to the disk file. As a result, data in one series may be
divided into more than
+ * one part and indexed separately.
+ */
+ private long indexBufferSize = 128 * 1024 * 1024L;
+
+ /**
+ * the index framework adopts sliding window model to preprocess the
original tv list in the
+ * subsequence matching task.
+ */
+ private int defaultIndexWindowRange = 10;
Review comment:
```suggestion
private int indexWindowRange = 10;
```
##########
File path:
server/src/main/java/org/apache/iotdb/db/qp/physical/sys/DropIndexPlan.java
##########
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.qp.physical.sys;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import org.apache.iotdb.db.exception.metadata.IllegalPathException;
+import org.apache.iotdb.db.index.common.IndexType;
+import org.apache.iotdb.db.metadata.PartialPath;
+import org.apache.iotdb.db.qp.logical.Operator;
+import org.apache.iotdb.db.qp.logical.Operator.OperatorType;
+import org.apache.iotdb.db.qp.physical.PhysicalPlan;
+
+public class DropIndexPlan extends PhysicalPlan {
+
+ protected List<PartialPath> paths;
+ private IndexType indexType;
+
+ public DropIndexPlan() {
+ super(false, Operator.OperatorType.DELETE_TIMESERIES);
Review comment:
Should it be `DROP_INDEX`?
```suggestion
super(false, Operator.OperatorType.DROP_INDEX);
```
##########
File path:
tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java
##########
@@ -708,6 +708,26 @@ public static ByteBuffer
readByteBufferWithSelfDescriptionLength(ByteBuffer buff
return byteBuffer;
}
+ /**
+ * read bytes from an inputStream where the length is specified at the head
of the inputStream.
+ * @param inputStream contains a int-type length and a stream
+ * @return bytebuffer
+ * @throws IOException if the read length doesn't equal to the self
description length.
+ */
+ public static ByteBuffer readByteBufferWithSelfDescriptionLength(InputStream
inputStream)
+ throws IOException {
+ int length = readInt(inputStream);
Review comment:
This method should also ensure that it is an int value in inputStream.
Maybe we should imply it in the method name or in comment?
##########
File path:
server/src/main/java/org/apache/iotdb/db/index/common/IndexConstant.java
##########
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.iotdb.db.index.common;
+
+public class IndexConstant {
+
+ // SQL show
+ public static final String ID = "ID";
+
+
+ public static final String META_DIR_NAME = "meta";
+ public static final String STORAGE_GROUP_INDEXING_SUFFIX = ".sg_indexing";
+ public static final String STORAGE_GROUP_INDEXED_SUFFIX = ".sg_index";
+
+ public static final String INDEXING_SUFFIX = ".indexing";
+ public static final String INDEXED_SUFFIX = ".index";
+
+ // whole matching
+ public static final int NON_SET_TOP_K = -1;
+ public static final String TOP_K = "TOP_K";
+
+ // subsequence matching: sliding window
+ public static final String INDEX_WINDOW_RANGE = "INDEX_WINDOW_RANGE";
+ public static final String INDEX_RANGE_STRATEGY = "INDEX_RANGE_STRATEGY";
+ public static final String INDEX_SLIDE_STEP = "INDEX_SLIDE_STEP";
+
+ public static final String INDEX_MAGIC = "IoTDBIndex";
+ public static final String DEFAULT_PROP_NAME = "DEFAULT";
+
+ public static final int INDEX_MAP_INIT_RESERVE_SIZE = 5;
+
Review comment:
There are many unused constants in this class, I wonder whether we
should define them in this PR or not : )
##########
File path: server/src/assembly/resources/conf/iotdb-engine.properties
##########
@@ -572,3 +572,22 @@
authorizer_provider_class=org.apache.iotdb.db.auth.authorizer.LocalFileAuthorize
#If OpenIdAuthorizer is enabled, then openID_url must be set.
#openID_url=
+
+# Uncomment following fields to configure the index root directory.
+# For Window platform, the index is as follows:
+# index_root_dir=data\\index
+# For Linux platform
+# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
+# index_root_dir=data/index
+
+# Is index enable
+enable_index=false
+
+# How many threads can concurrently build index. When <= 0, use CPU core
number.
+concurrent_index_build_thread=0
+
+# the default size of sliding window used for the subsequence matching in
index framework
+default_index_window_range=10
Review comment:
I think all configurations are the default value...
```suggestion
# size of sliding window used for the subsequence matching in index framework
index_window_range=10
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
