This is an automated email from the ASF dual-hosted git repository.
kangrong pushed a commit to branch f_index_dev
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/f_index_dev by this push:
new 2f2016d fix bug of whole matching index
2f2016d is described below
commit 2f2016db019e70b391a5c3aa68ed1bf7bc2a6b0e
Author: kr11 <3095717866.com>
AuthorDate: Fri May 14 11:27:17 2021 +0800
fix bug of whole matching index
查询时没考虑未创建索引的序列；flush之后involvedSet没有序列化，重启后就消失了导致错误
---
.../org/apache/iotdb/db/index/IndexProcessor.java | 3 +-
.../iotdb/db/index/algorithm/IoTDBIndex.java | 6 +-
.../apache/iotdb/db/index/algorithm/NoIndex.java | 2 +-
.../iotdb/db/index/algorithm/RTreeIndex.java | 28 ++--
.../iotdb/db/index/algorithm/elb/ELBIndex.java | 7 +-
.../iotdb/db/index/algorithm/mmhh/MMHHIndex.java | 57 ++++----
.../iotdb/db/index/algorithm/rtree/RTree.java | 7 +-
.../iotdb/db/index/algorithm/rtree/RTreeTest.java | 3 +-
.../apache/iotdb/db/index/it/DemoMMHHWindIT.java | 150 ++++++++++++---------
.../apache/iotdb/db/index/it/DemoRTreeWindIT.java | 148 +++++++++++---------
10 files changed, 236 insertions(+), 175 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/index/IndexProcessor.java
b/server/src/main/java/org/apache/iotdb/db/index/IndexProcessor.java
index 86e0ff7..e541479 100644
--- a/server/src/main/java/org/apache/iotdb/db/index/IndexProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/index/IndexProcessor.java
@@ -62,6 +62,7 @@ import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import static org.apache.iotdb.db.index.common.IndexType.ANY_FOR_QUERY;
+import static org.apache.iotdb.db.index.common.IndexType.ELB_INDEX;
/**
* Each {@code IndexProcessor} manages all index instances under an
<b>IndexSeries</b>.
@@ -430,7 +431,7 @@ public class IndexProcessor implements
Comparable<IndexProcessor> {
void endFlushMemTable() {
// wait until all flushing tasks end.
try {
- waitingFlushEndAndDo(() -> {});
+ waitingFlushEndAndDo(()-> allPathsIndexMap.forEach((k,index)->
index.serializeIndex()));
} catch (IOException ignored) {
// the exception is ignored
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/index/algorithm/IoTDBIndex.java
b/server/src/main/java/org/apache/iotdb/db/index/algorithm/IoTDBIndex.java
index 4cded51..6ccdc7c 100644
--- a/server/src/main/java/org/apache/iotdb/db/index/algorithm/IoTDBIndex.java
+++ b/server/src/main/java/org/apache/iotdb/db/index/algorithm/IoTDBIndex.java
@@ -82,8 +82,8 @@ public abstract class IoTDBIndex {
/** A new item has been pre-processed by the FeatureExtractor, now the index
can insert it. */
public abstract boolean buildNext() throws IndexManagerException;
- /** This index will be closed, it's time to serialize in-memory data to disk
for next open. */
- protected abstract void flushIndex();
+ /** It's time to serialize in-memory data to disk. */
+ public abstract void serializeIndex();
/**
* execute index query and return the result.
@@ -123,7 +123,7 @@ public abstract class IoTDBIndex {
/** Close the index, release resources of the index structure and the
feature extractor. */
public ByteBuffer closeAndRelease() throws IOException {
- flushIndex();
+ serializeIndex();
if (indexFeatureExtractor != null) {
return indexFeatureExtractor.closeAndRelease();
} else {
diff --git
a/server/src/main/java/org/apache/iotdb/db/index/algorithm/NoIndex.java
b/server/src/main/java/org/apache/iotdb/db/index/algorithm/NoIndex.java
index 1e9ff88..8888ad3 100644
--- a/server/src/main/java/org/apache/iotdb/db/index/algorithm/NoIndex.java
+++ b/server/src/main/java/org/apache/iotdb/db/index/algorithm/NoIndex.java
@@ -51,7 +51,7 @@ public class NoIndex extends IoTDBIndex {
}
@Override
- protected void flushIndex() {
+ public void serializeIndex() {
// NoIndex does nothing
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/index/algorithm/RTreeIndex.java
b/server/src/main/java/org/apache/iotdb/db/index/algorithm/RTreeIndex.java
index 653b1c7..a385cc7 100644
--- a/server/src/main/java/org/apache/iotdb/db/index/algorithm/RTreeIndex.java
+++ b/server/src/main/java/org/apache/iotdb/db/index/algorithm/RTreeIndex.java
@@ -80,6 +80,7 @@ import static
org.apache.iotdb.db.index.common.IndexConstant.SEED_PICKER;
import static org.apache.iotdb.db.index.common.IndexConstant.SERIES_LENGTH;
import static org.apache.iotdb.db.index.common.IndexConstant.THRESHOLD;
import static org.apache.iotdb.db.index.common.IndexConstant.TOP_K;
+import static org.apache.iotdb.db.index.common.IndexType.MMHH;
import static org.apache.iotdb.db.index.common.IndexType.RTREE_PAA;
/**
@@ -104,7 +105,9 @@ public abstract class RTreeIndex extends IoTDBIndex {
* <p>The range of dimension {@code i} is {@code [corner[i],
corner[i]+range[i]]}
*/
private final boolean usePointType;
- /** For generality, RTree only store ids of identifiers or others. */
+ /**
+ * For generality, RTree only store ids of identifiers or others.
+ */
private RTree<PartialPath> rTree;
protected float[] currentLowerBounds;
@@ -128,7 +131,7 @@ public abstract class RTreeIndex extends IoTDBIndex {
if (indexDirFile.exists()) {
logger.info("reload index {} from {}", RTREE_PAA, indexDir);
featureFile = IndexUtils.getIndexFile(indexDir + File.separator +
"rTreeFeature");
- deserializeFeatures();
+ deserializeRTree();
} else {
logger.info("mkdir index {} in {}", RTREE_PAA, indexDir);
indexDirFile.mkdirs();
@@ -136,23 +139,27 @@ public abstract class RTreeIndex extends IoTDBIndex {
}
}
- private void deserializeFeatures() {
+ private void deserializeRTree() {
if (!featureFile.exists()) {
return;
}
try (InputStream inputStream = new FileInputStream(featureFile)) {
- this.rTree = RTree.deserializePartialPath(inputStream);
+ logger.info("reload index {} from {}", RTREE_PAA, featureFile);
+ this.rTree = RTree.deserialize(inputStream, involvedPathSet);
+ logger.info("Deserialize RTreeIndex rTree: {}",
rTree.toString().substring(0, 10));
+ logger.info("Deserialize InvolvedSet: {}, {}", involvedPathSet.size(),
involvedPathSet);
} catch (IOException e) {
logger.error("Error when deserialize ELB features. Given up.", e);
}
}
@Override
- protected void flushIndex() {
+ public void serializeIndex() {
logger.info("RTreeIndex {} starts serialization", indexSeries);
+ logger.info("RTreeIndex RTree to serialized: {}",
rTree.toString().substring(0, 10));
+ logger.info("Serialize InvolvedSet: {}, {}", involvedPathSet.size(),
involvedPathSet);
try (OutputStream outputStream = new FileOutputStream(featureFile)) {
rTree.serialize(outputStream);
- rTree.clear();
} catch (IOException e) {
logger.error("Error when serialize router. Given up.", e);
}
@@ -241,6 +248,7 @@ public abstract class RTreeIndex extends IoTDBIndex {
return res;
}
+ @Override
public void endFlushTask() {
super.endFlushTask();
currentInsertPath = null;
@@ -267,12 +275,16 @@ public abstract class RTreeIndex extends IoTDBIndex {
// protected abstract BiConsumer<String, InputStream> getDeserializeFunc();
// protected abstract List<Identifier> getQueryCandidates(List<Integer>
candidateIds);
- /** */
+ /**
+ *
+ */
protected abstract float[] calcQueryFeature(double[] patterns);
public static class RTreeQueryStruct {
- /** features is represented by float array */
+ /**
+ * features is represented by float array
+ */
float[] patternFeatures;
// TriFunction<float[], float[], float[], Double> calcLowerDistFunc;
//
diff --git
a/server/src/main/java/org/apache/iotdb/db/index/algorithm/elb/ELBIndex.java
b/server/src/main/java/org/apache/iotdb/db/index/algorithm/elb/ELBIndex.java
index 99f8cee..84f149d 100644
--- a/server/src/main/java/org/apache/iotdb/db/index/algorithm/elb/ELBIndex.java
+++ b/server/src/main/java/org/apache/iotdb/db/index/algorithm/elb/ELBIndex.java
@@ -175,8 +175,13 @@ public class ELBIndex extends IoTDBIndex {
}
}
+// @Override
+// public void endFlushTask() {
+// super.endFlushTask();
+// }
+
@Override
- protected void flushIndex() {
+ public void serializeIndex() {
logger.info("ELBIndex {} start serialization", indexSeries);
try (OutputStream outputStream = new FileOutputStream(featureFile)) {
ReadWriteIOUtils.write(windowBlockFeatures.size(), outputStream);
diff --git
a/server/src/main/java/org/apache/iotdb/db/index/algorithm/mmhh/MMHHIndex.java
b/server/src/main/java/org/apache/iotdb/db/index/algorithm/mmhh/MMHHIndex.java
index 8367b95..48fd146 100644
---
a/server/src/main/java/org/apache/iotdb/db/index/algorithm/mmhh/MMHHIndex.java
+++
b/server/src/main/java/org/apache/iotdb/db/index/algorithm/mmhh/MMHHIndex.java
@@ -80,7 +80,9 @@ import static
org.apache.iotdb.db.index.common.IndexConstant.SERIES_LENGTH;
import static org.apache.iotdb.db.index.common.IndexConstant.TOP_K;
import static org.apache.iotdb.db.index.common.IndexType.MMHH;
-/** Refer to: Kang, Rong, et al. Maximum-margin hamming hashing. ICCV.
IEEE/CVF. 2019: 8252-8261. */
+/**
+ * Refer to: Kang, Rong, et al. Maximum-margin hamming hashing. ICCV.
IEEE/CVF. 2019: 8252-8261.
+ */
public class MMHHIndex extends IoTDBIndex {
private static final Logger logger =
LoggerFactory.getLogger(MMHHIndex.class);
@@ -109,8 +111,10 @@ public class MMHHIndex extends IoTDBIndex {
File indexDirFile = IndexUtils.getIndexFile(indexDir);
if (indexDirFile.exists()) {
- logger.info("reload index {} from {}", MMHH, indexDir);
+ logger.info("reload index {} from {}", MMHH, featureFile);
deserializeHashLookup();
+ logger.info("Deserialize MMHHIndex hashTable: {}", hashLookupTable);
+ logger.info("Deserialize InvolvedSet: {}, {}", involvedPathSet.size(),
involvedPathSet);
} else {
indexDirFile.mkdirs();
hashLookupTable = new HashMap<>();
@@ -145,7 +149,10 @@ public class MMHHIndex extends IoTDBIndex {
return res;
}
- /** should be concise into WholeIndex or IoTDBIndex, it's duplicate */
+ /**
+ * should be concise into WholeIndex or IoTDBIndex, it's duplicate
+ */
+ @Override
public void endFlushTask() {
super.endFlushTask();
currentInsertPath = null;
@@ -170,19 +177,21 @@ public class MMHHIndex extends IoTDBIndex {
}
@Override
- protected void flushIndex() {
+ public void serializeIndex() {
logger.info("MMHHIndex {} starts serialization", indexSeries);
+ logger.info("MMHHIndex hashTable to serialized: {}", hashLookupTable);
+ logger.info("Serialize InvolvedSet: {}, {}", involvedPathSet.size(),
involvedPathSet);
serializeHashLookup();
- hashLookupTable.clear();
logger.info("MMHHIndex {} finishes serialization", indexSeries);
}
private void deserializeHashLookup() {
+ hashLookupTable = new HashMap<>();
+
if (!featureFile.exists()) {
return;
}
try (InputStream inputStream = new FileInputStream(featureFile)) {
- hashLookupTable = new HashMap<>();
int tableSize = ReadWriteIOUtils.readInt(inputStream);
for (int i = 0; i < tableSize; i++) {
Long key = ReadWriteIOUtils.readLong(inputStream);
@@ -191,6 +200,7 @@ public class MMHHIndex extends IoTDBIndex {
for (int j = 0; j < bucketSize; j++) {
Long v = ReadWriteIOUtils.readLong(inputStream);
bucket.add(v);
+ involvedPathSet.add(seriesIdToPath(v));
itemSize++;
}
hashLookupTable.put(key, bucket);
@@ -222,7 +232,9 @@ public class MMHHIndex extends IoTDBIndex {
private static class MMHHQueryStruct {
- /** features is represented by float array */
+ /**
+ * features is represented by float array
+ */
// float[] patternFeatures;
// TriFunction<float[], float[], float[], Double> calcLowerDistFunc;
//
@@ -382,23 +394,6 @@ public class MMHHIndex extends IoTDBIndex {
if (!uninvolvedList.isEmpty()) {
Function<PartialPath, TVList> loadSeriesFunc =
getLoadSeriesFunc(context, tsDataType, mmhhFeatureExtractor);
- BiFunction<double[], TVList, Double> exactDistFunc =
getCalcExactDistFunc();
- //
- // PriorityQueue<DistSeries> topKPQ = new
PriorityQueue<>(struct.topK,
- // new DistSeriesComparator());
- // topKPQ.addAll(res);
- // double kthMinDist = topKPQ.isEmpty() ? Double.MAX_VALUE :
topKPQ.peek().dist;
- // for (PartialPath path : uninvolvedList) {
- // TVList tvList = loadSeriesFunc.apply(path);
- // double tempDist = exactDistFunc.apply(struct.patterns, tvList);
- // if (topKPQ.size() < struct.topK || tempDist < kthMinDist) {
- // if (topKPQ.size() == struct.topK) {
- // topKPQ.poll();
- // }
- // topKPQ.add(new DistSeries(tempDist, tvList, path));
- // kthMinDist = topKPQ.peek().dist;
- // }
- // }
for (PartialPath path : uninvolvedList) {
TVList rawData = loadSeriesFunc.apply(path);
res.add(new DistSeries(0, rawData, path));
@@ -445,7 +440,9 @@ public class MMHHIndex extends IoTDBIndex {
return res;
}
- /** if res has reached topK */
+ /**
+ * if res has reached topK
+ */
private boolean scanBucket(
long queryCode,
int doneIdx,
@@ -511,12 +508,16 @@ public class MMHHIndex extends IoTDBIndex {
}
}
- private DistSeries readRawData(
- int hammingDist, Long seriesId, Function<PartialPath, TVList>
loadSeriesFunc) {
+ private PartialPath seriesIdToPath(Long seriesId) {
int len = indexSeries.getNodeLength();
String[] nodes = Arrays.copyOf(indexSeries.getNodes(), len);
nodes[len - 2] = seriesId.toString();
- PartialPath path = new PartialPath(nodes);
+ return new PartialPath(nodes);
+ }
+
+ private DistSeries readRawData(
+ int hammingDist, Long seriesId, Function<PartialPath, TVList>
loadSeriesFunc) {
+ PartialPath path = seriesIdToPath(seriesId);
TVList rawData = loadSeriesFunc.apply(path);
return new DistSeries(hammingDist, rawData, path);
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/index/algorithm/rtree/RTree.java
b/server/src/main/java/org/apache/iotdb/db/index/algorithm/rtree/RTree.java
index ecf4fc5..7639447 100644
--- a/server/src/main/java/org/apache/iotdb/db/index/algorithm/rtree/RTree.java
+++ b/server/src/main/java/org/apache/iotdb/db/index/algorithm/rtree/RTree.java
@@ -431,7 +431,8 @@ public class RTree<T> {
}
}
- public static RTree<PartialPath> deserializePartialPath(InputStream
inputStream)
+ public static RTree<PartialPath> deserialize(InputStream inputStream,
+ Set<PartialPath> involvedPathSet)
throws IOException {
int dim = ReadWriteIOUtils.readInt(inputStream);
int nMaxPerNode = ReadWriteIOUtils.readInt(inputStream);
@@ -444,7 +445,9 @@ public class RTree<T> {
inputStream,
in -> {
try {
- return new PartialPath(ReadWriteIOUtils.readString(inputStream));
+ PartialPath path = new
PartialPath(ReadWriteIOUtils.readString(inputStream));
+ involvedPathSet.add(path);
+ return path;
} catch (IOException | IllegalPathException e) {
logger.error("read path error", e);
return null;
diff --git
a/server/src/test/java/org/apache/iotdb/db/index/algorithm/rtree/RTreeTest.java
b/server/src/test/java/org/apache/iotdb/db/index/algorithm/rtree/RTreeTest.java
index 753cbf7..d0fd10d 100644
---
a/server/src/test/java/org/apache/iotdb/db/index/algorithm/rtree/RTreeTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/index/algorithm/rtree/RTreeTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.iotdb.db.index.algorithm.rtree;
+import java.util.HashSet;
import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.index.algorithm.rtree.RTree.RNode;
import org.apache.iotdb.db.index.algorithm.rtree.RTree.SeedsPicker;
@@ -56,7 +57,7 @@ public class RTreeTest {
rTree.serialize(out);
InputStream in = new ByteArrayInputStream(out.toByteArray());
- RTree<PartialPath> rTree2 = RTree.deserializePartialPath(in);
+ RTree<PartialPath> rTree2 = RTree.deserialize(in, new HashSet<>());
Assert.assertEquals(rTree.toString(), rTree2.toString());
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/index/it/DemoMMHHWindIT.java
b/server/src/test/java/org/apache/iotdb/db/index/it/DemoMMHHWindIT.java
index a92a481..06ef98f 100644
--- a/server/src/test/java/org/apache/iotdb/db/index/it/DemoMMHHWindIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/index/it/DemoMMHHWindIT.java
@@ -18,7 +18,10 @@
*/
package org.apache.iotdb.db.index.it;
+import java.sql.SQLException;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.StartupException;
+import org.apache.iotdb.db.index.IndexManager;
import org.apache.iotdb.db.rescon.TVListAllocator;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.db.utils.datastructure.TVList;
@@ -85,12 +88,23 @@ public class DemoMMHHWindIT {
IoTDBDescriptor.getInstance().getConfig().setEnableIndex(false);
}
+ private void executeNonQuery(String sql) {
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement();) {
+ statement.execute(sql);
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+ }
+
private static void insertSQL(boolean createTS) throws
ClassNotFoundException {
Class.forName(Config.JDBC_DRIVER_NAME);
try (Connection connection =
- DriverManager.getConnection(
- Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
- Statement statement = connection.createStatement(); ) {
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement();) {
statement.execute(String.format("SET STORAGE GROUP TO %s",
storageGroupSub));
statement.execute(String.format("SET STORAGE GROUP TO %s",
storageGroupWhole));
System.out.println(String.format("SET STORAGE GROUP TO %s",
storageGroupSub));
@@ -191,29 +205,70 @@ public class DemoMMHHWindIT {
}
@Test
- public void checkReadWithoutCreateTS() throws ClassNotFoundException {
- checkRead(false);
+ public void checkReadWithoutCreateTS() throws ClassNotFoundException,
StartupException {
+ insertSQL(false);
+ System.out.println("<<<<<<< Query after insert");
+ // MMHH is approximate index. Before flushing, the index hasn't been built
thus the query is turned to scan and sort by euclidean distance.
+ // Therefore, the first query result might be slightly different from the
later three queries.
+ checkReads(false);
+ executeNonQuery("flush");
+ System.out.println("<<<<<<< Query after insert + flush");
+ checkReads(true);
+ IndexManager.getInstance().stop();
+ IndexManager.getInstance().start();
+ System.out.println("<<<<<<< Query after stop and restart");
+ checkReads(true);
+ executeNonQuery("flush");
+ System.out.println("<<<<<<< Query after stop and restart + flush");
+ checkReads(true);
+ }
+
+ private void checkReads(boolean assertResult) throws ClassNotFoundException {
+ String template = "SELECT TOP 3 speed FROM root.wind2.* WHERE speed LIKE
(%s)";
+ String q1Line =
"20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,"
+ +
"20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,"
+ + "20.0,20.0,20.0,20.0,20.0,"
+ +
"20.5,21.0,21.5,22.0,22.5,23.0,23.5,24.0,24.5,25.0,25.5,26.0,26.5,27.0,"
+ +
"27.5,28.0,28.5,29.0,29.5,30.0,29.5,29.0,28.5,28.0,27.5,27.0,26.5,26.0,"
+ + "25.5,25.0,24.5,24.0,23.5,23.0,22.5,22.0,21.5,21.0,20.5,"
+ +
"20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,"
+ +
"20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0";
+ String q2Best =
"17.548,16.949,16.906,16.993,17.630,16.945,17.284,16.831,16.706,17.216,"
+ +
"16.937,16.196,15.916,15.947,15.200,16.076,16.000,16.152,15.537,15.362,"
+ +
"15.737,15.656,15.995,15.698,15.857,15.706,16.643,16.779,16.383,17.665,"
+ +
"18.364,19.835,18.765,18.451,19.949,19.530,20.692,20.544,20.879,20.904,"
+ +
"21.953,21.726,22.571,23.718,24.009,23.462,23.782,23.871,24.508,24.553,"
+ +
"24.843,24.750,25.500,25.127,25.079,24.951,24.683,24.003,22.977,21.877,"
+ +
"21.850,21.100,20.499,19.988,19.376,18.960,19.471,17.961,17.570,18.652,"
+ +
"18.555,18.401,18.645,17.587,18.056,18.448,17.901,18.105,17.168,18.083,"
+ +
"18.283,16.916,17.099,17.115,18.685,17.483,16.859,16.968,16.339,17.476,"
+ +
"16.835,16.379,17.312,17.789,17.147,17.456,18.596,18.269,17.956,17.851";
+ String q3_76 =
"18.735,18.299,19.216,18.049,18.762,18.739,18.745,18.074,18.398,17.306,"
+ +
"17.455,17.401,16.886,16.683,16.413,16.469,17.458,17.429,17.450,16.701,"
+ +
"16.932,16.032,16.755,16.264,17.173,16.238,16.335,15.200,16.120,16.209,"
+ +
"18.333,18.754,18.621,18.898,19.394,21.456,20.956,22.844,23.484,23.219,"
+ +
"23.903,25.306,24.586,25.023,24.670,25.361,25.771,26.625,27.457,27.110,"
+ +
"26.849,26.500,26.552,26.922,26.056,25.468,25.656,25.830,25.439,25.058,"
+ +
"23.967,24.251,22.794,22.174,21.551,20.847,20.373,20.662,19.614,19.460,"
+ +
"18.355,17.562,18.276,18.907,18.524,18.557,18.104,18.487,18.885,19.310,"
+ +
"19.461,19.036,19.356,19.737,19.328,19.652,19.566,19.629,18.953,18.839,"
+ +
"18.754,19.221,19.215,19.359,19.324,19.729,19.575,19.720,19.591,19.761";
+ String gt_q1_line =
"Time,root.wind2.1417439639000.speed.(D_Ham=0),root.wind2.1417437998000.speed.(D_Ham=0),root.wind2.1417403228000.speed.(D_Ham=0),\n";
+ String gt_q2_best =
"Time,root.wind2.1417403228000.speed.(D_Ham=0),root.wind2.1417439639000.speed.(D_Ham=0),root.wind2.1417434715000.speed.(D_Ham=0),\n";
+ String gt_q3_series76 =
"Time,root.wind2.1417439639000.speed.(D_Ham=0),root.wind2.1417403228000.speed.(D_Ham=0),root.wind2.1417437998000.speed.(D_Ham=0),\n";
+ checkRead(String.format(template, q1Line), assertResult ? gt_q1_line :
null, true);
+ checkRead(String.format(template, q2Best), assertResult ? gt_q2_best :
null, true);
+ checkRead(String.format(template, q3_76), assertResult ? gt_q3_series76 :
null, true);
}
- private void checkRead(boolean createTS) throws ClassNotFoundException {
- insertSQL(createTS);
+ private void checkRead(String querySQL, String gt, boolean onlyHeader)
+ throws ClassNotFoundException {
Class.forName(Config.JDBC_DRIVER_NAME);
try (Connection connection =
- DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/",
"root", "root");
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root",
"root");
Statement statement = connection.createStatement()) {
- String q1Line =
- "20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,"
- +
"20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,"
- + "20.0,20.0,20.0,20.0,20.0,"
- +
"20.5,21.0,21.5,22.0,22.5,23.0,23.5,24.0,24.5,25.0,25.5,26.0,26.5,27.0,"
- +
"27.5,28.0,28.5,29.0,29.5,30.0,29.5,29.0,28.5,28.0,27.5,27.0,26.5,26.0,"
- + "25.5,25.0,24.5,24.0,23.5,23.0,22.5,22.0,21.5,21.0,20.5,"
- +
"20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,"
- +
"20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0";
- String querySQL =
- String.format("SELECT TOP 3 speed FROM root.wind2.* WHERE speed LIKE
(%s)", q1Line);
- System.out.println(querySQL);
- statement.setQueryTimeout(200);
+// System.out.println(querySQL);
+// statement.setQueryTimeout(200);
boolean hasIndex = statement.execute(querySQL);
// String gt = "Time,root.wind1.azq01.speed.17,\n";
Assert.assertTrue(hasIndex);
@@ -224,55 +279,20 @@ public class DemoMMHHWindIT {
sb.append(resultSetMetaData.getColumnName(i)).append(",");
}
sb.append("\n");
- while (resultSet.next()) {
- for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
- sb.append(resultSet.getString(i)).append(",");
+ if (!onlyHeader) {
+ while (resultSet.next()) {
+ for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
+ sb.append(resultSet.getString(i)).append(",");
+ }
+ sb.append("\n");
}
- sb.append("\n");
}
+ System.out.println(">>>>>>>>> Query Result:");
System.out.println(sb);
- // Assert.assertEquals(gt, sb.toString());
- }
- // ========================================
- String basePath =
"/Users/kangrong/tsResearch/tols/JINFENG/d2/out_sub_base.csv";
- List<Float> base = new ArrayList<>();
- try (BufferedReader csvReader = new BufferedReader(new
FileReader(basePath))) {
- String row;
- int idx = 0;
- while ((row = csvReader.readLine()) != null) {
- if (idx >= 1) {
- String[] data = row.split(",");
- long t = Long.parseLong(data[0]);
- float v = Float.parseFloat(data[1]);
- base.add(v);
- }
- idx++;
+ if (gt != null) {
+ Assert.assertEquals(gt, sb.toString());
}
}
- String q2Line = getStringFromList(base, 7600, 7600 + 100);
- querySQL =
- String.format("SELECT TOP 3 speed FROM root.wind2.* WHERE speed LIKE
(%s)", q2Line);
- System.out.println(querySQL);
- statement.setQueryTimeout(200);
- hasIndex = statement.execute(querySQL);
- // String gt = "Time,root.wind1.azq01.speed.17,\n";
- Assert.assertTrue(hasIndex);
- try (ResultSet resultSet = statement.getResultSet()) {
- ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
- StringBuilder sb = new StringBuilder();
- for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
- sb.append(resultSetMetaData.getColumnName(i)).append(",");
- }
- sb.append("\n");
- while (resultSet.next()) {
- for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
- sb.append(resultSet.getString(i)).append(",");
- }
- sb.append("\n");
- }
- System.out.println(sb);
- // Assert.assertEquals(gt, sb.toString());
- }
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
diff --git
a/server/src/test/java/org/apache/iotdb/db/index/it/DemoRTreeWindIT.java
b/server/src/test/java/org/apache/iotdb/db/index/it/DemoRTreeWindIT.java
index 1891d56..a85cfb4 100644
--- a/server/src/test/java/org/apache/iotdb/db/index/it/DemoRTreeWindIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/index/it/DemoRTreeWindIT.java
@@ -18,7 +18,10 @@
*/
package org.apache.iotdb.db.index.it;
+import java.sql.SQLException;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.StartupException;
+import org.apache.iotdb.db.index.IndexManager;
import org.apache.iotdb.db.rescon.TVListAllocator;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.db.utils.datastructure.TVList;
@@ -82,12 +85,23 @@ public class DemoRTreeWindIT {
IoTDBDescriptor.getInstance().getConfig().setEnableIndex(false);
}
+ private void executeNonQuery(String sql) {
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement();) {
+ statement.execute(sql);
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+ }
+
private static void insertSQL(boolean createTS) throws
ClassNotFoundException {
Class.forName(Config.JDBC_DRIVER_NAME);
try (Connection connection =
- DriverManager.getConnection(
- Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
- Statement statement = connection.createStatement(); ) {
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement();) {
statement.execute(String.format("SET STORAGE GROUP TO %s",
storageGroupSub));
statement.execute(String.format("SET STORAGE GROUP TO %s",
storageGroupWhole));
System.out.println(String.format("SET STORAGE GROUP TO %s",
storageGroupSub));
@@ -167,29 +181,68 @@ public class DemoRTreeWindIT {
}
@Test
- public void checkReadWithoutCreateTS() throws ClassNotFoundException {
- checkRead(false);
+ public void checkReadWithoutCreateTS() throws ClassNotFoundException,
StartupException {
+ insertSQL(false);
+ System.out.println("<<<<<<< Query after insert");
+ checkReads(true);
+ executeNonQuery("flush");
+ System.out.println("<<<<<<< Query after insert + flush");
+ checkReads(true);
+ IndexManager.getInstance().stop();
+ IndexManager.getInstance().start();
+ System.out.println("<<<<<<< Query after stop and restart");
+ checkReads(true);
+ executeNonQuery("flush");
+ System.out.println("<<<<<<< Query after stop and restart + flush");
+ checkReads(true);
+ }
+
+ private void checkReads(boolean assertResult) throws ClassNotFoundException {
+ String template = "SELECT TOP 3 speed FROM root.wind2.* WHERE speed LIKE
(%s)";
+ String q1Line =
"20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,"
+ +
"20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,"
+ + "20.0,20.0,20.0,20.0,20.0,"
+ +
"20.5,21.0,21.5,22.0,22.5,23.0,23.5,24.0,24.5,25.0,25.5,26.0,26.5,27.0,"
+ +
"27.5,28.0,28.5,29.0,29.5,30.0,29.5,29.0,28.5,28.0,27.5,27.0,26.5,26.0,"
+ + "25.5,25.0,24.5,24.0,23.5,23.0,22.5,22.0,21.5,21.0,20.5,"
+ +
"20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,"
+ +
"20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0";
+ String q2Best =
"17.548,16.949,16.906,16.993,17.630,16.945,17.284,16.831,16.706,17.216,"
+ +
"16.937,16.196,15.916,15.947,15.200,16.076,16.000,16.152,15.537,15.362,"
+ +
"15.737,15.656,15.995,15.698,15.857,15.706,16.643,16.779,16.383,17.665,"
+ +
"18.364,19.835,18.765,18.451,19.949,19.530,20.692,20.544,20.879,20.904,"
+ +
"21.953,21.726,22.571,23.718,24.009,23.462,23.782,23.871,24.508,24.553,"
+ +
"24.843,24.750,25.500,25.127,25.079,24.951,24.683,24.003,22.977,21.877,"
+ +
"21.850,21.100,20.499,19.988,19.376,18.960,19.471,17.961,17.570,18.652,"
+ +
"18.555,18.401,18.645,17.587,18.056,18.448,17.901,18.105,17.168,18.083,"
+ +
"18.283,16.916,17.099,17.115,18.685,17.483,16.859,16.968,16.339,17.476,"
+ +
"16.835,16.379,17.312,17.789,17.147,17.456,18.596,18.269,17.956,17.851";
+ String q3_76 =
"18.735,18.299,19.216,18.049,18.762,18.739,18.745,18.074,18.398,17.306,"
+ +
"17.455,17.401,16.886,16.683,16.413,16.469,17.458,17.429,17.450,16.701,"
+ +
"16.932,16.032,16.755,16.264,17.173,16.238,16.335,15.200,16.120,16.209,"
+ +
"18.333,18.754,18.621,18.898,19.394,21.456,20.956,22.844,23.484,23.219,"
+ +
"23.903,25.306,24.586,25.023,24.670,25.361,25.771,26.625,27.457,27.110,"
+ +
"26.849,26.500,26.552,26.922,26.056,25.468,25.656,25.830,25.439,25.058,"
+ +
"23.967,24.251,22.794,22.174,21.551,20.847,20.373,20.662,19.614,19.460,"
+ +
"18.355,17.562,18.276,18.907,18.524,18.557,18.104,18.487,18.885,19.310,"
+ +
"19.461,19.036,19.356,19.737,19.328,19.652,19.566,19.629,18.953,18.839,"
+ +
"18.754,19.221,19.215,19.359,19.324,19.729,19.575,19.720,19.591,19.761";
+ String gt_q1_line =
"Time,root.wind2.1417439639000.speed.(D=20.17),root.wind2.1417437998000.speed.(D=32.81),root.wind2.1417437450000.speed.(D=34.97),\n";
+ String gt_q2_best =
"Time,root.wind2.1417403228000.speed.(D=16.92),root.wind2.1417439639000.speed.(D=17.02),root.wind2.1417434715000.speed.(D=21.14),\n";
+ String gt_q3_series76 =
"Time,root.wind2.1417439639000.speed.(D=0.00),root.wind2.1417403228000.speed.(D=21.57),root.wind2.1417437998000.speed.(D=29.78),\n";
+ checkRead(String.format(template, q1Line), assertResult ? gt_q1_line :
null, true);
+ checkRead(String.format(template, q2Best), assertResult ? gt_q2_best :
null, true);
+ checkRead(String.format(template, q3_76), assertResult ? gt_q3_series76 :
null, true);
}
- private void checkRead(boolean createTS) throws ClassNotFoundException {
- insertSQL(createTS);
+ private void checkRead(String querySQL, String gt, boolean onlyHeader)
+ throws ClassNotFoundException {
Class.forName(Config.JDBC_DRIVER_NAME);
try (Connection connection =
- DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/",
"root", "root");
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root",
"root");
Statement statement = connection.createStatement()) {
- String q1Line =
- "20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,"
- +
"20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,"
- + "20.0,20.0,20.0,20.0,20.0,"
- +
"20.5,21.0,21.5,22.0,22.5,23.0,23.5,24.0,24.5,25.0,25.5,26.0,26.5,27.0,"
- +
"27.5,28.0,28.5,29.0,29.5,30.0,29.5,29.0,28.5,28.0,27.5,27.0,26.5,26.0,"
- + "25.5,25.0,24.5,24.0,23.5,23.0,22.5,22.0,21.5,21.0,20.5,"
- +
"20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,"
- +
"20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0";
- String querySQL =
- String.format("SELECT TOP 3 speed FROM root.wind2.* WHERE speed LIKE
(%s)", q1Line);
- System.out.println(querySQL);
- statement.setQueryTimeout(200);
+// System.out.println(querySQL);
+// statement.setQueryTimeout(200);
boolean hasIndex = statement.execute(querySQL);
// String gt = "Time,root.wind1.azq01.speed.17,\n";
Assert.assertTrue(hasIndex);
@@ -200,55 +253,20 @@ public class DemoRTreeWindIT {
sb.append(resultSetMetaData.getColumnName(i)).append(",");
}
sb.append("\n");
- while (resultSet.next()) {
- for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
- sb.append(resultSet.getString(i)).append(",");
+ if (!onlyHeader) {
+ while (resultSet.next()) {
+ for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
+ sb.append(resultSet.getString(i)).append(",");
+ }
+ sb.append("\n");
}
- sb.append("\n");
}
+ System.out.println(">>>>>>>>> Query Result:");
System.out.println(sb);
- // Assert.assertEquals(gt, sb.toString());
- }
- // ========================================
- String basePath =
"/Users/kangrong/tsResearch/tols/JINFENG/d2/out_sub_base.csv";
- List<Float> base = new ArrayList<>();
- try (BufferedReader csvReader = new BufferedReader(new
FileReader(basePath))) {
- String row;
- int idx = 0;
- while ((row = csvReader.readLine()) != null) {
- if (idx >= 1) {
- String[] data = row.split(",");
- long t = Long.parseLong(data[0]);
- float v = Float.parseFloat(data[1]);
- base.add(v);
- }
- idx++;
+ if (gt != null) {
+ Assert.assertEquals(gt, sb.toString());
}
}
- String q2Line = getStringFromList(base, 7600, 7600 + 100);
- querySQL =
- String.format("SELECT TOP 3 speed FROM root.wind2.* WHERE speed LIKE
(%s)", q2Line);
- System.out.println(querySQL);
- statement.setQueryTimeout(200);
- hasIndex = statement.execute(querySQL);
- // String gt = "Time,root.wind1.azq01.speed.17,\n";
- Assert.assertTrue(hasIndex);
- try (ResultSet resultSet = statement.getResultSet()) {
- ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
- StringBuilder sb = new StringBuilder();
- for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
- sb.append(resultSetMetaData.getColumnName(i)).append(",");
- }
- sb.append("\n");
- while (resultSet.next()) {
- for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
- sb.append(resultSet.getString(i)).append(",");
- }
- sb.append("\n");
- }
- System.out.println(sb);
- // Assert.assertEquals(gt, sb.toString());
- }
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
