This is an automated email from the ASF dual-hosted git repository.
kangrong pushed a commit to branch f_index_dev
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/f_index_dev by this push:
new aef6776 fix bug in add and remove index
aef6776 is described below
commit aef6776ae4e000e9f573bbd9f57e29b5dfe49be4
Author: kr11 <3095717866.com>
AuthorDate: Mon May 17 13:15:27 2021 +0800
fix bug in add and remove index
---
.../org/apache/iotdb/db/index/IndexManager.java | 13 +-
.../org/apache/iotdb/db/index/IndexProcessor.java | 7 +-
.../iotdb/db/index/router/ProtoIndexRouter.java | 80 ++++++++----
.../db/index/router/ProtoIndexRouterTest.java | 144 +++++++++++++++++----
4 files changed, 189 insertions(+), 55 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/index/IndexManager.java
b/server/src/main/java/org/apache/iotdb/db/index/IndexManager.java
index 0216565..41c7f12 100644
--- a/server/src/main/java/org/apache/iotdb/db/index/IndexManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/index/IndexManager.java
@@ -76,6 +76,11 @@ public class IndexManager implements IndexManagerMBean,
IService {
private final String indexDataDirPath;
private final IIndexRouter router;
+ @TestOnly
+ public CreateIndexProcessorFunc getCreateIndexProcessorFunc() {
+ return createIndexProcessorFunc;
+ }
+
/** A function interface to construct an index processor. */
private CreateIndexProcessorFunc createIndexProcessorFunc;
@@ -102,11 +107,15 @@ public class IndexManager implements IndexManagerMBean,
IService {
* @param indexType the type of index
* @return the feature directory path for this index.
*/
- private String getFeatureFileDirectory(PartialPath path, IndexType
indexType) {
+ public String getFeatureFileDirectory(PartialPath path, IndexType indexType)
{
return IndexUtils.removeIllegalStarInDir(
indexDataDirPath + File.separator + path.getFullPath() +
File.separator + indexType);
}
+ public String getIndexDirectory() {
+ return indexDataDirPath;
+ }
+
/**
* Given an IndexSeries, return its data file path (unused currently).
*
@@ -114,7 +123,7 @@ public class IndexManager implements IndexManagerMBean,
IService {
* @param indexType the type of index
* @return the feature directory path for this index.
*/
- private String getIndexDataDirectory(PartialPath path, IndexType indexType) {
+ public String getIndexDataDirectory(PartialPath path, IndexType indexType) {
return getFeatureFileDirectory(path, indexType);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/index/IndexProcessor.java
b/server/src/main/java/org/apache/iotdb/db/index/IndexProcessor.java
index 4548674..c1e821d 100644
--- a/server/src/main/java/org/apache/iotdb/db/index/IndexProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/index/IndexProcessor.java
@@ -37,6 +37,7 @@ import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.metadata.PartialPath;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.service.IoTDB;
+import org.apache.iotdb.db.utils.FileUtils;
import org.apache.iotdb.db.utils.datastructure.TVList;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
@@ -279,7 +280,7 @@ public class IndexProcessor implements
Comparable<IndexProcessor> {
closed = true;
if (deleteFiles) {
File dir = IndexUtils.getIndexFile(indexSeriesDirPath);
- dir.delete();
+ FileUtils.deleteDirectory(dir);
}
} finally {
lock.writeLock().unlock();
@@ -497,9 +498,11 @@ public class IndexProcessor implements
Comparable<IndexProcessor> {
}
// remove index file directories
File dir = IndexUtils.getIndexFile(getIndexDir(indexType));
- dir.delete();
+ FileUtils.deleteDirectory(dir);
allPathsIndexMap.remove(indexType);
+ indexLockMap.remove(indexType);
usableMap.remove(indexType);
+ previousDataBufferMap.remove(indexType);
}
}
} finally {
diff --git
a/server/src/main/java/org/apache/iotdb/db/index/router/ProtoIndexRouter.java
b/server/src/main/java/org/apache/iotdb/db/index/router/ProtoIndexRouter.java
index 3b55dc7..021b17d 100644
---
a/server/src/main/java/org/apache/iotdb/db/index/router/ProtoIndexRouter.java
+++
b/server/src/main/java/org/apache/iotdb/db/index/router/ProtoIndexRouter.java
@@ -206,12 +206,14 @@ public class ProtoIndexRouter implements IIndexRouter {
partialPath.toLowerCase();
if (partialPath.isFullPath()) {
String fullPath = partialPath.getFullPath();
- if (!fullPathProcessorMap.containsKey(fullPath)) {
+ if (!fullPathProcessorMap.containsKey(fullPath)
+ ||
!fullPathProcessorMap.get(fullPath).infos.containsKey(indexType)) {
throw new QueryIndexException("haven't create index on " + fullPath);
}
struct = fullPathProcessorMap.get(fullPath);
} else {
- if (!wildCardProcessorMap.containsKey(partialPath)) {
+ if (!wildCardProcessorMap.containsKey(partialPath)
+ ||
!wildCardProcessorMap.get(partialPath).infos.containsKey(indexType)) {
throw new QueryIndexException("haven't create index on " +
partialPath);
}
struct = wildCardProcessorMap.get(partialPath);
@@ -237,7 +239,9 @@ public class ProtoIndexRouter implements IIndexRouter {
// TODO prefixPath must be "prefix", i.e. ending with wildcard:
root.XX.XX.*
// the wildcard match is costly. We'd move router into MManager
Map<PartialPath, Map<IndexType, IndexInfo>> res = new HashMap<>();
- if (prefixPath != null) prefixPath.toLowerCase();
+ if (prefixPath != null) {
+ prefixPath.toLowerCase();
+ }
wildCardProcessorMap.forEach(
(indexSeries, struct) -> {
if (prefixPath == null || prefixPath.matchFullPath(indexSeries)) {
@@ -273,7 +277,12 @@ public class ProtoIndexRouter implements IIndexRouter {
lock.writeLock().lock();
IndexType indexType = indexInfo.getIndexType();
// record the relationship between storage group and the
- StorageGroupMNode storageGroupMNode =
mManager.getStorageGroupNodeByPath(partialPath);
+ StorageGroupMNode storageGroupMNode;
+ try {
+ storageGroupMNode = mManager.getStorageGroupNodeByPath(partialPath);
+ } catch (NullPointerException e) {
+ throw new MetadataException("please set storage group before create
index");
+ }
String storageGroupPath = storageGroupMNode.getPartialPath().getFullPath();
// add to pathMap
try {
@@ -286,7 +295,12 @@ public class ProtoIndexRouter implements IIndexRouter {
fullPathProcessorMap.put(
fullPath, new IndexProcessorStruct(processor, partialPath,
infoMap));
} else {
- fullPathProcessorMap.get(fullPath).infos.put(indexType, indexInfo);
+ Map<IndexType, IndexInfo> infoMap =
fullPathProcessorMap.get(fullPath).infos;
+ if (infoMap.containsKey(indexType)) {
+ // nothing to do, throw exception
+ throw new MetadataException(indexType + " has already been set on
" + fullPath);
+ }
+ infoMap.put(indexType, indexInfo);
}
IndexProcessorStruct pair = fullPathProcessorMap.get(fullPath);
pair.processor.refreshSeriesIndexMapFromMManager(pair.infos);
@@ -312,7 +326,12 @@ public class ProtoIndexRouter implements IIndexRouter {
wildCardProcessorMap.put(
partialPath, new IndexProcessorStruct(processor,
representativePath, infoMap));
} else {
- wildCardProcessorMap.get(partialPath).infos.put(indexType,
indexInfo);
+ Map<IndexType, IndexInfo> infoMap =
wildCardProcessorMap.get(partialPath).infos;
+ if (infoMap.containsKey(indexType)) {
+ // nothing to do, throw exception
+ throw new MetadataException(indexType + " has already been set on
" + partialPath);
+ }
+ infoMap.put(indexType, indexInfo);
}
IndexProcessorStruct pair = wildCardProcessorMap.get(partialPath);
pair.processor.refreshSeriesIndexMapFromMManager(pair.infos);
@@ -344,39 +363,54 @@ public class ProtoIndexRouter implements IIndexRouter {
// only the pair.left (indexType map) will be updated.
lock.writeLock().lock();
// record the relationship between storage group and the index processors
- StorageGroupMNode storageGroupMNode =
mManager.getStorageGroupNodeByPath(partialPath);
+ StorageGroupMNode storageGroupMNode;
+ try {
+ storageGroupMNode = mManager.getStorageGroupNodeByPath(partialPath);
+ } catch (NullPointerException e) {
+ throw new MetadataException("no storage group, remove nothing");
+ }
String storageGroupPath = storageGroupMNode.getPartialPath().getFullPath();
// remove from pathMap
+ boolean actualRemove = false;
try {
if (partialPath.isFullPath()) {
String fullPath = partialPath.getFullPath();
if (fullPathProcessorMap.containsKey(fullPath)) {
IndexProcessorStruct pair = fullPathProcessorMap.get(fullPath);
- pair.infos.remove(indexType);
- pair.processor.refreshSeriesIndexMapFromMManager(pair.infos);
- if (pair.infos.isEmpty()) {
- sgToFullPathMap.get(storageGroupPath).remove(fullPath);
- fullPathProcessorMap.remove(fullPath);
- pair.representativePath = null;
- pair.processor.close(true);
- pair.processor = null;
+ if (pair.infos.containsKey(indexType)) {
+ pair.infos.remove(indexType);
+ pair.processor.refreshSeriesIndexMapFromMManager(pair.infos);
+ if (pair.infos.isEmpty()) {
+ sgToFullPathMap.get(storageGroupPath).remove(fullPath);
+ fullPathProcessorMap.remove(fullPath);
+ pair.representativePath = null;
+ pair.processor.close(true);
+ pair.processor = null;
+ }
+ actualRemove = true;
}
}
} else {
if (wildCardProcessorMap.containsKey(partialPath)) {
IndexProcessorStruct pair = wildCardProcessorMap.get(partialPath);
- pair.infos.remove(indexType);
- pair.processor.refreshSeriesIndexMapFromMManager(pair.infos);
- if (pair.infos.isEmpty()) {
- sgToWildCardPathMap.get(storageGroupPath).remove(partialPath);
- wildCardProcessorMap.remove(partialPath);
- pair.representativePath = null;
- pair.processor.close(true);
- pair.processor = null;
+ if (pair.infos.containsKey(indexType)) {
+ pair.infos.remove(indexType);
+ pair.processor.refreshSeriesIndexMapFromMManager(pair.infos);
+ if (pair.infos.isEmpty()) {
+ sgToWildCardPathMap.get(storageGroupPath).remove(partialPath);
+ wildCardProcessorMap.remove(partialPath);
+ pair.representativePath = null;
+ pair.processor.close(true);
+ pair.processor = null;
+ }
+ actualRemove = true;
}
}
}
+ if (!actualRemove) {
+ throw new MetadataException(indexType + " hasn't been created on " +
partialPath);
+ }
serialize(false);
} finally {
lock.writeLock().unlock();
diff --git
a/server/src/test/java/org/apache/iotdb/db/index/router/ProtoIndexRouterTest.java
b/server/src/test/java/org/apache/iotdb/db/index/router/ProtoIndexRouterTest.java
index 86b577b..441623d 100644
---
a/server/src/test/java/org/apache/iotdb/db/index/router/ProtoIndexRouterTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/index/router/ProtoIndexRouterTest.java
@@ -17,14 +17,15 @@
*/
package org.apache.iotdb.db.index.router;
+import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
-import org.apache.iotdb.db.index.IndexProcessor;
+import org.apache.iotdb.db.index.IndexManager;
import org.apache.iotdb.db.index.common.IndexInfo;
+import org.apache.iotdb.db.index.common.IndexType;
import org.apache.iotdb.db.index.common.func.CreateIndexProcessorFunc;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.metadata.PartialPath;
import org.apache.iotdb.db.utils.EnvironmentUtils;
-import org.apache.iotdb.db.utils.FileUtils;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
@@ -56,10 +57,10 @@ public class ProtoIndexRouterTest {
private static final String direction3 = "root.wind2.3.direction";
private static final String index_sub = speed1;
private static final String index_full = "root.wind2.*.direction";
- private IndexInfo infoFull;
- private IndexInfo infoSub;
- private CreateIndexProcessorFunc fakeCreateFunc;
- private IndexInfo infoFull2;
+ private IndexInfo infoFull_RTREE;
+ private IndexInfo infoSub_ELB;
+ private CreateIndexProcessorFunc createFunc;
+ private IndexInfo infoFull2_NO_INDEX;
private void prepareMManager() throws MetadataException {
MManager mManager = MManager.getInstance();
@@ -98,43 +99,44 @@ public class ProtoIndexRouterTest {
props_full.put(PAA_DIM, "5");
Map<String, String> props_full2 = new HashMap<>();
props_full2.put(PAA_DIM, "10");
- this.infoSub = new IndexInfo(ELB_INDEX, 0, props_sub);
- this.infoFull = new IndexInfo(RTREE_PAA, 5, props_full);
- this.infoFull2 = new IndexInfo(NO_INDEX, 10, props_full2);
- this.fakeCreateFunc =
- (indexSeries, indexInfoMap) ->
- new IndexProcessor(
- indexSeries,
- testRouterDir + File.separator + "index_fake_" + indexSeries,
- indexInfoMap);
+ this.infoSub_ELB = new IndexInfo(ELB_INDEX, 0, props_sub);
+ this.infoFull_RTREE = new IndexInfo(RTREE_PAA, 5, props_full);
+ this.infoFull2_NO_INDEX = new IndexInfo(NO_INDEX, 10, props_full2);
+ this.createFunc = IndexManager.getInstance().getCreateIndexProcessorFunc();
+ // (indexSeries, indexInfoMap) ->
+ // new IndexProcessor(
+ // indexSeries,
+ // IndexManager.getInstance().getIndexDirectory() +
File.separator +
+ // "index_fake_" + indexSeries,
+ // indexInfoMap);
}
- private static final String testRouterDir = "test_protoIndexRouter";
+ // private static final String testRouterDir = "test_protoIndexRouter";
@Before
public void setUp() throws Exception {
EnvironmentUtils.envSetUp();
- if (!new File(testRouterDir).exists()) {
- new File(testRouterDir).mkdirs();
- }
+ // if (!new File(testRouterDir).exists()) {
+ // new File(testRouterDir).mkdirs();
+ // }
}
@After
public void tearDown() throws Exception {
EnvironmentUtils.cleanEnv();
- if (new File(testRouterDir).exists()) {
- FileUtils.deleteDirectory(new File(testRouterDir));
- }
+ // if (new File(testRouterDir).exists()) {
+ // FileUtils.deleteDirectory(new File(testRouterDir));
+ // }
}
@Test
public void testBasic() throws MetadataException, IOException {
prepareMManager();
- ProtoIndexRouter router = new ProtoIndexRouter(testRouterDir);
+ ProtoIndexRouter router = new
ProtoIndexRouter(IndexManager.getInstance().getIndexDirectory());
- router.addIndexIntoRouter(new PartialPath(index_sub), infoSub,
fakeCreateFunc, true);
- router.addIndexIntoRouter(new PartialPath(index_full), infoFull,
fakeCreateFunc, true);
- router.addIndexIntoRouter(new PartialPath(index_full), infoFull2,
fakeCreateFunc, true);
+ router.addIndexIntoRouter(new PartialPath(index_sub), infoSub_ELB,
createFunc, true);
+ router.addIndexIntoRouter(new PartialPath(index_full), infoFull_RTREE,
createFunc, true);
+ router.addIndexIntoRouter(new PartialPath(index_full), infoFull2_NO_INDEX,
createFunc, true);
// System.out.println(router.toString());
Assert.assertEquals(
"<{NO_INDEX=[type: NO_INDEX, time: 10, props: {PAA_DIM=10}],
RTREE_PAA=[type: RTREE_PAA, time: 5, props: {PAA_DIM=5}]}\n"
@@ -166,8 +168,8 @@ public class ProtoIndexRouterTest {
router.serialize(true);
Assert.assertEquals("", router.toString());
- IIndexRouter newRouter = new ProtoIndexRouter(testRouterDir);
- newRouter.deserializeAndReload(fakeCreateFunc);
+ IIndexRouter newRouter = new
ProtoIndexRouter(IndexManager.getInstance().getIndexDirectory());
+ newRouter.deserializeAndReload(createFunc);
Assert.assertEquals(
"<{NO_INDEX=[type: NO_INDEX, time: 10, props: {PAA_DIM=10}],
RTREE_PAA=[type: RTREE_PAA, time: 5, props: {PAA_DIM=5}]}\n"
+ "root.wind2.*.direction: {NO_INDEX=NO_INDEX,
RTREE_PAA=nMax:50,nMin:2,dim:4,seedsPicker:LINEARinner:0,leaf:1,item:0;\n"
@@ -194,4 +196,90 @@ public class ProtoIndexRouterTest {
newRouter.removeIndexFromRouter(new PartialPath(index_sub), ELB_INDEX);
Assert.assertEquals("", newRouter.toString());
}
+
+ @Test
+ public void testDuplicateAdd() throws MetadataException {
+ prepareMManager();
+ ProtoIndexRouter router = new
ProtoIndexRouter(IndexManager.getInstance().getIndexDirectory());
+
+ router.addIndexIntoRouter(new PartialPath(index_sub), infoSub_ELB,
createFunc, true);
+ router.addIndexIntoRouter(new PartialPath(index_full), infoFull_RTREE,
createFunc, true);
+ router.addIndexIntoRouter(new PartialPath(index_full), infoFull2_NO_INDEX,
createFunc, true);
+ try {
+ router.addIndexIntoRouter(new PartialPath(index_full),
infoFull2_NO_INDEX, createFunc, true);
+ Assert.fail();
+ } catch (MetadataException e) {
+ Assert.assertEquals(
+ "NO_INDEX has already been set on root.wind2.*.direction",
e.getMessage());
+ }
+ try {
+ router.addIndexIntoRouter(new PartialPath(index_sub), infoSub_ELB,
createFunc, true);
+ Assert.fail();
+ } catch (MetadataException e) {
+ Assert.assertEquals(
+ "ELB_INDEX has already been set on root.wind1.azq01.speed",
e.getMessage());
+ }
+ // System.out.println(router.toString());
+ }
+
+ private File getIndexDirFile(String path, IndexType indexType) throws
IllegalPathException {
+ return new File(
+ IndexManager.getInstance().getIndexDataDirectory(new
PartialPath(path), indexType));
+ }
+
+ @Test
+ public void testDuplicateDrop() throws MetadataException, IOException {
+ prepareMManager();
+ ProtoIndexRouter router = new
ProtoIndexRouter(IndexManager.getInstance().getIndexDirectory());
+ try {
+ router.removeIndexFromRouter(new PartialPath(index_sub), NO_INDEX);
+ Assert.fail();
+ } catch (MetadataException e) {
+ Assert.assertEquals("NO_INDEX hasn't been created on
root.wind1.azq01.speed", e.getMessage());
+ }
+ Assert.assertFalse(getIndexDirFile(index_sub, NO_INDEX).exists());
+ try {
+ router.removeIndexFromRouter(new PartialPath(index_full), NO_INDEX);
+ Assert.fail();
+ } catch (MetadataException e) {
+ Assert.assertEquals("NO_INDEX hasn't been created on
root.wind2.*.direction", e.getMessage());
+ }
+ Assert.assertFalse(getIndexDirFile(index_full, NO_INDEX).exists());
+ router.addIndexIntoRouter(new PartialPath(index_sub), infoSub_ELB,
createFunc, true);
+ Assert.assertTrue(getIndexDirFile(index_sub, ELB_INDEX).exists());
+ router.addIndexIntoRouter(new PartialPath(index_full), infoFull_RTREE,
createFunc, true);
+ Assert.assertTrue(getIndexDirFile(index_full, RTREE_PAA).exists());
+ router.removeIndexFromRouter(new PartialPath(index_sub), ELB_INDEX);
+ Assert.assertFalse(getIndexDirFile(index_sub, ELB_INDEX).exists());
+ try {
+ router.removeIndexFromRouter(new PartialPath(index_sub), ELB_INDEX);
+ Assert.assertFalse(getIndexDirFile(index_sub, ELB_INDEX).exists());
+ Assert.fail("should throw exception");
+ } catch (MetadataException e) {
+ Assert.assertEquals(
+ "ELB_INDEX hasn't been created on root.wind1.azq01.speed",
e.getMessage());
+ }
+ // System.out.println(router.toString());
+ }
+
+ @Test
+ public void testOperationWithoutStorageGroup() throws MetadataException,
IOException {
+ ProtoIndexRouter router = new
ProtoIndexRouter(IndexManager.getInstance().getIndexDirectory());
+ try {
+ router.addIndexIntoRouter(
+ new PartialPath(index_sub),
+ new IndexInfo(ELB_INDEX, 0, new HashMap<>()),
+ createFunc,
+ true);
+ Assert.fail();
+ } catch (MetadataException e) {
+ Assert.assertEquals("please set storage group before create index",
e.getMessage());
+ }
+ try {
+ router.removeIndexFromRouter(new PartialPath(index_sub), NO_INDEX);
+ Assert.fail();
+ } catch (MetadataException e) {
+ Assert.assertEquals("no storage group, remove nothing", e.getMessage());
+ }
+ }
}
