This is an automated email from the ASF dual-hosted git repository. xuekaifeng pushed a commit to branch xkf_id_table in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2b8cfedbdd8ce66cb348c4ec3e602bb65ee33c14 Author: 151250176 <[email protected]> AuthorDate: Tue Dec 14 11:01:51 2021 +0800 fix restart --- .../org/apache/iotdb/db/engine/StorageEngine.java | 1 + .../engine/storagegroup/StorageGroupProcessor.java | 7 +- .../service/TriggerRegistrationService.java | 13 +- .../org/apache/iotdb/db/metadata/MManager.java | 21 +- .../id_table/AppendOnlyDiskSchemaManager.java | 8 + .../iotdb/db/metadata/id_table/IDManager.java | 73 ----- .../iotdb/db/metadata/id_table/IDManagerImpl.java | 85 ----- .../apache/iotdb/db/metadata/id_table/IDTable.java | 352 ++------------------- .../{IDTable.java => IDTableHashmapImpl.java} | 33 +- .../iotdb/db/metadata/id_table/IDTableManager.java | 99 ++++++ .../metadata/id_table/entry/DeviceIDFactory.java | 11 +- .../iotdb/db/qp/physical/crud/InsertPlan.java | 12 + .../iotdb/db/qp/physical/crud/InsertRowPlan.java | 4 +- .../physical/crud/InsertRowsOfOneDevicePlan.java | 4 +- .../db/qp/physical/crud/InsertTabletPlan.java | 4 +- .../iotdb/db/query/executor/LastQueryExecutor.java | 11 +- .../db/query/executor/RawDataQueryExecutor.java | 7 + .../db/query/executor/fill/LastPointReader.java | 3 +- .../java/org/apache/iotdb/db/rest/RestService.java | 26 +- .../apache/iotdb/db/utils/EnvironmentUtils.java | 5 + ...est.java => IDTableHashmapImplLogFileTest.java} | 6 +- ... => IDTableHashmapImplResourceControlTest.java} | 7 +- ...est.java => IDTableHashmapImplRestartTest.java} | 158 +++++---- ...DTableTest.java => IDTableHashmapImplTest.java} | 33 +- ....java => InsertWithIDTableHashmapImplTest.java} | 12 +- ...t.java => LastQueryWithIDTableHashmapImpl.java} | 154 ++------- ...t.java => QueryWithIDTableHashmapImplTest.java} | 33 +- 27 files changed, 353 insertions(+), 829 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java index 063ab10..1a66684 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java @@ -1064,6 +1064,7 @@ public class StorageEngine implements IService { } else { IoTDB.metaManager.getSeriesSchemasAndReadLockDevice(insertPlan); insertPlan.setDeviceID(DeviceIDFactory.getInstance().getDeviceID(insertPlan.getDeviceId())); + insertPlan.setDevicePath(insertPlan.getDeviceId()); } } catch (MetadataException | IOException e) { throw new StorageEngineException(e); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java index bd454f5..5c0ed0d 100755 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java @@ -57,6 +57,7 @@ import org.apache.iotdb.db.exception.metadata.MetadataException; import org.apache.iotdb.db.exception.query.OutOfTTLException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.metadata.id_table.IDTable; +import org.apache.iotdb.db.metadata.id_table.IDTableManager; import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.db.qp.physical.crud.DeletePlan; @@ -411,7 +412,11 @@ public class StorageGroupProcessor { TimeUnit.MILLISECONDS); // use id table if (config.isEnableIDTable()) { - idTable = new IDTable(storageGroupSysDir); + try { + idTable = IDTableManager.getInstance().getIDTable(new PartialPath(logicalStorageGroupName)); + } catch (IllegalPathException e) { + logger.error("failed to create id table"); + } } recover(); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/trigger/service/TriggerRegistrationService.java b/server/src/main/java/org/apache/iotdb/db/engine/trigger/service/TriggerRegistrationService.java index f5f8f5f..72e7f00 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/trigger/service/TriggerRegistrationService.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/trigger/service/TriggerRegistrationService.java @@ -20,16 +20,15 @@ package org.apache.iotdb.db.engine.trigger.service; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory; import org.apache.iotdb.db.engine.trigger.api.Trigger; import org.apache.iotdb.db.engine.trigger.executor.TriggerExecutor; import org.apache.iotdb.db.exception.StartupException; -import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.TriggerExecutionException; import org.apache.iotdb.db.exception.TriggerManagementException; import org.apache.iotdb.db.exception.metadata.MetadataException; import org.apache.iotdb.db.metadata.id_table.IDTable; +import org.apache.iotdb.db.metadata.id_table.IDTableManager; import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.db.qp.physical.PhysicalPlan; @@ -159,10 +158,9 @@ public class TriggerRegistrationService implements IService { // update id table try { - IDTable idTable = - StorageEngine.getInstance().getProcessor(plan.getFullPath().getDevicePath()).getIdTable(); + IDTable idTable = IDTableManager.getInstance().getIDTable(plan.getFullPath().getDevicePath()); idTable.registerTrigger(plan.getFullPath(), measurementMNode); - } catch (StorageEngineException | MetadataException e) { + } catch (MetadataException e) { throw new TriggerManagementException(e.getMessage(), e); } } @@ -212,10 +210,9 @@ public class TriggerRegistrationService implements IService { // update id table try { PartialPath fullPath = executor.getMeasurementMNode().getPartialPath(); - IDTable idTable = - StorageEngine.getInstance().getProcessor(fullPath.getDevicePath()).getIdTable(); + IDTable idTable = IDTableManager.getInstance().getIDTable(fullPath.getDevicePath()); idTable.deregisterTrigger(fullPath, executor.getMeasurementMNode()); - } catch (StorageEngineException | MetadataException e) { + } catch (MetadataException e) { throw new TriggerManagementException(e.getMessage(), e); } } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java index eb63295..73fa4a6 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java @@ -24,7 +24,6 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory; import org.apache.iotdb.db.engine.trigger.executor.TriggerEngine; -import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.AliasAlreadyExistException; import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; import org.apache.iotdb.db.exception.metadata.DeleteFailedException; @@ -39,6 +38,7 @@ import org.apache.iotdb.db.exception.metadata.StorageGroupNotSetException; import org.apache.iotdb.db.exception.metadata.TemplateIsInUseException; import org.apache.iotdb.db.exception.metadata.UndefinedTemplateException; import org.apache.iotdb.db.metadata.id_table.IDTable; +import org.apache.iotdb.db.metadata.id_table.IDTableManager; import org.apache.iotdb.db.metadata.lastCache.LastCacheManager; import org.apache.iotdb.db.metadata.logfile.MLogReader; import org.apache.iotdb.db.metadata.logfile.MLogWriter; @@ -297,6 +297,7 @@ public class MManager { "spend {} ms to deserialize mtree from mlog.bin", System.currentTimeMillis() - time); return idx; } catch (Exception e) { + e.printStackTrace(); throw new IOException("Failed to parser mlog.bin for err:" + e); } } else { @@ -466,13 +467,8 @@ public class MManager { // update id table if (config.isEnableIDTable()) { - try { - IDTable idTable = - StorageEngine.getInstance().getProcessor(plan.getPath().getDevicePath()).getIdTable(); - idTable.createTimeseries(plan); - } catch (StorageEngineException e) { - logger.error("get id table error"); - } + IDTable idTable = IDTableManager.getInstance().getIDTable(plan.getPath().getDevicePath()); + idTable.createTimeseries(plan); } } @@ -584,13 +580,8 @@ public class MManager { // update id table if (config.isEnableIDTable()) { - try { - IDTable idTable = - StorageEngine.getInstance().getProcessor(plan.getPrefixPath()).getIdTable(); - idTable.createAlignedTimeseries(plan); - } catch (StorageEngineException e) { - logger.error("get id table error"); - } + IDTable idTable = IDTableManager.getInstance().getIDTable(plan.getPrefixPath()); + idTable.createAlignedTimeseries(plan); } } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/AppendOnlyDiskSchemaManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/AppendOnlyDiskSchemaManager.java index 77d32aa..4632b2f 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/AppendOnlyDiskSchemaManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/AppendOnlyDiskSchemaManager.java @@ -48,6 +48,7 @@ public class AppendOnlyDiskSchemaManager implements DiskSchemaManager { private static final Logger logger = LoggerFactory.getLogger(AppendOnlyDiskSchemaManager.class); public AppendOnlyDiskSchemaManager(File dir) { + try { initFile(dir); outputStream = new FileOutputStream(dataFile); @@ -57,6 +58,13 @@ public class AppendOnlyDiskSchemaManager implements DiskSchemaManager { } private void initFile(File dir) throws IOException { + // create dirs + if (dir.mkdirs()) { + logger.info( + "ID table create storage group system dir {} doesn't exist, create it", + dir.getParentFile()); + } + dataFile = new File(dir, FILE_NAME); if (dataFile.exists()) { loc = dataFile.length(); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDManager.java deleted file mode 100644 index 9cc5406..0000000 --- a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDManager.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.metadata.id_table; - -import org.apache.iotdb.db.metadata.id_table.entry.IDeviceID; -import org.apache.iotdb.db.metadata.id_table.entry.TimeseriesID; -import org.apache.iotdb.db.metadata.path.PartialPath; -import org.apache.iotdb.tsfile.utils.Pair; - -public interface IDManager { - - /** - * check whether a time series is exist if exist, check the type consistency if not exist, call - * MManager to create it - * - * @param seriesKey full path of the time series - * @return timeseries ID of this time series - */ - public TimeseriesID checkOrCreateIfNotExist(PartialPath seriesKey); - - /** - * upatde latest flushed time of one timeseries - * - * @param timeseriesID timeseries id - * @param flushedTime latest flushed time - */ - public void updateLatestFlushedTime(TimeseriesID timeseriesID, long flushedTime); - - /** - * upatde latest flushed time of one timeseries - * - * @param timeseriesID timeseries id - * @return latest flushed time of one timeseries - */ - public long getLatestFlushedTime(TimeseriesID timeseriesID); - - /** - * get latest time value pair of one timeseries - * - * @param timeseriesID timeseries id - * @return latest time value pair of one timeseries - */ - public Pair<Long, Object> getLastTimeValuePair(TimeseriesID timeseriesID); - - /** - * update latest time value pair of one timeseries - * - * @param timeseriesID timeseries id - * @param lastTimeValue latest time value pair of one timeseries - */ - public void updateLastTimeValuePair(TimeseriesID timeseriesID, Pair<Long, Object> lastTimeValue); - - public static IDeviceID getDeviceID() { - - return null; - } -} diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDManagerImpl.java b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDManagerImpl.java deleted file mode 100644 index 27003ec..0000000 --- a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDManagerImpl.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.metadata.id_table; - -import org.apache.iotdb.db.metadata.id_table.entry.TimeseriesID; -import org.apache.iotdb.db.metadata.path.PartialPath; -import org.apache.iotdb.tsfile.utils.Pair; - -import java.util.Map; - -public class IDManagerImpl implements IDManager { - - /** storage group name -> ID table */ - private Map<String, IDTable> storageGroupIDTableMap; - - /** - * check whether a time series is exist if exist, check the type consistency if not exist, call - * MManager to create it - * - * @param seriesKey full path of the time series - * @return timeseries ID of this time series - */ - @Override - public TimeseriesID checkOrCreateIfNotExist(PartialPath seriesKey) { - return null; - } - - /** - * upatde latest flushed time of one timeseries - * - * @param timeseriesID timeseries id - * @param flushedTime latest flushed time - */ - @Override - public void updateLatestFlushedTime(TimeseriesID timeseriesID, long flushedTime) {} - - /** - * upatde latest flushed time of one timeseries - * - * @param timeseriesID timeseries id - * @return latest flushed time of one timeseries - */ - @Override - public long getLatestFlushedTime(TimeseriesID timeseriesID) { - return 0; - } - - /** - * get latest time value pair of one timeseries - * - * @param timeseriesID timeseries id - * @return latest time value pair of one timeseries - */ - @Override - public Pair<Long, Object> getLastTimeValuePair(TimeseriesID timeseriesID) { - return null; - } - - /** - * update latest time value pair of one timeseries - * - * @param timeseriesID timeseries id - * @param lastTimeValue latest time value pair of one timeseries - */ - @Override - public void updateLastTimeValuePair( - TimeseriesID timeseriesID, Pair<Long, Object> lastTimeValue) {} -} diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTable.java b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTable.java index 1ef1586..cd7494a 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTable.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTable.java @@ -1,80 +1,33 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - package org.apache.iotdb.db.metadata.id_table; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; import org.apache.iotdb.db.exception.metadata.MetadataException; import org.apache.iotdb.db.metadata.id_table.entry.DeviceEntry; import org.apache.iotdb.db.metadata.id_table.entry.DeviceIDFactory; import org.apache.iotdb.db.metadata.id_table.entry.IDeviceID; -import org.apache.iotdb.db.metadata.id_table.entry.InsertMeasurementMNode; -import org.apache.iotdb.db.metadata.id_table.entry.SchemaEntry; import org.apache.iotdb.db.metadata.id_table.entry.TimeseriesID; import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; +import org.apache.iotdb.db.metadata.path.AlignedPath; import org.apache.iotdb.db.metadata.path.MeasurementPath; import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; -import org.apache.iotdb.db.service.IoTDB; import org.apache.iotdb.db.utils.TestOnly; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.TimeValuePair; -import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.util.HashMap; import java.util.Map; -/** id table belongs to a storage group and mapping timeseries path to it's schema */ -public class IDTable { - - // number of table slot - private static final int NUM_OF_SLOTS = 256; +public interface IDTable { /** logger */ - private static final Logger logger = LoggerFactory.getLogger(IDTable.class); - - /** - * 256 hashmap for avoiding rehash performance issue and lock competition device ID -> - * (measurement name -> schema entry) - */ - private Map<IDeviceID, DeviceEntry>[] idTables; + Logger logger = LoggerFactory.getLogger(IDTable.class); - /** disk schema manager to manage disk schema entry */ - private DiskSchemaManager diskSchemaManager; /** iotdb config */ - protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - - public IDTable(File storageGroupDir) { - idTables = new Map[NUM_OF_SLOTS]; - for (int i = 0; i < NUM_OF_SLOTS; i++) { - idTables[i] = new HashMap<>(); - } - diskSchemaManager = new AppendOnlyDiskSchemaManager(storageGroupDir); - } + IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); /** * create aligned timeseries @@ -82,24 +35,7 @@ public class IDTable { * @param plan create aligned timeseries plan * @throws MetadataException if the device is not aligned, throw it */ - public synchronized void createAlignedTimeseries(CreateAlignedTimeSeriesPlan plan) - throws MetadataException { - DeviceEntry deviceEntry = getDeviceEntry(plan.getPrefixPath(), true); - - for (int i = 0; i < plan.getMeasurements().size(); i++) { - PartialPath fullPath = - new PartialPath(plan.getPrefixPath().toString(), plan.getMeasurements().get(i)); - SchemaEntry schemaEntry = - new SchemaEntry( - plan.getDataTypes().get(i), - plan.getEncodings().get(i), - plan.getCompressors().get(i), - deviceEntry.getDeviceID(), - fullPath, - diskSchemaManager); - deviceEntry.putSchemaEntry(plan.getMeasurements().get(i), schemaEntry); - } - } + void createAlignedTimeseries(CreateAlignedTimeSeriesPlan plan) throws MetadataException; /** * create timeseries @@ -107,18 +43,7 @@ public class IDTable { * @param plan create timeseries plan * @throws MetadataException if the device is aligned, throw it */ - public synchronized void createTimeseries(CreateTimeSeriesPlan plan) throws MetadataException { - DeviceEntry deviceEntry = getDeviceEntry(plan.getPath().getDevicePath(), false); - SchemaEntry schemaEntry = - new SchemaEntry( - plan.getDataType(), - plan.getEncoding(), - plan.getCompressor(), - deviceEntry.getDeviceID(), - plan.getPath(), - diskSchemaManager); - deviceEntry.putSchemaEntry(plan.getPath().getMeasurement(), schemaEntry); - } + void createTimeseries(CreateTimeSeriesPlan plan) throws MetadataException; /** * check inserting timeseries existence and fill their measurement mnode @@ -127,62 +52,7 @@ public class IDTable { * @return reusable device id * @throws MetadataException if insert plan's aligned value is inconsistent with device */ - public synchronized IDeviceID getSeriesSchemas(InsertPlan plan) throws MetadataException { - PartialPath devicePath = plan.getDeviceId(); - String[] measurementList = plan.getMeasurements(); - IMeasurementMNode[] measurementMNodes = plan.getMeasurementMNodes(); - - // 1. get device entry and check align - DeviceEntry deviceEntry = getDeviceEntry(devicePath, plan.isAligned()); - - // 2. get schema of each measurement - for (int i = 0; i < measurementList.length; i++) { - try { - // get MeasurementMNode, auto create if absent - try { - IMeasurementMNode measurementMNode = - getOrCreateMeasurementIfNotExist(deviceEntry, plan, i); - - checkDataTypeMatch(plan, i, measurementMNode.getSchema().getType()); - measurementMNodes[i] = measurementMNode; - } catch (DataTypeMismatchException mismatchException) { - if (!config.isEnablePartialInsert()) { - throw mismatchException; - } else { - // mark failed measurement - plan.markFailedMeasurementInsertion(i, mismatchException); - } - } - } catch (MetadataException e) { - if (IoTDB.isClusterMode()) { - logger.debug( - "meet error when check {}.{}, message: {}", - devicePath, - measurementList[i], - e.getMessage()); - } else { - logger.warn( - "meet error when check {}.{}, message: {}", - devicePath, - measurementList[i], - e.getMessage()); - } - if (config.isEnablePartialInsert()) { - // mark failed measurement - plan.markFailedMeasurementInsertion(i, e); - } else { - throw e; - } - } - } - - // set reusable device id - plan.setDeviceID(deviceEntry.getDeviceID()); - // for last flushed time map - plan.setDeviceId(new PartialPath(deviceEntry.getDeviceID().toStringID())); - - return deviceEntry.getDeviceID(); - } + IDeviceID getSeriesSchemas(InsertPlan plan) throws MetadataException; /** * update latest flushed time of one timeseries @@ -191,10 +61,7 @@ public class IDTable { * @param flushTime latest flushed time * @throws MetadataException throw if this timeseries is not exist */ - public synchronized void updateLatestFlushTime(TimeseriesID timeseriesID, long flushTime) - throws MetadataException { - getSchemaEntry(timeseriesID).updateLastedFlushTime(flushTime); - } + void updateLatestFlushTime(TimeseriesID timeseriesID, long flushTime) throws MetadataException; /** * update latest flushed time of one timeseries @@ -203,10 +70,7 @@ public class IDTable { * @return latest flushed time of one timeseries * @throws MetadataException throw if this timeseries is not exist */ - public synchronized long getLatestFlushedTime(TimeseriesID timeseriesID) - throws MetadataException { - return getSchemaEntry(timeseriesID).getFlushTime(); - } + long getLatestFlushedTime(TimeseriesID timeseriesID) throws MetadataException; /** * register trigger to the timeseries @@ -215,13 +79,8 @@ public class IDTable { * @param measurementMNode the timeseries measurement mnode * @throws MetadataException if the timeseries is not exits */ - public synchronized void registerTrigger(PartialPath fullPath, IMeasurementMNode measurementMNode) - throws MetadataException { - boolean isAligned = measurementMNode.getParent().isAligned(); - DeviceEntry deviceEntry = getDeviceEntry(fullPath.getDevicePath(), isAligned); - - deviceEntry.getSchemaEntry(fullPath.getMeasurement()).setUsingTrigger(); - } + void registerTrigger(PartialPath fullPath, IMeasurementMNode measurementMNode) + throws MetadataException; /** * deregister trigger to the timeseries @@ -230,24 +89,15 @@ public class IDTable { * @param measurementMNode the timeseries measurement mnode * @throws MetadataException if the timeseries is not exits */ - public synchronized void deregisterTrigger( - PartialPath fullPath, IMeasurementMNode measurementMNode) throws MetadataException { - boolean isAligned = measurementMNode.getParent().isAligned(); - DeviceEntry deviceEntry = getDeviceEntry(fullPath.getDevicePath(), isAligned); - - deviceEntry.getSchemaEntry(fullPath.getMeasurement()).setUnUsingTrigger(); - } - + void deregisterTrigger(PartialPath fullPath, IMeasurementMNode measurementMNode) + throws MetadataException; /** * get last cache of the timeseies * * @param timeseriesID timeseries ID of the timeseries * @throws MetadataException if the timeseries is not exits */ - public synchronized TimeValuePair getLastCache(TimeseriesID timeseriesID) - throws MetadataException { - return getSchemaEntry(timeseriesID).getCachedLast(); - } + TimeValuePair getLastCache(TimeseriesID timeseriesID) throws MetadataException; /** * update last cache of the timeseies @@ -258,159 +108,12 @@ public class IDTable { * @param latestFlushedTime last flushed time * @throws MetadataException if the timeseries is not exits */ - public synchronized void updateLastCache( + void updateLastCache( TimeseriesID timeseriesID, TimeValuePair pair, boolean highPriorityUpdate, Long latestFlushedTime) - throws MetadataException { - getSchemaEntry(timeseriesID).updateCachedLast(pair, highPriorityUpdate, latestFlushedTime); - } - - /** - * check whether a time series is exist if exist, check the type consistency if not exist, call - * MManager to create it - * - * @return measurement MNode of the time series or null if type is not match - */ - private IMeasurementMNode getOrCreateMeasurementIfNotExist( - DeviceEntry deviceEntry, InsertPlan plan, int loc) throws MetadataException { - String measurementName = plan.getMeasurements()[loc]; - PartialPath seriesKey = new PartialPath(plan.getDeviceId().toString(), measurementName); - - SchemaEntry schemaEntry = deviceEntry.getSchemaEntry(measurementName); - - // if not exist, we create it - if (schemaEntry == null) { - // we have to copy plan's mnode for using id table's last cache - IMeasurementMNode[] insertPlanMNodeBackup = - new IMeasurementMNode[plan.getMeasurementMNodes().length]; - System.arraycopy( - plan.getMeasurementMNodes(), 0, insertPlanMNodeBackup, 0, insertPlanMNodeBackup.length); - try { - IoTDB.metaManager.getSeriesSchemasAndReadLockDevice(plan); - } catch (IOException e) { - throw new MetadataException(e); - } - - // if the timeseries is in template, mmanager will not create timeseries. so we have to put it - // in id table here - for (IMeasurementMNode measurementMNode : plan.getMeasurementMNodes()) { - if (measurementMNode != null && !deviceEntry.contains(measurementMNode.getName())) { - IMeasurementSchema schema = measurementMNode.getSchema(); - SchemaEntry curEntry = - new SchemaEntry( - schema.getType(), - schema.getEncodingType(), - schema.getCompressor(), - deviceEntry.getDeviceID(), - seriesKey, - diskSchemaManager); - deviceEntry.putSchemaEntry(measurementMNode.getName(), curEntry); - } - } - - // copy back measurement mnode list - System.arraycopy( - insertPlanMNodeBackup, 0, plan.getMeasurementMNodes(), 0, insertPlanMNodeBackup.length); - - schemaEntry = deviceEntry.getSchemaEntry(measurementName); - } - - // timeseries is using trigger, we should get trigger from mmanager - if (schemaEntry.isUsingTrigger()) { - IMeasurementMNode measurementMNode = IoTDB.metaManager.getMeasurementMNode(seriesKey); - return new InsertMeasurementMNode( - measurementName, schemaEntry, measurementMNode.getTriggerExecutor()); - } - - return new InsertMeasurementMNode(measurementName, schemaEntry); - } - - /** - * get device id from device path and check is aligned, - * - * @param deviceName device name of the time series - * @param isAligned whether the insert plan is aligned - * @return device entry of the timeseries - */ - private DeviceEntry getDeviceEntry(PartialPath deviceName, boolean isAligned) - throws MetadataException { - IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(deviceName); - int slot = calculateSlot(deviceID); - - DeviceEntry deviceEntry = idTables[slot].get(deviceID); - // new device - if (deviceEntry == null) { - deviceEntry = new DeviceEntry(deviceID); - deviceEntry.setAligned(isAligned); - idTables[slot].put(deviceID, deviceEntry); - - return deviceEntry; - } - - // check aligned - if (deviceEntry.isAligned() != isAligned) { - throw new MetadataException( - String.format( - "Timeseries under path [%s]'s align value is [%b], which is not consistent with insert plan", - deviceName, deviceEntry.isAligned())); - } - - // reuse device entry in map - return deviceEntry; - } - - /** - * calculate slot that this deviceID should in - * - * @param deviceID device id - * @return slot number - */ - private int calculateSlot(IDeviceID deviceID) { - return Math.abs(deviceID.hashCode()) % NUM_OF_SLOTS; - } - - /** - * get schema entry - * - * @param timeseriesID the timeseries ID - * @return schema entry of the timeseries - * @throws MetadataException throw if this timeseries is not exist - */ - private SchemaEntry getSchemaEntry(TimeseriesID timeseriesID) throws MetadataException { - IDeviceID deviceID = timeseriesID.getDeviceID(); - int slot = calculateSlot(deviceID); - - DeviceEntry deviceEntry = idTables[slot].get(deviceID); - if (deviceEntry == null) { - throw new MetadataException( - "get non exist timeseries's schema entry, timeseries id is: " + timeseriesID); - } - - SchemaEntry schemaEntry = deviceEntry.getSchemaEntry(timeseriesID.getMeasurement()); - if (schemaEntry == null) { - throw new MetadataException( - "get non exist timeseries's schema entry, timeseries id is: " + timeseriesID); - } - - return schemaEntry; - } - - // from mmanger - private void checkDataTypeMatch(InsertPlan plan, int loc, TSDataType dataType) - throws MetadataException { - TSDataType insertDataType = plan.getDataTypes()[loc]; - if (dataType != insertDataType) { - String measurement = plan.getMeasurements()[loc]; - logger.warn( - "DataType mismatch, Insert measurement {} type {}, metadata tree type {}", - measurement, - insertDataType, - dataType); - throw new DataTypeMismatchException(measurement, insertDataType, dataType); - } - } + throws MetadataException; /** * translate query path's device path to device id @@ -418,14 +121,25 @@ public class IDTable { * @param fullPath full query path * @return translated query path */ - public static PartialPath translateQueryPath(PartialPath fullPath) { + static PartialPath translateQueryPath(PartialPath fullPath) { // if not enable id table, just return original path if (!config.isEnableIDTable()) { return fullPath; } - TimeseriesID timeseriesID = new TimeseriesID(fullPath); try { + // handle aligned path + if (fullPath instanceof AlignedPath) { + AlignedPath cur = (AlignedPath) fullPath; + + return new AlignedPath( + DeviceIDFactory.getInstance().getDeviceID(cur).toStringID(), + cur.getMeasurementList(), + cur.getSchemaList()); + } + + // normal path + TimeseriesID timeseriesID = new TimeseriesID(fullPath); return new MeasurementPath( timeseriesID.getDeviceID().toStringID(), timeseriesID.getMeasurement(), @@ -438,12 +152,8 @@ public class IDTable { } @TestOnly - public Map<IDeviceID, DeviceEntry>[] getIdTables() { - return idTables; - } + Map<IDeviceID, DeviceEntry>[] getIdTables(); @TestOnly - public DiskSchemaManager getDiskSchemaManager() { - return diskSchemaManager; - } + DiskSchemaManager getDiskSchemaManager(); } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTable.java b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTableHashmapImpl.java similarity index 94% copy from server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTable.java copy to server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTableHashmapImpl.java index 1ef1586..12e7083 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTable.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTableHashmapImpl.java @@ -30,7 +30,6 @@ import org.apache.iotdb.db.metadata.id_table.entry.InsertMeasurementMNode; import org.apache.iotdb.db.metadata.id_table.entry.SchemaEntry; import org.apache.iotdb.db.metadata.id_table.entry.TimeseriesID; import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; -import org.apache.iotdb.db.metadata.path.MeasurementPath; import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; @@ -50,12 +49,12 @@ import java.util.HashMap; import java.util.Map; /** id table belongs to a storage group and mapping timeseries path to it's schema */ -public class IDTable { +public class IDTableHashmapImpl implements IDTable { // number of table slot private static final int NUM_OF_SLOTS = 256; /** logger */ - private static final Logger logger = LoggerFactory.getLogger(IDTable.class); + private static final Logger logger = LoggerFactory.getLogger(IDTableHashmapImpl.class); /** * 256 hashmap for avoiding rehash performance issue and lock competition device ID -> @@ -68,7 +67,7 @@ public class IDTable { /** iotdb config */ protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - public IDTable(File storageGroupDir) { + public IDTableHashmapImpl(File storageGroupDir) { idTables = new Map[NUM_OF_SLOTS]; for (int i = 0; i < NUM_OF_SLOTS; i++) { idTables[i] = new HashMap<>(); @@ -179,6 +178,7 @@ public class IDTable { // set reusable device id plan.setDeviceID(deviceEntry.getDeviceID()); // for last flushed time map + plan.setDevicePath(devicePath); plan.setDeviceId(new PartialPath(deviceEntry.getDeviceID().toStringID())); return deviceEntry.getDeviceID(); @@ -412,31 +412,6 @@ public class IDTable { } } - /** - * translate query path's device path to device id - * - * @param fullPath full query path - * @return translated query path - */ - public static PartialPath translateQueryPath(PartialPath fullPath) { - // if not enable id table, just return original path - if (!config.isEnableIDTable()) { - return fullPath; - } - - TimeseriesID timeseriesID = new TimeseriesID(fullPath); - try { - return new MeasurementPath( - timeseriesID.getDeviceID().toStringID(), - timeseriesID.getMeasurement(), - fullPath.getMeasurementSchema()); - } catch (MetadataException e) { - logger.error("Error when translate query path: " + fullPath); - } - - return null; - } - @TestOnly public Map<IDeviceID, DeviceEntry>[] getIdTables() { return idTables; diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTableManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTableManager.java new file mode 100644 index 0000000..7905bf1 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTableManager.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.id_table; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory; +import org.apache.iotdb.db.exception.metadata.MetadataException; +import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; +import org.apache.iotdb.db.metadata.path.PartialPath; +import org.apache.iotdb.db.service.IoTDB; +import org.apache.iotdb.tsfile.utils.FilePathUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.HashMap; + +public class IDTableManager { + + /** logger */ + Logger logger = LoggerFactory.getLogger(IDTableManager.class); + + /** storage group path -> id table */ + HashMap<String, IDTable> idTableMap; + + /** system dir */ + private final String systemDir = + FilePathUtils.regularizePath(IoTDBDescriptor.getInstance().getConfig().getSystemDir()) + + "storage_groups"; + + // region IDManager Singleton + private static class IDManagerHolder { + + private IDManagerHolder() { + // allowed to do nothing + } + + private static final IDTableManager INSTANCE = new IDTableManager(); + } + + /** + * get instance + * + * @return instance of the factory + */ + public static IDTableManager getInstance() { + return IDManagerHolder.INSTANCE; + } + + private IDTableManager() { + idTableMap = new HashMap<>(); + } + // endregion + + /** + * get id table by device path + * + * @param devicePath device path + * @return id table belongs to path's storage group + */ + public synchronized IDTable getIDTable(PartialPath devicePath) { + try { + IStorageGroupMNode storageGroupMNode = + IoTDB.metaManager.getStorageGroupNodeByPath(devicePath); + return idTableMap.computeIfAbsent( + storageGroupMNode.getFullPath(), + storageGroupPath -> + new IDTableHashmapImpl( + SystemFileFactory.INSTANCE.getFile( + systemDir + File.separator + storageGroupPath))); + } catch (MetadataException e) { + logger.error("get id table failed, path is: " + devicePath + ". caused by: " + e); + } + + return null; + } + + /** clear id table map */ + public void clear() { + idTableMap.clear(); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceIDFactory.java b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceIDFactory.java index 1fd2752..2884c53 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceIDFactory.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceIDFactory.java @@ -44,14 +44,15 @@ public class DeviceIDFactory { * @return instance of the factory */ public static DeviceIDFactory getInstance() { - return DeviceIDFactory.DeviceIDFactoryHolder.INSTANCE; + return DeviceIDFactoryHolder.INSTANCE; } private DeviceIDFactory() { - if (IoTDBDescriptor.getInstance() - .getConfig() - .getDeviceIDTransformationMethod() - .equals("SHA256")) { + if (IoTDBDescriptor.getInstance().getConfig().isEnableIDTable() + && IoTDBDescriptor.getInstance() + .getConfig() + .getDeviceIDTransformationMethod() + .equals("SHA256")) { getDeviceIDFunction = partialPath -> new SHA256DeviceID(partialPath.toString()); } else { getDeviceIDFunction = partialPath -> new PlainDeviceID(partialPath.toString()); diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java index 9276b71..a8f4d2e 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java @@ -33,7 +33,11 @@ import java.util.List; public abstract class InsertPlan extends PhysicalPlan { + // id form of device path protected PartialPath deviceId; + // device path + protected PartialPath devicePath; + protected boolean isAligned; protected String[] measurements; // get from client @@ -198,4 +202,12 @@ public abstract class InsertPlan extends PhysicalPlan { public void setDeviceID(IDeviceID deviceID) { this.deviceID = deviceID; } + + public PartialPath getDevicePath() { + return devicePath; + } + + public void setDevicePath(PartialPath devicePath) { + this.devicePath = devicePath; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowPlan.java index 6be7392..444a2ac 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowPlan.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowPlan.java @@ -335,7 +335,7 @@ public class InsertRowPlan extends InsertPlan { public void subSerialize(DataOutputStream stream) throws IOException { stream.writeLong(time); - putString(stream, deviceId.getFullPath()); + putString(stream, devicePath.getFullPath()); serializeMeasurementsAndValues(stream); } @@ -486,7 +486,7 @@ public class InsertRowPlan extends InsertPlan { public void subSerialize(ByteBuffer buffer) { buffer.putLong(time); - putString(buffer, deviceId.getFullPath()); + putString(buffer, devicePath.getFullPath()); serializeMeasurementsAndValues(buffer); } diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowsOfOneDevicePlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowsOfOneDevicePlan.java index 8e5c86c..0475447 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowsOfOneDevicePlan.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertRowsOfOneDevicePlan.java @@ -146,7 +146,7 @@ public class InsertRowsOfOneDevicePlan extends InsertPlan implements BatchPlan { public void serialize(DataOutputStream stream) throws IOException { int type = PhysicalPlanType.BATCH_INSERT_ONE_DEVICE.ordinal(); stream.writeByte((byte) type); - putString(stream, deviceId.getFullPath()); + putString(stream, devicePath.getFullPath()); stream.writeInt(rowPlans.length); for (InsertRowPlan plan : rowPlans) { @@ -163,7 +163,7 @@ public class InsertRowsOfOneDevicePlan extends InsertPlan implements BatchPlan { int type = PhysicalPlanType.BATCH_INSERT_ONE_DEVICE.ordinal(); buffer.put((byte) type); - putString(buffer, deviceId.getFullPath()); + putString(buffer, devicePath.getFullPath()); buffer.putInt(rowPlans.length); for (InsertRowPlan plan : rowPlans) { buffer.putLong(plan.getTime()); diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertTabletPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertTabletPlan.java index a65c6e1..b07ce60 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertTabletPlan.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertTabletPlan.java @@ -160,7 +160,7 @@ public class InsertTabletPlan extends InsertPlan { } public void subSerialize(DataOutputStream stream) throws IOException { - putString(stream, deviceId.getFullPath()); + putString(stream, devicePath.getFullPath()); writeMeasurements(stream); writeDataTypes(stream); writeTimes(stream); @@ -254,7 +254,7 @@ public class InsertTabletPlan extends InsertPlan { } public void subSerialize(ByteBuffer buffer) { - putString(buffer, deviceId.getFullPath()); + putString(buffer, devicePath.getFullPath()); writeMeasurements(buffer); writeDataTypes(buffer); writeTimes(buffer); diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java index f0c90a2..08abb66 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java +++ b/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.MetadataException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.metadata.id_table.IDTable; +import org.apache.iotdb.db.metadata.id_table.IDTableManager; import org.apache.iotdb.db.metadata.id_table.entry.TimeseriesID; import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; import org.apache.iotdb.db.metadata.path.MeasurementPath; @@ -335,10 +336,9 @@ public class LastQueryExecutor { @Override public TimeValuePair read() { try { - IDTable table = - StorageEngine.getInstance().getProcessor(fullPath.getDevicePath()).getIdTable(); + IDTable table = IDTableManager.getInstance().getIDTable(fullPath.getDevicePath()); return table.getLastCache(new TimeseriesID(fullPath)); - } catch (StorageEngineException | MetadataException e) { + } catch (MetadataException e) { logger.error("last query can't find storage group: path is: " + fullPath); } @@ -348,10 +348,9 @@ public class LastQueryExecutor { @Override public void write(TimeValuePair pair) { try { - IDTable table = - StorageEngine.getInstance().getProcessor(fullPath.getDevicePath()).getIdTable(); + IDTable table = IDTableManager.getInstance().getIDTable(fullPath.getDevicePath()); table.updateLastCache(new TimeseriesID(fullPath), pair, false, Long.MIN_VALUE); - } catch (StorageEngineException | MetadataException e) { + } catch (MetadataException e) { logger.error("last query can't find storage group: path is: " + fullPath); } } diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/RawDataQueryExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/RawDataQueryExecutor.java index 88481d9..39adb87 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/executor/RawDataQueryExecutor.java +++ b/server/src/main/java/org/apache/iotdb/db/query/executor/RawDataQueryExecutor.java @@ -41,6 +41,9 @@ import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet; import org.apache.iotdb.tsfile.read.query.timegenerator.TimeGenerator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -57,6 +60,8 @@ public class RawDataQueryExecutor { this.queryPlan = queryPlan; } + private static final Logger logger = LoggerFactory.getLogger(RawDataQueryExecutor.class); + /** without filter or with global time filter. */ public QueryDataSet executeWithoutValueFilter(QueryContext context) throws StorageEngineException, QueryProcessException { @@ -127,6 +132,8 @@ public class RawDataQueryExecutor { queryPlan.isAscending()); readersOfSelectedSeries.add(reader); } + } catch (Exception e) { + logger.error("Meet error when init series reader " + e); } finally { StorageEngine.getInstance().mergeUnLock(list); } diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/fill/LastPointReader.java b/server/src/main/java/org/apache/iotdb/db/query/executor/fill/LastPointReader.java index f5ee18a..4b6121b 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/executor/fill/LastPointReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/executor/fill/LastPointReader.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.query.executor.fill; import org.apache.iotdb.db.engine.querycontext.QueryDataSource; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.metadata.id_table.IDTable; import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.utils.FileLoaderUtils; @@ -70,7 +71,7 @@ public class LastPointReader { QueryDataSource dataSource, long queryTime, Filter timeFilter) { - this.seriesPath = seriesPath; + this.seriesPath = IDTable.translateQueryPath(seriesPath); this.dataType = dataType; this.dataSource = dataSource; this.context = context; diff --git a/server/src/main/java/org/apache/iotdb/db/rest/RestService.java b/server/src/main/java/org/apache/iotdb/db/rest/RestService.java index 75928f7..68b2e60 100644 --- a/server/src/main/java/org/apache/iotdb/db/rest/RestService.java +++ b/server/src/main/java/org/apache/iotdb/db/rest/RestService.java @@ -103,12 +103,12 @@ public class RestService implements IService { } private void serverStart() { - try { - server.start(); - } catch (Exception e) { - LOGGER.warn("RestService failed to start: {}", e.getMessage()); - server.destroy(); - } + // try { + // server.start(); + // } catch (Exception e) { + // LOGGER.warn("RestService failed to start: {}", e.getMessage()); + // server.destroy(); + // } } @Override @@ -129,13 +129,13 @@ public class RestService implements IService { @Override public void stop() { - try { - server.stop(); - } catch (Exception e) { - LOGGER.warn("RestService failed to stop: {}", e.getMessage()); - } finally { - server.destroy(); - } + // try { + // server.stop(); + // } catch (Exception e) { + // LOGGER.warn("RestService failed to stop: {}", e.getMessage()); + // } finally { + // server.destroy(); + // } } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/EnvironmentUtils.java index fc45633..374ee76 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/EnvironmentUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/EnvironmentUtils.java @@ -35,6 +35,7 @@ import org.apache.iotdb.db.exception.ContinuousQueryException; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.TriggerManagementException; import org.apache.iotdb.db.exception.UDFRegistrationException; +import org.apache.iotdb.db.metadata.id_table.IDTableManager; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.control.FileReaderManager; import org.apache.iotdb.db.query.control.QueryResourceManager; @@ -163,6 +164,9 @@ public class EnvironmentUtils { // clear tsFileResource manager info TsFileResourceManager.getInstance().clear(); + // clear id table manager + IDTableManager.getInstance().clear(); + // delete all directory cleanAllDir(); config.setSeqTsFileSize(oldSeqTsFileSize); @@ -305,6 +309,7 @@ public class EnvironmentUtils { shutdownDaemon(); stopDaemon(); IoTDB.metaManager.clear(); + IDTableManager.getInstance().clear(); TsFileResourceManager.getInstance().clear(); reactiveDaemon(); } diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableLogFileTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableHashmapImplLogFileTest.java similarity index 96% rename from server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableLogFileTest.java rename to server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableHashmapImplLogFileTest.java index 2d0d51c..10147a3 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableLogFileTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableHashmapImplLogFileTest.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.metadata.id_table; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.metadata.id_table.entry.DeviceIDFactory; import org.apache.iotdb.db.metadata.id_table.entry.DiskSchemaEntry; @@ -41,7 +40,7 @@ import java.util.HashSet; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -public class IDTableLogFileTest { +public class IDTableHashmapImplLogFileTest { private boolean isEnableIDTable = false; @@ -96,8 +95,7 @@ public class IDTableLogFileTest { new IMeasurementMNode[insertRowPlan.getMeasurements().length]); // call getSeriesSchemasAndReadLockDevice - IDTable idTable = - StorageEngine.getInstance().getProcessor(new PartialPath("root.laptop")).getIdTable(); + IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.laptop")); idTable.getSeriesSchemas(insertRowPlan); diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableResourceControlTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableHashmapImplResourceControlTest.java similarity index 96% copy from server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableResourceControlTest.java copy to server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableHashmapImplResourceControlTest.java index e24d463..a4047fa 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableResourceControlTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableHashmapImplResourceControlTest.java @@ -51,7 +51,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.fail; -public class IDTableResourceControlTest { +public class IDTableHashmapImplResourceControlTest { private final Planner processor = new Planner(); private boolean isEnableIDTable = false; @@ -97,10 +97,7 @@ public class IDTableResourceControlTest { IDeviceID idTableDeviceID = null; for (Map<IDeviceID, DeviceEntry> map : - StorageEngine.getInstance() - .getProcessor(new PartialPath("root.isp.d1")) - .getIdTable() - .getIdTables()) { + IDTableManager.getInstance().getIDTable(new PartialPath("root.isp.d1")).getIdTables()) { if (map == null) { continue; } diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableResourceControlTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableHashmapImplRestartTest.java similarity index 50% rename from server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableResourceControlTest.java rename to server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableHashmapImplRestartTest.java index e24d463..1634a83 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableResourceControlTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableHashmapImplRestartTest.java @@ -16,28 +16,23 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.metadata.id_table; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.StorageEngine; -import org.apache.iotdb.db.engine.memtable.IWritableMemChunkGroup; -import org.apache.iotdb.db.engine.storagegroup.TsFileProcessor; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.IllegalPathException; -import org.apache.iotdb.db.exception.metadata.MetadataException; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.metadata.id_table.entry.DeviceEntry; -import org.apache.iotdb.db.metadata.id_table.entry.IDeviceID; import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.db.qp.Planner; import org.apache.iotdb.db.qp.executor.PlanExecutor; -import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; +import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan; import org.apache.iotdb.db.qp.physical.crud.QueryPlan; import org.apache.iotdb.db.utils.EnvironmentUtils; -import org.apache.iotdb.tsfile.exception.filter.QueryFilterOptimizationException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.RowRecord; import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet; +import org.apache.iotdb.tsfile.utils.Binary; import org.junit.After; import org.junit.Assert; @@ -45,21 +40,35 @@ import org.junit.Before; import org.junit.Test; import java.io.IOException; -import java.util.Map; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; -public class IDTableResourceControlTest { +public class IDTableHashmapImplRestartTest { + private final Planner processor = new Planner(); private boolean isEnableIDTable = false; private String originalDeviceIDTransformationMethod = null; + Set<String> retSet = + new HashSet<>( + Arrays.asList( + "113\troot.isp.d1.s3\t100003\tINT64", + "113\troot.isp.d1.s4\t1003\tINT32", + "113\troot.isp.d1.s5\tfalse\tBOOLEAN", + "113\troot.isp.d1.s6\tmm3\tTEXT", + "113\troot.isp.d1.s1\t13.0\tDOUBLE", + "113\troot.isp.d1.s2\t23.0\tFLOAT")); + @Before public void before() { + IoTDBDescriptor.getInstance().getConfig().setAutoCreateSchemaEnabled(true); isEnableIDTable = IoTDBDescriptor.getInstance().getConfig().isEnableIDTable(); originalDeviceIDTransformationMethod = IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod(); @@ -79,92 +88,69 @@ public class IDTableResourceControlTest { } @Test - public void testDeviceIDReusing() - throws QueryProcessException, MetadataException, InterruptedException, - QueryFilterOptimizationException, StorageEngineException, IOException { - InsertRowPlan rowPlan = getInsertRowPlan(); + public void testRawDataQueryAfterRestart() throws Exception { + insertDataInMemory(); + + // restart + try { + EnvironmentUtils.restartDaemon(); + } catch (Exception e) { + Assert.fail(); + } PlanExecutor executor = new PlanExecutor(); - executor.insert(rowPlan); - QueryPlan queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from root.isp.d1"); QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); Assert.assertEquals(6, dataSet.getPaths().size()); + int count = 0; while (dataSet.hasNext()) { RowRecord record = dataSet.next(); - Assert.assertEquals(6, record.getFields().size()); + System.out.println(record); + count++; } - IDeviceID idTableDeviceID = null; - for (Map<IDeviceID, DeviceEntry> map : - StorageEngine.getInstance() - .getProcessor(new PartialPath("root.isp.d1")) - .getIdTable() - .getIdTables()) { - if (map == null) { - continue; - } - - for (IDeviceID deviceID : map.keySet()) { - if (idTableDeviceID == null) { - idTableDeviceID = deviceID; - } else { - fail("there should only be one device in id table"); - } - } - } + assertEquals(4, count); + + assertEquals(4, count); + } - assertNotNull(idTableDeviceID); - - int deviceCount = 0; - for (TsFileProcessor processor : - StorageEngine.getInstance() - .getProcessor(new PartialPath("root.isp.d1")) - .getWorkSequenceTsFileProcessors()) { - for (Map.Entry<IDeviceID, IWritableMemChunkGroup> entry : - processor.getWorkMemTable().getMemTableMap().entrySet()) { - // using '!=' to check is same device id - if (entry.getKey() != rowPlan.getDeviceID()) { - fail("memtable's device id is not same as insert plan's device id"); - } - - // using '!=' to check is same device id - if (entry.getKey() != idTableDeviceID) { - fail("memtable's device id is not same as insert plan's device id"); - } - - deviceCount++; - } + private void insertDataInMemory() throws IllegalPathException, QueryProcessException { + long[] times = new long[] {110L, 111L, 112L, 113L}; + List<Integer> dataTypes = new ArrayList<>(); + dataTypes.add(TSDataType.DOUBLE.ordinal()); + dataTypes.add(TSDataType.FLOAT.ordinal()); + dataTypes.add(TSDataType.INT64.ordinal()); + dataTypes.add(TSDataType.INT32.ordinal()); + dataTypes.add(TSDataType.BOOLEAN.ordinal()); + dataTypes.add(TSDataType.TEXT.ordinal()); + + Object[] columns = new Object[6]; + columns[0] = new double[4]; + columns[1] = new float[4]; + columns[2] = new long[4]; + columns[3] = new int[4]; + columns[4] = new boolean[4]; + columns[5] = new Binary[4]; + + for (int r = 0; r < 4; r++) { + ((double[]) columns[0])[r] = 10.0 + r; + ((float[]) columns[1])[r] = 20 + r; + ((long[]) columns[2])[r] = 100000 + r; + ((int[]) columns[3])[r] = 1000 + r; + ((boolean[]) columns[4])[r] = false; + ((Binary[]) columns[5])[r] = new Binary("mm" + r); } - assertEquals(1, deviceCount); - } + InsertTabletPlan tabletPlan = + new InsertTabletPlan( + new PartialPath("root.isp.d1"), + new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}, + dataTypes); + tabletPlan.setTimes(times); + tabletPlan.setColumns(columns); + tabletPlan.setRowCount(times.length); - private InsertRowPlan getInsertRowPlan() throws IllegalPathException { - long time = 110L; - TSDataType[] dataTypes = - new TSDataType[] { - TSDataType.DOUBLE, - TSDataType.FLOAT, - TSDataType.INT64, - TSDataType.INT32, - TSDataType.BOOLEAN, - TSDataType.TEXT - }; - - String[] columns = new String[6]; - columns[0] = 1.0 + ""; - columns[1] = 2 + ""; - columns[2] = 10000 + ""; - columns[3] = 100 + ""; - columns[4] = false + ""; - columns[5] = "hh" + 0; - - return new InsertRowPlan( - new PartialPath("root.isp.d1"), - time, - new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}, - dataTypes, - columns); + PlanExecutor executor = new PlanExecutor(); + executor.insertTablet(tabletPlan); } } diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableHashmapImplTest.java similarity index 95% rename from server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableTest.java rename to server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableHashmapImplTest.java index 21b49a6..740906f 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableHashmapImplTest.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.metadata.id_table; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.trigger.service.TriggerRegistrationService; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; @@ -59,7 +58,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -public class IDTableTest { +public class IDTableHashmapImplTest { private CompressionType compressionType; @@ -115,8 +114,7 @@ public class IDTableTest { manager.createAlignedTimeSeriesEntry(plan); - IDTable idTable = - StorageEngine.getInstance().getProcessor(new PartialPath("root.laptop")).getIdTable(); + IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.laptop")); // construct an insertRowPlan with mismatched data type long time = 1L; @@ -195,8 +193,7 @@ public class IDTableTest { manager.createAlignedTimeSeriesEntry(plan); - IDTable idTable = - StorageEngine.getInstance().getProcessor(new PartialPath("root.laptop")).getIdTable(); + IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.laptop")); // construct an insertRowPlan with mismatched data type long time = 1L; @@ -250,8 +247,7 @@ public class IDTableTest { compressionType, Collections.emptyMap()); - IDTable idTable = - StorageEngine.getInstance().getProcessor(new PartialPath("root.laptop")).getIdTable(); + IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.laptop")); long time = 1L; String[] columns = new String[1]; @@ -332,8 +328,7 @@ public class IDTableTest { new IMeasurementMNode[insertRowPlan.getMeasurements().length]); // call getSeriesSchemasAndReadLockDevice - IDTable idTable = - StorageEngine.getInstance().getProcessor(new PartialPath("root.laptop")).getIdTable(); + IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.laptop")); idTable.getSeriesSchemas(insertRowPlan); fail("should throw exception"); @@ -370,8 +365,7 @@ public class IDTableTest { new IMeasurementMNode[insertRowPlan.getMeasurements().length]); // call getSeriesSchemasAndReadLockDevice - IDTable idTable = - StorageEngine.getInstance().getProcessor(new PartialPath("root.laptop")).getIdTable(); + IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.laptop")); idTable.getSeriesSchemas(insertRowPlan); @@ -425,7 +419,7 @@ public class IDTableTest { } catch (Exception e) { fail("throw wrong exception"); } - } catch (MetadataException | StorageEngineException e) { + } catch (MetadataException e) { e.printStackTrace(); fail("throw exception"); } @@ -455,8 +449,7 @@ public class IDTableTest { new IMeasurementMNode[insertRowPlan.getMeasurements().length]); // call getSeriesSchemasAndReadLockDevice - IDTable idTable = - StorageEngine.getInstance().getProcessor(new PartialPath("root.laptop")).getIdTable(); + IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.laptop")); idTable.getSeriesSchemas(insertRowPlan); @@ -511,7 +504,7 @@ public class IDTableTest { } catch (Exception e) { fail("throw wrong exception"); } - } catch (MetadataException | StorageEngineException e) { + } catch (MetadataException e) { e.printStackTrace(); fail("throw exception"); } @@ -563,8 +556,7 @@ public class IDTableTest { new IMeasurementMNode[insertRowPlan.getMeasurements().length]); // call getSeriesSchemasAndReadLockDevice - IDTable idTable = - StorageEngine.getInstance().getProcessor(new PartialPath("root.laptop")).getIdTable(); + IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.laptop")); idTable.getSeriesSchemas(insertRowPlan); @@ -632,8 +624,7 @@ public class IDTableTest { new IMeasurementMNode[insertRowPlan.getMeasurements().length]); // call getSeriesSchemasAndReadLockDevice - IDTable idTable = - StorageEngine.getInstance().getProcessor(new PartialPath("root.laptop")).getIdTable(); + IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.laptop")); idTable.getSeriesSchemas(insertRowPlan); @@ -665,7 +656,7 @@ public class IDTableTest { idTable.updateLatestFlushTime(timeseriesID, 12L); assertEquals(12L, idTable.getLatestFlushedTime(timeseriesID)); - } catch (MetadataException | StorageEngineException e) { + } catch (MetadataException e) { e.printStackTrace(); fail("throw exception"); } diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/InsertWithIDTableTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/InsertWithIDTableHashmapImplTest.java similarity index 99% rename from server/src/test/java/org/apache/iotdb/db/metadata/id_table/InsertWithIDTableTest.java rename to server/src/test/java/org/apache/iotdb/db/metadata/id_table/InsertWithIDTableHashmapImplTest.java index b0d80ba..f559d57 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/InsertWithIDTableTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/InsertWithIDTableHashmapImplTest.java @@ -60,7 +60,7 @@ import java.util.List; import static org.junit.Assert.assertEquals; -public class InsertWithIDTableTest { +public class InsertWithIDTableHashmapImplTest { private final Planner processor = new Planner(); private boolean isEnableIDTable = false; @@ -99,11 +99,15 @@ public class InsertWithIDTableTest { QueryPlan queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from root.isp.d1"); QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); + int count = 0; assertEquals(6, dataSet.getPaths().size()); while (dataSet.hasNext()) { RowRecord record = dataSet.next(); assertEquals(6, record.getFields().size()); + count++; } + + assertEquals(1, count); } @Test @@ -120,11 +124,15 @@ public class InsertWithIDTableTest { QueryPlan queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from root.isp.d1.GPS"); QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); + int count = 0; assertEquals(1, dataSet.getPaths().size()); while (dataSet.hasNext()) { RowRecord record = dataSet.next(); assertEquals(3, record.getFields().size()); + count++; } + + assertEquals(1, count); } @Test @@ -402,6 +410,8 @@ public class InsertWithIDTableTest { } rowNum++; } + + assertEquals(4, rowNum); } @Test diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/QueryWithIDTableTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/LastQueryWithIDTableHashmapImpl.java similarity index 50% copy from server/src/test/java/org/apache/iotdb/db/metadata/id_table/QueryWithIDTableTest.java copy to server/src/test/java/org/apache/iotdb/db/metadata/id_table/LastQueryWithIDTableHashmapImpl.java index 39baabf..807b917 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/QueryWithIDTableTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/LastQueryWithIDTableHashmapImpl.java @@ -1,30 +1,9 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - package org.apache.iotdb.db.metadata.id_table; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.IllegalPathException; import org.apache.iotdb.db.exception.metadata.MetadataException; -import org.apache.iotdb.db.exception.metadata.StorageGroupNotSetException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.metadata.id_table.entry.TimeseriesID; import org.apache.iotdb.db.metadata.path.PartialPath; @@ -36,16 +15,9 @@ import org.apache.iotdb.db.qp.physical.crud.QueryPlan; import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.iotdb.tsfile.exception.filter.QueryFilterOptimizationException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.read.TimeValuePair; import org.apache.iotdb.tsfile.read.common.RowRecord; import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet; import org.apache.iotdb.tsfile.utils.Binary; -import org.apache.iotdb.tsfile.utils.TsPrimitiveType.TsBinary; -import org.apache.iotdb.tsfile.utils.TsPrimitiveType.TsBoolean; -import org.apache.iotdb.tsfile.utils.TsPrimitiveType.TsDouble; -import org.apache.iotdb.tsfile.utils.TsPrimitiveType.TsFloat; -import org.apache.iotdb.tsfile.utils.TsPrimitiveType.TsInt; -import org.apache.iotdb.tsfile.utils.TsPrimitiveType.TsLong; import org.junit.After; import org.junit.Assert; @@ -60,13 +32,16 @@ import java.util.List; import java.util.Set; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -public class QueryWithIDTableTest { +public class LastQueryWithIDTableHashmapImpl { private final Planner processor = new Planner(); private boolean isEnableIDTable = false; + private boolean originalEnableCache = false; + private String originalDeviceIDTransformationMethod = null; Set<String> retSet = @@ -85,6 +60,8 @@ public class QueryWithIDTableTest { isEnableIDTable = IoTDBDescriptor.getInstance().getConfig().isEnableIDTable(); originalDeviceIDTransformationMethod = IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod(); + originalEnableCache = IoTDBDescriptor.getInstance().getConfig().isLastCacheEnabled(); + IoTDBDescriptor.getInstance().getConfig().setEnableLastCache(false); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); @@ -97,86 +74,50 @@ public class QueryWithIDTableTest { IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); + IoTDBDescriptor.getInstance().getConfig().setEnableLastCache(originalEnableCache); EnvironmentUtils.cleanEnv(); } @Test - public void testRawDataQueryAfterFlush() - throws MetadataException, QueryProcessException, StorageEngineException, InterruptedException, - QueryFilterOptimizationException, IOException { - insertDataInDisk(); + public void testLastCacheQueryWithoutCache() + throws QueryProcessException, MetadataException, InterruptedException, + QueryFilterOptimizationException, StorageEngineException, IOException { + insertDataInMemory(); PlanExecutor executor = new PlanExecutor(); - QueryPlan queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from root.isp.d1"); + QueryPlan queryPlan = + (QueryPlan) processor.parseSQLToPhysicalPlan("select last * from root.isp.d1"); QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); - Assert.assertEquals(6, dataSet.getPaths().size()); + Assert.assertEquals(3, dataSet.getPaths().size()); int count = 0; while (dataSet.hasNext()) { RowRecord record = dataSet.next(); + assertTrue(retSet.contains(record.toString())); count++; } - assertEquals(8, count); - } + assertEquals(retSet.size(), count); - @Test - public void testLastCacheQuery() - throws QueryProcessException, MetadataException, InterruptedException, - QueryFilterOptimizationException, StorageEngineException, IOException { - insertDataInMemory(); + // flush and test again + PhysicalPlan flushPlan = processor.parseSQLToPhysicalPlan("flush"); + executor.processNonQuery(flushPlan); - PlanExecutor executor = new PlanExecutor(); - QueryPlan queryPlan = - (QueryPlan) processor.parseSQLToPhysicalPlan("select last * from root.isp.d1"); - QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); + dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); Assert.assertEquals(3, dataSet.getPaths().size()); - int count = 0; + count = 0; while (dataSet.hasNext()) { RowRecord record = dataSet.next(); assertTrue(retSet.contains(record.toString())); count++; } - assertEquals(retSet.size(), count); - // test it from id table - assertEquals( - new TimeValuePair(113L, new TsDouble(13.0d)), - StorageEngine.getInstance() - .getProcessor(new PartialPath("root.isp.d1")) - .getIdTable() + // assert id table is not refresh + assertNull( + IDTableManager.getInstance() + .getIDTable(new PartialPath("root.isp.d1")) .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s1")))); - assertEquals( - new TimeValuePair(113L, new TsFloat(23.0f)), - StorageEngine.getInstance() - .getProcessor(new PartialPath("root.isp.d1")) - .getIdTable() - .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s2")))); - assertEquals( - new TimeValuePair(113L, new TsLong(100003L)), - StorageEngine.getInstance() - .getProcessor(new PartialPath("root.isp.d1")) - .getIdTable() - .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s3")))); - assertEquals( - new TimeValuePair(113L, new TsInt(1003)), - StorageEngine.getInstance() - .getProcessor(new PartialPath("root.isp.d1")) - .getIdTable() - .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s4")))); - assertEquals( - new TimeValuePair(113L, new TsBoolean(false)), - StorageEngine.getInstance() - .getProcessor(new PartialPath("root.isp.d1")) - .getIdTable() - .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s5")))); - assertEquals( - new TimeValuePair(113L, new TsBinary(new Binary("mm3"))), - StorageEngine.getInstance() - .getProcessor(new PartialPath("root.isp.d1")) - .getIdTable() - .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s6")))); } private void insertDataInMemory() throws IllegalPathException, QueryProcessException { @@ -218,49 +159,4 @@ public class QueryWithIDTableTest { PlanExecutor executor = new PlanExecutor(); executor.insertTablet(tabletPlan); } - - private void insertDataInDisk() - throws IllegalPathException, QueryProcessException, StorageGroupNotSetException, - StorageEngineException { - long[] times = new long[] {10L, 11L, 12L, 13L}; - List<Integer> dataTypes = new ArrayList<>(); - dataTypes.add(TSDataType.DOUBLE.ordinal()); - dataTypes.add(TSDataType.FLOAT.ordinal()); - dataTypes.add(TSDataType.INT64.ordinal()); - dataTypes.add(TSDataType.INT32.ordinal()); - dataTypes.add(TSDataType.BOOLEAN.ordinal()); - dataTypes.add(TSDataType.TEXT.ordinal()); - - Object[] columns = new Object[6]; - columns[0] = new double[4]; - columns[1] = new float[4]; - columns[2] = new long[4]; - columns[3] = new int[4]; - columns[4] = new boolean[4]; - columns[5] = new Binary[4]; - - for (int r = 0; r < 4; r++) { - ((double[]) columns[0])[r] = 1.0 + r; - ((float[]) columns[1])[r] = 2 + r; - ((long[]) columns[2])[r] = 10000 + r; - ((int[]) columns[3])[r] = 100 + r; - ((boolean[]) columns[4])[r] = false; - ((Binary[]) columns[5])[r] = new Binary("hh" + r); - } - - InsertTabletPlan tabletPlan = - new InsertTabletPlan( - new PartialPath("root.isp.d1"), - new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}, - dataTypes); - tabletPlan.setTimes(times); - tabletPlan.setColumns(columns); - tabletPlan.setRowCount(times.length); - - PlanExecutor executor = new PlanExecutor(); - executor.insertTablet(tabletPlan); - - PhysicalPlan flushPlan = processor.parseSQLToPhysicalPlan("flush"); - executor.processNonQuery(flushPlan); - } } diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/QueryWithIDTableTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/QueryWithIDTableHashmapImplTest.java similarity index 92% rename from server/src/test/java/org/apache/iotdb/db/metadata/id_table/QueryWithIDTableTest.java rename to server/src/test/java/org/apache/iotdb/db/metadata/id_table/QueryWithIDTableHashmapImplTest.java index 39baabf..6952d53 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/QueryWithIDTableTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/QueryWithIDTableHashmapImplTest.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.metadata.id_table; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.IllegalPathException; import org.apache.iotdb.db.exception.metadata.MetadataException; @@ -62,7 +61,7 @@ import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -public class QueryWithIDTableTest { +public class QueryWithIDTableHashmapImplTest { private final Planner processor = new Planner(); private boolean isEnableIDTable = false; @@ -143,39 +142,33 @@ public class QueryWithIDTableTest { // test it from id table assertEquals( new TimeValuePair(113L, new TsDouble(13.0d)), - StorageEngine.getInstance() - .getProcessor(new PartialPath("root.isp.d1")) - .getIdTable() + IDTableManager.getInstance() + .getIDTable(new PartialPath("root.isp.d1")) .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s1")))); assertEquals( new TimeValuePair(113L, new TsFloat(23.0f)), - StorageEngine.getInstance() - .getProcessor(new PartialPath("root.isp.d1")) - .getIdTable() + IDTableManager.getInstance() + .getIDTable(new PartialPath("root.isp.d1")) .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s2")))); assertEquals( new TimeValuePair(113L, new TsLong(100003L)), - StorageEngine.getInstance() - .getProcessor(new PartialPath("root.isp.d1")) - .getIdTable() + IDTableManager.getInstance() + .getIDTable(new PartialPath("root.isp.d1")) .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s3")))); assertEquals( new TimeValuePair(113L, new TsInt(1003)), - StorageEngine.getInstance() - .getProcessor(new PartialPath("root.isp.d1")) - .getIdTable() + IDTableManager.getInstance() + .getIDTable(new PartialPath("root.isp.d1")) .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s4")))); assertEquals( new TimeValuePair(113L, new TsBoolean(false)), - StorageEngine.getInstance() - .getProcessor(new PartialPath("root.isp.d1")) - .getIdTable() + IDTableManager.getInstance() + .getIDTable(new PartialPath("root.isp.d1")) .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s5")))); assertEquals( new TimeValuePair(113L, new TsBinary(new Binary("mm3"))), - StorageEngine.getInstance() - .getProcessor(new PartialPath("root.isp.d1")) - .getIdTable() + IDTableManager.getInstance() + .getIDTable(new PartialPath("root.isp.d1")) .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s6")))); }
