This is an automated email from the ASF dual-hosted git repository. xuekaifeng pushed a commit to branch xkf_idtable_recovery in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1cae8a31010bb4379a17d02a78b2c076f9b4178d Merge: b49bb01 24d4b97 Author: 151250176 <[email protected]> AuthorDate: Wed Jan 19 12:03:25 2022 +0800 merge and fix typo .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 6 +- docs/UserGuide/Maintenance-Tools/Metric-Tool.md | 53 +- docs/zh/SystemDesign/StorageEngine/MergeManager.md | 7 +- docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md | 53 +- .../apache/iotdb/tsfile/TsFileSequenceRead.java | 9 +- .../db/integration/IoTDBNewTsFileCompactionIT.java | 1 + .../integration/IoTDBSizeTieredCompactionIT.java | 7 +- .../aligned/IOTDBInsertAlignedValuesIT.java | 3 + .../iotdb/metrics/dropwizard/MetricName.java | 12 +- .../prometheus/DropwizardMetricsExporter.java | 3 +- .../main/assembly/resources/conf/iotdb-metric.yml | 6 +- .../metrics/config/MetricConfigDescriptor.java | 2 +- .../interface/src/test/resources/iotdb-metric.yml | 4 +- pom.xml | 2 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 34 +- .../org/apache/iotdb/db/conf/IoTDBConstant.java | 7 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 9 - .../org/apache/iotdb/db/engine/StorageEngine.java | 4 +- .../db/engine/cache/TimeSeriesMetadataCache.java | 18 +- .../db/engine/compaction/CompactionScheduler.java | 23 +- .../engine/compaction/CompactionTaskManager.java | 30 + .../db/engine/compaction/CompactionUtils.java | 388 +++ .../AbstractCrossSpaceCompactionSelector.java | 10 +- .../cross/AbstractCrossSpaceCompactionTask.java | 1 - .../compaction/cross/CrossCompactionStrategy.java | 45 +- .../CrossSpaceCompactionExceptionHandler.java | 256 ++ .../cross/CrossSpaceCompactionTaskFactory.java | 16 +- .../inplace/InplaceCompactionRecoverTask.java | 110 - .../cross/inplace/InplaceCompactionTask.java | 244 -- .../inplace/manage/CrossSpaceMergeContext.java | 88 - .../cross/inplace/manage/MergeFuture.java | 151 - .../cross/inplace/manage/MergeManager.java | 365 --- .../cross/inplace/manage/MergeManagerMBean.java | 26 - .../cross/inplace/manage/MergeThreadPool.java | 54 - .../recover/InplaceCompactionLogAnalyzer.java | 353 --- .../inplace/recover/InplaceCompactionLogger.java | 150 - .../task/CleanLastCrossSpaceCompactionTask.java | 67 - .../cross/inplace/task/CrossSpaceMergeTask.java | 259 -- .../cross/inplace/task/MergeCallback.java | 40 - .../cross/inplace/task/MergeFileTask.java | 405 --- .../cross/inplace/task/MergeMultiChunkTask.java | 709 ----- .../RewriteCrossSpaceCompactionSelector.java} | 25 +- .../manage/CrossSpaceMergeResource.java | 103 +- .../RewriteCrossSpaceCompactionLogAnalyzer.java | 118 + .../recover/RewriteCrossSpaceCompactionLogger.java | 86 + .../selector/ICrossSpaceMergeFileSelector.java | 2 +- .../selector/IFileQueryMemMeasurement.java | 2 +- .../selector/IMergePathSelector.java | 2 +- .../selector/MaxFileMergeFileSelector.java | 4 +- .../selector/MaxSeriesMergeFileSelector.java | 4 +- .../selector/MergeFileStrategy.java | 2 +- .../selector/NaivePathSelector.java | 2 +- .../task/RewriteCrossCompactionRecoverTask.java} | 180 +- .../task/RewriteCrossSpaceCompactionTask.java | 296 ++ .../AbstractInnerSpaceCompactionSelector.java | 7 +- .../compaction/inner/InnerCompactionStrategy.java | 11 +- .../InnerSpaceCompactionExceptionHandler.java | 14 +- .../inner/InnerSpaceCompactionTaskFactory.java | 1 - .../SizeTieredCompactionRecoverTask.java | 16 +- .../sizetiered/SizeTieredCompactionSelector.java | 3 - .../inner/sizetiered/SizeTieredCompactionTask.java | 31 +- .../utils/AlignedSeriesCompactionExecutor.java | 3 + .../inner/utils/InnerSpaceCompactionUtils.java | 22 +- .../inner/utils/MultiTsFileDeviceIterator.java | 58 +- .../utils/SingleSeriesCompactionExecutor.java | 11 +- .../compaction/task/AbstractCompactionTask.java | 4 +- .../compaction/task/CompactionRecoverTask.java | 30 +- .../writer/AbstractCompactionWriter.java | 157 + .../writer/CrossSpaceCompactionWriter.java | 182 ++ .../writer/InnerSpaceCompactionWriter.java | 86 + .../engine/storagegroup/TsFileNameGenerator.java | 84 +- .../db/engine/storagegroup/TsFileResource.java | 5 + .../storagegroup/VirtualStorageGroupProcessor.java | 119 +- .../virtualSg/StorageGroupManager.java | 4 +- .../org/apache/iotdb/db/metadata/MManager.java | 44 +- .../db/metadata/idtable/entry/DeviceEntry.java | 3 +- .../db/metadata/idtable/entry/DeviceIDFactory.java | 4 +- .../db/metadata/idtable/entry/SHA256DeviceID.java | 6 +- .../apache/iotdb/db/qp/executor/PlanExecutor.java | 60 +- .../db/qp/logical/sys/ShowMergeStatusOperator.java | 40 - .../db/qp/physical/sys/ShowMergeStatusPlan.java | 27 - .../apache/iotdb/db/qp/physical/sys/ShowPlan.java | 1 - .../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 8 - .../db/query/control/QueryResourceManager.java | 17 + .../java/org/apache/iotdb/db/service/IoTDB.java | 2 - .../db/sync/sender/manage/SyncFileManager.java | 4 +- .../java/org/apache/iotdb/db/utils/MergeUtils.java | 127 +- .../engine/compaction/AbstractCompactionTest.java | 292 ++ .../engine/compaction/CompactionSchedulerTest.java | 70 +- .../compaction/CompactionTaskManagerTest.java | 63 +- .../db/engine/compaction/CompactionUtilsTest.java | 3344 ++++++++++++++++++++ .../cross/CrossSpaceCompactionExceptionTest.java | 498 +++ .../compaction/cross/CrossSpaceCompactionTest.java | 411 ++- .../cross/MaxFileMergeFileSelectorTest.java | 6 +- .../cross/MaxSeriesMergeFileSelectorTest.java | 4 +- .../db/engine/compaction/cross/MergeLogTest.java | 92 - .../engine/compaction/cross/MergeManagerTest.java | 146 - .../engine/compaction/cross/MergeOverLapTest.java | 205 -- .../db/engine/compaction/cross/MergePerfTest.java | 108 - .../db/engine/compaction/cross/MergeTaskTest.java | 575 ---- .../db/engine/compaction/cross/MergeTest.java | 3 - .../engine/compaction/cross/MergeUpgradeTest.java | 4 +- .../RewriteCrossSpaceCompactionRecoverTest.java | 506 +++ .../cross/RewriteCrossSpaceCompactionTest.java | 334 ++ .../inner/AbstractInnerSpaceCompactionTest.java | 26 +- .../inner/InnerCompactionSchedulerTest.java | 87 +- .../compaction/inner/InnerCompactionTest.java | 2 +- .../compaction/inner/InnerSeqCompactionTest.java | 2 +- .../inner/InnerSpaceCompactionExceptionTest.java | 46 +- .../inner/InnerSpaceCompactionUtilsOldTest.java | 2 +- .../compaction/inner/InnerUnseqCompactionTest.java | 45 +- .../SizeTieredCompactionHandleExceptionTest.java | 43 +- .../SizeTieredCompactionRecoverTest.java | 58 +- .../recover/SizeTieredCompactionRecoverTest.java | 128 +- .../task/FakedCrossSpaceCompactionTask.java | 21 +- .../task/FakedCrossSpaceCompactionTaskFactory.java | 16 +- .../task/FakedInnerSpaceCompactionTask.java | 3 - .../task/FakedInnerSpaceCompactionTaskFactory.java | 1 - .../compaction/utils/CompactionClearUtils.java | 72 +- .../utils/CompactionFileGeneratorUtils.java | 65 +- .../engine/storagegroup/FakedTsFileResource.java | 1 + .../storagegroup/StorageGroupProcessorTest.java | 2 +- .../db/metadata/idtable/IDTableRecoverTest.java | 12 +- .../query/reader/series/SeriesReaderTestUtil.java | 2 - .../iotdb/tsfile/file/header/ChunkHeader.java | 4 +- .../file/metadata/AlignedTimeSeriesMetadata.java | 6 +- .../read/TsFileAlignedSeriesReaderIterator.java | 3 + .../iotdb/tsfile/read/TsFileSequenceReader.java | 2 +- .../iotdb/tsfile/utils/TsFileGeneratorUtils.java | 285 ++ .../tsfile/write/chunk/AlignedChunkWriterImpl.java | 17 + .../iotdb/tsfile/write/chunk/ValueChunkWriter.java | 17 + .../apache/iotdb/tsfile/read/TsFileReaderTest.java | 6 +- .../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 83 +- .../iotdb/tsfile/write/TsFileWriteApiTest.java | 46 +- 134 files changed, 8307 insertions(+), 5504 deletions(-) diff --cc server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceEntry.java index 7f26357,de52b8a..e5cbd90 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceEntry.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceEntry.java @@@ -19,10 -19,8 +19,11 @@@ package org.apache.iotdb.db.metadata.idtable.entry; ++import org.apache.iotdb.db.utils.TestOnly; ++ import java.util.HashMap; import java.util.Map; +import java.util.Objects; - import org.apache.iotdb.db.utils.TestOnly; /** device entry in id table */ public class DeviceEntry { diff --cc server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceIDFactory.java index c4a9676,c4a9676..6ebee8f --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceIDFactory.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceIDFactory.java @@@ -89,9 -89,9 +89,9 @@@ public class DeviceIDFactory .getConfig() .getDeviceIDTransformationMethod() .equals("SHA256")) { -- getDeviceIDFunction = partialPath -> new SHA256DeviceID(partialPath.toString()); ++ getDeviceIDFunction = SHA256DeviceID::new; } else { -- getDeviceIDFunction = partialPath -> new PlainDeviceID(partialPath.toString()); ++ getDeviceIDFunction = PlainDeviceID::new; } } } diff --cc server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SHA256DeviceID.java index 7cb477e,7cb477e..62479bb --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SHA256DeviceID.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SHA256DeviceID.java @@@ -34,7 -34,7 +34,7 @@@ public class SHA256DeviceID implements long l3; long l4; -- private static final String SEPERATOR = "#"; ++ private static final String SEPARATOR = "#"; /** using lots of message digest for improving parallelism */ private static MessageDigest[] md; @@@ -71,7 -71,7 +71,7 @@@ * @param deviceID a sha 256 string */ private void fromSHA256String(String deviceID) { -- String[] part = deviceID.split(SEPERATOR); ++ String[] part = deviceID.split(SEPARATOR); l1 = Long.parseLong(part[0]); l2 = Long.parseLong(part[1]); l3 = Long.parseLong(part[2]); @@@ -143,6 -143,6 +143,6 @@@ @Override public String toStringID() { -- return l1 + SEPERATOR + l2 + SEPERATOR + l3 + SEPERATOR + l4; ++ return l1 + SEPARATOR + l2 + SEPARATOR + l3 + SEPARATOR + l4; } } diff --cc server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRecoverTest.java index 379caeb,0000000..e73eafe mode 100644,000000..100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRecoverTest.java @@@ -1,209 -1,0 +1,211 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.metadata.idtable; + - import static org.junit.Assert.assertEquals; - - import java.io.IOException; - import java.util.ArrayList; - import java.util.List; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.StorageEngineException; +import org.apache.iotdb.db.exception.metadata.IllegalPathException; +import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; +import org.apache.iotdb.db.metadata.path.PartialPath; +import org.apache.iotdb.db.qp.Planner; +import org.apache.iotdb.db.qp.executor.PlanExecutor; +import org.apache.iotdb.db.qp.physical.PhysicalPlan; +import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; +import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan; +import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.Binary; ++ +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + ++import java.io.IOException; ++import java.util.ArrayList; ++import java.util.List; ++ ++import static org.junit.Assert.assertEquals; ++ +public class IDTableRecoverTest { + private final Planner processor = new Planner(); + + private boolean isEnableIDTable = false; + + private String originalDeviceIDTransformationMethod = null; + + private boolean isEnableIDTableLogFile = false; + + @Before + public void before() { + IoTDBDescriptor.getInstance().getConfig().setAutoCreateSchemaEnabled(true); + isEnableIDTable = IoTDBDescriptor.getInstance().getConfig().isEnableIDTable(); + originalDeviceIDTransformationMethod = + IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod(); + isEnableIDTableLogFile = IoTDBDescriptor.getInstance().getConfig().isEnableIDTableLogFile(); + + IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(true); + EnvironmentUtils.envSetUp(); + } + + @After + public void clean() throws IOException, StorageEngineException { + IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); + IoTDBDescriptor.getInstance() + .getConfig() + .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(isEnableIDTableLogFile); + + EnvironmentUtils.cleanEnv(); + } + + @Test + public void testRecover() throws Exception { + insertDataInMemoryWithTablet(false); + insertDataInMemoryWithRecord(false); + + PlanExecutor executor = new PlanExecutor(); + PhysicalPlan flushPlan = processor.parseSQLToPhysicalPlan("flush"); + executor.processNonQuery(flushPlan); + + IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp")); + List<DeviceEntry> memoryList = idTable.getAllDeviceEntry(); + + // restart + try { + EnvironmentUtils.restartDaemon(); + } catch (Exception e) { + Assert.fail(); + } + + // check id table fields + + idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp.d1")); + List<DeviceEntry> recoverList = idTable.getAllDeviceEntry(); + + assertEquals(memoryList, recoverList); + } + + @Test + public void testRecoverAligned() throws Exception { + insertDataInMemoryWithTablet(true); + insertDataInMemoryWithRecord(false); + + PlanExecutor executor = new PlanExecutor(); + PhysicalPlan flushPlan = processor.parseSQLToPhysicalPlan("flush"); + executor.processNonQuery(flushPlan); + + IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp")); + List<DeviceEntry> memoryList = idTable.getAllDeviceEntry(); + + // restart + try { + EnvironmentUtils.restartDaemon(); + } catch (Exception e) { + Assert.fail(); + } + + // check id table fields + + idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp.d1")); + List<DeviceEntry> recoverList = idTable.getAllDeviceEntry(); + + assertEquals(memoryList, recoverList); + } + + private void insertDataInMemoryWithRecord(boolean isAligned) + throws IllegalPathException, QueryProcessException { + long time = 100L; + TSDataType[] dataTypes = + new TSDataType[] { + TSDataType.DOUBLE, + TSDataType.FLOAT, + TSDataType.INT64, + TSDataType.INT32, + TSDataType.BOOLEAN, + TSDataType.TEXT + }; + + String[] columns = new String[6]; + columns[0] = 1.0 + ""; + columns[1] = 2 + ""; + columns[2] = 10000 + ""; + columns[3] = 100 + ""; + columns[4] = false + ""; + columns[5] = "hh" + 0; + + InsertRowPlan insertRowPlan = + new InsertRowPlan( + new PartialPath("root.isp.d1"), + time, + new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}, + dataTypes, + columns); + insertRowPlan.setAligned(isAligned); + + PlanExecutor executor = new PlanExecutor(); + executor.insert(insertRowPlan); + } + + private void insertDataInMemoryWithTablet(boolean isAligned) + throws IllegalPathException, QueryProcessException { + long[] times = new long[] {110L, 111L, 112L, 113L}; + List<Integer> dataTypes = new ArrayList<>(); + dataTypes.add(TSDataType.DOUBLE.ordinal()); + dataTypes.add(TSDataType.FLOAT.ordinal()); + dataTypes.add(TSDataType.INT64.ordinal()); + dataTypes.add(TSDataType.INT32.ordinal()); + dataTypes.add(TSDataType.BOOLEAN.ordinal()); + dataTypes.add(TSDataType.TEXT.ordinal()); + + Object[] columns = new Object[6]; + columns[0] = new double[4]; + columns[1] = new float[4]; + columns[2] = new long[4]; + columns[3] = new int[4]; + columns[4] = new boolean[4]; + columns[5] = new Binary[4]; + + for (int r = 0; r < 4; r++) { + ((double[]) columns[0])[r] = 10.0 + r; + ((float[]) columns[1])[r] = 20 + r; + ((long[]) columns[2])[r] = 100000 + r; + ((int[]) columns[3])[r] = 1000 + r; + ((boolean[]) columns[4])[r] = false; + ((Binary[]) columns[5])[r] = new Binary("mm" + r); + } + + InsertTabletPlan tabletPlan = + new InsertTabletPlan( + new PartialPath("root.isp.d2"), + new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}, + dataTypes); + tabletPlan.setTimes(times); + tabletPlan.setColumns(columns); + tabletPlan.setRowCount(times.length); + tabletPlan.setAligned(isAligned); + + PlanExecutor executor = new PlanExecutor(); + executor.insertTablet(tabletPlan); + } +}
