This is an automated email from the ASF dual-hosted git repository. spricoder pushed a commit to branch feature/metric-detail in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 32133de04e246b2c464bd86500ebce14c45a27ad Author: spricoder <[email protected]> AuthorDate: Tue Sep 5 16:56:42 2023 +0800 Add database into memtable and split quantity of pointsIn into database level --- .../schemaengine/metric/SchemaEngineMemMetric.java | 19 ++++++-- .../db/storageengine/dataregion/DataRegion.java | 3 +- .../dataregion/memtable/AbstractMemTable.java | 50 +++++++++++++++++++--- .../dataregion/memtable/IMemTable.java | 2 + .../dataregion/memtable/PrimitiveMemTable.java | 13 +++--- .../wal/recover/file/TsFilePlanRedoer.java | 5 ++- .../file/UnsealedTsFileRecoverPerformer.java | 5 ++- .../rescon/memory/MemTableManager.java | 6 +-- .../dataregion/memtable/MemTableFlushTaskTest.java | 2 +- .../dataregion/memtable/MemtableBenchmark.java | 3 +- .../dataregion/memtable/PrimitiveMemTableTest.java | 15 ++++--- .../wal/checkpoint/CheckpointManagerTest.java | 6 ++- .../dataregion/wal/io/CheckpointFileTest.java | 7 ++- .../dataregion/wal/node/WALEntryHandlerTest.java | 19 ++++---- .../dataregion/wal/node/WALNodeTest.java | 9 ++-- .../wal/recover/WALRecoverManagerTest.java | 15 ++++--- .../wal/recover/file/TsFilePlanRedoerTest.java | 16 +++---- .../file/UnsealedTsFileRecoverPerformerTest.java | 6 +-- .../wal/utils/WALInsertNodeCacheTest.java | 9 ++-- 19 files changed, 139 insertions(+), 71 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/metric/SchemaEngineMemMetric.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/metric/SchemaEngineMemMetric.java index 3f5c6ec7211..4c8704dda35 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/metric/SchemaEngineMemMetric.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/metric/SchemaEngineMemMetric.java @@ -58,14 +58,18 @@ public class SchemaEngineMemMetric implements ISchemaEngineMetric { engineStatistics, ISchemaEngineStatistics::getTotalSeriesNumber, Tag.NAME.toString(), - TIME_SERES_CNT); + TIME_SERES_CNT, + Tag.DATABASE.toString(), + "total"); metricService.createAutoGauge( Metric.QUANTITY.toString(), MetricLevel.IMPORTANT, engineStatistics, ISchemaEngineStatistics::getTemplateSeriesNumber, Tag.NAME.toString(), - TEMPLATE_SERIES_CNT); + TEMPLATE_SERIES_CNT, + Tag.DATABASE.toString(), + "total"); metricService.createAutoGauge( Metric.MEM.toString(), MetricLevel.IMPORTANT, @@ -112,12 +116,19 @@ public class SchemaEngineMemMetric implements ISchemaEngineMetric { @Override public void unbindFrom(AbstractMetricService metricService) { metricService.remove( - MetricType.AUTO_GAUGE, Metric.QUANTITY.toString(), Tag.NAME.toString(), TIME_SERES_CNT); + MetricType.AUTO_GAUGE, + Metric.QUANTITY.toString(), + Tag.NAME.toString(), + TIME_SERES_CNT, + Tag.DATABASE.toString(), + "total"); metricService.remove( MetricType.AUTO_GAUGE, Metric.QUANTITY.toString(), Tag.NAME.toString(), - TEMPLATE_SERIES_CNT); + TEMPLATE_SERIES_CNT, + Tag.DATABASE.toString(), + "total"); metricService.remove( MetricType.AUTO_GAUGE, Metric.MEM.toString(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 75609c20f93..e0e639773bc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -678,7 +678,8 @@ public class DataRegion implements IDataRegionForQuery { private WALRecoverListener recoverUnsealedTsFile( TsFileResource unsealedTsFile, DataRegionRecoveryContext context, boolean isSeq) { UnsealedTsFileRecoverPerformer recoverPerformer = - new UnsealedTsFileRecoverPerformer(unsealedTsFile, isSeq, context.recoverPerformers::add); + new UnsealedTsFileRecoverPerformer( + unsealedTsFile, isSeq, context.recoverPerformers::add, databaseName); // remember to close UnsealedTsFileRecoverPerformer return WALRecoverManager.getInstance().addRecoverPerformer(recoverPerformer); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java index 3eab70464fc..f7a5dbda063 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java @@ -65,7 +65,8 @@ public abstract class AbstractMemTable implements IMemTable { private final Map<IDeviceID, IWritableMemChunkGroup> memTableMap; /** - * The initial value is true because we want to calculate the text data size when recover memTable. + * The initial value is true because we want to calculate the text data size when recover + * memTable. */ protected boolean disableMemControl = true; @@ -95,13 +96,22 @@ public abstract class AbstractMemTable implements IMemTable { private final long createdTime = System.currentTimeMillis(); + private String database; + private static final String METRIC_POINT_IN = "pointsIn"; protected AbstractMemTable() { + this.database = null; this.memTableMap = new HashMap<>(); } - protected AbstractMemTable(Map<IDeviceID, IWritableMemChunkGroup> memTableMap) { + protected AbstractMemTable(String database) { + this.database = database; + this.memTableMap = new HashMap<>(); + } + + protected AbstractMemTable(String database, Map<IDeviceID, IWritableMemChunkGroup> memTableMap) { + this.database = database; this.memTableMap = memTableMap; } @@ -194,7 +204,9 @@ public abstract class AbstractMemTable implements IMemTable { Metric.QUANTITY.toString(), MetricLevel.CORE, Tag.NAME.toString(), - METRIC_POINT_IN); + METRIC_POINT_IN, + Tag.DATABASE.toString(), + database); } @Override @@ -233,7 +245,9 @@ public abstract class AbstractMemTable implements IMemTable { Metric.QUANTITY.toString(), MetricLevel.CORE, Tag.NAME.toString(), - METRIC_POINT_IN); + METRIC_POINT_IN, + Tag.DATABASE.toString(), + database); } @Override @@ -252,7 +266,9 @@ public abstract class AbstractMemTable implements IMemTable { Metric.QUANTITY.toString(), MetricLevel.CORE, Tag.NAME.toString(), - METRIC_POINT_IN); + METRIC_POINT_IN, + Tag.DATABASE.toString(), + database); } catch (RuntimeException e) { throw new WriteProcessException(e); } @@ -274,7 +290,9 @@ public abstract class AbstractMemTable implements IMemTable { Metric.QUANTITY.toString(), MetricLevel.CORE, Tag.NAME.toString(), - METRIC_POINT_IN); + METRIC_POINT_IN, + Tag.DATABASE.toString(), + database); } catch (RuntimeException e) { throw new WriteProcessException(e); } @@ -590,6 +608,12 @@ public abstract class AbstractMemTable implements IMemTable { buffer.putLong(totalPointsNumThreshold); buffer.putLong(maxPlanIndex); buffer.putLong(minPlanIndex); + if (database == null || database.isEmpty()) { + buffer.putInt(0); + } else { + buffer.putInt(database.length()); + buffer.put(database.getBytes()); + } buffer.putInt(memTableMap.size()); for (Map.Entry<IDeviceID, IWritableMemChunkGroup> entry : memTableMap.entrySet()) { @@ -609,6 +633,12 @@ public abstract class AbstractMemTable implements IMemTable { totalPointsNumThreshold = stream.readLong(); maxPlanIndex = stream.readLong(); minPlanIndex = stream.readLong(); + int databaseLength = stream.readInt(); + if (databaseLength != 0) { + byte[] bytes = new byte[databaseLength]; + stream.read(bytes); + database = new String(bytes); + } int memTableMapSize = stream.readInt(); for (int i = 0; i < memTableMapSize; ++i) { @@ -645,11 +675,17 @@ public abstract class AbstractMemTable implements IMemTable { if (isSignal) { memTable = new NotifyFlushMemTable(); } else { - PrimitiveMemTable primitiveMemTable = new PrimitiveMemTable(); + // database will be updated when deserialize + PrimitiveMemTable primitiveMemTable = new PrimitiveMemTable(null); primitiveMemTable.deserialize(stream); memTable = primitiveMemTable; } return memTable; } } + + @Override + public String getDatabase() { + return database; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java index b4b0204080d..7ed96e7a8cf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java @@ -169,4 +169,6 @@ public interface IMemTable extends WALEntryValue { void setFlushStatus(FlushStatus flushStatus); Map<String, Long> getMaxTime(); + + String getDatabase(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTable.java index d4bf9c18743..bf88c5de627 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTable.java @@ -24,21 +24,24 @@ import java.util.Map; public class PrimitiveMemTable extends AbstractMemTable { - public PrimitiveMemTable() {} + public PrimitiveMemTable(String database) { + super(database); + } - public PrimitiveMemTable(boolean enableMemControl) { + public PrimitiveMemTable(String database, boolean enableMemControl) { + super(database); this.disableMemControl = !enableMemControl; } - public PrimitiveMemTable(Map<IDeviceID, IWritableMemChunkGroup> memTableMap) { - super(memTableMap); + public PrimitiveMemTable(String database, Map<IDeviceID, IWritableMemChunkGroup> memTableMap) { + super(database, memTableMap); } @Override public IMemTable copy() { Map<IDeviceID, IWritableMemChunkGroup> newMap = new HashMap<>(getMemTableMap()); - return new PrimitiveMemTable(newMap); + return new PrimitiveMemTable(getDatabase(), newMap); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java index d7a292cf36c..2a384ed2517 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java @@ -45,11 +45,12 @@ public class TsFilePlanRedoer { private final boolean sequence; // store data when redoing logs - private IMemTable recoveryMemTable = new PrimitiveMemTable(); + private IMemTable recoveryMemTable; - public TsFilePlanRedoer(TsFileResource tsFileResource, boolean sequence) { + public TsFilePlanRedoer(TsFileResource tsFileResource, boolean sequence, String database) { this.tsFileResource = tsFileResource; this.sequence = sequence; + this.recoveryMemTable = new PrimitiveMemTable(database); } void redoDelete(DeleteDataNode deleteDataNode) throws IOException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java index 742e996fd03..f50062004f2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java @@ -73,11 +73,12 @@ public class UnsealedTsFileRecoverPerformer extends AbstractTsFileRecoverPerform public UnsealedTsFileRecoverPerformer( TsFileResource tsFileResource, boolean sequence, - Consumer<UnsealedTsFileRecoverPerformer> callbackAfterUnsealedTsFileRecovered) { + Consumer<UnsealedTsFileRecoverPerformer> callbackAfterUnsealedTsFileRecovered, + String database) { super(tsFileResource); this.sequence = sequence; this.callbackAfterUnsealedTsFileRecovered = callbackAfterUnsealedTsFileRecovered; - this.walRedoer = new TsFilePlanRedoer(tsFileResource, sequence); + this.walRedoer = new TsFilePlanRedoer(tsFileResource, sequence, database); this.recoverListener = new WALRecoverListener(tsFileResource.getTsFilePath()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/MemTableManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/MemTableManager.java index 0359aec79a1..5646a48326c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/MemTableManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/MemTableManager.java @@ -47,12 +47,12 @@ public class MemTableManager { throws WriteProcessException { if (CONFIG.isEnableMemControl()) { currentMemtableNumber++; - return new PrimitiveMemTable(CONFIG.isEnableMemControl()); + return new PrimitiveMemTable(storageGroup, CONFIG.isEnableMemControl()); } if (!reachMaxMemtableNumber()) { currentMemtableNumber++; - return new PrimitiveMemTable(); + return new PrimitiveMemTable(storageGroup); } // wait until the total number of memtable is less than the system capacity @@ -60,7 +60,7 @@ public class MemTableManager { while (true) { if (!reachMaxMemtableNumber()) { currentMemtableNumber++; - return new PrimitiveMemTable(); + return new PrimitiveMemTable(storageGroup); } try { wait(WAIT_TIME); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java index 907ff94319d..dd758ef2683 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java @@ -52,7 +52,7 @@ public class MemTableFlushTaskTest { public void setUp() throws Exception { EnvironmentUtils.envSetUp(); writer = new RestorableTsFileIOWriter(FSFactoryProducer.getFSFactory().getFile(filePath)); - memTable = new PrimitiveMemTable(); + memTable = new PrimitiveMemTable(storageGroup); } @After diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemtableBenchmark.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemtableBenchmark.java index ca697b3257c..18f2c470a58 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemtableBenchmark.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemtableBenchmark.java @@ -29,6 +29,7 @@ import java.util.Collections; /** Memtable insert benchmark. Bench the Memtable and get its performance. */ public class MemtableBenchmark { + private static String database = "root.test"; private static String deviceId = "d0"; private static int numOfMeasurement = 10000; private static int numOfPoint = 1000; @@ -43,7 +44,7 @@ public class MemtableBenchmark { } public static void main(String[] args) throws IllegalPathException { - IMemTable memTable = new PrimitiveMemTable(); + IMemTable memTable = new PrimitiveMemTable(database); final long startTime = System.currentTimeMillis(); // cpu not locality for (int i = 0; i < numOfPoint; i++) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java index 86d55cc771c..99734bcc023 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java @@ -60,6 +60,7 @@ import static org.junit.Assert.assertEquals; public class PrimitiveMemTableTest { + String database = "root.test"; double delta; @Before @@ -114,7 +115,7 @@ public class PrimitiveMemTableTest { @Test public void simpleTest() throws IOException, QueryProcessException, MetadataException { - IMemTable memTable = new PrimitiveMemTable(); + IMemTable memTable = new PrimitiveMemTable(database); int count = 10; String deviceId = "d1"; String[] measurementId = new String[count]; @@ -161,7 +162,7 @@ public class PrimitiveMemTableTest { @Test public void totalSeriesNumberTest() throws IOException, QueryProcessException, MetadataException { - IMemTable memTable = new PrimitiveMemTable(); + IMemTable memTable = new PrimitiveMemTable(database); int count = 10; String deviceId = "d1"; String[] measurementId = new String[count]; @@ -211,7 +212,7 @@ public class PrimitiveMemTableTest { @Test public void queryWithDeletionTest() throws IOException, QueryProcessException, MetadataException { - IMemTable memTable = new PrimitiveMemTable(); + IMemTable memTable = new PrimitiveMemTable(database); int count = 10; String deviceId = "d1"; String[] measurementId = new String[count]; @@ -265,7 +266,7 @@ public class PrimitiveMemTableTest { @Test public void queryAlignChuckWithDeletionTest() throws IOException, QueryProcessException, MetadataException { - IMemTable memTable = new PrimitiveMemTable(); + IMemTable memTable = new PrimitiveMemTable(database); int count = 10; String deviceId = "d1"; String[] measurementId = new String[count]; @@ -426,7 +427,7 @@ public class PrimitiveMemTableTest { @Test public void testFloatType() throws IOException, QueryProcessException, MetadataException { - IMemTable memTable = new PrimitiveMemTable(); + IMemTable memTable = new PrimitiveMemTable(database); String deviceId = "d1"; int size = 100; write(memTable, deviceId, "s1", TSDataType.FLOAT, TSEncoding.RLE, size); @@ -435,7 +436,7 @@ public class PrimitiveMemTableTest { @Test public void testAllType() throws IOException, QueryProcessException, MetadataException, WriteProcessException { - IMemTable memTable = new PrimitiveMemTable(); + IMemTable memTable = new PrimitiveMemTable(database); int count = 10; String deviceId = "d1"; String[] measurementId = new String[count]; @@ -539,7 +540,7 @@ public class PrimitiveMemTableTest { @Test public void testSerializeSize() throws IOException, QueryProcessException, MetadataException, WriteProcessException { - IMemTable memTable = new PrimitiveMemTable(); + IMemTable memTable = new PrimitiveMemTable(database); int count = 10; String deviceId = "d1"; String[] measurementId = new String[count]; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManagerTest.java index a5039fd4d07..6b34810e7c5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManagerTest.java @@ -48,6 +48,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class CheckpointManagerTest { + private static final String database = "root.test"; private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); private static final String identifier = String.valueOf(Integer.MAX_VALUE); private static final String logDirectory = TestConstant.BASE_OUTPUT_PATH.concat("wal-test"); @@ -97,7 +98,7 @@ public class CheckpointManagerTest { () -> { String tsFilePath = logDirectory + File.separator + versionId + ".tsfile"; MemTableInfo memTableInfo = - new MemTableInfo(new PrimitiveMemTable(), tsFilePath, versionId); + new MemTableInfo(new PrimitiveMemTable(database), tsFilePath, versionId); versionId2memTableId.put(versionId, memTableInfo.getMemTableId()); checkpointManager.makeCreateMemTableCP(memTableInfo); if (versionId < memTablesNum / 2) { @@ -133,7 +134,8 @@ public class CheckpointManagerTest { while (size < config.getCheckpointFileSizeThresholdInByte()) { ++versionId; String tsFilePath = logDirectory + File.separator + versionId + ".tsfile"; - MemTableInfo memTableInfo = new MemTableInfo(new PrimitiveMemTable(), tsFilePath, versionId); + MemTableInfo memTableInfo = + new MemTableInfo(new PrimitiveMemTable(database), tsFilePath, versionId); versionId2memTableId.put(versionId, memTableInfo.getMemTableId()); Checkpoint checkpoint = new Checkpoint( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointFileTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointFileTest.java index 78e898c08c2..b3a28a2e52d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointFileTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointFileTest.java @@ -41,6 +41,7 @@ import static org.junit.Assert.assertEquals; public class CheckpointFileTest { private final File checkpointFile = new File(TestConstant.BASE_OUTPUT_PATH.concat("_0.checkpoint")); + private static final String database = "root.test"; @Before public void setUp() throws Exception { @@ -58,7 +59,8 @@ public class CheckpointFileTest { @Test public void testReadNormalFile() throws IOException { - MemTableInfo fakeMemTableInfo = new MemTableInfo(new PrimitiveMemTable(), "fake.tsfile", 0); + MemTableInfo fakeMemTableInfo = + new MemTableInfo(new PrimitiveMemTable(database), "fake.tsfile", 0); List<Checkpoint> expectedCheckpoints = new ArrayList<>(); expectedCheckpoints.add( new Checkpoint(CheckpointType.GLOBAL_MEMORY_TABLE_INFO, Collections.emptyList())); @@ -101,7 +103,8 @@ public class CheckpointFileTest { @Test public void testReadBrokenFile() throws IOException { - MemTableInfo fakeMemTableInfo = new MemTableInfo(new PrimitiveMemTable(), "fake.tsfile", 0); + MemTableInfo fakeMemTableInfo = + new MemTableInfo(new PrimitiveMemTable(database), "fake.tsfile", 0); List<Checkpoint> expectedCheckpoints = new ArrayList<>(); expectedCheckpoints.add( new Checkpoint(CheckpointType.GLOBAL_MEMORY_TABLE_INFO, Collections.emptyList())); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALEntryHandlerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALEntryHandlerTest.java index 39dda5f68c4..55835c7697e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALEntryHandlerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALEntryHandlerTest.java @@ -67,7 +67,8 @@ public class WALEntryHandlerTest { private static final String logDirectory2 = TestConstant.BASE_OUTPUT_PATH.concat("wal-test" + identifier2); - private static final String devicePath = "root.test_sg.test_d"; + private static final String databasePath = "root.test_sg"; + private static final String devicePath = databasePath + ".test_d"; private WALMode prevMode; private boolean prevIsClusterMode; private WALNode walNode1; @@ -98,7 +99,7 @@ public class WALEntryHandlerTest { @Test(expected = MemTablePinException.class) public void pinDeletedMemTable() throws Exception { - IMemTable memTable = new PrimitiveMemTable(); + IMemTable memTable = new PrimitiveMemTable(databasePath); walNode1.onMemTableCreated(memTable, logDirectory1 + "/" + "fake.tsfile"); WALFlushListener flushListener = walNode1.log( @@ -111,7 +112,7 @@ public class WALEntryHandlerTest { @Test public void pinMemTable() throws Exception { - IMemTable memTable = new PrimitiveMemTable(); + IMemTable memTable = new PrimitiveMemTable(databasePath); walNode1.onMemTableCreated(memTable, logDirectory1 + "/" + "fake.tsfile"); InsertRowNode node1 = getInsertRowNode(devicePath, System.currentTimeMillis()); node1.setSearchIndex(1); @@ -141,7 +142,7 @@ public class WALEntryHandlerTest { @Test(expected = MemTablePinException.class) public void unpinDeletedMemTable() throws Exception { - IMemTable memTable = new PrimitiveMemTable(); + IMemTable memTable = new PrimitiveMemTable(databasePath); walNode1.onMemTableCreated(memTable, logDirectory1 + "/" + "fake.tsfile"); WALFlushListener flushListener = walNode1.log( @@ -154,7 +155,7 @@ public class WALEntryHandlerTest { @Test public void unpinFlushedMemTable() throws Exception { - IMemTable memTable = new PrimitiveMemTable(); + IMemTable memTable = new PrimitiveMemTable(databasePath); walNode1.onMemTableCreated(memTable, logDirectory1 + "/" + "fake.tsfile"); WALFlushListener flushListener = walNode1.log( @@ -178,7 +179,7 @@ public class WALEntryHandlerTest { @Test public void unpinMemTable() throws Exception { - IMemTable memTable = new PrimitiveMemTable(); + IMemTable memTable = new PrimitiveMemTable(databasePath); walNode1.onMemTableCreated(memTable, logDirectory1 + "/" + "fake.tsfile"); InsertRowNode node1 = getInsertRowNode(devicePath, System.currentTimeMillis()); node1.setSearchIndex(1); @@ -207,7 +208,7 @@ public class WALEntryHandlerTest { @Test public void getUnFlushedValue() throws Exception { - IMemTable memTable = new PrimitiveMemTable(); + IMemTable memTable = new PrimitiveMemTable(databasePath); walNode1.onMemTableCreated(memTable, logDirectory1 + "/" + "fake.tsfile"); InsertRowNode node1 = getInsertRowNode(devicePath, System.currentTimeMillis()); node1.setSearchIndex(1); @@ -221,7 +222,7 @@ public class WALEntryHandlerTest { @Test public void getFlushedValue() throws Exception { - IMemTable memTable = new PrimitiveMemTable(); + IMemTable memTable = new PrimitiveMemTable(databasePath); walNode1.onMemTableCreated(memTable, logDirectory1 + "/" + "fake.tsfile"); InsertRowNode node1 = getInsertRowNode(devicePath, System.currentTimeMillis()); node1.setSearchIndex(1); @@ -247,7 +248,7 @@ public class WALEntryHandlerTest { String logDirectory = i % 2 == 0 ? logDirectory1 : logDirectory2; Callable<Void> writeTask = () -> { - IMemTable memTable = new PrimitiveMemTable(); + IMemTable memTable = new PrimitiveMemTable(databasePath); walNode.onMemTableCreated(memTable, logDirectory + "/" + "fake.tsfile"); List<WALFlushListener> walFlushListeners = new ArrayList<>(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNodeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNodeTest.java index 4d6d30883fe..0f19d3b48f8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNodeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNodeTest.java @@ -68,7 +68,8 @@ public class WALNodeTest { private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); private static final String identifier = String.valueOf(Integer.MAX_VALUE); private static final String logDirectory = TestConstant.BASE_OUTPUT_PATH.concat("wal-test"); - private static final String devicePath = "root.test_sg.test_d"; + private static final String databasePath = "root.test_sg"; + private static final String devicePath = databasePath + ".test_d"; private WALMode prevMode; private boolean prevIsClusterMode; private WALNode walNode; @@ -226,7 +227,7 @@ public class WALNodeTest { for (int i = 0; i < memTablesNum; ++i) { Callable<Void> writeTask = () -> { - IMemTable memTable = new PrimitiveMemTable(); + IMemTable memTable = new PrimitiveMemTable(databasePath); long memTableId = memTable.getMemTableId(); String tsFilePath = logDirectory + File.separator + memTableId + ".tsfile"; long firstFileVersionId = walNode.getCurrentLogVersion(); @@ -260,7 +261,7 @@ public class WALNodeTest { List<WALFlushListener> walFlushListeners = new ArrayList<>(); // write until log is rolled long time = 0; - IMemTable memTable = new PrimitiveMemTable(); + IMemTable memTable = new PrimitiveMemTable(databasePath); long memTableId = memTable.getMemTableId(); String tsFilePath = logDirectory + File.separator + memTableId + ".tsfile"; walNode.onMemTableCreated(memTable, tsFilePath); @@ -273,7 +274,7 @@ public class WALNodeTest { walFlushListeners.add(walFlushListener); } walNode.onMemTableFlushed(memTable); - walNode.onMemTableCreated(new PrimitiveMemTable(), tsFilePath); + walNode.onMemTableCreated(new PrimitiveMemTable(databasePath), tsFilePath); // check existence of _0-0-0.wal file and _1-0-1.wal file assertTrue( new File( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java index 1a0d3a07d8c..500fb2df1da 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java @@ -151,7 +151,7 @@ public class WALRecoverManagerTest { List<Future<Void>> futures = new ArrayList<>(); long firstWALVersionId = walBuffer.getCurrentWALFileVersion(); for (int i = 0; i < threadsNum; ++i) { - IMemTable fakeMemTable = new PrimitiveMemTable(); + IMemTable fakeMemTable = new PrimitiveMemTable(SG_NAME); long memTableId = fakeMemTable.getMemTableId(); Callable<Void> writeTask = () -> { @@ -185,7 +185,7 @@ public class WALRecoverManagerTest { Thread.sleep(1_000); // write normal .wal files long firstValidVersionId = walBuffer.getCurrentWALFileVersion(); - IMemTable targetMemTable = new PrimitiveMemTable(); + IMemTable targetMemTable = new PrimitiveMemTable(SG_NAME); WALEntry walEntry = new WALInfoEntry(targetMemTable.getMemTableId(), getInsertRowNode(DEVICE2_NAME, 4L), true); walBuffer.write(walEntry); @@ -210,7 +210,7 @@ public class WALRecoverManagerTest { List<Future<Void>> futures = new ArrayList<>(); long firstWALVersionId = walBuffer.getCurrentWALFileVersion(); for (int i = 0; i < threadsNum; ++i) { - IMemTable fakeMemTable = new PrimitiveMemTable(); + IMemTable fakeMemTable = new PrimitiveMemTable(SG_NAME); long memTableId = fakeMemTable.getMemTableId(); Callable<Void> writeTask = () -> { @@ -244,7 +244,7 @@ public class WALRecoverManagerTest { Thread.sleep(1_000); // write normal .wal files long firstValidVersionId = walBuffer.getCurrentWALFileVersion(); - IMemTable targetMemTable = new PrimitiveMemTable(); + IMemTable targetMemTable = new PrimitiveMemTable(SG_NAME); InsertRowNode insertRowNode = getInsertRowNode(DEVICE2_NAME, 4L); targetMemTable.insert(insertRowNode); @@ -415,7 +415,7 @@ public class WALRecoverManagerTest { tsFileWithWALResource = new TsFileResource(fileWithWAL); UnsealedTsFileRecoverPerformer recoverPerformer = new UnsealedTsFileRecoverPerformer( - tsFileWithWALResource, true, performer -> assertFalse(performer.canWrite())); + tsFileWithWALResource, true, performer -> assertFalse(performer.canWrite()), SG_NAME); recoverManager.addRecoverPerformer(recoverPerformer); recoverListeners.add(recoverPerformer.getRecoverListener()); @@ -425,7 +425,10 @@ public class WALRecoverManagerTest { tsFileWithoutWALResource = new TsFileResource(fileWithoutWAL); recoverPerformer = new UnsealedTsFileRecoverPerformer( - tsFileWithoutWALResource, true, performer -> assertFalse(performer.canWrite())); + tsFileWithoutWALResource, + true, + performer -> assertFalse(performer.canWrite()), + SG_NAME); recoverManager.addRecoverPerformer(recoverPerformer); recoverListeners.add(recoverPerformer.getRecoverListener()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoerTest.java index 5c440b16165..2c18ebf9902 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoerTest.java @@ -145,7 +145,7 @@ public class TsFilePlanRedoerTest { }); // redo InsertTabletPlan, vsg processor is used to test IdTable, don't test IdTable here - TsFilePlanRedoer planRedoer = new TsFilePlanRedoer(tsFileResource, true); + TsFilePlanRedoer planRedoer = new TsFilePlanRedoer(tsFileResource, true, SG_NAME); planRedoer.redoInsert(insertRowNode); // check data in memTable @@ -217,7 +217,7 @@ public class TsFilePlanRedoerTest { }); // redo InsertTabletPlan, vsg processor is used to test IdTable, don't test IdTable here - TsFilePlanRedoer planRedoer = new TsFilePlanRedoer(tsFileResource, true); + TsFilePlanRedoer planRedoer = new TsFilePlanRedoer(tsFileResource, true, SG_NAME); planRedoer.redoInsert(insertRowNode); // check data in memTable @@ -303,7 +303,7 @@ public class TsFilePlanRedoerTest { }); // redo InsertTabletPlan, vsg processor is used to test IdTable, don't test IdTable here - TsFilePlanRedoer planRedoer = new TsFilePlanRedoer(tsFileResource, true); + TsFilePlanRedoer planRedoer = new TsFilePlanRedoer(tsFileResource, true, SG_NAME); planRedoer.redoInsert(insertTabletNode); // check data in memTable @@ -408,7 +408,7 @@ public class TsFilePlanRedoerTest { }); // redo InsertTabletPlan, vsg processor is used to test IdTable, don't test IdTable here - TsFilePlanRedoer planRedoer = new TsFilePlanRedoer(tsFileResource, true); + TsFilePlanRedoer planRedoer = new TsFilePlanRedoer(tsFileResource, true, SG_NAME); planRedoer.redoInsert(insertTabletNode); // check data in memTable @@ -485,7 +485,7 @@ public class TsFilePlanRedoerTest { times.length); // redo InsertTabletPlan, vsg processor is used to test IdTable, don't test IdTable here - TsFilePlanRedoer planRedoer = new TsFilePlanRedoer(tsFileResource, true); + TsFilePlanRedoer planRedoer = new TsFilePlanRedoer(tsFileResource, true, SG_NAME); planRedoer.redoInsert(insertTabletNode); // check data in memTable @@ -536,7 +536,7 @@ public class TsFilePlanRedoerTest { times.length); // redo InsertTabletPlan, vsg processor is used to test IdTable, don't test IdTable here - TsFilePlanRedoer planRedoer = new TsFilePlanRedoer(tsFileResource, false); + TsFilePlanRedoer planRedoer = new TsFilePlanRedoer(tsFileResource, false, SG_NAME); planRedoer.redoInsert(insertTabletNode); // check data in memTable @@ -593,7 +593,7 @@ public class TsFilePlanRedoerTest { // redo DeleteDataNode, vsg processor is used to test IdTable, don't test IdTable here File modsFile = new File(FILE_NAME.concat(ModificationFile.FILE_SUFFIX)); assertFalse(modsFile.exists()); - TsFilePlanRedoer planRedoer = new TsFilePlanRedoer(tsFileResource, false); + TsFilePlanRedoer planRedoer = new TsFilePlanRedoer(tsFileResource, false, SG_NAME); planRedoer.redoDelete(deleteDataNode); assertTrue(modsFile.exists()); } @@ -666,7 +666,7 @@ public class TsFilePlanRedoerTest { columns, times.length); // redo InsertTabletPlan, data region is used to test IdTable, don't test IdTable here - TsFilePlanRedoer planRedoer = new TsFilePlanRedoer(tsFileResource, true); + TsFilePlanRedoer planRedoer = new TsFilePlanRedoer(tsFileResource, true, SG_NAME); insertTabletNode.setMeasurementSchemas(schemas); planRedoer.redoInsert(insertTabletNode); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java index f96f79955f4..e87c031f60b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java @@ -126,7 +126,7 @@ public class UnsealedTsFileRecoverPerformerTest { // vsg processor is used to test IdTable, don't test IdTable here try (UnsealedTsFileRecoverPerformer recoverPerformer = new UnsealedTsFileRecoverPerformer( - tsFileResource, true, performer -> assertFalse(performer.canWrite()))) { + tsFileResource, true, performer -> assertFalse(performer.canWrite()), SG_NAME)) { recoverPerformer.startRecovery(); assertTrue(recoverPerformer.hasCrashed()); assertTrue(recoverPerformer.canWrite()); @@ -185,7 +185,7 @@ public class UnsealedTsFileRecoverPerformerTest { // vsg processor is used to test IdTable, don't test IdTable here try (UnsealedTsFileRecoverPerformer recoverPerformer = new UnsealedTsFileRecoverPerformer( - tsFileResource, true, performer -> assertFalse(performer.canWrite()))) { + tsFileResource, true, performer -> assertFalse(performer.canWrite()), SG_NAME)) { recoverPerformer.startRecovery(); assertTrue(recoverPerformer.hasCrashed()); assertTrue(recoverPerformer.canWrite()); @@ -312,7 +312,7 @@ public class UnsealedTsFileRecoverPerformerTest { // vsg processor is used to test IdTable, don't test IdTable here try (UnsealedTsFileRecoverPerformer recoverPerformer = new UnsealedTsFileRecoverPerformer( - tsFileResource, true, performer -> assertFalse(performer.canWrite()))) { + tsFileResource, true, performer -> assertFalse(performer.canWrite()), SG_NAME)) { recoverPerformer.startRecovery(); assertTrue(recoverPerformer.hasCrashed()); assertTrue(recoverPerformer.canWrite()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCacheTest.java index 1518ffc4dcf..cd277191691 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCacheTest.java @@ -46,7 +46,8 @@ public class WALInsertNodeCacheTest { private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); private static final String identifier = String.valueOf(Integer.MAX_VALUE); private static final String logDirectory = TestConstant.BASE_OUTPUT_PATH.concat("wal-test"); - private static final String devicePath = "root.test_sg.test_d"; + private static final String databasePath = "root.test_sg"; + private static final String devicePath = databasePath + ".test_d"; private static final WALInsertNodeCache cache = WALInsertNodeCache.getInstance(); private WALMode prevMode; private boolean prevIsClusterMode; @@ -74,7 +75,7 @@ public class WALInsertNodeCacheTest { @Test public void testLoadUnsealedWALFile() throws Exception { - IMemTable memTable = new PrimitiveMemTable(); + IMemTable memTable = new PrimitiveMemTable(databasePath); walNode.onMemTableCreated(memTable, logDirectory + "/" + "fake.tsfile"); InsertRowNode node1 = getInsertRowNode(devicePath, System.currentTimeMillis()); node1.setSearchIndex(1); @@ -91,7 +92,7 @@ public class WALInsertNodeCacheTest { @Test public void testBatchLoad() throws Exception { // write memTable1 - IMemTable memTable1 = new PrimitiveMemTable(); + IMemTable memTable1 = new PrimitiveMemTable(databasePath); walNode.onMemTableCreated(memTable1, logDirectory + "/" + "fake1.tsfile"); InsertRowNode node1 = getInsertRowNode(devicePath, System.currentTimeMillis()); node1.setSearchIndex(1); @@ -102,7 +103,7 @@ public class WALInsertNodeCacheTest { WALFlushListener flushListener2 = walNode.log(memTable1.getMemTableId(), node2); WALEntryPosition position2 = flushListener2.getWalEntryHandler().getWalEntryPosition(); // write memTable2 - IMemTable memTable2 = new PrimitiveMemTable(); + IMemTable memTable2 = new PrimitiveMemTable(databasePath); walNode.onMemTableCreated(memTable2, logDirectory + "/" + "fake2.tsfile"); InsertRowNode node3 = getInsertRowNode(devicePath, System.currentTimeMillis()); node1.setSearchIndex(3);
