This is an automated email from the ASF dual-hosted git repository. jt2594838 pushed a commit to branch add_flat_device in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e3963bf2803e6adac2c71f014b98032e6f76be20 Author: Tian Jiang <[email protected]> AuthorDate: Thu Oct 15 09:43:16 2026 +0800 tmp --- .../relational/it/db/it/IoTDBLoadTsFileIT.java | 36 +++--- .../it/insertquery/IoTDBFlatDeviceIT.java | 129 +++++++++++++-------- .../iotdb/session/SessionConnectionTest.java | 10 ++ .../pipe/event/TsFileInsertionEventParserTest.java | 9 ++ .../table/FlatDeviceTableParserDeletionTest.java | 12 ++ .../sink/util/builder/PipeTsFileBuilderV2Test.java | 89 +++++++++++++- .../fragment/FragmentInstanceExecutionTest.java | 3 + .../source/FlatDevicePageReaderQueueTest.java | 6 + .../queryengine/plan/analyze/AnalyzeUtilsTest.java | 4 + .../plan/analyze/load/LoadTsFileAnalyzerTest.java | 4 + .../planner/node/write/InsertRowNodeSerdeTest.java | 2 + .../node/write/InsertRowsNodeSerdeTest.java | 4 + .../node/write/InsertTabletNodeSerdeTest.java | 2 + .../planner/node/DeviceTableScanNodeTest.java | 2 + .../plan/statement/InsertStatementTest.java | 3 + .../utils/writer/AbstractCompactionWriterTest.java | 5 + .../repair/RepairDataFileScanUtilTest.java | 4 + .../tablemodel/FlatDeviceCompactionTest.java | 41 +++++++ .../memtable/AlignedTVListIteratorTest.java | 2 + .../memtable/FlatDeviceMemTableTest.java | 3 + .../memtable/MemChunkDeserializeTest.java | 2 + .../dataregion/memtable/TsFileProcessorTest.java | 5 + .../modification/TableDeletionEntryTest.java | 9 ++ .../reader/chunk/AlignedChunkPageReaderTest.java | 16 +++ .../reader/chunk/MemAlignedChunkLoaderTest.java | 4 + .../common/AlignedPriorityMergeReaderTest.java | 2 + .../reader/common/FlatDevicePointReaderTest.java | 14 +++ .../tsfile/FlatDeviceAlignedChunkWriterTest.java | 5 + .../wal/recover/file/FlatDeviceRecoveryTest.java | 51 ++++++++ .../AlignedSparseWritePerformanceTest.java | 3 + .../db/utils/datastructure/AlignedTVListTest.java | 58 +++++++++ .../datastructure/FlatDeviceAlignedTVListTest.java | 63 ++++++++++ .../utils/datastructure/FlatDeviceEncoderTest.java | 21 ++++ .../schema/table/TsFileTableSchemaUtilTest.java | 13 +++ .../schema/table/column/TagColumnSchemaTest.java | 2 + 35 files changed, 567 insertions(+), 71 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java index 36e1b062cbb..b4b619c3afe 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java @@ -19,6 +19,8 @@ package org.apache.iotdb.relational.it.db.it; +import org.apache.iotdb.commons.schema.table.TsFileTableSchemaUtil; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.FlatDeviceAlignedChunkWriter; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.it.utils.TsFileTableGenerator; @@ -27,6 +29,7 @@ import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; @@ -35,7 +38,6 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.TsFileWriter; -import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -496,6 +498,8 @@ public class IoTDBLoadTsFileIT { return tableCreation; } + // LOAD a table file containing twenty timestamped rows with both FIELDs null. + // count(*) must retain every row while FIELD counts stay zero and projected values stay null. @Test public void testLoadWithAllFieldsNullRows() throws Exception { final List<IMeasurementSchema> schemas = @@ -566,10 +570,10 @@ public class IoTDBLoadTsFileIT { } } + // LOAD an ordinary table file into a flat_device table, including three devices at time 1. + // Tablet conversion must preserve both TAG columns and all four rows with their FIELD values. @Test public void testLoadIntoFlatDeviceTablePreservesTagColumns() throws Exception { - // Regression coverage: flat_device LOAD converts the source TsFile to Tablet batches, so TAG - // values must survive conversion together with rows that share the same timestamp. final String database = "flat_device_load"; final String table = "sensor"; final List<IMeasurementSchema> schemas = @@ -672,25 +676,23 @@ public class IoTDBLoadTsFileIT { table, schemas, Arrays.asList(ColumnCategory.TAG, ColumnCategory.TAG, ColumnCategory.FIELD))); - // Encode the on-disk contract directly so this test detects incompatible writer changes. - writer.addTsFileProperty("flat_device." + table, new byte[] {1}); + TsFileTableSchemaUtil.addFlatDeviceProperty(writer, table, true); writer.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create(new String[] {table})); - final AlignedChunkWriterImpl chunks = new AlignedChunkWriterImpl(schemas); - for (int row = 0; row < tags.length; row++) { + final FlatDeviceAlignedChunkWriter chunks = + new FlatDeviceAlignedChunkWriter(schemas, EncryptUtils.getEncryptParameter()); + // Physical flat rows are ordered by TAGs (null before empty) and then time. Keep the + // original timestamps so LOAD must also handle time regression between logical devices. + for (int row : new int[] {2, 3, 0, 1}) { final long time = row + 1; - chunks.getTimeChunkWriter().write(time); for (int tag = 0; tag < 2; tag++) { final String value = tags[row][tag]; - chunks - .getValueChunkWriterByIndex(tag) - .write( - time, - value == null - ? Binary.EMPTY_VALUE - : new Binary(value, TSFileConfig.STRING_CHARSET), - value == null); + chunks.write( + time, + value == null ? Binary.EMPTY_VALUE : new Binary(value, TSFileConfig.STRING_CHARSET), + value == null); } - chunks.getValueChunkWriterByIndex(2).write(time, time * 10, false); + chunks.write(time, time * 10, false); + chunks.write(time); } chunks.writeToFileWriter(writer); writer.endChunkGroup(); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBFlatDeviceIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBFlatDeviceIT.java index 170117f2101..ae85f25c780 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBFlatDeviceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBFlatDeviceIT.java @@ -18,13 +18,7 @@ package org.apache.iotdb.relational.it.insertquery; -import java.util.Random; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicLong; +import org.apache.iotdb.commons.schema.table.TsFileTableSchemaUtil; import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.env.EnvFactory; @@ -63,6 +57,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.Random; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -199,6 +200,9 @@ public class IoTDBFlatDeviceIT { } } + // Attempt to change flat_device on flat and ordinary tables, including setting the same value + // or DEFAULT. Every ALTER must be rejected and the existing flat layout property must remain + // true. @Test public void testFlatDevicePropertyCannotBeAltered() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -281,8 +285,8 @@ public class IoTDBFlatDeviceIT { } } - // Covers Tablet writes whose rows span multiple logical devices and verifies the shared physical - // group preserves time-plus-TAG ordering across flush. + // Write Tablets spanning multiple logical devices with shared timestamps and null TAGs. + // Flush must preserve every compound TAG/time row key in one physical table group. @Test public void testFlatDeviceTabletWriteUsesOneGroup() throws Exception { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -612,9 +616,13 @@ public class IoTDBFlatDeviceIT { } } + // Manual throughput workload: ten clients write disjoint batches of 100 devices across 500,000 + // devices and 100 timestamps, selecting up to 100 of 500 DOUBLE fields per Tablet. All workers + // must finish without write failures; counters report completed Tablets and point throughput. @Ignore @Test - public void testMassiveSeriesWrite() throws SQLException, ExecutionException, InterruptedException { + public void testMassiveSeriesWrite() + throws SQLException, ExecutionException, InterruptedException { int concurrency = 10; int deviceNum = 500000; int fieldNum = 500; @@ -651,50 +659,67 @@ public class IoTDBFlatDeviceIT { List<Future<Void>> tasks = new ArrayList<>(concurrency); for (int i = 0; i < concurrency; i++) { final int taskNum = i; - Callable<Void> task = () -> { - Random random = new Random(); - try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { - session.executeNonQueryStatement("USE " + DATABASE); - - for (int l = 0; l < loop; l++) { - for (int deviceStart = taskNum * devicePerTablet; deviceStart < deviceNum; deviceStart += devicePerTablet * concurrency) { - int fieldStart = random.nextInt(1, fieldNum + 1 - fieldPerTablet); - int fieldEnd = Math.min(fieldStart + fieldPerTablet, fieldNum); - - List<String> subColumnNames = new ArrayList<>(columnNames.subList(fieldStart, fieldEnd)); - subColumnNames.add(0, "device_id"); - List<TSDataType> subDataTypes = new ArrayList<>(dataTypes.subList(fieldStart, fieldEnd)); - subDataTypes.add(0, TSDataType.STRING); - ArrayList<ColumnCategory> subColumnCategories = new ArrayList<>( - columnCategories.subList(fieldStart, fieldEnd)); - subColumnCategories.add(0, ColumnCategory.TAG); - - Tablet tablet = new Tablet("test", subColumnNames, subDataTypes, subColumnCategories, devicePerTablet); - - for (int batchDeviceIndex = 0; batchDeviceIndex < devicePerTablet; batchDeviceIndex++) { - String deviceId = "d" + (deviceStart + batchDeviceIndex); - tablet.addTimestamp(batchDeviceIndex, l); - tablet.addValue(batchDeviceIndex, 0, deviceId); - - for (int f = 0; f < fieldEnd - fieldStart; f++) { - tablet.addValue(batchDeviceIndex, f + 1, (long) l * 1.0); + Callable<Void> task = + () -> { + Random random = new Random(); + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE " + DATABASE); + + for (int l = 0; l < loop; l++) { + for (int deviceStart = taskNum * devicePerTablet; + deviceStart < deviceNum; + deviceStart += devicePerTablet * concurrency) { + int fieldStart = random.nextInt(1, fieldNum + 1 - fieldPerTablet); + int fieldEnd = Math.min(fieldStart + fieldPerTablet, fieldNum); + + List<String> subColumnNames = + new ArrayList<>(columnNames.subList(fieldStart, fieldEnd)); + subColumnNames.add(0, "device_id"); + List<TSDataType> subDataTypes = + new ArrayList<>(dataTypes.subList(fieldStart, fieldEnd)); + subDataTypes.add(0, TSDataType.STRING); + ArrayList<ColumnCategory> subColumnCategories = + new ArrayList<>(columnCategories.subList(fieldStart, fieldEnd)); + subColumnCategories.add(0, ColumnCategory.TAG); + + Tablet tablet = + new Tablet( + "test", + subColumnNames, + subDataTypes, + subColumnCategories, + devicePerTablet); + + for (int batchDeviceIndex = 0; + batchDeviceIndex < devicePerTablet; + batchDeviceIndex++) { + String deviceId = "d" + (deviceStart + batchDeviceIndex); + tablet.addTimestamp(batchDeviceIndex, l); + tablet.addValue(batchDeviceIndex, 0, deviceId); + + for (int f = 0; f < fieldEnd - fieldStart; f++) { + tablet.addValue(batchDeviceIndex, f + 1, (long) l * 1.0); + } + } + + session.insert(tablet); + writtenPointNum.addAndGet((long) (fieldEnd - fieldStart) * devicePerTablet); + writtenTabletNum.addAndGet(1); + writtenTabletNumPerTask[taskNum]++; } + long elapsedTime = System.currentTimeMillis() - startTime; + System.out.printf( + "client %d loop %d finished after %dms, speed %fpt/s%n", + taskNum, l, elapsedTime, writtenPointNum.get() / (elapsedTime / 1000.0)); + System.out.printf("%d points written%n", writtenPointNum.get()); + System.out.printf( + "%d tablets written, %s%n", + writtenTabletNum.get(), Arrays.toString(writtenTabletNumPerTask)); } - - session.insert(tablet); - writtenPointNum.addAndGet((long) (fieldEnd - fieldStart) * devicePerTablet); - writtenTabletNum.addAndGet(1); - writtenTabletNumPerTask[taskNum] ++; } - long elapsedTime = System.currentTimeMillis() - startTime; - System.out.printf("client %d loop %d finished after %dms, speed %fpt/s%n", taskNum, l, elapsedTime, writtenPointNum.get() / (elapsedTime / 1000.0)); - System.out.printf("%d points written%n", writtenPointNum.get()); - System.out.printf("%d tablets written, %s%n", writtenTabletNum.get(), Arrays.toString(writtenTabletNumPerTask)); - } - } - return null; - }; + return null; + }; tasks.add(threadPool.submit(task)); } @@ -784,7 +809,9 @@ public class IoTDBFlatDeviceIT { assertArrayEquals( file.toString(), new byte[] {(byte) (flatDevice ? 1 : 0)}, - reader.getTsFileProperties().get("flat_device." + table)); + reader + .getTsFileProperties() + .get(TsFileTableSchemaUtil.getFlatDevicePropertyKey(table))); checkedFiles++; } } diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionConnectionTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionConnectionTest.java index 655a5147965..1e3e0304e02 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionConnectionTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionConnectionTest.java @@ -253,12 +253,16 @@ public class SessionConnectionTest { sessionConnection.checkTimeseriesExists("root.test1.dev1.s1", 500l); } + // Execute SHOW VERSION through the mocked session with a 500 ms timeout; the query must + // complete without a connection, statement, or redirect exception. @Test public void testExecuteQueryStatement() throws IoTDBConnectionException, StatementExecutionException, RedirectException { sessionConnection.executeQueryStatement("show version", 500l); } + // Repeated transport failures exhaust all eleven query attempts with reconnect unavailable. + // The connection exception must retain the final RPC transport failure as its cause. @Test public void testQueryRetryExhaustionPreservesTransportException() throws Exception { TTransportException firstFailure = new TTransportException("first read timeout"); @@ -279,6 +283,8 @@ public class SessionConnectionTest { } } + // Interrupt the caller before retrying a failed query RPC. Stop after the first attempt, + // preserve the interrupt flag, and expose the original transport failure as the cause. @Test public void testQueryInterruptedRetryPreservesTransportException() throws Exception { TTransportException rpcFailure = new TTransportException("read timeout"); @@ -299,6 +305,8 @@ public class SessionConnectionTest { } } + // The first query RPC times out and the second succeeds, with no live reconnect transport. + // Retry must return a dataset after exactly two RPC attempts. @Test public void testQueryRetryCanRecover() throws Exception { TSExecuteStatementResp response = @@ -319,6 +327,8 @@ public class SessionConnectionTest { } } + // The server repeatedly returns a retryable execution error. After eleven attempts, + // report a statement exception carrying the server status code rather than a transport error. @Test public void testQueryRetryExhaustionPreservesServerStatus() throws Exception { TSStatus status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionEventParserTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionEventParserTest.java index 7a77a44b7fa..4e978b7a930 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionEventParserTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionEventParserTest.java @@ -887,6 +887,9 @@ public class TsFileInsertionEventParserTest { testTreeParserSkipsUnnecessaryBitMaps(false); } + // Parse a table with a non-null TAG, a dense FIELD, and one null sparse FIELD under a limited + // chunk budget. Allocate a bitmap only for the sparse FIELD, even when columns split into + // Tablets. @Test public void testTableParserSkipsUnnecessaryBitMaps() throws Exception { final long originalPipeMaxReaderChunkSize = @@ -966,6 +969,8 @@ public class TsFileInsertionEventParserTest { } } + // Parse an ordinary table file with three TAGs and one FIELD. The resulting Tablet must + // preserve every column name, category, TAG value, and FIELD value in schema order. @Test public void testTableParserPreservesAllTagColumns() throws Exception { alignedTsFile = new File("table-parser-all-tags.tsfile"); @@ -1032,6 +1037,8 @@ public class TsFileInsertionEventParserTest { } } + // Parse a device whose third TAG is null and therefore absent from the shortened device ID. + // The table schema must retain that TAG column and mark its Tablet value null. @Test public void testTableParserPreservesTrailingNullTagColumns() throws Exception { alignedTsFile = new File("table-parser-trailing-tag.tsfile"); @@ -1082,6 +1089,8 @@ public class TsFileInsertionEventParserTest { } } + // Parse a file marked flat_device whose TAG values are stored in physical value chunks. + // Read both TAGs and the FIELD from those chunks, preserving column categories and both rows. @Test public void testTableParserReadsFlatDeviceTagChunks() throws Exception { alignedTsFile = new File("table-parser-flat-device-tags.tsfile"); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/FlatDeviceTableParserDeletionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/FlatDeviceTableParserDeletionTest.java index fd7a3881242..07a45cadc6c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/FlatDeviceTableParserDeletionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/FlatDeviceTableParserDeletionTest.java @@ -65,6 +65,8 @@ public class FlatDeviceTableParserDeletionTest { public void close() {} }; + // Apply a full-row deletion for TAG a to a flat chunk shared by devices a and b. + // Delete only a's row; the shared chunk must still yield b's timestamp and TAG. @Test public void testTagScopedDeletionCannotSkipWholeChunk() throws Exception { FlatDeviceDeletionEntry deletion = @@ -74,6 +76,8 @@ public class FlatDeviceTableParserDeletionTest { Assert.assertEquals(binary("b"), ((Binary[]) tablets.get(0).getValues()[0])[0]); } + // Read TAG-first rows at times 3, 1, and 2 with an unconditional deletion covering times 1..2. + // Time reset between devices must not bypass deletion; only the row at time 3 survives. @Test public void testDeletionSurvivesTimeResetAcrossLogicalDevices() throws Exception { List<Tablet> tablets = @@ -84,6 +88,8 @@ public class FlatDeviceTableParserDeletionTest { Assert.assertEquals(List.of(3L), times(tablets)); } + // Delete only s0 for TAG a in a chunk containing a and b. Both rows and TAGs must survive, + // with a's s0 null, its other FIELD intact, and b's s0 unaffected. @Test public void testScopedFieldDeletionKeepsOtherFieldsAndTags() throws Exception { List<Tablet> tablets = @@ -99,6 +105,8 @@ public class FlatDeviceTableParserDeletionTest { Assert.assertFalse(tablet.getBitMaps()[2].isMarked(1)); } + // A full-device deletion uses TAG order tag1, tag0 while the file stores tag0, tag1. + // Resolve the predicate by column names so only the intended a row is removed. @Test public void testTagMappingUsesDeletionColumnNames() throws Exception { FlatDeviceDeletionEntry deletion = @@ -111,6 +119,8 @@ public class FlatDeviceTableParserDeletionTest { List.of(2L), times(parse(new String[] {"a", "b"}, new long[] {1, 2}, deletion))); } + // Match a null TAG predicate against both an explicit null and a TAG absent from the file + // schema. Treat the missing TAG as null and delete precisely the matching rows. @Test public void testNullAndMissingTagsMatchNullPredicate() throws Exception { FlatDeviceDeletionEntry deletion = @@ -121,6 +131,8 @@ public class FlatDeviceTableParserDeletionTest { Assert.assertTrue(times(parse(new String[] {"a", "b"}, new long[] {1, 2}, deletion)).isEmpty()); } + // Combine overlapping deletions for TAG a over times 1..2 and TAG b at time 1. + // Keep each predicate attached to its own range so b at time 2 remains visible. @Test public void testOverlappingRangesRetainIndependentTagPredicates() throws Exception { FlatDeviceDeletionEntry first = deletion(new TagPredicate.SegmentExactMatch("a", 1), List.of()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTsFileBuilderV2Test.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTsFileBuilderV2Test.java index 199f6a3f4f4..2db1c616aab 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTsFileBuilderV2Test.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTsFileBuilderV2Test.java @@ -19,12 +19,16 @@ package org.apache.iotdb.db.pipe.sink.util.builder; +import org.apache.iotdb.commons.schema.table.TsFileTableSchemaUtil; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.pipe.event.common.tsfile.parser.TsFileInsertionEventParserMemoryBlock; +import org.apache.iotdb.db.pipe.event.common.tsfile.parser.table.TsFileInsertionEventTableParserTabletIterator; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.table.ITableCache; +import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; @@ -33,6 +37,7 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; @@ -49,6 +54,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -108,6 +114,8 @@ public class PipeTsFileBuilderV2Test { } } + // Fail the second database's primary writer by returning a directory as its file path. + // Fallback must produce both database files and remove all partially created primary paths. @Test public void testFallbackDeletesPrimaryTableModelFiles() throws Exception { final AtomicInteger fileIndex = new AtomicInteger(); @@ -142,6 +150,8 @@ public class PipeTsFileBuilderV2Test { } } + // Build a flat table file from a Tablet whose TAG schema uses ordinary encoding. + // The output TAG must use DICTIONARY, without changing FIELD encoding or the input schema. @Test public void testFlatDeviceTagUsesDictionaryEncoding() throws Exception { final String database = "pipe_flat_device_encoding_db"; @@ -186,8 +196,11 @@ public class PipeTsFileBuilderV2Test { } } + // Force the flat table's primary writer to fail, then parse the ordinary fallback file. + // Its layout marker must be false and all five rows must survive, distinguishing null and empty + // TAGs and preserving different logical devices that share the same timestamp. @Test - public void testFlatDeviceFallbackUsesDictionaryEncoding() throws Exception { + public void testFlatDeviceFallbackPreservesRowsWithOrdinaryLayout() throws Exception { final String database = "pipe_flat_device_fallback_db"; final String tableName = "flat_table"; final ITableCache tableCache = DataNodeTableCache.getInstance(); @@ -209,16 +222,41 @@ public class PipeTsFileBuilderV2Test { }; final List<Pair<String, File>> files = new ArrayList<>(); try { - builder.bufferTableModelTablet(database, createTableModelTablet(tableName)); + // Different logical devices share time 10; null and empty TAGs must remain distinct. + final Tablet tablet = + new Tablet( + tableName, + List.of("tag", "field"), + List.of(TSDataType.STRING, TSDataType.INT64), + List.of(ColumnCategory.TAG, ColumnCategory.FIELD), + 5); + final String[] tags = {null, null, "", "A", "B"}; + for (int rowIndex = 0; rowIndex < tags.length; rowIndex++) { + tablet.addTimestamp(rowIndex, rowIndex == 0 ? 5 : 10); + tablet.addValue(rowIndex, 0, tags[rowIndex]); + tablet.addValue(rowIndex, 1, rowIndex + 1L); + } + builder.bufferTableModelTablet(database, tablet); files.addAll(builder.convertTabletToTsFileWithDBInfo()); Assert.assertEquals(1, files.size()); try (final TsFileSequenceReader reader = new TsFileSequenceReader(files.get(0).right.getAbsolutePath())) { + // Footer inspection and the parser must share metadata with the table schema cached. + reader.setEnableCacheTableSchemaMap(); final TableSchema writtenSchema = reader.getTableSchemaMap().get(tableName); Assert.assertNotNull(writtenSchema); + Assert.assertFalse( + TsFileTableSchemaUtil.isFlatDevice(reader.getTsFileProperties(), tableName)); + Assert.assertArrayEquals( + new byte[] {0}, + reader + .getTsFileProperties() + .get(TsFileTableSchemaUtil.getFlatDevicePropertyKey(tableName))); + final List<String> rows = parseTableRows(reader); + Assert.assertEquals(5, rows.size()); Assert.assertEquals( - TSEncoding.DICTIONARY, writtenSchema.getColumnSchemas().get(0).getEncodingType()); + Set.of("5:null:1", "10:null:2", "10::3", "10:A:4", "10:B:5"), Set.copyOf(rows)); } } finally { builder.close(); @@ -227,6 +265,51 @@ public class PipeTsFileBuilderV2Test { } } + private static List<String> parseTableRows(TsFileSequenceReader reader) throws IOException { + final TsFileInsertionEventParserMemoryBlock memory = + new TsFileInsertionEventParserMemoryBlock() { + @Override + public long getMemoryUsageInBytes() { + return 1L << 30; + } + + @Override + public void forceResize(long newSizeInBytes) {} + + @Override + public void close() {} + }; + final TsFileInsertionEventTableParserTabletIterator parser = + new TsFileInsertionEventTableParserTabletIterator( + reader, + entry -> true, + memory, + memory, + memory, + memory, + memory, + PatternTreeMapFactory.getModsPatternTreeMap(), + Long.MIN_VALUE, + Long.MAX_VALUE); + final List<String> rows = new ArrayList<>(); + while (parser.hasNext()) { + final Tablet tablet = parser.next(); + for (int rowIndex = 0; rowIndex < tablet.getRowSize(); rowIndex++) { + final boolean tagIsNull = + tablet.getBitMaps() != null + && tablet.getBitMaps()[0] != null + && tablet.getBitMaps()[0].isMarked(rowIndex); + rows.add( + tablet.getTimestamp(rowIndex) + + ":" + + (tagIsNull ? "null" : ((Binary[]) tablet.getValues()[0])[rowIndex]) + + ":" + + ((long[]) tablet.getValues()[1])[rowIndex]); + } + } + return rows; + } + @Test public void testClassicTableModelBuilderDeletesEarlierDatabaseFilesOnFailure() throws Exception { final AtomicInteger fileIndex = new AtomicInteger(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java index b26b7390c71..5474b2e9629 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java @@ -315,6 +315,9 @@ public class FragmentInstanceExecutionTest { } } + // Two queries project different columns from an unsorted five-column MemTable, triggering + // partial clone and ownership transfer. Both must read all 100 rows correctly; the old source's + // reservation must match cleanup accounting and be released when its owning query ends. @Test public void testAlignedTVListPartialColumnCloneEndToEnd() throws InterruptedException { ExecutorService instanceNotificationExecutor = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FlatDevicePageReaderQueueTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FlatDevicePageReaderQueueTest.java index 0a33559c312..37e7067a345 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FlatDevicePageReaderQueueTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FlatDevicePageReaderQueueTest.java @@ -119,6 +119,8 @@ public class FlatDevicePageReaderQueueTest { ChunkCache.getInstance().clear(); } + // Skip one physical page by a time filter and another by deletion. The surviving reader must + // use physical page index 2, retain its modified flag, and decode the expected c-device rows. @Test public void testPhysicalPageLoadAfterSkippedPages() throws Exception { metadata.getTimeChunkMetadata().insertIntoSortedDeletions(new TimeRange(100, 101)); @@ -142,6 +144,8 @@ public class FlatDevicePageReaderQueueTest { assertEquals(100, block.getColumn(1).getLong(0)); } + // Queue pages for devices a, b, and c whose time ranges reset between TAGs. + // Ascending and descending merges must follow TAG then time order and preserve FIELD values. @Test public void testQueueTagTimeOrderBothDirections() throws Exception { for (boolean ascending : new boolean[] {true, false}) { @@ -174,6 +178,8 @@ public class FlatDevicePageReaderQueueTest { } } + // Combine queue and chunk time filters with full-row and FIELD deletions across flat pages. + // Only c at time 11 must remain, with its deleted FIELD null and no extra rows. @Test public void testQueueFilteringAndDeletion() throws Exception { metadata.getTimeChunkMetadata().insertIntoSortedDeletions(new TimeRange(100, 101)); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java index c34f195b0e0..754a39b734c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java @@ -64,6 +64,8 @@ import static org.junit.Assert.assertTrue; public class AnalyzeUtilsTest { + // Parse a DELETE predicate ts <= 100 for a table whose time column is named ts. + // Produce one deletion range from Long.MIN_VALUE through 100. @Test public void testParseDeletePredicateWithRenamedTimeColumn() { TsTable table = new TsTable("table1"); @@ -83,6 +85,8 @@ public class AnalyzeUtilsTest { assertEquals(100, entries.get(0).getEndTime()); } + // Analyze relational row and Tablet inserts before the physical flat grouping is selected. + // Partition routing must retain the complete table-plus-TAG device IDs for both insert forms. @Test public void testTableDataPartitionParamsKeepCompleteDeviceId() { // flat_device is deliberately absent from insert statements during analysis. Partition diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerTest.java index 966aec2d555..3858162262b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerTest.java @@ -350,6 +350,8 @@ public class LoadTsFileAnalyzerTest { } } + // Disable global schema auto-creation while LOAD still requests it. The analyzer must keep + // the effective global setting false and retain the per-LOAD request for permission checks. @Test public void testGlobalAutoCreateDisabledKeepsPerLoadAutoCreatePermission() throws Exception { final boolean originalAutoCreateSchemaEnabled = @@ -480,6 +482,8 @@ public class LoadTsFileAnalyzerTest { .anyMatch(device -> Arrays.equals(device.getSegments(), expectedSegments)); } + // Inspect source footer properties with a flat marker and table schema but no cached table. + // LOAD analysis must detect the flat layout directly from the source file. @Test public void testFlatDevicePropertyIsReadWithoutTableCache() throws Exception { final Map<String, byte[]> properties = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java index be799c89a4a..57128665f0c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java @@ -135,6 +135,8 @@ public class InsertRowNodeSerdeTest { Assert.assertFalse(tmpNode.isLastFragment()); } + // Round-trip an ordinary relational row through the legacy WAL format. Its 103-byte payload + // must retain the legacy node type and search index without adding a flat-device flag byte. @Test public void testDeserializeLegacyWALRelational() throws IOException { RelationalInsertRowNode insertRowNode = getRelationalInsertRowNodeWithMeasurementSchemas(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsNodeSerdeTest.java index 13e6eb7cd46..24385241aba 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsNodeSerdeTest.java @@ -411,6 +411,8 @@ public class InsertRowsNodeSerdeTest { } } + // Round-trip two ordinary relational rows through legacy batch WAL. The 202-byte payload + // must retain both rows and the search index without adding layout flags to its children. @Test public void testDeserializeLegacyWALRelational() throws IOException { RelationalInsertRowsNode insertRowsNode = @@ -487,6 +489,8 @@ public class InsertRowsNodeSerdeTest { Assert.assertFalse(tmpNode.isLastFragment()); } + // Serialize a batch containing one flat row and one ordinary row through plan and both WAL + // decoders. Use the flat batch node type while preserving each child's individual layout flag. @Test public void testRelationalFlatDeviceRowsSerde() throws IOException { RelationalInsertRowsNode insertRowsNode = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java index 2ded4f2e0ad..0bec1b6577f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java @@ -193,6 +193,8 @@ public class InsertTabletNodeSerdeTest { } } + // Round-trip an ordinary relational Tablet through legacy WAL. Its 232-byte payload must + // retain the legacy node type, values, and search index without a flat-device flag byte. @Test public void testDeserializeLegacyWALRelational() throws IOException { RelationalInsertTabletNode insertTabletNode = getRelationalInsertTabletNodeWithSchema("table1"); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNodeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNodeTest.java index a60360875a3..193b10d0ddf 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNodeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNodeTest.java @@ -33,6 +33,8 @@ import static org.junit.Assert.assertEquals; public class DeviceTableScanNodeTest { + // Clone and serialize scan nodes with flat_device both enabled and disabled. + // Each resulting node must retain the original layout flag. @Test public void testFlatDeviceFlagSurvivesCloneAndSerialization() { for (boolean flatDevice : new boolean[] {false, true}) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java index 7170643918b..85e6678487e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java @@ -172,6 +172,9 @@ public class InsertStatementTest { return tsTable; } + // Validate an insert with reordered TAGs, a missing TAG, and an incompatible FIELD type. + // Reorder and fill TAGs with DICTIONARY schemas, preserve valid values, and null the failed + // FIELD. @Test public void testValidate() { InsertRowStatement insertRowStatement = new InsertRowStatement(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriterTest.java index b544c2a14e5..1e3a2827a8b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriterTest.java @@ -61,6 +61,9 @@ public class AbstractCompactionWriterTest { private static final int SUB_TASK_ID = 0; + // Use the same single-segment ID for a flat table and an ordinary all-null-TAG device. + // The explicit flat flag selects table/file bounds; ordinary mode reads actual device metadata + // rather than inferring the layout from the number of ID segments. @Test public void testCrossCompactionUsesExplicitFlatDeviceProperty() throws IOException { IDeviceID device = IDeviceID.Factory.DEFAULT_FACTORY.create(new String[] {"table1"}); @@ -94,6 +97,8 @@ public class AbstractCompactionWriterTest { } } + // Store flat table bounds under the table-only ID in the ordinary device time index. + // Cross compaction must find that physical entry without requiring a logical TAG-device entry. @Test public void testCrossCompactionFlatDeviceUsesMergedTableTimeIndexEntry() throws IOException { IDeviceID device = IDeviceID.Factory.DEFAULT_FACTORY.create(new String[] {"table1"}); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtilTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtilTest.java index 433b99dd087..957cbcd01d4 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtilTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtilTest.java @@ -65,6 +65,8 @@ public class RepairDataFileScanUtilTest extends AbstractCompactionTest { super.tearDown(); } + // Scan a valid file with aligned null columns and non-aligned series across separate devices. + // Repair validation must report neither file corruption nor unsorted data or wrong statistics. @Test public void testScanNormalFile() throws IOException { TsFileResource resource = createEmptyFileAndResource(true); @@ -94,6 +96,8 @@ public class RepairDataFileScanUtilTest extends AbstractCompactionTest { Assert.assertFalse(scanUtil.hasUnsortedDataOrWrongStatistics()); } + // Scan flat rows with repeated null TAGs followed by a non-null TAG and a time reset. + // Null-first order must pass; a null TAG after a non-null TAG must be unsorted, not corrupt. @Test public void testFlatDeviceNullTagOrder() throws IOException { for (boolean unsorted : List.of(false, true)) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/FlatDeviceCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/FlatDeviceCompactionTest.java index 25befbc94e2..6bbbdd7c1ba 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/FlatDeviceCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/FlatDeviceCompactionTest.java @@ -95,11 +95,15 @@ public class FlatDeviceCompactionTest extends AbstractCompactionTest { Thread.currentThread().setName(oldThreadName); } + // Read-point compact two flat files containing four distinct TAG tuples at time 1. + // All four rows, their values, the table schema, and the flat layout marker must survive. @Test public void testReadPointCompactionKeepsFlatDeviceRowsWithSameTimestamp() throws Exception { assertFlatDeviceRowsSurviveCompaction(new ReadPointCompactionPerformer()); } + // Inner compact TAG tuples (a, b.c) and (a.b, c), which have the same display string. + // Count two distinct logical devices and four rows across the two source timestamps. @Test public void testInnerCompactionStatisticsKeepTagSegmentBoundaries() throws Exception { seqResources.add( @@ -116,6 +120,9 @@ public class FlatDeviceCompactionTest extends AbstractCompactionTest { Assert.assertEquals(4, readFlatDeviceRows(target).size()); } + // Cross compact seq/unseq files with colliding TAG display strings and duplicate full keys. + // Each target must count two distinct devices and two deduplicated rows, preserving TAG + // boundaries. @Test public void testCrossCompactionStatisticsKeepTagSegmentBoundaries() throws Exception { // Every target contains two distinct TAG tuples that both display as flat_table.a.b.c. @@ -147,6 +154,8 @@ public class FlatDeviceCompactionTest extends AbstractCompactionTest { } } + // Repair unsorted rows whose two TAG tuples share a display string, including a duplicate key. + // The rewritten file must contain and count two distinct logical devices and two rows. @Test public void testRepairStatisticsKeepTagSegmentBoundaries() throws Exception { TsFileResource source = createEmptyFileAndResource(true); @@ -172,11 +181,16 @@ public class FlatDeviceCompactionTest extends AbstractCompactionTest { Assert.assertEquals(2, readFlatDeviceRows(target).size()); } + // Read-point compact three overlapping unseq files with eight logical devices and 20 times + // per file. Preserve TAG/time keys, let newer duplicate values win, and keep one unseq target + // with table-only time bounds and accurate logical-device and row counts. @Test public void testInnerUnseqCompactionWithOverlappingFlatDeviceFiles() throws Exception { assertInnerUnseqFlatDeviceCompaction(3, 20, 8, 37, 11, false); } + // Run FAST INNER_UNSEQ on three overlapping flat files with eight devices and 20 times per file. + // Its read-point fallback must preserve unseq classification, all full keys, and newer values. @Test public void testFastInnerUnseqCompactionWithOverlappingFlatDeviceFiles() throws Exception { // INNER_UNSEQ defaults to FAST. Its flat fallback must preserve the unseq classification @@ -185,6 +199,9 @@ public class FlatDeviceCompactionTest extends AbstractCompactionTest { 3, 20, 8, 37, 11, false, new FastCompactionPerformer(false)); } + // Run FAST on eight overlapping flat unseq files with 32 devices, 113-row chunks, and 31-row + // pages. The fallback must preserve distinct same-time TAG keys across boundaries and deduplicate + // updates into one sorted unseq target with correct statistics. @Test public void testFastInnerUnseqCompactionWithFlatDevicePageAndChunkBoundaries() throws Exception { // Exercise the default performer with equal timestamps split across physical pages/chunks. @@ -192,6 +209,9 @@ public class FlatDeviceCompactionTest extends AbstractCompactionTest { 8, 67, 32, 113, 31, false, new FastCompactionPerformer(false)); } + // Use the INNER_UNSEQ FAST factory on eight overlapping files with deterministic device/time + // gaps and page/chunk splits. The single-list fallback must retain every surviving full key + // with newer values, correct statistics, and unseq target classification. @Test public void testFastInnerUnseqCompactionWithSparseFlatDeviceRows() throws Exception { // Unequal client progress leaves gaps in overlapping flushes. Use the INNER_UNSEQ factory @@ -200,6 +220,9 @@ public class FlatDeviceCompactionTest extends AbstractCompactionTest { 8, 67, 32, 113, 31, true, InnerUnseqCompactionPerformer.FAST.createInstance()); } + // A newer unseq file backfills earlier times and overwrites (a, x, 3), while b and c also use + // time 3. FAST fallback must preserve all seven distinct rows, newer values, and table bounds + // 1..6. @Test public void testFastInnerUnseqCompactionWithEarlierRowsInNewerFlatDeviceFile() throws Exception { // File version order differs from time order. The newer file both backfills earlier data @@ -246,6 +269,9 @@ public class FlatDeviceCompactionTest extends AbstractCompactionTest { Assert.assertEquals(6L, target.getEndTime(tableDevice).get().longValue()); } + // FAST cross compaction falls back for flat files from both seq and unseq spaces. + // Unseq values must win duplicate full keys; same-time b/c rows must survive in two seq targets + // with three logical devices and three rows each, leaving no unseq source files. @Test public void testFastCrossCompactionPreservesFlatDeviceSourceSpaces() throws Exception { // The fallback must retain both source spaces so unseq updates win duplicate full row keys @@ -283,11 +309,16 @@ public class FlatDeviceCompactionTest extends AbstractCompactionTest { rows); } + // Read-point compact eight overlapping unseq files with 32 devices and split equal-time keys + // across 113-row chunks and 31-row pages. Preserve and deduplicate by TAG/time into one sorted + // unseq file with accurate table bounds and row/device statistics. @Test public void testInnerUnseqCompactionWithFlatDevicePageAndChunkBoundaries() throws Exception { assertInnerUnseqFlatDeviceCompaction(8, 67, 32, 113, 31, false); } + // Read-point compact eight unseq files with sparse device/time progress and page/chunk splits. + // Retain every expected full key with the latest source value and correct target statistics. @Test public void testInnerUnseqCompactionWithSparseFlatDeviceRows() throws Exception { assertInnerUnseqFlatDeviceCompaction(8, 67, 32, 113, 31, true); @@ -414,6 +445,9 @@ public class FlatDeviceCompactionTest extends AbstractCompactionTest { Assert.assertFalse(resource.getTsFilePath(), scan.hasUnsortedDataOrWrongStatistics()); } + // Scan two chunks with TAGs separated by a FIELD and equal timestamps, with resource checks + // both on and off. Compare all TAGs across chunks: a regressing TAG or duplicate full key must + // be reported unsorted, while increasing TAGs pass despite table-only resource indexing. @Test public void testRepairChecksFlatDeviceRowKeys() throws Exception { // TAGs are separated by a FIELD in the physical schema. Compare both TAGs, including @@ -443,6 +477,9 @@ public class FlatDeviceCompactionTest extends AbstractCompactionTest { } } + // Repair descending times, same-time different TAGs, and a duplicate full key. + // Emit four TAG/time-sorted rows for two devices, count deduplicated rows, and index only the + // physical table ID with bounds 1..3; the repaired file must pass resource validation. @Test public void testRepairWritesOnlyTableTimeIndexAndCountsDeduplicatedRows() throws Exception { TsFileResource source = createEmptyFileAndResource(true); @@ -497,11 +534,15 @@ public class FlatDeviceCompactionTest extends AbstractCompactionTest { Assert.assertFalse(scan.hasUnsortedDataOrWrongStatistics()); } + // FAST inner compaction encounters flat files with four distinct TAG tuples at time 1. + // Its fallback must retain all four rows and the flat table schema and layout marker. @Test public void testFastCompactionKeepsFlatDeviceRowsWithSameTimestamp() throws Exception { assertFlatDeviceRowsSurviveCompaction(new FastCompactionPerformer(false)); } + // Read-chunk compaction encounters flat files with four distinct TAG tuples at time 1. + // Its fallback must preserve all full row keys, FIELD values, schema, and flat layout marker. @Test public void testReadChunkCompactionKeepsFlatDeviceRowsWithSameTimestamp() throws Exception { assertFlatDeviceRowsSurviveCompaction(new ReadChunkCompactionPerformer()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedTVListIteratorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedTVListIteratorTest.java index 5de2f8f3e06..113fc312992 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedTVListIteratorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedTVListIteratorTest.java @@ -559,6 +559,8 @@ public class AlignedTVListIteratorTest { 10000); } + // Read aligned rows in descending order with varying limits and offsets, including an offset + // past all data. Return the expected row count for each pagination setting. @Test public void testAlignedDescWithLimitAndOffset() throws IOException { testAligned( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/FlatDeviceMemTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/FlatDeviceMemTableTest.java index f4161c890b1..f9134cadaf4 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/FlatDeviceMemTableTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/FlatDeviceMemTableTest.java @@ -465,6 +465,9 @@ public class FlatDeviceMemTableTest { Assert.assertEquals(2, writer.getTimeChunkWriter().getStatistics().getCount()); } + // Flush dictionary-backed logical devices through single-list and threshold-split encoding. + // Expand both TAG chunks in TAG-first order, retain null TAGs, and merge duplicates using the + // latest non-null FIELD even when physical timestamps regress between devices. @Test public void testDictionaryFlushExpandsTagChunksInBothEncodePaths() throws Exception { int oldThreshold = IoTDBDescriptor.getInstance().getConfig().getTvListSortThreshold(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java index 6b4ac9645c5..0b77257413c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java @@ -228,6 +228,8 @@ public class MemChunkDeserializeTest { } } + // WAL round-trip 1,000 ordinary aligned rows spanning six value types after deleting times + // 100..200. The restored reader must omit that range and preserve all remaining typed values. @Test public void testAlignedSeries() throws IOException, QueryProcessException, MetadataException { // IMemTable memTable = new PrimitiveMemTable(storageGroup, dataRegionId); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java index e8012404964..c01b553907c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java @@ -1436,6 +1436,8 @@ public class TsFileProcessorTest { Assert.assertEquals(memTable1.memSize(), memTable2.memSize()); } + // Query an empty processor, insert 100 ordinary rows, and query them from memory. + // The values and timestamps must match each insert, and synchronous close must seal the resource. @Test public void testWriteAndClose() throws IOException, WriteProcessException, MetadataException, ExecutionException { @@ -1502,6 +1504,9 @@ public class TsFileProcessorTest { assertFlatDeviceResourceTimes(false); } + // Flush (a.b, c) and (a, b.c) at two timestamps in separate MemTables. + // Footer statistics must union complete TAG tuples across flushes, counting two devices and four + // rows. @Test public void testFlatDeviceStatisticsKeepTagSegmentBoundaries() throws Exception { processor = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntryTest.java index b12f8b1a70c..7ef1dd597b1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntryTest.java @@ -53,6 +53,9 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; public class TableDeletionEntryTest { + // Clone and round-trip a TAG-scoped flat deletion through buffer and stream serializers. + // Each copy must match overlapping table-only resource bounds without deleting the entire table, + // and still distinguish matching and nonmatching TAG values at row level. @Test public void testFlatDeviceDeletionWithTableTimeIndex() throws IOException { FlatDeviceDeletionEntry deletion = @@ -101,6 +104,8 @@ public class TableDeletionEntryTest { } } + // Apply partial and full time ranges to ordinary, TAG-scoped flat, and unconditional flat + // deletions. Only the TAG-scoped flat deletion must defer chunk removal to row-level checks. @Test public void testChunkDeletionForNormalAndFlatDevices() { // Normal-table predicates are already matched against the chunk's logical device. Only @@ -210,6 +215,8 @@ public class TableDeletionEntryTest { } } + // Evaluate diverse TAG predicates with reordered columns, missing indexes, and trailing nulls. + // Mapped-row matching and both serialized forms must agree with matching the logical device ID. @Test public void testMappedTagValuesMatching() throws IOException { IDeviceID device = Factory.DEFAULT_FACTORY.create(new String[] {"t", "a", ""}); @@ -258,6 +265,8 @@ public class TableDeletionEntryTest { } } + // Reuse a value array and TAG mapping while switching String/Binary values, nulls, and indexes. + // FullExactMatch and DeviceIn must use current contents rather than retain a stale match. @Test public void testReusableTagValues() { IDeviceID device = Factory.DEFAULT_FACTORY.create(new String[] {"t", "a"}); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/AlignedChunkPageReaderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/AlignedChunkPageReaderTest.java index c8137c04e5b..6456763fb03 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/AlignedChunkPageReaderTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/AlignedChunkPageReaderTest.java @@ -49,6 +49,9 @@ import static org.junit.Assert.assertTrue; public class AlignedChunkPageReaderTest { private static final List<TSDataType> TYPES = List.of(TSDataType.INT64); + // Read first, middle, and last physical pages repeatedly under three compression types, + // including regressing page times. Decode only the requested rows and leave chunk buffers + // unchanged. @Test public void testFirstMiddleLastPageAndUnchangedBuffers() throws IOException { for (CompressionType compression : @@ -70,6 +73,8 @@ public class AlignedChunkPageReaderTest { } } + // A time filter skips page 0 and a full time deletion skips page 1. + // List and single-page loading must preserve physical index 2 and return its correct values. @Test public void testPhysicalIndexesAfterFilteringAndDeletion() throws IOException { Chunk[] chunks = chunks(CompressionType.LZ4, new long[][] {{0, 1}, {10, 11}, {20, 21}}); @@ -86,6 +91,8 @@ public class AlignedChunkPageReaderTest { assertEquals(200, block.getColumn(0).getLong(0)); } + // Load page 1 with a partial time deletion, a FIELD deletion, and page/record time filters. + // Mark the reader modified and return only times 11 and 12, with the FIELD at 11 null. @Test public void testPartialTimeAndValueDeletionAndRecordFilter() throws IOException { Chunk[] chunks = chunks(CompressionType.LZ4, new long[][] {{0, 1}, {10, 11, 12, 13}}); @@ -101,6 +108,8 @@ public class AlignedChunkPageReaderTest { assertEquals(120, block.getColumn(0).getLong(1)); } + // Load a single-page chunk with a partial time filter and request an out-of-range page. + // Report physical index 0, return only times 2 and 3, and yield no reader for page 1. @Test public void testSinglePageAndMissingPage() throws IOException { Chunk[] chunks = chunks(CompressionType.LZ4, new long[][] {{1, 2, 3}}); @@ -113,6 +122,8 @@ public class AlignedChunkPageReaderTest { assertTrue(reader(chunks, null, 1, null).loadPageReaderList().isEmpty()); } + // Read table pages with one missing column and an empty value page for the other column. + // Retain both timestamped rows even when every projected FIELD is null. @Test public void testMissingColumnEmptyPageAndAllNullTableRows() throws IOException { Chunk[] chunks = chunks(CompressionType.LZ4, new long[][] {{1, 2}, {3, 4}}, true); @@ -129,6 +140,9 @@ public class AlignedChunkPageReaderTest { } } + // Corrupt the compressed time bodies surrounding a valid middle page, leaving headers intact. + // Single-page loading must decode the middle page successfully; constructing all readers must + // fail. @Test public void testDoesNotDecompressOtherPages() throws IOException { Chunk[] chunks = chunks(CompressionType.LZ4, new long[][] {{1, 2}, {3, 4}, {5, 6}}); @@ -152,6 +166,8 @@ public class AlignedChunkPageReaderTest { assertThrows(Exception.class, () -> new TableChunkReader(chunks[0], List.of(chunks[1]), null)); } + // Load only the second page after marking the chunk's data type modified. + // Propagate the modified flag so page statistics cannot be used as if the schema were unchanged. @Test public void testDataTypeModification() throws IOException { Chunk[] chunks = chunks(CompressionType.LZ4, new long[][] {{1, 2}, {3, 4}}); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoaderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoaderTest.java index 0bb38a12556..9ccbfcb03d4 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoaderTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoaderTest.java @@ -60,6 +60,10 @@ public class MemAlignedChunkLoaderTest { private static final String BINARY_STR = "ty love zm"; private static final int maxNumberOfPointsInPage = 1000; + // Load a mocked ordinary aligned MemChunk with two rows and six typed columns. + // Return its memory page, values, and statistics in batch/TsBlock APIs while rejecting + // unsupported + // disk chunk/page APIs; descending batch mode must retain the expected data. @Test public void testMemAlignedChunkLoader() throws IOException { AlignedReadOnlyMemChunk chunk = Mockito.mock(AlignedReadOnlyMemChunk.class); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/common/AlignedPriorityMergeReaderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/common/AlignedPriorityMergeReaderTest.java index ff02f97b064..4d6a747c250 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/common/AlignedPriorityMergeReaderTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/common/AlignedPriorityMergeReaderTest.java @@ -120,6 +120,8 @@ public class AlignedPriorityMergeReaderTest { new long[] {30, 20, 11}); } + // Merge same-time flat vectors with a FIELD between two TAG columns. + // Use projected TAG indexes 0 and 2 for ordering, preserving four distinct compound row keys. @Test public void testFlatDeviceRowsAreOrderedByTagColumns() throws IOException { // A flat-device aligned vector may contain fields between TAG columns. The row-key indices diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/common/FlatDevicePointReaderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/common/FlatDevicePointReaderTest.java index 80aefd4106d..823dac6df02 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/common/FlatDevicePointReaderTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/common/FlatDevicePointReaderTest.java @@ -40,6 +40,10 @@ public class FlatDevicePointReaderTest { private static final List<TSDataType> DATA_TYPES = List.of(TSDataType.STRING, TSDataType.INT64, TSDataType.STRING); + // Traverse a TAG-first page with interleaved FIELDs, null TAGs, and time resets in both + // directions. + // Repeated current-row access must be stable, each row must match its source, and the memory + // reservation must remain available for heap accounting after exhaustion. @Test public void testTagFirstPageInBothDirections() { // TAGs surround a FIELD column, and changing either TAG can move time backwards. @@ -68,6 +72,8 @@ public class FlatDevicePointReaderTest { } } + // Open an empty flat page in ascending and descending modes. + // Both readers must immediately report exhaustion and return null for current and next rows. @Test public void testEmptyPageInBothDirections() { TsBlock block = new TsBlockBuilder(DATA_TYPES).build(); @@ -80,6 +86,8 @@ public class FlatDevicePointReaderTest { } } + // Read a one-row flat page with a null FIELD in both directions. + // Return exactly that row with its TAGs and null value, then report exhaustion. @Test public void testSingleRowInBothDirections() { TsBlock block = @@ -94,6 +102,9 @@ public class FlatDevicePointReaderTest { } } + // Close a two-row flat reader after consuming one row, in both traversal directions. + // Subsequent access must report exhaustion, repeated close must be safe, and the reservation + // must remain available for the merge heap to release. @Test public void testCloseBeforePageIsExhausted() { TsBlock block = @@ -114,6 +125,9 @@ public class FlatDevicePointReaderTest { } } + // Merge older and newer TAG-first pages with overlapping full keys in both directions. + // Prefer newer FIELDs, fill newer nulls from older values, preserve six distinct keys, and + // release all merge memory after the final row. @Test public void testOverlappingPagesInBothDirections() throws IOException { TsBlock older = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/FlatDeviceAlignedChunkWriterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/FlatDeviceAlignedChunkWriterTest.java index ad7083843dd..cd2cf18c5fb 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/FlatDeviceAlignedChunkWriterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/FlatDeviceAlignedChunkWriterTest.java @@ -39,6 +39,8 @@ import java.util.ArrayList; import java.util.List; public class FlatDeviceAlignedChunkWriterTest { + // Write only the middle three entries of a timestamp batch containing sentinels and time reset. + // Track actual bounds 1..100 and count three rows without sealing a page on time regression. @Test public void testTimeBatchOffsetAndRegressingStatistics() { FlatDeviceAlignedChunkWriter writer = @@ -49,6 +51,9 @@ public class FlatDeviceAlignedChunkWriterTest { Assert.assertEquals(0, writer.getTimeChunkWriter().getNumOfPages()); } + // Write two nonzero-offset column regions with nulls and time resets across a three-row page + // boundary. Seal two pages by size, exclude the sentinel, and compute time/value statistics + // from the five selected rows for all six value types. @Test public void testColumnRegionsAcrossPageBoundary() { TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/FlatDeviceRecoveryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/FlatDeviceRecoveryTest.java index f272be0df21..d30d6e3cda9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/FlatDeviceRecoveryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/FlatDeviceRecoveryTest.java @@ -35,6 +35,8 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalIn import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.FlatDeviceCompactionUtils; +import org.apache.iotdb.db.storageengine.dataregion.compaction.repair.RepairDataFileScanUtil; import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; import org.apache.iotdb.db.storageengine.dataregion.memtable.PrimitiveMemTable; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; @@ -54,6 +56,7 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.filter.factory.TimeFilterApi; @@ -74,6 +77,7 @@ import java.util.List; import java.util.Set; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class FlatDeviceRecoveryTest { @@ -114,6 +118,8 @@ public class FlatDeviceRecoveryTest { EnvironmentUtils.cleanEnv(); } + // Replay B at time 50 while the flat resource's table bounds are 10..100. + // Table end time must not suppress an unflushed logical-device row. @Test public void testRowReplayBelowTableEndTime() throws Exception { setTableBounds(); @@ -122,6 +128,8 @@ public class FlatDeviceRecoveryTest { assertEquals(List.of("50:B:2"), readMemTable(redoer.getRecoveryMemTable())); } + // Replay a row batch for B at 50 and a null-TAG device at 60 within table bounds 10..100. + // Keep both rows and return them in logical-device order rather than skip by table end time. @Test public void testBatchRowReplayBelowTableEndTime() throws Exception { setTableBounds(); @@ -132,6 +140,8 @@ public class FlatDeviceRecoveryTest { assertEquals(List.of("60:null:3", "50:B:2"), readMemTable(redoer.getRecoveryMemTable())); } + // Replay a Tablet with null TAG at 50 and B at 60 below table end time 100. + // The null-TAG logical ID equals the physical ID, but both unflushed rows must still survive. @Test public void testTabletReplayIncludesNullTagBelowTableEndTime() throws Exception { setTableBounds(); @@ -158,6 +168,8 @@ public class FlatDeviceRecoveryTest { assertEquals(List.of("50:null:2", "60:B:3"), readMemTable(redoer.getRecoveryMemTable())); } + // Recover a snapshot containing B at 50 and an updated A at 100 inside flushed table bounds, + // then replay C at 60. Install and retain the snapshot; merge all rows with the updated A value. @Test public void testSnapshotRetainsRowsInsideFlushedTableRange() throws Exception { writeUnsealedFile(); @@ -177,6 +189,39 @@ public class FlatDeviceRecoveryTest { assertEquals(List.of("10:A:1", "100:A:3", "20:B:2", "50:B:2", "60:C:4"), readRecoveredFile()); } + // Recover a crashed unsealed file after every MemTable was flushed, with no WAL replay. + // Restore the table schema and flat footer marker from recovered chunks, preserve all three + // rows, and make compaction detection and resource/repair validation succeed. + @Test + public void testCrashRecoveryWithoutWalRestoresFlatMetadata() throws Exception { + writeUnsealedFile(); + try (UnsealedTsFileRecoverPerformer performer = + new UnsealedTsFileRecoverPerformer(resource, false, ignored -> {})) { + performer.startRecovery(); + assertTrue(performer.hasCrashed()); + // All MemTables were flushed before the crash, so no WAL entry registers this table. + performer.endRecovery(); + } + resource.close(); + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getPath())) { + assertEquals(Set.of(TABLE), reader.getTableSchemaMap().keySet()); + assertEquals( + List.of("tag", "s"), + reader.getTableSchemaMap().get(TABLE).getColumnSchemas().stream() + .map(schema -> schema.getMeasurementName()) + .toList()); + assertTrue(TsFileTableSchemaUtil.isFlatDevice(reader.getTsFileProperties(), TABLE)); + } + assertTrue(FlatDeviceCompactionUtils.containsFlatDevice(List.of(resource))); + RepairDataFileScanUtil scan = new RepairDataFileScanUtil(resource); + scan.scanTsFile(true); + assertFalse(scan.isBrokenFile()); + assertFalse(scan.hasUnsortedDataOrWrongStatistics()); + assertEquals(List.of("10:A:1", "100:A:1", "20:B:2"), readRecoveredFile()); + } + + // Recover flushed A/B rows, then replay an updated A at 100 plus B at 50 and null TAG at 60. + // Merge by complete TAG/time keys, retaining all five rows with the later WAL value for A. @Test public void testCrashRecoveryReplaysAndMergesDuplicateFlatKeys() throws Exception { writeUnsealedFile(); @@ -194,6 +239,8 @@ public class FlatDeviceRecoveryTest { List.of("60:null:5", "10:A:1", "100:A:3", "20:B:2", "50:B:4"), readRecoveredFile()); } + // Recover a TAG-A deletion over 0..200 that covers the shared flat chunk's entire time range. + // Keep B at 20 and preserve conservative table resource bounds 10..100. @Test public void testTagDeletionCoveringChunkKeepsResourceBounds() throws Exception { assertRecoveredBounds( @@ -204,6 +251,8 @@ public class FlatDeviceRecoveryTest { List.of("20:B:2")); } + // Recover a TAG-A deletion over 0..30 touching the chunk start. + // Keep A at 100 and B at 20 without trimming the shared table resource bounds 10..100. @Test public void testTagDeletionTouchingChunkBoundaryKeepsResourceBounds() throws Exception { assertRecoveredBounds( @@ -214,6 +263,8 @@ public class FlatDeviceRecoveryTest { List.of("100:A:1", "20:B:2")); } + // Recover a deletion over 0..30 with no TAG restriction. + // Unlike scoped deletion, trim the resource bounds to 31..100 and retain only A at 100. @Test public void testUnconditionalDeletionStillTrimsResourceBounds() throws Exception { assertRecoveredBounds( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedSparseWritePerformanceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedSparseWritePerformanceTest.java index 03e63fae0e7..a668d6933c9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedSparseWritePerformanceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedSparseWritePerformanceTest.java @@ -48,6 +48,9 @@ public class AlignedSparseWritePerformanceTest { private static volatile long benchmarkBlackhole; + // Opt-in benchmark comparing dense, sparse, all-null, null-prefix, and schema-extension writes. + // Defaults use 64 columns with one sparse column; validated workloads report CPU/allocation + // comparisons and avoided bitmap costs without asserting a fixed performance threshold. @Test public void sparseAlignedWriteBenchmark() { Assume.assumeTrue( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java index b826048c3aa..ca33a74cdf7 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java @@ -61,6 +61,8 @@ public class AlignedTVListTest { Assert.assertEquals(NUM_BYTES_OBJECT_REF, AlignedTVList.bitmapReferenceRamCost()); } + // Append 1,000 ascending rows to five INT64 columns. + // Each stored timestamp and all five values must match the inserted row index. @Test public void testAlignedTVList1() { List<TSDataType> dataTypes = new ArrayList<>(); @@ -87,6 +89,8 @@ public class AlignedTVListTest { } } + // Insert descending timestamps with six different value types, then sort the aligned list. + // Time must become ascending while every typed value remains attached to its row. @Test public void testAlignedTVList2() { List<TSDataType> dataTypes = new ArrayList<>(); @@ -117,6 +121,8 @@ public class AlignedTVListTest { } } + // Batch-insert a selected range of descending timestamps into five INT64 columns. + // The list must retain the batch's insertion order before sorting. @Test public void testAlignedTVLists() { List<TSDataType> dataTypes = new ArrayList<>(); @@ -140,6 +146,8 @@ public class AlignedTVListTest { } } + // Batch-insert descending timestamps with periodic null markers in all five columns. + // Preserve insertion order and expose the marked rows as all-null aligned values. @Test public void testAlignedTVListsWithBitMaps() { List<TSDataType> dataTypes = new ArrayList<>(); @@ -172,6 +180,9 @@ public class AlignedTVListTest { } } + // Write more than two dense array blocks, then introduce a null in the third block. + // Allocate only that compact bitmap, leave earlier bitmap slots absent, and charge exactly + // the new bitmap and container/reference memory without materializing another value array. @Test public void testBitmapIsAllocatedLazilyWithCompactBackingArray() { AlignedTVList tvList = @@ -206,6 +217,8 @@ public class AlignedTVListTest { tvList.calculateRamSize().getRamSize() - ramSizeBeforeBitmapAllocation); } + // Split a Tablet at offset 3 with failed statuses on both sides of an array boundary. + // Mark precisely the failed rows null in both columns without shifting neighboring bits. @Test public void testFailedStatusBitmapWithNonByteAlignedOffset() { AlignedTVList tvList = @@ -235,6 +248,8 @@ public class AlignedTVListTest { Assert.assertEquals(1, AlignedTVList.buildResultBitMapBytes(new TSStatus[8], 0, 0, 8).length); } + // Insert dense batches with empty input bitmaps and either absent or explicit success statuses. + // Neither form must allocate MemTable null bitmaps. @Test public void testEmptyInputBitmapsDoNotMaterializeMemTableBitmaps() { AlignedTVList tvList = AlignedTVListFactory.createAlignedList(List.of(TSDataType.INT64)); @@ -257,6 +272,8 @@ public class AlignedTVListTest { Assert.assertNull(tvList.getBitMaps()); } + // Insert a successful nonzero-offset Tablet range crossing an array boundary, with failures + // outside the range. Preserve selected values, leave bitmaps absent, and do not mutate statuses. @Test public void testSuccessfulTabletSubrangeWithFailuresOutsideRange() { // Failures outside the selected range must not affect successful rows across array boundaries. @@ -295,6 +312,9 @@ public class AlignedTVListTest { } } + // Append a nonzero-offset range to a populated block with both failed statuses and column nulls. + // Union their bits at the correct row offsets, preserve the old row, and leave wholly null + // subsequent column blocks unallocated. @Test public void testFailureAndNullBitmapUnionAcrossArrayBoundaries() { // Start inside a populated block: failure statuses and input nulls must be merged at the @@ -354,6 +374,9 @@ public class AlignedTVListTest { } } + // Populate one column while another remains null across blocks, then extend a new column. + // Materialize only the block receiving its first value, preserve prior implicit nulls, and + // charge new primitive arrays, lazy bitmaps, and schema container growth accurately. @Test public void testPrimitiveArraysAreAllocatedOnFirstWrite() { AlignedTVList tvList = @@ -411,6 +434,9 @@ public class AlignedTVListTest { tvList.calculateRamSize().getRamSize() - ramSizeBeforeExtendedColumnMaterialization); } + // Materialize a previously null column after crossing a block boundary. + // RAM growth must match the new array/bitmap/container costs; clones and projections must + // account consistently, and clear must leave only reusable empty-container costs. @Test public void testCalculateRamSizeCountsMaterializedPrimitiveArrays() { AlignedTVList tvList = @@ -457,6 +483,8 @@ public class AlignedTVListTest { tvList.calculateContainerRamCost(null), tvList.calculateRamSize().getRamSize()); } + // Write two blocks with one dense column and one never-materialized null column. + // Charge only existing primitive arrays plus shared block/container costs, with no bitmaps. @Test public void testCalculateRamSizeExcludesUnallocatedPrimitiveArrays() { AlignedTVList tvList = @@ -477,6 +505,8 @@ public class AlignedTVListTest { Assert.assertNull(tvList.getBitMaps()); } + // Batch-insert a block whose second column is entirely marked null. + // Allocate only the first column's array and represent the null column without a bitmap. @Test public void testBatchDoesNotAllocateAllNullPrimitiveArray() { AlignedTVList tvList = @@ -499,6 +529,9 @@ public class AlignedTVListTest { Assert.assertTrue(tvList.isNullValue(ARRAY_SIZE - 1, 1)); } + // Write all-null blocks, then append separate non-null values in each column. + // The all-value-deleted map must mark only wholly null rows, with lazy per-column bitmaps + // materialized only in the block containing actual values. @Test public void testImplicitNullBlocksAndAllValueDeletedMap() { AlignedTVList tvList = @@ -532,6 +565,9 @@ public class AlignedTVListTest { Assert.assertFalse(allValueDeletedMap.isMarked(ARRAY_SIZE + 2)); } + // Delete a range and then a whole column that has only implicit nulls. + // Report zero removed values without allocating bitmaps; deleting the populated column must + // mark its values null while leaving the already-null column's bitmap absent. @Test public void testDeletingImplicitNullColumnDoesNotMaterializeBitmaps() { AlignedTVList tvList = @@ -554,6 +590,8 @@ public class AlignedTVListTest { } } + // Write three columns with different periodic nulls and an initially unallocated column block. + // Across all blocks, the all-value-deleted map must equal the conjunction of column null states. @Test public void testAllValueDeletedMapMatchesPerColumnNullState() { AlignedTVList tvList = @@ -580,6 +618,8 @@ public class AlignedTVListTest { } } + // Create unallocated TEXT/INT64 blocks followed by a materialized value row. + // Cloning and WAL round-trip must preserve implicit null rows and the later typed values. @Test public void testNullPrimitiveArrayCanBeClonedAndSerialized() throws IOException { AlignedTVList tvList = @@ -610,6 +650,8 @@ public class AlignedTVListTest { "[value, 1]", deserializedTvList.getAlignedValue(ARRAY_SIZE + 1).toString()); } + // Clone five INT64 columns with descending timestamps and periodic null markers. + // The clone must preserve stored timestamps, aligned values, and binary-memory counters. @Test public void testClone() { List<TSDataType> dataTypes = new ArrayList<>(); @@ -651,6 +693,9 @@ public class AlignedTVListTest { } } + // Mix row and batch writes to INT32/TEXT columns, then delete values, extend columns, and clear. + // Binary-memory counters must retain allocated payload through logical deletion, track schema + // width changes, and reset on clear. @Test public void testCalculateChunkSize() { List<TSDataType> dataTypes = new ArrayList<>(); @@ -717,6 +762,9 @@ public class AlignedTVListTest { Assert.assertEquals(tvList.memoryBinaryChunkSize[1], 0); } + // Prepare a partial clone retaining column 1 in the source, including an implicit-null column. + // Preparation must leave source ownership intact; commit must move other columns to the clone + // and make the source's remaining RAM match the precomputed reservation. @Test public void testMovesUnclonedColumns() { List<TSDataType> dataTypes = new ArrayList<>(); @@ -749,6 +797,10 @@ public class AlignedTVListTest { Assert.assertEquals(retainedRamSize, tvList.calculateRamSize().getRamSize()); } + // Retain one of 256 populated columns during partial cloning. + // After excluding fixed schema containers, retained payload must be far smaller than full + // payload; + // committed source RAM must equal the retained-column estimate. @Test public void testPartialRamSizeScalesWithRetainedColumns() { int columnCount = 256; @@ -777,6 +829,8 @@ public class AlignedTVListTest { Assert.assertEquals(retainedRamSize, tvList.calculateRamSize().getRamSize()); } + // Partial-clone lists with and without sort indexes, retaining either a nullable column or time + // only. In every combination, reserved source memory must equal the post-commit cleanup cost. @Test public void testPartialReservationMatchesCleanupCalculation() { for (boolean createIndices : new boolean[] {false, true}) { @@ -816,6 +870,8 @@ public class AlignedTVListTest { } } + // Inject an incompatible array into a column so partial-clone preparation fails. + // The exception must leave source array/bitmap identities, null state, and valid values intact. @Test public void testPartialCloneFailureLeavesSourceUntouched() { AlignedTVList tvList = @@ -844,6 +900,8 @@ public class AlignedTVListTest { Assert.assertEquals(2L, tvList.getLongByValueIndex(0, 1)); } + // Release the middle column of a three-column list after its mixed nulls created a bitmap. + // Free that column's values and bitmap while preserving all values in the two retained columns. @Test public void testReleaseNonQueryColumnsWithBitmaps() { List<TSDataType> dataTypes = new ArrayList<>(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/FlatDeviceAlignedTVListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/FlatDeviceAlignedTVListTest.java index c65af55204a..21649193950 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/FlatDeviceAlignedTVListTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/FlatDeviceAlignedTVListTest.java @@ -81,6 +81,9 @@ public class FlatDeviceAlignedTVListTest { Assert.assertEquals(list.getRamSize(), list.calculateRamSize().getRamSize()); } + // Grow repeated-device row indexes using cache hits, map lookups, null TAGs, and schema edits, + // then delete FIELDs and rows. Incremental RAM accounting must match the full calculation + // through sorting, TAG/FIELD insertion, and deletion. @Test public void testIncrementalRamAccountingAcrossDeviceGrowthAndSchemaChanges() throws Exception { FlatDeviceAlignedTVList list = list(); @@ -105,6 +108,9 @@ public class FlatDeviceAlignedTVListTest { assertRamAccounting(list); } + // Write 257 rows with 100 of 500 FIELDs populated, then partial-clone with time-only, FIELD, + // or TAG retention. Reserved and calculated RAM must agree after ownership transfer, source + // clear, and query-column release, including device metadata and sparse block costs. @Test public void testSparseWideFieldRamAccountingAndOwnershipTransfer() throws Exception { List<TSDataType> types = new ArrayList<>(); @@ -144,6 +150,9 @@ public class FlatDeviceAlignedTVListTest { assertRamAccounting(list); } + // Batch-insert 129 rows with repeated and null TAGs, then clone, flush-snapshot, and WAL restore. + // Device RAM accounting must remain exact after further growth, clear, and reuse; independent + // snapshots must retain their original size. @Test public void testDeviceRamAccountingSurvivesSnapshotsWalAndReuse() throws Exception { FlatDeviceAlignedTVList list = list(); @@ -186,6 +195,9 @@ public class FlatDeviceAlignedTVListTest { assertRamAccounting(restored); } + // Insert interleaved devices with null and duplicate FIELD sort keys, then sort by device, + // FIELD, and time. Reorder device row indexes stably while keeping FIELD/time backing storage + // in insertion order and recognizing duplicate complete row keys. @Test public void testDeviceOrderAndFieldThenTimeOrder() { FlatDeviceAlignedTVList list = list(); @@ -214,6 +226,8 @@ public class FlatDeviceAlignedTVListTest { Assert.assertEquals(1, list.getValues().size()); } + // Delete the earliest and latest rows after sorting, then append a new logical device. + // Recompute live bounds as 20..30 without restoring the deleted times 1 and 100. @Test public void testAppendDoesNotRestoreDeletedTimeBounds() { FlatDeviceAlignedTVList list = list(); @@ -228,6 +242,9 @@ public class FlatDeviceAlignedTVListTest { Assert.assertEquals(30, list.getMaxTime()); } + // Delete a z-device row, append 1,025 a-device rows, sort, and WAL round-trip. + // The deleted row moves beyond the original bitmap range; preserve its deletion at sorted + // index 1,025 and compute live time bounds 2..1,026. @Test public void testWalDeletionAfterAppendMovesRowBeyondOriginalBitmap() throws Exception { FlatDeviceAlignedTVList list = list(); @@ -250,6 +267,9 @@ public class FlatDeviceAlignedTVListTest { Assert.assertEquals(1026, restored.getMaxTime()); } + // Insert a FIELD before an existing FIELD sort key, then append, sort, and transfer ownership. + // Shift logical column/sort mappings correctly, keep old rows null in the new FIELD, and + // preserve its value in the working clone after a time-only source transfer. @Test public void testInsertFieldKeepsLogicalMappingAndSortKeys() { FlatDeviceAlignedTVList list = list(); @@ -270,6 +290,9 @@ public class FlatDeviceAlignedTVListTest { Assert.assertEquals(3.5D, plan.getCloneList().getDoubleByValueIndex(1, 1), 0D); } + // Sort a flush snapshot independently, then sort and append to the original flat list. + // Device row-index arrays must not be shared; resorted source rows must include new devices + // while the snapshot retains its original three rows. @Test public void testAppendAfterSortAndIndependentFlushSnapshot() { FlatDeviceAlignedTVList list = list(); @@ -295,6 +318,8 @@ public class FlatDeviceAlignedTVListTest { Assert.assertEquals(3, snapshot.rowCount()); } + // Sort TAGs containing U+10000 and U+E000, whose UTF-16 and UTF-8 ordering differ. + // Device and row-key comparisons must use UTF-8 order, placing U+E000 first. @Test public void testUtf8DeviceOrder() { FlatDeviceAlignedTVList list = list(); @@ -305,6 +330,8 @@ public class FlatDeviceAlignedTVListTest { Assert.assertTrue(list.compareRowKey(0, list, 1) < 0); } + // WAL round-trip a sorted list with a FIELD sort key and a deleted b-device row. + // Restore TAG/FIELD sort indexes, the a-device FIELD-before-time order, and row deletion. @Test public void testWalPreservesFieldSortAndDeletion() throws Exception { FlatDeviceAlignedTVList list = list(); @@ -330,6 +357,8 @@ public class FlatDeviceAlignedTVListTest { Assert.assertEquals(1, restored.getValues().size()); } + // Partial-clone two same-time logical devices while retaining no FIELD columns in the source. + // The source must keep device identities and distinguish row keys; the clone must retain values. @Test public void testTimeOnlyOwnershipTransferRetainsDeviceKeys() { FlatDeviceAlignedTVList list = list(); @@ -344,6 +373,9 @@ public class FlatDeviceAlignedTVListTest { Assert.assertEquals(2L, plan.getCloneList().getLongByValueIndex(1, 1)); } + // Write 4,096 rows sharing four logical devices and compare with ordinary TAG-column storage. + // Store only FIELD arrays, use less RAM, and sort strictly by logical device and time while + // retrieving TAGs from device metadata. @Test public void testRepeatedDevicesUseLogicalDeviceMetadata() { FlatDeviceAlignedTVList list = list(); @@ -365,6 +397,8 @@ public class FlatDeviceAlignedTVListTest { Assert.assertEquals(4092L, list.getLongByValueIndex(list.getValueIndex(0), 1)); } + // Interleave a and b rows with descending times for b. + // Each DeviceInfo must record its exact original row indexes, row count, and min/max times. @Test public void testDeviceInfoTracksRowsAndTimeRange() { FlatDeviceAlignedTVList list = list(); @@ -386,6 +420,8 @@ public class FlatDeviceAlignedTVListTest { Assert.assertEquals(20, deviceInfoMap.get(deviceB).getEndTime()); } + // Create a flat list with physical table_name and two TAG columns, then WAL round-trip. + // Both original and restored logical device keys must retain the physical table name prefix. @Test public void testPhysicalDeviceTableNameIsKeptInLogicalDeviceKeys() throws Exception { IDeviceID physicalDevice = @@ -411,6 +447,8 @@ public class FlatDeviceAlignedTVListTest { Assert.assertTrue(recovered.getDeviceInfoMap().containsKey(logicalDevice)); } + // Insert same-time devices whose TAG is null, empty, or the literal string null. + // Keep three identities and distinct sorted row keys; only the actual null TAG is null. @Test public void testNullEmptyAndLiteralNullAreDistinct() { FlatDeviceAlignedTVList list = list(); @@ -427,6 +465,9 @@ public class FlatDeviceAlignedTVListTest { Assert.assertFalse(list.isSameRowKey(1, 2)); } + // Append a nonzero-offset Tablet range across blocks with a null TAG and a failed row status. + // Map null/failure positions correctly, preserve the other TAG for the null-TAG row, mark + // all columns null for the failed row, and retain the final selected FIELD value. @Test public void testTabletRangeNullsFailuresAndBlockBoundary() { FlatDeviceAlignedTVList list = list(); @@ -463,6 +504,9 @@ public class FlatDeviceAlignedTVListTest { Assert.assertEquals(1026L, list.getLongByValueIndex(list.rowCount() - 1, 1)); } + // Add a TAG after existing rows, then append rows with that TAG null or populated. + // Missing and null late TAGs must share the old identity, while a non-null late TAG creates + // a separate device; sorting must follow those identities and then time. @Test public void testLateTagPreservesDeviceIdentity() { FlatDeviceAlignedTVList list = list(); @@ -478,6 +522,9 @@ public class FlatDeviceAlignedTVListTest { Assert.assertEquals(2, list.getTime(2)); } + // Attempt every TAG deletion API on sorted and unsorted flat lists, including mixed FIELD/TAG + // deletion. Reject each operation, include table and TAG index in diagnostic messages, and + // leave device keys, row indexes, values, null states, and sorted state unchanged. @Test public void testTagDeletionIsRejectedWithoutChangingRows() { for (boolean sorted : new boolean[] {false, true}) { @@ -531,6 +578,8 @@ public class FlatDeviceAlignedTVListTest { } } + // Attempt TAG deletion on an empty list, outside the row time range, and on a null TAG. + // Reject the unsupported operation even when no populated TAG value would be affected. @Test public void testTagDeletionIsRejectedWithoutMatchingData() { FlatDeviceAlignedTVList list = list(); @@ -542,6 +591,9 @@ public class FlatDeviceAlignedTVListTest { () -> list.deleteMatchingRows(1, 1, row -> true, List.of(0))); } + // Delete FIELD values and row times from a sorted list, repeat deletions, then append a row. + // Count only newly deleted values/times, preserve TAG identity, and maintain live bounds and + // logical-device count when the existing device is reused. @Test public void testFieldAndRowDeletionPreserveTags() { FlatDeviceAlignedTVList list = list(); @@ -568,6 +620,8 @@ public class FlatDeviceAlignedTVListTest { Assert.assertEquals(3L, list.getLongByValueIndex(2, 1)); } + // WAL round-trip rows containing null and empty TAGs plus a deleted FIELD, before and after sort. + // Both forms must preserve row data, logical-device identities, and null/deletion state. @Test public void testWalBeforeAndAfterSort() throws Exception { FlatDeviceAlignedTVList list = list(); @@ -608,6 +662,9 @@ public class FlatDeviceAlignedTVListTest { } } + // Clone and transfer a flat list while retaining a TAG, a FIELD, or time only in the source. + // Appending to the working clone and clearing the source must leave both full and working + // clones' device metadata and TAG values intact. @Test public void testCloneAndPartialOwnershipTransfer() { for (Set<Integer> retained : List.of(Set.of(0), Set.of(1), Set.<Integer>of())) { @@ -626,6 +683,9 @@ public class FlatDeviceAlignedTVListTest { } } + // Mutate the caller's TAG bytes after insertion, then clear and reuse the flat list. + // Stored identity must be independent of those bytes; clear must release device metadata + // back to baseline RAM and allow insertion of a new device. @Test public void testClearReleasesDictionaryAndReusesList() { FlatDeviceAlignedTVList list = list(); @@ -641,6 +701,9 @@ public class FlatDeviceAlignedTVListTest { Assert.assertEquals(binary("new"), list.getBinaryByValueIndex(0, 0)); } + // Sort an independent flush snapshot and query a reordered TAG/FIELD projection with duplicates. + // Keep the source order intact, resolve projected TAGs/nulls correctly, and return two rows + // with the latest non-null FIELD for the duplicate b key. @Test public void testQueryProjectionAndFlushSortSnapshot() { FlatDeviceAlignedTVList source = list(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/FlatDeviceEncoderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/FlatDeviceEncoderTest.java index 7a6f3d4d4f2..e74a27aafac 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/FlatDeviceEncoderTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/FlatDeviceEncoderTest.java @@ -63,6 +63,10 @@ public class FlatDeviceEncoderTest { types, tags, IDeviceID.Factory.DEFAULT_FACTORY.create("table")); } + // Encode three historical/current lists with duplicate keys, per-FIELD null updates, deleted + // rows, and time resets between devices. Merge latest non-null values, retain older values + // when a newer row is deleted, and emit five TAG-first rows across two chunks and size-limited + // pages. @Test public void testDeviceMergeLatestNonNullAndPageChunkBoundaries() throws Exception { List<TSDataType> types = List.of(TSDataType.STRING, TSDataType.INT64, TSDataType.INT64); @@ -98,6 +102,9 @@ public class FlatDeviceEncoderTest { assertRow(rows.get(4), 3, "c", 30L, 300L); } + // Encode single-list and multi-list variants with duplicate updates, an all-null row, and a + // fully deleted device. Count exactly two emitted rows and two devices, excluding the deleted + // device while merging FIELD values independently. @Test public void testStatisticsCountEncodedRowsAfterDuplicateMergeAndDeletion() throws Exception { List<TSDataType> types = List.of(TSDataType.STRING, TSDataType.INT64, TSDataType.INT64); @@ -138,6 +145,9 @@ public class FlatDeviceEncoderTest { } } + // Exercise single-list encoding with repeated null updates, deleted winning duplicates, a + // deleted leading row/device, and all-null rows. Emit five rows using latest surviving non-null + // FIELDs and compute correct statistics across three-row chunks and two-row pages. @Test public void testSingleListLatestNonNullDeletionAndPageChunkBoundaries() throws Exception { List<TSDataType> types = List.of(TSDataType.STRING, TSDataType.INT64, TSDataType.INT64); @@ -177,6 +187,9 @@ public class FlatDeviceEncoderTest { assertRow(rows.get(4), 1, "c", 30L, 300L); } + // Encode duplicates distinguished by a FIELD sort key omitted from the output projection. + // Preserve that key distinction, merge latest non-null values, map reordered TAGs correctly, + // and emit null for a projected FIELD absent from the source. @Test public void testSingleListDuplicateFieldKeysWithColumnProjection() throws Exception { List<TSDataType> types = @@ -214,6 +227,9 @@ public class FlatDeviceEncoderTest { assertRow(rows.get(2), 1, "b", "x", 30L, "b", null); } + // Merge old and new schemas after adding a TAG and FIELD, including a key-only all-null row. + // Treat missing late TAGs as null, combine matching old/new keys, and preserve the distinct + // non-null late-TAG device and missing FIELD values. @Test public void testLateTagMissingFieldAndDeviceKeyOnly() throws Exception { FlatDeviceAlignedTVList old = list(List.of(TSDataType.STRING, TSDataType.INT64), List.of(0)); @@ -233,6 +249,8 @@ public class FlatDeviceEncoderTest { assertRow(rows.get(3), 2, "b", 20L, null, null); } + // Encode one list ordered by device, FIELD sort key, and time across all supported value types. + // Preserve null/typed FIELDs and four row keys in one page despite regressing timestamps. @Test public void testFieldSortThenTimeAndAllValueTypes() throws Exception { List<TSDataType> types = @@ -265,6 +283,9 @@ public class FlatDeviceEncoderTest { assertRow(rows.get(3), 3, "b", 0, true, 0.5F, 1.5D, 2L, "x", "u"); } + // Encode time-only duplicates, TAG-only rows with regressing time, fully deleted rows, and + // an empty list collection. Deduplicate time-only keys, expand TAG-only chunks without FIELD + // storage, and emit no chunks for deleted or empty input. @Test public void testTimeOnlyTagOnlyAndEmptyLists() throws Exception { FlatDeviceAlignedTVList timeOnly = list(List.of(), List.of()); diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/TsFileTableSchemaUtilTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/TsFileTableSchemaUtilTest.java index 237bae5bd76..0ab5ee5859b 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/TsFileTableSchemaUtilTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/TsFileTableSchemaUtilTest.java @@ -53,6 +53,8 @@ public class TsFileTableSchemaUtilTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + // Write flat and ordinary table layout markers to a TsFile footer and read them back. + // Preserve the one-byte true/false values and report an unmarked table as ordinary. @Test public void testFlatDevicePropertiesRoundTrip() throws IOException { final File file = temporaryFolder.newFile("table-properties.tsfile"); @@ -76,12 +78,16 @@ public class TsFileTableSchemaUtilTest { } } + // Inspect null or empty footer properties for a table layout marker. + // Both absent-property forms must default to ordinary layout. @Test public void testMissingFlatDevicePropertyDefaultsToFalse() { Assert.assertFalse(TsFileTableSchemaUtil.isFlatDevice(null, "table")); Assert.assertFalse(TsFileTableSchemaUtil.isFlatDevice(new HashMap<>(), "table")); } + // Write layout and device/row statistics for table names resembling property suffixes. + // Keep all three keys per table distinct and round-trip each marker and count without collisions. @Test public void testFlatDeviceStatisticsDoNotShadowTableMarkers() throws IOException { File file = temporaryFolder.newFile("collision-properties.tsfile"); @@ -116,6 +122,9 @@ public class TsFileTableSchemaUtilTest { } } + // Inspect legacy flat_device.t properties and a potentially colliding layout.t table name. + // Accept only the valid one-byte legacy marker; reject a long-sized value and avoid treating + // another table's legacy marker as t's layout flag. @Test public void testLegacyFlatDeviceMarker() { Assert.assertTrue( @@ -129,6 +138,8 @@ public class TsFileTableSchemaUtilTest { Map.of("flat_device.t", new byte[] {0}, "flat_device.layout.t", new byte[] {1}), "t")); } + // Read a layout property containing null, empty, unknown, or multi-byte values. + // Every malformed marker must default to ordinary layout. @Test public void testInvalidFlatDevicePropertyDefaultsToFalse() { final Map<String, byte[]> properties = new HashMap<>(); @@ -139,6 +150,8 @@ public class TsFileTableSchemaUtilTest { } } + // Convert a table containing TIME, TAG, ATTRIBUTE, and FIELD columns to a TsFile schema. + // Keep only TAG/FIELD columns in order with correct names, types, categories, and indexes. @Test public void testToTsFileTableSchemaNoAttribute() { // Create a TsTable with all column types diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/column/TagColumnSchemaTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/column/TagColumnSchemaTest.java index e8f37c6e084..16e3629eebd 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/column/TagColumnSchemaTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/column/TagColumnSchemaTest.java @@ -29,6 +29,8 @@ import java.util.Map; public class TagColumnSchemaTest { + // Create a TAG measurement schema with custom properties. + // Use DICTIONARY encoding immediately while preserving the TAG name, type, and properties. @Test public void testMeasurementSchemaUsesDictionaryEncodingFromCreation() { Map<String, String> props = Map.of("key", "value");
