This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch jira1330_12 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e8c0c68f788c86d416a18c07d9a71eb57f43519e Author: HouliangQi <[email protected]> AuthorDate: Thu Apr 29 16:03:57 2021 +0800 [IOTDB-1330]fix the load tsfile bug when the cross multi partition's tsfile only have one page (#3080) --- .../apache/iotdb/db/tools/TsFileRewriteTool.java | 15 +++--- .../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 16 ++++++ ...oTDBLoadExternalTsFileWithTimePartitionIT.java} | 44 ++++++++++++----- .../iotdb/db/utils/TsFileRewriteToolTest.java | 57 ++++++++++++++++++++++ 4 files changed, 111 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileRewriteTool.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileRewriteTool.java index 1cfd792..842fd00 100644 --- a/server/src/main/java/org/apache/iotdb/db/tools/TsFileRewriteTool.java +++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileRewriteTool.java @@ -270,17 +270,16 @@ public class TsFileRewriteTool implements AutoCloseable { } /** - * Due to TsFile version-3 changed the serialize way of integer in TEXT data and INT32 data with - * PLAIN encoding, and also add a sum statistic for BOOLEAN data, these types of data need to - * decode to points and rewrite in new TsFile. + * If the page have no statistics or crosses multi partitions, will return true, otherwise return + * false. */ protected boolean checkIfNeedToDecode( TSDataType dataType, TSEncoding encoding, PageHeader pageHeader) { - return dataType == TSDataType.BOOLEAN - || dataType == TSDataType.TEXT - || (dataType == TSDataType.INT32 && encoding == TSEncoding.PLAIN) - || StorageEngine.getTimePartition(pageHeader.getStartTime()) - != StorageEngine.getTimePartition(pageHeader.getEndTime()); + if (pageHeader.getStatistics() == null) { + return true; + } + return StorageEngine.getTimePartition(pageHeader.getStartTime()) + != StorageEngine.getTimePartition(pageHeader.getEndTime()); } /** diff --git a/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java b/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java index 8549858..a876044 100644 --- a/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java +++ b/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.db.tools.upgrade; +import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.modification.Deletion; import org.apache.iotdb.db.engine.modification.ModificationFile; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -237,6 +238,21 @@ public class TsFileOnlineUpgradeTool extends TsFileRewriteTool { return name[0] + "-0" + TsFileConstant.TSFILE_SUFFIX; } + /** + * Due to TsFile version-3 changed the serialize way of integer in TEXT data and INT32 data with + * PLAIN encoding, and also add a sum statistic for BOOLEAN data, these types of data need to + * decode to points and rewrite in new TsFile. + */ + @Override + protected boolean checkIfNeedToDecode( + TSDataType dataType, TSEncoding encoding, PageHeader pageHeader) { + return dataType == TSDataType.BOOLEAN + || dataType == TSDataType.TEXT + || (dataType == TSDataType.INT32 && encoding == TSEncoding.PLAIN) + || StorageEngine.getTimePartition(pageHeader.getStartTime()) + != StorageEngine.getTimePartition(pageHeader.getEndTime()); + } + @Override protected void decodeAndWritePageInToFiles( MeasurementSchema schema, diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileWithTimePartitionIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsFileWithTimePartitionIT.java similarity index 82% rename from server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileWithTimePartitionIT.java rename to server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsFileWithTimePartitionIT.java index 5de4c9e..d991123 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileWithTimePartitionIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsFileWithTimePartitionIT.java @@ -52,7 +52,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; -public class IoTDBLoadExternalTsfileWithTimePartitionIT { +public class IoTDBLoadExternalTsFileWithTimePartitionIT { String DOT = "."; String tempDir = "temp"; @@ -61,12 +61,16 @@ public class IoTDBLoadExternalTsfileWithTimePartitionIT { String[] devices = new String[] {"d1", "d2"}; String[] measurements = new String[] {"s1", "s2"}; - // generate several tsfiles, with timestamp from startTime(inclusive) to endTime(exclusive) + // generate several tsFiles, with timestamp from startTime(inclusive) to endTime(exclusive) long startTime = 0; long endTime = 1000_000; long timePartition = 100; // unit s + long recordTimeGap = 1000; + + int originalTsFileNum = 0; + boolean originalIsEnablePartition; long originalPartitionInterval; @@ -134,7 +138,7 @@ public class IoTDBLoadExternalTsfileWithTimePartitionIT { } /** create multiple tsfiles, each correspond to a time partition. */ - private void prepareData() throws IOException { + private void prepareData() { File dir = new File(tempDir); if (dir.exists()) { FileUtils.deleteDirectory(dir); @@ -144,8 +148,8 @@ public class IoTDBLoadExternalTsfileWithTimePartitionIT { File f; TsFileWriter tsFileWriter = null; int counter = 0; - for (long timestamp = startTime; timestamp < endTime; timestamp += 1000) { - if (timestamp % (timePartition * 1000) == 0) { + for (long timestamp = startTime; timestamp < endTime; timestamp += recordTimeGap) { + if (timestamp % (timePartition * recordTimeGap) == 0) { if (tsFileWriter != null) { tsFileWriter.flushAllChunkGroups(); tsFileWriter.close(); @@ -160,13 +164,14 @@ public class IoTDBLoadExternalTsfileWithTimePartitionIT { } tsFileWriter.flushAllChunkGroups(); tsFileWriter.close(); + originalTsFileNum = counter + 1; } catch (Throwable e) { e.printStackTrace(); } } @Test - public void loadTsfileWithTimePartition() { + public void loadTsFileWithTimePartition() { try (Connection connection = DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); Statement statement = connection.createStatement()) { @@ -174,16 +179,29 @@ public class IoTDBLoadExternalTsfileWithTimePartitionIT { statement.execute(String.format("load \"%s\"", new File(tempDir).getAbsolutePath())); String dataDir = config.getDataDirs()[0]; - File f = new File(dataDir, new PartialPath("sequence") + File.separator + "root.ln"); - System.out.println(Arrays.toString(f.list())); - Assert.assertEquals((endTime - startTime) / (timePartition * 1000), f.list().length); + // sequence/logical_sg/virtual_sg/time_partitions + File f = + new File( + dataDir, + new PartialPath("sequence") + File.separator + "root.ln" + File.separator + "0"); + Assert.assertEquals( + (endTime - startTime) / (timePartition), f.list().length * originalTsFileNum); + + int totalPartitionsNum = (int) ((endTime - startTime) / (timePartition) / originalTsFileNum); + int[] splitTsFilePartitions = new int[totalPartitionsNum]; + for (int i = 0; i < splitTsFilePartitions.length; i++) { + splitTsFilePartitions[i] = Integer.parseInt(f.list()[i]); + } + Arrays.sort(splitTsFilePartitions); - for (int i = 0; i < (endTime - startTime) / (timePartition * 1000); i++) { - Assert.assertEquals("" + i, f.list()[i]); + for (int i = 0; i < (endTime - startTime) / (timePartition) / originalTsFileNum; i++) { + Assert.assertEquals((i * originalTsFileNum), splitTsFilePartitions[i]); } + // each time partition folder should contain 2 files, the tsfile and the resource file - for (int i = 0; i < (endTime - startTime) / (timePartition * 1000); i++) { - Assert.assertEquals(2, new File(f.getAbsolutePath(), "" + i).list().length); + for (int i = 0; i < (endTime - startTime) / (timePartition) / originalTsFileNum; i++) { + Assert.assertEquals( + 2, new File(f.getAbsolutePath(), "" + i * originalTsFileNum).list().length); } } catch (SQLException | IllegalPathException throwables) { throwables.printStackTrace(); diff --git a/server/src/test/java/org/apache/iotdb/db/utils/TsFileRewriteToolTest.java b/server/src/test/java/org/apache/iotdb/db/utils/TsFileRewriteToolTest.java index 4cba60f..58126ef 100644 --- a/server/src/test/java/org/apache/iotdb/db/utils/TsFileRewriteToolTest.java +++ b/server/src/test/java/org/apache/iotdb/db/utils/TsFileRewriteToolTest.java @@ -181,6 +181,22 @@ public class TsFileRewriteToolTest { } } + @Test + public void loadFileWithOnlyOnePageTest() { + HashMap<String, List<String>> deviceSensorsMap = new HashMap<>(); + List<String> sensors = new ArrayList<>(); + sensors.add(SENSOR1); + deviceSensorsMap.put(DEVICE1, sensors); + createOneTsFileWithOnlyOnePage(deviceSensorsMap); + // try load the tsfile + String sql = "load \"" + path + "\"" + " true"; + try { + queryExecutor.processNonQuery(processor.parseSQLToPhysicalPlan(sql)); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + } + private void splitFileAndQueryCheck(HashMap<String, List<String>> deviceSensorsMap) { File tsFile = new File(path); TsFileResource tsFileResource = new TsFileResource(tsFile); @@ -203,6 +219,47 @@ public class TsFileRewriteToolTest { } } + private void createOneTsFileWithOnlyOnePage(HashMap<String, List<String>> deviceSensorsMap) { + try { + File f = FSFactoryProducer.getFSFactory().getFile(path); + TsFileWriter tsFileWriter = new TsFileWriter(f); + // add measurements into file schema + try { + for (Map.Entry<String, List<String>> entry : deviceSensorsMap.entrySet()) { + String device = entry.getKey(); + for (String sensor : entry.getValue()) { + tsFileWriter.registerTimeseries( + new Path(device, sensor), + new MeasurementSchema(sensor, TSDataType.INT64, TSEncoding.RLE)); + } + } + } catch (WriteProcessException e) { + Assert.fail(e.getMessage()); + } + + int count = 0; + for (long timestamp = 1; ; timestamp += newPartitionInterval) { + if (count == 2) { + break; + } + count++; + for (Map.Entry<String, List<String>> entry : deviceSensorsMap.entrySet()) { + String device = entry.getKey(); + TSRecord tsRecord = new TSRecord(timestamp, device); + for (String sensor : entry.getValue()) { + DataPoint dataPoint = new LongDataPoint(sensor, timestamp + VALUE_OFFSET); + tsRecord.addTuple(dataPoint); + } + tsFileWriter.write(tsRecord); + } + tsFileWriter.flushAllChunkGroups(); + } + tsFileWriter.close(); + } catch (Throwable e) { + Assert.fail(e.getMessage()); + } + } + private void createOneTsFile(HashMap<String, List<String>> deviceSensorsMap) { try { File f = FSFactoryProducer.getFSFactory().getFile(path);
