This is an automated email from the ASF dual-hosted git repository. marklau99 pushed a commit to branch IOTDB-4791 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f542692ec7b24203e02a46336c015ea42aa7fb9f Author: Liu Xuxin <[email protected]> AuthorDate: Fri Oct 28 18:54:22 2022 +0800 try to accelrate TsFileIOWriter.endFile() --- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 21 ++++++--------------- .../write/writer/tsmiterator/DiskTSMIterator.java | 7 ++++--- .../write/writer/tsmiterator/TSMIterator.java | 4 ++-- .../writer/TsFileIOWriterMemoryControlTest.java | 11 ++++++----- 4 files changed, 18 insertions(+), 25 deletions(-) diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java index a8de080e0e..cebef59af5 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java @@ -111,7 +111,6 @@ public class TsFileIOWriter implements AutoCloseable { protected File chunkMetadataTempFile; protected LocalTsFileOutput tempOutput; protected volatile boolean hasChunkMetadataInDisk = false; - protected String currentSeries = null; // record the total num of path in order to make bloom filter protected int pathCount = 0; protected boolean enableMemoryControl = false; @@ -349,6 +348,7 @@ public class TsFileIOWriter implements AutoCloseable { Queue<MetadataIndexNode> measurementMetadataIndexQueue = new ArrayDeque<>(); String currentDevice = null; String prevDevice = null; + Path currentPath = null; MetadataIndexNode currentIndexNode = new MetadataIndexNode(MetadataIndexNodeType.LEAF_MEASUREMENT); TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); @@ -361,25 +361,16 @@ public class TsFileIOWriter implements AutoCloseable { while (tsmIterator.hasNext()) { // read in all chunk metadata of one series // construct the timeseries metadata for this series - Pair<String, TimeseriesMetadata> timeseriesMetadataPair = tsmIterator.next(); + Pair<Path, TimeseriesMetadata> timeseriesMetadataPair = tsmIterator.next(); TimeseriesMetadata timeseriesMetadata = timeseriesMetadataPair.right; - currentSeries = timeseriesMetadataPair.left; + currentPath = timeseriesMetadataPair.left; indexCount++; // build bloom filter - filter.add(currentSeries); + filter.add(currentPath.getFullPath()); // construct the index tree node for the series - Path currentPath = null; - if (timeseriesMetadata.getTSDataType() == TSDataType.VECTOR) { - // this series is the time column of the aligned device - // the full series path will be like "root.sg.d." - // we remove the last . in the series id here - currentPath = new Path(currentSeries); - currentDevice = currentSeries.substring(0, currentSeries.length() - 1); - } else { - currentPath = new Path(currentSeries, true); - currentDevice = currentPath.getDevice(); - } + + currentDevice = currentPath.getDevice(); if (!currentDevice.equals(prevDevice)) { if (prevDevice != null) { addCurrentIndexNodeToQueue(currentIndexNode, measurementMetadataIndexQueue, out); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/DiskTSMIterator.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/DiskTSMIterator.java index fd02f1438a..2ed2b57f08 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/DiskTSMIterator.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/DiskTSMIterator.java @@ -24,6 +24,7 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.read.reader.LocalTsFileInput; import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; @@ -75,7 +76,7 @@ public class DiskTSMIterator extends TSMIterator { } @Override - public Pair<String, TimeseriesMetadata> next() { + public Pair<Path, TimeseriesMetadata> next() { try { if (remainsInFile) { // deserialize from file @@ -90,7 +91,7 @@ public class DiskTSMIterator extends TSMIterator { } } - private Pair<String, TimeseriesMetadata> getTimeSerisMetadataFromFile() throws IOException { + private Pair<Path, TimeseriesMetadata> getTimeSerisMetadataFromFile() throws IOException { if (currentPos == nextEndPosForDevice) { // deserialize the current device name currentDevice = ReadWriteIOUtils.readString(input.wrapAsInputStream()); @@ -118,7 +119,7 @@ public class DiskTSMIterator extends TSMIterator { } updateCurrentPos(); return new Pair<>( - currentDevice + "." + measurementUid, + new Path(currentDevice, measurementUid), constructOneTimeseriesMetadata(measurementUid, chunkMetadataList)); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/TSMIterator.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/TSMIterator.java index f11242f296..38425044ac 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/TSMIterator.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/TSMIterator.java @@ -70,10 +70,10 @@ public class TSMIterator { return iterator.hasNext(); } - public Pair<String, TimeseriesMetadata> next() throws IOException { + public Pair<Path, TimeseriesMetadata> next() throws IOException { Pair<Path, List<IChunkMetadata>> nextPair = iterator.next(); return new Pair<>( - nextPair.left.getFullPath(), + nextPair.left, constructOneTimeseriesMetadata(nextPair.left.getMeasurement(), nextPair.right)); } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriterMemoryControlTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriterMemoryControlTest.java index b7c6ff84db..2152318cc8 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriterMemoryControlTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriterMemoryControlTest.java @@ -28,6 +28,7 @@ import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; @@ -132,7 +133,7 @@ public class TsFileIOWriterMemoryControlTest { writer.chunkGroupMetadataList, writer.endPosInCMTForDevice); for (int i = 0; iterator.hasNext(); ++i) { - Pair<String, TimeseriesMetadata> timeseriesMetadataPair = iterator.next(); + Pair<Path, TimeseriesMetadata> timeseriesMetadataPair = iterator.next(); TimeseriesMetadata timeseriesMetadata = timeseriesMetadataPair.right; Assert.assertEquals(sortedSeriesId.get(i % 5), timeseriesMetadata.getMeasurementId()); Assert.assertEquals( @@ -169,8 +170,8 @@ public class TsFileIOWriterMemoryControlTest { TSMIterator.getTSMIteratorInDisk( writer.chunkMetadataTempFile, new ArrayList<>(), writer.endPosInCMTForDevice); for (int i = 0; iterator.hasNext(); ++i) { - Pair<String, TimeseriesMetadata> timeseriesMetadataPair = iterator.next(); - String fullPath = timeseriesMetadataPair.left; + Pair<Path, TimeseriesMetadata> timeseriesMetadataPair = iterator.next(); + String fullPath = timeseriesMetadataPair.left.getFullPath(); TimeseriesMetadata timeseriesMetadata = timeseriesMetadataPair.right; Assert.assertEquals(measurementIds.get(i), fullPath); Assert.assertEquals( @@ -233,8 +234,8 @@ public class TsFileIOWriterMemoryControlTest { TSMIterator.getTSMIteratorInDisk( writer.chunkMetadataTempFile, new ArrayList<>(), writer.endPosInCMTForDevice); for (int i = 0; i < originChunkMetadataList.size(); ++i) { - Pair<String, TimeseriesMetadata> timeseriesMetadataPair = iterator.next(); - Assert.assertEquals(seriesIds.get(i), timeseriesMetadataPair.left); + Pair<Path, TimeseriesMetadata> timeseriesMetadataPair = iterator.next(); + Assert.assertEquals(seriesIds.get(i), timeseriesMetadataPair.left.getFullPath()); Assert.assertEquals( originChunkMetadataList.get(i).getDataType(), timeseriesMetadataPair.right.getTSDataType());
