This is an automated email from the ASF dual-hosted git repository. sunzesong pushed a commit to branch jira_2217 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7a17553cbeb43b11f347b7d951e9db72b6318f59 Author: Zesong Sun <[email protected]> AuthorDate: Tue Dec 28 12:40:16 2021 +0800 Fix to iterator --- .../iotdb/tsfile/read/TsFileSequenceReader.java | 83 +++++++++++++++------- .../tsfile/write/MetadataIndexConstructorTest.java | 16 ++--- 2 files changed, 66 insertions(+), 33 deletions(-) diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java index 59b777a..fd34be0 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java @@ -649,19 +649,32 @@ public class TsFileSequenceReader implements AutoCloseable { } /** - * this function return all timeseries names in dictionary order + * this function return all timeseries names * * @return list of Paths * @throws IOException io error */ public List<Path> getAllPaths() throws IOException { - if (tsFileMetaData == null) { - readFileMetadata(); - } List<Path> paths = new ArrayList<>(); + for (String device : getAllDevices()) { + Map<String, TimeseriesMetadata> timeseriesMetadataMap = readDeviceMetadata(device); + for (String measurementId : timeseriesMetadataMap.keySet()) { + paths.add(new Path(device, measurementId)); + } + } + return paths; + } + + /** + * @return an iterator of timeseries list, in which names of timeseries are ordered in dictionary order + * @throws IOException io error + */ + public Iterator<List<Path>> getPathsIterator() throws IOException { + readFileMetadata(); MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex(); List<MetadataIndexEntry> metadataIndexEntryList = metadataIndexNode.getChildren(); + Queue<Pair<String, Pair<Long, Long>>> queue = new LinkedList<>(); for (int i = 0; i < metadataIndexEntryList.size(); i++) { MetadataIndexEntry metadataIndexEntry = metadataIndexEntryList.get(i); long endOffset = tsFileMetaData.getMetadataIndex().getEndOffset(); @@ -674,10 +687,37 @@ public class TsFileSequenceReader implements AutoCloseable { buffer, null, metadataIndexNode.getNodeType(), - paths, - false); + queue); } - return paths; + return new Iterator<List<Path>>() { + @Override + public boolean hasNext() { + return !queue.isEmpty(); + } + + @Override + public List<Path> next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + Pair<String, Pair<Long, Long>> startEndPair = queue.remove(); + List<Path> paths = new ArrayList<>(); + try { + List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>(); + ByteBuffer nextBuffer = readData(startEndPair.right.left, startEndPair.right.right); + while (nextBuffer.hasRemaining()) { + timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(nextBuffer, false)); + } + for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) { + paths.add(new Path(startEndPair.left, timeseriesMetadata.getMeasurementId())); + } + return paths; + } catch (IOException e) { + throw new TsFileRuntimeException( + "Error occurred while reading a time series metadata block."); + } + } + }; } private void getAllPaths( @@ -685,30 +725,24 @@ public class TsFileSequenceReader implements AutoCloseable { ByteBuffer buffer, String deviceId, MetadataIndexNodeType type, - List<Path> timeseriesMetadataMap, - boolean needChunkMetadata) + Queue<Pair<String, Pair<Long, Long>>> queue) throws IOException { try { - if (type.equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) { - List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>(); - while (buffer.hasRemaining()) { - timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer, needChunkMetadata)); - } - for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) { - timeseriesMetadataMap.add(new Path(deviceId, timeseriesMetadata.getMeasurementId())); - } - } else { - // deviceId should be determined by LEAF_DEVICE node - if (type.equals(MetadataIndexNodeType.LEAF_DEVICE)) { - deviceId = metadataIndex.getName(); - } + if (type.equals(MetadataIndexNodeType.LEAF_DEVICE)) { + deviceId = metadataIndex.getName(); + } MetadataIndexNode metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer); int metadataIndexListSize = metadataIndexNode.getChildren().size(); for (int i = 0; i < metadataIndexListSize; i++) { + long startOffset = metadataIndexNode.getChildren().get(i).getOffset(); long endOffset = metadataIndexNode.getEndOffset(); if (i != metadataIndexListSize - 1) { endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset(); } + if (metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) { + queue.add(new Pair<>(deviceId, new Pair<>(startOffset, endOffset))); + continue; + } ByteBuffer nextBuffer = readData(metadataIndexNode.getChildren().get(i).getOffset(), endOffset); getAllPaths( @@ -716,9 +750,8 @@ public class TsFileSequenceReader implements AutoCloseable { nextBuffer, deviceId, metadataIndexNode.getNodeType(), - timeseriesMetadataMap, - needChunkMetadata); - } + queue); + } } catch (BufferOverflowException e) { logger.error("Something error happened while getting all paths of file {}", file); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java index 5d33a84..1922445 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java @@ -49,11 +49,7 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -240,9 +236,13 @@ public class MetadataIndexConstructorTest { } } try (TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH)) { - List<Path> actualPaths = reader.getAllPaths(); - for (int i = 0; i < actualPaths.size(); i++) { - assertEquals(actualPaths.get(i).getFullPath(), correctPaths.get(i)); + Iterator<List<Path>> iterator = reader.getPathsIterator(); + int idx = 0; + while (iterator.hasNext()) { + for (Path actualPath : iterator.next()) { + assertEquals(actualPath.getFullPath(), correctPaths.get(idx)); + idx++; + } } } catch (IOException e) { e.printStackTrace();
