This is an automated email from the ASF dual-hosted git repository. sunzesong pushed a commit to branch jira_2217 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9ab5e8857053b819d0bd1fabb910be176b50d82b Author: Zesong Sun <[email protected]> AuthorDate: Tue Dec 28 11:09:48 2021 +0800 Add iterator to return timeseries Path in dictionary order --- .../iotdb/tsfile/read/TsFileSequenceReader.java | 71 ++++++++++++++++++++-- .../tsfile/write/MetadataIndexConstructorTest.java | 23 ++++++- 2 files changed, 86 insertions(+), 8 deletions(-) diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java index bfb8e31..59b777a 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java @@ -649,22 +649,83 @@ public class TsFileSequenceReader implements AutoCloseable { } /** - * this function return all timeseries names in this file + * this function return all timeseries names in dictionary order * * @return list of Paths * @throws IOException io error */ public List<Path> getAllPaths() throws IOException { + if (tsFileMetaData == null) { + readFileMetadata(); + } List<Path> paths = new ArrayList<>(); - for (String device : getAllDevices()) { - Map<String, TimeseriesMetadata> timeseriesMetadataMap = readDeviceMetadata(device); - for (String measurementId : timeseriesMetadataMap.keySet()) { - paths.add(new Path(device, measurementId)); + + MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex(); + List<MetadataIndexEntry> metadataIndexEntryList = metadataIndexNode.getChildren(); + for (int i = 0; i < metadataIndexEntryList.size(); i++) { + MetadataIndexEntry metadataIndexEntry = metadataIndexEntryList.get(i); + long endOffset = tsFileMetaData.getMetadataIndex().getEndOffset(); + if (i != metadataIndexEntryList.size() - 1) { + endOffset = metadataIndexEntryList.get(i + 1).getOffset(); } + ByteBuffer buffer = readData(metadataIndexEntry.getOffset(), endOffset); + getAllPaths( + metadataIndexEntry, + buffer, + null, + metadataIndexNode.getNodeType(), + paths, + false); } return paths; } + private void getAllPaths( + MetadataIndexEntry metadataIndex, + ByteBuffer buffer, + String deviceId, + MetadataIndexNodeType type, + List<Path> timeseriesMetadataMap, + boolean needChunkMetadata) + throws IOException { + try { + if (type.equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) { + List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>(); + while (buffer.hasRemaining()) { + timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer, needChunkMetadata)); + } + for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) { + timeseriesMetadataMap.add(new Path(deviceId, timeseriesMetadata.getMeasurementId())); + } + } else { + // deviceId should be determined by LEAF_DEVICE node + if (type.equals(MetadataIndexNodeType.LEAF_DEVICE)) { + deviceId = metadataIndex.getName(); + } + MetadataIndexNode metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer); + int metadataIndexListSize = metadataIndexNode.getChildren().size(); + for (int i = 0; i < metadataIndexListSize; i++) { + long endOffset = metadataIndexNode.getEndOffset(); + if (i != metadataIndexListSize - 1) { + endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset(); + } + ByteBuffer nextBuffer = + readData(metadataIndexNode.getChildren().get(i).getOffset(), endOffset); + getAllPaths( + metadataIndexNode.getChildren().get(i), + nextBuffer, + deviceId, + metadataIndexNode.getNodeType(), + timeseriesMetadataMap, + needChunkMetadata); + } + } + } catch (BufferOverflowException e) { + logger.error("Something error happened while getting all paths of file {}", file); + throw e; + } + } + private TimeseriesMetadata tryToGetFirstTimeseriesMetadata(MetadataIndexNode measurementNode) throws IOException { // Not aligned timeseries diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java index ee73f8f..5d33a84 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/MetadataIndexConstructorTest.java @@ -151,7 +151,7 @@ public class MetadataIndexConstructorTest { int[][] vectorMeasurement = new int[deviceNum][]; String[][] singleMeasurement = new String[deviceNum][]; for (int i = 0; i < deviceNum; i++) { - devices[i] = "d" + i; + devices[i] = "d" + generateIndexString(i, deviceNum); vectorMeasurement[i] = new int[0]; singleMeasurement[i] = new String[measurementNum]; for (int j = 0; j < measurementNum; j++) { @@ -204,8 +204,9 @@ public class MetadataIndexConstructorTest { List<String> correctDevices = new ArrayList<>(); // contains all device by sequence List<List<String>> correctFirstMeasurements = new ArrayList<>(); // contains first measurements of every leaf, group by device + List<String> correctPaths = new ArrayList<>(); // contains all paths by sequence generateCorrectResult( - correctDevices, correctFirstMeasurements, devices, vectorMeasurement, singleMeasurement); + correctDevices, correctFirstMeasurements, correctPaths, devices, vectorMeasurement, singleMeasurement); // 4. compare correct result with TsFile's metadata Arrays.sort(devices); // 4.1 make sure device in order @@ -229,6 +230,7 @@ public class MetadataIndexConstructorTest { e.printStackTrace(); fail(e.getMessage()); } + // 4.3 make sure split leaf correctly for (int j = 0; j < actualDevices.size(); j++) { for (int i = 0; i < actualMeasurements.get(j).size(); i++) { @@ -237,6 +239,15 @@ public class MetadataIndexConstructorTest { correctFirstMeasurements.get(j).get(i * conf.getMaxDegreeOfIndexNode())); } } + try (TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH)) { + List<Path> actualPaths = reader.getAllPaths(); + for (int i = 0; i < actualPaths.size(); i++) { + assertEquals(actualPaths.get(i).getFullPath(), correctPaths.get(i)); + } + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } } /** @@ -331,6 +342,7 @@ public class MetadataIndexConstructorTest { private void generateCorrectResult( List<String> correctDevices, List<List<String>> correctMeasurements, + List<String> correctPaths, String[] devices, int[][] vectorMeasurement, String[][] singleMeasurement) { @@ -341,16 +353,21 @@ public class MetadataIndexConstructorTest { List<String> measurements = new ArrayList<>(); // single-variable measurement if (singleMeasurement != null) { - measurements.addAll(Arrays.asList(singleMeasurement[i])); + for (String measurement : singleMeasurement[i]) { + measurements.add(measurement); + correctPaths.add(new Path(device, measurement).getFullPath()); + } } // multi-variable measurement for (int vectorIndex = 0; vectorIndex < vectorMeasurement[i].length; vectorIndex++) { measurements.add(""); + correctPaths.add(new Path(device, "").getFullPath()); int measurementNum = vectorMeasurement[i][vectorIndex]; for (int measurementIndex = 0; measurementIndex < measurementNum; measurementIndex++) { String measurementName = measurementPrefix + generateIndexString(measurementIndex, measurementNum); measurements.add(TsFileConstant.PATH_SEPARATOR + measurementName); + correctPaths.add(new Path(device, measurementName).getFullPath()); } } Collections.sort(measurements);
