This is an automated email from the ASF dual-hosted git repository. sunzesong pushed a commit to branch new_vector_jira1910 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2fa0c3d27465006f503753725a373bcfdfc4d519 Author: samperson1997 <[email protected]> AuthorDate: Tue Nov 2 15:51:45 2021 +0800 Support aligned timeseries in TsFile metadata index --- .../tsfile/file/metadata/AlignedChunkMetadata.java | 10 ---- .../iotdb/tsfile/file/metadata/ChunkMetadata.java | 11 ---- .../iotdb/tsfile/file/metadata/IChunkMetadata.java | 4 -- .../file/metadata/MetadataIndexConstructor.java | 6 +-- .../tsfile/file/metadata/TimeseriesMetadata.java | 9 ---- .../iotdb/tsfile/read/TsFileSequenceReader.java | 35 ------------- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 61 +++------------------- 7 files changed, 8 insertions(+), 128 deletions(-) diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java index f42bb3c..ee4f570 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java @@ -176,16 +176,6 @@ public class AlignedChunkMetadata implements IChunkMetadata { return 0; } - @Override - public boolean isTimeColumn() { - return false; - } - - @Override - public boolean isValueColumn() { - return false; - } - public Chunk getTimeChunk() throws IOException { return timeChunkMetadata.getChunkLoader().loadChunk((ChunkMetadata) timeChunkMetadata); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java index 33908b6..7fe70d2 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java @@ -19,7 +19,6 @@ package org.apache.iotdb.tsfile.file.metadata; import org.apache.iotdb.tsfile.common.cache.Accountable; -import org.apache.iotdb.tsfile.common.constant.TsFileConstant; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.common.TimeRange; @@ -335,16 +334,6 @@ public class ChunkMetadata implements Accountable, IChunkMetadata { return mask; } - @Override - public boolean isTimeColumn() { - return mask == TsFileConstant.TIME_COLUMN_MASK; - } - - @Override - public boolean isValueColumn() { - return mask == TsFileConstant.VALUE_COLUMN_MASK; - } - public void setMask(byte mask) { this.mask = mask; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java index 3893d15..1cc819f 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java @@ -74,8 +74,4 @@ public interface IChunkMetadata { int serializeTo(OutputStream outputStream, boolean serializeStatistic) throws IOException; byte getMask(); - - boolean isTimeColumn(); - - boolean isValueColumn(); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java index de20f40..062ffd6 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/MetadataIndexConstructor.java @@ -62,21 +62,17 @@ public class MetadataIndexConstructor { TimeseriesMetadata timeseriesMetadata; MetadataIndexNode currentIndexNode = new MetadataIndexNode(MetadataIndexNodeType.LEAF_MEASUREMENT); - int serializedTimeseriesMetadataNum = 0; for (int i = 0; i < entry.getValue().size(); i++) { timeseriesMetadata = entry.getValue().get(i); - if (serializedTimeseriesMetadataNum == 0 - || serializedTimeseriesMetadataNum >= config.getMaxDegreeOfIndexNode()) { + if (i % config.getMaxDegreeOfIndexNode() == 0) { if (currentIndexNode.isFull()) { addCurrentIndexNodeToQueue(currentIndexNode, measurementMetadataIndexQueue, out); currentIndexNode = new MetadataIndexNode(MetadataIndexNodeType.LEAF_MEASUREMENT); } currentIndexNode.addEntry( new MetadataIndexEntry(timeseriesMetadata.getMeasurementId(), out.getPosition())); - serializedTimeseriesMetadataNum = 0; } timeseriesMetadata.serializeTo(out.wrapAsStream()); - serializedTimeseriesMetadataNum++; } addCurrentIndexNodeToQueue(currentIndexNode, measurementMetadataIndexQueue, out); deviceMetadataIndexMap.put( diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java index 2881711..156d220 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java @@ -20,7 +20,6 @@ package org.apache.iotdb.tsfile.file.metadata; import org.apache.iotdb.tsfile.common.cache.Accountable; -import org.apache.iotdb.tsfile.common.constant.TsFileConstant; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader; @@ -148,14 +147,6 @@ public class TimeseriesMetadata implements Accountable, ITimeSeriesMetadata { return timeSeriesMetadataType; } - public boolean isTimeColumn() { - return timeSeriesMetadataType == TsFileConstant.TIME_COLUMN_MASK; - } - - public boolean isValueColumn() { - return timeSeriesMetadataType == TsFileConstant.VALUE_COLUMN_MASK; - } - public void setTimeSeriesMetadataType(byte timeSeriesMetadataType) { this.timeSeriesMetadataType = timeSeriesMetadataType; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java index 7588b84..c546068 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java @@ -370,41 +370,6 @@ public class TsFileSequenceReader implements AutoCloseable { return searchResult >= 0 ? timeseriesMetadataList.get(searchResult) : null; } - /** - * Find the leaf node that contains this vector, return all the needed subSensor and time column - * - * @param path path with time column - * @param subSensorList value columns that needed - * @return TimeseriesMetadata for the time column and all the needed subSensor, the order of the - * element in this list should be the same as subSensorList - */ - public List<TimeseriesMetadata> readTimeseriesMetadata(Path path, List<String> subSensorList) - throws IOException { - Pair<MetadataIndexEntry, Long> metadataIndexPair = getLeafMetadataIndexPair(path); - if (metadataIndexPair == null) { - return Collections.emptyList(); - } - Map<String, TimeseriesMetadata> timeseriesMetadataMap = new HashMap<>(); - ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right); - while (buffer.hasRemaining()) { - TimeseriesMetadata timeseriesMetadata; - try { - timeseriesMetadata = TimeseriesMetadata.deserializeFrom(buffer, true); - } catch (BufferOverflowException e) { - logger.error( - "Something error happened while deserializing TimeseriesMetadata of file {}", file); - throw e; - } - timeseriesMetadataMap.put(timeseriesMetadata.getMeasurementId(), timeseriesMetadata); - } - - List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>(); - for (String subSensor : subSensorList) { - timeseriesMetadataList.add(timeseriesMetadataMap.get(subSensor)); - } - return timeseriesMetadataList; - } - /* Find the leaf node that contains path, return all the sensors in that leaf node which are also in allSensors set */ public List<TimeseriesMetadata> readTimeseriesMetadata(Path path, Set<String> allSensors) throws IOException { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java index 3759780..524cb4f5 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java @@ -253,44 +253,17 @@ public class TsFileIOWriter { ReadWriteIOUtils.write(MetaMarker.SEPARATOR, out.wrapAsStream()); // group ChunkMetadata by series - // only contains ordinary path and time column of vector series Map<Path, List<IChunkMetadata>> chunkMetadataListMap = new TreeMap<>(); - // time column -> ChunkMetadataList TreeMap of value columns in vector - Map<Path, Map<Path, List<IChunkMetadata>>> vectorToPathsMap = new HashMap<>(); - for (ChunkGroupMetadata chunkGroupMetadata : chunkGroupMetadataList) { List<ChunkMetadata> chunkMetadatas = chunkGroupMetadata.getChunkMetadataList(); - int idx = 0; - while (idx < chunkMetadatas.size()) { - IChunkMetadata chunkMetadata = chunkMetadatas.get(idx); - if (chunkMetadata.getMask() == 0) { - Path series = new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid()); - chunkMetadataListMap.computeIfAbsent(series, k -> new ArrayList<>()).add(chunkMetadata); - idx++; - } else if (chunkMetadata.isTimeColumn()) { - // time column of a vector series - Path series = new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid()); - chunkMetadataListMap.computeIfAbsent(series, k -> new ArrayList<>()).add(chunkMetadata); - idx++; - Map<Path, List<IChunkMetadata>> chunkMetadataListMapInVector = - vectorToPathsMap.computeIfAbsent(series, key -> new TreeMap<>()); - - // value columns of a vector series - while (idx < chunkMetadatas.size() && chunkMetadatas.get(idx).isValueColumn()) { - chunkMetadata = chunkMetadatas.get(idx); - Path vectorSeries = - new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid()); - chunkMetadataListMapInVector - .computeIfAbsent(vectorSeries, k -> new ArrayList<>()) - .add(chunkMetadata); - idx++; - } - } + for (IChunkMetadata chunkMetadata : chunkMetadatas) { + Path series = new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid()); + chunkMetadataListMap.computeIfAbsent(series, k -> new ArrayList<>()).add(chunkMetadata); } } - MetadataIndexNode metadataIndex = flushMetadataIndex(chunkMetadataListMap, vectorToPathsMap); + MetadataIndexNode metadataIndex = flushMetadataIndex(chunkMetadataListMap); TsFileMetadata tsFileMetaData = new TsFileMetadata(); tsFileMetaData.setMetadataIndex(metadataIndex); tsFileMetaData.setMetaOffset(metaOffset); @@ -330,13 +303,9 @@ public class TsFileIOWriter { * Flush TsFileMetadata, including ChunkMetadataList and TimeseriesMetaData * * @param chunkMetadataListMap chunkMetadata that Path.mask == 0 - * @param vectorToPathsMap Map Path to chunkMataList, Key is Path(timeColumn) and Value is it's - * sub chunkMetadataListMap * @return MetadataIndexEntry list in TsFileMetadata */ - private MetadataIndexNode flushMetadataIndex( - Map<Path, List<IChunkMetadata>> chunkMetadataListMap, - Map<Path, Map<Path, List<IChunkMetadata>>> vectorToPathsMap) + private MetadataIndexNode flushMetadataIndex(Map<Path, List<IChunkMetadata>> chunkMetadataListMap) throws IOException { // convert ChunkMetadataList to this field @@ -344,7 +313,7 @@ public class TsFileIOWriter { // create device -> TimeseriesMetaDataList Map for (Map.Entry<Path, List<IChunkMetadata>> entry : chunkMetadataListMap.entrySet()) { // for ordinary path - flushOneChunkMetadata(entry.getKey(), entry.getValue(), vectorToPathsMap); + flushOneChunkMetadata(entry.getKey(), entry.getValue()); } // construct TsFileMetadata and return @@ -356,12 +325,8 @@ public class TsFileIOWriter { * * @param path Path of chunk * @param chunkMetadataList List of chunkMetadata about path(previous param) - * @param vectorToPathsMap Key is Path(timeColumn) and Value is it's sub chunkMetadataListMap */ - private void flushOneChunkMetadata( - Path path, - List<IChunkMetadata> chunkMetadataList, - Map<Path, Map<Path, List<IChunkMetadata>>> vectorToPathsMap) + private void flushOneChunkMetadata(Path path, List<IChunkMetadata> chunkMetadataList) throws IOException { // create TimeseriesMetaData PublicBAOS publicBAOS = new PublicBAOS(); @@ -391,18 +356,6 @@ public class TsFileIOWriter { deviceTimeseriesMetadataMap .computeIfAbsent(path.getDevice(), k -> new ArrayList<>()) .add(timeseriesMetadata); - - // for VECTOR - for (IChunkMetadata chunkMetadata : chunkMetadataList) { - // chunkMetadata is time column of a vector series - if (chunkMetadata.isTimeColumn()) { - Map<Path, List<IChunkMetadata>> vectorMap = vectorToPathsMap.get(path); - for (Map.Entry<Path, List<IChunkMetadata>> entry : vectorMap.entrySet()) { - flushOneChunkMetadata(entry.getKey(), entry.getValue(), vectorToPathsMap); - } - } - break; - } } /**
