qiaojialin commented on a change in pull request #1100:
URL: https://github.com/apache/incubator-iotdb/pull/1100#discussion_r415196107
##########
File path: server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
##########
@@ -72,7 +67,8 @@ public static void checkTsFileResource(TsFileResource
tsFileResource) throws IOE
public static void updateTsFileResource(TsFileMetadata metaData,
TsFileSequenceReader reader,
TsFileResource tsFileResource) throws IOException {
- for (String device : metaData.getDeviceMetadataIndex().keySet()) {
+ List<String> deviceList =
reader.getDevicesByMetadata(metaData.getDeviceMetadataIndex());
+ for (String device : deviceList) {
Review comment:
In this case, we'd better provide an API that read all
TimeseriesMetadata at a time to reduce IO.
##########
File path:
tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
##########
@@ -82,6 +82,10 @@
* The maximum number of data points in a page, default value is 1024 * 1024.
*/
private int maxNumberOfPointsInPage = 1024 * 1024;
+ /**
+ * The maximum number of index items in a metadataIndex node, default value
is 1024
+ */
+ private int maxNumberOfIndexItemsInNode = 5;
Review comment:
maxDegreeOfIndexNode or so
##########
File path:
tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
##########
@@ -68,16 +68,17 @@ public static TsFileMetadata deserializeFrom(ByteBuffer
buffer) {
// deviceMetadataIndex
int deviceNum = ReadWriteIOUtils.readInt(buffer);
- Map<String, Pair<Long, Integer>> deviceMetaDataMap = new HashMap<>();
+ List<MetadataIndex> deviceMetaDataList = new ArrayList<>();
if (deviceNum > 0) {
for (int i = 0; i < deviceNum; i++) {
- String deviceId = ReadWriteIOUtils.readString(buffer);
+ String name = ReadWriteIOUtils.readString(buffer);
long offset = ReadWriteIOUtils.readLong(buffer);
- int length = ReadWriteIOUtils.readInt(buffer);
- deviceMetaDataMap.put(deviceId, new Pair<>(offset, length));
+ ChildMetadataIndexType type = ChildMetadataIndexType
+ .deserialize(ReadWriteIOUtils.readShort(buffer));
Review comment:
replace these with MetadataIndex.deserializeFrom
##########
File path:
tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
##########
@@ -123,10 +124,11 @@ public int serializeTo(OutputStream outputStream) throws
IOException {
// deviceMetadataIndex
if (deviceMetadataIndex != null) {
byteLen += ReadWriteIOUtils.write(deviceMetadataIndex.size(),
outputStream);
- for (Map.Entry<String, Pair<Long, Integer>> entry :
deviceMetadataIndex.entrySet()) {
- byteLen += ReadWriteIOUtils.write(entry.getKey(), outputStream);
- byteLen += ReadWriteIOUtils.write(entry.getValue().left, outputStream);
- byteLen += ReadWriteIOUtils.write(entry.getValue().right,
outputStream);
+ for (MetadataIndex metadataIndex : deviceMetadataIndex) {
+ byteLen += ReadWriteIOUtils.write(metadataIndex.getName(),
outputStream);
+ byteLen += ReadWriteIOUtils.write(metadataIndex.getOffset(),
outputStream);
+ byteLen += ReadWriteIOUtils
+ .write(metadataIndex.getChildMetadataIndexType().serialize(),
outputStream);
Review comment:
replace these with metadataIndex.serializeTo
##########
File path:
tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
##########
@@ -208,11 +210,11 @@ public void setMetaOffset(long metaOffset) {
this.metaOffset = metaOffset;
}
- public Map<String, Pair<Long, Integer>> getDeviceMetadataIndex() {
+ public List<MetadataIndex> getDeviceMetadataIndex() {
Review comment:
```suggestion
public List<MetadataIndex> getMetadataIndex() {
```
##########
File path:
tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/ChildMetadataIndexType.java
##########
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.file.metadata.enums;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public enum ChildMetadataIndexType {
Review comment:
```suggestion
public enum =MetadataIndexNodeType {
```
##########
File path:
tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
##########
@@ -48,8 +48,8 @@
// bloom filter
private BloomFilter bloomFilter;
- // DeviceId -> offset and length of Map<String, TimeseriesMetadata>
- private Map<String, Pair<Long, Integer>> deviceMetadataIndex;
+ // List of <name, offset, childMetadataIndexType>
+ private List<MetadataIndex> deviceMetadataIndex;
Review comment:
```suggestion
private List<MetadataIndex> metadataIndex;
```
##########
File path: tsfile/src/main/java/org/apache/iotdb/tsfile/utils/MetadataIndex.java
##########
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.utils;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import org.apache.iotdb.tsfile.file.metadata.enums.ChildMetadataIndexType;
+
+public class MetadataIndex {
Review comment:
Actually this is a node of the tree index, so rename it to
MetadataIndexNode or TsMetaIndexNode or TsIndexNode is better..
##########
File path: tsfile/src/main/java/org/apache/iotdb/tsfile/utils/MetadataIndex.java
##########
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.utils;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import org.apache.iotdb.tsfile.file.metadata.enums.ChildMetadataIndexType;
+
+public class MetadataIndex {
Review comment:
```suggestion
public class MetadataIndexNode {
```
##########
File path:
tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/ChildMetadataIndexType.java
##########
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.file.metadata.enums;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public enum ChildMetadataIndexType {
+ DEVICE_INDEX, DEVICE, MEASUREMENT_INDEX, MEASUREMENT;
Review comment:
Internal_device, Leaf_device, Internal_Measurement, Leaf_Measurement
Give examples of the four nodes in the javadoc of this class
##########
File path:
tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
##########
@@ -208,11 +210,11 @@ public void setMetaOffset(long metaOffset) {
this.metaOffset = metaOffset;
}
- public Map<String, Pair<Long, Integer>> getDeviceMetadataIndex() {
+ public List<MetadataIndex> getDeviceMetadataIndex() {
return deviceMetadataIndex;
}
- public void setDeviceMetadataIndex(Map<String, Pair<Long, Integer>>
deviceMetadataIndex) {
+ public void setDeviceMetadataIndex(List<MetadataIndex> deviceMetadataIndex) {
Review comment:
```suggestion
public void setMetadataIndex(List<MetadataIndex> deviceMetadataIndex) {
```
##########
File path:
tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
##########
@@ -304,22 +311,120 @@ public void endFile() throws IOException {
deviceTimeseriesMetadataMap.computeIfAbsent(device, k -> new
ArrayList<>())
.add(timeseriesMetaData);
}
- // create DeviceMetaDataMap device -> Pair<TimeseriesMetaDataOffset,
TimeseriesMetaDataLength>
- Map<String, Pair<Long, Integer>> deviceMetadataMap = new HashMap<>();
+
+ // create TsFileMetadata
+ Map<String, Queue<MetadataIndex>> deviceMetadataIndexMap = new TreeMap<>();
+ int maxNumOfIndexItems = config.getMaxNumberOfIndexItemsInNode();
+
+ // for timeseriesMetadata of each device
for (Map.Entry<String, List<TimeseriesMetadata>> entry :
deviceTimeseriesMetadataMap
.entrySet()) {
- String device = entry.getKey();
- List<TimeseriesMetadata> timeseriesMetadataList = entry.getValue();
- long offsetOfFirstTimeseriesMetaDataInDevice = out.getPosition();
- int size = 0;
- for (TimeseriesMetadata timeseriesMetaData : timeseriesMetadataList) {
- size += timeseriesMetaData.serializeTo(out.wrapAsStream());
+ if (entry.getValue().isEmpty()) {
+ continue;
+ }
+ Queue<MetadataIndex> measurementMetadataIndexQueue = new ArrayDeque<>();
+ TimeseriesMetadata timeseriesMetadata;
+ for (int i = 0; i < entry.getValue().size(); i++) {
+ timeseriesMetadata = entry.getValue().get(i);
+ if (i % maxNumOfIndexItems == 0) {
+ measurementMetadataIndexQueue
+ .add(new MetadataIndex(timeseriesMetadata.getMeasurementId(),
out.getPosition(),
+ ChildMetadataIndexType.MEASUREMENT));
+ }
+ timeseriesMetadata.serializeTo(out.wrapAsStream());
+ }
+ measurementMetadataIndexQueue
+ .add(new MetadataIndex("", out.getPosition(),
ChildMetadataIndexType.MEASUREMENT));
+
+ int queueSize = measurementMetadataIndexQueue.size();
+ MetadataIndex metadataIndex;
+ while (queueSize > maxNumOfIndexItems) {
+ for (int i = 0; i < queueSize; i++) {
+ metadataIndex = measurementMetadataIndexQueue.poll();
+ if (i % maxNumOfIndexItems == 0) {
+ if (i != 0) {
+ addEmptyMetadataIndex(ChildMetadataIndexType.MEASUREMENT_INDEX);
Review comment:
```suggestion
serializeEmptyMetadataIndex(ChildMetadataIndexType.MEASUREMENT_INDEX);
```
##########
File path:
tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
##########
@@ -129,9 +129,9 @@ public static RestorableTsFileIOWriter
getWriterForAppendingDataOnCompletedTsFil
if (reader.isComplete()) {
reader.loadMetadataSize();
TsFileMetadata metaData = reader.readFileMetadata();
- for (Pair<Long, Integer> deviceMetaData :
metaData.getDeviceMetadataIndex().values()) {
- if (position > deviceMetaData.left) {
- position = deviceMetaData.left;
+ for (MetadataIndex deviceMetaData : metaData.getDeviceMetadataIndex())
{
+ if (position > deviceMetaData.getOffset()) {
+ position = deviceMetaData.getOffset();
Review comment:
This is a bug, deviceMetaData offset may index to a metadataIndex node,
not the first metadata offset.
Use TsFileMetadata.getMetaOffset()
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
