fanhualta commented on a change in pull request #736: Refactor TsFile
URL: https://github.com/apache/incubator-iotdb/pull/736#discussion_r377523969
##########
File path:
tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
##########
@@ -254,64 +260,51 @@ public void endFile(Schema schema) throws IOException {
// close file
out.close();
canWrite = false;
+ timeseriesMetadataMap = new TreeMap<>();
logger.info("output stream is closed");
}
/**
- * 1. group chunkGroupMetaDataList to TsDeviceMetadata 2. flush
TsDeviceMetadata 3. get
- * TsDeviceMetadataIndex
- *
- * @param chunkGroupMetaDataList all chunk group metadata in memory
- * @return TsDeviceMetadataIndex in TsFileMetaData
+ * @return tsOffsets in TsFileMetaData
*/
- private Map<String, TsDeviceMetadataIndex> flushTsDeviceMetaDataAndGetIndex(
- List<ChunkGroupMetaData> chunkGroupMetaDataList) throws IOException {
-
- Map<String, TsDeviceMetadataIndex> tsDeviceMetadataIndexMap = new
HashMap<>();
-
- long offset; /* offset for the flushing TsDeviceMetadata */
-
- TsDeviceMetadata currentTsDeviceMetadata;
-
- // flush TsDeviceMetadata by string order of deviceId
- for (Map.Entry<String, TsDeviceMetadata> entry :
getAllTsDeviceMetadata(chunkGroupMetaDataList)
- .entrySet()) {
- // update statistics in TsDeviceMetadata
- currentTsDeviceMetadata = entry.getValue();
-
- // flush tsChunkGroupBlockMetaData
- offset = out.getPosition();
- int size = currentTsDeviceMetadata.serializeTo(out.wrapAsStream());
-
- TsDeviceMetadataIndex tsDeviceMetadataIndex = new
TsDeviceMetadataIndex(offset, size,
- currentTsDeviceMetadata);
- tsDeviceMetadataIndexMap.put(entry.getKey(), tsDeviceMetadataIndex);
+ private long[] flushAllChunkMetadataList() throws IOException {
+ if (timeseriesMetadataMap.size() == 0) {
+ return new long[0];
}
-
- return tsDeviceMetadataIndexMap;
- }
-
- /**
- * group all chunk group metadata by device.
- *
- * @param chunkGroupMetaDataList all chunk group metadata
- * @return TsDeviceMetadata of all devices
- */
- private TreeMap<String, TsDeviceMetadata> getAllTsDeviceMetadata(
- List<ChunkGroupMetaData> chunkGroupMetaDataList) {
- String currentDevice;
- TreeMap<String, TsDeviceMetadata> tsDeviceMetadataMap = new TreeMap<>();
-
- for (ChunkGroupMetaData chunkGroupMetaData : chunkGroupMetaDataList) {
- currentDevice = chunkGroupMetaData.getDeviceID();
-
- if (!tsDeviceMetadataMap.containsKey(currentDevice)) {
- TsDeviceMetadata tsDeviceMetadata = new TsDeviceMetadata();
- tsDeviceMetadataMap.put(currentDevice, tsDeviceMetadata);
+ // convert ChunkMetadataList to this field
+ long[] tsOffsets = new long[timeseriesMetadataMap.size() + 1];
+ List<TimeseriesMetaData> tsMetadataList = new ArrayList<>();
+ // flush timeseriesMetadataList one by one
+ int i = 0;
+ for (Map.Entry<Path, List<ChunkMetaData>> entry :
timeseriesMetadataMap.entrySet()) {
+ Path path = entry.getKey();
+ String deviceId = path.getDevice();
+ if (!deviceOffsetsMap.containsKey(deviceId)) {
+ deviceOffsetsMap.put(deviceId, new int[2]);
+ deviceOffsetsMap.get(deviceId)[0] = i;
+ }
+ deviceOffsetsMap.get(deviceId)[1] = i + 1;
+ TimeseriesMetaData tsMetaData = new TimeseriesMetaData();
+
+ tsMetaData.setMeasurementId(path.getMeasurement());
+ tsMetaData.setOffsetOfChunkMetaDataList(out.getPosition());
+ int chunkMetadataSize = 0;
+ for (ChunkMetaData chunkMetadata : entry.getValue()) {
+ chunkMetadataSize += chunkMetadata.serializeTo(out.wrapAsStream());
}
-
tsDeviceMetadataMap.get(currentDevice).addChunkGroupMetaData(chunkGroupMetaData);
+ tsMetaData.setDataSizeOfChunkMetaDataList(chunkMetadataSize);
+ tsMetaData.setNumOfChunkMetaDatas(entry.getValue().size());
+ tsMetadataList.add(tsMetaData);
+ i++;
}
- return tsDeviceMetadataMap;
+
+ for (i = 0; i < tsMetadataList.size(); i++) {
+ tsOffsets[i] = out.getPosition();
+ int size = tsMetadataList.get(i).serializeTo(out.wrapAsStream());
+ tsOffsets[i+1] = tsOffsets[i] + size;
+ }
Review comment:
Please remove useless assignment and addition operations.
```
if(i > 0){
tsOffsets[0] = out.getPosition();
}
for (i = 0; i < tsMetadataList.size(); i++) {
int size = tsMetadataList.get(i).serializeTo(out.wrapAsStream());
tsOffsets[i+1] = out.getPosition();
}
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
