This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch TyLRU
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/TyLRU by this push:
new 2bb91f5 final version
2bb91f5 is described below
commit 2bb91f50c8630ff5a362ea9717d08f7acd047c49
Author: JackieTien97 <[email protected]>
AuthorDate: Fri May 15 14:39:23 2020 +0800
final version
---
.../apache/iotdb/db/engine/cache/ChunkCache.java | 4 +--
.../iotdb/db/engine/cache/ChunkMetadataCache.java | 32 +++++++++++-----------
.../db/engine/cache/TimeSeriesMetadataCache.java | 1 -
.../db/query/reader/chunk/DiskChunkLoader.java | 17 +++++++-----
.../chunk/metadata/DiskChunkMetadataLoader.java | 4 +--
.../iotdb/tsfile/read/TsFileSequenceReader.java | 4 ++-
6 files changed, 31 insertions(+), 31 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
index b2365fe..8e05f08 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -63,8 +63,7 @@ public class ChunkCache {
protected long calEntrySize(ChunkMetadata key, Chunk value) {
long currentSize;
if (count < 10) {
- currentSize = RamUsageEstimator.NUM_BYTES_OBJECT_REF +
RamUsageEstimator
- .shallowSizeOf(key.getChunkLoader()) +
RamUsageEstimator.sizeOf(value);
+ currentSize = RamUsageEstimator.NUM_BYTES_OBJECT_REF +
RamUsageEstimator.sizeOf(value);
averageSize = ((averageSize * count) + currentSize) / (++count);
} else if (count < 100000) {
count++;
@@ -74,7 +73,6 @@ public class ChunkCache {
count = 1;
currentSize = averageSize;
}
- key.setRAMSize(currentSize);
return currentSize;
}
};
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
index 80846c0..8826d76 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
@@ -48,6 +48,8 @@ public class ChunkMetadataCache {
private static final IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
private static final long MEMORY_THRESHOLD_IN_B =
config.getAllocateMemoryForChunkMetaDataCache();
private static final boolean cacheEnable = config.isMetaDataCacheEnable();
+ private static final long CHUNK_METADATA_FIXED_RAM_SIZE = 160;
+
/**
* key: file path dot deviceId dot sensorId.
* <p>
@@ -69,32 +71,30 @@ public class ChunkMetadataCache {
@Override
protected long calEntrySize(AccountableString key, List<ChunkMetadata>
value) {
if (value.isEmpty()) {
- return key.getString().getBytes().length;
+ return RamUsageEstimator.sizeOf(key) +
RamUsageEstimator.shallowSizeOf(value);
}
long entrySize;
if (count < 10) {
- long currentSize = RamUsageEstimator.shallowSizeOf(value.get(0)) +
RamUsageEstimator
- .shallowSizeOf(value.get(0).getStatistics()) + RamUsageEstimator
- .sizeOf(value.get(0).getMeasurementUid()) + RamUsageEstimator
- .shallowSizeOf(value.get(0).getChunkLoader());
+ long currentSize = CHUNK_METADATA_FIXED_RAM_SIZE + RamUsageEstimator
+ .sizeOf(value.get(0).getMeasurementUid());
averageSize = ((averageSize * count) + currentSize) / (++count);
IoTDBConfigDynamicAdapter.setChunkMetadataSizeInByte(averageSize);
- entrySize = RamUsageEstimator.sizeOf(key) + currentSize *
value.size() + RamUsageEstimator
- .shallowSizeOf(value);
+ entrySize = RamUsageEstimator.sizeOf(key)
+ + (currentSize + RamUsageEstimator.NUM_BYTES_OBJECT_REF) *
value.size()
+ + RamUsageEstimator.shallowSizeOf(value);
} else if (count < 100000) {
count++;
- entrySize = RamUsageEstimator.sizeOf(key) + averageSize *
value.size() + RamUsageEstimator
- .shallowSizeOf(value);
+ entrySize = RamUsageEstimator.sizeOf(key)
+ + (averageSize + RamUsageEstimator.NUM_BYTES_OBJECT_REF) *
value.size()
+ + RamUsageEstimator.shallowSizeOf(value);
} else {
- averageSize = RamUsageEstimator.shallowSizeOf(value.get(0)) +
RamUsageEstimator
- .shallowSizeOf(value.get(0).getStatistics()) + RamUsageEstimator
- .sizeOf(value.get(0).getMeasurementUid()) + RamUsageEstimator
- .shallowSizeOf(value.get(0).getChunkLoader());
+ averageSize = CHUNK_METADATA_FIXED_RAM_SIZE + RamUsageEstimator
+ .sizeOf(value.get(0).getMeasurementUid());
count = 1;
- entrySize = RamUsageEstimator.sizeOf(key) + averageSize *
value.size() + RamUsageEstimator
- .shallowSizeOf(value);
+ entrySize = RamUsageEstimator.sizeOf(key)
+ + (averageSize + RamUsageEstimator.NUM_BYTES_OBJECT_REF) *
value.size()
+ + RamUsageEstimator.shallowSizeOf(value);
}
- key.setRAMSize(entrySize);
return entrySize;
}
};
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index 33cf9f3..f2cf1d3 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -83,7 +83,6 @@ public class TimeSeriesMetadataCache {
count = 1;
currentSize = averageSize;
}
- key.setRAMSize(currentSize);
return currentSize;
}
};
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
index 11d6c15..e3c70f0 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
@@ -19,32 +19,35 @@
package org.apache.iotdb.db.query.reader.chunk;
+import java.io.IOException;
import org.apache.iotdb.db.engine.cache.ChunkCache;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
-import java.io.IOException;
-
/**
* To read one chunk from disk, and only used in iotdb server module
*/
public class DiskChunkLoader implements IChunkLoader {
- private TsFileSequenceReader reader;
+ private final TsFileResource resource;
- public DiskChunkLoader(TsFileSequenceReader reader) {
- this.reader = reader;
+ public DiskChunkLoader(TsFileResource resource) {
+ this.resource = resource;
}
@Override
public Chunk loadChunk(ChunkMetadata chunkMetaData) throws IOException {
- return ChunkCache.getInstance().get(chunkMetaData, reader);
+ TsFileSequenceReader tsFileSequenceReader =
+ FileReaderManager.getInstance().get(resource.getPath(),
resource.isClosed());
+ return ChunkCache.getInstance().get(chunkMetaData, tsFileSequenceReader);
}
@Override
public void close() throws IOException {
- reader.close();
+ // do nothing
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
index 2c5d8ea..7ba86b5 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
@@ -72,10 +72,8 @@ public class DiskChunkMetadataLoader implements
IChunkMetadataLoader {
QueryUtils.modifyChunkMetaData(chunkMetadataList, pathModifications);
}
- TsFileSequenceReader tsFileSequenceReader =
- FileReaderManager.getInstance().get(resource.getPath(),
resource.isClosed());
for (ChunkMetadata data : chunkMetadataList) {
- data.setChunkLoader(new DiskChunkLoader(tsFileSequenceReader));
+ data.setChunkLoader(new DiskChunkLoader(resource));
}
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index f924d83..0bf0cf1 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -959,7 +959,7 @@ public class TsFileSequenceReader implements AutoCloseable {
public List<ChunkMetadata> readChunkMetaDataList(TimeseriesMetadata
timeseriesMetaData)
throws IOException {
List<Pair<Long, Long>> versionInfo = tsFileMetaData.getVersionInfo();
- List<ChunkMetadata> chunkMetadataList = new ArrayList<>();
+ ArrayList<ChunkMetadata> chunkMetadataList = new ArrayList<>();
long startOffsetOfChunkMetadataList =
timeseriesMetaData.getOffsetOfChunkMetaDataList();
int dataSizeOfChunkMetadataList =
timeseriesMetaData.getDataSizeOfChunkMetaDataList();
@@ -970,6 +970,8 @@ public class TsFileSequenceReader implements AutoCloseable {
VersionUtils.applyVersion(chunkMetadataList, versionInfo);
+ // minimize the storage of an ArrayList instance.
+ chunkMetadataList.trimToSize();
return chunkMetadataList;
}
