This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 91fd0ea [IOTDB-680] Make LRUCache more accurate (#1212)
91fd0ea is described below
commit 91fd0ea2429d5db88825cb6c55fe3bb5ed1f2dff
Author: Jackie Tien <[email protected]>
AuthorDate: Sun May 17 15:48:23 2020 +0800
[IOTDB-680] Make LRUCache more accurate (#1212)
* refactor LRU and ramSize of different metadata
* change calculation of statistics
---
.../cache/AccountableString.java} | 49 +++++++++------
.../apache/iotdb/db/engine/cache/ChunkCache.java | 60 ++++++++++---------
.../iotdb/db/engine/cache/ChunkMetadataCache.java | 56 ++++++++++-------
.../iotdb/db/engine/cache/LRULinkedHashMap.java | 70 +++++++++++++++++-----
.../db/engine/cache/TimeSeriesMetadataCache.java | 57 ++++++++++++------
.../db/query/executor/fill/LastPointReader.java | 1 -
.../db/query/reader/chunk/DiskChunkLoader.java | 17 +++---
.../chunk/metadata/DiskChunkMetadataLoader.java | 11 ++--
.../chunk/metadata/MemChunkMetadataLoader.java | 2 +-
.../java/org/apache/iotdb/db/utils/MemUtils.java | 2 +-
.../iotdb/tsfile/common/cache/Accountable.java | 26 ++++++++
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 38 +++++++++---
.../tsfile/file/metadata/TimeseriesMetadata.java | 26 +++++++-
.../file/metadata/statistics/BinaryStatistics.java | 6 ++
.../metadata/statistics/BooleanStatistics.java | 8 +++
.../file/metadata/statistics/DoubleStatistics.java | 7 +++
.../file/metadata/statistics/FloatStatistics.java | 8 +++
.../metadata/statistics/IntegerStatistics.java | 8 +++
.../file/metadata/statistics/LongStatistics.java | 8 +++
.../file/metadata/statistics/Statistics.java | 2 +
.../iotdb/tsfile/read/TsFileSequenceReader.java | 4 +-
.../org/apache/iotdb/tsfile/read/common/Chunk.java | 15 ++++-
.../iotdb/tsfile/utils}/RamUsageEstimator.java | 22 +++----
23 files changed, 362 insertions(+), 141 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
b/server/src/main/java/org/apache/iotdb/db/engine/cache/AccountableString.java
similarity index 51%
copy from
server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
copy to
server/src/main/java/org/apache/iotdb/db/engine/cache/AccountableString.java
index 11d6c15..4a38f58 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/cache/AccountableString.java
@@ -16,35 +16,48 @@
* specific language governing permissions and limitations
* under the License.
*/
+package org.apache.iotdb.db.engine.cache;
-package org.apache.iotdb.db.query.reader.chunk;
+import java.util.Objects;
+import org.apache.iotdb.tsfile.common.cache.Accountable;
-import org.apache.iotdb.db.engine.cache.ChunkCache;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
-import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
-import org.apache.iotdb.tsfile.read.common.Chunk;
-import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
+public class AccountableString implements Accountable {
-import java.io.IOException;
+ private final String string;
+ private long ramSize;
-/**
- * To read one chunk from disk, and only used in iotdb server module
- */
-public class DiskChunkLoader implements IChunkLoader {
+ public AccountableString(String string) {
+ this.string = string;
+ }
- private TsFileSequenceReader reader;
+ public String getString() {
+ return string;
+ }
- public DiskChunkLoader(TsFileSequenceReader reader) {
- this.reader = reader;
+ @Override
+ public void setRamSize(long size) {
+ this.ramSize = size;
+ }
+
+ @Override
+ public long getRamSize() {
+ return ramSize;
}
@Override
- public Chunk loadChunk(ChunkMetadata chunkMetaData) throws IOException {
- return ChunkCache.getInstance().get(chunkMetaData, reader);
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ AccountableString that = (AccountableString) o;
+ return Objects.equals(string, that.string);
}
@Override
- public void close() throws IOException {
- reader.close();
+ public int hashCode() {
+ return Objects.hash(string);
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
index b4eefb3..2f25690 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -19,59 +19,61 @@
package org.apache.iotdb.db.engine.cache;
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
+import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
/**
- * This class is used to cache <code>Chunk</code> of
<code>ChunkMetaData</code> in IoTDB. The caching
- * strategy is LRU.
+ * This class is used to cache <code>Chunk</code> of
<code>ChunkMetaData</code> in IoTDB. The
+ * caching strategy is LRU.
*/
public class ChunkCache {
private static final Logger logger =
LoggerFactory.getLogger(ChunkCache.class);
private static final IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
- private static final long MEMORY_THRESHOLD_IN_CHUNK_CACHE =
config.getAllocateMemoryForChunkCache();
- private static boolean cacheEnable = config.isMetaDataCacheEnable();
+ private static final long MEMORY_THRESHOLD_IN_CHUNK_CACHE = config
+ .getAllocateMemoryForChunkCache();
+ private static final boolean CACHE_ENABLE = config.isMetaDataCacheEnable();
private final LRULinkedHashMap<ChunkMetadata, Chunk> lruCache;
- private AtomicLong cacheHitNum = new AtomicLong();
- private AtomicLong cacheRequestNum = new AtomicLong();
+ private final AtomicLong cacheHitNum = new AtomicLong();
+ private final AtomicLong cacheRequestNum = new AtomicLong();
private final ReadWriteLock lock = new ReentrantReadWriteLock();
private ChunkCache() {
- if (cacheEnable) {
+ if (CACHE_ENABLE) {
logger.info("ChunkCache size = " + MEMORY_THRESHOLD_IN_CHUNK_CACHE);
}
- lruCache = new LRULinkedHashMap<ChunkMetadata,
Chunk>(MEMORY_THRESHOLD_IN_CHUNK_CACHE, true) {
+ lruCache = new LRULinkedHashMap<ChunkMetadata,
Chunk>(MEMORY_THRESHOLD_IN_CHUNK_CACHE) {
@Override
protected long calEntrySize(ChunkMetadata key, Chunk value) {
+ long currentSize;
if (count < 10) {
- long currentSize = RamUsageEstimator.shallowSizeOf(key) +
RamUsageEstimator.sizeOf(value);
+ currentSize = RamUsageEstimator.NUM_BYTES_OBJECT_REF +
RamUsageEstimator.sizeOf(value);
averageSize = ((averageSize * count) + currentSize) / (++count);
- return currentSize;
} else if (count < 100000) {
count++;
- return averageSize;
+ currentSize = averageSize;
} else {
- averageSize = RamUsageEstimator.shallowSizeOf(key) +
RamUsageEstimator.sizeOf(value);
+ averageSize = RamUsageEstimator.NUM_BYTES_OBJECT_REF +
RamUsageEstimator.sizeOf(value);
count = 1;
- return averageSize;
+ currentSize = averageSize;
}
+ return currentSize;
}
};
}
@@ -81,9 +83,10 @@ public class ChunkCache {
}
public Chunk get(ChunkMetadata chunkMetaData, TsFileSequenceReader reader)
throws IOException {
- if (!cacheEnable) {
+ if (!CACHE_ENABLE) {
Chunk chunk = reader.readMemChunk(chunkMetaData);
- return new Chunk(chunk.getHeader(), chunk.getData().duplicate(),
chunk.getDeletedAt(), reader.getEndianType());
+ return new Chunk(chunk.getHeader(), chunk.getData().duplicate(),
chunk.getDeletedAt(),
+ reader.getEndianType());
}
cacheRequestNum.incrementAndGet();
@@ -94,7 +97,8 @@ public class ChunkCache {
cacheHitNum.incrementAndGet();
printCacheLog(true);
Chunk chunk = lruCache.get(chunkMetaData);
- return new Chunk(chunk.getHeader(), chunk.getData().duplicate(),
chunk.getDeletedAt(), reader.getEndianType());
+ return new Chunk(chunk.getHeader(), chunk.getData().duplicate(),
chunk.getDeletedAt(),
+ reader.getEndianType());
}
} finally {
lock.readLock().unlock();
@@ -113,12 +117,14 @@ public class ChunkCache {
cacheHitNum.incrementAndGet();
printCacheLog(true);
Chunk chunk = lruCache.get(chunkMetaData);
- return new Chunk(chunk.getHeader(), chunk.getData().duplicate(),
chunk.getDeletedAt(), reader.getEndianType());
+ return new Chunk(chunk.getHeader(), chunk.getData().duplicate(),
chunk.getDeletedAt(),
+ reader.getEndianType());
}
printCacheLog(false);
Chunk chunk = reader.readMemChunk(chunkMetaData);
lruCache.put(chunkMetaData, chunk);
- return new Chunk(chunk.getHeader(), chunk.getData().duplicate(),
chunk.getDeletedAt(), reader.getEndianType());
+ return new Chunk(chunk.getHeader(), chunk.getData().duplicate(),
chunk.getDeletedAt(),
+ reader.getEndianType());
} catch (IOException e) {
logger.error("something wrong happened while reading {}",
reader.getFileName());
throw e;
@@ -133,9 +139,9 @@ public class ChunkCache {
return;
}
logger.debug(
- "[ChunkMetaData cache {}hit] The number of requests for cache is
{}, hit rate is {}.",
- isHit ? "" : "didn't ", cacheRequestNum.get(),
- cacheHitNum.get() * 1.0 / cacheRequestNum.get());
+ "[ChunkMetaData cache {}hit] The number of requests for cache is {},
hit rate is {}.",
+ isHit ? "" : "didn't ", cacheRequestNum.get(),
+ cacheHitNum.get() * 1.0 / cacheRequestNum.get());
}
public double calculateChunkHitRatio() {
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
index 82adca5..8b5ee11 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
@@ -35,6 +35,7 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.BloomFilter;
+import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,45 +48,52 @@ public class ChunkMetadataCache {
private static final Logger logger =
LoggerFactory.getLogger(ChunkMetadataCache.class);
private static final IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
private static final long MEMORY_THRESHOLD_IN_B =
config.getAllocateMemoryForChunkMetaDataCache();
- private static boolean cacheEnable = config.isMetaDataCacheEnable();
+ private static final boolean CACHE_ENABLE = config.isMetaDataCacheEnable();
+
/**
* key: file path dot deviceId dot sensorId.
* <p>
* value: chunkMetaData list of one timeseries in the file.
*/
- private final LRULinkedHashMap<String, List<ChunkMetadata>> lruCache;
+ private final LRULinkedHashMap<AccountableString, List<ChunkMetadata>>
lruCache;
private final ReadWriteLock lock = new ReentrantReadWriteLock();
- private AtomicLong cacheHitNum = new AtomicLong();
- private AtomicLong cacheRequestNum = new AtomicLong();
+ private final AtomicLong cacheHitNum = new AtomicLong();
+ private final AtomicLong cacheRequestNum = new AtomicLong();
private ChunkMetadataCache(long memoryThreshold) {
- if (cacheEnable) {
+ if (CACHE_ENABLE) {
logger.info("ChunkMetadataCache size = " + memoryThreshold);
}
- lruCache = new LRULinkedHashMap<String,
List<ChunkMetadata>>(memoryThreshold, true) {
+ lruCache = new LRULinkedHashMap<AccountableString,
List<ChunkMetadata>>(memoryThreshold) {
@Override
- protected long calEntrySize(String key, List<ChunkMetadata> value) {
+ protected long calEntrySize(AccountableString key, List<ChunkMetadata>
value) {
if (value.isEmpty()) {
- return key.getBytes().length + averageSize * value.size();
+ return RamUsageEstimator.sizeOf(key) +
RamUsageEstimator.shallowSizeOf(value);
}
+ long entrySize;
if (count < 10) {
- long currentSize = RamUsageEstimator.shallowSizeOf(value.get(0)) +
RamUsageEstimator
- .shallowSizeOf(value.get(0).getStatistics());
+ long currentSize = value.get(0).calculateRamSize();
averageSize = ((averageSize * count) + currentSize) / (++count);
IoTDBConfigDynamicAdapter.setChunkMetadataSizeInByte(averageSize);
- return key.getBytes().length + currentSize * value.size();
+ entrySize = RamUsageEstimator.sizeOf(key)
+ + (currentSize + RamUsageEstimator.NUM_BYTES_OBJECT_REF) *
value.size()
+ + RamUsageEstimator.shallowSizeOf(value);
} else if (count < 100000) {
count++;
- return key.getBytes().length + averageSize * value.size();
+ entrySize = RamUsageEstimator.sizeOf(key)
+ + (averageSize + RamUsageEstimator.NUM_BYTES_OBJECT_REF) *
value.size()
+ + RamUsageEstimator.shallowSizeOf(value);
} else {
- averageSize = RamUsageEstimator.shallowSizeOf(value.get(0)) +
RamUsageEstimator
- .shallowSizeOf(value.get(0).getStatistics());
+ averageSize = value.get(0).calculateRamSize();
count = 1;
- return key.getBytes().length + averageSize * value.size();
+ entrySize = RamUsageEstimator.sizeOf(key)
+ + (averageSize + RamUsageEstimator.NUM_BYTES_OBJECT_REF) *
value.size()
+ + RamUsageEstimator.shallowSizeOf(value);
}
+ return entrySize;
}
};
}
@@ -99,7 +107,7 @@ public class ChunkMetadataCache {
*/
public List<ChunkMetadata> get(String filePath, Path seriesPath)
throws IOException {
- if (!cacheEnable) {
+ if (!CACHE_ENABLE) {
// bloom filter part
TsFileSequenceReader tsFileReader =
FileReaderManager.getInstance().get(filePath, true);
BloomFilter bloomFilter = tsFileReader.readBloomFilter();
@@ -115,8 +123,8 @@ public class ChunkMetadataCache {
return tsFileReader.getChunkMetadataList(seriesPath);
}
- String key = (filePath + IoTDBConstant.PATH_SEPARATOR
- + seriesPath.getDevice() + seriesPath.getMeasurement()).intern();
+ AccountableString key = new AccountableString(filePath +
IoTDBConstant.PATH_SEPARATOR
+ + seriesPath.getDevice() + seriesPath.getMeasurement());
cacheRequestNum.incrementAndGet();
@@ -148,7 +156,7 @@ public class ChunkMetadataCache {
List<ChunkMetadata> chunkMetaDataList = FileLoaderUtils
.getChunkMetadataList(seriesPath, filePath);
lruCache.put(key, chunkMetaDataList);
- return chunkMetaDataList;
+ return new ArrayList<>(chunkMetaDataList);
} finally {
lock.writeLock().unlock();
}
@@ -192,15 +200,19 @@ public class ChunkMetadataCache {
* clear LRUCache.
*/
public void clear() {
- synchronized (lruCache) {
+ lock.writeLock().lock();
+ if (lruCache != null) {
lruCache.clear();
}
+ lock.writeLock().unlock();
}
public void remove(TsFileResource resource) {
- synchronized (lruCache) {
- lruCache.entrySet().removeIf(e ->
e.getKey().startsWith(resource.getPath()));
+ lock.writeLock().lock();
+ if (resource != null) {
+ lruCache.entrySet().removeIf(e ->
e.getKey().getString().startsWith(resource.getPath()));
}
+ lock.writeLock().unlock();
}
/**
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/cache/LRULinkedHashMap.java
b/server/src/main/java/org/apache/iotdb/db/engine/cache/LRULinkedHashMap.java
index 8172588..004a30b 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/cache/LRULinkedHashMap.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/cache/LRULinkedHashMap.java
@@ -19,48 +19,82 @@
package org.apache.iotdb.db.engine.cache;
+import java.util.Iterator;
import java.util.LinkedHashMap;
-import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import org.apache.iotdb.tsfile.common.cache.Accountable;
/**
* This class is an LRU cache. <b>Note: It's not thread safe.</b>
*/
-public abstract class LRULinkedHashMap<K, V> extends LinkedHashMap<K, V> {
+public abstract class LRULinkedHashMap<K extends Accountable, V> {
- private static final long serialVersionUID = 1290160928914532649L;
private static final float LOAD_FACTOR_MAP = 0.75f;
private static final int INITIAL_CAPACITY = 128;
+ private static final float RETAIN_PERCENT = 0.9f;
+ private static final int MAP_ENTRY_SIZE = 40;
+
+ private final LinkedHashMap<K, V> linkedHashMap;
+
/**
* maximum memory threshold.
*/
- private long maxMemory;
+ private final long maxMemory;
/**
* current used memory.
*/
private long usedMemory;
+ /**
+ * memory size we need to retain while the cache is full
+ */
+ private final long retainMemory;
+
protected int count = 0;
protected long averageSize = 0;
- public LRULinkedHashMap(long maxMemory, boolean isLru) {
- super(INITIAL_CAPACITY, LOAD_FACTOR_MAP, isLru);
+ public LRULinkedHashMap(long maxMemory) {
+ this.linkedHashMap = new LinkedHashMap<>(INITIAL_CAPACITY,
LOAD_FACTOR_MAP);
this.maxMemory = maxMemory;
+ this.retainMemory = (long) (maxMemory * RETAIN_PERCENT);
}
- @Override
- protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
+ public V put(K key, V value) {
+ long size = calEntrySize(key, value) + MAP_ENTRY_SIZE;
+ key.setRamSize(size);
+ usedMemory += size;
+ V v = linkedHashMap.put(key, value);
if (usedMemory > maxMemory) {
- usedMemory -= calEntrySize(eldest.getKey(), eldest.getValue());
- return true;
- } else {
- return false;
+ Iterator<Entry<K, V>> iterator = linkedHashMap.entrySet().iterator();
+ while (usedMemory > retainMemory && iterator.hasNext()) {
+ Entry<K, V> entry = iterator.next();
+ usedMemory -= entry.getKey().getRamSize();
+ iterator.remove();
+ }
}
+ return v;
}
- @Override
- public V put(K key, V value) {
- usedMemory += calEntrySize(key, value);
- return super.put(key, value);
+ public V get(K key) {
+ return linkedHashMap.get(key);
+ }
+
+ public boolean containsKey(K key) {
+ return linkedHashMap.containsKey(key);
+ }
+
+ public void clear() {
+ linkedHashMap.clear();
+ usedMemory = 0;
+ }
+
+ public V remove(K key) {
+ V v = linkedHashMap.remove(key);
+ if (v != null && key != null) {
+ usedMemory -= key.getRamSize();
+ }
+ return v;
}
/**
@@ -87,6 +121,10 @@ public abstract class LRULinkedHashMap<K, V> extends
LinkedHashMap<K, V> {
return averageSize;
}
+ public Set<Entry<K, V>> entrySet() {
+ return linkedHashMap.entrySet();
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index 8a52a51..bdeb952 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -30,10 +30,12 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.query.control.FileReaderManager;
+import org.apache.iotdb.tsfile.common.cache.Accountable;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.BloomFilter;
+import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,38 +49,45 @@ public class TimeSeriesMetadataCache {
private static final IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
private static final long MEMORY_THRESHOLD_IN_TIME_SERIES_METADATA_CACHE =
config
.getAllocateMemoryForTimeSeriesMetaDataCache();
- private static boolean cacheEnable = config.isMetaDataCacheEnable();
+ private static final boolean CACHE_ENABLE = config.isMetaDataCacheEnable();
private final LRULinkedHashMap<TimeSeriesMetadataCacheKey,
TimeseriesMetadata> lruCache;
- private AtomicLong cacheHitNum = new AtomicLong();
- private AtomicLong cacheRequestNum = new AtomicLong();
+ private final AtomicLong cacheHitNum = new AtomicLong();
+ private final AtomicLong cacheRequestNum = new AtomicLong();
private final ReadWriteLock lock = new ReentrantReadWriteLock();
private TimeSeriesMetadataCache() {
- if (cacheEnable) {
+ if (CACHE_ENABLE) {
logger
.info("TimeseriesMetadataCache size = " +
MEMORY_THRESHOLD_IN_TIME_SERIES_METADATA_CACHE);
}
lruCache = new LRULinkedHashMap<TimeSeriesMetadataCacheKey,
TimeseriesMetadata>(
- MEMORY_THRESHOLD_IN_TIME_SERIES_METADATA_CACHE, true) {
+ MEMORY_THRESHOLD_IN_TIME_SERIES_METADATA_CACHE) {
@Override
protected long calEntrySize(TimeSeriesMetadataCacheKey key,
TimeseriesMetadata value) {
+ long currentSize;
if (count < 10) {
- long currentSize = RamUsageEstimator.shallowSizeOf(key) +
RamUsageEstimator.sizeOf(value);
+ currentSize = RamUsageEstimator.shallowSizeOf(key) +
RamUsageEstimator.sizeOf(key.device)
+ + RamUsageEstimator.sizeOf(key.measurement) +
RamUsageEstimator.shallowSizeOf(value)
+ + RamUsageEstimator.sizeOf(value.getMeasurementId()) +
RamUsageEstimator
+ .shallowSizeOf(value.getStatistics());
averageSize = ((averageSize * count) + currentSize) / (++count);
- return currentSize;
} else if (count < 100000) {
count++;
- return averageSize;
+ currentSize = averageSize;
} else {
- averageSize = RamUsageEstimator.shallowSizeOf(key) +
RamUsageEstimator.sizeOf(value);
+ averageSize = RamUsageEstimator.shallowSizeOf(key) +
RamUsageEstimator.sizeOf(key.device)
+ + RamUsageEstimator.sizeOf(key.measurement) +
RamUsageEstimator.shallowSizeOf(value)
+ + RamUsageEstimator.sizeOf(value.getMeasurementId()) +
RamUsageEstimator
+ .shallowSizeOf(value.getStatistics());
count = 1;
- return averageSize;
+ currentSize = averageSize;
}
+ return currentSize;
}
};
}
@@ -89,7 +98,7 @@ public class TimeSeriesMetadataCache {
public TimeseriesMetadata get(TimeSeriesMetadataCacheKey key, Set<String>
allSensors)
throws IOException {
- if (!cacheEnable) {
+ if (!CACHE_ENABLE) {
// bloom filter part
TsFileSequenceReader reader =
FileReaderManager.getInstance().get(key.filePath, true);
BloomFilter bloomFilter = reader.readBloomFilter();
@@ -107,7 +116,7 @@ public class TimeSeriesMetadataCache {
if (lruCache.containsKey(key)) {
cacheHitNum.incrementAndGet();
printCacheLog(true);
- return lruCache.get(key);
+ return new TimeseriesMetadata(lruCache.get(key));
}
} finally {
lock.readLock().unlock();
@@ -118,7 +127,7 @@ public class TimeSeriesMetadataCache {
if (lruCache.containsKey(key)) {
cacheHitNum.incrementAndGet();
printCacheLog(true);
- return lruCache.get(key);
+ return new TimeseriesMetadata(lruCache.get(key));
}
printCacheLog(false);
// bloom filter part
@@ -134,7 +143,7 @@ public class TimeSeriesMetadataCache {
timeSeriesMetadataList.forEach(timeseriesMetadata ->
lruCache.put(new TimeSeriesMetadataCacheKey(key.filePath, key.device,
timeseriesMetadata.getMeasurementId()), timeseriesMetadata));
- return lruCache.get(key);
+ return new TimeseriesMetadata(lruCache.get(key));
} catch (IOException e) {
logger.error("something wrong happened while reading {}", key.filePath);
throw e;
@@ -198,11 +207,14 @@ public class TimeSeriesMetadataCache {
lock.writeLock().unlock();
}
- public static class TimeSeriesMetadataCacheKey {
+ public static class TimeSeriesMetadataCacheKey implements Accountable {
+
+ private final String filePath;
+ private final String device;
+ private final String measurement;
+
+ private long ramSize;
- private String filePath;
- private String device;
- private String measurement;
public TimeSeriesMetadataCacheKey(String filePath, String device, String
measurement) {
this.filePath = filePath;
@@ -228,6 +240,15 @@ public class TimeSeriesMetadataCache {
public int hashCode() {
return Objects.hash(filePath, device, measurement);
}
+
+ public void setRamSize(long size) {
+ this.ramSize = size;
+ }
+
+ @Override
+ public long getRamSize() {
+ return ramSize;
+ }
}
/**
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/executor/fill/LastPointReader.java
b/server/src/main/java/org/apache/iotdb/db/query/executor/fill/LastPointReader.java
index 2e59c31..ac80a16 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/executor/fill/LastPointReader.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/executor/fill/LastPointReader.java
@@ -103,7 +103,6 @@ public class LastPointReader {
dataType);
} else {
List<ChunkMetadata> seqChunkMetadataList =
timeseriesMetadata.loadChunkMetadataList();
-
for (int i = seqChunkMetadataList.size() - 1; i >= 0; i--) {
lastPoint = getChunkLastPoint(seqChunkMetadataList.get(i));
// last point of this sequence chunk is valid, quit the loop
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
index 11d6c15..e3c70f0 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
@@ -19,32 +19,35 @@
package org.apache.iotdb.db.query.reader.chunk;
+import java.io.IOException;
import org.apache.iotdb.db.engine.cache.ChunkCache;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
-import java.io.IOException;
-
/**
* To read one chunk from disk, and only used in iotdb server module
*/
public class DiskChunkLoader implements IChunkLoader {
- private TsFileSequenceReader reader;
+ private final TsFileResource resource;
- public DiskChunkLoader(TsFileSequenceReader reader) {
- this.reader = reader;
+ public DiskChunkLoader(TsFileResource resource) {
+ this.resource = resource;
}
@Override
public Chunk loadChunk(ChunkMetadata chunkMetaData) throws IOException {
- return ChunkCache.getInstance().get(chunkMetaData, reader);
+ TsFileSequenceReader tsFileSequenceReader =
+ FileReaderManager.getInstance().get(resource.getPath(),
resource.isClosed());
+ return ChunkCache.getInstance().get(chunkMetaData, tsFileSequenceReader);
}
@Override
public void close() throws IOException {
- reader.close();
+ // do nothing
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
index 2c5d8ea..caf51f8 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
@@ -22,11 +22,9 @@ import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.context.QueryContext;
-import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.query.reader.chunk.DiskChunkLoader;
import org.apache.iotdb.db.utils.QueryUtils;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
-import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
@@ -64,18 +62,17 @@ public class DiskChunkMetadataLoader implements
IChunkMetadataLoader {
return chunkMetadataList;
}
- public static void setDiskChunkLoader(List<ChunkMetadata> chunkMetadataList,
TsFileResource resource, Path seriesPath, QueryContext context) throws
IOException {
+ public static void setDiskChunkLoader(List<ChunkMetadata> chunkMetadataList,
+ TsFileResource resource, Path seriesPath, QueryContext context) {
List<Modification> pathModifications =
- context.getPathModifications(resource.getModFile(),
seriesPath.getFullPath());
+ context.getPathModifications(resource.getModFile(),
seriesPath.getFullPath());
if (!pathModifications.isEmpty()) {
QueryUtils.modifyChunkMetaData(chunkMetadataList, pathModifications);
}
- TsFileSequenceReader tsFileSequenceReader =
- FileReaderManager.getInstance().get(resource.getPath(),
resource.isClosed());
for (ChunkMetadata data : chunkMetadataList) {
- data.setChunkLoader(new DiskChunkLoader(tsFileSequenceReader));
+ data.setChunkLoader(new DiskChunkLoader(resource));
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
index 4a54437..b0207aa 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
@@ -44,7 +44,7 @@ public class MemChunkMetadataLoader implements
IChunkMetadataLoader {
}
@Override
- public List<ChunkMetadata> loadChunkMetadataList() throws IOException {
+ public List<ChunkMetadata> loadChunkMetadataList() {
List<ChunkMetadata> chunkMetadataList = resource.getChunkMetadataList();
DiskChunkMetadataLoader.setDiskChunkLoader(chunkMetadataList, resource,
seriesPath, context);
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/MemUtils.java
b/server/src/main/java/org/apache/iotdb/db/utils/MemUtils.java
index 6f5b494..daef654 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/MemUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/MemUtils.java
@@ -20,9 +20,9 @@ package org.apache.iotdb.db.utils;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan;
-import org.apache.iotdb.db.engine.cache.RamUsageEstimator;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Binary;
+import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.record.datapoint.BooleanDataPoint;
import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/cache/Accountable.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/cache/Accountable.java
new file mode 100644
index 0000000..0cefcc6
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/cache/Accountable.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.tsfile.common.cache;
+
+public interface Accountable {
+
+ void setRamSize(long size);
+
+ long getRamSize();
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
index 12d96c7..603c555 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
@@ -18,20 +18,21 @@
*/
package org.apache.iotdb.tsfile.file.metadata;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
-import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.Objects;
+import org.apache.iotdb.tsfile.common.cache.Accountable;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
+import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
+import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
/**
* Metadata of one chunk.
*/
-public class ChunkMetadata {
+public class ChunkMetadata implements Accountable {
private String measurementUid;
@@ -62,6 +63,11 @@ public class ChunkMetadata {
private Statistics statistics;
+ private long ramSize;
+
+ private static final int CHUNK_METADATA_FIXED_RAM_SIZE = 80;
+
+
private ChunkMetadata() {
}
@@ -69,9 +75,9 @@ public class ChunkMetadata {
* constructor of ChunkMetaData.
*
* @param measurementUid measurement id
- * @param tsDataType time series data type
- * @param fileOffset file offset
- * @param statistics value statistics
+ * @param tsDataType time series data type
+ * @param fileOffset file offset
+ * @param statistics value statistics
*/
public ChunkMetadata(String measurementUid, TSDataType tsDataType, long
fileOffset,
Statistics statistics) {
@@ -209,4 +215,18 @@ public class ChunkMetadata {
public void setModified(boolean modified) {
this.modified = modified;
}
+
+ public long calculateRamSize() {
+ return CHUNK_METADATA_FIXED_RAM_SIZE +
RamUsageEstimator.sizeOf(measurementUid) + statistics
+ .calculateRamSize();
+ }
+
+ public void setRamSize(long size) {
+ this.ramSize = size;
+ }
+
+ @Override
+ public long getRamSize() {
+ return ramSize;
+ }
}
\ No newline at end of file
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index 0522ed7..d4d674f 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -23,12 +23,13 @@ import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.List;
+import org.apache.iotdb.tsfile.common.cache.Accountable;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-public class TimeseriesMetadata {
+public class TimeseriesMetadata implements Accountable {
private long startOffsetOfChunkMetaDataList;
private int chunkMetaDataListDataSize;
@@ -43,6 +44,20 @@ public class TimeseriesMetadata {
private IChunkMetadataLoader chunkMetadataLoader;
+ private long ramSize;
+
+ public TimeseriesMetadata() {
+ }
+
+ public TimeseriesMetadata(TimeseriesMetadata timeseriesMetadata) {
+ this.startOffsetOfChunkMetaDataList =
timeseriesMetadata.startOffsetOfChunkMetaDataList;
+ this.chunkMetaDataListDataSize =
timeseriesMetadata.chunkMetaDataListDataSize;
+ this.measurementId = timeseriesMetadata.measurementId;
+ this.tsDataType = timeseriesMetadata.tsDataType;
+ this.statistics = timeseriesMetadata.statistics;
+ this.modified = timeseriesMetadata.modified;
+ }
+
public static TimeseriesMetadata deserializeFrom(ByteBuffer buffer) {
TimeseriesMetadata timeseriesMetaData = new TimeseriesMetadata();
timeseriesMetaData.setMeasurementId(ReadWriteIOUtils.readString(buffer));
@@ -125,4 +140,13 @@ public class TimeseriesMetadata {
public void setModified(boolean modified) {
this.modified = modified;
}
+
+ public void setRamSize(long size) {
+ this.ramSize = size;
+ }
+
+ @Override
+ public long getRamSize() {
+ return ramSize;
+ }
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java
index 6ddd7b7..36c2b62 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.tsfile.file.metadata.statistics;
import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Binary;
+import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import java.io.IOException;
@@ -131,6 +132,11 @@ public class BinaryStatistics extends Statistics<Binary> {
}
@Override
+ public long calculateRamSize() {
+ return RamUsageEstimator.sizeOf(this);
+ }
+
+ @Override
public byte[] getMinValueBytes() {
throw new StatisticsClassException("Binary statistics does not support:
min");
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java
index d298055..77b91e6 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java
@@ -33,6 +33,9 @@ public class BooleanStatistics extends Statistics<Boolean> {
private boolean firstValue;
private boolean lastValue;
+ private static final int BOOLEAN_STATISTICS_FIXED_RAM_SIZE = 48;
+
+
@Override
public TSDataType getType() {
return TSDataType.BOOLEAN;
@@ -88,6 +91,11 @@ public class BooleanStatistics extends Statistics<Boolean> {
}
@Override
+ public long calculateRamSize() {
+ return BOOLEAN_STATISTICS_FIXED_RAM_SIZE;
+ }
+
+ @Override
public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java
index 6b87fc8..dddc0da 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java
@@ -35,6 +35,8 @@ public class DoubleStatistics extends Statistics<Double> {
private double lastValue;
private double sumValue;
+ private static final int DOUBLE_STATISTICS_FIXED_RAM_SIZE = 80;
+
@Override
public TSDataType getType() {
return TSDataType.DOUBLE;
@@ -116,6 +118,11 @@ public class DoubleStatistics extends Statistics<Double> {
}
@Override
+ public long calculateRamSize() {
+ return DOUBLE_STATISTICS_FIXED_RAM_SIZE;
+ }
+
+ @Override
public Double getMinValue() {
return minValue;
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java
index 16ce7be..dd3ad03 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java
@@ -38,6 +38,9 @@ public class FloatStatistics extends Statistics<Float> {
private float lastValue;
private double sumValue;
+ private static final int FLOAT_STATISTICS_FIXED_RAM_SIZE = 64;
+
+
@Override
public TSDataType getType() {
return TSDataType.FLOAT;
@@ -110,6 +113,11 @@ public class FloatStatistics extends Statistics<Float> {
}
@Override
+ public long calculateRamSize() {
+ return FLOAT_STATISTICS_FIXED_RAM_SIZE;
+ }
+
+ @Override
public Float getMinValue() {
return minValue;
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java
index e2c945d..e3f066a 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java
@@ -38,6 +38,9 @@ public class IntegerStatistics extends Statistics<Integer> {
private int lastValue;
private double sumValue;
+ private static final int INTEGER_STATISTICS_FIXED_RAM_SIZE = 64;
+
+
@Override
public TSDataType getType() {
return TSDataType.INT32;
@@ -111,6 +114,11 @@ public class IntegerStatistics extends Statistics<Integer>
{
}
@Override
+ public long calculateRamSize() {
+ return INTEGER_STATISTICS_FIXED_RAM_SIZE;
+ }
+
+ @Override
public Integer getMinValue() {
return minValue;
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java
index 075e582..55c61a8 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java
@@ -35,6 +35,9 @@ public class LongStatistics extends Statistics<Long> {
private long lastValue;
private double sumValue;
+ private static final int LONG_STATISTICS_FIXED_RAM_SIZE = 80;
+
+
@Override
public TSDataType getType() {
return TSDataType.INT64;
@@ -142,6 +145,11 @@ public class LongStatistics extends Statistics<Long> {
}
@Override
+ public long calculateRamSize() {
+ return LONG_STATISTICS_FIXED_RAM_SIZE;
+ }
+
+ @Override
protected void mergeStatisticsValue(Statistics stats) {
LongStatistics longStats = (LongStatistics) stats;
if (isEmpty) {
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
index 9be295f..130a2bf 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
@@ -411,6 +411,8 @@ public abstract class Statistics<T> {
this.count = count;
}
+ public abstract long calculateRamSize();
+
@Override
public String toString() {
return "startTime: " + startTime + " endTime: " + endTime + " count: " +
count;
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index f924d83..0bf0cf1 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -959,7 +959,7 @@ public class TsFileSequenceReader implements AutoCloseable {
public List<ChunkMetadata> readChunkMetaDataList(TimeseriesMetadata
timeseriesMetaData)
throws IOException {
List<Pair<Long, Long>> versionInfo = tsFileMetaData.getVersionInfo();
- List<ChunkMetadata> chunkMetadataList = new ArrayList<>();
+ ArrayList<ChunkMetadata> chunkMetadataList = new ArrayList<>();
long startOffsetOfChunkMetadataList =
timeseriesMetaData.getOffsetOfChunkMetaDataList();
int dataSizeOfChunkMetadataList =
timeseriesMetaData.getDataSizeOfChunkMetaDataList();
@@ -970,6 +970,8 @@ public class TsFileSequenceReader implements AutoCloseable {
VersionUtils.applyVersion(chunkMetadataList, versionInfo);
+ // minimize the storage of an ArrayList instance.
+ chunkMetadataList.trimToSize();
return chunkMetadataList;
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java
index 5120d08..b50b1cf 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java
@@ -20,13 +20,14 @@ package org.apache.iotdb.tsfile.read.common;
import java.nio.ByteBuffer;
+import org.apache.iotdb.tsfile.common.cache.Accountable;
import org.apache.iotdb.tsfile.encoding.common.EndianType;
import org.apache.iotdb.tsfile.file.header.ChunkHeader;
/**
* used in query.
*/
-public class Chunk {
+public class Chunk implements Accountable {
private ChunkHeader chunkHeader;
private ByteBuffer chunkData;
@@ -36,6 +37,8 @@ public class Chunk {
private long deletedAt;
private EndianType endianType;
+ private long ramSize;
+
public Chunk(ChunkHeader header, ByteBuffer buffer, long deletedAt,
EndianType endianType) {
this.chunkHeader = header;
this.chunkData = buffer;
@@ -62,4 +65,14 @@ public class Chunk {
public void setDeletedAt(long deletedAt) {
this.deletedAt = deletedAt;
}
+
+ @Override
+ public void setRamSize(long size) {
+ this.ramSize = size;
+ }
+
+ @Override
+ public long getRamSize() {
+ return ramSize;
+ }
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/cache/RamUsageEstimator.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/RamUsageEstimator.java
similarity index 98%
rename from
server/src/main/java/org/apache/iotdb/db/engine/cache/RamUsageEstimator.java
rename to
tsfile/src/main/java/org/apache/iotdb/tsfile/utils/RamUsageEstimator.java
index 74517f8..5089b78 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/cache/RamUsageEstimator.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/RamUsageEstimator.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.db.engine.cache;
+package org.apache.iotdb.tsfile.utils;
import java.lang.management.ManagementFactory;
import java.lang.reflect.Array;
@@ -50,7 +50,7 @@ public final class RamUsageEstimator {
/**
* JVM diagnostic features.
*/
- public static enum JvmFeature {
+ public enum JvmFeature {
OBJECT_REFERENCE_SIZE("Object reference size estimated using array index
scale"),
ARRAY_HEADER_SIZE("Array header size estimated using array based offset"),
FIELD_OFFSETS("Shallow instance size based on field offsets"),
@@ -71,7 +71,7 @@ public final class RamUsageEstimator {
/**
* JVM info string for debugging and reports.
*/
- public final static String JVM_INFO_STRING;
+ public static final String JVM_INFO_STRING;
/**
* One kilobyte bytes.
@@ -106,23 +106,23 @@ public final class RamUsageEstimator {
/**
* Number of bytes this jvm uses to represent an object reference.
*/
- public final static int NUM_BYTES_OBJECT_REF;
+ public static final int NUM_BYTES_OBJECT_REF;
/**
* Number of bytes to represent an object header (no fields, no alignments).
*/
- public final static int NUM_BYTES_OBJECT_HEADER;
+ public static final int NUM_BYTES_OBJECT_HEADER;
/**
* Number of bytes to represent an array header (no content, but with
alignments).
*/
- public final static int NUM_BYTES_ARRAY_HEADER;
+ public static final int NUM_BYTES_ARRAY_HEADER;
/**
* A constant specifying the object alignment boundary inside the JVM.
Objects will always take a
* full multiple of this constant, possibly wasting some space.
*/
- public final static int NUM_BYTES_OBJECT_ALIGNMENT;
+ public static final int NUM_BYTES_OBJECT_ALIGNMENT;
/**
* Sizes of primitive classes.
@@ -144,17 +144,17 @@ public final class RamUsageEstimator {
/**
* A handle to <code>sun.misc.Unsafe</code>.
*/
- private final static Object theUnsafe;
+ private static final Object theUnsafe;
/**
* A handle to <code>sun.misc.Unsafe#fieldOffset(Field)</code>.
*/
- private final static Method objectFieldOffsetMethod;
+ private static final Method objectFieldOffsetMethod;
/**
* All the supported "internal" JVM features detected at clinit.
*/
- private final static EnumSet<JvmFeature> supportedFeatures;
+ private static final EnumSet<JvmFeature> supportedFeatures;
/**
* Initialize constants and try to collect information about the JVM
internals.
@@ -957,7 +957,7 @@ public final class RamUsageEstimator {
*/
@Deprecated
public static final boolean JRE_IS_MINIMUM_JAVA6 =
- new Boolean(true).booleanValue(); // prevent inlining in foreign
class files
+ Boolean.TRUE; // prevent inlining in foreign class files
public static final boolean JRE_IS_MINIMUM_JAVA7;
public static final boolean JRE_IS_MINIMUM_JAVA8;
