LeiRui commented on a change in pull request #262: [IOTDB-144]meta data cache
for query
URL: https://github.com/apache/incubator-iotdb/pull/262#discussion_r306198470
##########
File path:
iotdb/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
##########
@@ -19,77 +19,144 @@
package org.apache.iotdb.db.engine.cache;
import java.io.IOException;
-import java.util.LinkedHashMap;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
-import java.util.Objects;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
+import org.apache.iotdb.tsfile.read.common.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * This class is used to cache <code>DeviceMetaDataCache</code> of tsfile in
IoTDB.
+ * This class is used to cache <code>List<ChunkMetaData></code> of tsfile in
IoTDB. The caching
+ * strategy is LRU.
*/
public class DeviceMetaDataCache {
private static final Logger logger =
LoggerFactory.getLogger(DeviceMetaDataCache.class);
+ private static final IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
- private static final int CACHE_SIZE = 100;
+ private static StorageEngine storageEngine = StorageEngine.getInstance();
+
+ private static final long MEMORY_THRESHOLD_IN_B = (long) (0.3 * config
+ .getAllocateMemoryForRead());
/**
- * key: the file path + deviceId.
+ * key: file path dot deviceId dot sensorId.
+ * <p>
+ * value: chunkMetaData list of one timeseries in the file.
*/
- private LinkedHashMap<String, TsDeviceMetadata> lruCache;
+ private LruLinkedHashMap<String, List<ChunkMetaData>> lruCache;
private AtomicLong cacheHintNum = new AtomicLong();
private AtomicLong cacheRequestNum = new AtomicLong();
- private DeviceMetaDataCache(int cacheSize) {
- lruCache = new LruLinkedHashMap(cacheSize, true);
+ /**
+ * approximate estimation of chunkMetaData size
+ */
+ private long chunkMetaDataSize = 0;
+
+ private DeviceMetaDataCache(long memoryThreshold) {
+ lruCache = new LruLinkedHashMap<String,
List<ChunkMetaData>>(memoryThreshold, true) {
+ @Override
+ protected long calEntrySize(String key, List<ChunkMetaData> value) {
+ if (chunkMetaDataSize == 0 && !value.isEmpty()) {
+ chunkMetaDataSize = RamUsageEstimator.sizeOf(value.get(0));
+ }
+ return value.size() * chunkMetaDataSize + key.length() * 2;
+ }
+ };
}
public static DeviceMetaDataCache getInstance() {
return RowGroupBlockMetaDataCacheSingleton.INSTANCE;
}
/**
- * get {@link TsDeviceMetadata}. THREAD SAFE.
+ * get {@link ChunkMetaData}. THREAD SAFE.
*/
- public TsDeviceMetadata get(String filePath, String deviceId, TsFileMetaData
fileMetaData)
+ public List<ChunkMetaData> get(String filePath, Path seriesPath)
throws IOException {
- // The key(the tsfile path and deviceId) for the lruCache
+ StringBuilder builder = new
StringBuilder(filePath).append(".").append(seriesPath.getDevice());
+ String pathDeviceStr = builder.toString();
+ String key =
builder.append(".").append(seriesPath.getMeasurement()).toString();
+ Object devicePathObject = pathDeviceStr.intern();
- String jointPath = filePath + deviceId;
- Object jointPathObject = jointPath.intern();
synchronized (lruCache) {
cacheRequestNum.incrementAndGet();
- if (lruCache.containsKey(jointPath)) {
+ if (lruCache.containsKey(key)) {
cacheHintNum.incrementAndGet();
if (logger.isDebugEnabled()) {
logger.debug(
- "Cache hint: the number of requests for cache is {}, "
+ "Cache hit: the number of requests for cache is {}, "
+ "the number of hints for cache is {}",
cacheRequestNum.get(), cacheHintNum.get());
}
- return lruCache.get(jointPath);
+ return new ArrayList<>(lruCache.get(key));
}
}
- synchronized (jointPathObject) {
+ synchronized (devicePathObject) {
synchronized (lruCache) {
- if (lruCache.containsKey(jointPath)) {
- return lruCache.get(jointPath);
+ if (lruCache.containsKey(key)) {
+ cacheHintNum.incrementAndGet();
+ return new ArrayList<>(lruCache.get(key));
}
}
if (logger.isDebugEnabled()) {
- logger.debug("Cache didn't hint: the number of requests for cache is
{}",
+ logger.debug("Cache didn't hit: the number of requests for cache is
{}",
cacheRequestNum.get());
}
+ TsFileMetaData fileMetaData =
TsFileMetaDataCache.getInstance().get(filePath);
TsDeviceMetadata blockMetaData = TsFileMetadataUtils
- .getTsRowGroupBlockMetaData(filePath, deviceId,
+ .getTsDeviceMetaData(filePath, seriesPath.getDevice(),
Review comment:
`seriesPath.getMeasurement` can be used in the `getTsDeviceMetaData` to help
filtering by adding the following logic:
```
if (!fileMetaData.getMeasurementSchema().containsKey(measurementId)) {
return null;
}
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
