This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 6eda5e4 [ISSUE-2688] LRULinkedHashMap does not work as an LRU Cache
(#2689)
6eda5e4 is described below
commit 6eda5e478beca32bfc042d9313420423cb97a2d9
Author: chenjun40 <[email protected]>
AuthorDate: Fri Feb 26 13:24:41 2021 +0800
[ISSUE-2688] LRULinkedHashMap does not work as an LRU Cache (#2689)
Co-authored-by: chenjun40 <[email protected]>
---
.../iotdb/db/engine/cache/ChunkMetadataCache.java | 7 ++++
.../iotdb/db/engine/cache/LRULinkedHashMap.java | 2 +-
.../db/engine/cache/ChunkMetadataCacheTest.java | 44 +++++++++++++++++++---
3 files changed, 47 insertions(+), 6 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
index 40db664..7fe2854 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
@@ -37,6 +37,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -219,6 +221,11 @@ public class ChunkMetadataCache {
return lruCache.isEmpty();
}
+ @TestOnly
+ public Set<Entry<AccountableString, List<ChunkMetadata>>> entrySet() {
+ return lruCache.entrySet();
+ }
+
/** singleton pattern. */
private static class ChunkMetadataCacheSingleton {
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/cache/LRULinkedHashMap.java
b/server/src/main/java/org/apache/iotdb/db/engine/cache/LRULinkedHashMap.java
index 82f56fa..6246574 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/cache/LRULinkedHashMap.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/cache/LRULinkedHashMap.java
@@ -48,7 +48,7 @@ public abstract class LRULinkedHashMap<K extends Accountable,
V> {
protected long averageSize = 0;
public LRULinkedHashMap(long maxMemory) {
- this.linkedHashMap = new LinkedHashMap<>(INITIAL_CAPACITY,
LOAD_FACTOR_MAP);
+ this.linkedHashMap = new LinkedHashMap<>(INITIAL_CAPACITY,
LOAD_FACTOR_MAP, true);
this.maxMemory = maxMemory;
this.retainMemory = (long) (maxMemory * RETAIN_PERCENT);
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java
index 0f96c0f..1e47343 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.db.engine.cache;
+import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.constant.TestConstant;
import org.apache.iotdb.db.engine.MetadataManagerHelper;
@@ -35,7 +36,9 @@ import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
@@ -48,6 +51,9 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.stream.Collectors;
public class ChunkMetadataCacheTest {
@@ -160,7 +166,6 @@ public class ChunkMetadataCacheTest {
List<TsFileResource> seqResources = queryDataSource.getSeqResources();
List<TsFileResource> unseqResources = queryDataSource.getUnseqResources();
-
Assert.assertEquals(1, seqResources.size());
Assert.assertEquals(3, unseqResources.size());
Assert.assertTrue(seqResources.get(0).isClosed());
@@ -168,9 +173,38 @@ public class ChunkMetadataCacheTest {
Assert.assertTrue(unseqResources.get(1).isClosed());
Assert.assertTrue(unseqResources.get(2).isClosed());
- List<ChunkMetadata> metaDataList =
- ChunkMetadataCache.getInstance()
- .get(seqResources.get(0).getTsFilePath(), new Path(storageGroup,
measurementId5), null);
- Assert.assertEquals(0, metaDataList.size());
+ String seqTsFilePath = seqResources.get(0).getTsFilePath();
+ TsFileSequenceReader tsFileReader =
FileReaderManager.getInstance().get(seqTsFilePath, true);
+ Map<String, List<TimeseriesMetadata>> stringListMap =
tsFileReader.getAllTimeseriesMetadata();
+ List<TimeseriesMetadata> timeseriesMetadataList =
stringListMap.get(storageGroup);
+
+ ChunkMetadataCache chunkMetadataCache = ChunkMetadataCache.getInstance();
+ chunkMetadataCache.clear();
+ double hitRatio = chunkMetadataCache.calculateChunkMetaDataHitRatio();
+ for (TimeseriesMetadata tsMd : timeseriesMetadataList) {
+ chunkMetadataCache.get(seqTsFilePath, new Path(storageGroup,
tsMd.getMeasurementId()), tsMd);
+ }
+ double decreasedHitRatio =
chunkMetadataCache.calculateChunkMetaDataHitRatio();
+ Assert.assertTrue("The hit rate should not increase", decreasedHitRatio <=
hitRatio);
+
+ // After an element is accessed, the newest element is put at the tail, so
the head is the
+ // eldest one
+ TimeseriesMetadata tsMd2 = timeseriesMetadataList.get(2);
+ chunkMetadataCache.get(seqTsFilePath, new Path(storageGroup,
tsMd2.getMeasurementId()), tsMd2);
+
+ String keyPrefix =
+ seqTsFilePath + IoTDBConstant.PATH_SEPARATOR + storageGroup +
IoTDBConstant.PATH_SEPARATOR;
+ AccountableString key0 =
+ new AccountableString(keyPrefix +
timeseriesMetadataList.get(0).getMeasurementId());
+ AccountableString key2 =
+ new AccountableString(keyPrefix +
timeseriesMetadataList.get(2).getMeasurementId());
+
+ List<AccountableString> keyList =
+
chunkMetadataCache.entrySet().stream().map(Entry::getKey).collect(Collectors.toList());
+ Assert.assertEquals("Now, the eldest(head) key should be key0",
keyList.get(0), key0);
+ Assert.assertEquals("Now, the newest(tail) key should be key2",
keyList.get(4), key2);
+
+ double increasedHitRatio =
ChunkMetadataCache.getInstance().calculateChunkMetaDataHitRatio();
+ Assert.assertTrue("The hit rate should increase", decreasedHitRatio <
increasedHitRatio);
}
}
