This is an automated email from the ASF dual-hosted git repository. JackieTien97 pushed a commit to branch ty/tsfile-resource-memory in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit aeb877dcec2a48c7c9e7a29bdafcc12b2d4f6d3c Author: JackieTien97 <[email protected]> AuthorDate: Fri Nov 20 17:44:20 2026 +0800 Fix heap accounting for sealed TsFile resources Estimate resource metadata and device indexes separately. Track the charge for each registered file so metadata changes, degradation, and removal update the time-index budget consistently. --- .../plan/relational/sql/ast/LoadTsFile.java | 24 +-- .../schedule/CompactionScheduleContext.java | 8 +- .../dataregion/modification/ModificationFile.java | 19 +- .../dataregion/tsfile/TsFileResource.java | 189 +++++++++++++++----- .../tsfile/timeindex/ArrayDeviceTimeIndex.java | 30 +++- .../dataregion/tsfile/timeindex/FileTimeIndex.java | 2 +- .../rescon/memory/TsFileResourceManager.java | 194 +++++++++++++++------ .../java/org/apache/iotdb/db/utils/MemUtils.java | 34 ++++ .../rescon/memory/ResourceManagerTest.java | 137 ++++++++++++++- 9 files changed, 518 insertions(+), 119 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java index 2e9adadb576..8768305c4e1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java @@ -30,6 +30,7 @@ import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator; +import org.apache.iotdb.db.utils.MemUtils; import org.apache.tsfile.utils.RamUsageEstimator; @@ -49,9 +50,6 @@ public class LoadTsFile extends Statement { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(LoadTsFile.class); - private static final long FILE_INSTANCE_SIZE = - RamUsageEstimator.shallowSizeOfInstance(File.class); - private String filePath; private int databaseLevel; // For loading to tree-model only @@ -389,14 +387,6 @@ public class LoadTsFile extends Statement { size += RamUsageEstimator.sizeOf(filePath); size += RamUsageEstimator.sizeOf(database); size += RamUsageEstimator.sizeOfMap(loadAttributes); - if (tsFiles != null) { - size += RamUsageEstimator.shallowSizeOf(tsFiles); - for (File file : tsFiles) { - if (file != null) { - size += FILE_INSTANCE_SIZE; - } - } - } if (resources != null) { size += RamUsageEstimator.shallowSizeOf(resources); for (TsFileResource resource : resources) { @@ -405,6 +395,18 @@ public class LoadTsFile extends Statement { } } } + if (tsFiles != null) { + size += RamUsageEstimator.shallowSizeOf(tsFiles); + for (int i = 0; i < tsFiles.size(); i++) { + File file = tsFiles.get(i); + // Analysis appends one resource (possibly null) for each processed file, in order. + TsFileResource resource = + resources != null && i < resources.size() ? resources.get(i) : null; + if (resource == null || resource.getTsFile() != file) { + size += MemUtils.getFileRamBytes(file); + } + } + } size += AstMemoryEstimationHelper.getEstimatedSizeOfLongList(writePointCountList); size += RamUsageEstimator.shallowSizeOf(isTableModel); return size; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleContext.java index 6f0f7872789..b9fde5ab83a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleContext.java @@ -89,9 +89,13 @@ public class CompactionScheduleContext { public void addResourceDeviceTimeIndex( TsFileResource tsFileResource, ArrayDeviceTimeIndex deviceTimeIndex) { - partitionFileDeviceInfoCache.put(tsFileResource, deviceTimeIndex); + ArrayDeviceTimeIndex previous = + partitionFileDeviceInfoCache.put(tsFileResource, deviceTimeIndex); + if (previous == deviceTimeIndex) { + return; + } long deviceTimeIndexSize = - tsFileResource.getDeviceTimeIndexRamSize().orElse(deviceTimeIndex.calculateRamSize()); + deviceTimeIndex.calculateRamSize() - (previous == null ? 0 : previous.calculateRamSize()); cachedDeviceInfoSize += deviceTimeIndexSize; CompactionMetrics.getInstance().addSelectionCachedDeviceTimeIndexSize(deviceTimeIndexSize); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java index 5ef17e9c01b..d9c62f02dd0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java @@ -28,7 +28,10 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion; import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Modification; import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.utils.MemUtils; +import org.apache.tsfile.utils.Accountable; +import org.apache.tsfile.utils.RamUsageEstimator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,6 +54,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.NoSuchElementException; import java.util.Set; +import java.util.concurrent.locks.AbstractQueuedSynchronizer; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; @@ -59,7 +63,14 @@ import static java.nio.file.StandardOpenOption.CREATE; import static org.apache.iotdb.db.utils.ModificationUtils.sortAndMerge; import static org.apache.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX; -public class ModificationFile implements AutoCloseable { +public class ModificationFile implements AutoCloseable, Accountable { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(ModificationFile.class) + + RamUsageEstimator.shallowSizeOfInstance(ReentrantReadWriteLock.class) + + RamUsageEstimator.shallowSizeOfInstance(ReentrantReadWriteLock.ReadLock.class) + + RamUsageEstimator.shallowSizeOfInstance(ReentrantReadWriteLock.WriteLock.class) + + RamUsageEstimator.shallowSizeOfInstance(AbstractQueuedSynchronizer.class); public static final String FILE_SUFFIX = ".mods2"; public static final String COMPACTION_FILE_SUFFIX = ".compaction.mods2"; @@ -92,6 +103,12 @@ public class ModificationFile implements AutoCloseable { } } + /** Long-lived handle and lock. Open streams and cascaded files have separate lifetimes. */ + @Override + public long ramBytesUsed() { + return INSTANCE_SIZE + MemUtils.getFileRamBytes(file); + } + public void writeLock() { this.lock.writeLock().lock(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index 5629173986a..f5f6e0f7061 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -48,8 +48,9 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDevice import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.PlainDeviceTimeIndex; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.TimeIndexLevel; import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; +import org.apache.iotdb.db.storageengine.rescon.memory.TsFileResourceManager.Registration; +import org.apache.iotdb.db.utils.MemUtils; import com.google.common.util.concurrent.RateLimiter; import org.apache.tsfile.file.metadata.IChunkMetadata; @@ -95,6 +96,7 @@ import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import static org.apache.iotdb.commons.conf.IoTDBConstant.FILE_NAME_SEPARATOR; import static org.apache.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX; @@ -104,8 +106,17 @@ public class TsFileResource implements PersistentResource, Cloneable { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(TsFileResource.class) - + RamUsageEstimator.shallowSizeOfInstance(TsFileRepairStatus.class) + + RamUsageEstimator.shallowSizeOfInstance(TsFileLock.class) + + 3L * RamUsageEstimator.shallowSizeOfInstance(AtomicReference.class) + + RamUsageEstimator.shallowSizeOfInstance(AtomicBoolean.class) + TsFileID.SHALLOW_SIZE; + private static final long TIER_SIZE = + RamUsageEstimator.shallowSizeOfInstance(AtomicInteger.class); + private static final long PAIR_SIZE = RamUsageEstimator.shallowSizeOfInstance(Pair.class); + private static final long TIME_VALUE_PAIR_SIZE = + RamUsageEstimator.shallowSizeOfInstance(TimeValuePair.class); + private static final AtomicReferenceFieldUpdater<TsFileResource, ITimeIndex> TIME_INDEX_UPDATER = + AtomicReferenceFieldUpdater.newUpdater(TsFileResource.class, ITimeIndex.class, "timeIndex"); private static final Logger LOGGER = LoggerFactory.getLogger(TsFileResource.class); @@ -174,7 +185,7 @@ public class TsFileResource implements PersistentResource, Cloneable { private TsFileID tsFileID; - private long deviceTimeIndexRamSize; + private volatile Registration memoryRegistration; private AtomicInteger tierLevel; @@ -213,7 +224,7 @@ public class TsFileResource implements PersistentResource, Cloneable { private InsertionCompactionCandidateStatus insertionCompactionCandidateStatus = InsertionCompactionCandidateStatus.NOT_CHECKED; - private Map<IDeviceID, List<Pair<String, TimeValuePair>>> lastValues; + private volatile Map<IDeviceID, List<Pair<String, TimeValuePair>>> lastValues; @TestOnly public TsFileResource() { @@ -371,6 +382,8 @@ public class TsFileResource implements PersistentResource, Cloneable { break; } } + } finally { + updateMemoryCostIfRegistered(true); } } @@ -396,10 +409,12 @@ public class TsFileResource implements PersistentResource, Cloneable { public void updateStartTime(IDeviceID device, long time) { timeIndex.updateStartTime(device, time); + updateMemoryCostIfRegistered(true); } public void updateEndTime(IDeviceID device, long time) { timeIndex.updateEndTime(device, time); + updateMemoryCostIfRegistered(true); } public boolean resourceFileExists() { @@ -560,6 +575,7 @@ public class TsFileResource implements PersistentResource, Cloneable { } else { exclusiveModFile = ModificationFile.getExclusiveMods(this); } + updateMemoryCostIfRegistered(); } } return exclusiveModFile; @@ -570,6 +586,7 @@ public class TsFileResource implements PersistentResource, Cloneable { synchronized (this) { if (compactionModFile == null) { compactionModFile = ModificationFile.getCompactionMods(this); + updateMemoryCostIfRegistered(); } } } @@ -580,7 +597,7 @@ public class TsFileResource implements PersistentResource, Cloneable { if (compactionModFileExists() && !useSharedModFile) { getCompactionModFile().remove(); } - compactionModFile = null; + setCompactionModFile(null); } @TestOnly @@ -589,6 +606,7 @@ public class TsFileResource implements PersistentResource, Cloneable { synchronized (this) { exclusiveModFile.close(); exclusiveModFile = null; + updateMemoryCostIfRegistered(); } } } @@ -596,6 +614,7 @@ public class TsFileResource implements PersistentResource, Cloneable { public void setFile(File file) { this.file = file; this.tsFileID = new TsFileID(file.getAbsolutePath()); + updateMemoryCostIfRegistered(); } public File getTsFile() { @@ -794,6 +813,7 @@ public class TsFileResource implements PersistentResource, Cloneable { pathToReadOnlyMemChunkMap = null; pathToTimeSeriesMetadataMap = null; timeIndex.close(); + updateMemoryCostIfRegistered(true); } public TsFileProcessor getProcessor() { @@ -885,7 +905,7 @@ public class TsFileResource implements PersistentResource, Cloneable { // Note that pipe can safely handle the case that the time index does not exist isEmpty(); if (getStatus() != TsFileResourceStatus.UNCLOSED) { - degradeTimeIndex(); + replaceDeviceTimeIndexWithFileTimeIndex(); } forceMarkDeleted(); try { @@ -1167,33 +1187,108 @@ public class TsFileResource implements PersistentResource, Cloneable { public void setExclusiveModFile(ModificationFile exclusiveModFile) { synchronized (this) { this.exclusiveModFile = exclusiveModFile; + updateMemoryCostIfRegistered(); } } - /** - * @return resource map size - */ + /** Estimated resident metadata of the file, excluding shared managers and query snapshot data. */ public long calculateRamSize() { - final ProgressIndex progressIndex = maxProgressIndex.get(); - if (timeIndex.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE) { - return INSTANCE_SIZE - + timeIndex.calculateRamSize() - + (Objects.nonNull(progressIndex) ? progressIndex.ramBytesUsed() : 0); + ITimeIndex index = timeIndex; + return calculateMetadataRamSize() + (index == null ? 0 : index.calculateRamSize()); + } + + /** The part that remains resident after a device index is replaced. */ + public long calculateMetadataRamSize() { + final ProgressIndex progress = maxProgressIndex.get(); + final ModificationFile exclusive = exclusiveModFile; + final ModificationFile compaction = compactionModFile; + long size = + INSTANCE_SIZE + + (tierLevel == null ? 0 : TIER_SIZE) + + MemUtils.getFileRamBytes(file) + + (progress == null || progress == MinimumProgressIndex.INSTANCE + ? 0 + : progress.ramBytesUsed()) + + (exclusive == null ? 0 : exclusive.ramBytesUsed()) + + (compaction == null || compaction == exclusive ? 0 : compaction.ramBytesUsed()); + final Future<ModificationFile> upgrade = exclusiveModFileFuture; + final CompletableFuture<String> recovery = sharedModFilePathFuture; + if (upgrade != null) { + size += RamUsageEstimator.shallowSizeOf(upgrade); + } + if (recovery != null) { + size += RamUsageEstimator.shallowSizeOf(recovery); } - if (deviceTimeIndexRamSize == 0) { - deviceTimeIndexRamSize = timeIndex.calculateRamSize(); + if (originTsFileResource == null) { + // LOAD and asynchronous recovery can mark the file sealed without calling close(). + if (pathToChunkMetadataListMap != null) { + size += RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP; + } + if (pathToReadOnlyMemChunkMap != null) { + size += RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP; + } + if (pathToTimeSeriesMetadataMap != null) { + size += RamUsageEstimator.SHALLOW_SIZE_OF_CONCURRENT_HASHMAP; + } } - return INSTANCE_SIZE - + deviceTimeIndexRamSize - + (Objects.nonNull(progressIndex) ? progressIndex.ramBytesUsed() : 0); + return size + estimateLastValuesSize(lastValues); } - // used for compaction + private static long estimateLastValuesSize( + Map<IDeviceID, List<Pair<String, TimeValuePair>>> values) { + if (values == null) { + return 0; + } + long size = RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP; + for (Map.Entry<IDeviceID, List<Pair<String, TimeValuePair>>> entry : values.entrySet()) { + List<Pair<String, TimeValuePair>> series = entry.getValue(); + size += + RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP_ENTRY + + RamUsageEstimator.NUM_BYTES_OBJECT_REF + + entry.getKey().ramBytesUsed() + + RamUsageEstimator.ARRAY_LIST_INSTANCE_SIZE + + RamUsageEstimator.sizeOfObjectArray(series.size()); + for (Pair<String, TimeValuePair> pair : series) { + if (pair == null) { + continue; + } + size += PAIR_SIZE + RamUsageEstimator.sizeOf(pair.left); + if (pair.right != null) { + size += TIME_VALUE_PAIR_SIZE; + if (pair.right.getValue() != null) { + size += MemUtils.getTsPrimitiveTypeRamBytes(pair.right.getValue()); + } + } + } + } + return size; + } + + /** A rebuilt device index must be measured by its owner when the resident index is file-level. */ public Optional<Long> getDeviceTimeIndexRamSize() { - if (!this.isClosed()) { - return Optional.empty(); + ITimeIndex index = timeIndex; + return isClosed() && index instanceof ArrayDeviceTimeIndex + ? Optional.of(index.calculateRamSize()) + : Optional.empty(); + } + + public Registration getMemoryRegistration() { + return memoryRegistration; + } + + public void setMemoryRegistration(Registration registration) { + memoryRegistration = registration; + } + + private void updateMemoryCostIfRegistered() { + updateMemoryCostIfRegistered(false); + } + + private void updateMemoryCostIfRegistered(boolean indexChanged) { + Registration registration = memoryRegistration; + if (registration != null) { + registration.refresh(indexChanged); } - return Optional.of(deviceTimeIndexRamSize); } public long getMaxPlanIndex() { @@ -1273,6 +1368,7 @@ public class TsFileResource implements PersistentResource, Cloneable { public void setTimeIndex(ITimeIndex timeIndex) { this.timeIndex = timeIndex; + updateMemoryCostIfRegistered(true); } /** @@ -1389,13 +1485,13 @@ public class TsFileResource implements PersistentResource, Cloneable { public void setTimeIndexType(byte type) { switch (type) { case ITimeIndex.ARRAY_DEVICE_TIME_INDEX_TYPE: - this.timeIndex = new ArrayDeviceTimeIndex(); + setTimeIndex(new ArrayDeviceTimeIndex()); break; case ITimeIndex.FILE_TIME_INDEX_TYPE: - this.timeIndex = new FileTimeIndex(); + setTimeIndex(new FileTimeIndex()); break; case ITimeIndex.PLAIN_DEVICE_TIME_INDEX_TYPE: - this.timeIndex = new PlainDeviceTimeIndex(); + setTimeIndex(new PlainDeviceTimeIndex()); break; default: throw new UnsupportedOperationException(); @@ -1404,19 +1500,23 @@ public class TsFileResource implements PersistentResource, Cloneable { /** the DeviceTimeIndex degrade to FileTimeIndex and release memory */ public long degradeTimeIndex() { - TimeIndexLevel timeIndexLevel = TimeIndexLevel.valueOf(getTimeIndexType()); - // if current timeIndex is FileTimeIndex, no need to degrade - if (timeIndexLevel == TimeIndexLevel.FILE_TIME_INDEX) { - return 0; + ITimeIndex previous = replaceDeviceTimeIndexWithFileTimeIndex(); + return previous == null ? 0 : previous.calculateRamSize() - FileTimeIndex.INSTANCE_SIZE; + } + + /** Drops the device index without scanning it. The caller may measure the previous index. */ + public ITimeIndex replaceDeviceTimeIndexWithFileTimeIndex() { + ITimeIndex current = timeIndex; + while (current.getTimeIndexType() != ITimeIndex.FILE_TIME_INDEX_TYPE) { + FileTimeIndex replacement = + new FileTimeIndex(current.getMinStartTime(), current.getMaxEndTime()); + if (TIME_INDEX_UPDATER.compareAndSet(this, current, replacement)) { + updateMemoryCostIfRegistered(true); + return current; + } + current = timeIndex; } - // get the minimum startTime - long startTime = timeIndex.getMinStartTime(); - // get the maximum endTime - long endTime = timeIndex.getMaxEndTime(); - // replace the DeviceTimeIndex with FileTimeIndex - timeIndex = new FileTimeIndex(startTime, endTime); - // deviceTimeIndexRamSize has already been calculated before - return deviceTimeIndexRamSize - timeIndex.calculateRamSize(); + return null; } public void deleteRemovedDeviceAndUpdateEndTime(Map<IDeviceID, Long> lastTimeForEachDevice) { @@ -1431,12 +1531,14 @@ public class TsFileResource implements PersistentResource, Cloneable { }); } timeIndex = newTimeIndex; + updateMemoryCostIfRegistered(true); } public void updateEndTime(Map<IDeviceID, Long> lastTimeForEachDevice) { for (Map.Entry<IDeviceID, Long> entry : lastTimeForEachDevice.entrySet()) { timeIndex.updateEndTime(entry.getKey(), entry.getValue()); } + updateMemoryCostIfRegistered(true); } /** @@ -1455,6 +1557,7 @@ public class TsFileResource implements PersistentResource, Cloneable { maxProgressIndex.updateAndGet( index -> index.updateToMinimumEqualOrIsAfterProgressIndex(progressIndex)); } + updateMemoryCostIfRegistered(); } public void setProgressIndex(ProgressIndex progressIndex) { @@ -1463,6 +1566,7 @@ public class TsFileResource implements PersistentResource, Cloneable { } maxProgressIndex.set(progressIndex); + updateMemoryCostIfRegistered(); } @Override @@ -1610,6 +1714,7 @@ public class TsFileResource implements PersistentResource, Cloneable { exclusiveModFileFuture = CompletableFuture.completedFuture(doUpgradeModFile(oldModFile, updateModMetrics)); } + updateMemoryCostIfRegistered(); } @SuppressWarnings({"java:S4042", "java:S899", "ResultOfMethodCallIgnored"}) @@ -1641,6 +1746,7 @@ public class TsFileResource implements PersistentResource, Cloneable { public void setSharedModFilePathFuture(CompletableFuture<String> sharedModFilePathFuture) { this.sharedModFilePathFuture = sharedModFilePathFuture; + updateMemoryCostIfRegistered(); } public boolean isUseSharedModFile() { @@ -1657,6 +1763,7 @@ public class TsFileResource implements PersistentResource, Cloneable { public void setCompactionModFile(ModificationFile compactionModFile) { this.compactionModFile = compactionModFile; + updateMemoryCostIfRegistered(); } public Map<IDeviceID, List<Pair<String, TimeValuePair>>> getLastValues() { @@ -1665,6 +1772,7 @@ public class TsFileResource implements PersistentResource, Cloneable { public void setLastValues(Map<IDeviceID, List<Pair<String, TimeValuePair>>> lastValues) { this.lastValues = lastValues; + updateMemoryCostIfRegistered(); } public TsFileResource shallowClone() { @@ -1684,7 +1792,6 @@ public class TsFileResource implements PersistentResource, Cloneable { cloned.isRecordedByMetric = this.isRecordedByMetric; cloned.tsFileRepairStatus = this.tsFileRepairStatus; cloned.settleTsFileCallBack = this.settleTsFileCallBack; - cloned.deviceTimeIndexRamSize = this.deviceTimeIndexRamSize; cloned.tsFileSize = this.tsFileSize; cloned.processor = this.processor; cloned.originTsFileResource = this.originTsFileResource; @@ -1706,6 +1813,8 @@ public class TsFileResource implements PersistentResource, Cloneable { } public TsFileResource shallowCloneForNative() throws CloneNotSupportedException { - return (TsFileResource) clone(); + TsFileResource cloned = (TsFileResource) clone(); + cloned.memoryRegistration = null; + return cloned; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java index 8af037a5924..9557cc3cfed 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java @@ -54,6 +54,14 @@ public class ArrayDeviceTimeIndex implements ITimeIndex { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(ArrayDeviceTimeIndex.class); + private static final long INTEGER_SIZE = RamUsageEstimator.shallowSizeOfInstance(Integer.class); + // Both ConcurrentHashMap.Node and HashMap.Node hold a hash and three references. + private static final long MAP_NODE_SIZE = + RamUsageEstimator.alignObjectSize( + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + + Integer.BYTES + + 3L * RamUsageEstimator.NUM_BYTES_OBJECT_REF); + private static final Logger logger = LoggerFactory.getLogger(ArrayDeviceTimeIndex.class); public static final int INIT_ARRAY_SIZE = 64; @@ -235,9 +243,27 @@ public class ArrayDeviceTimeIndex implements ITimeIndex { @Override public long calculateRamSize() { + int count = deviceToIndex.size(); + long deviceIdBytes = 0; + for (IDeviceID device : deviceToIndex.keySet()) { + deviceIdBytes += device.ramBytesUsed(); + } + // Capacity is an estimate: map implementations do not expose their backing array. + // ConcurrentHashMap grows at the threshold; HashMap grows on the next insertion. + boolean concurrent = deviceToIndex instanceof ConcurrentHashMap; + long requiredSlots = (4L * count - (concurrent ? 0 : 1)) / 3; + long buckets = + count == 0 ? 0 : Math.min(1L << 30, Math.max(16L, Long.highestOneBit(requiredSlots) << 1)); + long tableSize = buckets == 0 ? 0 : RamUsageEstimator.sizeOfObjectArray((int) buckets); return INSTANCE_SIZE - + RamUsageEstimator.sizeOfMap( - deviceToIndex, RamUsageEstimator.shallowSizeOfInstance(Integer.class)) + + (concurrent + ? RamUsageEstimator.SHALLOW_SIZE_OF_CONCURRENT_HASHMAP + : RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP) + + tableSize + + count * MAP_NODE_SIZE + + deviceIdBytes + // Indices are contiguous from zero. Integer.valueOf shares at least the first 128. + + Math.max(0, count - 128) * INTEGER_SIZE + RamUsageEstimator.sizeOf(startTimes) + RamUsageEstimator.sizeOf(endTimes); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java index f6f598ef9a4..d96747e92be 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java @@ -48,7 +48,7 @@ import java.util.Set; public class FileTimeIndex implements ITimeIndex { - private static final long INSTANCE_SIZE = + public static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(FileTimeIndex.class); private static final Logger logger = LoggerFactory.getLogger(FileTimeIndex.class); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TsFileResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TsFileResourceManager.java index 3221185db55..733db3f504d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TsFileResourceManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TsFileResourceManager.java @@ -26,11 +26,13 @@ import org.apache.iotdb.db.conf.DataNodeMemoryConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.TimeIndexLevel; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; +import org.apache.tsfile.utils.RamUsageEstimator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.util.TreeSet; public class TsFileResourceManager { @@ -42,12 +44,60 @@ public class TsFileResourceManager { /** The memory block of time index */ private final IMemoryBlock memoryBlock; - /** store the sealed TsFileResource, sorted by priority of TimeIndex */ - private final TreeSet<TsFileResource> sealedTsFileResources = - new TreeSet<>(TsFileResource::compareIndexDegradePriority); + // One registration and one TreeMap.Entry (key, value, parent, left, right and color) per file. + static final long REGISTRATION_RAM_BYTES = + RamUsageEstimator.shallowSizeOfInstance(Registration.class) + + RamUsageEstimator.alignObjectSize( + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + + 5L * RamUsageEstimator.NUM_BYTES_OBJECT_REF + + Byte.BYTES); + + private final TreeSet<Registration> sealedTsFileResources = new TreeSet<>(); + private long degradedTimeIndexNum; + private long nextRegistrationId; + private boolean degrading; + + /** The sort key and the charged bytes remain stable until the manager updates the entry. */ + public static final class Registration implements Comparable<Registration> { + private final TsFileResource resource; + private final long id; + private long indexBytes; + private File file; + private long startTime; + private boolean fileIndex; + private long bytes; + + private Registration(TsFileResource resource, long id) { + this.resource = resource; + this.id = id; + updatePriority(resource.getTimeIndex()); + } + + private void updatePriority(ITimeIndex index) { + file = resource.getTsFile(); + startTime = index.getMinStartTime(); + fileIndex = index.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE; + } + + private int comparePriority(Registration other) { + int cmp = Boolean.compare(fileIndex, other.fileIndex); + if (cmp == 0) { + cmp = Long.compare(startTime, other.startTime); + } + return cmp != 0 ? cmp : file.getAbsolutePath().compareTo(other.file.getAbsolutePath()); + } + + @Override + public int compareTo(Registration other) { + int cmp = comparePriority(other); + // Two previously distinct entries can acquire the same priority after degradation. + return cmp == 0 ? Long.compare(id, other.id) : cmp; + } - // degraded time index number - private long degradedTimeIndexNum = 0; + public void refresh(boolean indexChanged) { + TsFileResourceManager.getInstance().refresh(this, indexChanged); + } + } private TsFileResourceManager() { memoryBlock = @@ -61,82 +111,115 @@ public class TsFileResourceManager { this.memoryBlock.setTotalMemorySizeInBytes(timeIndexMemoryThreshold); } - public long getPriorityQueueSize() { + public synchronized long getPriorityQueueSize() { return sealedTsFileResources.size(); } - /** - * add the closed TsFileResource into priorityQueue and increase memory cost of timeIndex, once - * memory cost is larger than threshold, degradation is triggered. - */ - public synchronized void registerSealedTsFileResource(TsFileResource tsFileResource) { - if (!sealedTsFileResources.contains(tsFileResource)) { - sealedTsFileResources.add(tsFileResource); - memoryBlock.forceAllocateWithoutLimitation(tsFileResource.calculateRamSize()); - chooseTsFileResourceToDegrade(); + public synchronized void registerSealedTsFileResource(TsFileResource resource) { + if (resource.getMemoryRegistration() != null) { + return; + } + Registration entry = new Registration(resource, ++nextRegistrationId); + Registration previous = sealedTsFileResources.floor(entry); + if (previous == null || entry.comparePriority(previous) != 0) { + sealedTsFileResources.add(entry); + if (entry.fileIndex) { + degradedTimeIndexNum++; + } + // Publish before sampling so a concurrent metadata change cannot miss its charge. + resource.setMemoryRegistration(entry); + try { + refresh(entry, true); + } catch (RuntimeException | Error e) { + removeTsFileResource(resource); + throw e; + } } } - /** delete the TsFileResource in PriorityQueue when the source file is deleted */ - public synchronized void removeTsFileResource(TsFileResource tsFileResource) { - if (sealedTsFileResources.contains(tsFileResource)) { - sealedTsFileResources.remove(tsFileResource); - if (TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType()) - == TimeIndexLevel.FILE_TIME_INDEX) { - memoryBlock.release(tsFileResource.calculateRamSize()); + public synchronized void removeTsFileResource(TsFileResource resource) { + Registration entry = resource.getMemoryRegistration(); + if (entry == null) { + // Recovery can supply another resource describing the same file. + if (sealedTsFileResources.isEmpty()) { + return; + } + Registration key = new Registration(resource, Long.MIN_VALUE); + entry = sealedTsFileResources.ceiling(key); + if (entry == null || entry.comparePriority(key) != 0) { + return; + } + } + if (sealedTsFileResources.remove(entry)) { + entry.resource.setMemoryRegistration(null); + memoryBlock.release(entry.bytes); + if (entry.fileIndex) { degradedTimeIndexNum--; - } else { - memoryBlock.release(tsFileResource.calculateRamSize()); } } } - public void forceDegradeTsFileResource(TsFileResource resource) { - if (TimeIndexLevel.valueOf(resource.getTimeIndexType()) == TimeIndexLevel.FILE_TIME_INDEX) { + private synchronized void refresh(Registration entry, boolean indexChanged) { + TsFileResource resource = entry.resource; + if (resource.getMemoryRegistration() != entry) { return; } - logger.debug(StorageEngineMessages.FORCE_DEGRADE_TSFILE_RESOURCE, resource.getTsFilePath()); - synchronized (this) { - if (!sealedTsFileResources.remove(resource)) { - resource.degradeTimeIndex(); - return; + ITimeIndex index = resource.getTimeIndex(); + boolean fileIndex = index.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE; + if (entry.fileIndex != fileIndex + || entry.startTime != index.getMinStartTime() + || entry.file != resource.getTsFile()) { + sealedTsFileResources.remove(entry); + if (entry.fileIndex != fileIndex) { + degradedTimeIndexNum += fileIndex ? 1 : -1; } - long memoryReduce = resource.degradeTimeIndex(); - degradedTimeIndexNum++; - releaseTimeIndexMemCost(memoryReduce); - sealedTsFileResources.add(resource); + entry.updatePriority(index); + sealedTsFileResources.add(entry); + } + if (indexChanged) { + entry.indexBytes = index.calculateRamSize(); + } + long bytes = REGISTRATION_RAM_BYTES + resource.calculateMetadataRamSize() + entry.indexBytes; + long delta = bytes - entry.bytes; + entry.bytes = bytes; + if (delta > 0) { + memoryBlock.forceAllocateWithoutLimitation(delta); + if (!degrading + && memoryBlock.getUsedMemoryInBytes() > memoryBlock.getTotalMemorySizeInBytes()) { + degrading = true; + try { + chooseTsFileResourceToDegrade(); + } finally { + degrading = false; + } + } + } else if (delta < 0) { + memoryBlock.release(-delta); } } - /** once degradation is triggered, the total memory for timeIndex should reduce */ - private void releaseTimeIndexMemCost(long memCost) { - memoryBlock.release(memCost); + public void forceDegradeTsFileResource(TsFileResource resource) { + if (resource.replaceDeviceTimeIndexWithFileTimeIndex() != null) { + logger.debug(StorageEngineMessages.FORCE_DEGRADE_TSFILE_RESOURCE, resource.getTsFilePath()); + } } - /** - * choose the top TsFileResource in priorityQueue to degrade until the memory is smaller than - * threshold. - */ private void chooseTsFileResourceToDegrade() { while (memoryBlock.getUsedMemoryInBytes() > memoryBlock.getTotalMemorySizeInBytes()) { - TsFileResource tsFileResource = sealedTsFileResources.pollFirst(); - if (tsFileResource == null - || TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType()) - == TimeIndexLevel.FILE_TIME_INDEX) { + Registration entry = sealedTsFileResources.first(); + if (entry.fileIndex) { logger.debug(StorageEngineMessages.CANNOT_DEGRADE_TIME_INDEX_ALL_FILE_LEVEL); - sealedTsFileResources.add(tsFileResource); return; } - long memoryReduce = tsFileResource.degradeTimeIndex(); - logger.debug(StorageEngineMessages.DEGRADE_TSFILE_RESOURCE, tsFileResource.getTsFilePath()); - degradedTimeIndexNum++; - releaseTimeIndexMemCost(memoryReduce); - // add the polled tsFileResource to the priority queue - sealedTsFileResources.add(tsFileResource); + logger.debug(StorageEngineMessages.DEGRADE_TSFILE_RESOURCE, entry.file); + if (entry.resource.replaceDeviceTimeIndexWithFileTimeIndex() == null) { + // Another thread has already swapped the index and is waiting to publish its size. + refresh(entry, true); + } } } - public long getDegradedTimeIndexNum() { + public synchronized long getDegradedTimeIndexNum() { return degradedTimeIndexNum; } @@ -151,6 +234,7 @@ public class TsFileResourceManager { /** function for clearing TsFileManager */ @TestOnly public synchronized void clear() { + sealedTsFileResources.forEach(entry -> entry.resource.setMemoryRegistration(null)); this.sealedTsFileResources.clear(); this.memoryBlock.setUsedMemoryInBytes(0); this.degradedTimeIndexNum = 0; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java index b724c4d3c6f..ad05bf6bc8c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java @@ -30,6 +30,7 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.BooleanDataPoint; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -41,11 +42,44 @@ import org.apache.tsfile.write.record.datapoint.StringDataPoint; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.util.List; // Notice : methods in this class may not be accurate. public class MemUtils { + private static final long FILE_INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(File.class); + + private static final ClassValue<Long> PRIMITIVE_INSTANCE_SIZE = + new ClassValue<Long>() { + @Override + protected Long computeValue(Class<?> type) { + return RamUsageEstimator.shallowSizeOfInstance(type); + } + }; + + public static long getTsPrimitiveTypeRamBytes(TsPrimitiveType value) { + long bytes = PRIMITIVE_INSTANCE_SIZE.get(value.getClass()); + if (value instanceof TsPrimitiveType.TsBinary && value.getBinary() != null) { + bytes += value.getBinary().ramBytesUsed(); + } else if (value instanceof TsPrimitiveType.TsVector && value.getVector() != null) { + TsPrimitiveType[] elements = value.getVector(); + bytes += RamUsageEstimator.shallowSizeOf(elements); + for (TsPrimitiveType element : elements) { + if (element != null) { + bytes += getTsPrimitiveTypeRamBytes(element); + } + } + } + return bytes; + } + + /** Heap occupied by a file handle and its stored path. */ + public static long getFileRamBytes(File file) { + return file == null ? 0 : FILE_INSTANCE_SIZE + RamUsageEstimator.sizeOf(file.getPath()); + } + private static final Logger LOGGER = LoggerFactory.getLogger(MemUtils.class); private MemUtils() {} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java index 4236cbcd576..8926999c3f1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.storageengine.rescon.memory; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.consensus.index.impl.SimpleProgressIndex; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.db.conf.DataNodeMemoryConfig; import org.apache.iotdb.db.conf.IoTDBConfig; @@ -31,9 +32,12 @@ import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.TimeIndexLevel; import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.db.utils.MemUtils; import org.apache.iotdb.db.utils.constant.TestConstant; import org.apache.tsfile.enums.TSDataType; @@ -42,7 +46,11 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.fileSystem.FSFactoryProducer; +import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -56,10 +64,13 @@ import org.junit.Test; import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR; +import static org.apache.iotdb.db.storageengine.rescon.memory.TsFileResourceManager.REGISTRATION_RAM_BYTES; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class ResourceManagerTest { @@ -246,7 +257,7 @@ public class ResourceManagerTest { TimeIndexLevel.ARRAY_DEVICE_TIME_INDEX, TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType())); long previousRamSize = tsFileResource.calculateRamSize(); - long curTimeIndexMemoryThreshold = 4291; + long curTimeIndexMemoryThreshold = previousRamSize + REGISTRATION_RAM_BYTES; tsFileResourceManager.setTimeIndexMemoryThreshold(curTimeIndexMemoryThreshold); tsFileResourceManager.registerSealedTsFileResource(tsFileResource); assertEquals(0, previousRamSize - tsFileResource.calculateRamSize()); @@ -302,7 +313,9 @@ public class ResourceManagerTest { TimeIndexLevel.valueOf(tsFileResource2.getTimeIndexType())); long curTimeIndexMemoryThreshold = - degradedResource1.calculateRamSize() + tsFileResource2.calculateRamSize(); + degradedResource1.calculateRamSize() + + tsFileResource2.calculateRamSize() + + 2 * REGISTRATION_RAM_BYTES; tsFileResourceManager.setTimeIndexMemoryThreshold(curTimeIndexMemoryThreshold); tsFileResourceManager.registerSealedTsFileResource(tsFileResource1); assertEquals( @@ -340,7 +353,7 @@ public class ResourceManagerTest { TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType())); prepareFile(tsFileResource, i * ptNum, ptNum, 0); tsFileResource.degradeTimeIndex(); - curTimeIndexMemoryThreshold += tsFileResource.calculateRamSize(); + curTimeIndexMemoryThreshold += tsFileResource.calculateRamSize() + REGISTRATION_RAM_BYTES; } for (int i = 8; i < seqFileNum; i++) { File file = @@ -361,7 +374,7 @@ public class ResourceManagerTest { TimeIndexLevel.ARRAY_DEVICE_TIME_INDEX, TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType())); prepareFile(tsFileResource, i * ptNum, ptNum, 0); - curTimeIndexMemoryThreshold += tsFileResource.calculateRamSize(); + curTimeIndexMemoryThreshold += tsFileResource.calculateRamSize() + REGISTRATION_RAM_BYTES; } for (int i = 0; i < seqFileNum; i++) { @@ -459,13 +472,123 @@ public class ResourceManagerTest { tsFileResourceManager.registerSealedTsFileResource(tsFileResource); long resourceRamSizeBeforeDegrade = tsFileResource.calculateRamSize(); Assert.assertEquals( - resourceRamSizeBeforeDegrade, tsFileResourceManager.getTotalTimeIndexMemCost()); + resourceRamSizeBeforeDegrade + REGISTRATION_RAM_BYTES, + tsFileResourceManager.getTotalTimeIndexMemCost()); TsFileResourceManager.getInstance().forceDegradeTsFileResource(tsFileResource); Assert.assertEquals( - tsFileResource.calculateRamSize(), tsFileResourceManager.getTotalTimeIndexMemCost()); + tsFileResource.calculateRamSize() + REGISTRATION_RAM_BYTES, + tsFileResourceManager.getTotalTimeIndexMemCost()); Assert.assertTrue( - resourceRamSizeBeforeDegrade > tsFileResourceManager.getTotalTimeIndexMemCost()); + resourceRamSizeBeforeDegrade + REGISTRATION_RAM_BYTES + > tsFileResourceManager.getTotalTimeIndexMemCost()); Assert.assertTrue(tsFileResource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE); Assert.assertEquals(1, tsFileResourceManager.getDegradedTimeIndexNum()); } + + @Test + public void testResidentMetadataAndIndexSizeChanges() throws IOException { + File file = new File("target/0-1-0-0.tsfile"); + TsFileResource resource = new TsFileResource(file); + long before = resource.calculateRamSize(); + IDeviceID device = IDeviceID.Factory.DEFAULT_FACTORY.create("root.test.newDevice"); + resource.updateStartTime(device, 1); + assertTrue(resource.calculateRamSize() > before); + + resource.close(); + long sealed = resource.calculateRamSize(); + long indexSize = resource.getTimeIndex().calculateRamSize(); + assertEquals(Long.valueOf(indexSize), resource.getDeviceTimeIndexRamSize().get()); + long expectedRelease = indexSize - new FileTimeIndex().calculateRamSize(); + // A freshly sealed resource can also be degraded before any size calculation. + TsFileResource unmeasured = new TsFileResource(file); + unmeasured.updateStartTime(device, 1); + unmeasured.close(); + assertEquals(expectedRelease, unmeasured.degradeTimeIndex()); + assertEquals(expectedRelease, resource.degradeTimeIndex()); + assertEquals(sealed - expectedRelease, resource.calculateRamSize()); + assertEquals(0, resource.degradeTimeIndex()); + Assert.assertFalse(resource.getDeviceTimeIndexRamSize().isPresent()); + + long base = resource.calculateRamSize(); + File longerPath = new File("target/longer/directory/0-1-0-0.tsfile"); + resource.setFile(longerPath); + assertEquals( + base + MemUtils.getFileRamBytes(longerPath) - MemUtils.getFileRamBytes(file), + resource.calculateRamSize()); + long beforeMods = resource.calculateRamSize(); + long modSize = resource.getExclusiveModFile().ramBytesUsed(); + assertEquals(beforeMods + modSize, resource.calculateRamSize()); + resource.resetModFile(); + assertEquals(beforeMods, resource.calculateRamSize()); + } + + @Test + public void testRegisteredBytesFollowMetadataAndIndexChanges() throws Exception { + tsFileResourceManager.setTimeIndexMemoryThreshold(Long.MAX_VALUE); + TsFileResource remaining = new TsFileResource(new File("target/0-1-0-0.tsfile")); + remaining.close(); + tsFileResourceManager.registerSealedTsFileResource(remaining); + long baseline = tsFileResourceManager.getTotalTimeIndexMemCost(); + TsFileResource resource = new TsFileResource(new File("target/0-2-0-0.tsfile")); + resource.close(); + resource.setTimeIndex(new ArrayDeviceTimeIndex()); + tsFileResourceManager.registerSealedTsFileResource(resource); + tsFileResourceManager.registerSealedTsFileResource(resource); + long originalCharge = resource.calculateRamSize() + REGISTRATION_RAM_BYTES; + assertEquals(baseline + originalCharge, tsFileResourceManager.getTotalTimeIndexMemCost()); + resource.updateStartTime(IDeviceID.Factory.DEFAULT_FACTORY.create("root.test.later"), 7); + resource.setProgressIndex(new SimpleProgressIndex(1, 1)); + resource.getExclusiveModFile(); + long changedCharge = resource.calculateRamSize() + REGISTRATION_RAM_BYTES; + assertTrue(changedCharge > originalCharge); + assertEquals(baseline + changedCharge, tsFileResourceManager.getTotalTimeIndexMemCost()); + + resource.degradeTimeIndex(); + assertEquals(1, tsFileResourceManager.getDegradedTimeIndexNum()); + assertEquals( + baseline + resource.calculateRamSize() + REGISTRATION_RAM_BYTES, + tsFileResourceManager.getTotalTimeIndexMemCost()); + TsFileResource clone = resource.shallowCloneForNative(); + Assert.assertNull(clone.getMemoryRegistration()); + clone.getCompactionModFile(); + assertEquals( + baseline + resource.calculateRamSize() + REGISTRATION_RAM_BYTES, + tsFileResourceManager.getTotalTimeIndexMemCost()); + resource.setTimeIndex(new ArrayDeviceTimeIndex()); + assertEquals(0, tsFileResourceManager.getDegradedTimeIndexNum()); + assertEquals( + baseline + resource.calculateRamSize() + REGISTRATION_RAM_BYTES, + tsFileResourceManager.getTotalTimeIndexMemCost()); + tsFileResourceManager.removeTsFileResource(resource); + tsFileResourceManager.removeTsFileResource(resource); + assertEquals(baseline, tsFileResourceManager.getTotalTimeIndexMemCost()); + assertEquals(0, tsFileResourceManager.getDegradedTimeIndexNum()); + tsFileResourceManager.removeTsFileResource(remaining); + assertEquals(0, tsFileResourceManager.getTotalTimeIndexMemCost()); + } + + @Test + public void testLoadLastValuesReleasedFromRegisteredBytes() throws IOException { + tsFileResourceManager.setTimeIndexMemoryThreshold(Long.MAX_VALUE); + TsFileResource resource = new TsFileResource(new File("target/0-3-0-0.tsfile")); + resource.setTimeIndex(new FileTimeIndex(1, 2)); + resource.close(); + tsFileResourceManager.registerSealedTsFileResource(resource); + assertEquals(1, tsFileResourceManager.getDegradedTimeIndexNum()); + long baseline = tsFileResourceManager.getTotalTimeIndexMemCost(); + TsPrimitiveType.TsBinary value = new TsPrimitiveType.TsBinary(new Binary(new byte[1024])); + resource.setLastValues( + Collections.singletonMap( + IDeviceID.Factory.DEFAULT_FACTORY.create("root.test.d"), + Collections.singletonList(new Pair<>("s1", new TimeValuePair(1, value))))); + assertTrue(tsFileResourceManager.getTotalTimeIndexMemCost() >= baseline + 1024); + assertEquals( + resource.calculateRamSize() + REGISTRATION_RAM_BYTES, + tsFileResourceManager.getTotalTimeIndexMemCost()); + resource.setLastValues(null); + assertEquals(baseline, tsFileResourceManager.getTotalTimeIndexMemCost()); + tsFileResourceManager.removeTsFileResource(resource); + assertEquals(0, tsFileResourceManager.getDegradedTimeIndexNum()); + assertEquals(0, tsFileResourceManager.getTotalTimeIndexMemCost()); + } }
