This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch table_disk_usage_statistics_with_cache
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to
refs/heads/table_disk_usage_statistics_with_cache by this push:
new dfc79290caf use ServiceLoader
dfc79290caf is described below
commit dfc79290caf53ef596b758f4ced48659b0121c10
Author: shuwenwei <[email protected]>
AuthorDate: Tue Jan 20 16:21:55 2026 +0800
use ServiceLoader
---
.../storageengine/dataregion/tsfile/TsFileID.java | 2 +
.../dataregion/tsfile/TsFileResource.java | 2 +-
.../utils/TableDiskUsageStatisticUtil.java | 4 --
.../DefaultTableDiskUsageCacheProvider.java | 28 ++++++++++
.../tableDiskUsageCache/TableDiskUsageCache.java | 63 ++++++++++++++--------
.../TableDiskUsageCacheProvider.java | 24 +++++++++
.../TableDiskUsageCacheReader.java | 41 +++++++++++++-
.../TimePartitionTableSizeQueryContext.java | 29 +++++++++-
...r.java => TsFileTableDiskUsageCacheWriter.java} | 17 +++---
.../TsFileTableSizeCacheReader.java | 4 +-
.../TsFileTableSizeCacheWriter.java | 4 +-
.../utils/TsFileTableSizeCacheWriterTest.java | 14 ++---
.../apache/iotdb/db/utils/EnvironmentUtils.java | 2 +
13 files changed, 184 insertions(+), 50 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileID.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileID.java
index c9656382e3d..5c77fa78f54 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileID.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileID.java
@@ -26,6 +26,8 @@ import static
org.apache.tsfile.utils.FilePathUtils.splitTsFilePath;
public class TsFileID {
+ public static final long SHALLOW_SIZE = TsFileID.SHALLOW_SIZE;
+
public final int regionId;
public final long timePartitionId;
public final long timestamp;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
index b84cce9e8d2..4a0f1e4b72c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
@@ -101,7 +101,7 @@ public class TsFileResource implements PersistentResource,
Cloneable {
private static final long INSTANCE_SIZE =
RamUsageEstimator.shallowSizeOfInstance(TsFileResource.class)
+ RamUsageEstimator.shallowSizeOfInstance(TsFileRepairStatus.class)
- + RamUsageEstimator.shallowSizeOfInstance(TsFileID.class);
+ + TsFileID.SHALLOW_SIZE;
private static final Logger LOGGER =
LoggerFactory.getLogger(TsFileResource.class);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageStatisticUtil.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageStatisticUtil.java
index 2f0e5b48045..f5b11e446ef 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageStatisticUtil.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageStatisticUtil.java
@@ -86,10 +86,6 @@ public class TableDiskUsageStatisticUtil extends
DiskUsageStatisticUtil {
return true;
}
- public List<Pair<TsFileID, Long>> getTsFilesToQueryInCache() {
- return tsFilesToQueryInCache;
- }
-
@Override
protected void calculateNextFile(TsFileResource tsFileResource,
TsFileSequenceReader reader)
throws IOException {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/DefaultTableDiskUsageCacheProvider.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/DefaultTableDiskUsageCacheProvider.java
new file mode 100644
index 00000000000..5cd17a6ef3e
--- /dev/null
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/DefaultTableDiskUsageCacheProvider.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache;
+
+public class DefaultTableDiskUsageCacheProvider implements
TableDiskUsageCacheProvider {
+
+ @Override
+ public TableDiskUsageCache create() {
+ return new TableDiskUsageCache();
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCache.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCache.java
index 5b216876f07..8a1a6112f33 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCache.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCache.java
@@ -29,6 +29,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import java.util.ServiceLoader;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.LinkedBlockingQueue;
@@ -36,19 +37,19 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
public class TableDiskUsageCache {
- private static final Logger LOGGER =
LoggerFactory.getLogger(TableDiskUsageCache.class);
- private final BlockingQueue<Operation> queue = new LinkedBlockingQueue<>();
- private final Map<Integer, TableDiskUsageCacheWriter> writerMap = new
HashMap<>();
- private final ScheduledExecutorService scheduledExecutorService;
+ protected static final Logger LOGGER =
LoggerFactory.getLogger(TableDiskUsageCache.class);
+ protected final BlockingQueue<Operation> queue = new LinkedBlockingQueue<>();
+ protected final Map<Integer, TsFileTableDiskUsageCacheWriter> writerMap =
new HashMap<>();
+ protected final ScheduledExecutorService scheduledExecutorService;
- private TableDiskUsageCache() {
+ protected TableDiskUsageCache() {
scheduledExecutorService =
IoTDBThreadPoolFactory.newSingleThreadScheduledExecutor(
ThreadName.FILE_TIME_INDEX_RECORD.getName());
scheduledExecutorService.submit(this::run);
}
- private void run() {
+ protected void run() {
try {
while (!Thread.currentThread().isInterrupted()) {
try {
@@ -66,13 +67,13 @@ public class TableDiskUsageCache {
}
}
} finally {
- writerMap.values().forEach(TableDiskUsageCacheWriter::close);
+ writerMap.values().forEach(TsFileTableDiskUsageCacheWriter::close);
}
}
- private void checkAndMayCompact(long maxRunTime) {
+ protected void checkAndMayCompact(long maxRunTime) {
long startTime = System.currentTimeMillis();
- for (TableDiskUsageCacheWriter writer : writerMap.values()) {
+ for (TsFileTableDiskUsageCacheWriter writer : writerMap.values()) {
if (System.currentTimeMillis() - startTime > maxRunTime) {
break;
}
@@ -82,8 +83,8 @@ public class TableDiskUsageCache {
}
}
- private void checkAndMayCloseIdleWriter() {
- for (TableDiskUsageCacheWriter writer : writerMap.values()) {
+ protected void checkAndMayCloseIdleWriter() {
+ for (TsFileTableDiskUsageCacheWriter writer : writerMap.values()) {
writer.closeIfIdle();
}
}
@@ -96,7 +97,7 @@ public class TableDiskUsageCache {
}
public void write(String database, TsFileID originTsFileID, TsFileID
newTsFileID) {
- queue.add(new ReplaceOperation(database, originTsFileID, newTsFileID));
+ queue.add(new ReplaceTsFileOperation(database, originTsFileID,
newTsFileID));
}
public CompletableFuture<TsFileTableSizeCacheReader> startRead(String
database, int regionId) {
@@ -121,7 +122,14 @@ public class TableDiskUsageCache {
}
}
- public abstract static class Operation {
+ public void close() {
+ if (scheduledExecutorService != null) {
+ scheduledExecutorService.shutdownNow();
+ }
+ writerMap.values().forEach(TsFileTableDiskUsageCacheWriter::close);
+ }
+
+ protected abstract static class Operation {
protected final String database;
protected final int regionId;
@@ -143,9 +151,9 @@ public class TableDiskUsageCache {
@Override
public void apply(TableDiskUsageCache tableDiskUsageCache) throws
IOException {
try {
- TableDiskUsageCacheWriter writer =
+ TsFileTableDiskUsageCacheWriter writer =
tableDiskUsageCache.writerMap.computeIfAbsent(
- regionId, k -> new TableDiskUsageCacheWriter(database,
regionId));
+ regionId, k -> new TsFileTableDiskUsageCacheWriter(database,
regionId));
writer.flush();
writer.increaseActiveReaderNum();
future.complete(
@@ -168,9 +176,9 @@ public class TableDiskUsageCache {
@Override
public void apply(TableDiskUsageCache tableDiskUsageCache) throws
IOException {
- TableDiskUsageCacheWriter writer =
+ TsFileTableDiskUsageCacheWriter writer =
tableDiskUsageCache.writerMap.computeIfAbsent(
- regionId, k -> new TableDiskUsageCacheWriter(database,
regionId));
+ regionId, k -> new TsFileTableDiskUsageCacheWriter(database,
regionId));
writer.decreaseActiveReaderNum();
}
}
@@ -190,16 +198,16 @@ public class TableDiskUsageCache {
public void apply(TableDiskUsageCache tableDiskUsageCache) throws
IOException {
tableDiskUsageCache
.writerMap
- .computeIfAbsent(regionId, k -> new
TableDiskUsageCacheWriter(database, regionId))
+ .computeIfAbsent(regionId, k -> new
TsFileTableDiskUsageCacheWriter(database, regionId))
.write(tsFileID, tableSizeMap);
}
}
- private static class ReplaceOperation extends Operation {
+ private static class ReplaceTsFileOperation extends Operation {
private final TsFileID originTsFileID;
private final TsFileID newTsFileID;
- public ReplaceOperation(String database, TsFileID originTsFileID, TsFileID
newTsFileID) {
+ public ReplaceTsFileOperation(String database, TsFileID originTsFileID,
TsFileID newTsFileID) {
super(database, originTsFileID.regionId);
this.originTsFileID = originTsFileID;
this.newTsFileID = newTsFileID;
@@ -207,7 +215,7 @@ public class TableDiskUsageCache {
@Override
public void apply(TableDiskUsageCache tableDiskUsageCache) throws
IOException {
- TableDiskUsageCacheWriter writer =
tableDiskUsageCache.writerMap.get(regionId);
+ TsFileTableDiskUsageCacheWriter writer =
tableDiskUsageCache.writerMap.get(regionId);
if (writer != null) {
writer.write(originTsFileID, newTsFileID);
}
@@ -224,7 +232,7 @@ public class TableDiskUsageCache {
@Override
public void apply(TableDiskUsageCache tableDiskUsageCache) throws
IOException {
- TableDiskUsageCacheWriter writer =
tableDiskUsageCache.writerMap.remove(regionId);
+ TsFileTableDiskUsageCacheWriter writer =
tableDiskUsageCache.writerMap.remove(regionId);
if (writer != null) {
writer.close();
}
@@ -239,6 +247,15 @@ public class TableDiskUsageCache {
private static class InstanceHolder {
private InstanceHolder() {}
- private static final TableDiskUsageCache INSTANCE = new
TableDiskUsageCache();
+ private static final TableDiskUsageCache INSTANCE = loadInstance();
+
+ private static TableDiskUsageCache loadInstance() {
+ ServiceLoader<TableDiskUsageCacheProvider> loader =
+ ServiceLoader.load(TableDiskUsageCacheProvider.class);
+ for (TableDiskUsageCacheProvider provider : loader) {
+ return provider.create();
+ }
+ return new DefaultTableDiskUsageCacheProvider().create();
+ }
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheProvider.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheProvider.java
new file mode 100644
index 00000000000..cba896ed781
--- /dev/null
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheProvider.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache;
+
+public interface TableDiskUsageCacheProvider {
+ TableDiskUsageCache create();
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheReader.java
index ad801dedaf9..81f4b76b0d3 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheReader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheReader.java
@@ -20,11 +20,13 @@
package org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache;
import
org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext;
+import
org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
import
org.apache.iotdb.db.storageengine.dataregion.utils.TableDiskUsageStatisticUtil;
import org.apache.tsfile.utils.Pair;
+import org.apache.tsfile.utils.RamUsageEstimator;
import java.io.Closeable;
import java.io.IOException;
@@ -43,6 +45,8 @@ public class TableDiskUsageCacheReader implements Closeable {
private final Map<Long, TimePartitionTableSizeQueryContext>
timePartitionQueryContexts;
private CompletableFuture<TsFileTableSizeCacheReader> future;
private TsFileTableSizeCacheReader cacheFileReader;
+ private long acquiredMemory;
+ private boolean tsFileIdKeysPrepared = false;
private final Iterator<Map.Entry<Long, TimePartitionTableSizeQueryContext>>
timePartitionIterator;
@@ -64,9 +68,17 @@ public class TableDiskUsageCacheReader implements Closeable {
this.currentDatabaseOnlyHasOneTable = databaseHasOnlyOneTable;
this.context = context;
this.timePartitionIterator =
timePartitionQueryContexts.entrySet().iterator();
+ reserveMemory(
+ RamUsageEstimator.sizeOfMapWithKnownShallowSize(
+ timePartitionQueryContexts,
+ RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP,
+ RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP_ENTRY));
}
public boolean prepareCachedTsFileIDKeys(long startTime, long maxRunTime)
throws Exception {
+ if (tsFileIdKeysPrepared) {
+ return true;
+ }
if (this.cacheFileReader == null) {
this.future =
this.future == null
@@ -90,7 +102,15 @@ public class TableDiskUsageCacheReader implements Closeable
{
if (this.cacheFileReader == null) {
return false;
}
- return cacheFileReader.readFromKeyFile(timePartitionQueryContexts,
startTime, maxRunTime);
+ if (cacheFileReader.readFromKeyFile(timePartitionQueryContexts, startTime,
maxRunTime)) {
+ reserveMemory(
+ timePartitionQueryContexts.values().stream()
+
.mapToLong(TimePartitionTableSizeQueryContext::ramBytesUsedOfTsFileIDOffsetMap)
+ .sum());
+ tsFileIdKeysPrepared = true;
+ return true;
+ }
+ return false;
}
public boolean calculateNextFile() {
@@ -132,15 +152,34 @@ public class TableDiskUsageCacheReader implements
Closeable {
return dataRegion;
}
+ private void reserveMemory(long size) {
+ if (context.isPresent()) {
+ MemoryReservationManager memoryReservationContext =
+ context.get().getMemoryReservationContext();
+ memoryReservationContext.reserveMemoryCumulatively(size);
+ memoryReservationContext.reserveMemoryImmediately();
+ acquiredMemory += size;
+ }
+ }
+
@Override
public void close() throws IOException {
if (future != null) {
TableDiskUsageCache.getInstance().endRead(dataRegion.getDatabaseName(),
regionId);
future = null;
}
+ releaseMemory();
if (cacheFileReader != null) {
cacheFileReader.closeCurrentFile();
cacheFileReader = null;
}
}
+
+ private void releaseMemory() {
+ if (!context.isPresent()) {
+ return;
+ }
+
context.get().getMemoryReservationContext().releaseMemoryCumulatively(acquiredMemory);
+ acquiredMemory = 0;
+ }
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TimePartitionTableSizeQueryContext.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TimePartitionTableSizeQueryContext.java
index bc7f8271d99..9c5f1e35352 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TimePartitionTableSizeQueryContext.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TimePartitionTableSizeQueryContext.java
@@ -21,10 +21,15 @@ package
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
+import org.apache.tsfile.utils.Accountable;
+import org.apache.tsfile.utils.RamUsageEstimator;
+
import java.util.HashMap;
import java.util.Map;
-public class TimePartitionTableSizeQueryContext {
+public class TimePartitionTableSizeQueryContext implements Accountable {
+ private static final long SHALLOW_SIZE =
+
RamUsageEstimator.shallowSizeOf(TimePartitionTableSizeQueryContext.class);
private final Map<String, Long> tableSizeResultMap;
Map<TsFileID, Long> tsFileIDOffsetInValueFileMap;
@@ -65,4 +70,26 @@ public class TimePartitionTableSizeQueryContext {
public Long getCachedTsFileIdOffset(TsFileID tsFileID) {
return tsFileIDOffsetInValueFileMap == null ? null :
tsFileIDOffsetInValueFileMap.get(tsFileID);
}
+
+ @Override
+ public long ramBytesUsed() {
+ return SHALLOW_SIZE
+ + RamUsageEstimator.sizeOfMapWithKnownShallowSize(
+ tableSizeResultMap,
+ RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP,
+ RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP_ENTRY)
+ + ramBytesUsedOfTsFileIDOffsetMap();
+ }
+
+ // tsFileIDOffsetInValueFileMap should be null af first
+ public long ramBytesUsedOfTsFileIDOffsetMap() {
+ if (tsFileIDOffsetInValueFileMap == null) {
+ return 0;
+ }
+ return RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP
+ + tsFileIDOffsetInValueFileMap.size()
+ * (RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP_ENTRY
+ + Long.BYTES
+ + TsFileID.SHALLOW_SIZE);
+ }
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheWriter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableDiskUsageCacheWriter.java
similarity index 97%
rename from
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheWriter.java
rename to
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableDiskUsageCacheWriter.java
index dd6bd887111..fd1c79b06ed 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheWriter.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableDiskUsageCacheWriter.java
@@ -42,8 +42,9 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
-public class TableDiskUsageCacheWriter {
- private static final Logger logger =
LoggerFactory.getLogger(TableDiskUsageCacheWriter.class);
+public class TsFileTableDiskUsageCacheWriter {
+ private static final Logger logger =
+ LoggerFactory.getLogger(TsFileTableDiskUsageCacheWriter.class);
private static final String TSFILE_CACHE_KEY_FILENAME_PREFIX =
"TableSizeKeyFile_";
private static final String TSFILE_CACHE_VALUE_FILENAME_PREFIX =
"TableSizeValueFile_";
public static final int KEY_FILE_OFFSET_RECORD_LENGTH = 5 * Long.BYTES + 1;
@@ -60,7 +61,7 @@ public class TableDiskUsageCacheWriter {
private final File dir;
private TsFileTableSizeCacheWriter tsFileTableSizeCacheWriter;
- public TableDiskUsageCacheWriter(String database, int regionId) {
+ public TsFileTableDiskUsageCacheWriter(String database, int regionId) {
this.regionId = regionId;
this.dir = StorageEngine.getDataRegionSystemDir(database, regionId + "");
recoverTsFileTableSizeIndexFile(true);
@@ -176,7 +177,7 @@ public class TableDiskUsageCacheWriter {
int fileNum = tsFileManager.size(true) + tsFileManager.size(false);
int estimatedEntryNumInCacheFile = (int) (keyFileLength() /
KEY_FILE_OFFSET_RECORD_LENGTH);
int delta = estimatedEntryNumInCacheFile - fileNum;
- return delta > 0.2 * estimatedEntryNumInCacheFile || delta >= 1000;
+ return delta >= 1000;
}
public void compact() {
@@ -318,11 +319,9 @@ public class TableDiskUsageCacheWriter {
}
public void decreaseActiveReaderNum() {
- activeReaderNum--;
- }
-
- public int getActiveReaderNum() {
- return activeReaderNum;
+ if (activeReaderNum > 0) {
+ activeReaderNum--;
+ }
}
public void removeFiles() {}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeCacheReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeCacheReader.java
index 7895cee705a..74127625057 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeCacheReader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeCacheReader.java
@@ -177,10 +177,10 @@ public class TsFileTableSizeCacheReader {
TsFileID tsFileID =
new TsFileID(regionId, timePartition, timestamp, fileVersion,
compactionVersion);
KeyFileEntry keyFileEntry;
- if (type == TableDiskUsageCacheWriter.KEY_FILE_RECORD_TYPE_OFFSET) {
+ if (type == TsFileTableDiskUsageCacheWriter.KEY_FILE_RECORD_TYPE_OFFSET) {
long offset = ReadWriteIOUtils.readLong(inputStream);
keyFileEntry = new KeyFileEntry(tsFileID, offset);
- } else if (type ==
TableDiskUsageCacheWriter.KEY_FILE_RECORD_TYPE_REDIRECT) {
+ } else if (type ==
TsFileTableDiskUsageCacheWriter.KEY_FILE_RECORD_TYPE_REDIRECT) {
long originTimestamp = ReadWriteIOUtils.readLong(inputStream);
long originFileVersion = ReadWriteIOUtils.readLong(inputStream);
long originCompactionVersion = ReadWriteIOUtils.readLong(inputStream);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeCacheWriter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeCacheWriter.java
index 29287eb2af2..b073a5f2204 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeCacheWriter.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeCacheWriter.java
@@ -33,8 +33,8 @@ import java.nio.channels.FileChannel;
import java.nio.file.StandardOpenOption;
import java.util.Map;
-import static
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TableDiskUsageCacheWriter.KEY_FILE_RECORD_TYPE_OFFSET;
-import static
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TableDiskUsageCacheWriter.KEY_FILE_RECORD_TYPE_REDIRECT;
+import static
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TsFileTableDiskUsageCacheWriter.KEY_FILE_RECORD_TYPE_OFFSET;
+import static
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TsFileTableDiskUsageCacheWriter.KEY_FILE_RECORD_TYPE_REDIRECT;
public class TsFileTableSizeCacheWriter {
private final int regionId;
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileTableSizeCacheWriterTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileTableSizeCacheWriterTest.java
index 53cf08ab747..6517ff466d8 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileTableSizeCacheWriterTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileTableSizeCacheWriterTest.java
@@ -27,7 +27,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TableDiskUsageCacheWriter;
+import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TsFileTableDiskUsageCacheWriter;
import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TsFileTableSizeCacheReader;
import org.apache.tsfile.exception.write.WriteProcessException;
@@ -62,8 +62,8 @@ public class TsFileTableSizeCacheWriterTest extends
AbstractCompactionTest {
StorageEngine.getInstance().setDataRegion(new DataRegionId(0), dataRegion);
TsFileManager tsFileManager = new TsFileManager("root.test", "0", "");
Mockito.when(dataRegion.getTsFileManager()).thenReturn(tsFileManager);
- TableDiskUsageCacheWriter writer =
- new TableDiskUsageCacheWriter(dataRegion.getDatabaseName(), 0);
+ TsFileTableDiskUsageCacheWriter writer =
+ new TsFileTableDiskUsageCacheWriter(dataRegion.getDatabaseName(), 0);
File oldKeyFile = writer.getKeyFile();
File oldValueFile = writer.getValueFile();
Assert.assertEquals("TableSizeKeyFile_0", oldKeyFile.getName());
@@ -92,8 +92,8 @@ public class TsFileTableSizeCacheWriterTest extends
AbstractCompactionTest {
StorageEngine.getInstance().setDataRegion(new DataRegionId(0), dataRegion);
TsFileManager tsFileManager = new TsFileManager("root.test", "0", "");
Mockito.when(dataRegion.getTsFileManager()).thenReturn(tsFileManager);
- TableDiskUsageCacheWriter writer =
- new TableDiskUsageCacheWriter(dataRegion.getDatabaseName(), 0);
+ TsFileTableDiskUsageCacheWriter writer =
+ new TsFileTableDiskUsageCacheWriter(dataRegion.getDatabaseName(), 0);
File oldKeyFile = writer.getKeyFile();
File oldValueFile = writer.getValueFile();
Assert.assertEquals("TableSizeKeyFile_0", oldKeyFile.getName());
@@ -140,8 +140,8 @@ public class TsFileTableSizeCacheWriterTest extends
AbstractCompactionTest {
StorageEngine.getInstance().setDataRegion(new DataRegionId(0), dataRegion);
TsFileManager tsFileManager = new TsFileManager("root.test", "0", "");
Mockito.when(dataRegion.getTsFileManager()).thenReturn(tsFileManager);
- TableDiskUsageCacheWriter writer =
- new TableDiskUsageCacheWriter(dataRegion.getDatabaseName(), 0);
+ TsFileTableDiskUsageCacheWriter writer =
+ new TsFileTableDiskUsageCacheWriter(dataRegion.getDatabaseName(), 0);
File oldKeyFile = writer.getKeyFile();
File oldValueFile = writer.getValueFile();
Assert.assertEquals("TableSizeKeyFile_0", oldKeyFile.getName());
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index ebd4d3b7ae0..0de593390f4 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@ -39,6 +39,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.flush.FlushManager;
import
org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
import
org.apache.iotdb.db.storageengine.dataregion.read.control.QueryResourceManager;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndexCacheRecorder;
+import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TableDiskUsageCache;
import org.apache.iotdb.db.storageengine.dataregion.wal.WALManager;
import
org.apache.iotdb.db.storageengine.dataregion.wal.recover.WALRecoverManager;
import org.apache.iotdb.db.storageengine.rescon.disk.TierManager;
@@ -228,6 +229,7 @@ public class EnvironmentUtils {
cleanDir(path);
}
FileTimeIndexCacheRecorder.getInstance().close();
+ TableDiskUsageCache.getInstance().close();
// delete system info
cleanDir(config.getSystemDir());
// delete query
