This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch table_disk_usage_statistics_with_cache
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to
refs/heads/table_disk_usage_statistics_with_cache by this push:
new 50bceddda95 refactor
50bceddda95 is described below
commit 50bceddda9530a32aeeadea26baccd8f3b9fa63c
Author: shuwenwei <[email protected]>
AuthorDate: Thu Jan 22 17:20:59 2026 +0800
refactor
---
.../InformationSchemaContentSupplierFactory.java | 19 ++--
.../DataRegionTableSizeQueryContext.java | 112 +++++++++++++++++++++
.../TableDiskUsageCacheReader.java | 20 ++--
.../TimePartitionTableSizeQueryContext.java | 16 +++
.../TsFileTableSizeCacheReader.java | 25 +----
.../object/EmptyObjectTableSizeCacheReader.java | 8 +-
.../object/IObjectTableSizeCacheReader.java | 7 +-
7 files changed, 158 insertions(+), 49 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java
index a5519bfda33..6d5ef1ae908 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java
@@ -88,6 +88,7 @@ import
org.apache.iotdb.db.relational.grammar.sql.RelationalSqlKeywords;
import org.apache.iotdb.db.schemaengine.table.InformationSchemaUtils;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import
org.apache.iotdb.db.storageengine.dataregion.utils.StorageEngineTimePartitionIterator;
+import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.DataRegionTableSizeQueryContext;
import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TableDiskUsageCacheReader;
import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TimePartitionTableSizeQueryContext;
import org.apache.iotdb.db.utils.MathUtils;
@@ -113,7 +114,6 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
@@ -1198,8 +1198,8 @@ public class InformationSchemaContentSupplierFactory {
private boolean currentDatabaseOnlyHasOneTable;
private TableDiskUsageCacheReader currentDataRegionCacheReader;
- private final Map<Long, TimePartitionTableSizeQueryContext>
timePartitionsContextMap =
- new LinkedHashMap<>();
+ private DataRegionTableSizeQueryContext
currentDataRegionTableSizeQueryContext =
+ new DataRegionTableSizeQueryContext(false);
private final StorageEngineTimePartitionIterator dataRegionIterator;
@@ -1228,7 +1228,8 @@ public class InformationSchemaContentSupplierFactory {
if (tTableInfos == null || tTableInfos.isEmpty()) {
return false;
}
- timePartitionsContextMap.clear();
+ currentDataRegionTableSizeQueryContext =
+ new DataRegionTableSizeQueryContext(false);
return
PathUtils.isTableModelDatabase(dataRegion.getDatabaseName());
}),
Optional.empty());
@@ -1248,17 +1249,17 @@ public class InformationSchemaContentSupplierFactory {
for (Long timePartition :
currentDataRegion.getTsFileManager().getTimePartitions()) {
Map<String, Long> tablesToScan =
getTablesToScan(currentDataRegion, timePartition);
if (!tablesToScan.isEmpty()) {
- timePartitionsContextMap.put(
+ currentDataRegionTableSizeQueryContext.addTimePartition(
timePartition, new
TimePartitionTableSizeQueryContext(tablesToScan));
}
}
- if (timePartitionsContextMap.isEmpty()) {
+ if (currentDataRegionTableSizeQueryContext.isEmpty()) {
continue;
}
currentDataRegionCacheReader =
new TableDiskUsageCacheReader(
currentDataRegion,
- timePartitionsContextMap,
+ currentDataRegionTableSizeQueryContext,
currentDatabaseOnlyHasOneTable,
Optional.ofNullable(operatorContext.getInstanceContext()));
return true;
@@ -1358,7 +1359,9 @@ public class InformationSchemaContentSupplierFactory {
private TsBlock buildTsBlock() {
TsBlockBuilder builder = new TsBlockBuilder(dataTypes);
for (Map.Entry<Long, TimePartitionTableSizeQueryContext> entry :
- timePartitionsContextMap.entrySet()) {
+ currentDataRegionTableSizeQueryContext
+ .getTimePartitionTableSizeQueryContextMap()
+ .entrySet()) {
long timePartition = entry.getKey();
for (Map.Entry<String, Long> tableSizeEntry :
entry.getValue().getTableSizeResultMap().entrySet()) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/DataRegionTableSizeQueryContext.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/DataRegionTableSizeQueryContext.java
new file mode 100644
index 00000000000..521abc4a167
--- /dev/null
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/DataRegionTableSizeQueryContext.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache;
+
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
+
+import org.apache.tsfile.utils.Accountable;
+
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class DataRegionTableSizeQueryContext implements Accountable {
+
+ private final boolean needAllData;
+
+ private final Map<Long, TimePartitionTableSizeQueryContext>
+ timePartitionTableSizeQueryContextMap = new LinkedHashMap<>();
+ private int objectFileNum = 0;
+
+ private long previousUsedTimePartition;
+ private TimePartitionTableSizeQueryContext previousUsedTimePartitionContext;
+
+ public DataRegionTableSizeQueryContext(boolean needAllData) {
+ this.needAllData = needAllData;
+ }
+
+ public Map<Long, TimePartitionTableSizeQueryContext>
getTimePartitionTableSizeQueryContextMap() {
+ return timePartitionTableSizeQueryContextMap;
+ }
+
+ public boolean isEmpty() {
+ return timePartitionTableSizeQueryContextMap.isEmpty();
+ }
+
+ public int getObjectFileNum() {
+ return objectFileNum;
+ }
+
+ public long getObjectFileSize() {
+ long totalSize = 0;
+ for (TimePartitionTableSizeQueryContext timePartitionContext :
+ timePartitionTableSizeQueryContextMap.values()) {
+ totalSize += timePartitionContext.getObjectFileSize();
+ }
+ return totalSize;
+ }
+
+ public void addCachedTsFileIDAndOffsetInValueFile(TsFileID tsFileID, long
offset) {
+ switchTimePartition(tsFileID.timePartitionId);
+
previousUsedTimePartitionContext.addCachedTsFileIDAndOffsetInValueFile(tsFileID,
offset);
+ }
+
+ public void replaceCachedTsFileID(TsFileID originTsFileID, TsFileID
newTsFileID) {
+ switchTimePartition(originTsFileID.timePartitionId);
+ previousUsedTimePartitionContext.replaceCachedTsFileID(originTsFileID,
newTsFileID);
+ }
+
+ public void updateResult(String table, long size, long currentTimePartition)
{
+ switchTimePartition(currentTimePartition);
+ previousUsedTimePartitionContext.updateResult(table, size, needAllData);
+ }
+
+ private void switchTimePartition(long currentTimePartition) {
+ if (currentTimePartition != previousUsedTimePartition
+ || previousUsedTimePartitionContext == null) {
+ TimePartitionTableSizeQueryContext currentTimePartitionContext =
+ timePartitionTableSizeQueryContextMap.compute(
+ currentTimePartition,
+ (k, v) ->
+ (v == null && needAllData)
+ ? new TimePartitionTableSizeQueryContext(new HashMap<>())
+ : v);
+ if (currentTimePartitionContext == null) {
+ return;
+ }
+ previousUsedTimePartition = currentTimePartition;
+ previousUsedTimePartitionContext = currentTimePartitionContext;
+ }
+ }
+
+ public void addTimePartition(
+ long timePartition, TimePartitionTableSizeQueryContext
timePartitionTableSizeQueryContext) {
+ timePartitionTableSizeQueryContextMap.put(timePartition,
timePartitionTableSizeQueryContext);
+ }
+
+ public void updateObjectFileNum(int delta) {
+ this.objectFileNum += delta;
+ }
+
+ @Override
+ public long ramBytesUsed() {
+ return 0;
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheReader.java
index 80a8c8bd955..4c591f27266 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheReader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheReader.java
@@ -43,7 +43,7 @@ public class TableDiskUsageCacheReader implements Closeable {
private final DataRegion dataRegion;
private final int regionId;
- private final Map<Long, TimePartitionTableSizeQueryContext>
timePartitionQueryContexts;
+ private final DataRegionTableSizeQueryContext dataRegionContext;
private CompletableFuture<Pair<TsFileTableSizeCacheReader,
IObjectTableSizeCacheReader>>
prepareReaderFuture;
@@ -64,18 +64,19 @@ public class TableDiskUsageCacheReader implements Closeable
{
public TableDiskUsageCacheReader(
DataRegion dataRegion,
- Map<Long, TimePartitionTableSizeQueryContext> resultMap,
+ DataRegionTableSizeQueryContext dataRegionContext,
boolean databaseHasOnlyOneTable,
Optional<FragmentInstanceContext> context) {
this.dataRegion = dataRegion;
this.regionId = Integer.parseInt(dataRegion.getDataRegionIdString());
- this.timePartitionQueryContexts = resultMap;
+ this.dataRegionContext = dataRegionContext;
this.currentDatabaseOnlyHasOneTable = databaseHasOnlyOneTable;
this.context = context;
- this.timePartitionIterator =
timePartitionQueryContexts.entrySet().iterator();
+ this.timePartitionIterator =
+
dataRegionContext.getTimePartitionTableSizeQueryContextMap().entrySet().iterator();
reserveMemory(
RamUsageEstimator.sizeOfMapWithKnownShallowSize(
- timePartitionQueryContexts,
+ dataRegionContext.getTimePartitionTableSizeQueryContextMap(),
RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP,
RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP_ENTRY));
}
@@ -110,7 +111,7 @@ public class TableDiskUsageCacheReader implements Closeable
{
public boolean loadObjectFileTableSizeCache(long startTime, long maxRunTime)
throws Exception {
if (objectTableSizeCacheReader.loadObjectFileTableSize(
- timePartitionQueryContexts, startTime, maxRunTime)) {
+ dataRegionContext, startTime, maxRunTime)) {
objectTableSizeCacheReader.close();
return true;
}
@@ -121,10 +122,9 @@ public class TableDiskUsageCacheReader implements
Closeable {
if (tsFileIdKeysPrepared) {
return true;
}
- if (tsFileTableSizeCacheReader.readFromKeyFile(
- timePartitionQueryContexts, startTime, maxRunTime)) {
+ if (tsFileTableSizeCacheReader.readFromKeyFile(dataRegionContext,
startTime, maxRunTime)) {
reserveMemory(
- timePartitionQueryContexts.values().stream()
+
dataRegionContext.getTimePartitionTableSizeQueryContextMap().values().stream()
.mapToLong(TimePartitionTableSizeQueryContext::ramBytesUsedOfTsFileIDOffsetMap)
.sum());
tsFileIdKeysPrepared = true;
@@ -165,7 +165,7 @@ public class TableDiskUsageCacheReader implements Closeable
{
this.tsFileTableSizeCacheReader.openValueFile();
}
return tsFileTableSizeCacheReader.readFromValueFile(
- tsFilesToQueryInCacheIterator, timePartitionQueryContexts, startTime,
maxRunTime);
+ tsFilesToQueryInCacheIterator, dataRegionContext, startTime,
maxRunTime);
}
public DataRegion getDataRegion() {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TimePartitionTableSizeQueryContext.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TimePartitionTableSizeQueryContext.java
index 9c5f1e35352..242bc65d9ec 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TimePartitionTableSizeQueryContext.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TimePartitionTableSizeQueryContext.java
@@ -58,6 +58,14 @@ public class TimePartitionTableSizeQueryContext implements
Accountable {
tableSizeResultMap.computeIfPresent(table, (k, v) -> v + size);
}
+ public void updateResult(String table, long size, boolean needAllData) {
+ if (needAllData) {
+ tableSizeResultMap.compute(table, (k, v) -> (v == null ? 0 : v) + size);
+ } else {
+ tableSizeResultMap.computeIfPresent(table, (k, v) -> v + size);
+ }
+ }
+
public Map<String, Long> getTableSizeResultMap() {
return tableSizeResultMap;
}
@@ -71,6 +79,14 @@ public class TimePartitionTableSizeQueryContext implements
Accountable {
return tsFileIDOffsetInValueFileMap == null ? null :
tsFileIDOffsetInValueFileMap.get(tsFileID);
}
+ public long getObjectFileSize() {
+ long size = 0;
+ for (Long value : tableSizeResultMap.values()) {
+ size += value;
+ }
+ return size;
+ }
+
@Override
public long ramBytesUsed() {
return SHALLOW_SIZE
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeCacheReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeCacheReader.java
index 74127625057..351c2b10e9f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeCacheReader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TsFileTableSizeCacheReader.java
@@ -128,12 +128,8 @@ public class TsFileTableSizeCacheReader {
}
public boolean readFromKeyFile(
- Map<Long, TimePartitionTableSizeQueryContext> timePartitionContexts,
- long startTime,
- long maxRunTime)
+ DataRegionTableSizeQueryContext dataRegionContext, long startTime, long
maxRunTime)
throws IOException {
- long previousTimePartition = 0;
- TimePartitionTableSizeQueryContext timePartitionContext = null;
do {
if (keyFileLength == 0) {
return true;
@@ -144,16 +140,11 @@ public class TsFileTableSizeCacheReader {
}
try {
KeyFileEntry keyFileEntry = readOneEntryFromKeyFile();
- if (timePartitionContext == null
- || keyFileEntry.tsFileID.timePartitionId != previousTimePartition)
{
- previousTimePartition = keyFileEntry.tsFileID.timePartitionId;
- timePartitionContext =
timePartitionContexts.get(previousTimePartition);
- }
if (keyFileEntry.originTsFileID == null) {
- timePartitionContext.addCachedTsFileIDAndOffsetInValueFile(
+ dataRegionContext.addCachedTsFileIDAndOffsetInValueFile(
keyFileEntry.tsFileID, keyFileEntry.offset);
} else {
- timePartitionContext.replaceCachedTsFileID(
+ dataRegionContext.replaceCachedTsFileID(
keyFileEntry.tsFileID, keyFileEntry.originTsFileID);
}
} catch (IOException e) {
@@ -198,12 +189,10 @@ public class TsFileTableSizeCacheReader {
public boolean readFromValueFile(
Iterator<Pair<TsFileID, Long>> tsFilesToQueryInCache,
- Map<Long, TimePartitionTableSizeQueryContext> resultMap,
+ DataRegionTableSizeQueryContext dataRegionContext,
long startTime,
long maxRunTime)
throws IOException {
- long previousTimePartition = 0;
- TimePartitionTableSizeQueryContext currentTimePartition = null;
do {
if (!tsFilesToQueryInCache.hasNext()) {
closeCurrentFile();
@@ -211,10 +200,6 @@ public class TsFileTableSizeCacheReader {
}
Pair<TsFileID, Long> pair = tsFilesToQueryInCache.next();
long timePartition = pair.left.timePartitionId;
- if (currentTimePartition == null || timePartition !=
previousTimePartition) {
- currentTimePartition = resultMap.get(timePartition);
- previousTimePartition = timePartition;
- }
long offset = pair.right;
inputStream.seek(offset);
@@ -222,7 +207,7 @@ public class TsFileTableSizeCacheReader {
for (int i = 0; i < tableNum; i++) {
String tableName = ReadWriteIOUtils.readVarIntString(inputStream);
long size = ReadWriteIOUtils.readLong(inputStream);
- currentTimePartition.updateResult(tableName, size);
+ dataRegionContext.updateResult(tableName, size, timePartition);
}
} while (System.nanoTime() - startTime < maxRunTime);
return false;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/object/EmptyObjectTableSizeCacheReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/object/EmptyObjectTableSizeCacheReader.java
index d72ddc0212b..2020ab10a23 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/object/EmptyObjectTableSizeCacheReader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/object/EmptyObjectTableSizeCacheReader.java
@@ -19,17 +19,13 @@
package
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.object;
-import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TimePartitionTableSizeQueryContext;
-
-import java.util.Map;
+import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.DataRegionTableSizeQueryContext;
public class EmptyObjectTableSizeCacheReader implements
IObjectTableSizeCacheReader {
@Override
public boolean loadObjectFileTableSize(
- Map<Long, TimePartitionTableSizeQueryContext> timePartitionContexts,
- long startTime,
- long maxRunTime) {
+ DataRegionTableSizeQueryContext dataRegionContext, long startTime, long
maxRunTime) {
return true;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/object/IObjectTableSizeCacheReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/object/IObjectTableSizeCacheReader.java
index 88f641f65fe..cc38307f093 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/object/IObjectTableSizeCacheReader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/object/IObjectTableSizeCacheReader.java
@@ -19,16 +19,13 @@
package
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.object;
-import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TimePartitionTableSizeQueryContext;
+import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.DataRegionTableSizeQueryContext;
import java.io.IOException;
-import java.util.Map;
public interface IObjectTableSizeCacheReader {
boolean loadObjectFileTableSize(
- Map<Long, TimePartitionTableSizeQueryContext> timePartitionContexts,
- long startTime,
- long maxRunTime)
+ DataRegionTableSizeQueryContext dataRegionContext, long startTime, long
maxRunTime)
throws IOException;
void close();
