This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch table_disk_usage_statistics
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/table_disk_usage_statistics by
this push:
new b6787b4e808 fix
b6787b4e808 is described below
commit b6787b4e808370ac864209f775e1608b4051082f
Author: shuwenwei <[email protected]>
AuthorDate: Wed Dec 10 17:03:52 2025 +0800
fix
---
.../java/org/apache/iotdb/session/Session.java | 7 +-
.../InformationSchemaContentSupplierFactory.java | 5 +
.../plan/relational/sql/rewrite/ShowRewrite.java | 4 +-
.../execute/utils/MultiTsFileDeviceIterator.java | 4 +-
.../dataregion/utils/DiskUsageStatisticUtil.java | 111 +++++++++++++++++----
.../utils/TableDiskUsageStatisticUtil.java | 81 ++++++++++-----
.../utils/TreeDiskUsageStatisticUtil.java | 64 ++++++++----
7 files changed, 203 insertions(+), 73 deletions(-)
diff --git
a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java
b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java
index 8844f36c247..ea7c7c85cd7 100644
--- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java
+++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java
@@ -3074,10 +3074,9 @@ public class Session implements ISession {
this.columnEncodersMap
.getOrDefault(
measurementSchema.getType(),
- TSEncoding.valueOf(
- TSFileDescriptor.getInstance()
- .getConfig()
- .getValueEncoder(measurementSchema.getType())))
+ TSFileDescriptor.getInstance()
+ .getConfig()
+ .getValueEncoder(measurementSchema.getType()))
.serialize());
}
} else {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java
index 9115cd85025..08a35c9744d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java
@@ -1252,6 +1252,7 @@ public class InformationSchemaContentSupplierFactory {
private DataRegion currentDataRegion;
private long currentTimePartition;
private List<String> currentTablesToScan;
+ private boolean currentDatabaseOnlyHasOneTable;
private TableDiskUsageStatisticUtil statisticUtil;
private final StorageEngineTimePartitionIterator timePartitionIterator;
@@ -1307,6 +1308,7 @@ public class InformationSchemaContentSupplierFactory {
currentDataRegion.getTsFileManager(),
currentTimePartition,
currentTablesToScan,
+ currentDatabaseOnlyHasOneTable,
Optional.ofNullable(operatorContext.getInstanceContext()));
return true;
}
@@ -1325,10 +1327,12 @@ public class InformationSchemaContentSupplierFactory {
}
List<String> tablesToScan = new ArrayList<>(tTableInfos.size());
+ int totalValidTableCount = 0;
for (TTableInfo tTableInfo : tTableInfos) {
if (tTableInfo.getType() != TableType.BASE_TABLE.ordinal()) {
continue;
}
+ totalValidTableCount++;
if (pushDownFilter != null) {
Object[] row = new Object[5];
row[0] = new Binary(dataRegion.getDatabaseName(),
TSFileConfig.STRING_CHARSET);
@@ -1350,6 +1354,7 @@ public class InformationSchemaContentSupplierFactory {
paginationController.consumeLimit();
tablesToScan.add(tTableInfo.getTableName());
}
+ currentDatabaseOnlyHasOneTable = totalValidTableCount == 1;
return tablesToScan;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/rewrite/ShowRewrite.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/rewrite/ShowRewrite.java
index 63693ae099f..12fd67d5017 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/rewrite/ShowRewrite.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/rewrite/ShowRewrite.java
@@ -60,7 +60,7 @@ public final class ShowRewrite implements
StatementRewrite.Rewrite {
final Map<NodeRef<Parameter>, Expression> parameterLookup,
final WarningCollector warningCollector) {
final Visitor visitor = new Visitor();
- return (Statement) visitor.process(node);
+ return (Statement) visitor.process(node, null);
}
private static class Visitor extends AstVisitor<Node, Void> {
@@ -85,7 +85,7 @@ public final class ShowRewrite implements
StatementRewrite.Rewrite {
}
@Override
- protected Node visitCountStatement(final CountStatement countStatement,
Void context) {
+ protected Node visitCountStatement(final CountStatement countStatement,
final Void context) {
return simpleQuery(
new Select(
false,
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
index 076d2de5a03..4dfa5a05661 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
@@ -175,7 +175,7 @@ public class MultiTsFileDeviceIterator implements
AutoCloseable {
}
}
- public boolean hasNextDevice() {
+ public boolean hasNextDevice() throws IOException {
boolean hasNext = false;
for (TsFileDeviceIterator iterator : deviceIteratorMap.values()) {
hasNext =
@@ -196,7 +196,7 @@ public class MultiTsFileDeviceIterator implements
AutoCloseable {
* @return Pair of device full path and whether this device is aligned
*/
@SuppressWarnings({"squid:S135", "java:S2259"})
- public Pair<IDeviceID, Boolean> nextDevice() throws IllegalPathException {
+ public Pair<IDeviceID, Boolean> nextDevice() throws IllegalPathException,
IOException {
List<TsFileResource> toBeRemovedResources = new LinkedList<>();
Pair<IDeviceID, Boolean> minDevice = null;
// get the device from source files sorted from the newest to the oldest
by version
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/DiskUsageStatisticUtil.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/DiskUsageStatisticUtil.java
index 20e43010308..07296d217d2 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/DiskUsageStatisticUtil.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/DiskUsageStatisticUtil.java
@@ -25,11 +25,12 @@ import
org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManag
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.tsfile.file.IMetadataIndexEntry;
import org.apache.tsfile.file.header.ChunkGroupHeader;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.MetadataIndexNode;
-import org.apache.tsfile.file.metadata.TimeseriesMetadata;
+import org.apache.tsfile.file.metadata.enums.MetadataIndexNodeType;
import org.apache.tsfile.read.TsFileSequenceReader;
import org.apache.tsfile.utils.Pair;
import org.slf4j.Logger;
@@ -37,11 +38,11 @@ import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import java.util.Queue;
import java.util.function.LongConsumer;
@@ -108,7 +109,9 @@ public abstract class DiskUsageStatisticUtil implements
Closeable {
public void calculateNextFile() {
TsFileResource tsFileResource = iterator.next();
- if (tsFileResource.isDeleted()) {
+ if (tsFileResource.isDeleted() ||
calculateWithoutOpenFile(tsFileResource)) {
+ iterator.remove();
+ tsFileResource.readUnlock();
return;
}
FileReaderManager.getInstance().increaseFileReaderReference(tsFileResource,
true);
@@ -130,39 +133,72 @@ public abstract class DiskUsageStatisticUtil implements
Closeable {
}
}
+ protected abstract boolean calculateWithoutOpenFile(TsFileResource
tsFileResource);
+
protected abstract void calculateNextFile(
TsFileResource tsFileResource, TsFileSequenceReader reader)
throws IOException, IllegalPathException;
- protected long calculateStartOffsetOfChunkGroup(
+ protected Offsets calculateStartOffsetOfChunkGroupAndTimeseriesMetadata(
TsFileSequenceReader reader,
MetadataIndexNode firstMeasurementNodeOfCurrentDevice,
- Pair<IDeviceID, Boolean> deviceIsAlignedPair)
+ Pair<IDeviceID, Boolean> deviceIsAlignedPair,
+ long rootMeasurementNodeStartOffset)
throws IOException {
int chunkGroupHeaderSize =
new
ChunkGroupHeader(deviceIsAlignedPair.getLeft()).getSerializedSize();
if (deviceIsAlignedPair.getRight()) {
- TimeseriesMetadata timeColumnTimeseriesMetadata =
- reader.getTimeColumnMetadata(
- firstMeasurementNodeOfCurrentDevice,
timeSeriesMetadataIoSizeRecorder);
- IChunkMetadata iChunkMetadata =
timeColumnTimeseriesMetadata.getChunkMetadataList().get(0);
- return iChunkMetadata.getOffsetOfChunkHeader() - chunkGroupHeaderSize;
+ Pair<Long, Long> timeseriesMetadataOffsetPair =
+ getTimeColumnMetadataOffset(reader,
firstMeasurementNodeOfCurrentDevice);
+ IChunkMetadata firstChunkMetadata =
+ reader
+ .getChunkMetadataListByTimeseriesMetadataOffset(
+ timeseriesMetadataOffsetPair.getLeft(),
timeseriesMetadataOffsetPair.getRight())
+ .get(0);
+ return new Offsets(
+ firstChunkMetadata.getOffsetOfChunkHeader() - chunkGroupHeaderSize,
+ timeseriesMetadataOffsetPair.getLeft(),
+ rootMeasurementNodeStartOffset);
} else {
- List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
- reader.getDeviceTimeseriesMetadata(
- timeseriesMetadataList,
- firstMeasurementNodeOfCurrentDevice,
- Collections.emptySet(),
- true,
- timeSeriesMetadataIoSizeRecorder);
- long minOffset = Long.MAX_VALUE;
- for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) {
- for (IChunkMetadata chunkMetadata :
timeseriesMetadata.getChunkMetadataList()) {
- minOffset = Math.min(minOffset,
chunkMetadata.getOffsetOfChunkHeader());
+ Map<String, Pair<List<IChunkMetadata>, Pair<Long, Long>>>
timeseriesMetadataOffsetByDevice =
+ reader.getTimeseriesMetadataOffsetByDevice(
+ firstMeasurementNodeOfCurrentDevice, Collections.emptySet(),
true);
+ long minTimeseriesMetadataOffset = 0;
+ long minChunkOffset = Long.MAX_VALUE;
+ for (Map.Entry<String, Pair<List<IChunkMetadata>, Pair<Long, Long>>>
entry :
+ timeseriesMetadataOffsetByDevice.entrySet()) {
+ minTimeseriesMetadataOffset =
+ minTimeseriesMetadataOffset == 0
+ ? entry.getValue().getRight().getLeft()
+ : minTimeseriesMetadataOffset;
+ for (IChunkMetadata chunkMetadata : entry.getValue().getLeft()) {
+ minChunkOffset = Math.min(minChunkOffset,
chunkMetadata.getOffsetOfChunkHeader());
break;
}
}
- return minOffset - chunkGroupHeaderSize;
+ return new Offsets(
+ minChunkOffset - chunkGroupHeaderSize,
+ minTimeseriesMetadataOffset,
+ rootMeasurementNodeStartOffset);
+ }
+ }
+
+ private Pair<Long, Long> getTimeColumnMetadataOffset(
+ TsFileSequenceReader reader, MetadataIndexNode measurementNode) throws
IOException {
+ if (measurementNode.isDeviceLevel()) {
+ throw new IllegalArgumentException("device level metadata index node is
not supported");
+ }
+ List<IMetadataIndexEntry> children = measurementNode.getChildren();
+ long startOffset = children.get(0).getOffset();
+ long endOffset =
+ children.size() > 1 ? children.get(1).getOffset() :
measurementNode.getEndOffset();
+ if
(measurementNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
+ return new Pair<>(startOffset, endOffset);
+ } else {
+ MetadataIndexNode metadataIndexNode =
+ reader.readMetadataIndexNode(
+ startOffset, endOffset, false, timeSeriesMetadataIoSizeRecorder);
+ return getTimeColumnMetadataOffset(reader, metadataIndexNode);
}
}
@@ -170,4 +206,35 @@ public abstract class DiskUsageStatisticUtil implements
Closeable {
public void close() throws IOException {
releaseReadLocks();
}
+
+ protected static class Offsets {
+ protected final long firstChunkOffset;
+ protected final long firstTimeseriesMetadataOffset;
+ protected final long firstMeasurementNodeOffset;
+
+ public Offsets(
+ long firstChunkOffset,
+ long firstTimeseriesMetadataOffset,
+ long firstMeasurementNodeOffset) {
+ this.firstChunkOffset = firstChunkOffset;
+ this.firstTimeseriesMetadataOffset = firstTimeseriesMetadataOffset;
+ this.firstMeasurementNodeOffset = firstMeasurementNodeOffset;
+ }
+
+ protected long minusOffsetForTableModel(Offsets other) {
+ return firstChunkOffset
+ - other.firstChunkOffset
+ + firstTimeseriesMetadataOffset
+ - other.firstTimeseriesMetadataOffset
+ + firstMeasurementNodeOffset
+ - other.firstMeasurementNodeOffset;
+ }
+
+ protected long minusOffsetForTreeModel(Offsets other) {
+ return firstChunkOffset
+ - other.firstChunkOffset
+ + firstTimeseriesMetadataOffset
+ - other.firstTimeseriesMetadataOffset;
+ }
+ }
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageStatisticUtil.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageStatisticUtil.java
index 99c1cf227ae..f1236c1673e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageStatisticUtil.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageStatisticUtil.java
@@ -23,7 +23,6 @@ import
org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContex
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.MetadataIndexNode;
import org.apache.tsfile.file.metadata.TsFileMetadata;
@@ -43,18 +42,21 @@ public class TableDiskUsageStatisticUtil extends
DiskUsageStatisticUtil {
public static final long SHALLOW_SIZE =
RamUsageEstimator.shallowSizeOfInstance(TableDiskUsageStatisticUtil.class);
private final Map<String, Integer> tableIndexMap;
+ private final boolean databaseHasOnlyOneTable;
private final long[] resultArr;
public TableDiskUsageStatisticUtil(
TsFileManager tsFileManager,
long timePartition,
List<String> tableNames,
+ boolean databaseHasOnlyOneTable,
Optional<FragmentInstanceContext> context) {
super(tsFileManager, timePartition, context);
this.tableIndexMap = new HashMap<>();
for (int i = 0; i < tableNames.size(); i++) {
tableIndexMap.put(tableNames.get(i), i);
}
+ this.databaseHasOnlyOneTable = databaseHasOnlyOneTable;
this.resultArr = new long[tableNames.size()];
}
@@ -63,6 +65,15 @@ public class TableDiskUsageStatisticUtil extends
DiskUsageStatisticUtil {
return resultArr;
}
+ @Override
+ protected boolean calculateWithoutOpenFile(TsFileResource tsFileResource) {
+ if (!databaseHasOnlyOneTable || tsFileResource.anyModFileExists()) {
+ return false;
+ }
+ resultArr[0] += tsFileResource.getTsFileSize();
+ return true;
+ }
+
@Override
protected void calculateNextFile(TsFileResource tsFileResource,
TsFileSequenceReader reader)
throws IOException {
@@ -70,16 +81,14 @@ public class TableDiskUsageStatisticUtil extends
DiskUsageStatisticUtil {
if (!hasSatisfiedData(tsFileMetadata)) {
return;
}
- int allSatisfiedTableIndex = getAllSatisfiedTableIndex(tsFileMetadata);
- if (allSatisfiedTableIndex > 0) {
- // size of tsfile - size of (tsfile magic string + version number + all
metadata + metadata
- // marker)
+ Pair<Integer, Boolean> allSatisfiedTableIndexPair =
getAllSatisfiedTableIndex(tsFileMetadata);
+ int allSatisfiedTableIndex = allSatisfiedTableIndexPair.getLeft();
+ // the only one table in this tsfile might be deleted by mods, and it is
not the table we
+ // queried
+ boolean mayContainSearchedTable = allSatisfiedTableIndexPair.getRight();
+ if (allSatisfiedTableIndex != -1) {
resultArr[allSatisfiedTableIndex] +=
- (tsFileResource.getTsFileSize()
- - reader.getAllMetadataSize()
- - 1
- - TSFileConfig.MAGIC_STRING.getBytes().length
- - 1);
+ mayContainSearchedTable ? tsFileResource.getTsFileSize() : 0;
return;
}
calculateDiskUsageInBytesByOffset(reader);
@@ -91,13 +100,15 @@ public class TableDiskUsageStatisticUtil extends
DiskUsageStatisticUtil {
return
tableIndexMap.keySet().stream().anyMatch(tableMetadataIndexNodeMap::containsKey);
}
- private int getAllSatisfiedTableIndex(TsFileMetadata tsFileMetadata) {
+ private Pair<Integer, Boolean> getAllSatisfiedTableIndex(TsFileMetadata
tsFileMetadata) {
if (tsFileMetadata.getTableMetadataIndexNodeMap().size() != 1) {
- return -1;
+ return new Pair<>(-1, true);
}
String satisfiedTableName =
tsFileMetadata.getTableMetadataIndexNodeMap().keySet().iterator().next();
- return tableIndexMap.get(satisfiedTableName);
+ String searchedTableName = tableIndexMap.keySet().iterator().next();
+ return new Pair<>(
+ tableIndexMap.get(satisfiedTableName),
satisfiedTableName.equals(searchedTableName));
}
private void calculateDiskUsageInBytesByOffset(TsFileSequenceReader reader)
throws IOException {
@@ -106,7 +117,7 @@ public class TableDiskUsageStatisticUtil extends
DiskUsageStatisticUtil {
tsFileMetadata.getTableMetadataIndexNodeMap();
String nextTable = null;
Iterator<String> iterator = tableMetadataIndexNodeMap.keySet().iterator();
- Map<String, Long> tableOffsetMap = new HashMap<>();
+ Map<String, Offsets> tableOffsetMap = new HashMap<>();
while (iterator.hasNext()) {
String currentTable = iterator.next();
while (currentTable != null && tableIndexMap.containsKey(currentTable)) {
@@ -120,23 +131,42 @@ public class TableDiskUsageStatisticUtil extends
DiskUsageStatisticUtil {
}
private long calculateTableSize(
- Map<String, Long> tableOffsetMap,
+ Map<String, Offsets> tableOffsetMap,
TsFileMetadata tsFileMetadata,
TsFileSequenceReader reader,
String tableName,
- String nextTable) {
- long startOffset, endOffset;
+ String nextTable)
+ throws IOException {
+ Offsets startOffset = getTableOffset(tableOffsetMap, reader, tableName);
+ Offsets endOffset;
if (nextTable == null) {
- endOffset = tsFileMetadata.getMetaOffset();
+ long firstMeasurementNodeOffsetOfFirstTable;
+ String firstTableName =
+
tsFileMetadata.getTableMetadataIndexNodeMap().keySet().iterator().next();
+ Offsets firstTableOffset = tableOffsetMap.get(firstTableName);
+ if (firstTableOffset != null) {
+ firstMeasurementNodeOffsetOfFirstTable =
firstTableOffset.firstMeasurementNodeOffset;
+ } else {
+ TsFileDeviceIterator deviceIterator =
+ reader.getTableDevicesIteratorWithIsAligned(
+ tableName, timeSeriesMetadataIoSizeRecorder);
+ deviceIterator.next();
+ firstMeasurementNodeOffsetOfFirstTable =
+ deviceIterator.getCurrentDeviceMeasurementNodeOffset()[0];
+ }
+ endOffset =
+ new Offsets(
+ tsFileMetadata.getMetaOffset(),
+ firstMeasurementNodeOffsetOfFirstTable,
+ reader.getFileMetadataPos());
} else {
endOffset = getTableOffset(tableOffsetMap, reader, nextTable);
}
- startOffset = getTableOffset(tableOffsetMap, reader, tableName);
- return endOffset - startOffset;
+ return endOffset.minusOffsetForTableModel(startOffset);
}
- private long getTableOffset(
- Map<String, Long> tableOffsetMap, TsFileSequenceReader reader, String
tableName) {
+ private Offsets getTableOffset(
+ Map<String, Offsets> tableOffsetMap, TsFileSequenceReader reader, String
tableName) {
return tableOffsetMap.computeIfAbsent(
tableName,
k -> {
@@ -144,8 +174,11 @@ public class TableDiskUsageStatisticUtil extends
DiskUsageStatisticUtil {
TsFileDeviceIterator deviceIterator =
reader.getTableDevicesIteratorWithIsAligned(k,
timeSeriesMetadataIoSizeRecorder);
Pair<IDeviceID, Boolean> pair = deviceIterator.next();
- return calculateStartOffsetOfChunkGroup(
- reader,
deviceIterator.getFirstMeasurementNodeOfCurrentDevice(), pair);
+ return calculateStartOffsetOfChunkGroupAndTimeseriesMetadata(
+ reader,
+ deviceIterator.getFirstMeasurementNodeOfCurrentDevice(),
+ pair,
+ deviceIterator.getCurrentDeviceMeasurementNodeOffset()[0]);
} catch (IOException e) {
throw new RuntimeException(e);
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TreeDiskUsageStatisticUtil.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TreeDiskUsageStatisticUtil.java
index ae1c47b03e0..5d6f2aaed5e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TreeDiskUsageStatisticUtil.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TreeDiskUsageStatisticUtil.java
@@ -29,7 +29,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.MetadataIndexNode;
-import org.apache.tsfile.read.TsFileDeviceIterator;
+import org.apache.tsfile.read.LazyTsFileDeviceIterator;
import org.apache.tsfile.read.TsFileSequenceReader;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.utils.RamUsageEstimator;
@@ -75,58 +75,84 @@ public class TreeDiskUsageStatisticUtil extends
DiskUsageStatisticUtil {
return new long[] {result};
}
+ @Override
+ protected boolean calculateWithoutOpenFile(TsFileResource tsFileResource) {
+ return false;
+ }
+
@Override
protected void calculateNextFile(TsFileResource tsFileResource,
TsFileSequenceReader reader)
throws IOException, IllegalPathException {
- TsFileDeviceIterator deviceIterator =
reader.getAllDevicesIteratorWithIsAligned();
+ long firstDeviceMeasurementNodeOffset = -1;
+ LazyTsFileDeviceIterator deviceIterator = reader.getLazyDeviceIterator();
while (deviceIterator.hasNext()) {
- Pair<IDeviceID, Boolean> deviceIsAlignedPair = deviceIterator.next();
- if (!matchPathPattern(deviceIsAlignedPair.getLeft())) {
+ IDeviceID deviceIsAlignedPair = deviceIterator.next();
+ firstDeviceMeasurementNodeOffset =
+ firstDeviceMeasurementNodeOffset == -1
+ ? deviceIterator.getCurrentDeviceMeasurementNodeOffset()[0]
+ : firstDeviceMeasurementNodeOffset;
+ if (!matchPathPattern(deviceIsAlignedPair)) {
continue;
}
MetadataIndexNode nodeOfFirstMatchedDevice =
deviceIterator.getFirstMeasurementNodeOfCurrentDevice();
- Pair<IDeviceID, Boolean> nextNotMatchedDevice = null;
+ addMeasurementNodeSizeForCurrentDevice(deviceIterator);
+ IDeviceID nextNotMatchedDevice = null;
MetadataIndexNode nodeOfNextNotMatchedDevice = null;
while (deviceIterator.hasNext()) {
- Pair<IDeviceID, Boolean> currentDevice = deviceIterator.next();
- if (!matchPathPattern(currentDevice.getLeft())) {
+ IDeviceID currentDevice = deviceIterator.next();
+ if (!matchPathPattern(currentDevice)) {
nextNotMatchedDevice = currentDevice;
nodeOfNextNotMatchedDevice =
deviceIterator.getFirstMeasurementNodeOfCurrentDevice();
break;
}
+ addMeasurementNodeSizeForCurrentDevice(deviceIterator);
}
result +=
calculatePathPatternSize(
reader,
- deviceIsAlignedPair,
+ new Pair<>(deviceIsAlignedPair,
reader.isAlignedDevice(nodeOfFirstMatchedDevice)),
nodeOfFirstMatchedDevice,
- nextNotMatchedDevice,
- nodeOfNextNotMatchedDevice);
+ nodeOfNextNotMatchedDevice == null
+ ? null
+ : new Pair<>(
+ nextNotMatchedDevice,
reader.isAlignedDevice(nodeOfNextNotMatchedDevice)),
+ nodeOfNextNotMatchedDevice,
+ firstDeviceMeasurementNodeOffset);
if (isMatchedDeviceSequential) {
break;
}
}
}
+ private void addMeasurementNodeSizeForCurrentDevice(LazyTsFileDeviceIterator
iterator) {
+ long[] startEndPair = iterator.getCurrentDeviceMeasurementNodeOffset();
+ result += startEndPair[1] - startEndPair[0];
+ }
+
private long calculatePathPatternSize(
TsFileSequenceReader reader,
Pair<IDeviceID, Boolean> firstMatchedDevice,
MetadataIndexNode nodeOfFirstMatchedDevice,
Pair<IDeviceID, Boolean> nextNotMatchedDevice,
- MetadataIndexNode nodeOfNextNotMatchedDevice)
+ MetadataIndexNode nodeOfNextNotMatchedDevice,
+ long firstDeviceMeasurementNodeOffset)
throws IOException {
- long startOffset, endOffset;
+ Offsets chunkGroupTimeseriesMetadataStartOffsetPair,
chunkGroupTimeseriesMetadataEndOffsetPair;
if (nextNotMatchedDevice == null) {
- endOffset = reader.readFileMetadata().getMetaOffset();
+ chunkGroupTimeseriesMetadataEndOffsetPair =
+ new Offsets(
+ reader.readFileMetadata().getMetaOffset(),
firstDeviceMeasurementNodeOffset, 0);
} else {
- endOffset =
- calculateStartOffsetOfChunkGroup(
- reader, nodeOfNextNotMatchedDevice, nextNotMatchedDevice);
+ chunkGroupTimeseriesMetadataEndOffsetPair =
+ calculateStartOffsetOfChunkGroupAndTimeseriesMetadata(
+ reader, nodeOfNextNotMatchedDevice, nextNotMatchedDevice, 0);
}
- startOffset =
- calculateStartOffsetOfChunkGroup(reader, nodeOfFirstMatchedDevice,
firstMatchedDevice);
- return endOffset - startOffset;
+ chunkGroupTimeseriesMetadataStartOffsetPair =
+ calculateStartOffsetOfChunkGroupAndTimeseriesMetadata(
+ reader, nodeOfFirstMatchedDevice, firstMatchedDevice, 0);
+ return chunkGroupTimeseriesMetadataEndOffsetPair.minusOffsetForTreeModel(
+ chunkGroupTimeseriesMetadataStartOffsetPair);
}
private boolean matchPathPattern(IDeviceID deviceID) throws
IllegalPathException {
