This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch addMemoryControlForModEntriesInQuery in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 93c9afc531e4feb821ebee0c2c5c5055d7632555 Author: shuwenwei <[email protected]> AuthorDate: Tue Aug 12 16:33:22 2025 +0800 optimize performance --- .../queryengine/execution/MemoryEstimationHelper.java | 12 ++++++++++++ .../storageengine/dataregion/modification/ModEntry.java | 6 +++++- .../dataregion/modification/ModificationFile.java | 4 ---- .../dataregion/modification/TreeDeletionEntry.java | 17 ++++++++++++++--- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/MemoryEstimationHelper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/MemoryEstimationHelper.java index 7c548877a78..fd33bb767e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/MemoryEstimationHelper.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/MemoryEstimationHelper.java @@ -102,6 +102,18 @@ public class MemoryEstimationHelper { return totalSize; } + public static long getEstimatedSizeOfPartialPathNodes(@Nullable final PartialPath partialPath) { + if (partialPath == null) { + return 0; + } + long totalSize = 0; + String[] nodes = partialPath.getNodes(); + if (nodes != null && nodes.length > 0) { + totalSize += Arrays.stream(nodes).mapToLong(RamUsageEstimator::sizeOf).sum(); + } + return totalSize; + } + // This method should only be called if the content in the current PartialPath comes from other // structures whose memory cost have already been calculated. public static long getEstimatedSizeOfCopiedPartialPath(@Nullable final PartialPath partialPath) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java index 349f1b43ec2..462a0a21c39 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java @@ -28,6 +28,7 @@ import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.utils.Accountable; import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -178,7 +179,10 @@ public abstract class ModEntry } public static ModType deserialize(InputStream stream) throws IOException { - byte typeNum = ReadWriteIOUtils.readByte(stream); + int typeNum = stream.read(); + if (typeNum == -1) { + throw new EOFException(); + } switch (typeNum) { case 0x00: return TABLE_DELETION; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java index e72b06b9a1a..bc0e86a18c9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java @@ -304,10 +304,6 @@ public class ModificationFile implements AutoCloseable { } if (nextEntry == null) { try { - if (inputStream.available() == 0) { - close(); - return false; - } nextEntry = ModEntry.createFrom(inputStream); } catch (EOFException e) { close(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java index 3a755f0744f..c9fc8978f51 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java @@ -23,12 +23,14 @@ import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternUtil; +import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion; import org.apache.iotdb.db.utils.ModificationUtils; +import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.utils.RamUsageEstimator; @@ -108,7 +110,7 @@ public class TreeDeletionEntry extends ModEntry { public void deserialize(InputStream stream) throws IOException { super.deserialize(stream); try { - this.pathPattern = new MeasurementPath(ReadWriteIOUtils.readVarIntString(stream)); + this.pathPattern = getMeasurementPath(ReadWriteIOUtils.readVarIntString(stream)); } catch (IllegalPathException e) { throw new IOException(e); } @@ -118,12 +120,21 @@ public class TreeDeletionEntry extends ModEntry { public void deserialize(ByteBuffer buffer) { super.deserialize(buffer); try { - this.pathPattern = new MeasurementPath(ReadWriteIOUtils.readVarIntString(buffer)); + this.pathPattern = getMeasurementPath(ReadWriteIOUtils.readVarIntString(buffer)); } catch (IllegalPathException e) { throw new IllegalArgumentException(e); } } + private MeasurementPath getMeasurementPath(String path) throws IllegalPathException { + if (path.contains(TsFileConstant.BACK_QUOTE_STRING)) { + return new MeasurementPath(PathUtils.splitPathToDetachedNodes(path)); + } else { + String[] nodes = path.split(TsFileConstant.PATH_SEPARATER_NO_REGEX); + return new MeasurementPath(nodes); + } + } + @Override public boolean matches(PartialPath path) { return pathPattern.matchFullPath(path); @@ -235,6 +246,6 @@ public class TreeDeletionEntry extends ModEntry { public long ramBytesUsed() { return SHALLOW_SIZE + MemoryEstimationHelper.TIME_RANGE_INSTANCE_SIZE - + MemoryEstimationHelper.getEstimatedSizeOfPartialPath(pathPattern); + + MemoryEstimationHelper.getEstimatedSizeOfPartialPathNodes(pathPattern); } }
