This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch object_delet in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e2383e13ad0edac8e8a0f110e67a7e78a02720af Author: HTHou <[email protected]> AuthorDate: Tue Dec 16 09:53:55 2025 +0800 dev --- .../db/storageengine/dataregion/DataRegion.java | 65 ++++++++++++++++++++-- .../db/storageengine/rescon/disk/TierManager.java | 10 ++-- .../org/apache/iotdb/db/utils/ObjectTypeUtils.java | 21 +++++++ 3 files changed, 87 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 639b7e4c214..27a290b60fd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -158,6 +158,7 @@ import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.db.utils.DateTimeUtils; import org.apache.iotdb.db.utils.EncryptDBUtils; import org.apache.iotdb.db.utils.ModificationUtils; +import org.apache.iotdb.db.utils.ObjectTypeUtils; import org.apache.iotdb.db.utils.ObjectWriter; import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.rpc.RpcUtils; @@ -165,11 +166,13 @@ import org.apache.iotdb.rpc.TSStatusCode; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; +import com.google.common.io.BaseEncoding; import org.apache.thrift.TException; import org.apache.tsfile.external.commons.io.FileUtils; import org.apache.tsfile.external.commons.lang3.tuple.Triple; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.fileSystem.FSFactoryProducer; import org.apache.tsfile.fileSystem.FSType; @@ -186,6 +189,7 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -2762,8 +2766,10 @@ public class DataRegion implements IDataRegionForQuery { } } - if (TierManager.getInstance().checkObjectPathExist(dataRegionIdString, tableName)) { - deleteObjectFiles(tableName, modEntries); + List<File> matchedObjectDirs = + TierManager.getInstance().getAllMatchedObjectDirs(dataRegionIdString, tableName); + if (!matchedObjectDirs.isEmpty()) { + deleteObjectFiles(matchedObjectDirs, modEntries); } List<List<TsFileResource>> sealedTsFileResourceLists = new ArrayList<>(modEntries.size()); @@ -2934,9 +2940,58 @@ public class DataRegion implements IDataRegionForQuery { return walFlushListeners; } - private void deleteObjectFiles(String tableName, List<TableDeletionEntry> modEntries) { - for (TableDeletionEntry modEntry : modEntries) { - + private void deleteObjectFiles(List<File> matchedObjectDirs, List<TableDeletionEntry> modEntries) + throws IOException { + for (File matchedObjectDir : matchedObjectDirs) { + try (Stream<Path> paths = Files.walk(matchedObjectDir.toPath())) { + paths + .filter(Files::isRegularFile) + .filter( + path -> { + String name = path.getFileName().toString(); + return name.endsWith(".bin"); + }) + .forEach( + path -> { + Path relativePath = matchedObjectDir.getParentFile().toPath().relativize(path); + String[] ideviceIdSegments = new String[relativePath.getNameCount() - 2]; + for (int i = 0; i < ideviceIdSegments.length; i++) { + ideviceIdSegments[i] = + config.getRestrictObjectLimit() + ? relativePath.getName(i).toString() + : new String( + BaseEncoding.base32() + .omitPadding() + .decode(relativePath.getName(i).toString()), + StandardCharsets.UTF_8); + } + IDeviceID iDeviceID = Factory.DEFAULT_FACTORY.create(ideviceIdSegments); + String measurementId = + config.getRestrictObjectLimit() + ? relativePath.getName(relativePath.getNameCount() - 2).toString() + : new String( + BaseEncoding.base32() + .omitPadding() + .decode( + relativePath + .getName(relativePath.getNameCount() - 2) + .toString()), + StandardCharsets.UTF_8); + String fileName = path.getFileName().toString(); + long timestamp = Long.parseLong(fileName.substring(0, fileName.lastIndexOf('.'))); + logger.info( + "timestamp {}, measurementId {}, ideviceId {}", + timestamp, + measurementId, + iDeviceID); + for (TableDeletionEntry modEntry : modEntries) { + if (modEntry.affects(iDeviceID, timestamp, timestamp) + && modEntry.affects(measurementId)) { + ObjectTypeUtils.deleteObjectPath(path.toFile()); + } + } + }); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java index e173bb0abaf..a673e44dff9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java @@ -274,7 +274,8 @@ public class TierManager { return Optional.empty(); } - public boolean checkObjectPathExist(String regionIdStr, String... path) { + public List<File> getAllMatchedObjectDirs(String regionIdStr, String... path) { + List<File> matchedDirs = new ArrayList<>(); StringBuilder objectPath = new StringBuilder(); objectPath.append(regionIdStr); for (String str : path) { @@ -288,12 +289,13 @@ public class TierManager { .encode(str.getBytes(StandardCharsets.UTF_8))); } for (String objectDir : objectDirs) { - File objectFilePath = FSFactoryProducer.getFSFactory().getFile(objectDir, objectPath.toString()); + File objectFilePath = + FSFactoryProducer.getFSFactory().getFile(objectDir, objectPath.toString()); if (objectFilePath.exists()) { - return true; + matchedDirs.add(objectFilePath); } } - return false; + return matchedDirs; } public int getTiersNum() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java index ec1fd592617..2a03facbf96 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java @@ -294,6 +294,27 @@ public class ObjectTypeUtils { } } + public static void deleteObjectPath(File file) { + File tmpFile = new File(file.getPath() + ".tmp"); + File bakFile = new File(file.getPath() + ".back"); + for (int i = 0; i < 2; i++) { + if (file.exists()) { + FileMetrics.getInstance().decreaseObjectFileNum(1); + FileMetrics.getInstance().decreaseObjectFileSize(file.length()); + } + try { + deleteObjectFile(file); + deleteObjectFile(tmpFile); + deleteObjectFile(bakFile); + } catch (IOException e) { + logger.error("Failed to remove object file {}", file.getAbsolutePath(), e); + } + } + if (file.getParentFile().exists()) { + + } + } + private static void deleteObjectFile(File file) throws IOException { if (file.exists()) { logger.info("Remove object file {}, size is {}(byte)", file.getAbsolutePath(), file.length());
