This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch new_object_type in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1254bdfcce681b75e4dd70e9461bf0e8b0dedcd5 Author: Haonan <[email protected]> AuthorDate: Tue Aug 26 14:49:09 2025 +0800 Add object type file metric (#16257) --- .../iotdb/db/service/metrics/FileMetrics.java | 20 +++++ .../db/service/metrics/file/ObjectFileMetrics.java | 86 ++++++++++++++++++++++ .../iotdb/db/storageengine/StorageEngine.java | 16 ++-- .../db/storageengine/dataregion/DataRegion.java | 21 ++++++ .../org/apache/iotdb/db/utils/ObjectTypeUtils.java | 5 ++ 5 files changed, 143 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java index 8eca2ba3935..67cc75be749 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java @@ -22,6 +22,7 @@ package org.apache.iotdb.db.service.metrics; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.service.metrics.file.CompactionFileMetrics; import org.apache.iotdb.db.service.metrics.file.ModsFileMetrics; +import org.apache.iotdb.db.service.metrics.file.ObjectFileMetrics; import org.apache.iotdb.db.service.metrics.file.SystemRelatedFileMetrics; import org.apache.iotdb.db.service.metrics.file.TsFileMetrics; import org.apache.iotdb.db.service.metrics.file.WalFileMetrics; @@ -40,6 +41,7 @@ public class FileMetrics implements IMetricSet { private static final WalFileMetrics WAL_FILE_METRICS = new WalFileMetrics(); private static final SystemRelatedFileMetrics SYSTEM_RELATED_FILE_METRICS = new SystemRelatedFileMetrics(); + private static final ObjectFileMetrics OBJECT_FILE_METRICS = new ObjectFileMetrics(); @Override public void bindTo(AbstractMetricService metricService) { @@ -48,6 +50,7 @@ public class FileMetrics implements IMetricSet { COMPACTION_FILE_METRICS.bindTo(metricService); WAL_FILE_METRICS.bindTo(metricService); SYSTEM_RELATED_FILE_METRICS.bindTo(metricService); + OBJECT_FILE_METRICS.bindTo(metricService); } @Override @@ -57,6 +60,7 @@ public class FileMetrics implements IMetricSet { COMPACTION_FILE_METRICS.unbindFrom(metricService); WAL_FILE_METRICS.unbindFrom(metricService); SYSTEM_RELATED_FILE_METRICS.unbindFrom(metricService); + OBJECT_FILE_METRICS.unbindFrom(metricService); } // region TsFile Related Metrics Update @@ -104,6 +108,22 @@ public class FileMetrics implements IMetricSet { // endregion + public void increaseObjectFileNum(int num) { + OBJECT_FILE_METRICS.increaseObjectFileNum(num); + } + + public void decreaseObjectFileNum(int num) { + OBJECT_FILE_METRICS.decreaseObjectFileNum(num); + } + + public void increaseObjectFileSize(long size) { + OBJECT_FILE_METRICS.increaseObjectFileSize(size); + } + + public void decreaseObjectFileSize(long size) { + OBJECT_FILE_METRICS.decreaseObjectFileSize(size); + } + public Map<Integer, Long> getRegionSizeMap() { return TS_FILE_METRICS.getRegionSizeMap(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/ObjectFileMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/ObjectFileMetrics.java new file mode 100644 index 00000000000..71e68c235df --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/ObjectFileMetrics.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.metrics.file; + +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +public class ObjectFileMetrics implements IMetricSet { + private static final String OBJECT = "object"; + private final AtomicInteger objFileNum = new AtomicInteger(0); + private final AtomicLong objFileSize = new AtomicLong(0); + + @Override + public void bindTo(AbstractMetricService metricService) { + metricService.createAutoGauge( + Metric.FILE_SIZE.toString(), + MetricLevel.CORE, + this, + ObjectFileMetrics::getObjectFileSize, + Tag.NAME.toString(), + OBJECT); + metricService.createAutoGauge( + Metric.FILE_COUNT.toString(), + MetricLevel.CORE, + this, + ObjectFileMetrics::getObjectFileNum, + Tag.NAME.toString(), + OBJECT); + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + metricService.remove( + MetricType.AUTO_GAUGE, Metric.FILE_SIZE.toString(), Tag.NAME.toString(), OBJECT); + metricService.remove( + MetricType.AUTO_GAUGE, Metric.FILE_COUNT.toString(), Tag.NAME.toString(), OBJECT); + } + + public int getObjectFileNum() { + return objFileNum.get(); + } + + public long getObjectFileSize() { + return objFileSize.get(); + } + + public void increaseObjectFileNum(int num) { + objFileNum.addAndGet(num); + } + + public void decreaseObjectFileNum(int num) { + objFileNum.addAndGet(-num); + } + + public void increaseObjectFileSize(long size) { + objFileSize.addAndGet(size); + } + + public void decreaseObjectFileSize(long size) { + objFileSize.addAndGet(-size); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index bb1aecf05e3..89e1f0c7c98 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -1085,14 +1085,20 @@ public class StorageEngine implements IService { .filter( path -> { String name = path.getFileName().toString(); - return name.endsWith(".bin.back"); + return name.endsWith(".bin.back") || name.endsWith(".bin"); }) .forEach( path -> { - try { - Files.delete(path); - } catch (IOException e) { - LOGGER.error("Failed to delete: {} -> {}", path, e.getMessage()); + String name = path.getFileName().toString(); + if (name.endsWith(".bin.back")) { + try { + Files.delete(path); + } catch (IOException e) { + LOGGER.error("Failed to delete: {} -> {}", path, e.getMessage()); + } + } else if (name.endsWith(".bin")) { + FileMetrics.getInstance().increaseObjectFileNum(1); + FileMetrics.getInstance().increaseObjectFileSize(path.toFile().length()); } }); } catch (IOException e) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 6b3a05df7e5..eec169aa807 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -182,6 +182,7 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.ArrayList; @@ -1910,6 +1911,22 @@ public class DataRegion implements IDataRegionForQuery { private void deleteAllObjectFiles(List<String> folders) { for (String objectFolder : folders) { File dataRegionObjectFolder = fsFactory.getFile(objectFolder, dataRegionId); + try (Stream<Path> paths = Files.walk(dataRegionObjectFolder.toPath())) { + paths + .filter(Files::isRegularFile) + .filter( + path -> { + String name = path.getFileName().toString(); + return name.endsWith(".bin"); + }) + .forEach( + path -> { + FileMetrics.getInstance().decreaseObjectFileNum(1); + FileMetrics.getInstance().decreaseObjectFileSize(path.toFile().length()); + }); + } catch (IOException e) { + logger.error("Failed to check Object Files: {}", e.getMessage()); + } if (FSUtils.getFSType(dataRegionObjectFolder) != FSType.LOCAL) { try { fsFactory.deleteDirectory(dataRegionObjectFolder.getPath()); @@ -3381,11 +3398,15 @@ public class DataRegion implements IDataRegionForQuery { objectFile.toPath(), objectBackFile.toPath(), StandardCopyOption.REPLACE_EXISTING); Files.move( objectTmpFile.toPath(), objectFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + FileMetrics.getInstance().decreaseObjectFileNum(1); + FileMetrics.getInstance().decreaseObjectFileSize(objectBackFile.length()); Files.delete(objectBackFile.toPath()); } else { Files.move( objectTmpFile.toPath(), objectFile.toPath(), StandardCopyOption.REPLACE_EXISTING); } + FileMetrics.getInstance().increaseObjectFileNum(1); + FileMetrics.getInstance().increaseObjectFileSize(objectFile.length()); } getWALNode() .ifPresent(walNode -> walNode.log(TsFileProcessor.MEMTABLE_NOT_EXIST, objectNode)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java index ee2fcb8dc57..0531143fe8f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.utils; import org.apache.iotdb.commons.exception.ObjectFileNotExist; +import org.apache.iotdb.db.service.metrics.FileMetrics; import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; import org.apache.tsfile.common.conf.TSFileConfig; @@ -67,6 +68,10 @@ public class ObjectTypeUtils { File bakFile = new File(file.get().getPath() + ".back"); logger.info("Remove object file {}", file.get().getAbsolutePath()); for (int i = 0; i < 2; i++) { + if (file.get().exists()) { + FileMetrics.getInstance().decreaseObjectFileNum(1); + FileMetrics.getInstance().decreaseObjectFileSize(file.get().length()); + } try { Files.deleteIfExists(file.get().toPath()); Files.deleteIfExists(tmpFile.toPath());
