This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch object_type
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/object_type by this push:
new aad7d9c6599 Add object type file metric (#16257)
aad7d9c6599 is described below
commit aad7d9c6599cef35121e92b63d2345520a81c69a
Author: Haonan <[email protected]>
AuthorDate: Tue Aug 26 14:49:09 2025 +0800
Add object type file metric (#16257)
---
.../iotdb/db/service/metrics/FileMetrics.java | 20 +++++
.../db/service/metrics/file/ObjectFileMetrics.java | 86 ++++++++++++++++++++++
.../iotdb/db/storageengine/StorageEngine.java | 16 ++--
.../db/storageengine/dataregion/DataRegion.java | 22 ++++++
.../org/apache/iotdb/db/utils/ObjectTypeUtils.java | 5 ++
5 files changed, 144 insertions(+), 5 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
index 8eca2ba3935..67cc75be749 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.service.metrics;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.service.metrics.file.CompactionFileMetrics;
import org.apache.iotdb.db.service.metrics.file.ModsFileMetrics;
+import org.apache.iotdb.db.service.metrics.file.ObjectFileMetrics;
import org.apache.iotdb.db.service.metrics.file.SystemRelatedFileMetrics;
import org.apache.iotdb.db.service.metrics.file.TsFileMetrics;
import org.apache.iotdb.db.service.metrics.file.WalFileMetrics;
@@ -40,6 +41,7 @@ public class FileMetrics implements IMetricSet {
private static final WalFileMetrics WAL_FILE_METRICS = new WalFileMetrics();
private static final SystemRelatedFileMetrics SYSTEM_RELATED_FILE_METRICS =
new SystemRelatedFileMetrics();
+ private static final ObjectFileMetrics OBJECT_FILE_METRICS = new
ObjectFileMetrics();
@Override
public void bindTo(AbstractMetricService metricService) {
@@ -48,6 +50,7 @@ public class FileMetrics implements IMetricSet {
COMPACTION_FILE_METRICS.bindTo(metricService);
WAL_FILE_METRICS.bindTo(metricService);
SYSTEM_RELATED_FILE_METRICS.bindTo(metricService);
+ OBJECT_FILE_METRICS.bindTo(metricService);
}
@Override
@@ -57,6 +60,7 @@ public class FileMetrics implements IMetricSet {
COMPACTION_FILE_METRICS.unbindFrom(metricService);
WAL_FILE_METRICS.unbindFrom(metricService);
SYSTEM_RELATED_FILE_METRICS.unbindFrom(metricService);
+ OBJECT_FILE_METRICS.unbindFrom(metricService);
}
// region TsFile Related Metrics Update
@@ -104,6 +108,22 @@ public class FileMetrics implements IMetricSet {
// endregion
+ public void increaseObjectFileNum(int num) {
+ OBJECT_FILE_METRICS.increaseObjectFileNum(num);
+ }
+
+ public void decreaseObjectFileNum(int num) {
+ OBJECT_FILE_METRICS.decreaseObjectFileNum(num);
+ }
+
+ public void increaseObjectFileSize(long size) {
+ OBJECT_FILE_METRICS.increaseObjectFileSize(size);
+ }
+
+ public void decreaseObjectFileSize(long size) {
+ OBJECT_FILE_METRICS.decreaseObjectFileSize(size);
+ }
+
public Map<Integer, Long> getRegionSizeMap() {
return TS_FILE_METRICS.getRegionSizeMap();
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/ObjectFileMetrics.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/ObjectFileMetrics.java
new file mode 100644
index 00000000000..71e68c235df
--- /dev/null
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/ObjectFileMetrics.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.service.metrics.file;
+
+import org.apache.iotdb.commons.service.metric.enums.Metric;
+import org.apache.iotdb.commons.service.metric.enums.Tag;
+import org.apache.iotdb.metrics.AbstractMetricService;
+import org.apache.iotdb.metrics.metricsets.IMetricSet;
+import org.apache.iotdb.metrics.utils.MetricLevel;
+import org.apache.iotdb.metrics.utils.MetricType;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class ObjectFileMetrics implements IMetricSet {
+ private static final String OBJECT = "object";
+ private final AtomicInteger objFileNum = new AtomicInteger(0);
+ private final AtomicLong objFileSize = new AtomicLong(0);
+
+ @Override
+ public void bindTo(AbstractMetricService metricService) {
+ metricService.createAutoGauge(
+ Metric.FILE_SIZE.toString(),
+ MetricLevel.CORE,
+ this,
+ ObjectFileMetrics::getObjectFileSize,
+ Tag.NAME.toString(),
+ OBJECT);
+ metricService.createAutoGauge(
+ Metric.FILE_COUNT.toString(),
+ MetricLevel.CORE,
+ this,
+ ObjectFileMetrics::getObjectFileNum,
+ Tag.NAME.toString(),
+ OBJECT);
+ }
+
+ @Override
+ public void unbindFrom(AbstractMetricService metricService) {
+ metricService.remove(
+ MetricType.AUTO_GAUGE, Metric.FILE_SIZE.toString(),
Tag.NAME.toString(), OBJECT);
+ metricService.remove(
+ MetricType.AUTO_GAUGE, Metric.FILE_COUNT.toString(),
Tag.NAME.toString(), OBJECT);
+ }
+
+ public int getObjectFileNum() {
+ return objFileNum.get();
+ }
+
+ public long getObjectFileSize() {
+ return objFileSize.get();
+ }
+
+ public void increaseObjectFileNum(int num) {
+ objFileNum.addAndGet(num);
+ }
+
+ public void decreaseObjectFileNum(int num) {
+ objFileNum.addAndGet(-num);
+ }
+
+ public void increaseObjectFileSize(long size) {
+ objFileSize.addAndGet(size);
+ }
+
+ public void decreaseObjectFileSize(long size) {
+ objFileSize.addAndGet(-size);
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
index 20111516d7c..823983fced5 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
@@ -1087,14 +1087,20 @@ public class StorageEngine implements IService {
.filter(
path -> {
String name = path.getFileName().toString();
- return name.endsWith(".bin.back");
+ return name.endsWith(".bin.back") || name.endsWith(".bin");
})
.forEach(
path -> {
- try {
- Files.delete(path);
- } catch (IOException e) {
- LOGGER.error("Failed to delete: {} -> {}", path,
e.getMessage());
+ String name = path.getFileName().toString();
+ if (name.endsWith(".bin.back")) {
+ try {
+ Files.delete(path);
+ } catch (IOException e) {
+ LOGGER.error("Failed to delete: {} -> {}", path,
e.getMessage());
+ }
+ } else if (name.endsWith(".bin")) {
+ FileMetrics.getInstance().increaseObjectFileNum(1);
+
FileMetrics.getInstance().increaseObjectFileSize(path.toFile().length());
}
});
} catch (IOException e) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 166901363f6..4bb9bc2b7cb 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -175,6 +175,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
@@ -205,6 +206,7 @@ import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import static org.apache.iotdb.commons.conf.IoTDBConstant.FILE_NAME_SEPARATOR;
import static org.apache.iotdb.commons.utils.PathUtils.isTableModelDatabase;
@@ -1882,6 +1884,22 @@ public class DataRegion implements IDataRegionForQuery {
private void deleteAllObjectFiles(List<String> folders) {
for (String objectFolder : folders) {
File dataRegionObjectFolder = fsFactory.getFile(objectFolder,
dataRegionId);
+ try (Stream<Path> paths = Files.walk(dataRegionObjectFolder.toPath())) {
+ paths
+ .filter(Files::isRegularFile)
+ .filter(
+ path -> {
+ String name = path.getFileName().toString();
+ return name.endsWith(".bin");
+ })
+ .forEach(
+ path -> {
+ FileMetrics.getInstance().decreaseObjectFileNum(1);
+
FileMetrics.getInstance().decreaseObjectFileSize(path.toFile().length());
+ });
+ } catch (IOException e) {
+ logger.error("Failed to check Object Files: {}", e.getMessage());
+ }
if (FSUtils.getFSType(dataRegionObjectFolder) != FSType.LOCAL) {
try {
fsFactory.deleteDirectory(dataRegionObjectFolder.getPath());
@@ -3074,11 +3092,15 @@ public class DataRegion implements IDataRegionForQuery {
objectFile.toPath(), objectBackFile.toPath(),
StandardCopyOption.REPLACE_EXISTING);
Files.move(
objectTmpFile.toPath(), objectFile.toPath(),
StandardCopyOption.REPLACE_EXISTING);
+ FileMetrics.getInstance().decreaseObjectFileNum(1);
+
FileMetrics.getInstance().decreaseObjectFileSize(objectBackFile.length());
Files.delete(objectBackFile.toPath());
} else {
Files.move(
objectTmpFile.toPath(), objectFile.toPath(),
StandardCopyOption.REPLACE_EXISTING);
}
+ FileMetrics.getInstance().increaseObjectFileNum(1);
+ FileMetrics.getInstance().increaseObjectFileSize(objectFile.length());
}
getWALNode()
.ifPresent(walNode ->
walNode.log(TsFileProcessor.MEMTABLE_NOT_EXIST, objectNode));
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java
index ee2fcb8dc57..0531143fe8f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.utils;
import org.apache.iotdb.commons.exception.ObjectFileNotExist;
+import org.apache.iotdb.db.service.metrics.FileMetrics;
import org.apache.iotdb.db.storageengine.rescon.disk.TierManager;
import org.apache.tsfile.common.conf.TSFileConfig;
@@ -67,6 +68,10 @@ public class ObjectTypeUtils {
File bakFile = new File(file.get().getPath() + ".back");
logger.info("Remove object file {}", file.get().getAbsolutePath());
for (int i = 0; i < 2; i++) {
+ if (file.get().exists()) {
+ FileMetrics.getInstance().decreaseObjectFileNum(1);
+ FileMetrics.getInstance().decreaseObjectFileSize(file.get().length());
+ }
try {
Files.deleteIfExists(file.get().toPath());
Files.deleteIfExists(tmpFile.toPath());
