This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a commit to branch rel/1.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.2 by this push:
new facfd6f4226 Fix SYS_DISK_FREE_SPACE metric and add SYS_AVAILABLE_SPACE
metric (#11289) (#11298)
facfd6f4226 is described below
commit facfd6f4226abbbbd6aeb14a0bf0f5987b8c5ac8
Author: ZhangHongYin <[email protected]>
AuthorDate: Thu Oct 12 20:37:04 2023 -0500
Fix SYS_DISK_FREE_SPACE metric and add SYS_AVAILABLE_SPACE metric (#11289)
(#11298)
---
.../impl/DataNodeInternalRPCServiceImpl.java | 10 ++--
.../compaction/execute/utils/CompactionUtils.java | 57 ++++++++++++++++++++++
.../metrics/metricsets/system/SystemMetrics.java | 31 ++++++++++--
.../apache/iotdb/metrics/utils/SystemMetric.java | 1 +
4 files changed, 91 insertions(+), 8 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
index 3be407a8df0..56a9bea9961 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
@@ -1273,10 +1273,10 @@ public class DataNodeInternalRPCServiceImpl implements
IDataNodeRPCService.Iface
private void sampleDiskLoad(TLoadSample loadSample) {
final CommonConfig commonConfig =
CommonDescriptor.getInstance().getConfig();
- double freeDisk =
+ double availableDisk =
MetricService.getInstance()
.getAutoGauge(
- SystemMetric.SYS_DISK_FREE_SPACE.toString(),
+ SystemMetric.SYS_DISK_AVAILABLE_SPACE.toString(),
MetricLevel.CORE,
Tag.NAME.toString(),
SYSTEM)
@@ -1290,9 +1290,9 @@ public class DataNodeInternalRPCServiceImpl implements
IDataNodeRPCService.Iface
SYSTEM)
.value();
- if (freeDisk != 0 && totalDisk != 0) {
- double freeDiskRatio = freeDisk / totalDisk;
- loadSample.setFreeDiskSpace(freeDisk);
+ if (availableDisk != 0 && totalDisk != 0) {
+ double freeDiskRatio = availableDisk / totalDisk;
+ loadSample.setFreeDiskSpace(availableDisk);
loadSample.setDiskUsageRate(1d - freeDiskRatio);
// Reset NodeStatus if necessary
if (freeDiskRatio < commonConfig.getDiskSpaceWarningThreshold()) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
index a5160b7a3bf..a733d80941d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
@@ -19,13 +19,18 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.commons.service.metric.MetricService;
+import org.apache.iotdb.commons.service.metric.enums.Tag;
import org.apache.iotdb.db.service.metrics.FileMetrics;
import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
import
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex;
+import org.apache.iotdb.metrics.utils.MetricLevel;
+import org.apache.iotdb.metrics.utils.SystemMetric;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
@@ -68,6 +73,7 @@ import java.util.Set;
public class CompactionUtils {
private static final Logger logger =
LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME);
+ private static final String SYSTEM = "system";
private CompactionUtils() {}
@@ -436,4 +442,55 @@ public class CompactionUtils {
}
return true;
}
+
+ public static void deleteSourceTsFileAndUpdateFileMetrics(
+ List<TsFileResource> sourceSeqResourceList, List<TsFileResource>
sourceUnseqResourceList) {
+ deleteSourceTsFileAndUpdateFileMetrics(sourceSeqResourceList, true);
+ deleteSourceTsFileAndUpdateFileMetrics(sourceUnseqResourceList, false);
+ }
+
+ public static void deleteSourceTsFileAndUpdateFileMetrics(
+ List<TsFileResource> resources, boolean seq) {
+ List<TsFileResource> removeResources = new ArrayList<>();
+ for (TsFileResource resource : resources) {
+ if (!resource.remove()) {
+ logger.warn(
+ "[Compaction] delete file failed, file path is {}",
+ resource.getTsFile().getAbsolutePath());
+ } else {
+ logger.info("[Compaction] delete file: {}",
resource.getTsFile().getAbsolutePath());
+ removeResources.add(resource);
+ }
+ }
+ FileMetrics.getInstance().deleteTsFile(seq, resources);
+ }
+
+ public static boolean isDiskHasSpace() {
+ return isDiskHasSpace(0d);
+ }
+
+ public static boolean isDiskHasSpace(double redundancy) {
+ double availableDisk =
+ MetricService.getInstance()
+ .getAutoGauge(
+ SystemMetric.SYS_DISK_AVAILABLE_SPACE.toString(),
+ MetricLevel.CORE,
+ Tag.NAME.toString(),
+ SYSTEM)
+ .value();
+ double totalDisk =
+ MetricService.getInstance()
+ .getAutoGauge(
+ SystemMetric.SYS_DISK_TOTAL_SPACE.toString(),
+ MetricLevel.CORE,
+ Tag.NAME.toString(),
+ SYSTEM)
+ .value();
+
+ if (availableDisk != 0 && totalDisk != 0) {
+ return availableDisk / totalDisk
+ >
CommonDescriptor.getInstance().getConfig().getDiskSpaceWarningThreshold() +
redundancy;
+ }
+ return true;
+ }
}
diff --git
a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/system/SystemMetrics.java
b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/system/SystemMetrics.java
index d61b7b71120..dd9054dd9a0 100644
---
a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/system/SystemMetrics.java
+++
b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/system/SystemMetrics.java
@@ -50,6 +50,7 @@ public class SystemMetrics implements IMetricSet {
private final Set<FileStore> fileStores = new HashSet<>();
private final AtomicReference<List<String>> diskDirs =
new AtomicReference<>(Collections.emptyList());
+ private static final String FAILED_TO_STATISTIC = "Failed to statistic the
size of {}, because";
public SystemMetrics() {
this.osMxBean = (OperatingSystemMXBean)
ManagementFactory.getOperatingSystemMXBean();
@@ -204,6 +205,13 @@ public class SystemMetrics implements IMetricSet {
SystemMetrics::getSystemDiskFreeSpace,
SystemTag.NAME.toString(),
SYSTEM);
+ metricService.createAutoGauge(
+ SystemMetric.SYS_DISK_AVAILABLE_SPACE.toString(),
+ MetricLevel.CORE,
+ this,
+ SystemMetrics::getSystemDiskAvailableSpace,
+ SystemTag.NAME.toString(),
+ SYSTEM);
}
private void removeSystemDiskInfo(AbstractMetricService metricService) {
@@ -217,6 +225,11 @@ public class SystemMetrics implements IMetricSet {
SystemMetric.SYS_DISK_FREE_SPACE.toString(),
SystemTag.NAME.toString(),
SYSTEM);
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ SystemMetric.SYS_DISK_AVAILABLE_SPACE.toString(),
+ SystemTag.NAME.toString(),
+ SYSTEM);
diskDirs.get().clear();
fileStores.clear();
@@ -228,7 +241,7 @@ public class SystemMetrics implements IMetricSet {
try {
sysTotalSpace += fileStore.getTotalSpace();
} catch (IOException e) {
- logger.error("Failed to statistic the size of {}, because", fileStore,
e);
+ logger.error(FAILED_TO_STATISTIC, fileStore, e);
}
}
return sysTotalSpace;
@@ -238,14 +251,26 @@ public class SystemMetrics implements IMetricSet {
long sysFreeSpace = 0L;
for (FileStore fileStore : fileStores) {
try {
- sysFreeSpace += fileStore.getUsableSpace();
+ sysFreeSpace += fileStore.getUnallocatedSpace();
} catch (IOException e) {
- logger.error("Failed to statistic the size of {}, because", fileStore,
e);
+ logger.error(FAILED_TO_STATISTIC, fileStore, e);
}
}
return sysFreeSpace;
}
+ public long getSystemDiskAvailableSpace() {
+ long sysAvailableSpace = 0L;
+ for (FileStore fileStore : fileStores) {
+ try {
+ sysAvailableSpace += fileStore.getUsableSpace();
+ } catch (IOException e) {
+ logger.error(FAILED_TO_STATISTIC, fileStore, e);
+ }
+ }
+ return sysAvailableSpace;
+ }
+
public static SystemMetrics getInstance() {
return SystemMetricsHolder.INSTANCE;
}
diff --git
a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/utils/SystemMetric.java
b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/utils/SystemMetric.java
index e04586c0f11..2247628169c 100644
---
a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/utils/SystemMetric.java
+++
b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/utils/SystemMetric.java
@@ -58,6 +58,7 @@ public enum SystemMetric {
SYS_COMMITTED_VM_SIZE("sys_committed_vm_size"),
SYS_DISK_TOTAL_SPACE("sys_disk_total_space"),
SYS_DISK_FREE_SPACE("sys_disk_free_space"),
+ SYS_DISK_AVAILABLE_SPACE("sys_disk_available_space"),
// cpu related
MODULE_CPU_USAGE("module_cpu_usage"),
POOL_CPU_USAGE("pool_cpu_usage"),
