This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 214c49a2108 new iot_consensus_throttle_threshold_in_byte (#12333)
214c49a2108 is described below
commit 214c49a21080d1e74496a43d7fa15f898e131f70
Author: YuFengLiu <[email protected]>
AuthorDate: Sun Apr 14 14:45:57 2024 +0800
new iot_consensus_throttle_threshold_in_byte (#12333)
---
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 44 +++++++++++++++++--
.../metrics/metricsets/system/SystemMetrics.java | 49 +++++++++++-----------
.../resources/conf/iotdb-common.properties | 3 +-
3 files changed, 67 insertions(+), 29 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 048afadc585..14438eb845e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -49,6 +49,7 @@ import
org.apache.iotdb.db.utils.datastructure.TVListSortAlgorithm;
import org.apache.iotdb.external.api.IPropertiesLoader;
import org.apache.iotdb.metrics.config.MetricConfigDescriptor;
import org.apache.iotdb.metrics.config.ReloadLevel;
+import org.apache.iotdb.metrics.metricsets.system.SystemMetrics;
import org.apache.iotdb.metrics.reporter.iotdb.IoTDBInternalMemoryReporter;
import org.apache.iotdb.metrics.reporter.iotdb.IoTDBInternalReporter;
import org.apache.iotdb.metrics.utils.InternalReporterType;
@@ -72,11 +73,13 @@ import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
+import java.nio.file.FileStore;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import java.util.ServiceLoader;
+import java.util.Set;
public class IoTDBDescriptor {
@@ -86,6 +89,14 @@ public class IoTDBDescriptor {
private final IoTDBConfig conf = new IoTDBConfig();
+ private static final long MAX_THROTTLE_THRESHOLD = 600 * 1024 * 1024 * 1024L;
+
+ private static final long MIN_THROTTLE_THRESHOLD = 50 * 1024 * 1024 * 1024L;
+
+ private static final double MAX_DIR_USE_PROPORTION = 0.8;
+
+ private static final double MIN_DIR_USE_PROPORTION = 0.5;
+
protected IoTDBDescriptor() {
loadProps();
ServiceLoader<IPropertiesLoader> propertiesLoaderServiceLoader =
@@ -327,8 +338,6 @@ public class IoTDBDescriptor {
initMemoryAllocate(properties);
- loadWALProps(properties);
-
String systemDir = properties.getProperty("dn_system_dir");
if (systemDir == null) {
systemDir = properties.getProperty("base_dir");
@@ -971,6 +980,8 @@ public class IoTDBDescriptor {
commonDescriptor.loadCommonProps(properties);
commonDescriptor.initCommonConfigDir(conf.getSystemDir());
+ loadWALProps(properties);
+
// Timed flush memtable
loadTimedService(properties);
@@ -1271,7 +1282,7 @@ public class IoTDBDescriptor {
Long.parseLong(
properties.getProperty(
"iot_consensus_throttle_threshold_in_byte",
- Long.toString(conf.getThrottleThreshold())));
+ Long.toString(getThrottleThresholdWithDirs())));
if (throttleDownThresholdInByte > 0) {
conf.setThrottleThreshold(throttleDownThresholdInByte);
}
@@ -1286,6 +1297,33 @@ public class IoTDBDescriptor {
}
}
+ public long getThrottleThresholdWithDirs() {
+ ArrayList<String> dataDiskDirs = new
ArrayList<>(Arrays.asList(conf.getDataDirs()));
+ ArrayList<String> walDiskDirs =
+ new
ArrayList<>(Arrays.asList(commonDescriptor.getConfig().getWalDirs()));
+ Set<FileStore> dataFileStores = SystemMetrics.getFileStores(dataDiskDirs);
+ Set<FileStore> walFileStores = SystemMetrics.getFileStores(walDiskDirs);
+ double dirUseProportion = 0;
+ dataFileStores.retainAll(walFileStores);
+ // if there is no common disk between data and wal, use more usableSpace.
+ if (dataFileStores.isEmpty()) {
+ dirUseProportion = MAX_DIR_USE_PROPORTION;
+ } else {
+ dirUseProportion = MIN_DIR_USE_PROPORTION;
+ }
+ long newThrottleThreshold = Long.MAX_VALUE;
+ for (FileStore fileStore : walFileStores) {
+ try {
+ newThrottleThreshold = Math.min(newThrottleThreshold,
fileStore.getUsableSpace());
+ } catch (IOException e) {
+ LOGGER.error("Failed to get file size of {}, because", fileStore, e);
+ }
+ }
+ newThrottleThreshold = (long) (newThrottleThreshold * dirUseProportion *
walFileStores.size());
+ // the new throttle threshold should between MIN_THROTTLE_THRESHOLD and
MAX_THROTTLE_THRESHOLD
+ return Math.max(Math.min(newThrottleThreshold, MAX_THROTTLE_THRESHOLD),
MIN_THROTTLE_THRESHOLD);
+ }
+
private void loadAutoCreateSchemaProps(Properties properties) {
conf.setAutoCreateSchemaEnabled(
Boolean.parseBoolean(
diff --git
a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/system/SystemMetrics.java
b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/system/SystemMetrics.java
index 2450057d219..aed6ee78074 100644
---
a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/system/SystemMetrics.java
+++
b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/system/SystemMetrics.java
@@ -38,19 +38,15 @@ import java.nio.file.FileStore;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import java.util.concurrent.atomic.AtomicReference;
public class SystemMetrics implements IMetricSet {
private static final Logger logger =
LoggerFactory.getLogger(SystemMetrics.class);
private static final String SYSTEM = "system";
private final com.sun.management.OperatingSystemMXBean osMxBean;
- private final Set<FileStore> fileStores = new HashSet<>();
- private final AtomicReference<List<String>> diskDirs =
- new AtomicReference<>(Collections.emptyList());
+ private Set<FileStore> fileStores = new HashSet<>();
private static final String FAILED_TO_STATISTIC = "Failed to statistic the
size of {}, because";
public SystemMetrics() {
@@ -62,29 +58,34 @@ public class SystemMetrics implements IMetricSet {
.getMetricConfig()
.getMetricLevel()
.equals(MetricLevel.OFF)) {
- this.diskDirs.set(diskDirs);
- for (String diskDir : this.diskDirs.get()) {
- if (!FSUtils.isLocal(diskDir)) {
- continue;
- }
- Path path = Paths.get(diskDir);
- FileStore fileStore = null;
+ this.fileStores = getFileStores(diskDirs);
+ }
+ }
+
+ public static Set<FileStore> getFileStores(List<String> dirs) {
+ Set<FileStore> fileStoreSet = new HashSet<>();
+ for (String diskDir : dirs) {
+ if (!FSUtils.isLocal(diskDir)) {
+ continue;
+ }
+ Path path = Paths.get(diskDir);
+ FileStore fileStore = null;
+ try {
+ fileStore = Files.getFileStore(path);
+ } catch (IOException e) {
+ // check parent if path is not exists
+ path = path.getParent();
try {
fileStore = Files.getFileStore(path);
- } catch (IOException e) {
- // check parent if path is not exists
- path = path.getParent();
- try {
- fileStore = Files.getFileStore(path);
- } catch (IOException innerException) {
- logger.error("Failed to get storage path of {}, because", diskDir,
innerException);
- }
- }
- if (null != fileStore) {
- fileStores.add(fileStore);
+ } catch (IOException innerException) {
+ logger.error("Failed to get storage path of {}, because", diskDir,
innerException);
}
}
+ if (null != fileStore) {
+ fileStoreSet.add(fileStore);
+ }
}
+ return fileStoreSet;
}
@Override
@@ -236,8 +237,6 @@ public class SystemMetrics implements IMetricSet {
SystemMetric.SYS_DISK_AVAILABLE_SPACE.toString(),
SystemTag.NAME.toString(),
SYSTEM);
-
- diskDirs.get().clear();
fileStores.clear();
}
diff --git
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties
index cd1920cc643..fc2da76d388 100644
---
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -741,7 +741,8 @@ data_replication_factor=1
# delete_wal_files_period_in_ms=20000
# The minimum size of wal files when throttle down in IoTConsensus
-# If it's a value smaller than 0, use the default value 50 * 1024 * 1024 *
1024 bytes (50GB).
+# If this value is not set, it will be carefully chosen according to the
available disk space.
+# If this value is set smaller than 0, it will default to 50 * 1024 * 1024 *
1024 bytes (50GB).
# Datatype: long
# iot_consensus_throttle_threshold_in_byte=53687091200
