OneSizeFitsQuorum commented on code in PR #12333:
URL: https://github.com/apache/iotdb/pull/12333#discussion_r1564430529
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java:
##########
@@ -1286,6 +1293,59 @@ private void loadWALHotModifiedProps(Properties
properties) {
}
}
+ public long getThrottleThresholdWithDirs() {
+ ArrayList<String> dataDiskDirs = new
ArrayList<>(Arrays.asList(conf.getDataDirs()));
+ ArrayList<String> walDiskDirs =
+ new
ArrayList<>(Arrays.asList(commonDescriptor.getConfig().getWalDirs()));
+ Set<FileStore> dataFileStores = getFileStores(dataDiskDirs);
+ Set<FileStore> walFileStores = getFileStores(walDiskDirs);
+ double dirUseProportion = 0;
+ Set<FileStore> intersectFileStores = new HashSet<>(dataFileStores);
+ intersectFileStores.retainAll(walFileStores);
+ if (intersectFileStores.isEmpty()) {
+ dirUseProportion = 0.8;
+ } else {
+ dirUseProportion = 0.5;
+ }
+ long newThrottleThreshold = Long.MAX_VALUE;
+ for (FileStore fileStore : walFileStores) {
+ try {
+ newThrottleThreshold = Math.min(newThrottleThreshold,
fileStore.getUsableSpace());
+ } catch (IOException e) {
+ LOGGER.error("Failed to get file size of {}, because", fileStore, e);
+ }
+ }
+ newThrottleThreshold = (long) (newThrottleThreshold * dirUseProportion *
walFileStores.size());
+ return Math.max(
+ Math.min(newThrottleThreshold, 600 * 1024 * 1024 * 1024L), 50 * 1024 *
1024 * 1024L);
+ }
+
+ public Set<FileStore> getFileStores(List<String> dirs) {
Review Comment:
consider abstract a basic function here with systemmetric
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java:
##########
@@ -1286,6 +1293,59 @@ private void loadWALHotModifiedProps(Properties
properties) {
}
}
+ public long getThrottleThresholdWithDirs() {
+ ArrayList<String> dataDiskDirs = new
ArrayList<>(Arrays.asList(conf.getDataDirs()));
+ ArrayList<String> walDiskDirs =
+ new
ArrayList<>(Arrays.asList(commonDescriptor.getConfig().getWalDirs()));
+ Set<FileStore> dataFileStores = getFileStores(dataDiskDirs);
+ Set<FileStore> walFileStores = getFileStores(walDiskDirs);
+ double dirUseProportion = 0;
+ Set<FileStore> intersectFileStores = new HashSet<>(dataFileStores);
+ intersectFileStores.retainAll(walFileStores);
+ if (intersectFileStores.isEmpty()) {
+ dirUseProportion = 0.8;
Review Comment:
remove these magic number and add some commets for this
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java:
##########
@@ -1286,6 +1293,59 @@ private void loadWALHotModifiedProps(Properties
properties) {
}
}
+ public long getThrottleThresholdWithDirs() {
+ ArrayList<String> dataDiskDirs = new
ArrayList<>(Arrays.asList(conf.getDataDirs()));
+ ArrayList<String> walDiskDirs =
+ new
ArrayList<>(Arrays.asList(commonDescriptor.getConfig().getWalDirs()));
+ Set<FileStore> dataFileStores = getFileStores(dataDiskDirs);
+ Set<FileStore> walFileStores = getFileStores(walDiskDirs);
+ double dirUseProportion = 0;
+ Set<FileStore> intersectFileStores = new HashSet<>(dataFileStores);
Review Comment:
why copy ?
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java:
##########
@@ -1286,6 +1293,59 @@ private void loadWALHotModifiedProps(Properties
properties) {
}
}
+ public long getThrottleThresholdWithDirs() {
+ ArrayList<String> dataDiskDirs = new
ArrayList<>(Arrays.asList(conf.getDataDirs()));
+ ArrayList<String> walDiskDirs =
+ new
ArrayList<>(Arrays.asList(commonDescriptor.getConfig().getWalDirs()));
+ Set<FileStore> dataFileStores = getFileStores(dataDiskDirs);
+ Set<FileStore> walFileStores = getFileStores(walDiskDirs);
+ double dirUseProportion = 0;
+ Set<FileStore> intersectFileStores = new HashSet<>(dataFileStores);
+ intersectFileStores.retainAll(walFileStores);
+ if (intersectFileStores.isEmpty()) {
+ dirUseProportion = 0.8;
+ } else {
+ dirUseProportion = 0.5;
+ }
+ long newThrottleThreshold = Long.MAX_VALUE;
+ for (FileStore fileStore : walFileStores) {
+ try {
+ newThrottleThreshold = Math.min(newThrottleThreshold,
fileStore.getUsableSpace());
+ } catch (IOException e) {
+ LOGGER.error("Failed to get file size of {}, because", fileStore, e);
+ }
+ }
+ newThrottleThreshold = (long) (newThrottleThreshold * dirUseProportion *
walFileStores.size());
+ return Math.max(
+ Math.min(newThrottleThreshold, 600 * 1024 * 1024 * 1024L), 50 * 1024 *
1024 * 1024L);
Review Comment:
remove magic number
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java:
##########
@@ -1271,7 +1278,7 @@ private void loadWALHotModifiedProps(Properties
properties) {
Long.parseLong(
properties.getProperty(
"iot_consensus_throttle_threshold_in_byte",
- Long.toString(conf.getThrottleThreshold())));
+ Long.toString(getThrottleThresholdWithDirs())));
Review Comment:
if iot_consensus_throttle_threshold_in_byte is set, we do not need to
calculte getThrottleThresholdWithDirs
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
