Re: [PR] new iot_consensus_throttle_threshold_in_byte [iotdb]

[via GitHub]

OneSizeFitsQuorum merged PR #12333:
URL: https://github.com/apache/iotdb/pull/12333


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: reviews-unsubscr...@iotdb.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

Re: [PR] new iot_consensus_throttle_threshold_in_byte [iotdb]

[via GitHub]

OneSizeFitsQuorum commented on code in PR #12333:
URL: https://github.com/apache/iotdb/pull/12333#discussion_r1564430529


##
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java:
##
@@ -1286,6 +1293,59 @@ private void loadWALHotModifiedProps(Properties 
properties) {
 }
   }
 
+  public long getThrottleThresholdWithDirs() {
+ArrayList dataDiskDirs = new 
ArrayList<>(Arrays.asList(conf.getDataDirs()));
+ArrayList walDiskDirs =
+new 
ArrayList<>(Arrays.asList(commonDescriptor.getConfig().getWalDirs()));
+Set dataFileStores = getFileStores(dataDiskDirs);
+Set walFileStores = getFileStores(walDiskDirs);
+double dirUseProportion = 0;
+Set intersectFileStores = new HashSet<>(dataFileStores);
+intersectFileStores.retainAll(walFileStores);
+if (intersectFileStores.isEmpty()) {
+  dirUseProportion = 0.8;
+} else {
+  dirUseProportion = 0.5;
+}
+long newThrottleThreshold = Long.MAX_VALUE;
+for (FileStore fileStore : walFileStores) {
+  try {
+newThrottleThreshold = Math.min(newThrottleThreshold, 
fileStore.getUsableSpace());
+  } catch (IOException e) {
+LOGGER.error("Failed to get file size of {}, because", fileStore, e);
+  }
+}
+newThrottleThreshold = (long) (newThrottleThreshold * dirUseProportion * 
walFileStores.size());
+return Math.max(
+Math.min(newThrottleThreshold, 600 * 1024 * 1024 * 1024L), 50 * 1024 * 
1024 * 1024L);
+  }
+
+  public Set getFileStores(List dirs) {

Review Comment:
   consider abstract a basic function here with systemmetric



##
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java:
##
@@ -1286,6 +1293,59 @@ private void loadWALHotModifiedProps(Properties 
properties) {
 }
   }
 
+  public long getThrottleThresholdWithDirs() {
+ArrayList dataDiskDirs = new 
ArrayList<>(Arrays.asList(conf.getDataDirs()));
+ArrayList walDiskDirs =
+new 
ArrayList<>(Arrays.asList(commonDescriptor.getConfig().getWalDirs()));
+Set dataFileStores = getFileStores(dataDiskDirs);
+Set walFileStores = getFileStores(walDiskDirs);
+double dirUseProportion = 0;
+Set intersectFileStores = new HashSet<>(dataFileStores);
+intersectFileStores.retainAll(walFileStores);
+if (intersectFileStores.isEmpty()) {
+  dirUseProportion = 0.8;

Review Comment:
   remove these magic number and add some commets for this



##
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java:
##
@@ -1286,6 +1293,59 @@ private void loadWALHotModifiedProps(Properties 
properties) {
 }
   }
 
+  public long getThrottleThresholdWithDirs() {
+ArrayList dataDiskDirs = new 
ArrayList<>(Arrays.asList(conf.getDataDirs()));
+ArrayList walDiskDirs =
+new 
ArrayList<>(Arrays.asList(commonDescriptor.getConfig().getWalDirs()));
+Set dataFileStores = getFileStores(dataDiskDirs);
+Set walFileStores = getFileStores(walDiskDirs);
+double dirUseProportion = 0;
+Set intersectFileStores = new HashSet<>(dataFileStores);

Review Comment:
   why copy ?



##
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java:
##
@@ -1286,6 +1293,59 @@ private void loadWALHotModifiedProps(Properties 
properties) {
 }
   }
 
+  public long getThrottleThresholdWithDirs() {
+ArrayList dataDiskDirs = new 
ArrayList<>(Arrays.asList(conf.getDataDirs()));
+ArrayList walDiskDirs =
+new 
ArrayList<>(Arrays.asList(commonDescriptor.getConfig().getWalDirs()));
+Set dataFileStores = getFileStores(dataDiskDirs);
+Set walFileStores = getFileStores(walDiskDirs);
+double dirUseProportion = 0;
+Set intersectFileStores = new HashSet<>(dataFileStores);
+intersectFileStores.retainAll(walFileStores);
+if (intersectFileStores.isEmpty()) {
+  dirUseProportion = 0.8;
+} else {
+  dirUseProportion = 0.5;
+}
+long newThrottleThreshold = Long.MAX_VALUE;
+for (FileStore fileStore : walFileStores) {
+  try {
+newThrottleThreshold = Math.min(newThrottleThreshold, 
fileStore.getUsableSpace());
+  } catch (IOException e) {
+LOGGER.error("Failed to get file size of {}, because", fileStore, e);
+  }
+}
+newThrottleThreshold = (long) (newThrottleThreshold * dirUseProportion * 
walFileStores.size());
+return Math.max(
+Math.min(newThrottleThreshold, 600 * 1024 * 1024 * 1024L), 50 * 1024 * 
1024 * 1024L);

Review Comment:
   remove magic number



##
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java:
##
@@ -1271,7 +1278,7 @@ private void loadWALHotModifiedProps(Properties 
properties) {
 Long.parseLong(
 properties.getProperty(
 "iot_consensus_throttle_threshold_in_byte",
-Long.toString(conf.getThrottleThreshold(;
+Long.toStr

[PR] new iot_consensus_throttle_threshold_in_byte [iotdb]

[via GitHub]

l2280212 opened a new pull request, #12333:
URL: https://github.com/apache/iotdb/pull/12333

   This PR disign a new calculation method for 
iot_consensus_throttle_threshold_in_byte.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: reviews-unsubscr...@iotdb.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org