This is an automated email from the ASF dual-hosted git repository.
zuston pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new 1d61e67ba [#2120] improvement(common): Support basic RssConf updating
while reconfiguring is enabled (#2121)
1d61e67ba is described below
commit 1d61e67baa6263b570e5789034fd8ce2bc8062bd
Author: maobaolong <[email protected]>
AuthorDate: Wed Sep 18 19:45:11 2024 +0800
[#2120] improvement(common): Support basic RssConf updating while
reconfiguring is enabled (#2121)
### What changes were proposed in this pull request?
Support Update the RssConf while reload config from config file
### Why are the changes needed?
Fix: #2120
### Does this PR introduce _any_ user-facing change?
User can get the reconfigured value from dashboard.
### How was this patch tested?
Tested by dashboard conf button.
```
rss.server.huge-partition.size.hard.limit 700000
-->
rss.server.huge-partition.size.hard.limit 800000
-->
rss.server.huge-partition.size.hard.limit 900000
-->
# rss.server.huge-partition.size.hard.limit 700000
```
<img width="1703" alt="image"
src="https://github.com/user-attachments/assets/8339b457-5407-47b5-b3ec-021e9c85dd42";>
<img width="1674" alt="image"
src="https://github.com/user-attachments/assets/58bf854b-f825-4959-9640-4ae5bb76e99f";>
<img width="1623" alt="image"
src="https://github.com/user-attachments/assets/81073e97-244b-4086-b915-37530fade93e";>
<img width="610" alt="image"
src="https://github.com/user-attachments/assets/7117ee27-ff19-49c9-b9f9-67b523ee0af0";>
---
.../uniffle/common/ReconfigurableConfManager.java | 37 ++++++++++++++++------
.../common/ReconfigurableConfManagerTest.java | 2 ++
2 files changed, 30 insertions(+), 9 deletions(-)
diff --git
a/common/src/main/java/org/apache/uniffle/common/ReconfigurableConfManager.java
b/common/src/main/java/org/apache/uniffle/common/ReconfigurableConfManager.java
index 3a69cd53e..1c9219ebd 100644
---
a/common/src/main/java/org/apache/uniffle/common/ReconfigurableConfManager.java
+++
b/common/src/main/java/org/apache/uniffle/common/ReconfigurableConfManager.java
@@ -21,6 +21,7 @@ import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
+import java.util.Optional;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
@@ -58,7 +59,8 @@ public class ReconfigurableConfManager<T> {
}
private void initialize(RssConf rssConf, Supplier<RssConf> confSupplier) {
- this.rssConf = new RssConf(rssConf);
+ // Reconfigure for the given rssConf
+ this.rssConf = rssConf;
if (confSupplier != null) {
this.updateConfOptions = new ArrayList<>();
this.scheduledThreadPoolExecutor =
@@ -106,14 +108,19 @@ public class ReconfigurableConfManager<T> {
return;
}
for (ConfigOption<T> configOption : updateConfOptions) {
- T val = latestConf.get(configOption);
- if (!Objects.equals(val, rssConf.get(configOption))) {
- LOGGER.info(
- "Update the config option: {} from {} -> {}",
- configOption.key(),
- rssConf.get(configOption),
- val);
- rssConf.set(configOption, val);
+ Optional<T> valOptional = latestConf.getOptional(configOption);
+ if (valOptional.isPresent()) {
+ T val = valOptional.get();
+ if (!Objects.equals(val, rssConf.get(configOption))) {
+ LOGGER.info(
+ "Update the config option: {} from {} -> {}",
+ configOption.key(),
+ rssConf.get(configOption),
+ val);
+ rssConf.set(configOption, val);
+ }
+ } else {
+ rssConf.remove(configOption.key());
}
}
}
@@ -126,12 +133,24 @@ public class ReconfigurableConfManager<T> {
this.updateConfOptions.add(configOption);
}
+ /**
+ * Initialize the reconfigurable conf manager and reconfigure for the given
rss conf.
+ *
+ * @param rssConf the rss conf to be reconfigured
+ * @param rssConfFilePath the rss conf file path for reloading
+ */
public static void init(RssConf rssConf, String rssConfFilePath) {
ReconfigurableConfManager manager =
new ReconfigurableConfManager(rssConf, rssConfFilePath,
rssConf.getClass());
reconfigurableConfManager = manager;
}
+ /**
+ * Initialize the reconfigurable conf manager and reconfigure for the given
rss conf.
+ *
+ * @param rssConf the rss conf to be reconfigured
+ * @param confSupplier the supplier of rss conf
+ */
@VisibleForTesting
protected static void initForTest(RssConf rssConf, Supplier<RssConf>
confSupplier) {
ReconfigurableConfManager manager = new ReconfigurableConfManager(rssConf,
confSupplier);
diff --git
a/common/src/test/java/org/apache/uniffle/common/ReconfigurableConfManagerTest.java
b/common/src/test/java/org/apache/uniffle/common/ReconfigurableConfManagerTest.java
index a43a63ed1..b44ae8006 100644
---
a/common/src/test/java/org/apache/uniffle/common/ReconfigurableConfManagerTest.java
+++
b/common/src/test/java/org/apache/uniffle/common/ReconfigurableConfManagerTest.java
@@ -71,6 +71,8 @@ public class ReconfigurableConfManagerTest {
.until(() -> portReconfigurable.get().equals(100));
assertEquals(200, rpcReconfigurable.get());
assertEquals(Arrays.asList("/d1"), typeReconfigurable.get());
+ // The base config should be updated too
+ assertEquals(200, base.getInteger(RPC_SERVER_PORT));
}
@Test
