This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3dd5570 [Config] Support custom config handler (#6577)
3dd5570 is described below
commit 3dd55701ba455d25f8ff0c1b339fc12595527aa8
Author: ccoffline <[email protected]>
AuthorDate: Sun Nov 7 17:39:24 2021 +0800
[Config] Support custom config handler (#6577)
Support custom config handler callback and types.
---
.../java/org/apache/doris/common/ConfigBase.java | 204 +++++++++------------
.../apache/doris/http/rest/SetConfigAction.java | 50 ++---
.../apache/doris/httpv2/rest/SetConfigAction.java | 68 ++-----
.../doris/httpv2/rest/manager/NodeAction.java | 33 ++--
.../java/org/apache/doris/qe/ShowExecutor.java | 24 +--
.../doris/analysis/AdminSetConfigStmtTest.java | 2 +-
6 files changed, 137 insertions(+), 244 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ConfigBase.java
b/fe/fe-core/src/main/java/org/apache/doris/common/ConfigBase.java
index b7298aa..1287c19 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/ConfigBase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/ConfigBase.java
@@ -17,6 +17,7 @@
package org.apache.doris.common;
+import org.apache.doris.catalog.Catalog;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -39,21 +40,36 @@ import java.util.Map;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
public class ConfigBase {
private static final Logger LOG = LogManager.getLogger(ConfigBase.class);
@Retention(RetentionPolicy.RUNTIME)
- public static @interface ConfField {
+ public @interface ConfField {
String value() default "";
boolean mutable() default false;
boolean masterOnly() default false;
String comment() default "";
+ Class<? extends ConfHandler> callback() default
DefaultConfHandler.class;
+ }
+
+ public interface ConfHandler {
+ void handle(Field field, String confVal) throws Exception;
+ }
+
+ static class DefaultConfHandler implements ConfHandler {
+ @Override
+ public void handle(Field field, String confVal) throws Exception{
+ setConfigField(field, confVal);
+ }
}
private static String confFile;
private static String customConfFile;
public static Class<? extends ConfigBase> confClass;
+ public static Map<String, Field> confFields;
private static String ldapConfFile;
private static String ldapCustomConfFile;
@@ -66,6 +82,15 @@ public class ConfigBase {
if (!isLdapConfig) {
confClass = this.getClass();
confFile = configFile;
+ confFields = Maps.newHashMap();
+ for (Field field : confClass.getFields()) {
+ ConfField confField = field.getAnnotation(ConfField.class);
+ if (confField == null) {
+ continue;
+ }
+ confFields.put(confField.value().equals("") ? field.getName()
: confField.value(), field);
+ }
+
initConf(confFile);
} else {
ldapConfClass = this.getClass();
@@ -90,42 +115,48 @@ public class ConfigBase {
replacedByEnv(props);
setFields(props, isLdapConfig);
}
-
- public static HashMap<String, String> dump() throws Exception {
- HashMap<String, String> map = new HashMap<String, String>();
- Field[] fields = confClass.getFields();
+
+ public static HashMap<String, String> dump() {
+ HashMap<String, String> map = new HashMap<>();
+ Field[] fields = confClass.getFields();
for (Field f : fields) {
- if (f.getAnnotation(ConfField.class) == null) {
- continue;
+ ConfField anno = f.getAnnotation(ConfField.class);
+ if (anno != null) {
+ map.put(anno.value().isEmpty() ? f.getName() : anno.value(),
getConfValue(f));
}
- if (f.getType().isArray()) {
- switch (f.getType().getSimpleName()) {
+ }
+ return map;
+ }
+
+ public static String getConfValue(Field field) {
+ try {
+ if (field.getType().isArray()) {
+ switch (field.getType().getSimpleName()) {
+ case "boolean[]":
+ return Arrays.toString((boolean[]) field.get(null));
+ case "char[]":
+ return Arrays.toString((char[]) field.get(null));
+ case "byte[]":
+ return Arrays.toString((byte[]) field.get(null));
case "short[]":
- map.put(f.getName(), Arrays.toString((short[])
f.get(null)));
- break;
+ return Arrays.toString((short[]) field.get(null));
case "int[]":
- map.put(f.getName(), Arrays.toString((int[])
f.get(null)));
- break;
+ return Arrays.toString((int[]) field.get(null));
case "long[]":
- map.put(f.getName(), Arrays.toString((long[])
f.get(null)));
- break;
+ return Arrays.toString((long[]) field.get(null));
+ case "float[]":
+ return Arrays.toString((float[]) field.get(null));
case "double[]":
- map.put(f.getName(), Arrays.toString((double[])
f.get(null)));
- break;
- case "boolean[]":
- map.put(f.getName(), Arrays.toString((boolean[])
f.get(null)));
- break;
- case "String[]":
- map.put(f.getName(), Arrays.toString((String[])
f.get(null)));
- break;
+ return Arrays.toString((double[]) field.get(null));
default:
- throw new Exception("unknown type: " +
f.getType().getSimpleName());
- }
+ return Arrays.toString((Object[]) field.get(null));
+ }
} else {
- map.put(f.getName(), f.get(null).toString());
+ return String.valueOf(field.get(null));
}
+ } catch (Exception e) {
+ return String.format("Failed to get config %s: %s",
field.getName(), e.getMessage());
}
- return map;
}
// there is some config in fe.conf like:
@@ -176,7 +207,7 @@ public class ConfigBase {
}
}
- public static void setConfigField(Field f, String confVal) throws
IllegalAccessException, Exception {
+ public static void setConfigField(Field f, String confVal) throws
Exception {
confVal = confVal.trim();
String[] sa = confVal.split(",");
@@ -258,114 +289,57 @@ public class ConfigBase {
throw new IllegalArgumentException("type mismatch");
}
- public static Map<String, Field> getAllMutableConfigs() {
- Map<String, Field> mutableConfigs = Maps.newHashMap();
- Field fields[] = ConfigBase.confClass.getFields();
- for (Field field : fields) {
- ConfField confField = field.getAnnotation(ConfField.class);
- if (confField == null) {
- continue;
- }
- if (!confField.mutable()) {
- continue;
- }
- mutableConfigs.put(confField.value().equals("") ? field.getName()
: confField.value(), field);
- }
-
- return mutableConfigs;
- }
-
public synchronized static void setMutableConfig(String key, String value)
throws DdlException {
- Map<String, Field> mutableConfigs = getAllMutableConfigs();
- Field field = mutableConfigs.get(key);
+ Field field = confFields.get(key);
if (field == null) {
- throw new DdlException("Config '" + key + "' does not exist or is
not mutable");
+ throw new DdlException("Config '" + key + "' does not exist");
+ }
+
+ ConfField anno = field.getAnnotation(ConfField.class);
+ if (!anno.mutable()) {
+ throw new DdlException("Config '" + key + "' is not mutable");
+ }
+ if (anno.masterOnly() && !Catalog.getCurrentCatalog().isMaster()){
+ throw new DdlException("Config '" + key + "' is master only");
}
try {
- ConfigBase.setConfigField(field, value);
+ anno.callback().newInstance().handle(field, value);
} catch (Exception e) {
throw new DdlException("Failed to set config '" + key + "'. err: "
+ e.getMessage());
}
-
+
LOG.info("set config {} to {}", key, value);
}
- public synchronized static List<List<String>> getConfigInfo(PatternMatcher
matcher) throws DdlException {
- List<List<String>> configs = Lists.newArrayList();
- Field[] fields = confClass.getFields();
- for (Field f : fields) {
- List<String> config = Lists.newArrayList();
+ public synchronized static List<List<String>> getConfigInfo(PatternMatcher
matcher) {
+ return
confFields.entrySet().stream().sorted(Map.Entry.comparingByKey()).flatMap(e -> {
+ String confKey = e.getKey();
+ Field f = e.getValue();
ConfField anno = f.getAnnotation(ConfField.class);
- if (anno == null) {
- continue;
- }
-
- String confKey = anno.value().equals("") ? f.getName() :
anno.value();
- if (matcher != null && !matcher.match(confKey)) {
- continue;
- }
- String confVal;
- try {
- switch (f.getType().getSimpleName()) {
- case "short":
- case "int":
- case "long":
- case "double":
- case "boolean":
- case "String":
- confVal = String.valueOf(f.get(null));
- break;
- case "short[]":
- confVal = Arrays.toString((short[])f.get(null));
- break;
- case "int[]":
- confVal = Arrays.toString((int[])f.get(null));
- break;
- case "long[]":
- confVal = Arrays.toString((long[])f.get(null));
- break;
- case "double[]":
- confVal = Arrays.toString((double[])f.get(null));
- break;
- case "boolean[]":
- confVal = Arrays.toString((boolean[])f.get(null));
- break;
- case "String[]":
- confVal = Arrays.toString((String[])f.get(null));
- break;
- default:
- throw new DdlException("unknown type: " +
f.getType().getSimpleName());
- }
- } catch (IllegalArgumentException | IllegalAccessException e) {
- throw new DdlException("Failed to get config '" + confKey +
"'. err: " + e.getMessage());
+ if (matcher == null || matcher.match(confKey)) {
+ List<String> config = Lists.newArrayList();
+ config.add(confKey);
+ config.add(getConfValue(f));
+ config.add(f.getType().getSimpleName());
+ config.add(String.valueOf(anno.mutable()));
+ config.add(String.valueOf(anno.masterOnly()));
+ config.add(anno.comment());
+ return Stream.of(config);
+ } else {
+ return Stream.empty();
}
-
- config.add(confKey);
- config.add(Strings.nullToEmpty(confVal));
- config.add(f.getType().getSimpleName());
- config.add(String.valueOf(anno.mutable()));
- config.add(String.valueOf(anno.masterOnly()));
- config.add(anno.comment());
- configs.add(config);
- }
-
- return configs;
+ }).collect(Collectors.toList());
}
public synchronized static boolean checkIsMasterOnly(String key) {
- Map<String, Field> mutableConfigs = getAllMutableConfigs();
- Field f = mutableConfigs.get(key);
+ Field f = confFields.get(key);
if (f == null) {
return false;
}
ConfField anno = f.getAnnotation(ConfField.class);
- if (anno == null) {
- return false;
- }
-
- return anno.masterOnly();
+ return anno != null && anno.mutable() && anno.masterOnly();
}
// use synchronized to make sure only one thread modify this file
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java
b/fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java
index a79a2be..f096b10 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java
@@ -19,9 +19,7 @@ package org.apache.doris.http.rest;
import io.netty.handler.codec.http.HttpMethod;
-import org.apache.doris.catalog.Catalog;
import org.apache.doris.common.ConfigBase;
-import org.apache.doris.common.ConfigBase.ConfField;
import org.apache.doris.common.DdlException;
import org.apache.doris.http.ActionController;
import org.apache.doris.http.BaseRequest;
@@ -37,7 +35,6 @@ import org.apache.logging.log4j.Logger;
import org.codehaus.jackson.map.ObjectMapper;
import java.io.IOException;
-import java.lang.reflect.Field;
import java.util.List;
import java.util.Map;
@@ -86,37 +83,20 @@ public class SetConfigAction extends RestBaseAction {
LOG.debug("get config from url: {}, need persist: {}", configs,
needPersist);
- Field[] fields = ConfigBase.confClass.getFields();
- for (Field f : fields) {
- // ensure that field has "@ConfField" annotation
- ConfField anno = f.getAnnotation(ConfField.class);
- if (anno == null || !anno.mutable()) {
- continue;
- }
-
- if (anno.masterOnly() && !Catalog.getCurrentCatalog().isMaster()) {
- continue;
- }
-
- // ensure that field has property string
- String confKey = anno.value().equals("") ? f.getName() :
anno.value();
- List<String> confVals = configs.get(confKey);
- if (confVals == null || confVals.isEmpty()) {
- continue;
- }
-
- if (confVals.size() > 1) {
- continue;
- }
-
+ for (Map.Entry<String, List<String>> config : configs.entrySet()) {
+ String confKey = config.getKey();
+ List<String> confValue = config.getValue();
try {
- ConfigBase.setConfigField(f, confVals.get(0));
- } catch (Exception e) {
- LOG.warn("failed to set config {}:{}", confKey,
confVals.get(0), e);
- continue;
+ if (confValue != null && confValue.size() == 1) {
+ ConfigBase.setMutableConfig(confKey, confValue.get(0));
+ setConfigs.put(confKey, confValue.get(0));
+ } else {
+ throw new DdlException("conf value size != 1");
+ }
+ } catch (DdlException e) {
+ LOG.warn("failed to set config {}:{}", confKey, confValue, e);
+ errConfigs.put(confKey, String.valueOf(confValue));
}
-
- setConfigs.put(confKey, confVals.get(0));
}
String persistMsg = "";
@@ -130,12 +110,6 @@ public class SetConfigAction extends RestBaseAction {
}
}
- for (String key : configs.keySet()) {
- if (!setConfigs.containsKey(key)) {
- errConfigs.put(key, configs.get(key).toString());
- }
- }
-
Map<String, Object> resultMap = Maps.newHashMap();
resultMap.put("set", setConfigs);
resultMap.put("err", errConfigs);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/SetConfigAction.java
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/SetConfigAction.java
index 7124c33..014efcf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/SetConfigAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/SetConfigAction.java
@@ -20,9 +20,8 @@ package org.apache.doris.httpv2.rest;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;
-import org.apache.doris.catalog.Catalog;
import org.apache.doris.common.ConfigBase;
-import org.apache.doris.common.ConfigBase.ConfField;
+import org.apache.doris.common.DdlException;
import org.apache.doris.httpv2.entity.ResponseEntityBuilder;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
@@ -39,10 +38,9 @@ import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
-import java.lang.reflect.Field;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import java.util.stream.Collectors;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@@ -88,49 +86,20 @@ public class SetConfigAction extends RestBaseController {
LOG.debug("get config from url: {}, need persist: {}", configs,
needPersist);
- Field[] fields = ConfigBase.confClass.getFields();
- for (Field f : fields) {
- // ensure that field has "@ConfField" annotation
- ConfField anno = f.getAnnotation(ConfField.class);
-
- if (anno == null) {
- continue;
- }
-
- // ensure that field has property string
- String confKey = anno.value().equals("") ? f.getName() :
anno.value();
- String[] confVals = configs.get(confKey);
- if (confVals == null) {
- continue;
- }
-
- if (confVals.length != 1) {
- errConfigs.add(new ErrConfig(confKey, "", "No or multiple
configuration values."));
- continue;
- }
-
- if (!anno.mutable()) {
- errConfigs.add(new ErrConfig(confKey, confVals[0], "Not
support dynamic modification."));
- continue;
- }
-
- if (anno.masterOnly() && !Catalog.getCurrentCatalog().isMaster()) {
- errConfigs.add(new ErrConfig(confKey, confVals[0], "Not
support modification on non-master"));
- continue;
- }
-
+ for (Map.Entry<String, String[]> config : configs.entrySet()) {
+ String confKey = config.getKey();
+ String[] confValue = config.getValue();
try {
- ConfigBase.setConfigField(f, confVals[0]);
- } catch (IllegalArgumentException e){
- errConfigs.add(new ErrConfig(confKey, confVals[0],
"Unsupported configuration value type."));
- continue;
- } catch (Exception e) {
- LOG.warn("failed to set config {}:{}, {}", confKey,
confVals[0], e.getMessage());
- errConfigs.add(new ErrConfig(confKey, confVals[0],
e.getMessage()));
- continue;
+ if (confValue != null && confValue.length == 1) {
+ ConfigBase.setMutableConfig(confKey, confValue[0]);
+ setConfigs.put(confKey, confValue[0]);
+ } else {
+ throw new DdlException("conf value size != 1");
+ }
+ } catch (DdlException e) {
+ LOG.warn("failed to set config {}:{}", confKey,
Arrays.toString(confValue), e);
+ errConfigs.add(new ErrConfig(confKey,
Arrays.toString(confValue), e.getMessage()));
}
-
- setConfigs.put(confKey, confVals[0]);
}
String persistMsg = "";
@@ -144,15 +113,6 @@ public class SetConfigAction extends RestBaseController {
}
}
- List<String> errConfigNames =
errConfigs.stream().map(ErrConfig::getConfigName).collect(Collectors.toList());
- for (String key : configs.keySet()) {
- if (!setConfigs.containsKey(key) && !errConfigNames.contains(key))
{
- String[] confVals = configs.get(key);
- String confVal = confVals.length == 1 ? confVals[0] : "invalid
value";
- errConfigs.add(new ErrConfig(key, confVal, "invalid config"));
- }
- }
-
return ResponseEntityBuilder.ok(new SetConfigEntity(setConfigs,
errConfigs, persistMsg));
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java
index bd6ec87..fb82397 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java
@@ -21,7 +21,6 @@ import org.apache.doris.catalog.Catalog;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.ConfigBase;
-import org.apache.doris.common.DdlException;
import org.apache.doris.common.MarkedCountDownLatch;
import org.apache.doris.common.Pair;
import org.apache.doris.common.ThreadPoolManager;
@@ -54,7 +53,6 @@ import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
-import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
@@ -251,25 +249,20 @@ public class NodeAction extends RestBaseController {
executeCheckPassword(request, response);
checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(),
PrivPredicate.ADMIN);
+ List<List<String>> configs = ConfigBase.getConfigInfo(null);
+ // Sort all configs by config key.
+ configs.sort(Comparator.comparing(o -> o.get(0)));
+
+ // reorder the fields
List<List<String>> results = Lists.newArrayList();
- try {
- List<List<String>> configs = ConfigBase.getConfigInfo(null);
- // Sort all configs by config key.
- Collections.sort(configs, Comparator.comparing(o -> o.get(0)));
-
- // reorder the fields
- for (List<String> config : configs) {
- List<String> list = Lists.newArrayList();
- list.add(config.get(0));
- list.add(config.get(2));
- list.add(config.get(4));
- list.add(config.get(1));
- list.add(config.get(3));
- results.add(list);
- }
- } catch (DdlException e) {
- LOG.warn(e);
- return ResponseEntityBuilder.internalError(e.getMessage());
+ for (List<String> config : configs) {
+ List<String> list = Lists.newArrayList();
+ list.add(config.get(0));
+ list.add(config.get(2));
+ list.add(config.get(4));
+ list.add(config.get(1));
+ list.add(config.get(3));
+ results.add(list);
}
return results;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 3263f8c..2e03b71 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -1755,23 +1755,15 @@ public class ShowExecutor {
private void handleAdminShowConfig() throws AnalysisException {
AdminShowConfigStmt showStmt = (AdminShowConfigStmt) stmt;
List<List<String>> results;
- try {
- PatternMatcher matcher = null;
- if (showStmt.getPattern() != null) {
- matcher =
PatternMatcher.createMysqlPattern(showStmt.getPattern(),
- CaseSensibility.CONFIG.getCaseSensibility());
- }
- results = ConfigBase.getConfigInfo(matcher);
- // Sort all configs by config key.
- Collections.sort(results, new Comparator<List<String>>() {
- @Override
- public int compare(List<String> o1, List<String> o2) {
- return o1.get(0).compareTo(o2.get(0));
- }
- });
- } catch (DdlException e) {
- throw new AnalysisException(e.getMessage());
+
+ PatternMatcher matcher = null;
+ if (showStmt.getPattern() != null) {
+ matcher = PatternMatcher.createMysqlPattern(showStmt.getPattern(),
+ CaseSensibility.CONFIG.getCaseSensibility());
}
+ results = ConfigBase.getConfigInfo(matcher);
+ // Sort all configs by config key.
+ results.sort(Comparator.comparing(o -> o.get(0)));
resultSet = new ShowResultSet(showStmt.getMetaData(), results);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/AdminSetConfigStmtTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/AdminSetConfigStmtTest.java
index f6ce740..52b6758 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/analysis/AdminSetConfigStmtTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/analysis/AdminSetConfigStmtTest.java
@@ -58,7 +58,7 @@ public class AdminSetConfigStmtTest {
String stmt = "admin set frontend config(\"unknown_config\" =
\"unknown\");";
AdminSetConfigStmt adminSetConfigStmt = (AdminSetConfigStmt)
UtFrameUtils.parseAndAnalyzeStmt(stmt, connectContext);
expectedEx.expect(DdlException.class);
- expectedEx.expectMessage("errCode = 2, detailMessage = Config
'unknown_config' does not exist or is not mutable");
+ expectedEx.expectMessage("errCode = 2, detailMessage = Config
'unknown_config' does not exist");
Catalog.getCurrentCatalog().setConfig(adminSetConfigStmt);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
