This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8c88e356519 [opt](variable) force update some variable by variable
version (#41607)
8c88e356519 is described below
commit 8c88e356519c95e64b624c2669b5ffb9749d4c71
Author: morrySnow <[email protected]>
AuthorDate: Thu Nov 7 14:10:30 2024 +0800
[opt](variable) force update some variable by variable version (#41607)
variable version:
000-100: doris-2.0.x
100-200: doris-2.1.x
200-300: doris-3.0.x
update variables
000:
nereids_timeout_second = 30 if original value is 5
100:
enable_nereids_dml = true
enable_nereids_dml_with_pipeline = true
enable_nereids_planner = true
enable_fallback_to_original_planner = true
enable_pipeline_x_engine = true
200:
enable_fallback_to_original_planner = false
---
.../main/java/org/apache/doris/catalog/Env.java | 1 +
.../org/apache/doris/common/util/DocGenerator.java | 6 ++-
.../java/org/apache/doris/qe/GlobalVariable.java | 10 ++++
.../java/org/apache/doris/qe/SessionVariable.java | 3 +-
.../main/java/org/apache/doris/qe/VariableMgr.java | 55 +++++++++++++++++++++-
5 files changed, 70 insertions(+), 5 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 822c1954540..e5ef8e0dc72 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -1596,6 +1596,7 @@ public class Env {
// Set initial root password if master FE first time launch.
auth.setInitialRootPassword(Config.initial_root_password);
} else {
+ VariableMgr.forceUpdateVariables();
if (journalVersion <= FeMetaVersion.VERSION_114) {
// if journal version is less than 114, which means it is
upgraded from version before 2.0.
// When upgrading from 1.2 to 2.0,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java
index c0a544d825d..003dcdcf598 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java
@@ -256,8 +256,10 @@ public class DocGenerator {
}
sb.append("\n\n");
}
- sb.append(VAR_READ_ONLY[lang.idx]).append("`").append(varAttr.flag()
== VariableMgr.READ_ONLY).append("`\n\n");
- sb.append(VAR_GLOBAL_ONLY[lang.idx]).append("`").append(varAttr.flag()
== VariableMgr.GLOBAL).append("`\n\n");
+ sb.append(VAR_READ_ONLY[lang.idx]).append("`")
+ .append((varAttr.flag() & VariableMgr.READ_ONLY) !=
0).append("`\n\n");
+ sb.append(VAR_GLOBAL_ONLY[lang.idx]).append("`")
+ .append((varAttr.flag() & VariableMgr.GLOBAL) !=
0).append("`\n\n");
return sb.toString();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java
index b449a4bc8e9..45a6acba6dc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java
@@ -33,6 +33,13 @@ import java.util.List;
// NOTE: If you want access your variable safe, please hold VariableMgr's lock
before access.
public final class GlobalVariable {
+ public static final int VARIABLE_VERSION_0 = 0;
+ public static final int VARIABLE_VERSION_100 = 100;
+ public static final int VARIABLE_VERSION_200 = 200;
+ public static final int VARIABLE_VERSION_300 = 300;
+ public static final int CURRENT_VARIABLE_VERSION = VARIABLE_VERSION_300;
+ public static final String VARIABLE_VERSION = "variable_version";
+
public static final String VERSION_COMMENT = "version_comment";
public static final String VERSION = "version";
public static final String LOWER_CASE_TABLE_NAMES =
"lower_case_table_names";
@@ -69,6 +76,9 @@ public final class GlobalVariable {
public static final String ENABLE_FETCH_ICEBERG_STATS =
"enable_fetch_iceberg_stats";
+ @VariableMgr.VarAttr(name = VARIABLE_VERSION, flag = VariableMgr.INVISIBLE
+ | VariableMgr.READ_ONLY | VariableMgr.GLOBAL)
+ public static int variableVersion = CURRENT_VARIABLE_VERSION;
@VariableMgr.VarAttr(name = VERSION_COMMENT, flag = VariableMgr.READ_ONLY)
public static String versionComment = "Doris version "
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 88919c290d8..45df14a6e6f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -1518,7 +1518,7 @@ public class SessionVariable implements Serializable,
Writable {
@VariableMgr.VarAttr(name = ENABLE_NEREIDS_TIMEOUT, needForward = true)
public boolean enableNereidsTimeout = true;
- @VariableMgr.VarAttr(name = "nereids_timeout_second", needForward = true)
+ @VariableMgr.VarAttr(name = NEREIDS_TIMEOUT_SECOND, needForward = true)
public int nereidsTimeoutSecond = 30;
@VariableMgr.VarAttr(name = ENABLE_PUSH_DOWN_NO_GROUP_AGG)
@@ -3928,7 +3928,6 @@ public class SessionVariable implements Serializable,
Writable {
if (attr == null) {
continue;
}
-
if (!root.containsKey(attr.name())) {
continue;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
index a69ca0776e2..b7a4f8bb710 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
@@ -469,6 +469,11 @@ public class VariableMgr {
try {
String json = info.getPersistJsonString();
JSONObject root = (JSONObject) JSONValue.parse(json);
+ // if not variable version, we set it to 0 to ensure we could
force set global variable.
+ boolean hasVariableVersion =
root.containsKey(GlobalVariable.VARIABLE_VERSION);
+ if (!hasVariableVersion) {
+ GlobalVariable.variableVersion =
GlobalVariable.VARIABLE_VERSION_0;
+ }
for (Object varName : root.keySet()) {
VarContext varContext = ctxByVarName.get((String) varName);
if (Env.isCheckpointThread()) {
@@ -728,11 +733,15 @@ public class VariableMgr {
rlock.lock();
try {
for (Map.Entry<String, VarContext> entry :
ctxByDisplayVarName.entrySet()) {
- // not show removed variables
VarAttr varAttr =
entry.getValue().getField().getAnnotation(VarAttr.class);
+ // not show removed variables
if (VariableAnnotation.REMOVED.equals(varAttr.varType())) {
continue;
}
+ // not show invisible variables
+ if ((VariableMgr.INVISIBLE & varAttr.flag()) != 0) {
+ continue;
+ }
// Filter variable not match to the regex.
if (matcher != null && !matcher.match(entry.getKey())) {
continue;
@@ -947,4 +956,48 @@ public class VariableMgr {
}
return builder;
}
+
+ public static void forceUpdateVariables() {
+ int currentVariableVersion = GlobalVariable.variableVersion;
+ if (currentVariableVersion == GlobalVariable.VARIABLE_VERSION_0) {
+ // update from 2.0.15 or below to 2.0.16 or higher
+ if (VariableMgr.newSessionVariable().nereidsTimeoutSecond == 5) {
+ VariableMgr.refreshDefaultSessionVariables("update variable
version",
+ SessionVariable.NEREIDS_TIMEOUT_SECOND, "30");
+ }
+ }
+ if (currentVariableVersion < GlobalVariable.VARIABLE_VERSION_100) {
+ // update from 2.1.6 or below to 2.1.7 or higher
+ VariableMgr.refreshDefaultSessionVariables("update variable
version",
+ SessionVariable.ENABLE_NEREIDS_DML,
+ String.valueOf(true));
+ VariableMgr.refreshDefaultSessionVariables("update variable
version",
+ SessionVariable.ENABLE_NEREIDS_DML_WITH_PIPELINE,
+ String.valueOf(true));
+ VariableMgr.refreshDefaultSessionVariables("update variable
version",
+ SessionVariable.ENABLE_NEREIDS_PLANNER,
+ String.valueOf(true));
+ VariableMgr.refreshDefaultSessionVariables("update variable
version",
+ SessionVariable.ENABLE_FALLBACK_TO_ORIGINAL_PLANNER,
+ String.valueOf(true));
+ VariableMgr.refreshDefaultSessionVariables("update variable
version",
+ SessionVariable.ENABLE_PIPELINE_X_ENGINE,
+ String.valueOf(true));
+ }
+ if (currentVariableVersion < GlobalVariable.VARIABLE_VERSION_200) {
+ // update from 3.0.2 or below to 3.0.3 or higher
+ VariableMgr.refreshDefaultSessionVariables("update variable
version",
+ SessionVariable.ENABLE_FALLBACK_TO_ORIGINAL_PLANNER,
+ String.valueOf(false));
+ }
+ if (currentVariableVersion < GlobalVariable.VARIABLE_VERSION_300) {
+ // update to master
+ // do nothing
+ }
+ if (currentVariableVersion < GlobalVariable.CURRENT_VARIABLE_VERSION) {
+ VariableMgr.refreshDefaultSessionVariables("update variable
version",
+ GlobalVariable.VARIABLE_VERSION,
+ String.valueOf(GlobalVariable.CURRENT_VARIABLE_VERSION));
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
