This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit ae9d17623f2ee7122c165ed1fbcb7dfdb518a815 Author: Calvin Kirs <[email protected]> AuthorDate: Thu Sep 14 10:02:35 2023 +0800 [Improve](metadata)Start the script to set metadata_failure_recovery (#24308) --- bin/start_fe.sh | 12 +++++++++--- .../admin-manual/maint-monitor/metadata-operation.md | 14 ++++++-------- .../admin-manual/maint-monitor/metadata-operation.md | 17 ++++++++--------- .../src/main/java/org/apache/doris/common/Config.java | 10 ---------- fe/fe-core/src/main/java/org/apache/doris/DorisFE.java | 6 ++++++ .../src/main/java/org/apache/doris/catalog/Env.java | 3 ++- .../main/java/org/apache/doris/common/FeConstants.java | 2 ++ .../org/apache/doris/journal/bdbje/BDBEnvironment.java | 6 ++++-- .../org/apache/doris/journal/bdbje/BDBJEJournal.java | 2 +- 9 files changed, 38 insertions(+), 34 deletions(-) diff --git a/bin/start_fe.sh b/bin/start_fe.sh index 82d0e78501..f83ab7ca84 100755 --- a/bin/start_fe.sh +++ b/bin/start_fe.sh @@ -32,6 +32,7 @@ OPTS="$(getopt \ -l 'helper:' \ -l 'image:' \ -l 'version' \ + -l 'metadata_failure_recovery' \ -- "$@")" eval set -- "${OPTS}" @@ -41,6 +42,7 @@ HELPER='' IMAGE_PATH='' IMAGE_TOOL='' OPT_VERSION='' +METADATA_FAILURE_RECOVERY='' while true; do case "$1" in --daemon) @@ -51,6 +53,10 @@ while true; do OPT_VERSION="--version" shift ;; + --metadata_failure_recovery) + METADATA_FAILURE_RECOVERY="-r" + shift + ;; --helper) HELPER="$2" shift 2 @@ -215,15 +221,15 @@ fi if [[ "${IMAGE_TOOL}" -eq 1 ]]; then if [[ -n "${IMAGE_PATH}" ]]; then - ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} org.apache.doris.DorisFE -i "${IMAGE_PATH}" + ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} org.apache.doris.DorisFE -i "${IMAGE_PATH} ${METADATA_FAILURE_RECOVERY}" else echo "Internal Error. USE IMAGE_TOOL like : ./start_fe.sh --image image_path" fi elif [[ "${RUN_DAEMON}" -eq 1 ]]; then - nohup ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} -XX:-OmitStackTraceInFastThrow -XX:OnOutOfMemoryError="kill -9 %p" org.apache.doris.DorisFE ${HELPER:+${HELPER}} "$@" >>"${LOG_DIR}/fe.out" 2>&1 </dev/null & + nohup ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} -XX:-OmitStackTraceInFastThrow -XX:OnOutOfMemoryError="kill -9 %p" org.apache.doris.DorisFE ${HELPER:+${HELPER}} "${METADATA_FAILURE_RECOVERY}" "$@" >>"${LOG_DIR}/fe.out" 2>&1 </dev/null & else export DORIS_LOG_TO_STDERR=1 - ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} -XX:-OmitStackTraceInFastThrow -XX:OnOutOfMemoryError="kill -9 %p" org.apache.doris.DorisFE ${HELPER:+${HELPER}} ${OPT_VERSION:+${OPT_VERSION}} "$@" </dev/null + ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} -XX:-OmitStackTraceInFastThrow -XX:OnOutOfMemoryError="kill -9 %p" org.apache.doris.DorisFE ${HELPER:+${HELPER}} ${OPT_VERSION:+${OPT_VERSION}} "${METADATA_FAILURE_RECOVERY}" "$@" </dev/null fi echo $! >"${pidfile}" diff --git a/docs/en/docs/admin-manual/maint-monitor/metadata-operation.md b/docs/en/docs/admin-manual/maint-monitor/metadata-operation.md index f502a046d7..e9bd2f947d 100644 --- a/docs/en/docs/admin-manual/maint-monitor/metadata-operation.md +++ b/docs/en/docs/admin-manual/maint-monitor/metadata-operation.md @@ -190,12 +190,10 @@ FE may fail to start bdbje and synchronize between FEs for some reasons. Phenome 3. The following operations are performed on the FE nodes selected in step 2. 1. If the node is an OBSERVER, first change the `role=OBSERVER` in the `meta_dir/image/ROLE` file to `role=FOLLOWER`. (Recovery from the OBSERVER node will be more cumbersome, first follow the steps here, followed by a separate description) - 2. Add configuration in fe.conf: `metadata_failure_recovery=true`. - 3. Run `sh bin/start_fe.sh` to start the FE - 4. If normal, the FE will start in the role of MASTER, similar to the description in the previous section `Start a single node FE`. You should see the words `transfer from XXXX to MASTER` in fe.log. - 5. After the start-up is completed, connect to the FE first, and execute some query imports to check whether normal access is possible. If the operation is not normal, it may be wrong. It is recommended to read the above steps carefully and try again with the metadata previously backed up. If not, the problem may be more serious. - 6. If successful, through the `show frontends;` command, you should see all the FEs you added before, and the current FE is master. - 7. Delete the `metadata_failure_recovery=true` configuration item in fe.conf, or set it to `false`, and restart the FE (**Important**). + 2. Run `sh bin/start_fe.sh --metadata_failure_recovery` to start the FE + 3. If normal, the FE will start in the role of MASTER, similar to the description in the previous section `Start a single node FE`. You should see the words `transfer from XXXX to MASTER` in fe.log. + 4. After the start-up is completed, connect to the FE first, and execute some query imports to check whether normal access is possible. If the operation is not normal, it may be wrong. It is recommended to read the above steps carefully and try again with the metadata previously backed up. If not, the problem may be more serious. + 5. If successful, through the `show frontends;` command, you should see all the FEs you added before, and the current FE is master. > If you are recovering metadata from an OBSERVER node, after completing the above steps, you will find that the current FE role is OBSERVER, but `IsMaster` appears as `true`. This is because the "OBSERVER" seen here is recorded in Doris's metadata, but whether it is master or not, is recorded in bdbje's metadata. Because we recovered from an OBSERVER node, there was inconsistency. Please take the following steps to fix this problem (we will fix it in a later version): @@ -207,7 +205,7 @@ FE may fail to start bdbje and synchronize between FEs for some reasons. Phenome > 5. After confirming that the new FOLLOWER is working properly, the new FOLLOWER metadata is used to perform a failure recovery operation again. > 6. The purpose of the above steps is to manufacture a metadata of FOLLOWER node artificially, and then use this metadata to restart fault recovery. This avoids inconsistencies in recovering metadata from OBSERVER. - >The meaning of `metadata_failure_recovery = true` is to empty the metadata of `bdbje`. In this way, bdbje will not contact other FEs before, but start as a separate FE. This parameter needs to be set to true only when restoring startup. After recovery, it must be set to false. Otherwise, once restarted, the metadata of bdbje will be emptied again, which will make other FEs unable to work properly. + >The meaning of `metadata_failure_recovery` is to empty the metadata of `bdbje`. In this way, bdbje will not contact other FEs before, but start as a separate FE. This parameter needs to be set to true only when restoring startup. After recovery, it must be set to false. Otherwise, once restarted, the metadata of bdbje will be emptied again, which will make other FEs unable to work properly. 4. After the successful execution of step 3, we delete the previous FEs from the metadata by using the `ALTER SYSTEM DROP FOLLOWER/OBSERVER` command and add them again by adding new FEs. @@ -244,7 +242,7 @@ FE currently has the following ports 1. edit_log_port - If this port needs to be replaced, it needs to be restored with reference to the operations in the `Failure Recovery` section. Because the port has been persisted into bdbje's own metadata (also recorded in Doris's own metadata), it is necessary to clear bdbje's metadata by setting `metadata_failure_recovery=true`. + If this port needs to be replaced, it needs to be restored with reference to the operations in the `Failure Recovery` section. Because the port has been persisted into bdbje's own metadata (also recorded in Doris's own metadata), it is necessary to clear bdbje's metadata by setting `metadata_failure_recovery` when Fe start. 2. http_port diff --git a/docs/zh-CN/docs/admin-manual/maint-monitor/metadata-operation.md b/docs/zh-CN/docs/admin-manual/maint-monitor/metadata-operation.md index 8baa6957bc..39081d5989 100644 --- a/docs/zh-CN/docs/admin-manual/maint-monitor/metadata-operation.md +++ b/docs/zh-CN/docs/admin-manual/maint-monitor/metadata-operation.md @@ -189,13 +189,12 @@ FE 有可能因为某些原因出现无法启动 bdbje、FE 之间无法同步 3. 以下操作都在由第2步中选择出来的 FE 节点上进行。 - 1. 如果该节点是一个 OBSERVER，先将 `meta_dir/image/ROLE` 文件中的 `role=OBSERVER` 改为 `role=FOLLOWER`。（从 OBSERVER 节点恢复会比较麻烦，先按这里的步骤操作，后面会有单独说明） - 2. 在 fe.conf 中添加配置：`metadata_failure_recovery=true`。 - 3. 执行 `sh bin/start_fe.sh` 启动这个 FE。 - 4. 如果正常，这个 FE 会以 MASTER 的角色启动，类似于前面 `启动单节点 FE` 一节中的描述。在 fe.log 应该会看到 `transfer from XXXX to MASTER` 等字样。 - 5. 启动完成后，先连接到这个 FE，执行一些查询导入，检查是否能够正常访问。如果不正常，有可能是操作有误，建议仔细阅读以上步骤，用之前备份的元数据再试一次。如果还是不行，问题可能就比较严重了。 - 6. 如果成功，通过 `show frontends;` 命令，应该可以看到之前所添加的所有 FE，并且当前 FE 是 master。 - 7. 将 fe.conf 中的 `metadata_failure_recovery=true` 配置项删除，或者设置为 `false`，然后重启这个 FE（**重要**）。 + 1. 如果该节点是一个 OBSERVER，先将 `meta_dir/image/ROLE` 文件中的 `role=OBSERVER` 改为 `role=FOLLOWER`。（从 OBSERVER 节点恢复会比较麻烦，先按这里的步骤操作，后面会有单独说明）) + 2. 执行 `sh bin/start_fe.sh --metadata_failure_recovery` 启动这个 FE。 + 3. 如果正常，这个 FE 会以 MASTER 的角色启动，类似于前面 `启动单节点 FE` 一节中的描述。在 fe.log 应该会看到 `transfer from XXXX to MASTER` 等字样。 + 4. 启动完成后，先连接到这个 FE，执行一些查询导入，检查是否能够正常访问。如果不正常，有可能是操作有误，建议仔细阅读以上步骤，用之前备份的元数据再试一次。如果还是不行，问题可能就比较严重了。 + 5. 如果成功，通过 `show frontends;` 命令，应该可以看到之前所添加的所有 FE，并且当前 FE 是 master。 + 6. 后重启这个 FE（**重要**）。 > 如果你是从一个 OBSERVER 节点的元数据进行恢复的，那么完成如上步骤后，通过 `show frontends;` 语句你会发现，当前这个 FE 的角色为 OBSERVER，但是 `IsMaster` 显示为 `true`。这是因为，这里看到的 “OBSERVER” 是记录在 Doris 的元数据中的，而是否是 master，是记录在 bdbje 的元数据中的。因为我们是从一个 OBSERVER 节点恢复的，所以这里出现了不一致。请按如下步骤修复这个问题（这个问题我们会在之后的某个版本修复）： @@ -207,7 +206,7 @@ FE 有可能因为某些原因出现无法启动 bdbje、FE 之间无法同步 > 5. 确认这个新的 FOLLOWER 是可以正常工作之后，用这个新的 FOLLOWER 的元数据，重新执行一遍故障恢复操作。 > 6. 以上这些步骤的目的，其实就是人为的制造出一个 FOLLOWER 节点的元数据，然后用这个元数据，重新开始故障恢复。这样就避免了从 OBSERVER 恢复元数据所遇到的不一致的问题。 - > `metadata_failure_recovery=true` 的含义是，清空 "bdbje" 的元数据。这样 bdbje 就不会再联系之前的其他 FE 了，而作为一个独立的 FE 启动。这个参数只有在恢复启动时才需要设置为 true。恢复完成后，一定要设置为 false，否则一旦重启，bdbje 的元数据又会被清空，导致其他 FE 无法正常工作。 + > `metadata_failure_recovery` 的含义是，清空 "bdbje" 的元数据。这样 bdbje 就不会再联系之前的其他 FE 了，而作为一个独立的 FE 启动。这个参数只有在恢复启动时才需要设置为 true。恢复完成后，一定要设置为 false，否则一旦重启，bdbje 的元数据又会被清空，导致其他 FE 无法正常工作。 4. 第3步执行成功后，我们再通过 `ALTER SYSTEM DROP FOLLOWER/OBSERVER` 命令，将之前的其他的 FE 从元数据删除后，按加入新 FE 的方式，重新把这些 FE 添加一遍。 @@ -244,7 +243,7 @@ FE 目前有以下几个端口 1. edit_log_port - 如果需要更换这个端口，则需要参照 `故障恢复` 一节中的操作，进行恢复。因为该端口已经被持久化到 bdbje 自己的元数据中（同时也记录在 Doris 自己的元数据中），需要通过设置 `metadata_failure_recovery=true` 来清空 bdbje 的元数据。 + 如果需要更换这个端口，则需要参照 `故障恢复` 一节中的操作，进行恢复。因为该端口已经被持久化到 bdbje 自己的元数据中（同时也记录在 Doris 自己的元数据中），需要启动 FE 时通过指定 `--metadata_failure_recovery` 来清空 bdbje 的元数据。 2. http_port diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 8509f43615..5f2173af7c 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -271,16 +271,6 @@ public class Config extends ConfigBase { + "each element is a CIDR representation of the network address"}) public static String priority_networks = ""; - @ConfField(description = {"是否重置 BDBJE 的复制组，如果所有的可选节点都无法启动，" - + "可以将元数据拷贝到另一个节点，并将这个配置设置为 true，尝试重启 FE。更多信息请参阅官网的元数据故障恢复文档。", - "If true, FE will reset bdbje replication group(that is, to remove all electable nodes info) " - + "and is supposed to start as Master. " - + "If all the electable nodes can not start, we can copy the meta data " - + "to another node and set this config to true to try to restart the FE. " - + "For more information, please refer to the metadata failure recovery document " - + "on the official website."}) - public static String metadata_failure_recovery = "false"; - @ConfField(mutable = true, description = {"是否忽略元数据延迟，如果 FE 的元数据延迟超过这个阈值，" + "则非 Master FE 仍然提供读服务。这个配置可以用于当 Master FE 因为某些原因停止了较长时间，" + "但是仍然希望非 Master FE 可以提供读服务。", diff --git a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java index 07394d9cd4..e3cf89410d 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java @@ -20,6 +20,7 @@ package org.apache.doris; import org.apache.doris.catalog.Env; import org.apache.doris.common.CommandLineOptions; import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.LdapConfig; import org.apache.doris.common.Log4jConfig; import org.apache.doris.common.ThreadPoolManager; @@ -255,6 +256,8 @@ public class DorisFE { options.addOption("f", "from", true, "Specify the start scan key"); options.addOption("t", "to", true, "Specify the end scan key"); options.addOption("m", "metaversion", true, "Specify the meta version to decode log value"); + options.addOption("r", FeConstants.METADATA_FAILURE_RECOVERY_KEY, false, + "Check if the specified metadata recover is valid"); CommandLine cmd = null; try { @@ -288,6 +291,9 @@ public class DorisFE { } return new CommandLineOptions(false, "", null, imagePath); } + if (cmd.hasOption('r') || cmd.hasOption(FeConstants.METADATA_FAILURE_RECOVERY_KEY)) { + System.setProperty(FeConstants.METADATA_FAILURE_RECOVERY_KEY, "true"); + } if (cmd.hasOption('b') || cmd.hasOption("bdb")) { if (cmd.hasOption('l') || cmd.hasOption("listdb")) { // list bdb je databases diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 3754ea416b..a6c06056f8 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -1596,7 +1596,8 @@ public class Env { * frontend log is deleted because of checkpoint. */ private void checkCurrentNodeExist() { - if (Config.metadata_failure_recovery.equals("true")) { + boolean metadataFailureRecovery = null != System.getProperty(FeConstants.METADATA_FAILURE_RECOVERY_KEY); + if (metadataFailureRecovery) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java b/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java index e9e853eb9f..0845d593e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java @@ -92,4 +92,6 @@ public class FeConstants { public static String FS_PREFIX_FILE = "file"; public static final String INTERNAL_DB_NAME = "__internal_schema"; public static String TEMP_MATERIZLIZE_DVIEW_PREFIX = "internal_tmp_materialized_view_"; + + public static String METADATA_FAILURE_RECOVERY_KEY = "metadata_failure_recovery"; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java index 92bd5c2ef2..5dec4a4c57 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java @@ -19,6 +19,7 @@ package org.apache.doris.journal.bdbje; import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; import org.apache.doris.ha.BDBHA; import org.apache.doris.ha.BDBStateChangeListener; import org.apache.doris.ha.FrontendNodeType; @@ -90,13 +91,14 @@ public class BDBEnvironment { // The setup() method opens the environment and database public void setup(File envHome, String selfNodeName, String selfNodeHostPort, String helperHostPort, boolean isElectable) { - + boolean metadataFailureRecovery = null != System.getProperty(FeConstants.METADATA_FAILURE_RECOVERY_KEY); // Almost never used, just in case the master can not restart - if (Config.metadata_failure_recovery.equals("true")) { + if (metadataFailureRecovery) { if (!isElectable) { LOG.error("Current node is not in the electable_nodes list. will exit"); System.exit(-1); } + LOG.info("start group reset"); DbResetRepGroup resetUtility = new DbResetRepGroup( envHome, PALO_JOURNAL_GROUP, selfNodeName, selfNodeHostPort); resetUtility.reset(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java index 7fbdb276c3..f865bba4a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java @@ -312,7 +312,7 @@ public class BDBJEJournal implements Journal { // CHECKSTYLE IGNORE THIS LINE: B Env.getServingEnv().isElectable()); } catch (Exception e) { if (e instanceof DatabaseNotFoundException) { - LOG.error("It is not allowed to set metadata_failure_recovery to true " + LOG.error("It is not allowed to set metadata_failure_recovery" + "when meta dir or bdbje dir is empty， which may mean it is " + "the first time to start this node"); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
