This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit cebf5fede50aee94bc855bd5007b5d90ba919797 Author: Mingyu Chen <[email protected]> AuthorDate: Tue Oct 17 22:53:31 2023 +0800 [improvement](meta) allow to ignore unknown image module (#25450) Add new FE config `ignore_unknown_metadata_module`. Default is false. If set to true, when reading metadata image file, and there are unknown modules, these modules will be ignored and skipped. This is mainly used in downgrade operation, old version can be compatible with new version Image file. --- .../src/main/java/org/apache/doris/common/Config.java | 14 ++++++++++++++ .../java/org/apache/doris/persist/meta/MetaReader.java | 16 +++++++++++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 8fc192a6d94..789bb06d5cc 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2192,6 +2192,7 @@ public class Config extends ConfigBase { + "If this database conflicts with a user's own database, please modify this field to replace " + "the name of the Doris built-in MySQL database with a different name."}) public static String mysqldb_replace_name = "mysql"; + @ConfField(description = { "设置允许跨域访问的特定域名,默认允许任何域名跨域访问", "Set the specific domain name that allows cross-domain access. " @@ -2199,4 +2200,17 @@ public class Config extends ConfigBase { }) public static String access_control_allowed_origin_domain = "*"; + @ConfField(description = { + "是否忽略 Image 文件中未知的模块。如果为 true，不在 PersistMetaModules.MODULE_NAMES 中的元数据模块将被忽略并跳过。" + + "默认为 false，如果 Image 文件中包含未知的模块，Doris 将会抛出异常。" + + "该参数主要用于降级操作中，老版本可以兼容新版本的 Image 文件。", + "Whether to ignore unknown modules in Image file. " + + "If true, metadata modules not in PersistMetaModules.MODULE_NAMES " + + "will be ignored and skipped. Default is false, if Image file contains unknown modules, " + + "Doris will throw exception. " + + "This parameter is mainly used in downgrade operation, " + + "old version can be compatible with new version Image file." + }) + public static boolean ignore_unknown_metadata_module = false; + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaReader.java b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaReader.java index 62e02bde429..8024105fe26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaReader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaReader.java @@ -18,6 +18,7 @@ package org.apache.doris.persist.meta; import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import com.google.common.base.Preconditions; @@ -101,7 +102,7 @@ public class MetaReader { } // skip deprecated modules if (PersistMetaModules.DEPRECATED_MODULE_NAMES.contains(metaIndex.name)) { - LOG.warn("meta modules {} is deprecated, ignore and skip it"); + LOG.warn("meta modules {} is deprecated, ignore and skip it", metaIndex.name); // If this is the last module, nothing need to do. if (i < metaFooter.metaIndices.size() - 1) { IOUtils.skipFully(dis, metaFooter.metaIndices.get(i + 1).offset - metaIndex.offset); @@ -110,8 +111,17 @@ public class MetaReader { } MetaPersistMethod persistMethod = PersistMetaModules.MODULES_MAP.get(metaIndex.name); if (persistMethod == null) { - throw new IOException("Unknown meta module: " + metaIndex.name + ". Known modules: " - + PersistMetaModules.MODULE_NAMES); + if (Config.ignore_unknown_metadata_module) { + LOG.warn("meta modules {} is unknown, ignore and skip it", metaIndex.name); + // If this is the last module, nothing need to do. + if (i < metaFooter.metaIndices.size() - 1) { + IOUtils.skipFully(dis, metaFooter.metaIndices.get(i + 1).offset - metaIndex.offset); + } + continue; + } else { + throw new IOException("Unknown meta module: " + metaIndex.name + ". Known modules: " + + PersistMetaModules.MODULE_NAMES); + } } checksum = (long) persistMethod.readMethod.invoke(env, dis, checksum); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
