This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 35968c9956afbd2e63a7deaa555decd29e067341 Author: Mingyu Chen <[email protected]> AuthorDate: Thu Jul 20 10:42:29 2023 +0800 [improvement](catalog) return the root cause of error when forwarding init request to master FE (#22001) --- .../java/org/apache/doris/common/util/Util.java | 18 ++++++++++++++++ .../org/apache/doris/qe/MasterCatalogExecutor.java | 6 ++++++ .../apache/doris/service/FrontendServiceImpl.java | 25 ++++++++++++++++------ .../apache/doris/common/util/DebugUtilTest.java | 16 ++++++++++++++ 4 files changed, 59 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java index f83be84e29..8d9c2c6b0c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java @@ -41,6 +41,8 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.io.StringWriter; import java.net.URL; import java.net.URLConnection; import java.util.ArrayList; @@ -616,4 +618,20 @@ public class Util { } return rootCause; } + + // Return the stack of the root cause + public static String getRootCauseStack(Throwable t) { + String rootStack = "unknown"; + if (t == null) { + return rootStack; + } + Throwable p = t; + while (p.getCause() != null) { + p = p.getCause(); + } + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + p.printStackTrace(pw); + return sw.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterCatalogExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterCatalogExecutor.java index fe21f04066..2bd72642fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterCatalogExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterCatalogExecutor.java @@ -19,6 +19,7 @@ package org.apache.doris.qe; import org.apache.doris.catalog.Env; import org.apache.doris.common.ClientPool; +import org.apache.doris.common.UserException; import org.apache.doris.thrift.FrontendService; import org.apache.doris.thrift.TInitExternalCtlMetaRequest; import org.apache.doris.thrift.TInitExternalCtlMetaResult; @@ -35,6 +36,8 @@ import org.apache.logging.log4j.Logger; public class MasterCatalogExecutor { private static final Logger LOG = LogManager.getLogger(MasterCatalogExecutor.class); + public static final String STATUS_OK = "OK"; + private int waitTimeoutMs; public MasterCatalogExecutor(int waitTimeoutMs) { @@ -64,6 +67,9 @@ public class MasterCatalogExecutor { try { TInitExternalCtlMetaResult result = client.initExternalCtlMeta(request); ConnectContext.get().getEnv().getJournalObservable().waitOn(result.maxJournalId, waitTimeoutMs); + if (!result.getStatus().equalsIgnoreCase(STATUS_OK)) { + throw new UserException(result.getStatus()); + } isReturnToPool = true; } catch (Exception e) { LOG.warn("Failed to finish forward init operation, please try again. ", e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 462f476c15..93b4d7f687 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -57,6 +57,7 @@ import org.apache.doris.common.ThriftServerEventProcessor; import org.apache.doris.common.UserException; import org.apache.doris.common.Version; import org.apache.doris.common.annotation.LogException; +import org.apache.doris.common.util.Util; import org.apache.doris.cooldown.CooldownDelete; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; @@ -69,6 +70,7 @@ import org.apache.doris.planner.StreamLoadPlanner; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ConnectProcessor; import org.apache.doris.qe.DdlExecutor; +import org.apache.doris.qe.MasterCatalogExecutor; import org.apache.doris.qe.QeProcessorImpl; import org.apache.doris.qe.QueryState; import org.apache.doris.qe.VariableMgr; @@ -1988,10 +1990,15 @@ public class FrontendServiceImpl implements FrontendService.Iface { if (!(catalog instanceof ExternalCatalog)) { throw new TException("Only support forward ExternalCatalog init operation."); } - ((ExternalCatalog) catalog).makeSureInitialized(); TInitExternalCtlMetaResult result = new TInitExternalCtlMetaResult(); - result.setMaxJournalId(Env.getCurrentEnv().getMaxJournalId()); - result.setStatus("OK"); + try { + ((ExternalCatalog) catalog).makeSureInitialized(); + result.setMaxJournalId(Env.getCurrentEnv().getMaxJournalId()); + result.setStatus(MasterCatalogExecutor.STATUS_OK); + } catch (Throwable t) { + LOG.warn("init catalog failed. catalog: {}", catalog.getName(), t); + result.setStatus(Util.getRootCauseStack(t)); + } return result; } @@ -2007,10 +2014,16 @@ public class FrontendServiceImpl implements FrontendService.Iface { if (!(db instanceof ExternalDatabase)) { throw new TException("Only support forward ExternalDatabase init operation."); } - ((ExternalDatabase) db).makeSureInitialized(); + TInitExternalCtlMetaResult result = new TInitExternalCtlMetaResult(); - result.setMaxJournalId(Env.getCurrentEnv().getMaxJournalId()); - result.setStatus("OK"); + try { + ((ExternalDatabase) db).makeSureInitialized(); + result.setMaxJournalId(Env.getCurrentEnv().getMaxJournalId()); + result.setStatus(MasterCatalogExecutor.STATUS_OK); + } catch (Throwable t) { + LOG.warn("init database failed. catalog.database: {}", catalog.getName(), db.getFullName(), t); + result.setStatus(Util.getRootCauseStack(t)); + } return result; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/DebugUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/DebugUtilTest.java index 0dec8873f1..bebe65cd2e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/DebugUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/DebugUtilTest.java @@ -17,6 +17,7 @@ package org.apache.doris.common.util; +import org.apache.doris.common.DdlException; import org.apache.doris.common.Pair; import org.junit.Assert; @@ -81,4 +82,19 @@ public class DebugUtilTest { Assert.assertEquals(result.first, Double.valueOf(1.1497809458523989)); Assert.assertEquals(result.second, "GB"); } + + @Test + public void testUtilGetStackTrace() { + Exception e1 = new Exception("exception1"); + DdlException e2 = new DdlException("exception2", e1); + e2.printStackTrace(); + System.out.println(Util.getRootCauseStack(e2)); + Assert.assertTrue(Util.getRootCauseStack(e2).contains("java.lang.Exception: exception1")); + + DdlException e3 = new DdlException("only one exception"); + System.out.println(Util.getRootCauseStack(e3)); + Assert.assertTrue(Util.getRootCauseStack(e3) + .contains("org.apache.doris.common.DdlException: errCode = 2, detailMessage = only one exception")); + Assert.assertEquals("unknown", Util.getRootCauseStack(null)); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
