Repository: hive Updated Branches: refs/heads/master f8bc4868e -> 3c4a9c6bc
HIVE-18915: Better client logging when a HoS session can't be opened (Aihua Xu, reviewed by Sahil Takiar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3c4a9c6b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3c4a9c6b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3c4a9c6b Branch: refs/heads/master Commit: 3c4a9c6bc37dfc870800523a6d26f667d0b43818 Parents: f8bc486 Author: Aihua Xu <[email protected]> Authored: Thu Apr 12 17:12:56 2018 -0700 Committer: Aihua Xu <[email protected]> Committed: Mon Apr 30 16:35:41 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/ql/ErrorMsg.java | 2 +- .../ql/exec/spark/session/SparkSessionImpl.java | 30 +++++++++----------- .../session/TestSparkSessionManagerImpl.java | 6 ++-- 3 files changed, 17 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/3c4a9c6b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 7d33fa3..94dd636 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -576,7 +576,7 @@ public enum ErrorMsg { SPARK_CREATE_CLIENT_INTERRUPTED(30040, "Interrupted while creating Spark client for session {0}", true), SPARK_CREATE_CLIENT_ERROR(30041, - "Failed to create Spark client for Spark session {0}", true), + "Failed to create Spark client for Spark session {0}: {1}", true), SPARK_CREATE_CLIENT_INVALID_RESOURCE_REQUEST(30042, "Failed to create Spark client due to invalid resource request: {0}", true), SPARK_CREATE_CLIENT_CLOSED_SESSION(30043, http://git-wip-us.apache.org/repos/asf/hive/blob/3c4a9c6b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java index 2d5d03e..189de19 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java @@ -71,7 +71,6 @@ public class SparkSessionImpl implements SparkSession { private HiveSparkClient hiveSparkClient; private Path scratchDir; private final Object dirLock = new Object(); - private String matchedString = null; public SparkSessionImpl() { sessionId = makeSessionId(); @@ -195,6 +194,7 @@ public class SparkSessionImpl implements SparkSession { @VisibleForTesting HiveException getHiveException(Throwable e) { Throwable oe = e; + StringBuilder matchedString = new StringBuilder(); while (e != null) { if (e instanceof TimeoutException) { return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_TIMEOUT); @@ -202,31 +202,26 @@ public class SparkSessionImpl implements SparkSession { return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_INTERRUPTED, sessionId); } else if (e instanceof RuntimeException) { String sts = Throwables.getStackTraceAsString(e); - if (matches(sts, AM_TIMEOUT_ERR)) { + if (matches(sts, AM_TIMEOUT_ERR, matchedString)) { return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_TIMEOUT); - } else if (matches(sts, UNKNOWN_QUEUE_ERR) || matches(sts, STOPPED_QUEUE_ERR)) { - return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_INVALID_QUEUE, matchedString); - } else if (matches(sts, FULL_QUEUE_ERR)) { - return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_QUEUE_FULL, matchedString); - } else if (matches(sts, INVALILD_MEM_ERR) || matches(sts, INVALID_CORE_ERR)) { + } else if (matches(sts, UNKNOWN_QUEUE_ERR, matchedString) || matches(sts, STOPPED_QUEUE_ERR, matchedString)) { + return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_INVALID_QUEUE, matchedString.toString()); + } else if (matches(sts, FULL_QUEUE_ERR, matchedString)) { + return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_QUEUE_FULL, matchedString.toString()); + } else if (matches(sts, INVALILD_MEM_ERR, matchedString) || matches(sts, INVALID_CORE_ERR, matchedString)) { return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_INVALID_RESOURCE_REQUEST, - matchedString); + matchedString.toString()); } else { - return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, sessionId); + return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, sessionId, Throwables.getRootCause(e).getMessage()); } } e = e.getCause(); } - return new HiveException(oe, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, sessionId); + return new HiveException(oe, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, sessionId, Throwables.getRootCause(oe).getMessage()); } - @VisibleForTesting - String getMatchedString() { - return matchedString; - } - - private boolean matches(String input, String regex) { + private boolean matches(String input, String regex, StringBuilder matchedString) { if (!errorPatterns.containsKey(regex)) { LOG.warn("No error pattern found for regex: {}", regex); return false; @@ -235,7 +230,8 @@ public class SparkSessionImpl implements SparkSession { Matcher m = p.matcher(input); boolean result = m.find(); if (result && m.groupCount() == 1) { - this.matchedString = m.group(1); + // assume matchedString is empty + matchedString.append(m.group(1)); } return result; } http://git-wip-us.apache.org/repos/asf/hive/blob/3c4a9c6b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java index fe95ce0..5924b8b 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java @@ -183,8 +183,8 @@ public class TestSparkSessionManagerImpl { "initial executor number 5 must between min executor number10 and max executor number 50"); // Other exceptions which defaults to SPARK_CREATE_CLIENT_ERROR - e = new Exception(); - checkHiveException(ss, e, ErrorMsg.SPARK_CREATE_CLIENT_ERROR); + e = new Exception("Other exception"); + checkHiveException(ss, e, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, "Other exception"); } private void checkHiveException(SparkSessionImpl ss, Throwable e, ErrorMsg expectedErrMsg) { @@ -196,7 +196,7 @@ public class TestSparkSessionManagerImpl { HiveException he = ss.getHiveException(e); assertEquals(expectedErrMsg, he.getCanonicalErrorMsg()); if (expectedMatchedStr != null) { - assertEquals(expectedMatchedStr, ss.getMatchedString()); + assertTrue(he.getMessage().contains(expectedMatchedStr)); } }
