Repository: hive Updated Branches: refs/heads/master e8b87bfb0 -> d67d52c8d
HIVE-20488: SparkSubmitSparkClient#launchDriver should parse exceptions, not just errors (Bharathkrishna Guruvayoor Murali, reviewed by Sahil Takiar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d67d52c8 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d67d52c8 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d67d52c8 Branch: refs/heads/master Commit: d67d52c8d56d41f5d8ca5c605ee10373dafb5bd1 Parents: e8b87bf Author: Bharathkrishna Guruvayoor Murali <bhar...@cloudera.com> Authored: Thu Oct 18 10:36:02 2018 -0500 Committer: Sahil Takiar <stak...@cloudera.com> Committed: Thu Oct 18 10:36:41 2018 -0500 ---------------------------------------------------------------------- .../apache/hive/spark/client/SparkClientUtilities.java | 7 ++++++- .../apache/hive/spark/client/SparkSubmitSparkClient.java | 5 ++--- .../org/apache/hive/spark/client/TestSparkClient.java | 11 +++++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/d67d52c8/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java ---------------------------------------------------------------------- diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java index d3cb3dd..b434d8f 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java @@ -19,6 +19,7 @@ package org.apache.hive.spark.client; import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import java.io.File; @@ -49,7 +50,7 @@ public class SparkClientUtilities { public static final String HIVE_KRYO_REG_NAME = "org.apache.hive.spark.HiveKryoRegistrator"; private static final String HIVE_KRYO_REG_JAR_NAME = "hive-kryo-registrator"; - + private static final ImmutableList<String> ERROR_KEYWORDS = ImmutableList.of("error", "exception"); /** * Add new elements to the classpath. * @@ -186,4 +187,8 @@ public class SparkClientUtilities { Thread.currentThread().setContextClassLoader(newLoader); } } + + public static boolean containsErrorKeyword(String line) { + return ERROR_KEYWORDS.stream().anyMatch(x -> StringUtils.containsIgnoreCase(line, x)); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/d67d52c8/spark-client/src/main/java/org/apache/hive/spark/client/SparkSubmitSparkClient.java ---------------------------------------------------------------------- diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkSubmitSparkClient.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkSubmitSparkClient.java index 1879829..f42fee2 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkSubmitSparkClient.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkSubmitSparkClient.java @@ -34,8 +34,6 @@ import java.util.concurrent.FutureTask; import java.util.regex.Pattern; import java.util.regex.Matcher; -import org.apache.commons.lang3.StringUtils; - import org.apache.hadoop.hive.common.log.LogRedirector; import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; @@ -44,6 +42,7 @@ import org.apache.hive.spark.client.rpc.RpcServer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hive.spark.client.SparkClientUtilities.containsErrorKeyword; /** * Extends the {@link AbstractSparkClient} and launches a child process to run Spark's {@code @@ -228,7 +227,7 @@ class SparkSubmitSparkClient extends AbstractSparkClient { List<String> errorMessages = new ArrayList<>(); synchronized (childErrorLog) { for (String line : childErrorLog) { - if (StringUtils.containsIgnoreCase(line, "Error")) { + if (containsErrorKeyword(line)) { errorMessages.add("\"" + line + "\""); } } http://git-wip-us.apache.org/repos/asf/hive/blob/d67d52c8/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java ---------------------------------------------------------------------- diff --git a/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java b/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java index 996b24e..0e1557e 100644 --- a/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java +++ b/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java @@ -301,6 +301,17 @@ public class TestSparkClient { }); } + @Test + public void testErrorParsing() { + assertTrue(SparkClientUtilities.containsErrorKeyword("Error.. Test")); + assertTrue(SparkClientUtilities.containsErrorKeyword("This line has error..")); + assertTrue(SparkClientUtilities.containsErrorKeyword("Test that line has ExcePtion..")); + assertTrue(SparkClientUtilities.containsErrorKeyword("Here is eRRor in line..")); + assertTrue(SparkClientUtilities.containsErrorKeyword("Here is ExceptioNn in line..")); + assertTrue(SparkClientUtilities.containsErrorKeyword("Here is ERROR and Exception in line..")); + assertFalse(SparkClientUtilities.containsErrorKeyword("No problems in this line")); + } + private static final Logger LOG = LoggerFactory.getLogger(TestSparkClient.class); private <T extends Serializable> JobHandle.Listener<T> newListener() {
