[
https://issues.apache.org/jira/browse/KYLIN-4663?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
xieqifeng updated KYLIN-4663:
-----------------------------
Description:
{code:java}
stderr:
YARN Diagnostics:
spark-submit start failed
at
org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:193)
at
org.apache.kylin.job.impl.threadpool.DefaultScheduler$JobRunner.run(DefaultScheduler.java:114)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kylin.job.exception.ExecuteException:
java.lang.RuntimeException: livy execute failed.
livy get status failed. state is dead. log is at
scala.Option.map(Option.scala:146)
at
org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:366)
at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:143)
at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:924)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:933)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
stderr:
YARN Diagnostics:
spark-submit start failed
at
org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:193)
at
org.apache.kylin.job.execution.DefaultChainedExecutable.doWork(DefaultChainedExecutable.java:71)
at
org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:178)
... 4 more
Caused by: java.lang.RuntimeException: livy execute failed.
livy get status failed. state is dead. log is at
scala.Option.map(Option.scala:146)
at
org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:366)
at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:143)
at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:924)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:933)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
stderr:
YARN Diagnostics:
spark-submit start failed
at
org.apache.kylin.common.livy.LivyRestExecutor.execute(LivyRestExecutor.java:85)
at
org.apache.kylin.source.hive.MRHiveDictUtil.runLivySqlJob(MRHiveDictUtil.java:225)
at
org.apache.kylin.source.hive.CreateFlatHiveTableByLivyStep.createFlatHiveTable(CreateFlatHiveTableByLivyStep.java:45)
at
org.apache.kylin.source.hive.CreateFlatHiveTableByLivyStep.doWork(CreateFlatHiveTableByLivyStep.java:60)
at
org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:178)
... 6 more
{code}
{code:java}
kylin.engine.livy-conf.livy-key.file=hdfs:///kylin/kylin-job-3.0.0-SNAPSHOT.jar
kylin.engine.livy-conf.livy-arr.jars=hdfs:///kylin/hbase-client-2.0.0-cdh6.0.1.jar,hdfs:///kylin/hbase-common-2.0.0-cdh6.0.1.jar,hdfs:///kylin/hbase-hadoop-compat-2.0.0-cdh6.0.1.jar,hdfs:///kylin/hbase-hadoop2-compat-2.0.0-cdh6.0.1.jar,hdfs:///kylin/hbase-server-2.0.0-cdh6.0.1.jar,hdfs:///kylin/htrace-core-3.2.0-incubating.jar,hdfs:///kylin/metrics-core-3.2.0.jar
{code}
was:
{{stderr:
YARN Diagnostics:
spark-submit start failed
at
org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:193)
at
org.apache.kylin.job.impl.threadpool.DefaultScheduler$JobRunner.run(DefaultScheduler.java:114)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kylin.job.exception.ExecuteException:
java.lang.RuntimeException: livy execute failed.
livy get status failed. state is dead. log is at
scala.Option.map(Option.scala:146)
at
org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:366)
at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:143)
at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
at
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:924)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:933)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
stderr:
YARN Diagnostics:
spark-submit start failed
at
org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:193)
at
org.apache.kylin.job.execution.DefaultChainedExecutable.doWork(DefaultChainedExecutable.java:71)
at
org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:178)
... 4 more
Caused by: java.lang.RuntimeException: livy execute failed.
livy get status failed. state is dead. log is at
scala.Option.map(Option.scala:146)
at
org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:366)
at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:143)
at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
at
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:924)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:933)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
stderr:
YARN Diagnostics:
spark-submit start failed
at
org.apache.kylin.common.livy.LivyRestExecutor.execute(LivyRestExecutor.java:85)
at
org.apache.kylin.source.hive.MRHiveDictUtil.runLivySqlJob(MRHiveDictUtil.java:225)
at
org.apache.kylin.source.hive.CreateFlatHiveTableByLivyStep.createFlatHiveTable(CreateFlatHiveTableByLivyStep.java:45)
at
org.apache.kylin.source.hive.CreateFlatHiveTableByLivyStep.doWork(CreateFlatHiveTableByLivyStep.java:60)
at
org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:178)
... 6 more}}
> in kylin 3.1.0,use livy submit spark can't get the clear information from log
> -----------------------------------------------------------------------------
>
> Key: KYLIN-4663
> URL: https://issues.apache.org/jira/browse/KYLIN-4663
> Project: Kylin
> Issue Type: Improvement
> Components: Spark Engine
> Affects Versions: v3.1.0
> Reporter: xieqifeng
> Priority: Major
>
> {code:java}
> stderr:
> YARN Diagnostics:
> spark-submit start failed
> at
> org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:193)
> at
> org.apache.kylin.job.impl.threadpool.DefaultScheduler$JobRunner.run(DefaultScheduler.java:114)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: org.apache.kylin.job.exception.ExecuteException:
> java.lang.RuntimeException: livy execute failed.
> livy get status failed. state is dead. log is at
> scala.Option.map(Option.scala:146)
> at
> org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:366)
> at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:143)
> at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
> at
> org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:924)
> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:933)
> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> stderr:
> YARN Diagnostics:
> spark-submit start failed
> at
> org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:193)
> at
> org.apache.kylin.job.execution.DefaultChainedExecutable.doWork(DefaultChainedExecutable.java:71)
> at
> org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:178)
> ... 4 more
> Caused by: java.lang.RuntimeException: livy execute failed.
> livy get status failed. state is dead. log is at
> scala.Option.map(Option.scala:146)
> at
> org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:366)
> at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:143)
> at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
> at
> org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:924)
> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:933)
> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> stderr:
> YARN Diagnostics:
> spark-submit start failed
> at
> org.apache.kylin.common.livy.LivyRestExecutor.execute(LivyRestExecutor.java:85)
> at
> org.apache.kylin.source.hive.MRHiveDictUtil.runLivySqlJob(MRHiveDictUtil.java:225)
> at
> org.apache.kylin.source.hive.CreateFlatHiveTableByLivyStep.createFlatHiveTable(CreateFlatHiveTableByLivyStep.java:45)
> at
> org.apache.kylin.source.hive.CreateFlatHiveTableByLivyStep.doWork(CreateFlatHiveTableByLivyStep.java:60)
> at
> org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:178)
> ... 6 more
> {code}
> {code:java}
> kylin.engine.livy-conf.livy-key.file=hdfs:///kylin/kylin-job-3.0.0-SNAPSHOT.jar
> kylin.engine.livy-conf.livy-arr.jars=hdfs:///kylin/hbase-client-2.0.0-cdh6.0.1.jar,hdfs:///kylin/hbase-common-2.0.0-cdh6.0.1.jar,hdfs:///kylin/hbase-hadoop-compat-2.0.0-cdh6.0.1.jar,hdfs:///kylin/hbase-hadoop2-compat-2.0.0-cdh6.0.1.jar,hdfs:///kylin/hbase-server-2.0.0-cdh6.0.1.jar,hdfs:///kylin/htrace-core-3.2.0-incubating.jar,hdfs:///kylin/metrics-core-3.2.0.jar
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
