Philipp Dallig created ZEPPELIN-4781: ----------------------------------------
Summary: SparkUI-Link is not displayed in ZeppelinUI Key: ZEPPELIN-4781 URL: https://issues.apache.org/jira/browse/ZEPPELIN-4781 Project: Zeppelin Issue Type: Bug Components: spark Affects Versions: 0.9.0 Reporter: Philipp Dallig Hi I remember Zeppelin showing a link to SparkUI when a Spark interpreter is running. Now the link is gone. versions: - zeppelin -> master branch - spark -> spark3 (Preview version) I am a little engaged in this problem. I added the following code to my local installation to debug the problem. {code:java} diff --git a/spark/spark3-shims/src/main/scala/org/apache/zeppelin/spark/Spark3Shims.java b/spark/spark3-shims/src/main/scala/org/apache/zeppelin/spark/Spark3Shims.java index b213041df..8930bc036 100644 --- a/spark/spark3-shims/src/main/scala/org/apache/zeppelin/spark/Spark3Shims.java +++ b/spark/spark3-shims/src/main/scala/org/apache/zeppelin/spark/Spark3Shims.java @@ -20,7 +20,9 @@ package org.apache.zeppelin.spark; import org.apache.commons.lang.StringUtils; import org.apache.spark.SparkContext; +import org.apache.spark.SparkFirehoseListener; import org.apache.spark.scheduler.SparkListener; +import org.apache.spark.scheduler.SparkListenerEvent; import org.apache.spark.scheduler.SparkListenerJobStart; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; @@ -30,6 +32,8 @@ import org.apache.spark.sql.types.StructType; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.ResultMessages; import org.apache.zeppelin.tabledata.TableDataUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; @@ -39,25 +43,39 @@ public class Spark3Shims extends SparkShims { private SparkSession sparkSession; + private static final Logger LOGGER = LoggerFactory.getLogger(Spark3Shims.class); + public Spark3Shims(Properties properties, Object entryPoint) { super(properties); this.sparkSession = (SparkSession) entryPoint; } + @Override public void setupSparkListener(final String master, final String sparkWebUrl, final InterpreterContext context) { SparkContext sc = SparkContext.getOrCreate(); + LOGGER.info("Spark3Shims - setupSparkListener"); sc.addSparkListener(new SparkListener() { @Override public void onJobStart(SparkListenerJobStart jobStart) { + LOGGER.info("SparkUI"); + LOGGER.info("spark.ui.enabled: {}", sc.getConf().getBoolean("spark.ui.enabled", true)); + LOGGER.info("zeppelin.spark.ui.hidden: {}", Boolean.parseBoolean(properties.getProperty("zeppelin.spark.ui.hidden", "false"))); if (sc.getConf().getBoolean("spark.ui.enabled", true) && !Boolean.parseBoolean(properties.getProperty("zeppelin.spark.ui.hidden", "false"))) { buildSparkJobUrl(master, sparkWebUrl, jobStart.jobId(), jobStart.properties(), context); } } }); + sc.addSparkListener(new SparkFirehoseListener( ) { + @Override + public void onEvent(SparkListenerEvent event) { + LOGGER.info("Event: {}", event.getClass()); + super.onEvent(event); + } + }); } @Override {code} The spark interpreter has now additional output. {code:none} INFO [2020-04-29 09:53:36,628] ({FIFOScheduler-interpreter_1800946569-Worker-1} SparkShims.java[loadShims]:61) - Initializing shims for Spark 3.x INFO [2020-04-29 09:53:36,629] ({FIFOScheduler-interpreter_1800946569-Worker-1} Spark3Shims.java[setupSparkListener]:58) - Spark3Shims - setupSparkListener INFO [2020-04-29 09:53:37,602] ({FIFOScheduler-interpreter_1800946569-Worker-1} AbstractScheduler.java[runJob]:152) - Job paragraph_1585936744837_-1110526008 finished by scheduler interpreter_1800946569 INFO [2020-04-29 09:53:46,570] ({spark-listener-group-shared} Spark3Shims.java[onEvent]:75) - Event: class org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate INFO [2020-04-29 09:53:47,840] ({spark-listener-group-shared} Spark3Shims.java[onEvent]:75) - Event: class org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate; {code} I hope that someone can help to fix this problem. It seems that the Event `jobStart` is never called by Spark. -- This message was sent by Atlassian Jira (v8.3.4#803005)