Repository: incubator-livy Updated Branches: refs/heads/master 4fb5516d2 -> 0442eb01a
[LIVY-495] Add thriftserver UI ## What changes were proposed in this pull request? The PR adds a new table in the Session tab dedicated to thrift-server sessions. The table contains the active sessions with a link to the corresponding Livy session. Moreover, the same information is also exposed through a REST endpoint (`"thriftserver/sessions"`). ## How was this patch tested? Manual tests. A screenshot of the UI is:  Author: Marco Gaido <mga...@hortonworks.com> Closes #114 from mgaido91/LIVY-495. Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/0442eb01 Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/0442eb01 Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/0442eb01 Branch: refs/heads/master Commit: 0442eb01aa293d540fdf23ddd86b86c4278b83b9 Parents: 4fb5516 Author: Marco Gaido <mga...@hortonworks.com> Authored: Thu Oct 25 12:44:20 2018 -0700 Committer: Marcelo Vanzin <van...@cloudera.com> Committed: Thu Oct 25 12:44:20 2018 -0700 ---------------------------------------------------------------------- .../livy/server/ui/static/js/all-sessions.js | 4 +- .../org/apache/livy/server/LivyServer.scala | 16 +++-- .../livy/server/ThriftServerFactory.scala | 6 ++ .../org/apache/livy/server/ui/UIServlet.scala | 64 ++++++++++++-------- .../ui/static/html/thrift-sessions-table.html | 51 ++++++++++++++++ .../livy/server/ui/static/js/thrift-sessions.js | 50 +++++++++++++++ .../thriftserver/ThriftServerFactoryImpl.scala | 7 +++ .../thriftserver/ui/ThriftJsonServlet.scala | 55 +++++++++++++++++ 8 files changed, 222 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/0442eb01/server/src/main/resources/org/apache/livy/server/ui/static/js/all-sessions.js ---------------------------------------------------------------------- diff --git a/server/src/main/resources/org/apache/livy/server/ui/static/js/all-sessions.js b/server/src/main/resources/org/apache/livy/server/ui/static/js/all-sessions.js index 64b06df..90de331 100644 --- a/server/src/main/resources/org/apache/livy/server/ui/static/js/all-sessions.js +++ b/server/src/main/resources/org/apache/livy/server/ui/static/js/all-sessions.js @@ -26,7 +26,7 @@ function loadSessionsTable(sessions) { tdWrap(session.kind) + tdWrap(session.state) + tdWrap(logLinks(session, "session")) + - "</tr>" + "</tr>" ); }); } @@ -39,7 +39,7 @@ function loadBatchesTable(sessions) { tdWrap(appIdLink(session)) + tdWrap(session.state) + tdWrap(logLinks(session, "batch")) + - "</tr>" + "</tr>" ); }); } http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/0442eb01/server/src/main/scala/org/apache/livy/server/LivyServer.scala ---------------------------------------------------------------------- diff --git a/server/src/main/scala/org/apache/livy/server/LivyServer.scala b/server/src/main/scala/org/apache/livy/server/LivyServer.scala index 525f560..5f3bfd1 100644 --- a/server/src/main/scala/org/apache/livy/server/LivyServer.scala +++ b/server/src/main/scala/org/apache/livy/server/LivyServer.scala @@ -93,6 +93,12 @@ class LivyServer extends Logging { livyConf.set(LIVY_SPARK_SCALA_VERSION.key, sparkScalaVersion(formattedSparkVersion, scalaVersionFromSparkSubmit, livyConf)) + val thriftServerFactory = if (livyConf.getBoolean(LivyConf.THRIFT_SERVER_ENABLED)) { + Some(ThriftServerFactory.getInstance) + } else { + None + } + if (UserGroupInformation.isSecurityEnabled) { // If Hadoop security is enabled, run kinit periodically. runKinit() should be called // before any Hadoop operation, otherwise Kerberos exception will be thrown. @@ -211,10 +217,13 @@ class LivyServer extends Logging { mount(context, batchServlet, "/batches/*") if (livyConf.getBoolean(UI_ENABLED)) { - val uiServlet = new UIServlet(basePath) + val uiServlet = new UIServlet(basePath, livyConf) mount(context, uiServlet, "/ui/*") mount(context, staticResourceServlet, "/static/*") mount(context, uiRedirectServlet(basePath + "/ui/"), "/*") + thriftServerFactory.foreach { factory => + mount(context, factory.getServlet(basePath), factory.getServletMappings: _*) + } } else { mount(context, uiRedirectServlet(basePath + "/metrics"), "/*") } @@ -278,9 +287,8 @@ class LivyServer extends Logging { } }) - if (livyConf.getBoolean(LivyConf.THRIFT_SERVER_ENABLED)) { - ThriftServerFactory.getInstance.start( - livyConf, interactiveSessionManager, sessionStore, accessManager) + thriftServerFactory.foreach { + _.start(livyConf, interactiveSessionManager, sessionStore, accessManager) } _serverUrl = Some(s"${server.protocol}://${server.host}:${server.port}") http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/0442eb01/server/src/main/scala/org/apache/livy/server/ThriftServerFactory.scala ---------------------------------------------------------------------- diff --git a/server/src/main/scala/org/apache/livy/server/ThriftServerFactory.scala b/server/src/main/scala/org/apache/livy/server/ThriftServerFactory.scala index b6f7d9d..00c3c9f 100644 --- a/server/src/main/scala/org/apache/livy/server/ThriftServerFactory.scala +++ b/server/src/main/scala/org/apache/livy/server/ThriftServerFactory.scala @@ -17,6 +17,8 @@ package org.apache.livy.server +import javax.servlet.Servlet + import org.apache.livy.LivyConf import org.apache.livy.server.recovery.SessionStore import org.apache.livy.sessions.InteractiveSessionManager @@ -30,6 +32,10 @@ trait ThriftServerFactory { livySessionManager: InteractiveSessionManager, sessionStore: SessionStore, accessManager: AccessManager): Unit + + def getServlet(basePath: String): Servlet + + def getServletMappings: Seq[String] } object ThriftServerFactory { http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/0442eb01/server/src/main/scala/org/apache/livy/server/ui/UIServlet.scala ---------------------------------------------------------------------- diff --git a/server/src/main/scala/org/apache/livy/server/ui/UIServlet.scala b/server/src/main/scala/org/apache/livy/server/ui/UIServlet.scala index 47d6eae..7cbd135 100644 --- a/server/src/main/scala/org/apache/livy/server/ui/UIServlet.scala +++ b/server/src/main/scala/org/apache/livy/server/ui/UIServlet.scala @@ -21,18 +21,40 @@ import scala.xml.Node import org.scalatra.ScalatraServlet -class UIServlet(val basePath: String) extends ScalatraServlet { +import org.apache.livy.LivyConf + +class UIServlet(val basePath: String, livyConf: LivyConf) extends ScalatraServlet { before() { contentType = "text/html" } - sealed trait Page { val name: String } + private trait Page { + val name: String + def getNavCrumbs: Seq[Node] = Seq.empty + } private case class SimplePage(name: String) extends Page - private case class AllSessionsPage(name: String = "Sessions") extends Page + + private case class AllSessionsPage(name: String = "Sessions") extends Page { + override def getNavCrumbs: Seq[Node] = <li class="active"><a href="#">Sessions</a></li> + } private case class SessionPage(id: Int) extends Page { val name: String = "Session " + id + override def getNavCrumbs: Seq[Node] = { + <li><a href={basePath + "/ui"}>Sessions</a></li> ++ + <li class="active"><a href="#">{name}</a></li> + } } private case class LogPage(sessionType: String, id: Int) extends Page { val sessionName: String = sessionType + " " + id val name: String = sessionName + " Log" + override def getNavCrumbs: Seq[Node] = { + val sessionLink = if (sessionType == "Session") { + basePath + "/ui/session/" + id + } else { + "#" + } + <li><a href={basePath + "/ui"}>Sessions</a></li> ++ + <li><a href={sessionLink}>{sessionName}</a></li> ++ + <li class="active"><a href="#">Log</a></li> + } } private def getHeader(pageName: String): Seq[Node] = @@ -55,7 +77,7 @@ class UIServlet(val basePath: String) extends ScalatraServlet { <title>Livy - {pageName}</title> </head> - private def wrapNavTabs(tabs: Seq[Node]): Seq[Node] = + private def wrapNavCrumbs(crumbs: Seq[Node]): Seq[Node] = <nav class="navbar navbar-default"> <div class="container-fluid"> <div class="navbar-header"> @@ -65,32 +87,12 @@ class UIServlet(val basePath: String) extends ScalatraServlet { </div> <div class="collapse navbar-collapse"> <ul class="nav navbar-nav"> - {tabs} + {crumbs} </ul> </div> </div> </nav> - private def getNavBar(page: Page): Seq[Node] = { - val tabs: Seq[Node] = page match { - case _: AllSessionsPage => <li class="active"><a href="#">Sessions</a></li> - case sessionPage: SessionPage => { - <li><a href={basePath + "/ui"}>Sessions</a></li> ++ - <li class="active"><a href="#">{sessionPage.name}</a></li> - } - case logPage: LogPage => { - val sessionLink = if (logPage.sessionType == "Session") { - basePath + "/ui/session/" + logPage.id - } else "#" - <li><a href={basePath + "/ui"}>Sessions</a></li> ++ - <li><a href={sessionLink}>{logPage.sessionName}</a></li> ++ - <li class="active"><a href="#">Log</a></li> - } - case _ => Seq.empty - } - wrapNavTabs(tabs) - } - private def createPage(pageInfo: Page, pageContents: Seq[Node]): Seq[Node] = <html> {getHeader(pageInfo.name)} @@ -102,15 +104,27 @@ class UIServlet(val basePath: String) extends ScalatraServlet { </body> </html> + private def getNavBar(page: Page): Seq[Node] = wrapNavCrumbs(page.getNavCrumbs) + notFound { createPage(SimplePage("404"), <h3>404 No Such Page</h3>) } + private def thriftSessionsTable: Seq[Node] = { + if (livyConf.getBoolean(LivyConf.THRIFT_SERVER_ENABLED)) { + <div id="thrift-sessions"></div> ++ + <script src={s"$basePath/static/js/thrift-sessions.js"}></script> + } else { + Seq.empty + } + } + get("/") { val content = <div id="all-sessions"> <div id="interactive-sessions"></div> <div id="batches"></div> + {thriftSessionsTable} <script src={basePath + "/static/js/all-sessions.js"}></script> </div> http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/0442eb01/thriftserver/server/src/main/resources/org/apache/livy/server/ui/static/html/thrift-sessions-table.html ---------------------------------------------------------------------- diff --git a/thriftserver/server/src/main/resources/org/apache/livy/server/ui/static/html/thrift-sessions-table.html b/thriftserver/server/src/main/resources/org/apache/livy/server/ui/static/html/thrift-sessions-table.html new file mode 100644 index 0000000..243d74d --- /dev/null +++ b/thriftserver/server/src/main/resources/org/apache/livy/server/ui/static/html/thrift-sessions-table.html @@ -0,0 +1,51 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<h4 class="sessions-template">JDBC/ODBC Open Sessions</h4> + +<table id="thrift-sessions-table" + class="table table-striped sessions-table sessions-template"> + <thead class="sessions-table-head"> + <tr> + <th> + <span data-toggle="tooltip" + title="JDBC/ODBC session Id for the session"> + Session Id + </span> + </th> + <th> + <span data-toggle="tooltip" + title="Livy Interactive Session Id for this session. Links to the Session Summary Page"> + Livy Session Id + </span> + </th> + <th> + <span data-toggle="tooltip" title="Remote user who submitted this session"> + Owner + </span> + </th> + <th> + <span data-toggle="tooltip" + title="Creation time of the session"> + Created At + </span> + </th> + </tr> + </thead> + <tbody class="sessions-table-body"> + </tbody> +</table> http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/0442eb01/thriftserver/server/src/main/resources/org/apache/livy/server/ui/static/js/thrift-sessions.js ---------------------------------------------------------------------- diff --git a/thriftserver/server/src/main/resources/org/apache/livy/server/ui/static/js/thrift-sessions.js b/thriftserver/server/src/main/resources/org/apache/livy/server/ui/static/js/thrift-sessions.js new file mode 100644 index 0000000..74f8572 --- /dev/null +++ b/thriftserver/server/src/main/resources/org/apache/livy/server/ui/static/js/thrift-sessions.js @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +function loadThriftSessionsTable(sessions) { + $.each(sessions, function(index, session) { + $("#thrift-sessions-table .sessions-table-body").append( + "<tr>" + + tdWrap(session.sessionId) + + tdWrap(uiLink("session/" + session.livySessionId, session.livySessionId)) + + tdWrap(session.owner) + + tdWrap(session.createdAt) + + "</tr>" + ); + }); +} + +var numSessions = 0; + +$(document).ready(function () { + var sessionsReq = $.getJSON(location.origin + prependBasePath("/thriftserver/sessions"), function(response) { + if (response && response.total > 0) { + $("#thrift-sessions").load(prependBasePath("/static/html/thrift-sessions-table.html .sessions-template"), function() { + loadThriftSessionsTable(response.sessions); + $("#thrift-sessions-table").DataTable(); + $('#thrift-sessions [data-toggle="tooltip"]').tooltip(); + }); + } + numSessions = response.total; + }); + + $.when(sessionsReq).done(function () { + if (numSessions == 0) { + $("#thrift-sessions").append('<h4>No open JDBC/ODBC sessions.</h4>'); + } + }); +}); http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/0442eb01/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ThriftServerFactoryImpl.scala ---------------------------------------------------------------------- diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ThriftServerFactoryImpl.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ThriftServerFactoryImpl.scala index 5cc5707..7669580 100644 --- a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ThriftServerFactoryImpl.scala +++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ThriftServerFactoryImpl.scala @@ -17,10 +17,13 @@ package org.apache.livy.thriftserver +import javax.servlet.Servlet + import org.apache.livy.LivyConf import org.apache.livy.server.{AccessManager, ThriftServerFactory} import org.apache.livy.server.recovery.SessionStore import org.apache.livy.sessions.InteractiveSessionManager +import org.apache.livy.thriftserver.ui.ThriftJsonServlet class ThriftServerFactoryImpl extends ThriftServerFactory { override def start( @@ -34,4 +37,8 @@ class ThriftServerFactoryImpl extends ThriftServerFactory { } LivyThriftServer.start(livyConf, livySessionManager, sessionStore, accessManager) } + + override def getServlet(basePath: String): Servlet = new ThriftJsonServlet(basePath) + + override def getServletMappings: Seq[String] = Seq("/thriftserver/*") } http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/0442eb01/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ui/ThriftJsonServlet.scala ---------------------------------------------------------------------- diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ui/ThriftJsonServlet.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ui/ThriftJsonServlet.scala new file mode 100644 index 0000000..a7db3e9 --- /dev/null +++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ui/ThriftJsonServlet.scala @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.livy.thriftserver.ui + +import java.text.SimpleDateFormat + +import org.apache.livy.server.JsonServlet +import org.apache.livy.thriftserver.LivyThriftServer + + +class ThriftJsonServlet(val basePath: String) extends JsonServlet { + + private val df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z") + + case class SessionInfo( + sessionId: String, + livySessionId: String, + owner: String, + createdAt: String) + + get("/sessions") { + val thriftSessions = LivyThriftServer.getInstance.map { server => + val sessionManager = server.getSessionManager() + sessionManager.getSessions.map { sessionHandle => + val info = sessionManager.getSessionInfo(sessionHandle) + SessionInfo(sessionHandle.getSessionId.toString, + sessionManager.livySessionId(sessionHandle).map(_.toString).getOrElse(""), + info.username, + df.format(info.creationTime)) + }.toSeq + }.getOrElse(Seq.empty) + val from = params.get("from").map(_.toInt).getOrElse(0) + val size = params.get("size").map(_.toInt).getOrElse(100) + + Map( + "from" -> from, + "total" -> thriftSessions.length, + "sessions" -> thriftSessions.view(from, from + size)) + } +}
