Repository: spark Updated Branches: refs/heads/master 205e6d586 -> 06a13ecca
[SPARK-16808][CORE] History Server main page does not honor APPLICATION_WEB_PROXY_BASE ## What changes were proposed in this pull request? Application links generated on the history server UI no longer (regression from 1.6) contain the configured spark.ui.proxyBase in the links. To address this, made the uiRoot available globally to all javascripts for Web UI. Updated the mustache template (historypage-template.html) to include the uiroot for rendering links to the applications. The existing test was not sufficient to verify the scenario where ajax call is used to populate the application listing template, so added a new selenium test case to cover this scenario. ## How was this patch tested? Existing tests and a new unit test. No visual changes to the UI. Author: Vinayak <[email protected]> Closes #15742 from vijoshi/SPARK-16808_master. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/06a13ecc Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/06a13ecc Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/06a13ecc Branch: refs/heads/master Commit: 06a13ecca728e431c66fafb333b3bcff808e1afd Parents: 205e6d5 Author: Vinayak <[email protected]> Authored: Wed Nov 9 10:40:14 2016 -0800 Committer: Marcelo Vanzin <[email protected]> Committed: Wed Nov 9 10:40:14 2016 -0800 ---------------------------------------------------------------------- .../spark/ui/static/historypage-template.html | 6 +- .../org/apache/spark/ui/static/historypage.js | 6 +- .../org/apache/spark/ui/static/webui.js | 6 ++ .../scala/org/apache/spark/ui/UIUtils.scala | 1 + .../deploy/history/HistoryServerSuite.scala | 80 +++++++++++++++++++- 5 files changed, 92 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/06a13ecc/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html ---------------------------------------------------------------------- diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html index 1fd6ef4..42e2d9a 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html @@ -68,16 +68,16 @@ <tbody> {{#applications}} <tr> - <td class="rowGroupColumn"><span title="{{id}}"><a href="/history/{{id}}/{{num}}/jobs/">{{id}}</a></span></td> + <td class="rowGroupColumn"><span title="{{id}}"><a href="{{uiroot}}/history/{{id}}/{{num}}/jobs/">{{id}}</a></span></td> <td class="rowGroupColumn">{{name}}</td> {{#attempts}} - <td class="attemptIDSpan"><a href="/history/{{id}}/{{attemptId}}/jobs/">{{attemptId}}</a></td> + <td class="attemptIDSpan"><a href="{{uiroot}}/history/{{id}}/{{attemptId}}/jobs/">{{attemptId}}</a></td> <td>{{startTime}}</td> <td>{{endTime}}</td> <td><span title="{{duration}}" class="durationClass">{{duration}}</span></td> <td>{{sparkUser}}</td> <td>{{lastUpdated}}</td> - <td><a href="/api/v1/applications/{{id}}/{{num}}/logs" class="btn btn-info btn-mini">Download</a></td> + <td><a href="{{uiroot}}/api/v1/applications/{{id}}/{{num}}/logs" class="btn btn-info btn-mini">Download</a></td> {{/attempts}} </tr> {{/applications}} http://git-wip-us.apache.org/repos/asf/spark/blob/06a13ecc/core/src/main/resources/org/apache/spark/ui/static/historypage.js ---------------------------------------------------------------------- diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js index 2a32e18..6c0ec8d 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js @@ -119,7 +119,11 @@ $(document).ready(function() { } } - var data = {"applications": array} + var data = { + "uiroot": uiRoot, + "applications": array + } + $.get("static/historypage-template.html", function(template) { historySummary.append(Mustache.render($(template).filter("#history-summary-template").html(),data)); var selector = "#history-summary-table"; http://git-wip-us.apache.org/repos/asf/spark/blob/06a13ecc/core/src/main/resources/org/apache/spark/ui/static/webui.js ---------------------------------------------------------------------- diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.js b/core/src/main/resources/org/apache/spark/ui/static/webui.js index e37307a..0fa1fcf 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/webui.js +++ b/core/src/main/resources/org/apache/spark/ui/static/webui.js @@ -15,6 +15,12 @@ * limitations under the License. */ +var uiRoot = ""; + +function setUIRoot(val) { + uiRoot = val; +} + function collapseTablePageLoad(name, table){ if (window.localStorage.getItem(name) == "true") { // Set it to false so that the click function can revert it http://git-wip-us.apache.org/repos/asf/spark/blob/06a13ecc/core/src/main/scala/org/apache/spark/ui/UIUtils.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index 66b097a..57f6f2f 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -171,6 +171,7 @@ private[spark] object UIUtils extends Logging { <script src={prependBaseUri("/static/timeline-view.js")}></script> <script src={prependBaseUri("/static/log-view.js")}></script> <script src={prependBaseUri("/static/webui.js")}></script> + <script>setUIRoot('{UIUtils.uiRoot}')</script> } def vizHeaderNodes: Seq[Node] = { http://git-wip-us.apache.org/repos/asf/spark/blob/06a13ecc/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala index a595bc1..715811a 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala @@ -29,6 +29,8 @@ import com.codahale.metrics.Counter import com.google.common.io.{ByteStreams, Files} import org.apache.commons.io.{FileUtils, IOUtils} import org.apache.hadoop.fs.{FileStatus, FileSystem, Path} +import org.eclipse.jetty.proxy.ProxyServlet +import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder} import org.json4s.JsonAST._ import org.json4s.jackson.JsonMethods import org.json4s.jackson.JsonMethods._ @@ -258,8 +260,7 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers getContentAndCode("foobar")._1 should be (HttpServletResponse.SC_NOT_FOUND) } - test("relative links are prefixed with uiRoot (spark.ui.proxyBase)") { - val proxyBaseBeforeTest = System.getProperty("spark.ui.proxyBase") + test("static relative links are prefixed with uiRoot (spark.ui.proxyBase)") { val uiRoot = Option(System.getenv("APPLICATION_WEB_PROXY_BASE")).getOrElse("/testwebproxybase") val page = new HistoryPage(server) val request = mock[HttpServletRequest] @@ -267,7 +268,6 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers // when System.setProperty("spark.ui.proxyBase", uiRoot) val response = page.render(request) - System.setProperty("spark.ui.proxyBase", Option(proxyBaseBeforeTest).getOrElse("")) // then val urls = response \\ "@href" map (_.toString) @@ -275,6 +275,80 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers all (siteRelativeLinks) should startWith (uiRoot) } + test("ajax rendered relative links are prefixed with uiRoot (spark.ui.proxyBase)") { + val uiRoot = "/testwebproxybase" + System.setProperty("spark.ui.proxyBase", uiRoot) + + server.stop() + + val conf = new SparkConf() + .set("spark.history.fs.logDirectory", logDir) + .set("spark.history.fs.update.interval", "0") + .set("spark.testing", "true") + + provider = new FsHistoryProvider(conf) + provider.checkForLogs() + val securityManager = new SecurityManager(conf) + + server = new HistoryServer(conf, provider, securityManager, 18080) + server.initialize() + server.bind() + + val port = server.boundPort + + val servlet = new ProxyServlet { + override def rewriteTarget(request: HttpServletRequest): String = { + // servlet acts like a proxy that redirects calls made on + // spark.ui.proxyBase context path to the normal servlet handlers operating off "/" + val sb = request.getRequestURL() + + if (request.getQueryString() != null) { + sb.append(s"?${request.getQueryString()}") + } + + val proxyidx = sb.indexOf(uiRoot) + sb.delete(proxyidx, proxyidx + uiRoot.length).toString + } + } + + val contextHandler = new ServletContextHandler + val holder = new ServletHolder(servlet) + contextHandler.setContextPath(uiRoot) + contextHandler.addServlet(holder, "/") + server.attachHandler(contextHandler) + + implicit val webDriver: WebDriver = new HtmlUnitDriver(true) { + getWebClient.getOptions.setThrowExceptionOnScriptError(false) + } + + try { + val url = s"http://localhost:$port"; + + go to s"$url$uiRoot" + + // expect the ajax call to finish in 5 seconds + implicitlyWait(org.scalatest.time.Span(5, org.scalatest.time.Seconds)) + + // once this findAll call returns, we know the ajax load of the table completed + findAll(ClassNameQuery("odd")) + + val links = findAll(TagNameQuery("a")) + .map(_.attribute("href")) + .filter(_.isDefined) + .map(_.get) + .filter(_.startsWith(url)).toList + + // there are atleast some URL links that were generated via javascript, + // and they all contain the spark.ui.proxyBase (uiRoot) + links.length should be > 4 + all(links) should startWith(url + uiRoot) + } finally { + contextHandler.stop() + quit() + } + + } + test("incomplete apps get refreshed") { implicit val webDriver: WebDriver = new HtmlUnitDriver --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
