Repository: tez Updated Branches: refs/heads/master d227b99a4 -> 24a8a95bd
TEZ-3494. Support relative url for tez-ui.history-url.base config (hitesh via sree) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/24a8a95b Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/24a8a95b Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/24a8a95b Branch: refs/heads/master Commit: 24a8a95bd2093377154760cf57ebe4f56c6197f4 Parents: d227b99 Author: Sreenath Somarajapuram <[email protected]> Authored: Wed Jan 25 21:55:20 2017 +0530 Committer: Sreenath Somarajapuram <[email protected]> Committed: Wed Jan 25 21:55:20 2017 +0530 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../apache/tez/dag/api/TezConfiguration.java | 12 ++++++ .../tez/dag/app/rm/TaskSchedulerManager.java | 14 +++++-- .../dag/app/rm/TestTaskSchedulerManager.java | 42 +++++++++++++++----- 4 files changed, 56 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/24a8a95b/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 4d43bca..d6798ff 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -177,6 +177,7 @@ ALL CHANGES: TEZ-3583. Tez UI: UTs are flaky because of a dependency issue TEZ-3580. Tez UI: Pagination broken on queries page TEZ-3575. RM have started forwarding origin. Use that in AMWebController for CORS support + TEZ-3494. Support relative url for tez-ui.history-url.base config Release 0.8.5: Unreleased http://git-wip-us.apache.org/repos/asf/tez/blob/24a8a95b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index e27cdf8..8747616 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -1335,6 +1335,18 @@ public class TezConfiguration extends Configuration { TEZ_PREFIX + "yarn.ats.max.events.per.batch"; public static final int YARN_ATS_MAX_EVENTS_PER_BATCH_DEFAULT = 5; + /** + * Boolean value. Default true. + * Whether to fix the history url if it has not been configured correctly i.e. it does not have a + * scheme in the value. By default, the url will be prepended with a scheme (http) if there is + * none present. + */ + @Private + @ConfigurationScope(Scope.AM) + public static final String TEZ_AM_UI_HISTORY_URL_SCHEME_CHECK_ENABLED = + TEZ_PREFIX + "am.ui.history.url.scheme.check.enabled"; + public static final boolean TEZ_AM_UI_HISTORY_URL_SCHEME_CHECK_ENABLED_DEFAULT = true; + /** * Int value. Time, in milliseconds, to wait for an event before sending a batch to ATS. http://git-wip-us.apache.org/repos/asf/tez/blob/24a8a95b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java index e68c9b8..7989e5f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java @@ -988,9 +988,17 @@ public class TaskSchedulerManager extends AbstractService implements .replaceAll(HISTORY_URL_BASE, historyUrlBase) .replaceAll("([^:])/{2,}", "$1/"); - // make sure we have a valid scheme - if (!historyUrl.startsWith("http")) { - historyUrl = "http://"; + historyUrl; + // FIXME: we should not need this check in the first place. + // Might be better to remove it in the next release and not fix badly configured URLs. + boolean checkSchemePrefix = config.getBoolean( + TezConfiguration.TEZ_AM_UI_HISTORY_URL_SCHEME_CHECK_ENABLED, + TezConfiguration.TEZ_AM_UI_HISTORY_URL_SCHEME_CHECK_ENABLED_DEFAULT); + + if (checkSchemePrefix) { + // make sure we have a valid scheme + if (!historyUrl.startsWith("http")) { + historyUrl = "http://"; + historyUrl; + } } } http://git-wip-us.apache.org/repos/asf/tez/blob/24a8a95b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java index 791bb7f..66c209a 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java @@ -394,33 +394,55 @@ public class TestTaskSchedulerManager { // ensure history url is empty when timeline server is not the logging class conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://ui-host:9999";); - assertTrue("http://ui-host:9999/#/tez-app/TEST_APP_ID"; - .equals(schedulerHandler.getHistoryUrl())); + assertEquals("http://ui-host:9999/#/tez-app/TEST_APP_ID";, + schedulerHandler.getHistoryUrl()); // ensure the trailing / in history url is handled conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://ui-host:9998/";); - assertTrue("http://ui-host:9998/#/tez-app/TEST_APP_ID"; - .equals(schedulerHandler.getHistoryUrl())); + assertEquals("http://ui-host:9998/#/tez-app/TEST_APP_ID";, + schedulerHandler.getHistoryUrl()); // ensure missing scheme in history url is handled conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "ui-host:9998/"); - Assert.assertTrue("http://ui-host:9998/#/tez-app/TEST_APP_ID"; - .equals(schedulerHandler.getHistoryUrl())); + assertEquals("http://ui-host:9998/#/tez-app/TEST_APP_ID";, + schedulerHandler.getHistoryUrl()); // handle bad template ex without begining / conf.set(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE, "__HISTORY_URL_BASE__#/somepath"); - assertTrue("http://ui-host:9998/#/somepath"; - .equals(schedulerHandler.getHistoryUrl())); + assertEquals("http://ui-host:9998/#/somepath";, + schedulerHandler.getHistoryUrl()); conf.set(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE, "__HISTORY_URL_BASE__?viewPath=tez-app/__APPLICATION_ID__"); conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://localhost/ui/tez";); - assertTrue("http://localhost/ui/tez?viewPath=tez-app/TEST_APP_ID"; - .equals(schedulerHandler.getHistoryUrl())); + assertEquals("http://localhost/ui/tez?viewPath=tez-app/TEST_APP_ID";, + schedulerHandler.getHistoryUrl()); } + @Test (timeout = 5000) + public void testHistoryUrlWithoutScheme() throws Exception { + Configuration conf = schedulerHandler.appContext.getAMConf(); + final ApplicationId mockApplicationId = mock(ApplicationId.class); + doReturn("TEST_APP_ID").when(mockApplicationId).toString(); + doReturn(mockApplicationId).when(mockAppContext).getApplicationID(); + + conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "/foo/bar/"); + conf.setBoolean(TezConfiguration.TEZ_AM_UI_HISTORY_URL_SCHEME_CHECK_ENABLED, false); + assertEquals("/foo/bar/#/tez-app/TEST_APP_ID", + schedulerHandler.getHistoryUrl()); + + conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "ui-host:9998/foo/bar/"); + assertEquals("ui-host:9998/foo/bar/#/tez-app/TEST_APP_ID", + schedulerHandler.getHistoryUrl()); + + conf.setBoolean(TezConfiguration.TEZ_AM_UI_HISTORY_URL_SCHEME_CHECK_ENABLED, true); + conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "ui-host:9998/foo/bar/"); + assertEquals("http://ui-host:9998/foo/bar/#/tez-app/TEST_APP_ID";, + schedulerHandler.getHistoryUrl()); + } + @Test(timeout = 5000) public void testNoSchedulerSpecified() throws IOException { try {
