Repository: tez Updated Branches: refs/heads/master 06a90db1a -> 4f9b7fdf2
TEZ-3559. TEZ_LIB_URIS doesn't work with schemes different than the defaultFS (Eric Badge via jeagles) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/4f9b7fdf Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/4f9b7fdf Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/4f9b7fdf Branch: refs/heads/master Commit: 4f9b7fdf2049c2a96d6c6cb047abb91c58cf2737 Parents: 06a90db Author: Jonathan Eagles <[email protected]> Authored: Thu Dec 15 17:31:12 2016 -0600 Committer: Jonathan Eagles <[email protected]> Committed: Thu Dec 15 17:32:58 2016 -0600 ---------------------------------------------------------------------- CHANGES.txt | 3 ++ .../org/apache/tez/client/TezClientUtils.java | 5 ++-- .../apache/tez/client/TestTezClientUtils.java | 31 ++++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/4f9b7fdf/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index cabe368..2c91a9e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3559. TEZ_LIB_URIS doesn't work with schemes different than the defaultFS TEZ-3558. CartesianProduct is missing from the ExampleDriver class TEZ-3549. TaskAttemptImpl does not initialize TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS correctly TEZ-3552. Shuffle split array when size-based sorting is turned off. @@ -158,6 +159,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3559. TEZ_LIB_URIS doesn't work with schemes different than the defaultFS TEZ-3549. TaskAttemptImpl does not initialize TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS correctly TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768 to Tez TEZ-3536. NPE in WebUIService start when host resolution fails. @@ -668,6 +670,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3559. TEZ_LIB_URIS doesn't work with schemes different than the defaultFS TEZ-3549. TaskAttemptImpl does not initialize TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS correctly TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768 to Tez TEZ-3536. NPE in WebUIService start when host resolution fails. http://git-wip-us.apache.org/repos/asf/tez/blob/4f9b7fdf/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java index f440b6f..9e040e2 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java @@ -270,7 +270,7 @@ public class TezClientUtils { } LocalResourceVisibility lrVisibility; - if (checkAncestorPermissionsForAllUsers(conf, url.getFile(), + if (checkAncestorPermissionsForAllUsers(conf, p, FsAction.EXECUTE) && fStatus.getPermission().getOtherAction().implies(FsAction.READ)) { lrVisibility = LocalResourceVisibility.PUBLIC; @@ -972,9 +972,8 @@ public class TezClientUtils { + ( javaOpts != null ? javaOpts : ""); } - private static boolean checkAncestorPermissionsForAllUsers(Configuration conf, String uri, + private static boolean checkAncestorPermissionsForAllUsers(Configuration conf, Path pathComponent, FsAction permission) throws IOException { - Path pathComponent = new Path(uri); FileSystem fs = pathComponent.getFileSystem(conf); if (Shell.WINDOWS && fs instanceof LocalFileSystem) { http://git-wip-us.apache.org/repos/asf/tez/blob/4f9b7fdf/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java index 49aae20..a5e9d3c 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java @@ -207,6 +207,37 @@ public class TestTezClientUtils { assertFalse(localizedMap.isEmpty()); } + + /** + * + */ + @Test (timeout=5000) + public void testTezDefaultFS() throws Exception { + FileSystem localFs = FileSystem.getLocal(new Configuration()); + StringBuilder tezLibUris = new StringBuilder(); + + Path topDir = new Path(TEST_ROOT_DIR, "testTezDefaultFS"); + if (localFs.exists(topDir)) { + localFs.delete(topDir, true); + } + localFs.mkdirs(topDir); + + Path file = new Path(topDir, "file.jar"); + + Assert.assertTrue(localFs.createNewFile(file)); + tezLibUris.append(localFs.makeQualified(file).toString()); + + TezConfiguration conf = new TezConfiguration(); + conf.set(TezConfiguration.TEZ_LIB_URIS, tezLibUris.toString()); + conf.set("fs.defaultFS", "hdfs:///localhost:1234"); + Credentials credentials = new Credentials(); + Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>(); + TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap); + + Assert.assertTrue(localFs.delete(file, true)); + Assert.assertTrue(localFs.delete(topDir, true)); + } + /** * */
