Repository: tez Updated Branches: refs/heads/branch-0.8 f232e6606 -> 6f8ddf429
TEZ-3559. TEZ_LIB_URIS doesn't work with schemes different than the defaultFS (Eric Badge via jeagles) (cherry picked from commit 4f9b7fdf2049c2a96d6c6cb047abb91c58cf2737) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/6f8ddf42 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/6f8ddf42 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/6f8ddf42 Branch: refs/heads/branch-0.8 Commit: 6f8ddf4290cbf592dd662f6fa6883ce6b10d3d03 Parents: f232e66 Author: Jonathan Eagles <[email protected]> Authored: Thu Dec 15 17:31:12 2016 -0600 Committer: Jonathan Eagles <[email protected]> Committed: Thu Dec 15 17:34:49 2016 -0600 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ .../org/apache/tez/client/TezClientUtils.java | 5 ++-- .../apache/tez/client/TestTezClientUtils.java | 31 ++++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/6f8ddf42/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index a224ffe..c740bf3 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3559. TEZ_LIB_URIS doesn't work with schemes different than the defaultFS TEZ-3549. TaskAttemptImpl does not initialize TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS correctly TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768 to Tez TEZ-3507. Task logs link when editing url from one task to another. @@ -534,6 +535,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3559. TEZ_LIB_URIS doesn't work with schemes different than the defaultFS TEZ-3549. TaskAttemptImpl does not initialize TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS correctly TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768 to Tez TEZ-3507. Task logs link when editing url from one task to another. http://git-wip-us.apache.org/repos/asf/tez/blob/6f8ddf42/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java index eb1a95e..1d286f8 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java @@ -272,7 +272,7 @@ public class TezClientUtils { } LocalResourceVisibility lrVisibility; - if (checkAncestorPermissionsForAllUsers(conf, url.getFile(), + if (checkAncestorPermissionsForAllUsers(conf, p, FsAction.EXECUTE) && fStatus.getPermission().getOtherAction().implies(FsAction.READ)) { lrVisibility = LocalResourceVisibility.PUBLIC; @@ -1015,9 +1015,8 @@ public class TezClientUtils { + ( javaOpts != null ? javaOpts : ""); } - private static boolean checkAncestorPermissionsForAllUsers(Configuration conf, String uri, + private static boolean checkAncestorPermissionsForAllUsers(Configuration conf, Path pathComponent, FsAction permission) throws IOException { - Path pathComponent = new Path(uri); FileSystem fs = pathComponent.getFileSystem(conf); if (Shell.WINDOWS && fs instanceof LocalFileSystem) { http://git-wip-us.apache.org/repos/asf/tez/blob/6f8ddf42/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java index 2c69d77..c1198d0 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java @@ -209,6 +209,37 @@ public class TestTezClientUtils { assertFalse(localizedMap.isEmpty()); } + + /** + * + */ + @Test (timeout=5000) + public void testTezDefaultFS() throws Exception { + FileSystem localFs = FileSystem.getLocal(new Configuration()); + StringBuilder tezLibUris = new StringBuilder(); + + Path topDir = new Path(TEST_ROOT_DIR, "testTezDefaultFS"); + if (localFs.exists(topDir)) { + localFs.delete(topDir, true); + } + localFs.mkdirs(topDir); + + Path file = new Path(topDir, "file.jar"); + + Assert.assertTrue(localFs.createNewFile(file)); + tezLibUris.append(localFs.makeQualified(file).toString()); + + TezConfiguration conf = new TezConfiguration(); + conf.set(TezConfiguration.TEZ_LIB_URIS, tezLibUris.toString()); + conf.set("fs.defaultFS", "hdfs:///localhost:1234"); + Credentials credentials = new Credentials(); + Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>(); + TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap); + + Assert.assertTrue(localFs.delete(file, true)); + Assert.assertTrue(localFs.delete(topDir, true)); + } + /** * */
