Repository: tez Updated Branches: refs/heads/branch-0.7 e849f0981 -> 4a2316ba8
TEZ-3559. TEZ_LIB_URIS doesn't work with schemes different than the defaultFS (Eric Badge via jeagles) (cherry picked from commit 4f9b7fdf2049c2a96d6c6cb047abb91c58cf2737) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/4a2316ba Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/4a2316ba Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/4a2316ba Branch: refs/heads/branch-0.7 Commit: 4a2316ba803e5058f0724a3d5efc16c6c4895679 Parents: e849f09 Author: Jonathan Eagles <[email protected]> Authored: Thu Dec 15 17:31:12 2016 -0600 Committer: Jonathan Eagles <[email protected]> Committed: Thu Dec 15 17:42:54 2016 -0600 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../org/apache/tez/client/TezClientUtils.java | 5 ++-- .../apache/tez/client/TestTezClientUtils.java | 31 ++++++++++++++++++++ 3 files changed, 34 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/4a2316ba/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index c652351..7204600 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3559. TEZ_LIB_URIS doesn't work with schemes different than the defaultFS TEZ-3549. TaskAttemptImpl does not initialize TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS correctly TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768 to Tez TEZ-3507. Task logs link when editing url from one task to another. http://git-wip-us.apache.org/repos/asf/tez/blob/4a2316ba/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java index 7a5bf7c..67e9979 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java @@ -269,7 +269,7 @@ public class TezClientUtils { } LocalResourceVisibility lrVisibility; - if (checkAncestorPermissionsForAllUsers(conf, url.getFile(), + if (checkAncestorPermissionsForAllUsers(conf, p, FsAction.EXECUTE) && fStatus.getPermission().getOtherAction().implies(FsAction.READ)) { lrVisibility = LocalResourceVisibility.PUBLIC; @@ -1004,9 +1004,8 @@ public class TezClientUtils { + ( javaOpts != null ? javaOpts : ""); } - private static boolean checkAncestorPermissionsForAllUsers(Configuration conf, String uri, + private static boolean checkAncestorPermissionsForAllUsers(Configuration conf, Path pathComponent, FsAction permission) throws IOException { - Path pathComponent = new Path(uri); FileSystem fs = pathComponent.getFileSystem(conf); if (Shell.WINDOWS && fs instanceof LocalFileSystem) { http://git-wip-us.apache.org/repos/asf/tez/blob/4a2316ba/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java index c630db4..efa6ed4 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java @@ -209,6 +209,37 @@ public class TestTezClientUtils { assertFalse(localizedMap.isEmpty()); } + + /** + * + */ + @Test (timeout=5000) + public void testTezDefaultFS() throws Exception { + FileSystem localFs = FileSystem.getLocal(new Configuration()); + StringBuilder tezLibUris = new StringBuilder(); + + Path topDir = new Path(TEST_ROOT_DIR, "testTezDefaultFS"); + if (localFs.exists(topDir)) { + localFs.delete(topDir, true); + } + localFs.mkdirs(topDir); + + Path file = new Path(topDir, "file.jar"); + + Assert.assertTrue(localFs.createNewFile(file)); + tezLibUris.append(localFs.makeQualified(file).toString()); + + TezConfiguration conf = new TezConfiguration(); + conf.set(TezConfiguration.TEZ_LIB_URIS, tezLibUris.toString()); + conf.set("fs.defaultFS", "hdfs:///localhost:1234"); + Credentials credentials = new Credentials(); + Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>(); + TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap); + + Assert.assertTrue(localFs.delete(file, true)); + Assert.assertTrue(localFs.delete(topDir, true)); + } + /** * */
