Author: vinodkv
Date: Wed Apr 10 21:59:51 2013
New Revision: 1466705
URL: http://svn.apache.org/r1466705
Log:
HADOOP-8731. Fixed TrackerDistributedCacheManager to address failure in the
test TestTrackerDistributedCacheManager on Windows. Contributed by Ivan Mitic.
Modified:
hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1466705&r1=1466704&r2=1466705&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Wed Apr 10
21:59:51 2013
@@ -255,6 +255,9 @@ Branch-hadoop-1-win (branched from branc
HADOOP-9463. branch-1-win fails to build with OpenJDK7.
(Ivan Mitic via suresh)
+ HADOOP-8731. Fixed TrackerDistributedCacheManager to address failure in the
+ test TestTrackerDistributedCacheManager on Windows. (Ivan Mitic via
vinodkv)
+
Merged from branch-1
HDFS-385. Backport: Add support for an experimental API that allows a
Modified:
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java?rev=1466705&r1=1466704&r2=1466705&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
(original)
+++
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
Wed Apr 10 21:59:51 2013
@@ -56,6 +56,7 @@ import org.apache.hadoop.mapreduce.JobID
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.RunJar;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.mapreduce.security.TokenCache;
/**
@@ -326,7 +327,17 @@ public class TrackerDistributedCacheMana
if (!checkPermissionOfOther(fs, current, FsAction.READ)) {
return false;
}
- return ancestorsHaveExecutePermissions(fs, current.getParent());
+ if (Shell.WINDOWS && fs instanceof LocalFileSystem) {
+ // Relax the requirement for public cache on LFS on Windows since default
+ // permissions are "700" all the way up to the drive letter. In this
+ // model, the only requirement for a user is to give EVERYONE group
+ // permission on the file and the file will be considered public.
+ // This code path is only hit when fs.default.name is file:/// (mainly
+ // in test).
+ return true;
+ } else {
+ return ancestorsHaveExecutePermissions(fs, current.getParent());
+ }
}
/**
@@ -425,7 +436,6 @@ public class TrackerDistributedCacheMana
}
Path workFile = new Path(workDir, parchive.getName());
sourceFs.copyToLocalFile(sourcePath, workFile);
- localFs.setPermission(workFile, permission);
if (isArchive) {
String tmpArchive = workFile.getName().toLowerCase();
File srcFile = new File(workFile.toString());
@@ -445,7 +455,6 @@ public class TrackerDistributedCacheMana
// else will not do anyhting
// and copy the file into the dir as it is
}
- FileUtil.chmod(destDir.toString(), "ugo+rx", true);
}
// promote the output to the final location
if (!localFs.rename(workDir, finalDir)) {
@@ -458,6 +467,15 @@ public class TrackerDistributedCacheMana
return 0;
}
+ // chmod after the above rename operation since rename does not transfer
+ // the original permissions to the target on all platforms
+ if (isArchive) {
+ FileUtil.chmod(finalDir.toString(), "ugo+rx", true);
+ } else {
+ Path finalWorkFile = new Path(finalDir, parchive.getName());
+ localFs.setPermission(finalWorkFile, permission);
+ }
+
LOG.info(String.format("Cached %s as %s",
source.toString(), destination.toString()));
long cacheSize =
Modified:
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java?rev=1466705&r1=1466704&r2=1466705&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
(original)
+++
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
Wed Apr 10 21:59:51 2013
@@ -56,6 +56,7 @@ import org.apache.hadoop.filecache.TaskD
import org.apache.hadoop.filecache.TrackerDistributedCacheManager;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Shell;
public class TestTrackerDistributedCacheManager extends TestCase {
private static final Log LOG =
@@ -95,19 +96,20 @@ public class TestTrackerDistributedCache
if (!TEST_ROOT.exists()) {
TEST_ROOT.mkdirs();
}
+ FileUtil.setPermission(TEST_ROOT, new FsPermission("755"));
conf = new Configuration();
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");
fs = FileSystem.get(conf);
// This test suite will fail if any ancestor directory of the
- // test directory is not world-searchable (ie +x).
+ // test directory is not public.
// We prefer to fail the test in an obvious manner up front
// during setUp() rather than in a subtle way later.
assertTrue("Test root directory " + TEST_ROOT + " and all of its " +
"parent directories must have a+x permissions",
- TrackerDistributedCacheManager.ancestorsHaveExecutePermissions(
- fs, new Path(TEST_ROOT.toString())));
+ TrackerDistributedCacheManager.isPublic(
+ conf, new Path(TEST_ROOT.toString()).toUri()));
// Prepare the tests' mapred-local-dir
ROOT_MAPRED_LOCAL_DIR = new File(TEST_ROOT_DIR, "mapred/local");
