Author: cdouglas
Date: Thu Mar 31 08:12:48 2011
New Revision: 1087209
URL: http://svn.apache.org/viewvc?rev=1087209&view=rev
Log:
MAPREDUCE-2409. Distinguish distributed cache artifacts localized as
files, archives. Contributed by Siddharth Seth
Modified:
hadoop/common/branches/branch-0.20-security/CHANGES.txt
hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
Modified: hadoop/common/branches/branch-0.20-security/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/CHANGES.txt?rev=1087209&r1=1087208&r2=1087209&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security/CHANGES.txt Thu Mar 31 08:12:48
2011
@@ -18,7 +18,10 @@ Release 0.20.204.0 - unreleased
HDFS-1773. Do not show decommissioned datanodes, which are not in both
include and exclude lists, on web and JMX interfaces.
(Tanping Wang via szetszwo)
-
+
+ MAPREDUCE-2409. Distinguish distributed cache artifacts localized as
+ files, archives. (Siddharth Seth via cdouglas)
+
IMPROVEMENTS
HDFS-1541. Not marking datanodes dead when namenode in safemode.
Modified:
hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java?rev=1087209&r1=1087208&r2=1087209&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
(original)
+++
hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
Thu Mar 31 08:12:48 2011
@@ -147,7 +147,7 @@ public class TrackerDistributedCacheMana
boolean isPublic, CacheFile file) throws IOException {
String key;
String user = getLocalizedCacheOwner(isPublic);
- key = getKey(cache, conf, confFileStamp, user);
+ key = getKey(cache, conf, confFileStamp, user, isArchive);
CacheStatus lcacheStatus;
Path localizedPath = null;
Path localPath = null;
@@ -573,11 +573,12 @@ public class TrackerDistributedCacheMana
return true;
}
- String getKey(URI cache, Configuration conf, long timeStamp, String user)
- throws IOException {
- return makeRelative(cache, conf) + String.valueOf(timeStamp) + user;
+ String getKey(URI cache, Configuration conf, long timeStamp, String user,
+ boolean isArchive) throws IOException {
+ return (isArchive ? "a" : "f") + "^" + makeRelative(cache, conf)
+ + String.valueOf(timeStamp) + user;
}
-
+
/**
* This method create symlinks for all files in a given dir in another
* directory.
Modified:
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java?rev=1087209&r1=1087208&r2=1087209&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
(original)
+++
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
Thu Mar 31 08:12:48 2011
@@ -319,7 +319,51 @@ public class TestTrackerDistributedCache
checkLocalizedPath(true);
checkLocalizedPath(false);
}
-
+
+ public void testSameNameFileArchiveCache() throws IOException,
+ InterruptedException {
+ if (!canRun()) {
+ return;
+ }
+ TrackerDistributedCacheManager manager = new
TrackerDistributedCacheManager(
+ conf, taskController);
+ String userName = getJobOwnerName();
+ File workDir = new File(TEST_ROOT_DIR, "workdir");
+ Path cacheFile = new Path(TEST_ROOT_DIR, "fileArchiveCacheFile");
+
+ createPublicTempFile(cacheFile);
+ Configuration conf1 = new Configuration(conf);
+ conf1.set("user.name", userName);
+
+ DistributedCache.addCacheFile(cacheFile.toUri(), conf1);
+ DistributedCache.addCacheArchive(cacheFile.toUri(), conf1);
+ TrackerDistributedCacheManager.determineTimestamps(conf1);
+ TrackerDistributedCacheManager.determineCacheVisibilities(conf1);
+ dumpState(conf1);
+
+ TaskDistributedCacheManager handle = manager
+ .newTaskDistributedCacheManager(new JobID("jt", 1), conf1);
+ handle.setupCache(conf1, TaskTracker.getPublicDistributedCacheDir(),
+ TaskTracker.getPrivateDistributedCacheDir(userName));
+
+ TaskDistributedCacheManager.CacheFile cFile =
handle.getCacheFiles().get(0);
+ TaskDistributedCacheManager.CacheFile cArchive = handle.getCacheFiles()
+ .get(1);
+
+ String distCacheDir = TaskTracker.getPublicDistributedCacheDir();
+
+ Path localizedPathForFile = manager.getLocalCache(cacheFile.toUri(), conf1,
+ distCacheDir, fs.getFileStatus(cacheFile), false, cFile.timestamp,
+ true, cFile);
+
+ Path localizedPathForArchive = manager.getLocalCache(cacheFile.toUri(),
+ conf1, distCacheDir, fs.getFileStatus(cacheFile), true,
+ cArchive.timestamp, true, cArchive);
+ assertNotSame("File and Archive resolve to the same path: "
+ + localizedPathForFile + ". Should differ.", localizedPathForFile,
+ localizedPathForArchive);
+ }
+
private void appendStringArray(StringBuilder buffer, String[] data) {
if (data != null && data.length != 0) {
buffer.append(data[0]);
