Author: omalley
Date: Tue Mar 8 06:00:01 2011
New Revision: 1079246
URL: http://svn.apache.org/viewvc?rev=1079246&view=rev
Log:
commit a84bbbe1ae10f1c01613641c2b77c8371ef59c73
Author: Ravi Gummadi <[email protected]>
Date: Fri Jan 14 14:58:17 2011 +0530
: Fix Gridmix to handle distributed cache files with out
visibilities available in trace file. Patch is available at
(gravi)
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestDistCacheEmulation.java
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java?rev=1079246&r1=1079245&r2=1079246&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
Tue Mar 8 06:00:01 2011
@@ -326,7 +326,11 @@ class DistributedCacheEmulator {
FileSystem fs = FileSystem.get(conf);
String user = jobConf.getUser();
for (int i = 0; i < files.length; i++) {
- if (isLocalDistCacheFile(files[i], user, visibilities[i])) {
+ // Check if visibilities are available because older hadoop versions
+ // didn't have public, private Distributed Caches separately.
+ boolean visibility =
+ (visibilities == null) ? true : Boolean.valueOf(visibilities[i]);
+ if (isLocalDistCacheFile(files[i], user, visibility)) {
// local FS based dist cache file.
// Create this file on the pseudo local FS on the fly (i.e. when the
// simulated job is submitted).
@@ -334,7 +338,7 @@ class DistributedCacheEmulator {
}
// dist cache file on hdfs
String mappedPath = mapDistCacheFilePath(files[i], timeStamps[i],
- Boolean.valueOf(visibilities[i]), user);
+ visibility, user);
// No need to add a dist cache file path to the list if
// (1) the mapped path is already there in the list OR
@@ -363,9 +367,8 @@ class DistributedCacheEmulator {
* distributed cache file
*/
private boolean isLocalDistCacheFile(String filePath, String user,
- String visibility) {
- return (!Boolean.valueOf(visibility)
- && filePath.contains(user + "/.staging"));
+ boolean visibility) {
+ return (!visibility && filePath.contains(user + "/.staging"));
}
/**
@@ -500,7 +503,11 @@ class DistributedCacheEmulator {
String user = jobConf.getUser();
for (int i = 0; i < files.length; i++) {
- if (isLocalDistCacheFile(files[i], user, visibilities[i])) {
+ // Check if visibilities are available because older hadoop versions
+ // didn't have public, private Distributed Caches separately.
+ boolean visibility =
+ (visibilities == null) ? true : Boolean.valueOf(visibilities[i]);
+ if (isLocalDistCacheFile(files[i], user, visibility)) {
// local FS based dist cache file.
// Create this file on the pseudo local FS.
String fileId = MD5Hash.digest(files[i] +
timeStamps[i]).toString();
@@ -515,7 +522,7 @@ class DistributedCacheEmulator {
// hdfs based dist cache file.
// Get the mapped HDFS path on simulated cluster
String mappedPath = mapDistCacheFilePath(files[i], timeStamps[i],
- Boolean.valueOf(visibilities[i]), user);
+ visibility, user);
cacheFiles.add(mappedPath);
}
}
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestDistCacheEmulation.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestDistCacheEmulation.java?rev=1079246&r1=1079245&r2=1079246&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestDistCacheEmulation.java
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestDistCacheEmulation.java
Tue Mar 8 06:00:01 2011
@@ -451,6 +451,30 @@ public class TestDistCacheEmulation {
}
/**
+ * Verify if configureDistCacheFiles() works fine when there are distributed
+ * cache files set but visibilities are not set. This is to handle history
+ * traces of older hadoop version where there are no private/public
+ * Distributed Caches.
+ * @throws IOException
+ */
+ private void validateWithOutVisibilities() throws IOException {
+ Configuration conf = new Configuration();// configuration for simulated job
+ JobConf jobConf = new JobConf();
+ String user = "user1";
+ jobConf.setUser(user);
+ String[] files = {"/tmp/hdfs1.txt", "/tmp/"+ user + "/.staging/file1"};
+ jobConf.setStrings(MRJobConfig.CACHE_FILES, files);
+ jobConf.setStrings(MRJobConfig.CACHE_FILES_SIZES, "12,200");
+ jobConf.setStrings(MRJobConfig.CACHE_FILE_TIMESTAMPS, "56789,98345");
+ dce.configureDistCacheFiles(conf, jobConf);
+ assertEquals("Configuring of HDFS-based dist cache files by gridmix is "
+ + "wrong.", files.length,
+ conf.getStrings(MRJobConfig.CACHE_FILES).length);
+ assertNull("Configuring of local-FS-based dist cache files by gridmix is "
+ + "wrong.", conf.get("tmpfiles"));
+ }
+
+ /**
* Test if Gridmix can configure config properties related to Distributed
* Cache properly. Also verify if Gridmix can handle deprecated config
* properties related to Distributed Cache.
@@ -472,11 +496,18 @@ public class TestDistCacheEmulation {
+ DistributedCacheEmulator.GRIDMIX_EMULATE_DISTRIBUTEDCACHE
+ " is wrong.", dce.shouldEmulateDistCacheLoad());
+ // Validate if DistributedCacheEmulator can handle a JobStory with out
+ // Distributed Cache files properly.
validateJobConfWithOutDCFiles(conf, jobConf);
// Validate if Gridmix can configure dist cache files properly if there are
// HDFS-based dist cache files and localFS-based dist cache files in trace
// for a job. Set old config properties and validate.
validateJobConfWithDCFiles(conf, jobConf);
+
+ // Use new JobConf as JobStory conf and check if configureDistCacheFiles()
+ // doesn't throw NPE when there are dist cache files set but visibilities
+ // are not set.
+ validateWithOutVisibilities();
}
}
