Author: vinodkv
Date: Tue Jul 30 20:04:37 2013
New Revision: 1508595
URL: http://svn.apache.org/r1508595
Log:
MAPREDUCE-5385. Fixed a bug with JobContext getCacheFiles API. Contributed by
Omkar Vinit Joshi.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/map/WrappedMapper.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1508595&r1=1508594&r2=1508595&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Jul 30
20:04:37 2013
@@ -197,6 +197,9 @@ Release 2.1.1-beta - UNRELEASED
BUG FIXES
+ MAPREDUCE-5385. Fixed a bug with JobContext getCacheFiles API. (Omkar Vinit
+ Joshi via vinodkv)
+
Release 2.1.0-beta - 2013-08-06
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java?rev=1508595&r1=1508594&r2=1508595&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java
Tue Jul 30 20:04:37 2013
@@ -89,22 +89,26 @@ public class TestMRWithDistributedCache
@Override
public void setup(Context context) throws IOException {
Configuration conf = context.getConfiguration();
- Path[] files = context.getLocalCacheFiles();
- Path[] archives = context.getLocalCacheArchives();
+ Path[] localFiles = context.getLocalCacheFiles();
+ URI[] files = context.getCacheFiles();
+ Path[] localArchives = context.getLocalCacheArchives();
+ URI[] archives = context.getCacheArchives();
FileSystem fs = LocalFileSystem.get(conf);
// Check that 2 files and 2 archives are present
+ TestCase.assertEquals(2, localFiles.length);
+ TestCase.assertEquals(2, localArchives.length);
TestCase.assertEquals(2, files.length);
TestCase.assertEquals(2, archives.length);
// Check lengths of the files
- TestCase.assertEquals(1, fs.getFileStatus(files[0]).getLen());
- TestCase.assertTrue(fs.getFileStatus(files[1]).getLen() > 1);
+ TestCase.assertEquals(1, fs.getFileStatus(localFiles[0]).getLen());
+ TestCase.assertTrue(fs.getFileStatus(localFiles[1]).getLen() > 1);
// Check extraction of the archive
- TestCase.assertTrue(fs.exists(new Path(archives[0],
+ TestCase.assertTrue(fs.exists(new Path(localArchives[0],
"distributed.jar.inside3")));
- TestCase.assertTrue(fs.exists(new Path(archives[1],
+ TestCase.assertTrue(fs.exists(new Path(localArchives[1],
"distributed.jar.inside4")));
// Check the class loaders
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/map/WrappedMapper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/map/WrappedMapper.java?rev=1508595&r1=1508594&r2=1508595&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/map/WrappedMapper.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/map/WrappedMapper.java
Tue Jul 30 20:04:37 2013
@@ -144,7 +144,7 @@ public class WrappedMapper<KEYIN, VALUEI
@Override
public URI[] getCacheFiles() throws IOException {
- return mapContext.getCacheArchives();
+ return mapContext.getCacheFiles();
}
@Override
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java?rev=1508595&r1=1508594&r2=1508595&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
Tue Jul 30 20:04:37 2013
@@ -422,16 +422,20 @@ public class TestMRJobs {
@Override
public void setup(Context context) throws IOException {
Configuration conf = context.getConfiguration();
- Path[] files = context.getLocalCacheFiles();
- Path[] archives = context.getLocalCacheArchives();
+ Path[] localFiles = context.getLocalCacheFiles();
+ URI[] files = context.getCacheFiles();
+ Path[] localArchives = context.getLocalCacheArchives();
+ URI[] archives = context.getCacheArchives();
// Check that 4 (2 + appjar + DistrubutedCacheChecker jar) files
// and 2 archives are present
+ Assert.assertEquals(4, localFiles.length);
Assert.assertEquals(4, files.length);
+ Assert.assertEquals(2, localArchives.length);
Assert.assertEquals(2, archives.length);
// Check lengths of the files
- Map<String, Path> filesMap = pathsToMap(files);
+ Map<String, Path> filesMap = pathsToMap(localFiles);
Assert.assertTrue(filesMap.containsKey("distributed.first.symlink"));
Assert.assertEquals(1, localFs.getFileStatus(
filesMap.get("distributed.first.symlink")).getLen());
@@ -440,7 +444,7 @@ public class TestMRJobs {
filesMap.get("distributed.second.jar")).getLen() > 1);
// Check extraction of the archive
- Map<String, Path> archivesMap = pathsToMap(archives);
+ Map<String, Path> archivesMap = pathsToMap(localArchives);
Assert.assertTrue(archivesMap.containsKey("distributed.third.jar"));
Assert.assertTrue(localFs.exists(new Path(
archivesMap.get("distributed.third.jar"), "distributed.jar.inside3")));
