Author: bobby
Date: Mon Apr 23 19:44:30 2012
New Revision: 1329403
URL: http://svn.apache.org/viewvc?rev=1329403&view=rev
Log:
MAPREDUCE-4133. MR over viewfs is broken (John George via bobby)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java
hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1329403&r1=1329402&r2=1329403&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Mon Apr 23
19:44:30 2012
@@ -395,6 +395,7 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4190. Improve web UI for task attempts userlog link (Tom Graves
via bobby)
+ MAPREDUCE-4133. MR over viewfs is broken (John George via bobby)
Release 0.23.2 - UNRELEASED
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1329403&r1=1329402&r2=1329403&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
Mon Apr 23 19:44:30 2012
@@ -827,7 +827,7 @@ public class JobImpl implements org.apac
|| (sysMaxReduces == 1);
long sysMaxBytes = conf.getLong(MRJobConfig.JOB_UBERTASK_MAXBYTES,
- fs.getDefaultBlockSize()); // FIXME: this is wrong; get FS from
+ fs.getDefaultBlockSize(this.remoteJobSubmitDir)); // FIXME: this is
wrong; get FS from
// [File?]InputFormat and default block size
// from that
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java?rev=1329403&r1=1329402&r2=1329403&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java
Mon Apr 23 19:44:30 2012
@@ -128,7 +128,7 @@ public class DistributedFSCheck extends
if (rootStatus.isFile()) {
nrFiles++;
// For a regular file generate <fName,offset> pairs
- long blockSize = fs.getDefaultBlockSize();
+ long blockSize = fs.getDefaultBlockSize(rootFile);
long fileLength = rootStatus.getLen();
for(long offset = 0; offset < fileLength; offset += blockSize)
writer.append(new Text(rootFile.toString()), new LongWritable(offset));
@@ -160,15 +160,16 @@ public class DistributedFSCheck extends
) throws IOException {
// open file
FSDataInputStream in = null;
+ Path p = new Path(name);
try {
- in = fs.open(new Path(name));
+ in = fs.open(p);
} catch(IOException e) {
return name + "@(missing)";
}
in.seek(offset);
long actualSize = 0;
try {
- long blockSize = fs.getDefaultBlockSize();
+ long blockSize = fs.getDefaultBlockSize(p);
reporter.setStatus("reading " + name + "@" +
offset + "/" + blockSize);
for( int curSize = bufferSize;
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java?rev=1329403&r1=1329402&r2=1329403&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
Mon Apr 23 19:44:30 2012
@@ -287,7 +287,7 @@ public class JobHistory {
FSDataOutputStream out = logDirFs.create(logFile,
new FsPermission(JobHistory.HISTORY_FILE_PERMISSION),
true, defaultBufferSize,
- logDirFs.getDefaultReplication(),
+ logDirFs.getDefaultReplication(logFile),
jobHistoryBlockSize, null);
EventWriter writer = new EventWriter(out);
@@ -306,8 +306,8 @@ public class JobHistory {
jobFileOut = logDirFs.create(logDirConfPath,
new FsPermission(JobHistory.HISTORY_FILE_PERMISSION),
true, defaultBufferSize,
- logDirFs.getDefaultReplication(),
- logDirFs.getDefaultBlockSize(), null);
+ logDirFs.getDefaultReplication(logDirConfPath),
+ logDirFs.getDefaultBlockSize(logDirConfPath), null);
jobConf.writeXml(jobFileOut);
jobFileOut.close();
}
