Modified: hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java?rev=1519787&r1=1519786&r2=1519787&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java (original) +++ hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java Tue Sep 3 18:30:05 2013 @@ -20,6 +20,7 @@ package org.apache.hadoop.mapred; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.EnumSet; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -49,6 +50,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; @@ -118,8 +120,10 @@ public class ResourceMgrDelegate extends try { Set<String> appTypes = new HashSet<String>(1); appTypes.add(MRJobConfig.MR_APPLICATION_TYPE); + EnumSet<YarnApplicationState> appStates = + EnumSet.noneOf(YarnApplicationState.class); return TypeConverter.fromYarnApps( - client.getApplications(appTypes), this.conf); + client.getApplications(appTypes, appStates), this.conf); } catch (YarnException e) { throw new IOException(e); } @@ -299,12 +303,28 @@ public class ResourceMgrDelegate extends } @Override - public List<ApplicationReport> getApplications( - Set<String> applicationTypes) throws YarnException, IOException { + public List<ApplicationReport> getApplications(Set<String> applicationTypes) + throws YarnException, + IOException { return client.getApplications(applicationTypes); } @Override + public List<ApplicationReport> getApplications( + EnumSet<YarnApplicationState> applicationStates) throws YarnException, + IOException { + return client.getApplications(applicationStates); + } + + @Override + public List<ApplicationReport> getApplications( + Set<String> applicationTypes, + EnumSet<YarnApplicationState> applicationStates) + throws YarnException, IOException { + return client.getApplications(applicationTypes, applicationStates); + } + + @Override public YarnClusterMetrics getYarnClusterMetrics() throws YarnException, IOException { return client.getYarnClusterMetrics();
Modified: hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java?rev=1519787&r1=1519786&r2=1519787&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java (original) +++ hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java Tue Sep 3 18:30:05 2013 @@ -209,6 +209,9 @@ public class TestDFSIO implements Tool { .build(); FileSystem fs = cluster.getFileSystem(); bench.createControlFile(fs, DEFAULT_NR_BYTES, DEFAULT_NR_FILES); + + /** Check write here, as it is required for other tests */ + testWrite(); } @AfterClass @@ -220,8 +223,7 @@ public class TestDFSIO implements Tool { cluster.shutdown(); } - @Test - public void testWrite() throws Exception { + public static void testWrite() throws Exception { FileSystem fs = cluster.getFileSystem(); long tStart = System.currentTimeMillis(); bench.writeTest(fs); @@ -229,7 +231,7 @@ public class TestDFSIO implements Tool { bench.analyzeResult(fs, TestType.TEST_TYPE_WRITE, execTime); } - @Test + @Test (timeout = 3000) public void testRead() throws Exception { FileSystem fs = cluster.getFileSystem(); long tStart = System.currentTimeMillis(); @@ -238,7 +240,7 @@ public class TestDFSIO implements Tool { bench.analyzeResult(fs, TestType.TEST_TYPE_READ, execTime); } - @Test + @Test (timeout = 3000) public void testReadRandom() throws Exception { FileSystem fs = cluster.getFileSystem(); long tStart = System.currentTimeMillis(); @@ -248,7 +250,7 @@ public class TestDFSIO implements Tool { bench.analyzeResult(fs, TestType.TEST_TYPE_READ_RANDOM, execTime); } - @Test + @Test (timeout = 3000) public void testReadBackward() throws Exception { FileSystem fs = cluster.getFileSystem(); long tStart = System.currentTimeMillis(); @@ -258,7 +260,7 @@ public class TestDFSIO implements Tool { bench.analyzeResult(fs, TestType.TEST_TYPE_READ_BACKWARD, execTime); } - @Test + @Test (timeout = 3000) public void testReadSkip() throws Exception { FileSystem fs = cluster.getFileSystem(); long tStart = System.currentTimeMillis(); @@ -268,7 +270,7 @@ public class TestDFSIO implements Tool { bench.analyzeResult(fs, TestType.TEST_TYPE_READ_SKIP, execTime); } - @Test + @Test (timeout = 3000) public void testAppend() throws Exception { FileSystem fs = cluster.getFileSystem(); long tStart = System.currentTimeMillis(); Propchange: hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-examples/ ------------------------------------------------------------------------------ Merged /hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-examples:r1513206-1519783 Modified: hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraInputFormat.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraInputFormat.java?rev=1519787&r1=1519786&r2=1519787&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraInputFormat.java (original) +++ hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraInputFormat.java Tue Sep 3 18:30:05 2013 @@ -60,48 +60,6 @@ public class TeraInputFormat extends Fil private static MRJobConfig lastContext = null; private static List<InputSplit> lastResult = null; - static class TeraFileSplit extends FileSplit { - static private String[] ZERO_LOCATIONS = new String[0]; - - private String[] locations; - - public TeraFileSplit() { - locations = ZERO_LOCATIONS; - } - public TeraFileSplit(Path file, long start, long length, String[] hosts) { - super(file, start, length, hosts); - try { - locations = super.getLocations(); - } catch (IOException e) { - locations = ZERO_LOCATIONS; - } - } - - // XXXXXX should this also be null-protected? - protected void setLocations(String[] hosts) { - locations = hosts; - } - - @Override - public String[] getLocations() { - return locations; - } - - public String toString() { - StringBuffer result = new StringBuffer(); - result.append(getPath()); - result.append(" from "); - result.append(getStart()); - result.append(" length "); - result.append(getLength()); - for(String host: getLocations()) { - result.append(" "); - result.append(host); - } - return result.toString(); - } - } - static class TextSampler implements IndexedSortable { private ArrayList<Text> records = new ArrayList<Text>(); @@ -325,11 +283,6 @@ public class TeraInputFormat extends Fil return new TeraRecordReader(); } - protected FileSplit makeSplit(Path file, long start, long length, - String[] hosts) { - return new TeraFileSplit(file, start, length, hosts); - } - @Override public List<InputSplit> getSplits(JobContext job) throws IOException { if (job == lastContext) { @@ -343,7 +296,7 @@ public class TeraInputFormat extends Fil System.out.println("Spent " + (t2 - t1) + "ms computing base-splits."); if (job.getConfiguration().getBoolean(TeraScheduler.USE, true)) { TeraScheduler scheduler = new TeraScheduler( - lastResult.toArray(new TeraFileSplit[0]), job.getConfiguration()); + lastResult.toArray(new FileSplit[0]), job.getConfiguration()); lastResult = scheduler.getNewFileSplits(); t3 = System.currentTimeMillis(); System.out.println("Spent " + (t3 - t2) + "ms computing TeraScheduler splits."); Modified: hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java?rev=1519787&r1=1519786&r2=1519787&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java (original) +++ hadoop/common/branches/YARN-321/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java Tue Sep 3 18:30:05 2013 @@ -24,7 +24,6 @@ import java.util.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.examples.terasort.TeraInputFormat.TeraFileSplit; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.lib.input.FileSplit; import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; @@ -214,8 +213,9 @@ class TeraScheduler { for(int i=0; i < splits.length; ++i) { if (splits[i].isAssigned) { // copy the split and fix up the locations - ((TeraFileSplit) realSplits[i]).setLocations - (new String[]{splits[i].locations.get(0).hostname}); + String[] newLocations = {splits[i].locations.get(0).hostname}; + realSplits[i] = new FileSplit(realSplits[i].getPath(), + realSplits[i].getStart(), realSplits[i].getLength(), newLocations); result[left++] = realSplits[i]; } else { result[right--] = realSplits[i];
