Author: szetszwo Date: Fri Feb 28 19:37:47 2014 New Revision: 1573043 URL: http://svn.apache.org/r1573043 Log: Merge r1569890 through r1573042 from trunk.
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/ (props changed) hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt (contents, props changed) hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java Propchange: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1572723-1573042 Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt?rev=1573043&r1=1573042&r2=1573043&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt Fri Feb 28 19:37:47 2014 @@ -153,6 +153,9 @@ Release 2.5.0 - UNRELEASED MAPREDUCE-5688. TestStagingCleanup fails intermittently with JDK7 (Mit Desai via jeagles) + MAPREDUCE-5665. Add audience annotations to MiniMRYarnCluster and + MiniMRCluster. (Anubhav Dhoot via kasha) + OPTIMIZATIONS BUG FIXES @@ -199,6 +202,9 @@ Release 2.4.0 - UNRELEASED to the YARN's web-app proxy with the correct scheme prefix. (Jian He via vinodkv) + MAPREDUCE-5768. TestMRJobs.testContainerRollingLog fails on trunk (Gera + Shegalov via jlowe) + Release 2.3.1 - UNRELEASED INCOMPATIBLE CHANGES Propchange: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1572723-1573042 Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java?rev=1573043&r1=1573042&r2=1573043&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java Fri Feb 28 19:37:47 2014 @@ -22,6 +22,8 @@ import java.util.Random; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; @@ -40,6 +42,8 @@ import org.apache.hadoop.security.UserGr * instead */ @Deprecated +@InterfaceAudience.Public +@InterfaceStability.Evolving public class MiniMRCluster { private static final Log LOG = LogFactory.getLog(MiniMRCluster.class); Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java?rev=1573043&r1=1573042&r2=1573043&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java Fri Feb 28 19:37:47 2014 @@ -26,6 +26,8 @@ import java.net.UnknownHostException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileContext; @@ -55,6 +57,8 @@ import org.apache.hadoop.yarn.webapp.uti * Configures and starts the MR-specific components in the YARN cluster. * */ +@InterfaceAudience.Public +@InterfaceStability.Evolving public class MiniMRYarnCluster extends MiniYARNCluster { public static final String APPJAR = JarFinder.getJar(LocalContainerLauncher.class); Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java?rev=1573043&r1=1573042&r2=1573043&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java Fri Feb 28 19:37:47 2014 @@ -23,7 +23,6 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; -import java.io.InputStreamReader; import java.io.IOException; import java.io.StringReader; import java.net.URI; @@ -82,8 +81,10 @@ import org.apache.hadoop.security.token. import org.apache.hadoop.util.JarFinder; import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.log4j.Level; import org.junit.AfterClass; import org.junit.Assert; @@ -492,28 +493,17 @@ public class TestMRJobs { LOG.info("Checking for glob: " + absSyslogGlob); final FileStatus[] syslogs = localFs.globStatus(absSyslogGlob); for (FileStatus slog : syslogs) { - // check all syslogs for the container - // - final FileStatus[] sysSiblings = localFs.globStatus(new Path( - slog.getPath().getParent(), TaskLog.LogName.SYSLOG + "*")); - boolean foundAppMaster = false; - floop: - for (FileStatus f : sysSiblings) { - final BufferedReader reader = new BufferedReader( - new InputStreamReader(localFs.open(f.getPath()))); - String line; - try { - while ((line = reader.readLine()) != null) { - if (line.contains(MRJobConfig.APPLICATION_MASTER_CLASS)) { - foundAppMaster = true; - break floop; - } - } - } finally { - reader.close(); - } + boolean foundAppMaster = job.isUber(); + final Path containerPathComponent = slog.getPath().getParent(); + if (!foundAppMaster) { + final ContainerId cid = ConverterUtils.toContainerId( + containerPathComponent.getName()); + foundAppMaster = (cid.getId() == 1); } + final FileStatus[] sysSiblings = localFs.globStatus(new Path( + containerPathComponent, TaskLog.LogName.SYSLOG + "*")); + if (foundAppMaster) { numAppMasters++; } else {