Author: szetszwo
Date: Fri Apr 5 02:43:29 2013
New Revision: 1464815
URL: http://svn.apache.org/r1464815
Log:
Merge r1462698 through r1464807 from trunk.
Added:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/dev-support/
- copied from r1464807,
hadoop/common/trunk/hadoop-tools/hadoop-gridmix/dev-support/
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml
- copied unchanged from r1464807,
hadoop/common/trunk/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java
- copied unchanged from r1464807,
hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestDistCacheEmulation.java
- copied unchanged from r1464807,
hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestDistCacheEmulation.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridMixClasses.java
- copied unchanged from r1464807,
hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridMixClasses.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSubmission.java
- copied unchanged from r1464807,
hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSubmission.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestLoadJob.java
- copied unchanged from r1464807,
hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestLoadJob.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestSleepJob.java
- copied unchanged from r1464807,
hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestSleepJob.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount.json
- copied unchanged from r1464807,
hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount.json
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount2.json
- copied unchanged from r1464807,
hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount2.json
Modified:
hadoop/common/branches/HDFS-2802/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/pom.xml
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobMonitor.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobFactory.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobProducer.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
Propchange: hadoop/common/branches/HDFS-2802/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-2.0.4-alpha:r1463804
Merged /hadoop/common/trunk:r1462698-1464807
Modified: hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/pom.xml?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/pom.xml
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/pom.xml Fri
Apr 5 02:43:29 2013
@@ -91,10 +91,25 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <findbugsXmlOutput>true</findbugsXmlOutput>
+ <xmlOutput>true</xmlOutput>
+
<excludeFilterFile>${basedir}/dev-support/findbugs-exclude.xml</excludeFilterFile>
+ <effort>Max</effort>
+ </configuration>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
@@ -116,6 +131,15 @@
</executions>
</plugin>
<plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>src/test/resources/data/*</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
Fri Apr 5 02:43:29 2013
@@ -36,7 +36,9 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapred.gridmix.GenerateData.DataStatistics;
import org.apache.hadoop.mapred.gridmix.Statistics.JobStats;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
@@ -693,7 +695,7 @@ public class Gridmix extends Configured
try {
res = ToolRunner.run(new Configuration(), new Gridmix(argv), argv);
} finally {
- System.exit(res);
+ ExitUtil.terminate(res);
}
}
@@ -800,6 +802,10 @@ public class Gridmix extends Configured
*/
void abort();
}
-
+ // it is need for tests
+ protected Summarizer getSummarizer() {
+ return summarizer;
+ }
+
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobMonitor.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobMonitor.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobMonitor.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobMonitor.java
Fri Apr 5 02:43:29 2013
@@ -86,9 +86,7 @@ class JobMonitor implements Gridmix.Comp
* Add a running job's status to the polling queue.
*/
public void add(JobStats job) throws InterruptedException {
- synchronized (runningJobs) {
runningJobs.put(job);
- }
}
/**
@@ -147,12 +145,10 @@ class JobMonitor implements Gridmix.Comp
boolean shutdown;
while (true) {
try {
- synchronized (runningJobs) {
- synchronized (mJobs) {
- graceful = JobMonitor.this.graceful;
- shutdown = JobMonitor.this.shutdown;
- runningJobs.drainTo(mJobs);
- }
+ synchronized (mJobs) {
+ graceful = JobMonitor.this.graceful;
+ shutdown = JobMonitor.this.shutdown;
+ runningJobs.drainTo(mJobs);
}
// shutdown conditions; either shutdown requested and all jobs
@@ -160,11 +156,9 @@ class JobMonitor implements Gridmix.Comp
// submitted jobs not in the monitored set
if (shutdown) {
if (!graceful) {
- synchronized (runningJobs) {
- while (!runningJobs.isEmpty()) {
- synchronized (mJobs) {
- runningJobs.drainTo(mJobs);
- }
+ while (!runningJobs.isEmpty()) {
+ synchronized (mJobs) {
+ runningJobs.drainTo(mJobs);
}
}
break;
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java
Fri Apr 5 02:43:29 2013
@@ -175,4 +175,8 @@ public class SerialJobFactory extends Jo
LOG.info(" Starting Serial submission ");
this.rThread.start();
}
+ // it is need for test
+ void setDistCacheEmulator(DistributedCacheEmulator e) {
+ jobCreator.setDistCacheEmulator(e);
+ }
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java
Fri Apr 5 02:43:29 2013
@@ -84,7 +84,7 @@ public class SleepJob extends GridmixJob
String[] hosts) throws IOException {
super(conf, submissionMillis, jobdesc, outRoot, ugi, seq);
this.fakeLocations = numLocations;
- this.hosts = hosts;
+ this.hosts = hosts.clone();
this.selector = (fakeLocations > 0)? new Selector(hosts.length, (float)
fakeLocations
/ hosts.length, rand.get()) : null;
this.mapTasksOnly = conf.getBoolean(SLEEPJOB_MAPTASK_ONLY, false);
@@ -289,9 +289,9 @@ public class SleepJob extends GridmixJob
this.id = id;
this.sleepDuration = sleepDuration;
nSpec = reduceDurations.length;
- this.reduceDurations = reduceDurations;
+ this.reduceDurations = reduceDurations.clone();
this.nMaps = nMaps;
- this.locations = locations;
+ this.locations = locations.clone();
}
@Override
@@ -349,7 +349,7 @@ public class SleepJob extends GridmixJob
@Override
public String[] getLocations() {
- return locations;
+ return locations.clone();
}
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java
Fri Apr 5 02:43:29 2013
@@ -93,7 +93,7 @@ implements ResourceUsageEmulatorPlugin {
private static final float DEFAULT_HEAP_LOAD_RATIO = 0.1F;
- public static int ONE_MB = 1024 * 1024;
+ public static final int ONE_MB = 1024 * 1024;
/**
* Defines the core heap usage emulation algorithm. This engine is expected
@@ -129,7 +129,8 @@ implements ResourceUsageEmulatorPlugin {
public static class DefaultHeapUsageEmulator
implements HeapUsageEmulatorCore {
// store the unit loads in a list
- protected static ArrayList<Object> heapSpace = new ArrayList<Object>();
+ protected static final ArrayList<Object> heapSpace =
+ new ArrayList<Object>();
/**
* Increase heap usage by current process by the given amount.
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobFactory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobFactory.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobFactory.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobFactory.java
Fri Apr 5 02:43:29 2013
@@ -29,13 +29,13 @@ import java.util.concurrent.CountDownLat
/**
* Component generating random job traces for testing on a single node.
*/
-class DebugJobFactory {
+public class DebugJobFactory {
interface Debuggable {
ArrayList<JobStory> getSubmitted();
}
- public static JobFactory getFactory(
+ public static JobFactory<?> getFactory(
JobSubmitter submitter, Path scratch, int numJobs, Configuration conf,
CountDownLatch startFlag, UserResolver resolver) throws IOException {
GridmixJobSubmissionPolicy policy = GridmixJobSubmissionPolicy.getPolicy(
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobProducer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobProducer.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobProducer.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobProducer.java
Fri Apr 5 02:43:29 2013
@@ -216,7 +216,7 @@ public class DebugJobProducer implements
if (user == null) {
user = String.format("foobar%d", id);
}
- GridmixTestUtils.createHomeAndStagingDirectory(user, (JobConf)conf);
+ GridmixTestUtils.createHomeAndStagingDirectory(user, conf);
return user;
}
@@ -300,7 +300,7 @@ public class DebugJobProducer implements
@Override
public String getQueueName() {
- String qName = "q" + ((id % 2) + 1);
+ String qName = "default";
return qName;
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java
Fri Apr 5 02:43:29 2013
@@ -4,55 +4,76 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.mapred.MiniMRCluster;
-import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MiniMRClientCluster;
+import org.apache.hadoop.mapred.MiniMRClientClusterFactory;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
-import org.apache.hadoop.security.Groups;
import java.io.IOException;
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
*/
public class GridmixTestUtils {
private static final Log LOG = LogFactory.getLog(GridmixTestUtils.class);
static final Path DEST = new Path("/gridmix");
static FileSystem dfs = null;
static MiniDFSCluster dfsCluster = null;
- static MiniMRCluster mrCluster = null;
+ static MiniMRClientCluster mrvl = null;
+ protected static final String GRIDMIX_USE_QUEUE_IN_TRACE =
+ "gridmix.job-submission.use-queue-in-trace";
+ protected static final String GRIDMIX_DEFAULT_QUEUE =
+ "gridmix.job-submission.default-queue";
- public static void initCluster() throws IOException {
+ public static void initCluster(Class<?> caller) throws IOException {
Configuration conf = new Configuration();
- conf.set("mapred.queue.names", "default,q1,q2");
- dfsCluster = new MiniDFSCluster(conf, 3, true, null);
+// conf.set("mapred.queue.names", "default,q1,q2");
+ conf.set("mapred.queue.names", "default");
+ conf.set("yarn.scheduler.capacity.root.queues", "default");
+ conf.set("yarn.scheduler.capacity.root.default.capacity", "100.0");
+
+
+ conf.setBoolean(GRIDMIX_USE_QUEUE_IN_TRACE, false);
+ conf.set(GRIDMIX_DEFAULT_QUEUE, "default");
+
+
+ dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true)
+ .build();// MiniDFSCluster(conf, 3, true, null);
dfs = dfsCluster.getFileSystem();
conf.set(JTConfig.JT_RETIREJOBS, "false");
- mrCluster = new MiniMRCluster(3, dfs.getUri().toString(), 1, null, null,
- new JobConf(conf));
+ mrvl = MiniMRClientClusterFactory.create(caller, 2, conf);
+
+ conf = mrvl.getConfig();
+ String[] files = conf.getStrings(MRJobConfig.CACHE_FILES);
+ if (files != null) {
+ String[] timestamps = new String[files.length];
+ for (int i = 0; i < files.length; i++) {
+ timestamps[i] = Long.toString(System.currentTimeMillis());
+ }
+ conf.setStrings(MRJobConfig.CACHE_FILE_TIMESTAMPS, timestamps);
+ }
+
}
public static void shutdownCluster() throws IOException {
- if (mrCluster != null) {
- mrCluster.shutdown();
+ if (mrvl != null) {
+ mrvl.stop();
}
if (dfsCluster != null) {
dfsCluster.shutdown();
@@ -61,23 +82,25 @@ public class GridmixTestUtils {
/**
* Methods to generate the home directory for dummy users.
- *
+ *
* @param conf
*/
- public static void createHomeAndStagingDirectory(String user, JobConf conf) {
+ public static void createHomeAndStagingDirectory(String user,
+ Configuration conf) {
try {
FileSystem fs = dfsCluster.getFileSystem();
String path = "/user/" + user;
Path homeDirectory = new Path(path);
- if(fs.exists(homeDirectory)) {
- fs.delete(homeDirectory,true);
- }
- LOG.info("Creating Home directory : " + homeDirectory);
- fs.mkdirs(homeDirectory);
- changePermission(user,homeDirectory, fs);
- Path stagingArea =
- new Path(conf.get("mapreduce.jobtracker.staging.root.dir",
- "/tmp/hadoop/mapred/staging"));
+ if (!fs.exists(homeDirectory)) {
+ LOG.info("Creating Home directory : " + homeDirectory);
+ fs.mkdirs(homeDirectory);
+ changePermission(user, homeDirectory, fs);
+
+ }
+ changePermission(user, homeDirectory, fs);
+ Path stagingArea = new Path(
+ conf.get("mapreduce.jobtracker.staging.root.dir",
+ "/tmp/hadoop/mapred/staging"));
LOG.info("Creating Staging root directory : " + stagingArea);
fs.mkdirs(stagingArea);
fs.setPermission(stagingArea, new FsPermission((short) 0777));
@@ -87,7 +110,7 @@ public class GridmixTestUtils {
}
static void changePermission(String user, Path homeDirectory, FileSystem fs)
- throws IOException {
+ throws IOException {
fs.setOwner(homeDirectory, user, "");
}
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java
Fri Apr 5 02:43:29 2013
@@ -133,7 +133,7 @@ public class TestGridmixSummary {
/**
* A fake {@link JobFactory}.
*/
- @SuppressWarnings("unchecked")
+ @SuppressWarnings("rawtypes")
private static class FakeJobFactory extends JobFactory {
/**
* A fake {@link JobStoryProducer} for {@link FakeJobFactory}.
@@ -167,7 +167,7 @@ public class TestGridmixSummary {
* Test {@link ExecutionSummarizer}.
*/
@Test
- @SuppressWarnings("unchecked")
+ @SuppressWarnings({ "unchecked", "rawtypes" })
public void testExecutionSummarizer() throws IOException {
Configuration conf = new Configuration();
@@ -360,7 +360,6 @@ public class TestGridmixSummary {
* Test {@link ClusterSummarizer}.
*/
@Test
- @SuppressWarnings("deprecation")
public void testClusterSummarizer() throws IOException {
ClusterSummarizer cs = new ClusterSummarizer();
Configuration conf = new Configuration();
@@ -374,13 +373,13 @@ public class TestGridmixSummary {
assertEquals("JT name mismatch", jt, cs.getJobTrackerInfo());
assertEquals("NN name mismatch", nn, cs.getNamenodeInfo());
- ClusterStats cstats = ClusterStats.getClusterStats();
+ ClusterStats cStats = ClusterStats.getClusterStats();
conf.set(JTConfig.JT_IPC_ADDRESS, "local");
conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "local");
JobClient jc = new JobClient(conf);
- cstats.setClusterMetric(jc.getClusterStatus());
+ cStats.setClusterMetric(jc.getClusterStatus());
- cs.update(cstats);
+ cs.update(cStats);
// test
assertEquals("Cluster summary test failed!", 1, cs.getMaxMapTasks());
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
Fri Apr 5 02:43:29 2013
@@ -875,7 +875,7 @@ public class StreamJob implements Tool {
IdentifierResolver.TEXT_ID));
jobConf_.setClass("stream.map.output.reader.class",
idResolver.getOutputReaderClass(), OutputReader.class);
- if (isMapperACommand) {
+ if (isMapperACommand || jobConf_.get("stream.map.output") != null) {
// if mapper is a command, then map output key/value classes come from
the
// idResolver
jobConf_.setMapOutputKeyClass(idResolver.getOutputKeyClass());
@@ -891,7 +891,7 @@ public class StreamJob implements Tool {
IdentifierResolver.TEXT_ID));
jobConf_.setClass("stream.reduce.output.reader.class",
idResolver.getOutputReaderClass(), OutputReader.class);
- if (isReducerACommand) {
+ if (isReducerACommand || jobConf_.get("stream.reduce.output") != null) {
// if reducer is a command, then output key/value classes come from the
// idResolver
jobConf_.setOutputKeyClass(idResolver.getOutputKeyClass());
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java
Fri Apr 5 02:43:29 2013
@@ -24,6 +24,7 @@ import org.apache.hadoop.mapred.MapReduc
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.junit.Before;
import org.junit.Test;
@@ -177,6 +178,17 @@ public class TestStreamingOutputKeyValue
args.add("0");
super.testCommandLine();
}
+
+ @Test
+ public void testDefaultToIdentityReducer() throws Exception {
+ args.add("-mapper");args.add(map);
+ args.add("-jobconf");
+ args.add("mapreduce.task.files.preserve.failedtasks=true");
+ args.add("-jobconf");
+ args.add("stream.tmpdir="+System.getProperty("test.build.data","/tmp"));
+ args.add("-inputformat");args.add(TextInputFormat.class.getName());
+ super.testCommandLine();
+ }
@Override
@Test
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
Fri Apr 5 02:43:29 2013
@@ -43,7 +43,6 @@ public class TrApp
// Note the dots translated to underscore:
// property names have been escaped in PipeMapRed.safeEnvVarName()
expectDefined("mapreduce_cluster_local_dir");
- expect("mapred_output_format_class",
"org.apache.hadoop.mapred.TextOutputFormat");
expect("mapreduce_map_output_key_class", "org.apache.hadoop.io.Text");
expect("mapreduce_map_output_value_class", "org.apache.hadoop.io.Text");
