Author: omalley
Date: Sun Jun 19 06:30:55 2011
New Revision: 1137300
URL: http://svn.apache.org/viewvc?rev=1137300&view=rev
Log:
Fix up the merges from the 205 branch.
Added:
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/QueueMetrics.java
- copied unchanged from r1131280,
hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapred/QueueMetrics.java
hadoop/common/branches/branch-0.20-security-204/src/test/org/apache/hadoop/mapred/TestQueueMetrics.java
- copied unchanged from r1131280,
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/mapred/TestQueueMetrics.java
Modified:
hadoop/common/branches/branch-0.20-security-204/CHANGES.txt (props
changed)
hadoop/common/branches/branch-0.20-security-204/ivy/libraries.properties
hadoop/common/branches/branch-0.20-security-204/src/contrib/build-contrib.xml
hadoop/common/branches/branch-0.20-security-204/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java
hadoop/common/branches/branch-0.20-security-204/src/contrib/data_join/ivy.xml
hadoop/common/branches/branch-0.20-security-204/src/contrib/fairscheduler/ivy.xml
hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/build.xml
hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestHdfsProxy.java
hadoop/common/branches/branch-0.20-security-204/src/contrib/index/ivy.xml
hadoop/common/branches/branch-0.20-security-204/src/contrib/streaming/ivy/libraries.properties
hadoop/common/branches/branch-0.20-security-204/src/core/org/apache/hadoop/fs/FileUtil.java
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobTracker.java
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/Queue.java
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/QueueManager.java
Propchange: hadoop/common/branches/branch-0.20-security-204/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sun Jun 19 06:30:55 2011
@@ -1,5 +1,5 @@
/hadoop/common/branches/branch-0.20/CHANGES.txt:826138,826568,829987,831184,833001,880632,898713,909245,909723,960946,1044225
-/hadoop/common/branches/branch-0.20-security/CHANGES.txt:1097202,1098837,1100336,1101315,1101629,1101729,1102378,1102869,1103940,1125139,1125170,1125587,1125589,1127362,1130409,1131277,1131286,1131290,1131299,1131461,1131737,1134140
+/hadoop/common/branches/branch-0.20-security/CHANGES.txt:1097202,1098837,1100336,1101315,1101629,1101729,1102378,1102869,1103940,1125139,1125170,1125587,1125589,1127362,1130409,1131277,1131280,1131286,1131290,1131299,1131461,1131737,1134140
/hadoop/common/branches/branch-0.20-security-203/CHANGES.txt:1096071,1097012-1099333,1102071,1128115
/hadoop/common/branches/branch-0.20-security-205/CHANGES.txt:1132788,1133133,1133274,1133282,1133475
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
Modified:
hadoop/common/branches/branch-0.20-security-204/ivy/libraries.properties
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/ivy/libraries.properties?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/ivy/libraries.properties
(original)
+++ hadoop/common/branches/branch-0.20-security-204/ivy/libraries.properties
Sun Jun 19 06:30:55 2011
@@ -47,6 +47,7 @@ hsqldb.version=1.8.0.10
ivy.version=2.1.0
jasper.version=5.5.12
+jackson.version=1.0.1
#not able to figureout the version of jsp & jsp-api version to get it resolved
throught ivy
# but still declared here as we are going to have a local copy from the lib
folder
jsp.version=2.1
Modified:
hadoop/common/branches/branch-0.20-security-204/src/contrib/build-contrib.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/contrib/build-contrib.xml?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-204/src/contrib/build-contrib.xml
(original)
+++
hadoop/common/branches/branch-0.20-security-204/src/contrib/build-contrib.xml
Sun Jun 19 06:30:55 2011
@@ -283,7 +283,7 @@
<mkdir dir="${hadoop.log.dir}"/>
<junit
printsummary="yes" showoutput="${test.output}"
- haltonfailure="no" fork="yes" maxmemory="256m"
+ haltonfailure="no" fork="yes" maxmemory="512m"
errorProperty="tests.failed" failureProperty="tests.failed"
timeout="${test.timeout}">
Modified:
hadoop/common/branches/branch-0.20-security-204/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-204/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java
(original)
+++
hadoop/common/branches/branch-0.20-security-204/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java
Sun Jun 19 06:30:55 2011
@@ -432,7 +432,8 @@ public class TestCapacityScheduler exten
queues.clear();
for (String qName : newQueues) {
try {
- queues.put(qName, new Queue(qName, acls, Queue.QueueState.RUNNING));
+ queues.put(qName, new Queue(qName, acls, Queue.QueueState.RUNNING,
+ QueueMetrics.create(qName, new Configuration())));
} catch (Throwable t) {
throw new RuntimeException("Unable to initialize queue " + qName, t);
}
Modified:
hadoop/common/branches/branch-0.20-security-204/src/contrib/data_join/ivy.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/contrib/data_join/ivy.xml?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-204/src/contrib/data_join/ivy.xml
(original)
+++
hadoop/common/branches/branch-0.20-security-204/src/contrib/data_join/ivy.xml
Sun Jun 19 06:30:55 2011
@@ -40,5 +40,29 @@
name="commons-math"
rev="${commons-math.version}"
conf="common->default"/>
+ <dependency org="junit"
+ name="junit"
+ rev="${junit.version}"
+ conf="common->default"/>
+ <dependency org="org.mortbay.jetty"
+ name="jetty-util"
+ rev="${jetty-util.version}"
+ conf="common->master"/>
+ <dependency org="org.mortbay.jetty"
+ name="jetty"
+ rev="${jetty.version}"
+ conf="common->default"/>
+ <dependency org="org.codehaus.jackson"
+ name="jackson-core-asl"
+ rev="${jackson.version}"
+ conf="common->default"/>
+ <dependency org="org.codehaus.jackson"
+ name="jackson-mapper-asl"
+ rev="${jackson.version}"
+ conf="common->default"/>
+ <dependency org="commons-httpclient"
+ name="commons-httpclient"
+ rev="${commons-httpclient.version}"
+ conf="common->master"/>
</dependencies>
</ivy-module>
Modified:
hadoop/common/branches/branch-0.20-security-204/src/contrib/fairscheduler/ivy.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/contrib/fairscheduler/ivy.xml?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-204/src/contrib/fairscheduler/ivy.xml
(original)
+++
hadoop/common/branches/branch-0.20-security-204/src/contrib/fairscheduler/ivy.xml
Sun Jun 19 06:30:55 2011
@@ -47,8 +47,32 @@
rev="${jsp-api-2.1.version}"
conf="common->master"/>
<dependency org="org.mortbay.jetty"
- name="jsp-2.1"
- rev="${jsp-2.1.version}"
+ name="jetty"
+ rev="${jetty.version}"
+ conf="common->default"/>
+ <dependency org="org.codehaus.jackson"
+ name="jackson-core-asl"
+ rev="${jackson.version}"
+ conf="common->default"/>
+ <dependency org="org.codehaus.jackson"
+ name="jackson-mapper-asl"
+ rev="${jackson.version}"
+ conf="common->default"/>
+ <dependency org="commons-httpclient"
+ name="commons-httpclient"
+ rev="${commons-httpclient.version}"
+ conf="common->master"/>
+ <dependency org="commons-configuration"
+ name="commons-configuration"
+ rev="${commons-configuration.version}"
+ conf="common->master"/>
+ <dependency org="org.apache.commons"
+ name="commons-math"
+ rev="${commons-math.version}"
+ conf="common->master"/>
+ <dependency org="commons-lang"
+ name="commons-lang"
+ rev="${commons-lang.version}"
conf="common->master"/>
</dependencies>
</ivy-module>
Modified:
hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/build.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/build.xml?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/build.xml
(original)
+++
hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/build.xml
Sun Jun 19 06:30:55 2011
@@ -80,7 +80,8 @@
</and>
</condition>
- <property name="ivy.settings.file"
location="${hadoop.root}/ivy/ivysettings.xml"/>
+ <property name="ivy.settings.file"
+ location="${hadoop.root}/ivy/ivysettings.xml"/>
<target name="ivy-init" depends="ivy-init-antlib">
<ivy:settings id="${ant.project.name}.ivy.settings"/>
@@ -239,7 +240,8 @@
<copy file="${proxy.conf.dir}/tomcat-web.xml"
todir="${src.test.resources}"/>
<copy file="${proxy.conf.dir}/tomcat-forward-web.xml"
todir="${src.test.resources}"/>
- <junit fork="yes" printsummary="yes" errorProperty="tests.failed"
failureProperty="tests.failed">
+ <junit fork="yes" printsummary="yes" errorProperty="tests.failed"
+ failureProperty="tests.failed" showoutput="${test.output}">
<classpath refid="test.classpath"/>
<sysproperty key="test.build.data" value="${build.test}/data"/>
<sysproperty key="build.test" value="${build.test}"/>
Modified:
hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestHdfsProxy.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestHdfsProxy.java?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestHdfsProxy.java
(original)
+++
hadoop/common/branches/branch-0.20-security-204/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestHdfsProxy.java
Sun Jun 19 06:30:55 2011
@@ -26,6 +26,7 @@ import java.util.Random;
import junit.framework.TestCase;
+import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.log4j.Level;
@@ -46,13 +47,8 @@ import org.apache.hadoop.security.author
* A JUnit test for HdfsProxy
*/
public class TestHdfsProxy extends TestCase {
- {
- ((Log4JLogger) LogFactory.getLog("org.apache.hadoop.hdfs.StateChange"))
- .getLogger().setLevel(Level.OFF);
- ((Log4JLogger) DataNode.LOG).getLogger().setLevel(Level.OFF);
- ((Log4JLogger) FSNamesystem.LOG).getLogger().setLevel(Level.OFF);
- }
+ static final Log LOG = LogFactory.getLog(TestHdfsProxy.class);
static final URI LOCAL_FS = URI.create("file:///");
private static final int NFILES = 10;
@@ -221,12 +217,13 @@ public class TestHdfsProxy extends TestC
final FileSystem localfs = FileSystem.get(LOCAL_FS, dfsConf);
final FileSystem hdfs = cluster.getFileSystem();
final Configuration proxyConf = new Configuration(false);
- proxyConf.set("hdfsproxy.dfs.namenode.address", hdfs.getUri().getHost()
+ ":"
- + hdfs.getUri().getPort());
+ proxyConf.set("hdfsproxy.dfs.namenode.address", hdfs.getUri().getHost() +
+ ":" + hdfs.getUri().getPort());
proxyConf.set("hdfsproxy.https.address", "localhost:0");
final String namenode = hdfs.getUri().toString();
if (namenode.startsWith("hdfs://")) {
MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR + "/srcdat");
+
hdfs.copyFromLocalFile
(new Path("file:///" + TEST_ROOT_DIR + "/srcdat"),
new Path(namenode + "/destdat" ));
@@ -240,7 +237,8 @@ public class TestHdfsProxy extends TestC
final String realProxyAddr = proxyAddr.getHostName() + ":"
+ proxy.getPort();
final Path proxyUrl = new Path("hftp://"; + realProxyAddr);
- final FileSystem hftp = proxyUrl.getFileSystem(dfsConf);
+ final FileSystem hftp = proxyUrl.getFileSystem(dfsConf);
+
FileUtil.copy(hftp, new Path(proxyUrl, "/destdat"),
hdfs, new Path(namenode + "/copied1"),
false, true, proxyConf);
@@ -260,13 +258,21 @@ public class TestHdfsProxy extends TestC
deldir(localfs, TEST_ROOT_DIR + "/srcdat");
deldir(localfs, TEST_ROOT_DIR + "/copied2");
}
- } finally {
if (cluster != null) {
cluster.shutdown();
}
if (proxy != null) {
proxy.stop();
}
+ } catch (Exception t) {
+ LOG.fatal("caught exception in test", t);
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ if (proxy != null) {
+ proxy.stop();
+ }
+ throw t;
}
}
}
Modified:
hadoop/common/branches/branch-0.20-security-204/src/contrib/index/ivy.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/contrib/index/ivy.xml?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/contrib/index/ivy.xml
(original)
+++ hadoop/common/branches/branch-0.20-security-204/src/contrib/index/ivy.xml
Sun Jun 19 06:30:55 2011
@@ -32,6 +32,18 @@
name="log4j"
rev="${log4j.version}"
conf="common->master"/>
+ <dependency org="junit"
+ name="junit"
+ rev="${junit.version}"
+ conf="common->default"/>
+ <dependency org="org.mortbay.jetty"
+ name="jetty-util"
+ rev="${jetty-util.version}"
+ conf="common->master"/>
+ <dependency org="org.mortbay.jetty"
+ name="jetty"
+ rev="${jetty.version}"
+ conf="common->default"/>
<dependency org="org.apache.lucene"
name="lucene-core"
rev="${lucene-core.version}"
@@ -44,5 +56,9 @@
name="commons-math"
rev="${commons-math.version}"
conf="common->default"/>
+ <dependency org="org.codehaus.jackson"
+ name="jackson-core-asl"
+ rev="${jackson.version}"
+ conf="common->default"/>
</dependencies>
</ivy-module>
Modified:
hadoop/common/branches/branch-0.20-security-204/src/contrib/streaming/ivy/libraries.properties
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/contrib/streaming/ivy/libraries.properties?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-204/src/contrib/streaming/ivy/libraries.properties
(original)
+++
hadoop/common/branches/branch-0.20-security-204/src/contrib/streaming/ivy/libraries.properties
Sun Jun 19 06:30:55 2011
@@ -4,6 +4,5 @@
#Please list the dependencies name with version if they are different from the
ones
#listed in the global libraries.properties file (in alphabetical order)
-jackson.version=1.0.1
commons-configuration.version=1.6
commons-math.version=2.1
Modified:
hadoop/common/branches/branch-0.20-security-204/src/core/org/apache/hadoop/fs/FileUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/core/org/apache/hadoop/fs/FileUtil.java?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-204/src/core/org/apache/hadoop/fs/FileUtil.java
(original)
+++
hadoop/common/branches/branch-0.20-security-204/src/core/org/apache/hadoop/fs/FileUtil.java
Sun Jun 19 06:30:55 2011
@@ -22,17 +22,20 @@ import java.io.*;
import java.util.Enumeration;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
-import org.mortbay.log.Log;
/**
* A collection of file-processing util methods
*/
public class FileUtil {
+ private static final Log LOG = LogFactory.getLog(FileUtil.class);
+
/**
* convert an array of FileStatus to an array of Path
*
@@ -598,9 +601,9 @@ public class FileUtil {
try {
shExec.execute();
}catch(IOException e) {
- if(Log.isDebugEnabled()) {
- Log.debug("Error while changing permission : " + filename
- +" Exception: " + StringUtils.stringifyException(e));
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Error while changing permission : " + filename
+ +" Exception: " + StringUtils.stringifyException(e));
}
}
return shExec.getExitCode();
Modified:
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
(original)
+++
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
Sun Jun 19 06:30:55 2011
@@ -93,8 +93,8 @@ public class TrackerDistributedCacheMana
private static final Random random = new Random();
- BaseDirManager baseDirManager = new BaseDirManager();
- CleanupThread cleanupThread;
+ protected BaseDirManager baseDirManager = new BaseDirManager();
+ protected CleanupThread cleanupThread;
public TrackerDistributedCacheManager(Configuration conf,
TaskController controller
@@ -874,7 +874,7 @@ public class TrackerDistributedCacheMana
/**
* A thread to check and cleanup the unused files periodically
*/
- private class CleanupThread extends Thread {
+ protected class CleanupThread extends Thread {
// How often do we check if we need to clean up cache files?
private long cleanUpCheckPeriod = 60000L; // 1 minute
public CleanupThread(Configuration conf) {
@@ -882,6 +882,7 @@ public class TrackerDistributedCacheMana
conf.getLong("mapreduce.tasktracker.distributedcache.checkperiod",
cleanUpCheckPeriod);
}
+
private volatile boolean running = true;
public void stopRunning() {
@@ -894,19 +895,33 @@ public class TrackerDistributedCacheMana
try {
Thread.sleep(cleanUpCheckPeriod);
baseDirManager.checkAndCleanup();
- } catch (Exception e) {
+ } catch (IOException e) {
LOG.error("Exception in DistributedCache CleanupThread.", e);
- // This thread should keep running and never crash.
+ } catch(InterruptedException e) {
+ LOG.info("Cleanup...",e);
+ //To force us to exit cleanly
+ running = false;
+ } catch (Throwable t) {
+ exitTaskTracker(t);
}
}
}
+
+ /**
+ * Exit the task tracker because of a fatal error.
+ */
+ protected void exitTaskTracker(Throwable t) {
+ LOG.fatal("Distributed Cache cleanup thread received runtime exception."
+
+ " Exiting the TaskTracker", t);
+ Runtime.getRuntime().exit(-1);
+ }
}
/**
* This class holds properties of each base directories and is responsible
* for clean up unused cache files in base directories.
*/
- private class BaseDirManager {
+ protected class BaseDirManager {
// For holding the properties of each cache directory
private class CacheDir {
Modified:
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Sun Jun 19 06:30:55 2011
@@ -91,6 +91,7 @@ public class JobInProgress {
JobStatus status;
String jobFile = null;
Path localJobFile = null;
+ final QueueMetrics queueMetrics;
TaskInProgress maps[] = new TaskInProgress[0];
TaskInProgress reduces[] = new TaskInProgress[0];
@@ -339,8 +340,9 @@ public class JobInProgress {
this.resourceEstimator = new ResourceEstimator(this);
this.status = new JobStatus(jobid, 0.0f, 0.0f, JobStatus.PREP);
this.status.setUsername(conf.getUser());
+ String queueName = conf.getQueueName();
this.profile = new JobProfile(conf.getUser(), jobid, "", "",
- conf.getJobName(), conf.getQueueName());
+ conf.getJobName(), queueName);
this.memoryPerMap = conf.getMemoryForMapTask();
this.memoryPerReduce = conf.getMemoryForReduceTask();
this.maxTaskFailuresPerTracker = conf.getMaxTaskFailuresPerTracker();
@@ -377,6 +379,7 @@ public class JobInProgress {
this.status = new JobStatus(jobId, 0.0f, 0.0f, JobStatus.PREP);
this.status.setUsername(jobInfo.getUser().toString());
this.jobtracker.getInstrumentation().addPrepJob(conf, jobId);
+ // Add the queue-level metric below (after the profile has been
initialized)
this.startTime = jobtracker.getClock().getTime();
status.setStartTime(startTime);
this.localFs = jobtracker.getLocalFileSystem();
@@ -424,9 +427,9 @@ public class JobInProgress {
this.priority = conf.getJobPriority();
this.status.setJobPriority(this.priority);
+ String queueName = conf.getQueueName();
this.profile = new JobProfile(user, jobId,
- jobFile, url, conf.getJobName(),
- conf.getQueueName());
+ jobFile, url, conf.getJobName(), queueName);
Queue queue = this.jobtracker.getQueueManager().getQueue(queueName);
if (queue == null) {
@@ -487,7 +490,15 @@ public class JobInProgress {
FileSystem.closeAllForUGI(UserGroupInformation.getCurrentUser());
}
}
-
+
+ /**
+ * Get the QueueMetrics object associated with this job
+ * @return QueueMetrics
+ */
+ public QueueMetrics getQueueMetrics() {
+ return this.queueMetrics;
+ }
+
private void checkTaskLimits() throws IOException {
// if the number of tasks is larger than a configured value
// then fail the job.
@@ -695,6 +706,8 @@ public class JobInProgress {
jobtracker.getInstrumentation().addWaitingMaps(getJobID(), numMapTasks);
jobtracker.getInstrumentation().addWaitingReduces(getJobID(),
numReduceTasks);
+ this.queueMetrics.addWaitingMaps(getJobID(), numMapTasks);
+ this.queueMetrics.addWaitingReduces(getJobID(), numReduceTasks);
maps = new TaskInProgress[numMapTasks];
for(int i=0; i < numMapTasks; ++i) {
@@ -1695,6 +1708,7 @@ public class JobInProgress {
if (tip.getActiveTasks().size() > 1)
speculativeMapTasks++;
metrics.launchMap(id);
+ this.queueMetrics.launchMap(id);
} else {
++runningReduceTasks;
name = Values.REDUCE.name();
@@ -1702,6 +1716,7 @@ public class JobInProgress {
if (tip.getActiveTasks().size() > 1)
speculativeReduceTasks++;
metrics.launchReduce(id);
+ this.queueMetrics.launchReduce(id);
}
// Note that the logs are for the scheduled tasks only. Tasks that join on
// restart has already their logs in place.
@@ -1852,9 +1867,11 @@ public class JobInProgress {
map.put(taskTracker, info);
if (type == TaskType.MAP) {
jobtracker.getInstrumentation().addReservedMapSlots(reservedSlots);
+ this.queueMetrics.addReservedMapSlots(reservedSlots);
}
else {
jobtracker.getInstrumentation().addReservedReduceSlots(reservedSlots);
+ this.queueMetrics.addReservedReduceSlots(reservedSlots);
}
jobtracker.incrementReservations(type, reservedSlots);
}
@@ -1884,10 +1901,12 @@ public class JobInProgress {
map.remove(taskTracker);
if (type == TaskType.MAP) {
jobtracker.getInstrumentation().decReservedMapSlots(info.getNumSlots());
+ this.queueMetrics.decReservedMapSlots(info.getNumSlots());
}
else {
jobtracker.getInstrumentation().decReservedReduceSlots(
info.getNumSlots());
+ this.queueMetrics.decReservedReduceSlots(info.getNumSlots());
}
jobtracker.decrementReservations(type, info.getNumSlots());
}
@@ -2596,6 +2615,7 @@ public class JobInProgress {
}
finishedMapTasks += 1;
metrics.completeMap(taskid);
+ this.queueMetrics.completeMap(taskid);
// remove the completed map from the resp running caches
retireMap(tip);
if ((finishedMapTasks + failedMapTIPs) == (numMapTasks)) {
@@ -2611,6 +2631,7 @@ public class JobInProgress {
}
finishedReduceTasks += 1;
metrics.completeReduce(taskid);
+ this.queueMetrics.completeReduce(taskid);
// remove the completed reduces from the running reducers set
retireReduce(tip);
if ((finishedReduceTasks + failedReduceTIPs) == (numReduceTasks)) {
@@ -2655,14 +2676,18 @@ public class JobInProgress {
//update the metrics
if (oldState == JobStatus.PREP) {
this.jobtracker.getInstrumentation().decPrepJob(conf, jobId);
+ this.queueMetrics.decPrepJob(conf, jobId);
} else if (oldState == JobStatus.RUNNING) {
this.jobtracker.getInstrumentation().decRunningJob(conf, jobId);
+ this.queueMetrics.decRunningJob(conf, jobId);
}
if (newState == JobStatus.PREP) {
this.jobtracker.getInstrumentation().addPrepJob(conf, jobId);
+ this.queueMetrics.addPrepJob(conf, jobId);
} else if (newState == JobStatus.RUNNING) {
this.jobtracker.getInstrumentation().addRunningJob(conf, jobId);
+ this.queueMetrics.addRunningJob(conf, jobId);
}
}
@@ -2717,6 +2742,7 @@ public class JobInProgress {
garbageCollect();
metrics.completeJob(this.conf, this.status.getJobID());
+ this.queueMetrics.completeJob(this.conf, this.status.getJobID());
}
}
@@ -2757,9 +2783,11 @@ public class JobInProgress {
if (jobTerminationState == JobStatus.FAILED) {
jobtracker.getInstrumentation().failedJob(
this.conf, this.status.getJobID());
+ this.queueMetrics.failedJob(this.conf, this.status.getJobID());
} else {
jobtracker.getInstrumentation().killedJob(
this.conf, this.status.getJobID());
+ this.queueMetrics.killedJob(this.conf, this.status.getJobID());
}
}
}
@@ -2910,9 +2938,11 @@ public class JobInProgress {
if (tip.isMapTask() && !metricsDone) {
runningMapTasks -= 1;
metrics.failedMap(taskid);
+ this.queueMetrics.failedMap(taskid);
} else if (!metricsDone) {
runningReduceTasks -= 1;
metrics.failedReduce(taskid);
+ this.queueMetrics.failedReduce(taskid);
}
}
@@ -3155,6 +3185,8 @@ public class JobInProgress {
// Let the JobTracker know that a job is complete
jobtracker.getInstrumentation().decWaitingMaps(getJobID(),
pendingMaps());
jobtracker.getInstrumentation().decWaitingReduces(getJobID(),
pendingReduces());
+ this.queueMetrics.decWaitingMaps(getJobID(), pendingMaps());
+ this.queueMetrics.decWaitingReduces(getJobID(), pendingReduces());
jobtracker.storeCompletedJob(this);
jobtracker.finalizeJob(this);
Modified:
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobTracker.java
(original)
+++
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobTracker.java
Sun Jun 19 06:30:55 2011
@@ -4028,6 +4028,8 @@ public class JobTracker implements MRCon
}
}
myInstrumentation.submitJob(job.getJobConf(), jobId);
+ job.getQueueMetrics().submitJob(job.getJobConf(), jobId);
+
LOG.info("Job " + jobId + " added successfully for user '"
+ job.getJobConf().getUser() + "' to queue '"
+ job.getJobConf().getQueueName() + "'");
Modified:
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
(original)
+++
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
Sun Jun 19 06:30:55 2011
@@ -61,6 +61,7 @@ class LocalJobRunner implements JobSubmi
private final TaskController taskController = new DefaultTaskController();
private JobTrackerInstrumentation myMetrics = null;
+ private QueueMetrics queueMetrics = null;
private static final String jobDir = "localRunner/";
@@ -207,8 +208,10 @@ class LocalJobRunner implements JobSubmi
map.setConf(localConf);
map_tasks += 1;
myMetrics.launchMap(mapId);
+ queueMetrics.launchMap(mapId);
map.run(localConf, this);
myMetrics.completeMap(mapId);
+ queueMetrics.completeMap(mapId);
map_tasks -= 1;
updateCounters(map);
} else {
@@ -253,8 +256,10 @@ class LocalJobRunner implements JobSubmi
reduce.setConf(localConf);
reduce_tasks += 1;
myMetrics.launchReduce(reduce.getTaskID());
+ queueMetrics.launchReduce(reduce.getTaskID());
reduce.run(localConf, this);
myMetrics.completeReduce(reduce.getTaskID());
+ queueMetrics.completeReduce(reduce.getTaskID());
reduce_tasks -= 1;
updateCounters(reduce);
} else {
@@ -413,6 +418,7 @@ class LocalJobRunner implements JobSubmi
this.fs = FileSystem.getLocal(conf);
this.conf = conf;
myMetrics = JobTrackerInstrumentation.create(null, new JobConf(conf));
+ queueMetrics = QueueMetrics.create(conf.getQueueName(), new JobConf(conf));
taskController.setConf(conf);
}
Modified:
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/Queue.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/Queue.java?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/Queue.java
(original)
+++
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/Queue.java
Sun Jun 19 06:30:55 2011
@@ -37,6 +37,7 @@ class Queue {
private String name;
private Map<String,AccessControlList> acls;
private QueueState state = QueueState.RUNNING;
+ private QueueMetrics queueMetrics;
/**
* An Object that can be used by schedulers to fill in
@@ -69,10 +70,12 @@ class Queue {
* @param acls ACLs for the queue
* @param state state of the queue
*/
- Queue(String name, Map<String, AccessControlList> acls, QueueState state) {
+ Queue(String name, Map<String, AccessControlList> acls, QueueState state,
+ QueueMetrics metrics) {
this.name = name;
this.acls = acls;
this.state = state;
+ this.queueMetrics = metrics;
}
/**
@@ -149,4 +152,12 @@ class Queue {
void setSchedulingInfo(Object schedulingInfo) {
this.schedulingInfo = schedulingInfo;
}
+
+ /**
+ * Return the QueueMetrics object for this queue
+ * @return QueueMetrics
+ */
+ public QueueMetrics getMetrics() {
+ return this.queueMetrics;
+ }
}
Modified:
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/QueueManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/QueueManager.java?rev=1137300&r1=1137299&r2=1137300&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/QueueManager.java
(original)
+++
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/QueueManager.java
Sun Jun 19 06:30:55 2011
@@ -118,7 +118,7 @@ class QueueManager {
LOG.error("The queue, " + name + " does not have a configured ACL
list");
}
queues.put(name, new Queue(name, getQueueAcls(name, conf),
- getQueueState(name, conf)));
+ getQueueState(name, conf), QueueMetrics.create(name, conf)));
}
return queues;
@@ -136,7 +136,17 @@ class QueueManager {
public synchronized Set<String> getQueues() {
return queues.keySet();
}
-
+
+ /**
+ * Return a specific queue configured in the system.
+ *
+ * @param queueName Name of the queue requested
+ * @return Queue object corresponding to queueName
+ */
+ public synchronized Queue getQueue(String queueName) {
+ return queues.get(queueName);
+ }
+
/**
* Return true if the given user is part of the ACL for the given
* {@link QueueACL} name for the given queue.
