Author: acmurthy
Date: Wed Nov 21 11:16:15 2007
New Revision: 597185
URL: http://svn.apache.org/viewvc?rev=597185&view=rev
Log:
HADOOP-1843. Removed code from Configuration and JobConf deprecated by
HADOOP-785 and a minor fix to Configuration.toString. Specifically the
important change is that mapred-default.xml is no longer supported and
Configuration no longer supports the notion of default/final resources.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/contrib/abacus/src/java/org/apache/hadoop/abacus/ValueAggregatorJob.java
lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/conf/TestConfiguration.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=597185&r1=597184&r2=597185&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Wed Nov 21 11:16:15 2007
@@ -10,6 +10,12 @@
the value configured on the jobtracker. InterTrackerProtocol
version changed from 5 to 6. (Michael Bieniosek via omalley)
+ HADOOP-1843. Removed code from Configuration and JobConf deprecated by
+ HADOOP-785 and a minor fix to Configuration.toString. Specifically the
+ important change is that mapred-default.xml is no longer supported and
+ Configuration no longer supports the notion of default/final resources.
+ (acmurthy)
+
NEW FEATURES
HADOOP-1857. Ability to run a script when a task fails to capture stack
Modified:
lucene/hadoop/trunk/src/contrib/abacus/src/java/org/apache/hadoop/abacus/ValueAggregatorJob.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/abacus/src/java/org/apache/hadoop/abacus/ValueAggregatorJob.java?rev=597185&r1=597184&r2=597185&view=diff
==============================================================================
---
lucene/hadoop/trunk/src/contrib/abacus/src/java/org/apache/hadoop/abacus/ValueAggregatorJob.java
(original)
+++
lucene/hadoop/trunk/src/contrib/abacus/src/java/org/apache/hadoop/abacus/ValueAggregatorJob.java
Wed Nov 21 11:16:15 2007
@@ -133,7 +133,7 @@
JobConf theJob = new JobConf(ValueAggregatorJob.class);
if (specFile != null) {
- theJob.addDefaultResource(specFile);
+ theJob.addResource(specFile);
}
FileSystem fs = FileSystem.get(theJob);
theJob.setJobName("ValueAggregatorJob: " + jobName);
Modified:
lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java?rev=597185&r1=597184&r2=597185&view=diff
==============================================================================
---
lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
(original)
+++
lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
Wed Nov 21 11:16:15 2007
@@ -656,18 +656,18 @@
msg("hadoopAliasConf_ = " + hadoopAliasConf_);
config_ = new Configuration();
if (!cluster_.equals("default")) {
- config_.addFinalResource(new Path(getHadoopAliasConfFile()));
+ config_.addResource(new Path(getHadoopAliasConfFile()));
} else {
// use only defaults: hadoop-default.xml and hadoop-site.xml
}
System.out.println("additionalConfSpec_:" + additionalConfSpec_);
if (additionalConfSpec_ != null) {
- config_.addDefaultResource(new Path(additionalConfSpec_));
+ config_.addResource(new Path(additionalConfSpec_));
}
Iterator it = configPath_.iterator();
while (it.hasNext()) {
String pathName = (String) it.next();
- config_.addFinalResource(new Path(pathName));
+ config_.addResource(new Path(pathName));
}
// general MapRed job properties
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java?rev=597185&r1=597184&r2=597185&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java Wed
Nov 21 11:16:15 2007
@@ -129,17 +129,6 @@
private boolean quietmode = true;
/**
- * @deprecated Remove in hadoop-0.16.0 via HADOOP-1843
- */
- @Deprecated
- private ArrayList<Object> defaultResources = new ArrayList<Object>();
- /**
- * @deprecated Remove in hadoop-0.16.0 via HADOOP-1843
- */
- @Deprecated
- private ArrayList<Object> finalResources = new ArrayList<Object>();
-
- /**
* List of configuration resources.
*/
private ArrayList<Object> resources = new ArrayList<Object>();
@@ -164,8 +153,8 @@
if (LOG.isDebugEnabled()) {
LOG.debug(StringUtils.stringifyException(new IOException("config()")));
}
- defaultResources.add("hadoop-default.xml");
- finalResources.add("hadoop-site.xml");
+ resources.add("hadoop-default.xml");
+ resources.add("hadoop-site.xml");
}
/**
@@ -179,8 +168,6 @@
LOG.debug(StringUtils.stringifyException
(new IOException("config(config)")));
}
- this.defaultResources = (ArrayList)other.defaultResources.clone();
- this.finalResources = (ArrayList)other.finalResources.clone();
this.resources = (ArrayList)other.resources.clone();
if (other.properties != null)
this.properties = (Properties)other.properties.clone();
@@ -189,60 +176,6 @@
this.finalParameters = new HashSet<String>(other.finalParameters);
}
- /**
- * Add a default resource.
- * @deprecated Use [EMAIL PROTECTED] #addResource(String)} instead
- */
- @Deprecated
- public void addDefaultResource(String name) {
- addResource(defaultResources, name);
- }
-
- /**
- * Add a default resource.
- * @deprecated Use [EMAIL PROTECTED] #addResource(URL)} instead
- */
- @Deprecated
- public void addDefaultResource(URL url) {
- addResource(defaultResources, url);
- }
-
- /**
- * Add a default resource.
- * @deprecated Use [EMAIL PROTECTED] #addResource(Path)} instead
- */
- @Deprecated
- public void addDefaultResource(Path file) {
- addResource(defaultResources, file);
- }
-
- /**
- * Add a final resource.
- * @deprecated Use [EMAIL PROTECTED] #addResource(String)} instead
- */
- @Deprecated
- public void addFinalResource(String name) {
- addResource(finalResources, name);
- }
-
- /**
- * Add a final resource.
- * @deprecated Use [EMAIL PROTECTED] #addResource(URL)} instead
- */
- @Deprecated
- public void addFinalResource(URL url) {
- addResource(finalResources, url);
- }
-
- /**
- * Add a final resource.
- * @deprecated Use [EMAIL PROTECTED] #addResource(Path)} instead
- */
- @Deprecated
- public void addFinalResource(Path file) {
- addResource(finalResources, file);
- }
-
/**
* Add a configuration resource.
*
@@ -743,8 +676,6 @@
private synchronized Properties getProps() {
if (properties == null) {
properties = new Properties();
- loadResources(properties, defaultResources, quietmode);
- loadResources(properties, finalResources, quietmode);
loadResources(properties, resources, quietmode);
if (overlay!= null)
properties.putAll(overlay);
@@ -956,10 +887,7 @@
public String toString() {
StringBuffer sb = new StringBuffer();
sb.append("Configuration: ");
- sb.append("defaults: ");
- toString(defaultResources, sb);
- sb.append("final: ");
- toString(finalResources, sb);
+ toString(resources, sb);
return sb.toString();
}
@@ -967,7 +895,7 @@
ListIterator i = resources.listIterator();
while (i.hasNext()) {
if (i.nextIndex() != 0) {
- sb.append(" , ");
+ sb.append(", ");
}
sb.append(i.next());
}
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java?rev=597185&r1=597184&r2=597185&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java Wed Nov
21 11:16:15 2007
@@ -112,9 +112,7 @@
/**
* Construct a map/reduce job configuration.
*/
- public JobConf() {
- checkWarnAndLoadMapredDefault();
- }
+ public JobConf() {}
/**
* Construct a map/reduce job configuration.
@@ -122,7 +120,6 @@
* @param exampleClass a class whose containing jar is used as the job's jar.
*/
public JobConf(Class exampleClass) {
- checkWarnAndLoadMapredDefault();
setJarByClass(exampleClass);
}
@@ -133,7 +130,6 @@
*/
public JobConf(Configuration conf) {
super(conf);
- checkWarnAndLoadMapredDefault();
}
@@ -162,25 +158,9 @@
*/
public JobConf(Path config) {
super();
- checkWarnAndLoadMapredDefault();
addResource(config);
}
- /**
- * Checks if <b>mapred-default.xml</b> is on the CLASSPATH, if so
- * it warns the user and loads it as a [EMAIL PROTECTED] Configuration}
resource.
- *
- * @deprecated Remove in hadoop-0.16.0 via HADOOP-1843
- */
- private void checkWarnAndLoadMapredDefault() {
- URL mapredDefaultConf = getClassLoader().getResource("mapred-default.xml");
- if (mapredDefaultConf != null) {
- LOG.warn("Deprecated resource 'mapred-default.xml' is being loaded, " +
- "please discontinue its usage!");
- addResource("mapred-default.xml");
- }
- }
-
/**
* Get the user jar for the map-reduce job.
*
Modified:
lucene/hadoop/trunk/src/test/org/apache/hadoop/conf/TestConfiguration.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/conf/TestConfiguration.java?rev=597185&r1=597184&r2=597185&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/conf/TestConfiguration.java
(original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/conf/TestConfiguration.java
Wed Nov 21 11:16:15 2007
@@ -185,6 +185,19 @@
//two spaces one after "this", one before "contains"
assertEquals("this contains a comment", conf.get("my.comment"));
}
+
+ public void testToString() throws IOException {
+ out=new BufferedWriter(new FileWriter(CONFIG));
+ startConfig();
+ endConfig();
+ Path fileResource = new Path(CONFIG);
+ conf.addResource(fileResource);
+
+ String expectedOutput =
+ "Configuration: hadoop-default.xml, hadoop-site.xml, " +
+ fileResource.toString();
+ assertEquals(expectedOutput, conf.toString());
+ }
BufferedWriter out;
