Author: acmurthy
Date: Mon Jan 7 11:04:38 2008
New Revision: 609741
URL: http://svn.apache.org/viewvc?rev=609741&view=rev
Log:
Removed support for deprecated mapred.child.heap.size and fixed some
indentation issues in TaskRunner.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/conf/hadoop-default.xml
lucene/hadoop/trunk/docs/hadoop-default.html
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=609741&r1=609740&r2=609741&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Mon Jan 7 11:04:38 2008
@@ -345,6 +345,11 @@
HADOOP-2523. Fix TestDFSShell.testFilePermissions on Windows.
(Raghu Angadi via nigel)
+ HADOOP-2535. Removed support for deprecated mapred.child.heap.size and
+ fixed some indentation issues in TaskRunner. (acmurthy)
+ Configuration changes to hadoop-default.xml:
+ remove mapred.child.heap.size
+
Release 0.15.2 - 2008-01-02
BUG FIXES
Modified: lucene/hadoop/trunk/conf/hadoop-default.xml
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/conf/hadoop-default.xml?rev=609741&r1=609740&r2=609741&view=diff
==============================================================================
--- lucene/hadoop/trunk/conf/hadoop-default.xml (original)
+++ lucene/hadoop/trunk/conf/hadoop-default.xml Mon Jan 7 11:04:38 2008
@@ -648,13 +648,10 @@
<property>
<name>mapred.child.java.opts</name>
<value>-Xmx200m</value>
- <description>Java opts for the task tracker child processes. Subsumes
- 'mapred.child.heap.size' (If a mapred.child.heap.size value is found
- in a configuration, its maximum heap size will be used and a warning
- emitted that heap.size has been deprecated). Also, the following symbol,
- if present, will be interpolated: @taskid@ is replaced by current TaskID.
- Any other occurrences of '@' will go unchanged. For
- example, to enable verbose gc logging to a file named for the taskid in
+ <description>Java opts for the task tracker child processes.
+ The following symbol, if present, will be interpolated: @taskid@ is replaced
+ by current TaskID. Any other occurrences of '@' will go unchanged.
+ For example, to enable verbose gc logging to a file named for the taskid in
/tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
-Xmx1024m -verbose:gc -Xloggc:/tmp/@[EMAIL PROTECTED]
</description>
Modified: lucene/hadoop/trunk/docs/hadoop-default.html
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/docs/hadoop-default.html?rev=609741&r1=609740&r2=609741&view=diff
==============================================================================
--- lucene/hadoop/trunk/docs/hadoop-default.html (original)
+++ lucene/hadoop/trunk/docs/hadoop-default.html Mon Jan 7 11:04:38 2008
@@ -163,6 +163,18 @@
directories, for redundancy. </td>
</tr>
<tr>
+<td><a name="dfs.permissions">dfs.permissions</a></td><td>false</td><td>
+ If "true", enable permission checking in HDFS.
+ If "false", permission checking is turned off,
+ but all other behavior is unchanged.
+ Switching from one parameter value to the other does not change the mode,
+ owner or group of files or directories.
+ </td>
+</tr>
+<tr>
+<td><a
name="dfs.permissions.supergroup">dfs.permissions.supergroup</a></td><td>supergroup</td><td>The
name of the group of super-users.</td>
+</tr>
+<tr>
<td><a
name="dfs.client.buffer.dir">dfs.client.buffer.dir</a></td><td>${hadoop.tmp.dir}/dfs/tmp</td><td>Determines
where on the local filesystem an DFS client
should store its blocks before it sends them to the datanode.
</td>
@@ -383,13 +395,10 @@
</td>
</tr>
<tr>
-<td><a
name="mapred.child.java.opts">mapred.child.java.opts</a></td><td>-Xmx200m</td><td>Java
opts for the task tracker child processes. Subsumes
- 'mapred.child.heap.size' (If a mapred.child.heap.size value is found
- in a configuration, its maximum heap size will be used and a warning
- emitted that heap.size has been deprecated). Also, the following symbol,
- if present, will be interpolated: @taskid@ is replaced by current TaskID.
- Any other occurrences of '@' will go unchanged. For
- example, to enable verbose gc logging to a file named for the taskid in
+<td><a
name="mapred.child.java.opts">mapred.child.java.opts</a></td><td>-Xmx200m</td><td>Java
opts for the task tracker child processes.
+ The following symbol, if present, will be interpolated: @taskid@ is replaced
+ by current TaskID. Any other occurrences of '@' will go unchanged.
+ For example, to enable verbose gc logging to a file named for the taskid in
/tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
-Xmx1024m -verbose:gc -Xloggc:/tmp/@[EMAIL PROTECTED]
</td>
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java?rev=609741&r1=609740&r2=609741&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java Mon
Jan 7 11:04:38 2008
@@ -269,10 +269,7 @@
vargs.add(jvm.toString());
- // Add child java ops. Also, mapred.child.heap.size has been superceded
- // by // mapred.child.java.opts. Manage case where both are present
- // letting the mapred.child.heap.size win over any setting of heap size
in
- // mapred.child.java.opts (Emit a warning that heap.size is deprecated).
+ // Add child (task) java-vm options.
//
// The following symbols if present in mapred.child.java.opts value are
// replaced:
@@ -284,65 +281,67 @@
// connect with jconsole and the likes to watch child memory, threads
// and get thread dumps.
//
- // <name>mapred.child.optional.jvm.args</name>
- // <value>-verbose:gc -Xloggc:/tmp/@[EMAIL PROTECTED] \
- // -Dcom.sun.management.jmxremote.authenticate=false \
- // -Dcom.sun.management.jmxremote.ssl=false \
- // </value>
- //
- String javaOpts = handleDeprecatedHeapSize(
-
conf.get("mapred.child.java.opts", "-Xmx200m"),
-
conf.get("mapred.child.heap.size"));
- javaOpts = replaceAll(javaOpts, "@taskid@", taskid);
- String [] javaOptsSplit = javaOpts.split(" ");
- //Add java.library.path; necessary for native-hadoop libraries
- String libraryPath = System.getProperty("java.library.path");
- if (libraryPath != null) {
- boolean hasLibrary = false;
- for(int i=0; i<javaOptsSplit.length ;i++) {
- if(javaOptsSplit[i].startsWith("-Djava.library.path=")) {
- javaOptsSplit[i] += sep + libraryPath;
- hasLibrary = true;
- break;
- }
+ // <property>
+ // <name>mapred.child.java.opts</name>
+ // <value>-verbose:gc -Xloggc:/tmp/@[EMAIL PROTECTED] \
+ // -Dcom.sun.management.jmxremote.authenticate=false \
+ // -Dcom.sun.management.jmxremote.ssl=false \
+ // </value>
+ // </property>
+ //
+ String javaOpts = conf.get("mapred.child.java.opts", "-Xmx200m");
+ javaOpts = replaceAll(javaOpts, "@taskid@", taskid);
+ String [] javaOptsSplit = javaOpts.split(" ");
+
+ //Add java.library.path; necessary for native-hadoop libraries
+ String libraryPath = System.getProperty("java.library.path");
+ if (libraryPath != null) {
+ boolean hasLibrary = false;
+ for(int i=0; i<javaOptsSplit.length ;i++) {
+ if(javaOptsSplit[i].startsWith("-Djava.library.path=")) {
+ javaOptsSplit[i] += sep + libraryPath;
+ hasLibrary = true;
+ break;
}
- if(!hasLibrary)
- vargs.add("-Djava.library.path=" + libraryPath);
- }
- for (int i = 0; i < javaOptsSplit.length; i++) {
- vargs.add(javaOptsSplit[i]);
}
-
- // Add classpath.
- vargs.add("-classpath");
- vargs.add(classPath.toString());
+ if(!hasLibrary)
+ vargs.add("-Djava.library.path=" + libraryPath);
+ }
+ for (int i = 0; i < javaOptsSplit.length; i++) {
+ vargs.add(javaOptsSplit[i]);
+ }
- // Setup the log4j prop
- long logSize = TaskLog.getTaskLogLength(conf);
- vargs.add("-Dhadoop.log.dir=" +
- new File(System.getProperty("hadoop.log.dir")
- ).getAbsolutePath());
- vargs.add("-Dhadoop.root.logger=INFO,TLA");
- vargs.add("-Dhadoop.tasklog.taskid=" + taskid);
- vargs.add("-Dhadoop.tasklog.totalLogFileSize=" + logSize);
+ // Add classpath.
+ vargs.add("-classpath");
+ vargs.add(classPath.toString());
- // Add main class and its arguments
- vargs.add(TaskTracker.Child.class.getName()); // main of Child
- // pass umbilical address
- InetSocketAddress address = tracker.getTaskTrackerReportAddress();
- vargs.add(address.getAddress().getHostAddress());
- vargs.add(Integer.toString(address.getPort()));
- vargs.add(taskid); // pass task identifier
+ // Setup the log4j prop
+ long logSize = TaskLog.getTaskLogLength(conf);
+ vargs.add("-Dhadoop.log.dir=" +
+ new File(System.getProperty("hadoop.log.dir")
+ ).getAbsolutePath());
+ vargs.add("-Dhadoop.root.logger=INFO,TLA");
+ vargs.add("-Dhadoop.tasklog.taskid=" + taskid);
+ vargs.add("-Dhadoop.tasklog.totalLogFileSize=" + logSize);
- // Run java
- File stdout = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDOUT);
- File stderr = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDERR);
- stdout.getParentFile().mkdirs();
+ // Add main class and its arguments
+ vargs.add(TaskTracker.Child.class.getName()); // main of Child
+ // pass umbilical address
+ InetSocketAddress address = tracker.getTaskTrackerReportAddress();
+ vargs.add(address.getAddress().getHostAddress());
+ vargs.add(Integer.toString(address.getPort()));
+ vargs.add(taskid); // pass task identifier
+
+ // Set up the redirection of the task's stdout and stderr streams
+ File stdout = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDOUT);
+ File stderr = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDERR);
+ stdout.getParentFile().mkdirs();
+ List<String> wrappedCommand =
+ TaskLog.captureOutAndError(vargs, stdout, stderr, logSize);
+
+ // Run the task as child of the parent TaskTracker process
+ runChild(wrappedCommand, workDir, taskid);
- List<String> wrappedCommand =
- TaskLog.captureOutAndError(vargs, stdout, stderr, logSize);
- runChild(wrappedCommand, workDir, taskid);
-
} catch (FSError e) {
LOG.fatal("FSError", e);
try {
@@ -378,36 +377,6 @@
}
tracker.reportTaskFinished(t.getTaskId());
}
- }
-
-
- /**
- * Handle deprecated mapred.child.heap.size.
- * If present, interpolate into mapred.child.java.opts value with
- * warning.
- * @param javaOpts Value of mapred.child.java.opts property.
- * @param heapSize Value of mapred.child.heap.size property.
- * @return A <code>javaOpts</code> with <code>heapSize</code>
- * interpolated if present.
- */
- private String handleDeprecatedHeapSize(String javaOpts,
- final String heapSize) {
- if (heapSize == null || heapSize.length() <= 0) {
- return javaOpts;
- }
- final String MX = "-Xmx";
- int index = javaOpts.indexOf(MX);
- if (index < 0) {
- javaOpts = javaOpts + " " + MX + heapSize;
- } else {
- int end = javaOpts.indexOf(" ", index + MX.length());
- javaOpts = javaOpts.substring(0, index + MX.length()) +
- heapSize + ((end < 0)? "": javaOpts.substring(end));
- }
- LOG.warn("mapred.child.heap.size is deprecated. Use " +
- "mapred.child.java.opt instead. Meantime, mapred.child.heap.size
" +
- "is interpolated into mapred.child.java.opt: " + javaOpts);
- return javaOpts;
}
/**
