svn commit: r378095 - in /lucene/hadoop/trunk: conf/hadoop-default.xml src/java/org/apache/hadoop/mapred/TaskInProgress.java src/test/org/apache/hadoop/fs/TestFileSystem.java

[cutting]

Author: cutting
Date: Wed Feb 15 13:24:25 2006
New Revision: 378095

URL: http://svn.apache.org/viewcvs?rev=378095&view=rev
Log:
Make speculative execution optional, since it can break map tasks that have 
side effects.  Also fix TestFileSystem to be safe to use with speculative 
execution.

Modified:
    lucene/hadoop/trunk/conf/hadoop-default.xml
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskInProgress.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java

Modified: lucene/hadoop/trunk/conf/hadoop-default.xml
URL: 
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/conf/hadoop-default.xml?rev=378095&r1=378094&r2=378095&view=diff
==============================================================================
--- lucene/hadoop/trunk/conf/hadoop-default.xml (original)
+++ lucene/hadoop/trunk/conf/hadoop-default.xml Wed Feb 15 13:24:25 2006
@@ -203,6 +203,13 @@
   combining them and writing to disk.</description>
 </property>
 
+<property>
+  <name>mapred.speculative.execution</name>
+  <value>true</value>
+  <description>If true, then multiple instances of some map tasks may
+  be executed in parallel.</description>
+</property>
+
 
 <!-- ipc properties -->
 

Modified: 
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskInProgress.java
URL: 
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskInProgress.java?rev=378095&r1=378094&r2=378095&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskInProgress.java 
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskInProgress.java 
Wed Feb 15 13:24:25 2006
@@ -407,7 +407,8 @@
         // REMIND - mjc - these constants should be examined
         // in more depth eventually...
         //
-        if (isMapTask() && 
+        if (isMapTask() &&
+            conf.getBoolean("mapred.speculative.execution", true) &&
             (averageProgress - progress >= SPECULATIVE_GAP) &&
             (System.currentTimeMillis() - startTime >= SPECULATIVE_LAG)) {
             return true;

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java
URL: 
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java?rev=378095&r1=378094&r2=378095&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java 
(original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java Wed 
Feb 15 13:24:25 2006
@@ -103,6 +103,9 @@
     private byte[] buffer = new byte[BUFFER_SIZE];
     private FileSystem fs;
     private boolean fastCheck;
+
+    // a random suffix per task
+    private String suffix = "-"+random.nextLong();
     
     {
       try {
@@ -131,7 +134,9 @@
       random.setSeed(seed);
       reporter.setStatus("creating " + name);
 
-      OutputStream out = fs.create(new File(DATA_DIR, name));
+      // write to temp file initially to permit parallel execution
+      File tempFile = new File(DATA_DIR, name+suffix);
+      OutputStream out = fs.create(tempFile);
 
       long written = 0;
       try {
@@ -150,6 +155,8 @@
       } finally {
         out.close();
       }
+      // rename to final location
+      fs.rename(tempFile, new File(DATA_DIR, name));
 
       collector.collect(new UTF8("bytes"), new LongWritable(written));