svn commit: r675541 - in /hadoop/core/trunk: CHANGES.txt src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingKeyValue.java

[ddas]

Author: ddas
Date: Thu Jul 10 05:18:27 2008
New Revision: 675541

URL: http://svn.apache.org/viewvc?rev=675541&view=rev
Log:
HADOOP-3711. Fixes the Streaming input parsing to properly find the separator. 
Contributed by Amareshwari Sriramadasu.

Modified:
    hadoop/core/trunk/CHANGES.txt
    
hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
    
hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingKeyValue.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=675541&r1=675540&r2=675541&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Jul 10 05:18:27 2008
@@ -84,6 +84,9 @@
     HADOOP-3640. Fix the read method in the NativeS3InputStream. (tomwhite via
     omalley)
 
+    HADOOP-3711. Fixes the Streaming input parsing to properly find the 
+    separator. (Amareshwari Sriramadasu via ddas)
+
 Release 0.18.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
URL: 
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java?rev=675541&r1=675540&r2=675541&view=diff
==============================================================================
--- 
hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
 (original)
+++ 
hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
 Thu Jul 10 05:18:27 2008
@@ -322,10 +322,10 @@
     byte[] separator = getFieldSeparator();
     
     // Need to find numKeyFields separators
-    int pos = UTF8ByteArrayUtils.findBytes(line, 0, line.length, separator);
+    int pos = UTF8ByteArrayUtils.findBytes(line, 0, length, separator);
     for(int k=1; k<numKeyFields && pos!=-1; k++) {
       pos = UTF8ByteArrayUtils.findBytes(line, pos + separator.length, 
-          line.length, separator);
+          length, separator);
     }
     try {
       if (pos == -1) {

Modified: 
hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingKeyValue.java
URL: 
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingKeyValue.java?rev=675541&r1=675540&r2=675541&view=diff
==============================================================================
--- 
hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingKeyValue.java
 (original)
+++ 
hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingKeyValue.java
 Thu Jul 10 05:18:27 2008
@@ -35,9 +35,14 @@
   // Third line of input does not have any tab character.
   // So, the whole line is the key and value is empty.
   protected String input = 
-    "roses are \tred\t\n\tviolets are blue\nbunnies are pink\n";
+    "roses are \tred\t\n\tviolets are blue\nbunnies are pink\n" +
+    "this is for testing a big\tinput line\n" +
+    "small input\n";
   protected String outputExpect = 
-    "\tviolets are blue\nbunnies are pink\t\nroses are \tred\t\n";
+    "\tviolets are blue\nbunnies are pink\t\n" + 
+    "roses are \tred\t\n" +
+    "small input\t\n" +
+    "this is for testing a big\tinput line\n";
 
   private StreamJob job;