RE: java.lang.RuntimeException: java.io.EOFException at org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:103)
You are right, there is no readInt , I have only 2 String fields in MsRead Here are lines: I'll also will send both files in attch. Thanks in advance for your help. @Override public void readFields(DataInput in) throws IOException { query_id = in.readUTF(); record = in.readUTF(); } @Override public void write(DataOutput out) throws IOException { out.writeUTF(query_id); out.writeUTF(record); } public static class FirstComparator extends WritableComparator { private static final Text.Comparator TEXT_COMPARATOR = new Text.Comparator(); public FirstComparator() { super(MsRead.class); } @Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { try { int firstL1 = WritableUtils.decodeVIntSize(b1[s1]) + readVInt(b1, s1); int firstL2 = WritableUtils.decodeVIntSize(b2[s2]) + readVInt(b2, s2); return TEXT_COMPARATOR.compare(b1, s1, firstL1, b2, s2, firstL2); } catch (IOException e) { throw new IllegalArgumentException(e); } } @Override public int compare(WritableComparable a, WritableComparable b) { if (a instanceof MsRead b instanceof MsRead) { //System.err.println(COMPARE + ((MsRead)a).getType() + \t + ((MsRead)b).getType() + \t //+ (((MsRead) a).toString().compareTo(((MsRead) b).toString(; return (((MsRead) a).toString().compareTo(((MsRead) b).toString())); } return super.compare(a, b); } } @Override public int compareTo(MsRead o) { return this.toString().compareTo(o.toString()); } @Override public boolean equals(Object right) { if (right instanceof MsRead ) { return (query_id.equals(((MsRead)right).query_id)); } else return false; } @Override public int hashCode() { return query_id.hashCode() ; } Date: Wed, 29 Sep 2010 22:27:15 -0700 Subject: Re: java.lang.RuntimeException: java.io.EOFException at org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:103) From: yuzhih...@gmail.com To: common-user@hadoop.apache.org Your MsRead.readFields() doesn't contain readInt(). Can you show us the lines around line 84 of MsRead.java ? On Wed, Sep 29, 2010 at 2:44 PM, Tali K ncherr...@hotmail.com wrote: HI All, I am getting this Exception on a cluster(10 nodes) when I am running simple hadoop map / reduce job. I don't have this Exception while running it on my desktop in hadoop's pseudo distributed mode. Can somebody help? I would really appreciate it. 10/09/29 14:28:34 INFO mapred.JobClient: map 100% reduce 30% 10/09/29 14:28:36 INFO mapred.JobClient: Task Id : attempt_201009291306_0004_r_00_0, Status : FAILED java.lang.RuntimeException: java.io.EOFException at org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:103) at org.apache.hadoop.mapred.Merger$MergeQueue.lessThan(Merger.java:373) at org.apache.hadoop.util.PriorityQueue.upHeap(PriorityQueue.java:123) at org.apache.hadoop.util.PriorityQueue.put(PriorityQueue.java:50) at org.apache.hadoop.mapred.Merger$MergeQueue.merge(Merger.java:447) at org.apache.hadoop.mapred.Merger$MergeQueue.merge(Merger.java:381) at org.apache.hadoop.mapred.Merger.merge(Merger.java:107) at org.apache.hadoop.mapred.Merger.merge(Merger.java:93) at org.apache.hadoop.mapred.ReduceTask$ReduceCopier.createKVIterator(ReduceTask.java:2316) at org.apache.hadoop.mapred.ReduceTask$ReduceCopier.access$400(ReduceTask.java:576) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389) at org.apache.hadoop.mapred.Child.main(Child.java:170) Caused by: java.io.EOFException at java.io.DataInputStream.readInt(DataInputStream.java:375) at speeditup.MsRead.readFields(MsRead.java:84) at org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:97) ... 11 more Here is a class that has WritableComparator.compare. It has only 2 strings max length 20 characters for each. public class MsRead implements WritableComparable MsRead { private static final Log LOG = LogFactory.getLog(speeditup.CalculateMinEvalue.class); private String query_id; private String record; public String getRecord() { return record; } public void setRecord(String record) { this.record = record; } public String getQuery_id
RE: java.lang.RuntimeException: java.io.EOFException at org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:103)
May be that additional information can help. That is a message from the tasktracker where the error occured. 2010-09-29 14:28:30,761 INFO org.apache.hadoop.mapred.ReduceTask: GetMapEventsThread exiting 2010-09-29 14:28:30,761 INFO org.apache.hadoop.mapred.ReduceTask: getMapsEventsThread joined. 2010-09-29 14:28:30,761 INFO org.apache.hadoop.mapred.ReduceTask: Closed ram manager 2010-09-29 14:28:30,764 INFO org.apache.hadoop.mapred.ReduceTask: Interleaved on-disk merge complete: 0 files left. 2010-09-29 14:28:30,764 INFO org.apache.hadoop.mapred.ReduceTask: In-memory merge complete: 14 files left. 2010-09-29 14:28:30,794 INFO org.apache.hadoop.mapred.Merger: Merging 14 sorted segments 2010-09-29 14:28:30,798 WARN org.apache.hadoop.mapred.TaskTracker: Error running child
RE: java.lang.RuntimeException: java.io.EOFException at org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:103)
I have only 2 Strings, no lists or primitives, and I am not concatenating strings. So all my class has is : String a; String b; Date: Thu, 30 Sep 2010 10:42:38 -0700 Subject: Re: java.lang.RuntimeException: java.io.EOFException at org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:103) From: yuzhih...@gmail.com To: common-user@hadoop.apache.org Line 84 is empty. Line 83 is: out.writeUTF(query_id); Please send the stack trace that corresponds to your attachment. From previous discussion: In the very begining of readFields(), clear all available fields (lists, primitives, etc). The best way to to do that is to create a clearFields() function, that will be called both from readFields() and from the empty constructor. On Thu, Sep 30, 2010 at 10:16 AM, Tali K ncherr...@hotmail.com wrote: *You are right, there is no readInt , I have only 2 String fields in MsRead Here are lines: I'll also will send both files in attch. Thanks in advance for your help.* @Override public void readFields(DataInput in) throws IOException { query_id = in.readUTF(); record = in.readUTF(); } @Override public void write(DataOutput out) throws IOException { out.writeUTF(query_id); out.writeUTF(record); } public static class FirstComparator extends WritableComparator { private static final Text.Comparator TEXT_COMPARATOR = new Text.Comparator(); public FirstComparator() { super(MsRead.class); } @Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { try { int firstL1 = WritableUtils.decodeVIntSize(b1[s1]) + readVInt(b1, s1); int firstL2 = WritableUtils.decodeVIntSize(b2[s2]) + readVInt(b2, s2); return TEXT_COMPARATOR.compare(b1, s1, firstL1, b2, s2, firstL2); } catch (IOException e) { throw new IllegalArgumentException(e); } } @Override public int compare(WritableComparable a, WritableComparable b) { if (a instanceof MsRead b instanceof MsRead) { //System.err.println(COMPARE + ((MsRead)a).getType() + \t + ((MsRead)b).getType() + \t // + (((MsRead) a).toString().compareTo(((MsRead) b).toString(; return (((MsRead) a).toString().compareTo(((MsRead) b).toString())); } return super.compare(a, b); } } @Override public int compareTo(MsRead o) { return this.toString().compareTo(o.toString()); } @Override public boolean equals(Object right) { if (right instanceof MsRead ) { return (query_id.equals(((MsRead)right).query_id)); } else return false; } @Override public int hashCode() { return query_id.hashCode() ; } Date: Wed, 29 Sep 2010 22:27:15 -0700 Subject: Re: java.lang.RuntimeException: java.io.EOFException at org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:103) From: yuzhih...@gmail.com To: common-user@hadoop.apache.org Your MsRead.readFields() doesn't contain readInt(). Can you show us the lines around line 84 of MsRead.java ? On Wed, Sep 29, 2010 at 2:44 PM, Tali K ncherr...@hotmail.com wrote: HI All, I am getting this Exception on a cluster(10 nodes) when I am running simple hadoop map / reduce job. I don't have this Exception while running it on my desktop in hadoop's pseudo distributed mode. Can somebody help? I would really appreciate it. 10/09/29 14:28:34 INFO mapred.JobClient: map 100% reduce 30% 10/09/29 14:28:36 INFO mapred.JobClient: Task Id : attempt_201009291306_0004_r_00_0, Status : FAILED java.lang.RuntimeException: java.io.EOFException at org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:103) at org.apache.hadoop.mapred.Merger$MergeQueue.lessThan(Merger.java:373) at org.apache.hadoop.util.PriorityQueue.upHeap(PriorityQueue.java:123) at org.apache.hadoop.util.PriorityQueue.put(PriorityQueue.java:50) at org.apache.hadoop.mapred.Merger$MergeQueue.merge(Merger.java:447) at org.apache.hadoop.mapred.Merger$MergeQueue.merge(Merger.java:381) at org.apache.hadoop.mapred.Merger.merge(Merger.java:107) at org.apache.hadoop.mapred.Merger.merge(Merger.java:93) at org.apache.hadoop.mapred.ReduceTask$ReduceCopier.createKVIterator(ReduceTask.java:2316) at org.apache.hadoop.mapred.ReduceTask$ReduceCopier.access$400(ReduceTask.java:576) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389) at org.apache.hadoop.mapred.Child.main(Child.java:170) Caused by: java.io.EOFException at java.io.DataInputStream.readInt(DataInputStream.java:375) at speeditup.MsRead.readFields(MsRead.java:84) at org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:97) ... 11 more Here is a class that has WritableComparator.compare. It has only 2 strings max length 20 characters for each. public class MsRead
Re: java.lang.RuntimeException: java.io.EOFException at org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:103)
Your MsRead.readFields() doesn't contain readInt(). Can you show us the lines around line 84 of MsRead.java ? On Wed, Sep 29, 2010 at 2:44 PM, Tali K ncherr...@hotmail.com wrote: HI All, I am getting this Exception on a cluster(10 nodes) when I am running simple hadoop map / reduce job. I don't have this Exception while running it on my desktop in hadoop's pseudo distributed mode. Can somebody help? I would really appreciate it. 10/09/29 14:28:34 INFO mapred.JobClient: map 100% reduce 30% 10/09/29 14:28:36 INFO mapred.JobClient: Task Id : attempt_201009291306_0004_r_00_0, Status : FAILED java.lang.RuntimeException: java.io.EOFException at org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:103) at org.apache.hadoop.mapred.Merger$MergeQueue.lessThan(Merger.java:373) at org.apache.hadoop.util.PriorityQueue.upHeap(PriorityQueue.java:123) at org.apache.hadoop.util.PriorityQueue.put(PriorityQueue.java:50) at org.apache.hadoop.mapred.Merger$MergeQueue.merge(Merger.java:447) at org.apache.hadoop.mapred.Merger$MergeQueue.merge(Merger.java:381) at org.apache.hadoop.mapred.Merger.merge(Merger.java:107) at org.apache.hadoop.mapred.Merger.merge(Merger.java:93) at org.apache.hadoop.mapred.ReduceTask$ReduceCopier.createKVIterator(ReduceTask.java:2316) at org.apache.hadoop.mapred.ReduceTask$ReduceCopier.access$400(ReduceTask.java:576) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389) at org.apache.hadoop.mapred.Child.main(Child.java:170) Caused by: java.io.EOFException at java.io.DataInputStream.readInt(DataInputStream.java:375) at speeditup.MsRead.readFields(MsRead.java:84) at org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:97) ... 11 more Here is a class that has WritableComparator.compare. It has only 2 strings max length 20 characters for each. public class MsRead implements WritableComparable MsRead { private static final Log LOG = LogFactory.getLog(speeditup.CalculateMinEvalue.class); private String query_id; private String record; public String getRecord() { return record; } public void setRecord(String record) { this.record = record; } public String getQuery_id() { return query_id; } public void setQuery_id(String queryId) { query_id = queryId; } public MsRead() { ; } public MsRead(String a, String r) { setQuery_id(a); setRecord(r); } @Override public void readFields(DataInput in) throws IOException { LOG.debug(**myreadFields +); LOG.warn(**myreadFields +); LOG.info(**myreadFields + ); query_id = in.readUTF(); record = in.readUTF(); } @Override public void write(DataOutput out) throws IOException { out.writeUTF(query_id); out.writeUTF(record); } public static class FirstComparator extends WritableComparator { private static final Text.Comparator TEXT_COMPARATOR = new Text.Comparator(); public FirstComparator() { super(MsRead.class); } @Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { try { int firstL1 = WritableUtils.decodeVIntSize(b1[s1]) + readVInt(b1, s1); int firstL2 = WritableUtils.decodeVIntSize(b2[s2]) + readVInt(b2, s2); return TEXT_COMPARATOR.compare(b1, s1, firstL1, b2, s2, firstL2); } catch (IOException e) { throw new IllegalArgumentException(e); } } @Override public int compare(WritableComparable a, WritableComparable b) { if (a instanceof MsRead b instanceof MsRead) { //System.err.println(COMPARE + ((MsRead)a).getType() + \t + ((MsRead)b).getType() + \t //+ (((MsRead) a).toString().compareTo(((MsRead) b).toString(; return (((MsRead) a).toString().compareTo(((MsRead) b).toString())); } return super.compare(a, b); } } @Override public int compareTo(MsRead o) { return this.toString().compareTo(o.toString()); } @Override public boolean equals(Object right) { if (right instanceof MsRead ) { return (query_id.equals(((MsRead)right).query_id)); } else return false; } @Override public int hashCode() { return query_id.hashCode() ; } @Override public String toString() { return query_id; } public String toOutputString() { return record; } }