Author: cutting
Date: Thu Jan 18 15:03:42 2007
New Revision: 497624
URL: http://svn.apache.org/viewvc?view=rev&rev=497624
Log:
HADOOP-735. Switch generated record code to use BytesWritable to represent
fields of type 'buffer'. Contributed by Milind.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryInputArchive.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryOutputArchive.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvInputArchive.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvOutputArchive.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/InputArchive.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/OutputArchive.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Utils.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlInputArchive.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlOutputArchive.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBuffer.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JCompType.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JRecord.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JType.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/ant/RccTask.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/FromCpp.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/TestRecordIO.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/TestWritable.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/ToCpp.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=497624&r1=497623&r2=497624
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Thu Jan 18 15:03:42 2007
@@ -44,6 +44,9 @@
13. HADOOP-902. Fix a NullPointerException in HDFS client when
closing output streams. (Raghu Angadi via cutting)
+14. HADOOP-735. Switch generated record code to use BytesWritable to
+ represent fields of type 'buffer'. (Milind Bhandarkar via cutting)
+
Release 0.10.1 - 2007-01-10
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryInputArchive.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryInputArchive.java?view=diff&rev=497624&r1=497623&r2=497624
==============================================================================
---
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryInputArchive.java
(original)
+++
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryInputArchive.java
Thu Jan 18 15:03:42 2007
@@ -20,9 +20,9 @@
import java.io.DataInput;
import java.io.IOException;
-import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.InputStream;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableUtils;
@@ -87,13 +87,11 @@
return text;
}
- public ByteArrayOutputStream readBuffer(String tag) throws IOException {
- int len = readInt(tag);
- ByteArrayOutputStream buf = new ByteArrayOutputStream(len);
- byte[] arr = new byte[len];
- in.readFully(arr);
- buf.write(arr, 0, len);
- return buf;
+ public BytesWritable readBuffer(String tag) throws IOException {
+ int len = WritableUtils.readVInt(in);
+ byte[] barr = new byte[len];
+ in.readFully(barr);
+ return new BytesWritable(barr);
}
public void readRecord(Record r, String tag) throws IOException {
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryOutputArchive.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryOutputArchive.java?view=diff&rev=497624&r1=497623&r2=497624
==============================================================================
---
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryOutputArchive.java
(original)
+++
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryOutputArchive.java
Thu Jan 18 15:03:42 2007
@@ -19,12 +19,12 @@
package org.apache.hadoop.record;
import java.io.IOException;
-import java.io.ByteArrayOutputStream;
import java.util.TreeMap;
import java.util.ArrayList;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.OutputStream;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableUtils;
@@ -74,11 +74,12 @@
s.write(out);
}
- public void writeBuffer(ByteArrayOutputStream buf, String tag)
+ public void writeBuffer(BytesWritable buf, String tag)
throws IOException {
- byte[] barr = buf.toByteArray();
- writeInt(barr.length, tag);
- out.write(barr);
+ byte[] barr = buf.get();
+ int len = buf.getSize();
+ WritableUtils.writeVInt(out, len);
+ out.write(barr, 0, len);
}
public void writeRecord(Record r, String tag) throws IOException {
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvInputArchive.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvInputArchive.java?view=diff&rev=497624&r1=497623&r2=497624
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvInputArchive.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvInputArchive.java
Thu Jan 18 15:03:42 2007
@@ -21,9 +21,9 @@
import java.io.InputStreamReader;
import java.io.InputStream;
import java.io.IOException;
-import java.io.ByteArrayOutputStream;
import java.io.PushbackReader;
import java.io.UnsupportedEncodingException;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
/**
@@ -127,7 +127,7 @@
}
- public ByteArrayOutputStream readBuffer(String tag) throws IOException {
+ public BytesWritable readBuffer(String tag) throws IOException {
String sval = readField(tag);
return Utils.fromCSVBuffer(sval);
}
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvOutputArchive.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvOutputArchive.java?view=diff&rev=497624&r1=497623&r2=497624
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvOutputArchive.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvOutputArchive.java
Thu Jan 18 15:03:42 2007
@@ -19,12 +19,12 @@
package org.apache.hadoop.record;
import java.io.IOException;
-import java.io.ByteArrayOutputStream;
import java.util.TreeMap;
import java.util.ArrayList;
import java.io.PrintStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
/**
@@ -97,7 +97,7 @@
throwExceptionOnError(tag);
}
- public void writeBuffer(ByteArrayOutputStream buf, String tag)
+ public void writeBuffer(BytesWritable buf, String tag)
throws IOException {
printCommaUnlessFirst();
stream.print(Utils.toCSVBuffer(buf));
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/InputArchive.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/InputArchive.java?view=diff&rev=497624&r1=497623&r2=497624
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/InputArchive.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/InputArchive.java Thu
Jan 18 15:03:42 2007
@@ -19,7 +19,7 @@
package org.apache.hadoop.record;
import java.io.IOException;
-import java.io.ByteArrayOutputStream;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
/**
@@ -35,7 +35,7 @@
public float readFloat(String tag) throws IOException;
public double readDouble(String tag) throws IOException;
public Text readString(String tag) throws IOException;
- public ByteArrayOutputStream readBuffer(String tag) throws IOException;
+ public BytesWritable readBuffer(String tag) throws IOException;
public void readRecord(Record r, String tag) throws IOException;
public void startRecord(String tag) throws IOException;
public void endRecord(String tag) throws IOException;
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/OutputArchive.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/OutputArchive.java?view=diff&rev=497624&r1=497623&r2=497624
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/OutputArchive.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/OutputArchive.java
Thu Jan 18 15:03:42 2007
@@ -19,9 +19,9 @@
package org.apache.hadoop.record;
import java.io.IOException;
-import java.io.ByteArrayOutputStream;
import java.util.TreeMap;
import java.util.ArrayList;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
/**
@@ -37,7 +37,7 @@
public void writeFloat(float f, String tag) throws IOException;
public void writeDouble(double d, String tag) throws IOException;
public void writeString(Text s, String tag) throws IOException;
- public void writeBuffer(ByteArrayOutputStream buf, String tag)
+ public void writeBuffer(BytesWritable buf, String tag)
throws IOException;
public void writeRecord(Record r, String tag) throws IOException;
public void startRecord(Record r, String tag) throws IOException;
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Utils.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Utils.java?view=diff&rev=497624&r1=497623&r2=497624
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Utils.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Utils.java Thu Jan 18
15:03:42 2007
@@ -19,11 +19,8 @@
package org.apache.hadoop.record;
import java.io.ByteArrayOutputStream;
-import java.io.DataInput;
-import java.io.DataOutput;
import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.nio.charset.CharacterCodingException;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
/**
@@ -36,32 +33,6 @@
private Utils() {
}
- /**
- * equals function that actually compares two buffers.
- *
- * @param one First buffer
- * @param two Second buffer
- * @return true if one and two contain exactly the same content, else
false.
- */
- public static boolean bufEquals(ByteArrayOutputStream one,
- ByteArrayOutputStream two) {
- if (one == two) {
- return true;
- }
- byte[] onearray = one.toByteArray();
- byte[] twoarray = two.toByteArray();
- boolean ret = (onearray.length == twoarray.length);
- if (!ret) {
- return ret;
- }
- for (int idx = 0; idx < onearray.length; idx++) {
- if (onearray[idx] != twoarray[idx]) {
- return false;
- }
- }
- return true;
- }
-
public static final char[] hexchars = { '0', '1', '2', '3', '4', '5',
'6', '7', '8', '9', 'A', 'B',
'C', 'D', 'E', 'F' };
@@ -200,10 +171,11 @@
* @param s
* @return
*/
- static String toXMLBuffer(ByteArrayOutputStream s) {
- byte[] barr = s.toByteArray();
- StringBuffer sb = new StringBuffer(2*barr.length);
- for (int idx = 0; idx < barr.length; idx++) {
+ static String toXMLBuffer(BytesWritable s) {
+ byte[] barr = s.get();
+ int bsize = s.getSize();
+ StringBuffer sb = new StringBuffer(2*bsize);
+ for (int idx = 0; idx < bsize; idx++) {
sb.append(Integer.toHexString((int)barr[idx]));
}
return sb.toString();
@@ -215,10 +187,9 @@
* @throws java.io.IOException
* @return
*/
- static ByteArrayOutputStream fromXMLBuffer(String s)
+ static BytesWritable fromXMLBuffer(String s)
throws IOException {
- ByteArrayOutputStream stream = new ByteArrayOutputStream();
- if (s.length() == 0) { return stream; }
+ if (s.length() == 0) { return new BytesWritable(); }
int blen = s.length()/2;
byte[] barr = new byte[blen];
for (int idx = 0; idx < blen; idx++) {
@@ -226,8 +197,7 @@
char c2 = s.charAt(2*idx+1);
barr[idx] = Byte.parseByte(""+c1+c2, 16);
}
- stream.write(barr);
- return stream;
+ return new BytesWritable(barr);
}
/**
@@ -235,11 +205,12 @@
* @param buf
* @return
*/
- static String toCSVBuffer(ByteArrayOutputStream buf) {
- byte[] barr = buf.toByteArray();
- StringBuffer sb = new StringBuffer(barr.length+1);
+ static String toCSVBuffer(BytesWritable buf) {
+ byte[] barr = buf.get();
+ int bsize = buf.getSize();
+ StringBuffer sb = new StringBuffer(bsize+1);
sb.append('#');
- for(int idx = 0; idx < barr.length; idx++) {
+ for(int idx = 0; idx < bsize; idx++) {
sb.append(Integer.toHexString((int)barr[idx]));
}
return sb.toString();
@@ -247,18 +218,17 @@
/**
* Converts a CSV-serialized representation of buffer to a new
- * ByteArrayOutputStream.
+ * BytesWritable.
* @param s CSV-serialized representation of buffer
* @throws java.io.IOException
- * @return Deserialized ByteArrayOutputStream
+ * @return Deserialized BytesWritable
*/
- static ByteArrayOutputStream fromCSVBuffer(String s)
+ static BytesWritable fromCSVBuffer(String s)
throws IOException {
if (s.charAt(0) != '#') {
throw new IOException("Error deserializing buffer.");
}
- ByteArrayOutputStream stream = new ByteArrayOutputStream();
- if (s.length() == 1) { return stream; }
+ if (s.length() == 1) { return new BytesWritable(); }
int blen = (s.length()-1)/2;
byte[] barr = new byte[blen];
for (int idx = 0; idx < blen; idx++) {
@@ -266,7 +236,6 @@
char c2 = s.charAt(2*idx+2);
barr[idx] = Byte.parseByte(""+c1+c2, 16);
}
- stream.write(barr);
- return stream;
+ return new BytesWritable(barr);
}
}
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlInputArchive.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlInputArchive.java?view=diff&rev=497624&r1=497623&r2=497624
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlInputArchive.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlInputArchive.java
Thu Jan 18 15:03:42 2007
@@ -20,7 +20,6 @@
import java.io.InputStream;
import java.io.IOException;
-import java.io.ByteArrayOutputStream;
import java.util.ArrayList;
import org.xml.sax.*;
@@ -28,6 +27,7 @@
import javax.xml.parsers.SAXParserFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
/**
*
@@ -207,7 +207,7 @@
return Utils.fromXMLString(v.getValue());
}
- public ByteArrayOutputStream readBuffer(String tag) throws IOException {
+ public BytesWritable readBuffer(String tag) throws IOException {
Value v = next();
if (!"string".equals(v.getType())) {
throw new IOException("Error deserializing "+tag+".");
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlOutputArchive.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlOutputArchive.java?view=diff&rev=497624&r1=497623&r2=497624
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlOutputArchive.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlOutputArchive.java
Thu Jan 18 15:03:42 2007
@@ -19,12 +19,12 @@
package org.apache.hadoop.record;
import java.io.IOException;
-import java.io.ByteArrayOutputStream;
import java.util.TreeMap;
import java.util.ArrayList;
import java.io.PrintStream;
import java.io.OutputStream;
import java.util.Stack;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
/**
@@ -199,7 +199,7 @@
printEndEnvelope(tag);
}
- public void writeBuffer(ByteArrayOutputStream buf, String tag)
+ public void writeBuffer(BytesWritable buf, String tag)
throws IOException {
printBeginEnvelope(tag);
stream.print("<string>");
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBuffer.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBuffer.java?view=diff&rev=497624&r1=497623&r2=497624
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBuffer.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBuffer.java
Thu Jan 18 15:03:42 2007
@@ -26,7 +26,7 @@
/** Creates a new instance of JBuffer */
public JBuffer() {
- super(" ::std::string", "java.io.ByteArrayOutputStream", "Buffer",
"java.io.ByteArrayOutputStream");
+ super(" ::std::string", "BytesWritable", "Buffer", "BytesWritable");
}
public String genCppGetSet(String fname, int fIdx) {
@@ -34,7 +34,7 @@
cgetFunc += " return m"+fname+";\n";
cgetFunc += " }\n";
String getFunc = " virtual "+getCppType()+"& get"+fname+"() {\n";
- getFunc += " bs_.set("+fIdx+");return m"+fname+";\n";
+ getFunc += " return m"+fname+";\n";
getFunc += " }\n";
return cgetFunc + getFunc;
}
@@ -46,7 +46,7 @@
public String genJavaReadWrapper(String fname, String tag, boolean decl) {
String ret = "";
if (decl) {
- ret = " java.io.ByteArrayOutputStream "+fname+";\n";
+ ret = " BytesWritable "+fname+";\n";
}
return ret + " "+fname+"=a_.readBuffer(\""+tag+"\");\n";
}
@@ -58,9 +58,10 @@
public String genJavaCompareTo(String fname, String other) {
StringBuffer sb = new StringBuffer();
sb.append(" {\n");
- sb.append(" byte[] my = "+fname+".toByteArray();\n");
- sb.append(" byte[] ur = "+other+".toByteArray();\n");
- sb.append(" ret =
WritableComparator.compareBytes(my,0,my.length,ur,0,ur.length);\n");
+ sb.append(" byte[] my = "+fname+".get();\n");
+ sb.append(" byte[] ur = "+other+".get();\n");
+ sb.append(" ret = WritableComparator.compareBytes(my,0,"+
+ fname+".getSize(),ur,0,"+other+".getSize());\n");
sb.append(" }\n");
return sb.toString();
}
@@ -70,11 +71,11 @@
}
public String genJavaEquals(String fname, String peer) {
- return " ret =
org.apache.hadoop.record.Utils.bufEquals("+fname+","+peer+");\n";
+ return " ret = "+fname+".equals("+peer+");\n";
}
public String genJavaHashCode(String fname) {
- return " ret = "+fname+".toString().hashCode();\n";
+ return " ret = "+fname+".hashCode();\n";
}
public String genJavaSlurpBytes(String b, String s, String l) {
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JCompType.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JCompType.java?view=diff&rev=497624&r1=497623&r2=497624
==============================================================================
---
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JCompType.java
(original)
+++
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JCompType.java
Thu Jan 18 15:03:42 2007
@@ -36,7 +36,7 @@
cgetFunc += " return m"+fname+";\n";
cgetFunc += " }\n";
String getFunc = " virtual "+getCppType()+"& get"+fname+"() {\n";
- getFunc += " bs_.set("+fIdx+");return m"+fname+";\n";
+ getFunc += " return m"+fname+";\n";
getFunc += " }\n";
return cgetFunc + getFunc;
}
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JRecord.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JRecord.java?view=diff&rev=497624&r1=497623&r2=497624
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JRecord.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JRecord.java
Thu Jan 18 15:03:42 2007
@@ -139,13 +139,11 @@
JField jf = (JField) i.next();
hh.write(jf.genCppDecl());
}
- hh.write(" mutable std::bitset<"+mFields.size()+"> bs_;\n");
hh.write("public:\n");
hh.write(" virtual void serialize(::hadoop::OArchive& a_, const char*
tag) const;\n");
hh.write(" virtual void deserialize(::hadoop::IArchive& a_, const
char* tag);\n");
hh.write(" virtual const ::std::string& type() const;\n");
hh.write(" virtual const ::std::string& signature() const;\n");
- hh.write(" virtual bool validate() const;\n");
hh.write(" virtual bool operator<(const "+getName()+"& peer_)
const;\n");
hh.write(" virtual bool operator==(const "+getName()+"& peer_)
const;\n");
hh.write(" virtual ~"+getName()+"() {};\n");
@@ -159,7 +157,6 @@
hh.write("} // end namespace "+ns[i]+"\n");
}
cc.write("void "+getCppFQName()+"::serialize(::hadoop::OArchive& a_,
const char* tag) const {\n");
- cc.write(" if (!validate()) throw new ::hadoop::IOException(\"All
fields not set.\");\n");
cc.write(" a_.startRecord(*this,tag);\n");
fIdx = 0;
for (Iterator i = mFields.iterator(); i.hasNext(); fIdx++) {
@@ -170,7 +167,6 @@
} else {
cc.write(" a_.serialize("+name+",\""+jf.getTag()+"\");\n");
}
- cc.write(" bs_.reset("+fIdx+");\n");
}
cc.write(" a_.endRecord(*this,tag);\n");
cc.write(" return;\n");
@@ -187,23 +183,11 @@
} else {
cc.write(" a_.deserialize("+name+",\""+jf.getTag()+"\");\n");
}
- cc.write(" bs_.set("+fIdx+");\n");
}
cc.write(" a_.endRecord(*this,tag);\n");
cc.write(" return;\n");
cc.write("}\n");
- cc.write("bool "+getCppFQName()+"::validate() const {\n");
- cc.write(" if (bs_.size() != bs_.count()) return false;\n");
- for (Iterator i = mFields.iterator(); i.hasNext(); fIdx++) {
- JField jf = (JField) i.next();
- JType type = jf.getType();
- if (type instanceof JRecord) {
- cc.write(" if (!"+jf.getName()+".validate()) return
false;\n");
- }
- }
- cc.write(" return true;\n");
- cc.write("}\n");
cc.write("bool "+getCppFQName()+"::operator< (const
"+getCppFQName()+"& peer_) const {\n");
cc.write(" return (1\n");
@@ -261,6 +245,7 @@
jj.write("import org.apache.hadoop.io.WritableComparator;\n");
jj.write("import org.apache.hadoop.io.WritableComparable;\n");
jj.write("import org.apache.hadoop.io.WritableUtils;\n");
+ jj.write("import org.apache.hadoop.io.BytesWritable;\n");
jj.write("import org.apache.hadoop.io.Text;\n\n");
jj.write("public class "+getName()+" implements
org.apache.hadoop.record.Record, WritableComparable {\n");
jj.write(" private static final Log LOG= LogFactory.getLog(\""+
@@ -269,23 +254,17 @@
JField jf = (JField) i.next();
jj.write(jf.genJavaDecl());
}
- jj.write(" private java.util.BitSet bs_;\n");
- jj.write(" public "+getName()+"() {\n");
- jj.write(" bs_ = new java.util.BitSet("+(mFields.size()+1)+");\n");
- jj.write(" bs_.set("+mFields.size()+");\n");
- jj.write(" }\n");
+ jj.write(" public "+getName()+"() { }\n");
+
jj.write(" public "+getName()+"(\n");
int fIdx = 0;
- int fLen = mFields.size();
for (Iterator i = mFields.iterator(); i.hasNext(); fIdx++) {
JField jf = (JField) i.next();
jj.write(jf.genJavaConstructorParam(fIdx));
- jj.write((fLen-1 == fIdx)?"":",\n");
+ jj.write((!i.hasNext())?"":",\n");
}
jj.write(") {\n");
- jj.write(" bs_ = new java.util.BitSet("+(mFields.size()+1)+");\n");
- jj.write(" bs_.set("+mFields.size()+");\n");
fIdx = 0;
for (Iterator i = mFields.iterator(); i.hasNext(); fIdx++) {
JField jf = (JField) i.next();
@@ -298,24 +277,19 @@
jj.write(jf.genJavaGetSet(fIdx));
}
jj.write(" public void
serialize(org.apache.hadoop.record.OutputArchive a_, String tag) throws
java.io.IOException {\n");
- jj.write(" if (!validate()) throw new java.io.IOException(\"All
fields not set:\");\n");
jj.write(" a_.startRecord(this,tag);\n");
- fIdx = 0;
- for (Iterator i = mFields.iterator(); i.hasNext(); fIdx++) {
+ for (Iterator i = mFields.iterator(); i.hasNext();) {
JField jf = (JField) i.next();
jj.write(jf.genJavaWriteMethodName());
- jj.write(" bs_.clear("+fIdx+");\n");
}
jj.write(" a_.endRecord(this,tag);\n");
jj.write(" }\n");
jj.write(" public void
deserialize(org.apache.hadoop.record.InputArchive a_, String tag) throws
java.io.IOException {\n");
jj.write(" a_.startRecord(tag);\n");
- fIdx = 0;
- for (Iterator i = mFields.iterator(); i.hasNext(); fIdx++) {
+ for (Iterator i = mFields.iterator(); i.hasNext();) {
JField jf = (JField) i.next();
jj.write(jf.genJavaReadMethodName());
- jj.write(" bs_.set("+fIdx+");\n");
}
jj.write(" a_.endRecord(tag);\n");
jj.write("}\n");
@@ -335,9 +309,8 @@
jj.write(" a_.endRecord(this,\"\");\n");
jj.write(" return new String(s.toByteArray(), \"UTF-8\");\n");
jj.write(" } catch (Throwable ex) {\n");
- jj.write(" ex.printStackTrace();\n");
+ jj.write(" throw new RuntimeException(ex);\n");
jj.write(" }\n");
- jj.write(" return \"ERROR\";\n");
jj.write(" }\n");
jj.write(" public void write(java.io.DataOutput out) throws
java.io.IOException {\n");
@@ -349,18 +322,6 @@
jj.write(" org.apache.hadoop.record.BinaryInputArchive archive =
new org.apache.hadoop.record.BinaryInputArchive(in);\n");
jj.write(" deserialize(archive, \"\");\n");
jj.write(" }\n");
-
- jj.write(" public boolean validate() {\n");
- jj.write(" if (bs_.cardinality() != bs_.length()) return false;\n");
- for (Iterator i = mFields.iterator(); i.hasNext(); fIdx++) {
- JField jf = (JField) i.next();
- JType type = jf.getType();
- if (type instanceof JRecord) {
- jj.write(" if (!"+jf.getName()+".validate()) return
false;\n");
- }
- }
- jj.write(" return true;\n");
- jj.write("}\n");
jj.write(" public int compareTo (Object peer_) throws
ClassCastException {\n");
jj.write(" if (!(peer_ instanceof "+getName()+")) {\n");
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JType.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JType.java?view=diff&rev=497624&r1=497623&r2=497624
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JType.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JType.java
Thu Jan 18 15:03:42 2007
@@ -61,7 +61,7 @@
getFunc += " return m"+fname+";\n";
getFunc += " }\n";
String setFunc = " virtual void set"+fname+"("+mCppName+" m_) {\n";
- setFunc += " m"+fname+"=m_; bs_.set("+fIdx+");\n";
+ setFunc += " m"+fname+"=m_;\n";
setFunc += " }\n";
return getFunc+setFunc;
}
@@ -71,7 +71,7 @@
getFunc += " return m"+fname+";\n";
getFunc += " }\n";
String setFunc = " public void set"+fname+"("+mJavaName+" m_) {\n";
- setFunc += " m"+fname+"=m_; bs_.set("+fIdx+");\n";
+ setFunc += " m"+fname+"=m_;\n";
setFunc += " }\n";
return getFunc+setFunc;
}
@@ -143,6 +143,6 @@
}
String genJavaConstructorSet(String fname, int fIdx) {
- return " m"+fname+"=m"+fIdx+"; bs_.set("+fIdx+");\n";
+ return " m"+fname+"=m"+fIdx+";\n";
}
}
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/ant/RccTask.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/ant/RccTask.java?view=diff&rev=497624&r1=497623&r2=497624
==============================================================================
---
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/ant/RccTask.java
(original)
+++
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/ant/RccTask.java
Thu Jan 18 15:03:42 2007
@@ -52,7 +52,7 @@
private String language = "java";
private File src;
private File dest = new File(".");
- private final Vector<FileSet> filesets = new Vector();
+ private final Vector<FileSet> filesets = new Vector<FileSet>();
private boolean failOnError = true;
/** Creates a new instance of RccTask */
Modified:
lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/FromCpp.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/FromCpp.java?view=diff&rev=497624&r1=497623&r2=497624
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/FromCpp.java
(original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/FromCpp.java Thu
Jan 18 15:03:42 2007
@@ -26,6 +26,7 @@
import java.util.ArrayList;
import java.util.TreeMap;
import junit.framework.*;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
/**
@@ -56,7 +57,7 @@
r1.setIntVal(4567);
r1.setLongVal(0x5a5a5a5a5a5aL);
r1.setStringVal(new Text("random text"));
- r1.setBufferVal(new ByteArrayOutputStream(20));
+ r1.setBufferVal(new BytesWritable());
r1.setVectorVal(new ArrayList());
r1.setMapVal(new TreeMap());
FileInputStream istream = new FileInputStream(tmpfile);
@@ -82,7 +83,7 @@
r1.setIntVal(4567);
r1.setLongVal(0x5a5a5a5a5a5aL);
r1.setStringVal(new Text("random text"));
- r1.setBufferVal(new ByteArrayOutputStream(20));
+ r1.setBufferVal(new BytesWritable());
r1.setVectorVal(new ArrayList());
r1.setMapVal(new TreeMap());
FileInputStream istream = new FileInputStream(tmpfile);
@@ -108,7 +109,7 @@
r1.setIntVal(4567);
r1.setLongVal(0x5a5a5a5a5a5aL);
r1.setStringVal(new Text("random text"));
- r1.setBufferVal(new ByteArrayOutputStream(20));
+ r1.setBufferVal(new BytesWritable());
r1.setVectorVal(new ArrayList());
r1.setMapVal(new TreeMap());
FileInputStream istream = new FileInputStream(tmpfile);
Modified:
lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/TestRecordIO.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/TestRecordIO.java?view=diff&rev=497624&r1=497623&r2=497624
==============================================================================
---
lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/TestRecordIO.java
(original)
+++
lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/TestRecordIO.java
Thu Jan 18 15:03:42 2007
@@ -22,12 +22,12 @@
import junit.framework.*;
import org.apache.hadoop.record.RecordWriter;
import org.apache.hadoop.record.RecordReader;
-import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.util.ArrayList;
import java.util.TreeMap;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
/**
@@ -60,7 +60,7 @@
r1.setIntVal(4567);
r1.setLongVal(0x5a5a5a5a5a5aL);
r1.setStringVal(new Text("random text"));
- r1.setBufferVal(new ByteArrayOutputStream(20));
+ r1.setBufferVal(new BytesWritable());
r1.setVectorVal(new ArrayList());
r1.setMapVal(new TreeMap());
RecRecord0 r0 = new RecRecord0();
@@ -94,7 +94,7 @@
r1.setIntVal(4567);
r1.setLongVal(0x5a5a5a5a5a5aL);
r1.setStringVal(new Text("random text"));
- r1.setBufferVal(new ByteArrayOutputStream(20));
+ r1.setBufferVal(new BytesWritable());
r1.setVectorVal(new ArrayList());
r1.setMapVal(new TreeMap());
RecRecord0 r0 = new RecRecord0();
@@ -109,11 +109,36 @@
istream.close();
tmpfile.delete();
assertTrue("Serialized and deserialized records do not match.",
r1.equals(r2));
+
} catch (IOException ex) {
ex.printStackTrace();
- }
+ }
}
+ public void testToString() {
+ try {
+ RecRecord1 r1 = new RecRecord1();
+ r1.setBoolVal(true);
+ r1.setByteVal((byte)0x66);
+ r1.setFloatVal(3.145F);
+ r1.setDoubleVal(1.5234);
+ r1.setIntVal(4567);
+ r1.setLongVal(0x5a5a5a5a5a5aL);
+ r1.setStringVal(new Text("random text"));
+ r1.setBufferVal(new BytesWritable());
+ r1.setVectorVal(new ArrayList());
+ r1.setMapVal(new TreeMap());
+ RecRecord0 r0 = new RecRecord0();
+ r0.setStringVal(new Text("other random text"));
+ r1.setRecordVal(r0);
+ System.err.println("Illustrating toString bug"+r1.toString());
+ System.err.println("Illustrating toString bug"+r1.toString());
+ } catch (Throwable ex) {
+ assertTrue("Record.toString cannot be invoked twice in
succession."+
+ "This bug has been fixed in the latest version.", false);
+ }
+ }
+
public void testXml() {
File tmpfile;
try {
@@ -128,7 +153,7 @@
r1.setIntVal(4567);
r1.setLongVal(0x5a5a5a5a5a5aL);
r1.setStringVal(new Text("ran\002dom < %text<&more"));
- r1.setBufferVal(new ByteArrayOutputStream(20));
+ r1.setBufferVal(new BytesWritable());
r1.setVectorVal(new ArrayList());
r1.setMapVal(new TreeMap());
RecRecord0 r0 = new RecRecord0();
Modified:
lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/TestWritable.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/TestWritable.java?view=diff&rev=497624&r1=497623&r2=497624
==============================================================================
---
lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/TestWritable.java
(original)
+++
lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/TestWritable.java
Thu Jan 18 15:03:42 2007
@@ -63,8 +63,6 @@
for (int length = 0; length < MAX_LENGTH;
length+= random.nextInt(MAX_LENGTH/10)+1) {
- //LOG.info("creating; entries = " + length);
-
// create a file with length entries
SequenceFile.Writer writer =
new SequenceFile.Writer(fs, conf, file,
@@ -76,9 +74,7 @@
byte[] data = new byte[random.nextInt(10)];
random.nextBytes(data);
RecBuffer value = new RecBuffer();
- ByteArrayOutputStream strm = new ByteArrayOutputStream(data.length);
- strm.write(data);
- value.setData(strm);
+ value.setData(new BytesWritable(data));
writer.append(key, value);
}
} finally {
@@ -92,9 +88,7 @@
for (int i = 0; i < 3; i++) {
int numSplits =
random.nextInt(MAX_LENGTH/(SequenceFile.SYNC_INTERVAL/20))+1;
- //LOG.info("splitting: requesting = " + numSplits);
InputSplit[] splits = format.getSplits(job, numSplits);
- //LOG.info("splitting: got = " + splits.length);
// check each split
BitSet bits = new BitSet(length);
@@ -104,15 +98,10 @@
try {
int count = 0;
while (reader.next(key, value)) {
- // if (bits.get(key.get())) {
- // LOG.info("splits["+j+"]="+splits[j]+" : " + key.get());
- // LOG.info("@"+reader.getPos());
- // }
assertFalse("Key in multiple partitions.",
bits.get(key.getData()));
bits.set(key.getData());
count++;
}
- //LOG.info("splits["+j+"]="+splits[j]+" count=" + count);
} finally {
reader.close();
}
Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/ToCpp.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/ToCpp.java?view=diff&rev=497624&r1=497623&r2=497624
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/ToCpp.java
(original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/ToCpp.java Thu
Jan 18 15:03:42 2007
@@ -26,6 +26,7 @@
import java.io.FileOutputStream;
import java.util.ArrayList;
import java.util.TreeMap;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
/**
@@ -58,7 +59,7 @@
r1.setIntVal(4567);
r1.setLongVal(0x5a5a5a5a5a5aL);
r1.setStringVal(new Text("random text"));
- r1.setBufferVal(new ByteArrayOutputStream(20));
+ r1.setBufferVal(new BytesWritable());
r1.setVectorVal(new ArrayList());
r1.setMapVal(new TreeMap());
out.write(r1);
@@ -82,7 +83,7 @@
r1.setIntVal(4567);
r1.setLongVal(0x5a5a5a5a5a5aL);
r1.setStringVal(new Text("random text"));
- r1.setBufferVal(new ByteArrayOutputStream(20));
+ r1.setBufferVal(new BytesWritable());
r1.setVectorVal(new ArrayList());
r1.setMapVal(new TreeMap());
out.write(r1);
@@ -106,7 +107,7 @@
r1.setIntVal(4567);
r1.setLongVal(0x5a5a5a5a5a5aL);
r1.setStringVal(new Text("random text"));
- r1.setBufferVal(new ByteArrayOutputStream(20));
+ r1.setBufferVal(new BytesWritable());
r1.setVectorVal(new ArrayList());
r1.setMapVal(new TreeMap());
out.write(r1);
