Updated Branches: refs/heads/0.2-dev-mr-formats b0893e13f -> 996ed6a65
BLUR-ID:18 New Writables Signed-off-by: Aaron McCurry <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/996ed6a6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/996ed6a6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/996ed6a6 Branch: refs/heads/0.2-dev-mr-formats Commit: 996ed6a65120ff9252c00db30d2ec96413eba0cd Parents: 95bd2a0 Author: Gagan <[email protected]> Authored: Mon Nov 5 18:57:43 2012 +0530 Committer: Aaron McCurry <[email protected]> Committed: Thu Nov 8 22:20:40 2012 -0500 ---------------------------------------------------------------------- .../java/org/apache/blur/mr/BlurDocLocation.java | 85 ++++++++++ .../main/java/org/apache/blur/mr/BlurDocument.java | 90 ++++++++++ .../main/java/org/apache/blur/mr/BlurField.java | 127 +++++++++++++++ .../org/apache/blur/mr/BlurDocLocationTest.java | 43 +++++ .../java/org/apache/blur/mr/BlurDocumentTest.java | 55 +++++++ .../java/org/apache/blur/mr/BlurFieldTest.java | 96 +++++++++++ 6 files changed, 496 insertions(+), 0 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/996ed6a6/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurDocLocation.java ---------------------------------------------------------------------- diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurDocLocation.java b/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurDocLocation.java new file mode 100644 index 0000000..59b7d94 --- /dev/null +++ b/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurDocLocation.java @@ -0,0 +1,85 @@ +package org.apache.blur.mr; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.blur.mapreduce.IOUtil; +import org.apache.blur.thrift.generated.DocLocation; +import org.apache.hadoop.io.Writable; + +public class BlurDocLocation implements Writable{ + private int docId; + private int shardIndexId; + + public BlurDocLocation(){ + + } + + public BlurDocLocation(DocLocation docLocation){ + docId = docLocation.getDoc(); + shardIndexId = docLocation.getShardIndex(); + } + + + @Override + public void readFields(DataInput in) throws IOException { + docId = IOUtil.readVInt(in); + shardIndexId = IOUtil.readVInt(in); + } + + @Override + public void write(DataOutput out) throws IOException { + IOUtil.writeVInt(out, docId); + IOUtil.writeVInt(out, shardIndexId); + } + + public int getDocId() { + return docId; + } + + public void setDocId(int docId) { + this.docId = docId; + } + + public int getShardIndexId() { + return shardIndexId; + } + + public void setShardIndexId(int shardIndexId) { + this.shardIndexId = shardIndexId; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + docId; + result = prime * result + shardIndexId; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + BlurDocLocation other = (BlurDocLocation) obj; + if (docId != other.docId) + return false; + if (shardIndexId != other.shardIndexId) + return false; + return true; + } + + @Override + public String toString() { + return "BlurDocLocation [docId=" + docId + ", shardIndexId=" + shardIndexId + + "]"; + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/996ed6a6/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurDocument.java ---------------------------------------------------------------------- diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurDocument.java b/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurDocument.java new file mode 100644 index 0000000..31cc547 --- /dev/null +++ b/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurDocument.java @@ -0,0 +1,90 @@ +package org.apache.blur.mr; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.blur.mapreduce.IOUtil; +import org.apache.blur.thrift.generated.Document; +import org.apache.blur.thrift.generated.Field; +import org.apache.hadoop.io.Writable; + +public class BlurDocument implements Writable { + private List<BlurField> blurFields; + + public BlurDocument() { + blurFields = new ArrayList<BlurField>(); + } + + public BlurDocument(Document document) { + blurFields = new ArrayList<BlurField>(); + loadDocumentToBlurDocument(document); + } + + private void loadDocumentToBlurDocument(Document document) { + for (Field field : document.getFields()) { + blurFields.add(new BlurField(field)); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + int size = IOUtil.readVInt(in); + for (int i = 0; i < size; i++) { + BlurField blurField = new BlurField(); + blurField.readFields(in); + blurFields.add(blurField); + } + } + + @Override + public void write(DataOutput out) throws IOException { + IOUtil.writeVInt(out, blurFields.size()); + for (BlurField blurField : blurFields) { + blurField.write(out); + } + } + + public List<BlurField> getBlurFields() { + return blurFields; + } + + public void setBlurFields(List<BlurField> blurFields) { + this.blurFields = blurFields; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((blurFields == null) ? 0 : blurFields.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + BlurDocument other = (BlurDocument) obj; + if (blurFields == null) { + if (other.blurFields != null) + return false; + } else if (!blurFields.equals(other.blurFields)) + return false; + return true; + } + + @Override + public String toString() { + return "BlurDocument [blurFields=" + blurFields.toString() + "]"; + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/996ed6a6/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurField.java ---------------------------------------------------------------------- diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurField.java b/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurField.java new file mode 100644 index 0000000..30a543d --- /dev/null +++ b/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurField.java @@ -0,0 +1,127 @@ +package org.apache.blur.mr; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Arrays; + +import org.apache.blur.mapreduce.IOUtil; +import org.apache.blur.thrift.generated.Field; +import org.apache.blur.thrift.generated.TYPE; +import org.apache.hadoop.io.Writable; + +public class BlurField implements Writable{ + private String fieldName; + private TYPE fieldType; + private double fieldBoost = 1.0; + private byte[] fieldValue; + + public BlurField(){ + + } + + public BlurField(Field field){ + fieldName = field.getName(); + fieldType = field.getType(); + fieldBoost = field.getBoost(); + fieldValue = field.getValue(); + } + + + @Override + public void readFields(DataInput in) throws IOException { + fieldName = IOUtil.readString(in); + fieldType = TYPE.findByValue(IOUtil.readVInt(in)); + fieldBoost = in.readDouble(); + int length = IOUtil.readVInt(in); + fieldValue = new byte[length]; + in.readFully(fieldValue); + } + + @Override + public void write(DataOutput out) throws IOException { + IOUtil.writeString(out, fieldName); + IOUtil.writeVInt(out, fieldType.getValue()); + out.writeDouble(fieldBoost); + IOUtil.writeVInt(out,fieldValue.length); + out.write(fieldValue); + } + + public String getFieldName() { + return fieldName; + } + + public void setFieldName(String fieldName) { + this.fieldName = fieldName; + } + + public TYPE getFieldTypeValue() { + return fieldType; + } + + public void setFieldTypeValue(TYPE fieldType) { + this.fieldType = fieldType; + } + + public double getFieldBoost() { + return fieldBoost; + } + + public void setFieldBoost(double fieldBoost) { + this.fieldBoost = fieldBoost; + } + + public byte[] getFieldValue() { + return fieldValue; + } + + public void setFieldValue(byte[] fieldValue) { + this.fieldValue = fieldValue; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + long temp; + temp = Double.doubleToLongBits(fieldBoost); + result = prime * result + (int) (temp ^ (temp >>> 32)); + result = prime * result + ((fieldName == null) ? 0 : fieldName.hashCode()); + result = prime * result + fieldType.getValue(); + result = prime * result + Arrays.hashCode(fieldValue); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + BlurField other = (BlurField) obj; + if (Double.doubleToLongBits(fieldBoost) != Double + .doubleToLongBits(other.fieldBoost)) + return false; + if (fieldName == null) { + if (other.fieldName != null) + return false; + } else if (!fieldName.equals(other.fieldName)) + return false; + if (fieldType.getValue() != other.fieldType.getValue()) + return false; + if (!Arrays.equals(fieldValue, other.fieldValue)) + return false; + return true; + } + + @Override + public String toString() { + return "BlurField [fieldName=" + fieldName + ", fieldTypeValue=" + + fieldType.getValue() + ", fieldBoost=" + fieldBoost + ", fieldValue=" + + Arrays.toString(fieldValue) + "]"; + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/996ed6a6/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurDocLocationTest.java ---------------------------------------------------------------------- diff --git a/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurDocLocationTest.java b/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurDocLocationTest.java new file mode 100644 index 0000000..c27c95a --- /dev/null +++ b/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurDocLocationTest.java @@ -0,0 +1,43 @@ +package org.apache.blur.mr; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; + +import org.apache.blur.thrift.generated.DocLocation; +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.DataOutputBuffer; +import org.junit.Before; +import org.junit.Test; + +public class BlurDocLocationTest { + + private BlurDocLocation blurDocLocationBefore; + private BlurDocLocation blurDocLocationAfter; + + @Before + public void setUp(){ + DocLocation docLocation = new DocLocation(); + docLocation.setDoc(0001); + docLocation.setShardIndex(001); + blurDocLocationBefore = new BlurDocLocation(docLocation); + } + + @Test + public void testBlurDocLocationWritable() throws IOException{ + DataOutputBuffer dob = new DataOutputBuffer(); + blurDocLocationBefore.write(dob); + + DataInputBuffer dib = new DataInputBuffer(); + dib.reset(dob.getData(), dob.getLength()); + + blurDocLocationAfter = new BlurDocLocation(); + blurDocLocationAfter.readFields(dib); + + assertTrue(blurDocLocationBefore.equals(blurDocLocationAfter)); + assertEquals(blurDocLocationBefore.getDocId(), blurDocLocationAfter.getDocId()); + assertEquals(blurDocLocationBefore.getShardIndexId(), blurDocLocationAfter.getShardIndexId()); + + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/996ed6a6/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurDocumentTest.java ---------------------------------------------------------------------- diff --git a/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurDocumentTest.java b/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurDocumentTest.java new file mode 100644 index 0000000..6f7e872 --- /dev/null +++ b/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurDocumentTest.java @@ -0,0 +1,55 @@ +package org.apache.blur.mr; + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.apache.blur.thrift.generated.Document; +import org.apache.blur.thrift.generated.Field; +import org.apache.blur.thrift.generated.TYPE; +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.DataOutputBuffer; +import org.junit.Before; +import org.junit.Test; + +public class BlurDocumentTest { + private BlurDocument blurDocumentBefore; + private BlurDocument blurDocumentAfter; + + @Before + public void setup(){ + Field field1 = new Field(); + field1.setName("Field1"); + field1.setBoost(2.0); + field1.setType(TYPE.INT); + field1.setValue(ByteBuffer.allocate((Integer.SIZE)/8).putInt(5).array()); + + Field field2 = new Field(); + field2.setName("Field2"); + field2.setBoost(2.0); + field2.setType(TYPE.STRING); + String value = "test"; + field2.setValue(value.getBytes()); + + Document d = new Document(); + d.addToFields(field1); + d.addToFields(field2); + + blurDocumentBefore = new BlurDocument(d); + } + + @Test + public void testBlurDocumentWritable() throws IOException{ + DataOutputBuffer dob = new DataOutputBuffer(); + blurDocumentBefore.write(dob); + + DataInputBuffer dib = new DataInputBuffer(); + dib.reset(dob.getData(), dob.getLength()); + + blurDocumentAfter = new BlurDocument(); + blurDocumentAfter.readFields(dib); + + assertTrue(blurDocumentBefore.equals(blurDocumentAfter)); + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/996ed6a6/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurFieldTest.java ---------------------------------------------------------------------- diff --git a/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurFieldTest.java b/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurFieldTest.java new file mode 100644 index 0000000..0d1e95c --- /dev/null +++ b/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurFieldTest.java @@ -0,0 +1,96 @@ +package org.apache.blur.mr; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.apache.blur.thrift.generated.Field; +import org.apache.blur.thrift.generated.TYPE; +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.DataOutputBuffer; +import org.junit.Test; + +public class BlurFieldTest { + + + @Test + public void testBlurFieldWritableforIntValue() throws IOException{ + BlurField blurFieldBefore; + BlurField blurFieldAfter; + Field field = new Field(); + field.setName("Field1"); + field.setBoost(2.0); + field.setType(TYPE.INT); + field.setValue(ByteBuffer.allocate((Integer.SIZE)/8).putInt(5).array()); + blurFieldBefore = new BlurField(field); + + DataOutputBuffer dob = new DataOutputBuffer(); + blurFieldBefore.write(dob); + + DataInputBuffer dib = new DataInputBuffer(); + dib.reset(dob.getData(), dob.getLength()); + + blurFieldAfter = new BlurField(); + blurFieldAfter.readFields(dib); + + assertTrue(blurFieldBefore.equals(blurFieldAfter)); + assertEquals(blurFieldBefore.getFieldName(),blurFieldAfter.getFieldName()); + assertEquals((ByteBuffer.wrap(blurFieldBefore.getFieldValue()).getInt()),(ByteBuffer.wrap(blurFieldAfter.getFieldValue()).getInt())); + + } + + @Test + public void testBlurFieldWritableforDoubleValue() throws IOException{ + BlurField blurFieldBefore; + BlurField blurFieldAfter; + Field field = new Field(); + field.setName("Field1"); + field.setBoost(2.0); + field.setType(TYPE.DOUBLE); + field.setValue(ByteBuffer.allocate((Double.SIZE)/8).putDouble(10.0).array()); + blurFieldBefore = new BlurField(field); + + DataOutputBuffer dob = new DataOutputBuffer(); + blurFieldBefore.write(dob); + + DataInputBuffer dib = new DataInputBuffer(); + dib.reset(dob.getData(), dob.getLength()); + + blurFieldAfter = new BlurField(); + blurFieldAfter.readFields(dib); + + assertTrue(blurFieldBefore.equals(blurFieldAfter)); + assertEquals(blurFieldBefore.getFieldName(),blurFieldAfter.getFieldName()); + assertEquals(String.valueOf((ByteBuffer.wrap(blurFieldBefore.getFieldValue()).getDouble())),String.valueOf((ByteBuffer.wrap(blurFieldAfter.getFieldValue()).getDouble()))); + + } + + @Test + public void testBlurFieldWritableforStringValue() throws IOException{ + BlurField blurFieldBefore; + BlurField blurFieldAfter; + Field field = new Field(); + field.setName("Field1"); + field.setBoost(2.0); + field.setType(TYPE.STRING); + field.setValue(new String("Test").getBytes()); + blurFieldBefore = new BlurField(field); + + DataOutputBuffer dob = new DataOutputBuffer(); + blurFieldBefore.write(dob); + + DataInputBuffer dib = new DataInputBuffer(); + dib.reset(dob.getData(), dob.getLength()); + + blurFieldAfter = new BlurField(); + blurFieldAfter.readFields(dib); + + assertTrue(blurFieldBefore.equals(blurFieldAfter)); + assertEquals(blurFieldBefore.getFieldName(),blurFieldAfter.getFieldName()); + + assertEquals(new String(blurFieldBefore.getFieldValue()), new String(blurFieldAfter.getFieldValue())); + + } +}
