Refactoring some code.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/7821c8f4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/7821c8f4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/7821c8f4 Branch: refs/heads/0.2-dev Commit: 7821c8f4b976b0a084a721c75c4957cb6a66886b Parents: 64943aa Author: Aaron McCurry <[email protected]> Authored: Sat Nov 24 20:52:41 2012 -0500 Committer: Aaron McCurry <[email protected]> Committed: Sat Nov 24 20:52:41 2012 -0500 ---------------------------------------------------------------------- .../blur/manager/writer/TransactionRecorder.java | 2 +- .../org/apache/blur/thrift/BlurShardServer.java | 12 +- .../org/apache/blur/thrift/lucene/Convert.java | 444 --------------- .../apache/blur/utils/ThriftLuceneConversion.java | 443 ++++++++++++++ .../org/apache/blur/manager/IndexManagerTest.java | 13 - .../org/apache/blur/thrift/ITBlurClusterTest.java | 6 +- .../blur/testsuite/SearchLuceneApiTable.java | 6 +- 7 files changed, 456 insertions(+), 470 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7821c8f4/src/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java b/src/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java index 72d19d1..4901be3 100644 --- a/src/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java +++ b/src/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java @@ -16,7 +16,7 @@ package org.apache.blur.manager.writer; * See the License for the specific language governing permissions and * limitations under the License. */ -import static org.apache.blur.thrift.lucene.Convert.toLucene; +import static org.apache.blur.utils.ThriftLuceneConversion.toLucene; import java.io.ByteArrayInputStream; import java.io.DataInputStream; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7821c8f4/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java index 088370f..032aa3c 100644 --- a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java +++ b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java @@ -16,10 +16,10 @@ package org.apache.blur.thrift; * See the License for the specific language governing permissions and * limitations under the License. */ -import static org.apache.blur.thrift.lucene.Convert.toThrift; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_CACHE_MAX_QUERYCACHE_ELEMENTS; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_CACHE_MAX_TIMETOLIVE; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_DATA_FETCH_THREAD_COUNT; +import static org.apache.blur.utils.ThriftLuceneConversion.toThrift; import java.io.IOException; import java.nio.ByteBuffer; @@ -61,9 +61,9 @@ import org.apache.blur.thrift.generated.TableStats; import org.apache.blur.thrift.generated.Term; import org.apache.blur.thrift.generated.TopFieldDocs; import org.apache.blur.thrift.generated.UpdatePackage; -import org.apache.blur.thrift.lucene.Convert; import org.apache.blur.utils.BlurConstants; import org.apache.blur.utils.BlurUtil; +import org.apache.blur.utils.ThriftLuceneConversion; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.Filter; import org.apache.lucene.search.IndexSearcher; @@ -216,10 +216,10 @@ public class BlurShardServer extends TableAdmin implements Iface { Collection<SearchAction> searchersToSearch = getSearchActions(tableDescriptor, shardIndexes, searchers); List<Future<TopFieldDocs>> futures = new ArrayList<Future<TopFieldDocs>>(searchersToSearch.size()); - Query query = Convert.toLuceneQuery(queryArgs.query); - Filter filter = Convert.toLuceneFilter(queryArgs); - Sort sort = Convert.toLuceneSort(queryArgs); - ScoreDoc after = Convert.toLucene(queryArgs.getAfter()); + Query query = ThriftLuceneConversion.toLuceneQuery(queryArgs.query); + Filter filter = ThriftLuceneConversion.toLuceneFilter(queryArgs); + Sort sort = ThriftLuceneConversion.toLuceneSort(queryArgs); + ScoreDoc after = ThriftLuceneConversion.toLucene(queryArgs.getAfter()); boolean doDocScores = queryArgs.isDoDocScores(); boolean doMaxScore = queryArgs.isDoMaxScore(); int numberToFetch = queryArgs.getNumberToFetch(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7821c8f4/src/blur-core/src/main/java/org/apache/blur/thrift/lucene/Convert.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/lucene/Convert.java b/src/blur-core/src/main/java/org/apache/blur/thrift/lucene/Convert.java deleted file mode 100644 index 2a3ce3e..0000000 --- a/src/blur-core/src/main/java/org/apache/blur/thrift/lucene/Convert.java +++ /dev/null @@ -1,444 +0,0 @@ -package org.apache.blur.thrift.lucene; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; - -import org.apache.blur.log.Log; -import org.apache.blur.log.LogFactory; -import org.apache.blur.lucene.serializer.QueryWritable; -import org.apache.blur.thrift.generated.Field; -import org.apache.blur.thrift.generated.QueryArgs; -import org.apache.blur.thrift.generated.ScoreDoc; -import org.apache.blur.thrift.generated.SortField; -import org.apache.blur.thrift.generated.SortType; -import org.apache.blur.thrift.generated.TYPE; -import org.apache.blur.thrift.generated.TopFieldDocs; -import org.apache.blur.utils.BlurUtil; -import org.apache.hadoop.io.DataInputBuffer; -import org.apache.hadoop.io.DataOutputBuffer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.DoubleField; -import org.apache.lucene.document.Field.Store; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.FloatField; -import org.apache.lucene.document.IntField; -import org.apache.lucene.document.LongField; -import org.apache.lucene.document.StoredField; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField.Type; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.util.BytesRef; - -public class Convert { - - private static final Log LOG = LogFactory.getLog(Convert.class); - - public static TopFieldDocs toThrift(TopDocs topDocs) { - return toThrift(new org.apache.lucene.search.TopFieldDocs(topDocs.totalHits, topDocs.scoreDocs, null, Float.NaN)); - } - - public static org.apache.lucene.search.ScoreDoc toLucene(org.apache.blur.thrift.generated.ScoreDoc scoreDoc) { - if (scoreDoc == null) { - return null; - } - long docLocation = scoreDoc.getDocLocation(); - return new org.apache.lucene.search.ScoreDoc(BlurUtil.getDocumentId(docLocation), (float) scoreDoc.score, BlurUtil.getShardIndex(docLocation)); - } - - public static org.apache.blur.thrift.generated.Document toThrift(org.apache.lucene.document.Document document) { - org.apache.blur.thrift.generated.Document result = new org.apache.blur.thrift.generated.Document(); - List<IndexableField> fields = document.getFields(); - for (IndexableField field : fields) { - result.addToFields(toThrift(field)); - } - return result; - } - - public static Field toThrift(IndexableField field) { - Field result = new Field(); - result.setBoost(field.boost()); - result.setName(field.name()); - Number number = field.numericValue(); - if (number != null) { - if (number instanceof Byte || number instanceof Short || number instanceof Integer) { - result.setType(TYPE.INT); - result.setValue(toByteBuffer((Integer) number)); - } else if (number instanceof Long) { - result.setType(TYPE.LONG); - result.setValue(toByteBuffer((Long) number)); - } else if (number instanceof Float) { - result.setType(TYPE.FLOAT); - result.setValue(toByteBuffer((Float) number)); - } else if (number instanceof Double) { - result.setType(TYPE.DOUBLE); - result.setValue(toByteBuffer((Double) number)); - } else { - throw new IllegalArgumentException("cannot store numeric type " + number.getClass()); - } - } else { - BytesRef bytes = field.binaryValue(); - if (bytes != null) { - result.setType(TYPE.BINARY); - result.setValue(toByteBuffer(bytes)); - } else { - result.setType(TYPE.STRING); - result.setValue(toByteBuffer(field.stringValue())); - } - } - return result; - } - - public static ByteBuffer toByteBuffer(String val) { - if (val == null) { - return null; - } - return toByteBuffer(new BytesRef(val)); - } - - public static ByteBuffer toByteBuffer(BytesRef val) { - if (val == null) { - return null; - } - return flip(ByteBuffer.allocate(val.length).put(val.bytes, val.offset, val.length)); - } - - private static ByteBuffer flip(ByteBuffer buffer) { - buffer.flip(); - return buffer; - } - - public static ByteBuffer toByteBuffer(Double val) { - if (val == null) { - return null; - } - return flip(ByteBuffer.allocate(8).putDouble(val)); - } - - public static ByteBuffer toByteBuffer(Long val) { - if (val == null) { - return null; - } - return flip(ByteBuffer.allocate(8).putLong(val)); - } - - public static ByteBuffer toByteBuffer(Float val) { - if (val == null) { - return null; - } - return flip(ByteBuffer.allocate(4).putFloat(val)); - } - - public static ByteBuffer toByteBuffer(Integer val) { - if (val == null) { - return null; - } - return flip(ByteBuffer.allocate(4).putInt(val)); - } - - public static String toString(org.apache.blur.thrift.generated.Document document) { - StringBuilder builder = new StringBuilder(); - builder.append('{'); - for (Field f : document.getFields()) { - if (builder.length() != 0) { - builder.append(','); - } - builder.append(toString(f)); - } - builder.append('}'); - return builder.toString(); - } - - public static String toString(Field f) { - StringBuilder builder = new StringBuilder(); - builder.append('{'); - String name = f.getName(); - builder.append(name); - builder.append(','); - - TYPE type = f.getType(); - builder.append(type.name()); - builder.append(','); - - byte[] value = f.getValue(); - builder.append(toString(type, value)); - builder.append(','); - - double boost = f.getBoost(); - builder.append(boost); - builder.append('}'); - return builder.toString(); - } - - public static String toString(TYPE type, byte[] value) { - switch (type) { - case STRING: - return new BytesRef(value).utf8ToString(); - case INT: - return Integer.toString(ByteBuffer.wrap(value).getInt()); - case LONG: - return Long.toString(ByteBuffer.wrap(value).getLong()); - case FLOAT: - return Float.toString(ByteBuffer.wrap(value).getFloat()); - case DOUBLE: - return Double.toString(ByteBuffer.wrap(value).getDouble()); - case BINARY: - return ByteBuffer.wrap(value).toString(); - default: - throw new RuntimeException("Type unknown."); - } - } - - public static Document toLucene(org.apache.blur.thrift.generated.Document document) throws IOException { - Document result = new Document(); - List<Field> fields = document.getFields(); - for (Field field : fields) { - result.add(toLucene(field)); - } - return result; - } - - public static IndexableField toLucene(Field field) throws IOException { - TYPE type = field.getType(); - org.apache.lucene.document.Field result; - switch (type) { - case STRING: - result = new StringField(field.getName(), toString(field.getValue()), Store.YES); - break; - case TEXT: - result = new TextField(field.getName(), toString(field.getValue()), Store.YES); - break; - case INT: - result = new IntField(field.getName(), toInteger(field.getValue()), Store.YES); - break; - case FLOAT: - result = new FloatField(field.getName(), toFloat(field.getValue()), Store.YES); - break; - case LONG: - result = new LongField(field.getName(), toLong(field.getValue()), Store.YES); - break; - case DOUBLE: - result = new DoubleField(field.getName(), toDouble(field.getValue()), Store.YES); - break; - case BINARY: - result = new StoredField(field.getName(), field.getValue()); - break; - default: - throw new RuntimeException("Not supported"); - } - FieldType fieldType = result.fieldType(); - if (fieldType.indexed() && !fieldType.omitNorms()) { - result.setBoost((float) field.getBoost()); - } - return result; - } - - public static double toDouble(byte[] value) { - throw new RuntimeException("Not implemented"); - } - - public static long toLong(byte[] value) { - throw new RuntimeException("Not implemented"); - } - - public static float toFloat(byte[] value) { - throw new RuntimeException("Not implemented"); - } - - public static int toInteger(byte[] value) { - throw new RuntimeException("Not implemented"); - } - - public static String toString(byte[] value) throws IOException { - return new String(value, "UTF-8"); - } - - public static Term toLucene(org.apache.blur.thrift.generated.Term term) { - return new Term(term.getField(), new BytesRef(term.getBytes())); - } - - public static Term[] toLucene(org.apache.blur.thrift.generated.Term[] terms) { - Term[] result = new Term[terms.length]; - for (int i = 0; i < terms.length; i++) { - result[i] = toLucene(terms[i]); - } - return result; - } - - public static List<Document> toLucene(List<org.apache.blur.thrift.generated.Document> documents) throws IOException { - List<Document> docs = new ArrayList<Document>(); - for (org.apache.blur.thrift.generated.Document doc : documents) { - docs.add(toLucene(doc)); - } - return docs; - } - - public static Query[] toLucene(ByteBuffer[] queries) throws IOException { - Query[] result = new Query[queries.length]; - for (int i = 0; i < queries.length; i++) { - result[i] = toLuceneQuery(queries[i]); - } - return result; - } - - public static Query toLuceneQuery(ByteBuffer byteBuffer) throws IOException { - DataInputBuffer in = new DataInputBuffer(); - in.reset(byteBuffer.array(),byteBuffer.arrayOffset() + byteBuffer.position() , byteBuffer.remaining()); - QueryWritable qw = new QueryWritable(); - qw.readFields(in); - return qw.getQuery(); - } - - public static Query toLucene(QueryArgs queryArgs) throws IOException { - return toLuceneQuery(queryArgs.getQuery()); - } - - public static Query toLuceneQuery(byte[] bs) throws IOException { - DataInputBuffer in = new DataInputBuffer(); - in.reset(bs, bs.length); - QueryWritable qw = new QueryWritable(); - qw.readFields(in); - return qw.getQuery(); - } - - public static byte[] toBytes(Query query) throws IOException { - DataOutputBuffer out = new DataOutputBuffer(); - QueryWritable qw = new QueryWritable(); - qw.setQuery(query); - qw.write(out); - out.close(); - - byte[] data = out.getData(); - int length = out.getLength(); - byte[] buf = new byte[length]; - System.arraycopy(data, 0, buf, 0, length); - return buf; - } - - public static Filter toLuceneFilter(QueryArgs queryArgs) { - byte[] filter = queryArgs.getFilter(); - return null; - } - - public static Sort toLuceneSort(QueryArgs queryArgs) { - return null; - } - - public static TopFieldDocs toThrift(org.apache.lucene.search.TopFieldDocs topFieldDocs) { - TopFieldDocs result = new TopFieldDocs(); - result.setFields(toThrift(topFieldDocs.fields)); - result.setMaxScore(topFieldDocs.getMaxScore()); - result.setScoreDocs(toThrift(topFieldDocs.scoreDocs)); - result.setTotalHits(topFieldDocs.totalHits); - return result; - } - - public static List<ScoreDoc> toThrift(org.apache.lucene.search.ScoreDoc[] scoreDocs) { - List<ScoreDoc> result = new ArrayList<ScoreDoc>(scoreDocs.length); - for (int i = 0; i < scoreDocs.length; i++) { - org.apache.lucene.search.ScoreDoc scoreDoc = scoreDocs[i]; - if (scoreDoc instanceof org.apache.lucene.search.FieldDoc) { - result.add(toThrift((org.apache.lucene.search.FieldDoc) scoreDoc)); - } else { - result.add(toThrift(scoreDoc)); - } - } - return result; - } - - public List<ScoreDoc> toThrift(org.apache.lucene.search.FieldDoc[] fieldDocs) { - List<ScoreDoc> result = new ArrayList<ScoreDoc>(fieldDocs.length); - for (int i = 0; i < fieldDocs.length; i++) { - result.add(toThrift(fieldDocs[i])); - } - return result; - } - - public static List<SortField> toThrift(org.apache.lucene.search.SortField[] fields) { - if (fields == null) { - return null; - } - List<SortField> result = new ArrayList<SortField>(fields.length); - for (int i = 0; i < fields.length; i++) { - result.add(toThrift(fields[i])); - } - return result; - } - - public static SortField toThrift(org.apache.lucene.search.SortField sortField) { - if (sortField == null) { - return null; - } - SortField result = new SortField(); - result.setField(sortField.getField()); - result.setReverse(sortField.getReverse()); - result.setType(toThrift(sortField.getType())); - return result; - } - - public static ScoreDoc toThrift(org.apache.lucene.search.ScoreDoc scoreDoc) { - if (scoreDoc == null) { - return null; - } - ScoreDoc result = new ScoreDoc(); - result.setDocLocation(BlurUtil.getDocLocation(scoreDoc.shardIndex, scoreDoc.doc)); - result.setScore(scoreDoc.score); - return result; - } - - public static ScoreDoc toThrift(org.apache.lucene.search.FieldDoc fieldDoc) { - if (fieldDoc == null) { - return null; - } - ScoreDoc result = new ScoreDoc(); - result.setDocLocation(BlurUtil.getDocLocation(fieldDoc.shardIndex, fieldDoc.doc)); - result.setFields(convert(fieldDoc.fields)); - result.setScore(fieldDoc.score); - return result; - } - - public static SortType toThrift(Type type) { - if (type == null) { - return null; - } - switch (type) { - case BYTE: - return SortType.BYTE; - case BYTES: - return SortType.BYTES; - case SCORE: - return SortType.SCORE; - case DOC: - return SortType.DOC; - case STRING: - return SortType.STRING; - case INT: - return SortType.INT; - case FLOAT: - return SortType.FLOAT; - case LONG: - return SortType.LONG; - case DOUBLE: - return SortType.DOUBLE; - case SHORT: - return SortType.SHORT; - case STRING_VAL: - return SortType.STRING_VAL; - default: - throw new RuntimeException("Not supported"); - } - } - - public static List<ByteBuffer> convert(Object[] fields) { - // throw new RuntimeException("Not supported YET!"); - LOG.error("Not supported YET!"); - return null; - } -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7821c8f4/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java b/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java new file mode 100644 index 0000000..543c82c --- /dev/null +++ b/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java @@ -0,0 +1,443 @@ +package org.apache.blur.utils; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.apache.blur.log.Log; +import org.apache.blur.log.LogFactory; +import org.apache.blur.lucene.serializer.QueryWritable; +import org.apache.blur.thrift.generated.Field; +import org.apache.blur.thrift.generated.QueryArgs; +import org.apache.blur.thrift.generated.ScoreDoc; +import org.apache.blur.thrift.generated.SortField; +import org.apache.blur.thrift.generated.SortType; +import org.apache.blur.thrift.generated.TYPE; +import org.apache.blur.thrift.generated.TopFieldDocs; +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoubleField; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.FloatField; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.LongField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField.Type; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.util.BytesRef; + +public class ThriftLuceneConversion { + + private static final Log LOG = LogFactory.getLog(ThriftLuceneConversion.class); + + public static TopFieldDocs toThrift(TopDocs topDocs) { + return toThrift(new org.apache.lucene.search.TopFieldDocs(topDocs.totalHits, topDocs.scoreDocs, null, Float.NaN)); + } + + public static org.apache.lucene.search.ScoreDoc toLucene(org.apache.blur.thrift.generated.ScoreDoc scoreDoc) { + if (scoreDoc == null) { + return null; + } + long docLocation = scoreDoc.getDocLocation(); + return new org.apache.lucene.search.ScoreDoc(BlurUtil.getDocumentId(docLocation), (float) scoreDoc.score, BlurUtil.getShardIndex(docLocation)); + } + + public static org.apache.blur.thrift.generated.Document toThrift(org.apache.lucene.document.Document document) { + org.apache.blur.thrift.generated.Document result = new org.apache.blur.thrift.generated.Document(); + List<IndexableField> fields = document.getFields(); + for (IndexableField field : fields) { + result.addToFields(toThrift(field)); + } + return result; + } + + public static Field toThrift(IndexableField field) { + Field result = new Field(); + result.setBoost(field.boost()); + result.setName(field.name()); + Number number = field.numericValue(); + if (number != null) { + if (number instanceof Byte || number instanceof Short || number instanceof Integer) { + result.setType(TYPE.INT); + result.setValue(toByteBuffer((Integer) number)); + } else if (number instanceof Long) { + result.setType(TYPE.LONG); + result.setValue(toByteBuffer((Long) number)); + } else if (number instanceof Float) { + result.setType(TYPE.FLOAT); + result.setValue(toByteBuffer((Float) number)); + } else if (number instanceof Double) { + result.setType(TYPE.DOUBLE); + result.setValue(toByteBuffer((Double) number)); + } else { + throw new IllegalArgumentException("cannot store numeric type " + number.getClass()); + } + } else { + BytesRef bytes = field.binaryValue(); + if (bytes != null) { + result.setType(TYPE.BINARY); + result.setValue(toByteBuffer(bytes)); + } else { + result.setType(TYPE.STRING); + result.setValue(toByteBuffer(field.stringValue())); + } + } + return result; + } + + public static ByteBuffer toByteBuffer(String val) { + if (val == null) { + return null; + } + return toByteBuffer(new BytesRef(val)); + } + + public static ByteBuffer toByteBuffer(BytesRef val) { + if (val == null) { + return null; + } + return flip(ByteBuffer.allocate(val.length).put(val.bytes, val.offset, val.length)); + } + + private static ByteBuffer flip(ByteBuffer buffer) { + buffer.flip(); + return buffer; + } + + public static ByteBuffer toByteBuffer(Double val) { + if (val == null) { + return null; + } + return flip(ByteBuffer.allocate(8).putDouble(val)); + } + + public static ByteBuffer toByteBuffer(Long val) { + if (val == null) { + return null; + } + return flip(ByteBuffer.allocate(8).putLong(val)); + } + + public static ByteBuffer toByteBuffer(Float val) { + if (val == null) { + return null; + } + return flip(ByteBuffer.allocate(4).putFloat(val)); + } + + public static ByteBuffer toByteBuffer(Integer val) { + if (val == null) { + return null; + } + return flip(ByteBuffer.allocate(4).putInt(val)); + } + + public static String toString(org.apache.blur.thrift.generated.Document document) { + StringBuilder builder = new StringBuilder(); + builder.append('{'); + for (Field f : document.getFields()) { + if (builder.length() != 0) { + builder.append(','); + } + builder.append(toString(f)); + } + builder.append('}'); + return builder.toString(); + } + + public static String toString(Field f) { + StringBuilder builder = new StringBuilder(); + builder.append('{'); + String name = f.getName(); + builder.append(name); + builder.append(','); + + TYPE type = f.getType(); + builder.append(type.name()); + builder.append(','); + + byte[] value = f.getValue(); + builder.append(toString(type, value)); + builder.append(','); + + double boost = f.getBoost(); + builder.append(boost); + builder.append('}'); + return builder.toString(); + } + + public static String toString(TYPE type, byte[] value) { + switch (type) { + case STRING: + return new BytesRef(value).utf8ToString(); + case INT: + return Integer.toString(ByteBuffer.wrap(value).getInt()); + case LONG: + return Long.toString(ByteBuffer.wrap(value).getLong()); + case FLOAT: + return Float.toString(ByteBuffer.wrap(value).getFloat()); + case DOUBLE: + return Double.toString(ByteBuffer.wrap(value).getDouble()); + case BINARY: + return ByteBuffer.wrap(value).toString(); + default: + throw new RuntimeException("Type unknown."); + } + } + + public static Document toLucene(org.apache.blur.thrift.generated.Document document) throws IOException { + Document result = new Document(); + List<Field> fields = document.getFields(); + for (Field field : fields) { + result.add(toLucene(field)); + } + return result; + } + + public static IndexableField toLucene(Field field) throws IOException { + TYPE type = field.getType(); + org.apache.lucene.document.Field result; + switch (type) { + case STRING: + result = new StringField(field.getName(), toString(field.getValue()), Store.YES); + break; + case TEXT: + result = new TextField(field.getName(), toString(field.getValue()), Store.YES); + break; + case INT: + result = new IntField(field.getName(), toInteger(field.getValue()), Store.YES); + break; + case FLOAT: + result = new FloatField(field.getName(), toFloat(field.getValue()), Store.YES); + break; + case LONG: + result = new LongField(field.getName(), toLong(field.getValue()), Store.YES); + break; + case DOUBLE: + result = new DoubleField(field.getName(), toDouble(field.getValue()), Store.YES); + break; + case BINARY: + result = new StoredField(field.getName(), field.getValue()); + break; + default: + throw new RuntimeException("Not supported"); + } + FieldType fieldType = result.fieldType(); + if (fieldType.indexed() && !fieldType.omitNorms()) { + result.setBoost((float) field.getBoost()); + } + return result; + } + + public static double toDouble(byte[] value) { + throw new RuntimeException("Not implemented"); + } + + public static long toLong(byte[] value) { + throw new RuntimeException("Not implemented"); + } + + public static float toFloat(byte[] value) { + throw new RuntimeException("Not implemented"); + } + + public static int toInteger(byte[] value) { + throw new RuntimeException("Not implemented"); + } + + public static String toString(byte[] value) throws IOException { + return new String(value, "UTF-8"); + } + + public static Term toLucene(org.apache.blur.thrift.generated.Term term) { + return new Term(term.getField(), new BytesRef(term.getBytes())); + } + + public static Term[] toLucene(org.apache.blur.thrift.generated.Term[] terms) { + Term[] result = new Term[terms.length]; + for (int i = 0; i < terms.length; i++) { + result[i] = toLucene(terms[i]); + } + return result; + } + + public static List<Document> toLucene(List<org.apache.blur.thrift.generated.Document> documents) throws IOException { + List<Document> docs = new ArrayList<Document>(); + for (org.apache.blur.thrift.generated.Document doc : documents) { + docs.add(toLucene(doc)); + } + return docs; + } + + public static Query[] toLucene(ByteBuffer[] queries) throws IOException { + Query[] result = new Query[queries.length]; + for (int i = 0; i < queries.length; i++) { + result[i] = toLuceneQuery(queries[i]); + } + return result; + } + + public static Query toLuceneQuery(ByteBuffer byteBuffer) throws IOException { + DataInputBuffer in = new DataInputBuffer(); + in.reset(byteBuffer.array(),byteBuffer.arrayOffset() + byteBuffer.position() , byteBuffer.remaining()); + QueryWritable qw = new QueryWritable(); + qw.readFields(in); + return qw.getQuery(); + } + + public static Query toLucene(QueryArgs queryArgs) throws IOException { + return toLuceneQuery(queryArgs.getQuery()); + } + + public static Query toLuceneQuery(byte[] bs) throws IOException { + DataInputBuffer in = new DataInputBuffer(); + in.reset(bs, bs.length); + QueryWritable qw = new QueryWritable(); + qw.readFields(in); + return qw.getQuery(); + } + + public static byte[] toBytes(Query query) throws IOException { + DataOutputBuffer out = new DataOutputBuffer(); + QueryWritable qw = new QueryWritable(); + qw.setQuery(query); + qw.write(out); + out.close(); + + byte[] data = out.getData(); + int length = out.getLength(); + byte[] buf = new byte[length]; + System.arraycopy(data, 0, buf, 0, length); + return buf; + } + + public static Filter toLuceneFilter(QueryArgs queryArgs) { + byte[] filter = queryArgs.getFilter(); + return null; + } + + public static Sort toLuceneSort(QueryArgs queryArgs) { + return null; + } + + public static TopFieldDocs toThrift(org.apache.lucene.search.TopFieldDocs topFieldDocs) { + TopFieldDocs result = new TopFieldDocs(); + result.setFields(toThrift(topFieldDocs.fields)); + result.setMaxScore(topFieldDocs.getMaxScore()); + result.setScoreDocs(toThrift(topFieldDocs.scoreDocs)); + result.setTotalHits(topFieldDocs.totalHits); + return result; + } + + public static List<ScoreDoc> toThrift(org.apache.lucene.search.ScoreDoc[] scoreDocs) { + List<ScoreDoc> result = new ArrayList<ScoreDoc>(scoreDocs.length); + for (int i = 0; i < scoreDocs.length; i++) { + org.apache.lucene.search.ScoreDoc scoreDoc = scoreDocs[i]; + if (scoreDoc instanceof org.apache.lucene.search.FieldDoc) { + result.add(toThrift((org.apache.lucene.search.FieldDoc) scoreDoc)); + } else { + result.add(toThrift(scoreDoc)); + } + } + return result; + } + + public List<ScoreDoc> toThrift(org.apache.lucene.search.FieldDoc[] fieldDocs) { + List<ScoreDoc> result = new ArrayList<ScoreDoc>(fieldDocs.length); + for (int i = 0; i < fieldDocs.length; i++) { + result.add(toThrift(fieldDocs[i])); + } + return result; + } + + public static List<SortField> toThrift(org.apache.lucene.search.SortField[] fields) { + if (fields == null) { + return null; + } + List<SortField> result = new ArrayList<SortField>(fields.length); + for (int i = 0; i < fields.length; i++) { + result.add(toThrift(fields[i])); + } + return result; + } + + public static SortField toThrift(org.apache.lucene.search.SortField sortField) { + if (sortField == null) { + return null; + } + SortField result = new SortField(); + result.setField(sortField.getField()); + result.setReverse(sortField.getReverse()); + result.setType(toThrift(sortField.getType())); + return result; + } + + public static ScoreDoc toThrift(org.apache.lucene.search.ScoreDoc scoreDoc) { + if (scoreDoc == null) { + return null; + } + ScoreDoc result = new ScoreDoc(); + result.setDocLocation(BlurUtil.getDocLocation(scoreDoc.shardIndex, scoreDoc.doc)); + result.setScore(scoreDoc.score); + return result; + } + + public static ScoreDoc toThrift(org.apache.lucene.search.FieldDoc fieldDoc) { + if (fieldDoc == null) { + return null; + } + ScoreDoc result = new ScoreDoc(); + result.setDocLocation(BlurUtil.getDocLocation(fieldDoc.shardIndex, fieldDoc.doc)); + result.setFields(convert(fieldDoc.fields)); + result.setScore(fieldDoc.score); + return result; + } + + public static SortType toThrift(Type type) { + if (type == null) { + return null; + } + switch (type) { + case BYTE: + return SortType.BYTE; + case BYTES: + return SortType.BYTES; + case SCORE: + return SortType.SCORE; + case DOC: + return SortType.DOC; + case STRING: + return SortType.STRING; + case INT: + return SortType.INT; + case FLOAT: + return SortType.FLOAT; + case LONG: + return SortType.LONG; + case DOUBLE: + return SortType.DOUBLE; + case SHORT: + return SortType.SHORT; + case STRING_VAL: + return SortType.STRING_VAL; + default: + throw new RuntimeException("Not supported"); + } + } + + public static List<ByteBuffer> convert(Object[] fields) { + // throw new RuntimeException("Not supported YET!"); + LOG.error("Not supported YET!"); + return null; + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7821c8f4/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java b/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java index 5fb73e6..82e3d92 100644 --- a/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java +++ b/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java @@ -23,14 +23,10 @@ import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; import org.apache.blur.manager.indexserver.LocalIndexServer; import org.apache.blur.metrics.BlurMetrics; import org.apache.blur.thrift.generated.BlurException; -import org.apache.blur.thrift.generated.Schema; import org.apache.blur.utils.BlurConstants; import org.apache.blur.utils.BlurUtil; import org.apache.hadoop.conf.Configuration; @@ -94,15 +90,6 @@ public class IndexManagerTest { assertEquals(0, indexManager.recordFrequency(TABLE, FAMILY, "testcol1", "NO VALUE")); } - @Test - public void testSchema() throws Exception { - Schema schema = indexManager.schema(TABLE); - assertEquals(TABLE, schema.table); - Map<String, Set<String>> columnFamilies = schema.columnFamilies; - assertEquals(new TreeSet<String>(Arrays.asList(FAMILY, FAMILY2)), new TreeSet<String>(columnFamilies.keySet())); - assertEquals(new TreeSet<String>(Arrays.asList("testcol1", "testcol2", "testcol3", "testcol12", "testcol13")), new TreeSet<String>(columnFamilies.get(FAMILY))); - assertEquals(new TreeSet<String>(Arrays.asList("testcol18")), new TreeSet<String>(columnFamilies.get(FAMILY2))); - } @Test public void testTerms() throws Exception { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7821c8f4/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java b/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java index dd0e4b1..35c9fea 100644 --- a/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java +++ b/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java @@ -40,7 +40,7 @@ import org.apache.blur.thrift.generated.TYPE; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.thrift.generated.TopFieldDocs; import org.apache.blur.thrift.generated.UpdatePackage; -import org.apache.blur.thrift.lucene.Convert; +import org.apache.blur.utils.ThriftLuceneConversion; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; @@ -133,7 +133,7 @@ public class ITBlurClusterTest { QueryArgs queryArgs = new QueryArgs(); Term term = new Term("value", "test"); TermQuery query = new TermQuery(term); - queryArgs.setQuery(Convert.toBytes(query)); + queryArgs.setQuery(ThriftLuceneConversion.toBytes(query)); List<TopFieldDocs> results = client.search(session, queryArgs); long totalHits = 0; for (TopFieldDocs fieldDocs : results) { @@ -175,7 +175,7 @@ public class ITBlurClusterTest { QueryArgs queryArgs = new QueryArgs(); Term term = new Term("value", "test"); TermQuery query = new TermQuery(term); - queryArgs.setQuery(Convert.toBytes(query)); + queryArgs.setQuery(ThriftLuceneConversion.toBytes(query)); List<TopFieldDocs> results = client.search(session, queryArgs); long totalHits = 0; for (TopFieldDocs fieldDocs : results) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7821c8f4/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/SearchLuceneApiTable.java ---------------------------------------------------------------------- diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/SearchLuceneApiTable.java b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/SearchLuceneApiTable.java index 3c344e3..5e03669 100644 --- a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/SearchLuceneApiTable.java +++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/SearchLuceneApiTable.java @@ -28,7 +28,7 @@ import org.apache.blur.thrift.generated.ShardServer.Client; import org.apache.blur.thrift.generated.QueryArgs; import org.apache.blur.thrift.generated.Session; import org.apache.blur.thrift.generated.TopFieldDocs; -import org.apache.blur.thrift.lucene.Convert; +import org.apache.blur.utils.ThriftLuceneConversion; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.queryparser.classic.ParseException; @@ -85,9 +85,9 @@ public class SearchLuceneApiTable { System.out.println("================"); for (TopFieldDocs topFieldDocs : topDocs) { System.out.println(topFieldDocs); - List<Document> docs = client.doc(session, Convert.toThrift(topFieldDocs.getScoreDocs()), null); + List<Document> docs = client.doc(session, ThriftLuceneConversion.toThrift(topFieldDocs.getScoreDocs()), null); for (Document document : docs) { - System.out.println(Convert.toString(document)); + System.out.println(ThriftLuceneConversion.toString(document)); } }
