Fixing Blur sort feature, was broken due to a method that was not implemented but silently performed the query.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/ea01b949 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/ea01b949 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/ea01b949 Branch: refs/heads/apache-blur-0.2 Commit: ea01b9491b819c508b1e330596b45aae025a2210 Parents: 561b49b Author: Aaron McCurry <[email protected]> Authored: Tue Jun 17 13:41:47 2014 -0400 Committer: Aaron McCurry <[email protected]> Committed: Tue Jun 17 13:41:47 2014 -0400 ---------------------------------------------------------------------- .../BlurResultPeekableIteratorComparator.java | 1 + .../java/org/apache/blur/utils/BlurUtil.java | 39 ++++++++++- .../org/apache/blur/thrift/BlurClusterTest.java | 72 ++++++++++++++++++++ .../apache/blur/thrift/SortFieldComparator.java | 9 ++- 4 files changed, 117 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ea01b949/blur-core/src/main/java/org/apache/blur/manager/results/BlurResultPeekableIteratorComparator.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/results/BlurResultPeekableIteratorComparator.java b/blur-core/src/main/java/org/apache/blur/manager/results/BlurResultPeekableIteratorComparator.java index 6d5c596..c28bd1c 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/results/BlurResultPeekableIteratorComparator.java +++ b/blur-core/src/main/java/org/apache/blur/manager/results/BlurResultPeekableIteratorComparator.java @@ -57,6 +57,7 @@ public class BlurResultPeekableIteratorComparator implements Comparator<Peekable return compare; } } + return 0; } if (sortFields1 == null && sortFields2 == null) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ea01b949/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java index 02e6f63..ed4ab0c 100644 --- a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java +++ b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java @@ -38,6 +38,7 @@ import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.Date; @@ -1331,8 +1332,42 @@ public class BlurUtil { } public static List<SortFieldResult> convertToSortFields(Object[] fields) { - // TODO Auto-generated method stub - return null; + if (fields == null) { + return null; + } + if (fields.length == 0) { + return Arrays.asList(); + } + List<SortFieldResult> result = new ArrayList<SortFieldResult>(); + for (Object o : fields) { + result.add(convertToSortField(o)); + } + return result; + } + + public static SortFieldResult convertToSortField(Object o) { + if (o == null) { + return SortFieldResult.nullValue(true); + } else if (o instanceof BytesRef) { + return SortFieldResult.binaryValue(getBytes((BytesRef) o)); + } else if (o instanceof String) { + return SortFieldResult.stringValue((String) o); + } else if (o instanceof Double) { + return SortFieldResult.doubleValue((Double) o); + } else if (o instanceof Integer) { + return SortFieldResult.intValue((Integer) o); + } else if (o instanceof Long) { + return SortFieldResult.longValue((Long) o); + } else if (o instanceof byte[]) { + return SortFieldResult.binaryValue((byte[]) o); + } + throw new RuntimeException("Object [" + o + "] is not supported"); + } + + private static byte[] getBytes(BytesRef ref) { + byte[] buf = new byte[ref.length]; + System.arraycopy(ref.bytes, ref.offset, buf, 0, ref.length); + return buf; } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ea01b949/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java index eb12d5d..72858b6 100644 --- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java +++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java @@ -50,16 +50,19 @@ import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.BlurQuery; import org.apache.blur.thrift.generated.BlurResult; import org.apache.blur.thrift.generated.BlurResults; +import org.apache.blur.thrift.generated.Column; import org.apache.blur.thrift.generated.ColumnDefinition; import org.apache.blur.thrift.generated.ErrorType; import org.apache.blur.thrift.generated.Facet; import org.apache.blur.thrift.generated.FetchResult; import org.apache.blur.thrift.generated.Query; +import org.apache.blur.thrift.generated.Record; import org.apache.blur.thrift.generated.RecordMutation; import org.apache.blur.thrift.generated.RowMutation; import org.apache.blur.thrift.generated.Schema; import org.apache.blur.thrift.generated.Selector; import org.apache.blur.thrift.generated.ShardState; +import org.apache.blur.thrift.generated.SortField; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.thrift.generated.TableStats; import org.apache.blur.thrift.util.BlurThriftHelper; @@ -274,6 +277,13 @@ public class BlurClusterTest { RowMutation rowMutation = BlurThriftHelper.newRowMutation(tableName, rowId, mutation); mutations.add(rowMutation); } + ColumnDefinition columnDefinition = new ColumnDefinition(); + columnDefinition.setFamily("test"); + columnDefinition.setColumnName("facet"); + columnDefinition.setFieldLessIndexed(true); + columnDefinition.setFieldType("string"); + columnDefinition.setSortable(true); + client.addColumnDefinition(tableName, columnDefinition); long s = System.nanoTime(); client.mutateBatch(mutations); long e = System.nanoTime(); @@ -325,6 +335,68 @@ public class BlurClusterTest { } + @Test + public void testSortedQueryWithSelector() throws BlurException, TException, IOException, InterruptedException { + final String tableName = "testSortedQueryWithSelector"; + createTable(tableName); + loadTable(tableName); + + Iface client = getClient(); + + BlurQuery blurQueryRow = new BlurQuery(); + Query queryRow = new Query(); + queryRow.setQuery("test.test:value"); + queryRow.setRowQuery(false); + blurQueryRow.setQuery(queryRow); + blurQueryRow.addToSortFields(new SortField("test", "facet", false)); + + blurQueryRow.setUseCacheIfPresent(false); + blurQueryRow.setCacheResult(false); + Selector selector = new Selector(); + selector.setRecordOnly(true); + blurQueryRow.setSelector(selector); + + BlurResults resultsRow = client.query(tableName, blurQueryRow); + long totalResults = resultsRow.getTotalResults(); + + assertEquals(numberOfDocs, resultsRow.getTotalResults()); + + String lastValue = null; + long totalFetched = 0; + do { + for (BlurResult blurResult : resultsRow.getResults()) { + FetchResult fetchResult = blurResult.getFetchResult(); + Record record = fetchResult.getRecordResult().getRecord(); + if (lastValue == null) { + lastValue = getColumnValue(record, "facet"); + } else { + String currentValue = getColumnValue(record, "facet"); + if (currentValue.compareTo(lastValue) < 0) { + fail("Current Value of [" + currentValue + "] can not be less than lastValue of [" + lastValue + "]"); + } + lastValue = currentValue; + } + totalFetched++; + } + int size = resultsRow.getResults().size(); + totalResults -= size; + if (totalResults > 0) { + blurQueryRow.setStart(blurQueryRow.getStart() + size); + resultsRow = client.query(tableName, blurQueryRow); + } + } while (totalResults > 0); + assertEquals(numberOfDocs, totalFetched); + } + + private String getColumnValue(Record record, String columnName) { + for (Column col : record.getColumns()) { + if (col.getName().equals(columnName)) { + return col.getValue(); + } + } + return null; + } + // @Test public void testQueryWithSelectorForDeepPagingPerformance() throws BlurException, TException, IOException, InterruptedException { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ea01b949/blur-thrift/src/main/java/org/apache/blur/thrift/SortFieldComparator.java ---------------------------------------------------------------------- diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/SortFieldComparator.java b/blur-thrift/src/main/java/org/apache/blur/thrift/SortFieldComparator.java index 5c93f5a..ee3043e 100644 --- a/blur-thrift/src/main/java/org/apache/blur/thrift/SortFieldComparator.java +++ b/blur-thrift/src/main/java/org/apache/blur/thrift/SortFieldComparator.java @@ -16,6 +16,7 @@ */ package org.apache.blur.thrift; +import java.nio.ByteBuffer; import java.util.Comparator; import org.apache.blur.thrift.generated.SortFieldResult; @@ -43,7 +44,7 @@ public class SortFieldComparator implements Comparator<SortFieldResult> { case INT_VALUE: // DOUBLE_VALUE return ((Integer) obj1).compareTo((Integer) obj2); case BINARY_VALUE: // BINARY_VALUE - return compare((byte[]) obj1, (byte[]) obj2); + return compare((ByteBuffer) obj1, (ByteBuffer) obj2); default: throw new RuntimeException("Unsupported type of [" + field + "]"); } @@ -51,7 +52,11 @@ public class SortFieldComparator implements Comparator<SortFieldResult> { return lastComparison; } - public int compare(byte[] b1, byte[] b2) { + public static int compare(ByteBuffer b1, ByteBuffer b2) { + return compareBytes(b1.array(), b1.arrayOffset(), b1.limit(), b2.array(), b2.arrayOffset(), b2.limit()); + } + + public static int compare(byte[] b1, byte[] b2) { return compareBytes(b1, 0, b1.length, b2, 0, b2.length); }
