Simple highlighting is now built in, will likely need more work.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/3dd4c2e7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/3dd4c2e7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/3dd4c2e7 Branch: refs/heads/master Commit: 3dd4c2e72fd20f7e50bbb23f384387318825fe57 Parents: ab468c7 Author: Aaron McCurry <[email protected]> Authored: Wed Jun 12 22:23:30 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Wed Jun 12 22:23:30 2013 -0400 ---------------------------------------------------------------------- .../org/apache/blur/manager/IndexManager.java | 61 +- .../blur/thrift/BlurControllerServer.java | 6 + .../org/apache/blur/thrift/BlurShardServer.java | 9 + .../org/apache/blur/utils/HighlightHelper.java | 97 ++- .../apache/blur/manager/IndexManagerTest.java | 202 ++++-- .../main/java/org/apache/blur/shell/Main.java | 25 +- .../org/apache/blur/shell/QueryCommand.java | 5 + .../thrift/generated/AnalyzerDefinition.java | 48 +- .../org/apache/blur/thrift/generated/Blur.java | 468 ++++++------ .../apache/blur/thrift/generated/BlurQuery.java | 36 +- .../blur/thrift/generated/BlurQueryStatus.java | 48 +- .../blur/thrift/generated/BlurResults.java | 148 ++-- .../blur/thrift/generated/ColumnDefinition.java | 48 +- .../generated/ColumnFamilyDefinition.java | 48 +- .../blur/thrift/generated/ColumnPreCache.java | 32 +- .../blur/thrift/generated/HighlightOptions.java | 252 +++++-- .../apache/blur/thrift/generated/Record.java | 196 +---- .../org/apache/blur/thrift/generated/Row.java | 36 +- .../blur/thrift/generated/RowMutation.java | 36 +- .../apache/blur/thrift/generated/Schema.java | 72 +- .../apache/blur/thrift/generated/Selector.java | 104 +-- .../blur/thrift/generated/TableDescriptor.java | 76 +- .../src/main/scripts/interface/Blur.thrift | 18 +- .../main/scripts/interface/gen-html/Blur.html | 9 +- .../thrift/generated/AnalyzerDefinition.java | 48 +- .../org/apache/blur/thrift/generated/Blur.java | 468 ++++++------ .../apache/blur/thrift/generated/BlurQuery.java | 36 +- .../blur/thrift/generated/BlurQueryStatus.java | 48 +- .../blur/thrift/generated/BlurResults.java | 148 ++-- .../blur/thrift/generated/ColumnDefinition.java | 48 +- .../generated/ColumnFamilyDefinition.java | 48 +- .../blur/thrift/generated/ColumnPreCache.java | 32 +- .../blur/thrift/generated/HighlightOptions.java | 252 +++++-- .../apache/blur/thrift/generated/Record.java | 196 +---- .../org/apache/blur/thrift/generated/Row.java | 36 +- .../blur/thrift/generated/RowMutation.java | 36 +- .../apache/blur/thrift/generated/Schema.java | 72 +- .../apache/blur/thrift/generated/Selector.java | 104 +-- .../blur/thrift/generated/TableDescriptor.java | 76 +- .../src/main/scripts/interface/gen-js/Blur.js | 416 +++++------ .../main/scripts/interface/gen-js/Blur_types.js | 725 +++++++++---------- .../scripts/interface/gen-perl/Blur/Blur.pm | 270 +++---- .../scripts/interface/gen-perl/Blur/Types.pm | 500 ++++++------- .../main/scripts/interface/gen-rb/blur_types.rb | 15 +- .../src/main/scripts/interface/genAndReplace.sh | 4 +- 45 files changed, 2873 insertions(+), 2785 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3dd4c2e7/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java index 8578f8a..9be2b92 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java +++ b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java @@ -69,6 +69,7 @@ import org.apache.blur.thrift.generated.Column; import org.apache.blur.thrift.generated.ExpertQuery; import org.apache.blur.thrift.generated.FetchResult; import org.apache.blur.thrift.generated.FetchRowResult; +import org.apache.blur.thrift.generated.HighlightOptions; import org.apache.blur.thrift.generated.Record; import org.apache.blur.thrift.generated.RecordMutation; import org.apache.blur.thrift.generated.RecordMutationType; @@ -85,6 +86,7 @@ import org.apache.blur.utils.BlurUtil; import org.apache.blur.utils.ForkJoin; import org.apache.blur.utils.ForkJoin.Merger; import org.apache.blur.utils.ForkJoin.ParallelCall; +import org.apache.blur.utils.HighlightHelper; import org.apache.blur.utils.ResetableDocumentStoredFieldVisitor; import org.apache.lucene.document.Document; import org.apache.lucene.index.AtomicReader; @@ -105,6 +107,7 @@ import org.apache.lucene.search.FilteredQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -213,10 +216,11 @@ public class IndexManager { searcher = index.getIndexReader(); usedCache = false; } + BlurAnalyzer analyzer = _indexServer.getAnalyzer(table); + + Query highlightQuery = getHighlightQuery(selector, table, analyzer); - Query highlightQuery = getHighlightQuery(selector); - - fetchRow(searcher.getIndexReader(), table, selector, fetchResult, highlightQuery); + fetchRow(searcher.getIndexReader(), table, selector, fetchResult, highlightQuery, analyzer); if (fetchResult.rowResult != null) { if (fetchResult.rowResult.row != null && fetchResult.rowResult.row.records != null) { _recordsMeter.mark(fetchResult.rowResult.row.records.size()); @@ -242,9 +246,24 @@ public class IndexManager { } } - private Query getHighlightQuery(Selector selector) { - // TODO Auto-generated method stub - return null; + private Query getHighlightQuery(Selector selector, String table, BlurAnalyzer analyzer) throws ParseException, + BlurException { + HighlightOptions highlightOptions = selector.getHighlightOptions(); + if (highlightOptions == null) { + return null; + } + SimpleQuery simpleQuery = highlightOptions.getSimpleQuery(); + if (simpleQuery == null) { + return null; + } + + TableContext context = getTableContext(table); + Filter preFilter = QueryParserUtil.parseFilter(table, simpleQuery.preSuperFilter, false, analyzer, _filterCache, + context); + Filter postFilter = QueryParserUtil.parseFilter(table, simpleQuery.postSuperFilter, true, analyzer, _filterCache, + context); + return QueryParserUtil.parseQuery(simpleQuery.queryStr, simpleQuery.superQueryOn, analyzer, postFilter, preFilter, + getScoreType(simpleQuery.type), context); } private void populateSelector(String table, Selector selector) throws IOException, BlurException { @@ -445,8 +464,13 @@ public class IndexManager { return _statusManager.queryStatusIdList(table); } - public static void fetchRow(IndexReader reader, String table, Selector selector, FetchResult fetchResult, Query highlightQuery) - throws CorruptIndexException, IOException { + public static void fetchRow(IndexReader reader, String table, Selector selector, FetchResult fetchResult, + Query highlightQuery) throws CorruptIndexException, IOException { + fetchRow(reader, table, selector, fetchResult, highlightQuery, null); + } + + public static void fetchRow(IndexReader reader, String table, Selector selector, FetchResult fetchResult, + Query highlightQuery, BlurAnalyzer analyzer) throws CorruptIndexException, IOException { fetchResult.table = table; String locationId = selector.locationId; int lastSlash = locationId.lastIndexOf('/'); @@ -475,6 +499,16 @@ public class IndexManager { fetchResult.deleted = false; reader.document(docId, fieldVisitor); Document document = fieldVisitor.getDocument(); + if (highlightQuery != null && analyzer != null) { + HighlightOptions highlightOptions = selector.getHighlightOptions(); + String preTag = highlightOptions.getPreTag(); + String postTag = highlightOptions.getPostTag(); + try { + document = HighlightHelper.highlight(docId, document, highlightQuery, analyzer, reader, preTag, postTag); + } catch (InvalidTokenOffsetsException e) { + LOG.error("Unknown error while tring to highlight", e); + } + } fieldVisitor.reset(); fetchResult.recordResult = getRecord(document); return; @@ -494,7 +528,16 @@ public class IndexManager { fetchResult.rowResult = new FetchRowResult(); fetchResult.rowResult.row = new Row(rowId, null, recordCount); } else { - List<Document> docs = BlurUtil.fetchDocuments(reader, term, fieldVisitor, selector); + List<Document> docs; + if (highlightQuery != null && analyzer != null) { + HighlightOptions highlightOptions = selector.getHighlightOptions(); + String preTag = highlightOptions.getPreTag(); + String postTag = highlightOptions.getPostTag(); + docs = HighlightHelper.highlightDocuments(reader, term, fieldVisitor, selector, highlightQuery, analyzer, + preTag, postTag); + } else { + docs = BlurUtil.fetchDocuments(reader, term, fieldVisitor, selector); + } fetchResult.rowResult = new FetchRowResult(getRow(docs)); } return; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3dd4c2e7/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java index 20837b3..da0c6fd 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java @@ -64,6 +64,7 @@ import org.apache.blur.thrift.generated.BlurQueryStatus; import org.apache.blur.thrift.generated.BlurResult; import org.apache.blur.thrift.generated.BlurResults; import org.apache.blur.thrift.generated.FetchResult; +import org.apache.blur.thrift.generated.HighlightOptions; import org.apache.blur.thrift.generated.RowMutation; import org.apache.blur.thrift.generated.Schema; import org.apache.blur.thrift.generated.Selector; @@ -320,6 +321,11 @@ public class BlurControllerServer extends TableAdmin implements Iface { if (!blurQuery.simpleQuery.superQueryOn) { selector.setRecordOnly(true); } + } else { + HighlightOptions highlightOptions = selector.getHighlightOptions(); + if (highlightOptions != null && highlightOptions.getSimpleQuery() == null) { + highlightOptions.setSimpleQuery(blurQuery.getSimpleQuery()); + } } blurQuery.setSelector(null); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3dd4c2e7/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java index 609b7d0..db2a817 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java @@ -47,6 +47,7 @@ import org.apache.blur.thrift.generated.BlurQuery; import org.apache.blur.thrift.generated.BlurQueryStatus; import org.apache.blur.thrift.generated.BlurResults; import org.apache.blur.thrift.generated.FetchResult; +import org.apache.blur.thrift.generated.HighlightOptions; import org.apache.blur.thrift.generated.RowMutation; import org.apache.blur.thrift.generated.Schema; import org.apache.blur.thrift.generated.Selector; @@ -92,6 +93,14 @@ public class BlurShardServer extends TableAdmin implements Iface { _queryChecker.checkQuery(blurQuery); try { BlurQuery original = new BlurQuery(blurQuery); + Selector selector = original.getSelector(); + if (selector != null) { + HighlightOptions highlightOptions = selector.getHighlightOptions(); + if (highlightOptions != null && highlightOptions.getSimpleQuery() == null) { + highlightOptions.setSimpleQuery(blurQuery.getSimpleQuery()); + } + } + if (blurQuery.useCacheIfPresent) { LOG.debug("Using cache for query [{0}] on table [{1}].", blurQuery, table); QueryCacheKey key = QueryCache.getNormalizedBlurQueryKey(table, blurQuery); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3dd4c2e7/blur-core/src/main/java/org/apache/blur/utils/HighlightHelper.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/utils/HighlightHelper.java b/blur-core/src/main/java/org/apache/blur/utils/HighlightHelper.java index 64214f4..af33240 100644 --- a/blur-core/src/main/java/org/apache/blur/utils/HighlightHelper.java +++ b/blur-core/src/main/java/org/apache/blur/utils/HighlightHelper.java @@ -1,20 +1,30 @@ package org.apache.blur.utils; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; +import java.util.List; import org.apache.blur.analysis.BlurAnalyzer; +import org.apache.blur.log.Log; +import org.apache.blur.log.LogFactory; +import org.apache.blur.lucene.search.SuperQuery; +import org.apache.blur.thrift.generated.Selector; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.NumericRangeQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.highlight.Highlighter; import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; import org.apache.lucene.search.highlight.QueryScorer; @@ -23,42 +33,90 @@ import org.apache.lucene.search.highlight.TextFragment; import org.apache.lucene.search.highlight.TokenSources; public class HighlightHelper { - - private static final Collection<String> FIELDS_NOT_TO_HIGHLIGHT = new HashSet<String>() {{ + + private static final Log LOG = LogFactory.getLog(HighlightHelper.class); + + private static final Collection<String> FIELDS_NOT_TO_HIGHLIGHT = new HashSet<String>() { + { add(BlurConstants.SUPER); add(BlurConstants.ROW_ID); add(BlurConstants.RECORD_ID); add(BlurConstants.PRIME_DOC); add(BlurConstants.FAMILY); - }}; + } + }; + + public static List<Document> highlightDocuments(IndexReader reader, Term term, + ResetableDocumentStoredFieldVisitor fieldSelector, Selector selector, Query highlightQuery, + BlurAnalyzer analyzer, String preTag, String postTag) throws IOException { + IndexSearcher indexSearcher = new IndexSearcher(reader); + int docFreq = reader.docFreq(term); + BooleanQuery booleanQueryForFamily = null; + BooleanQuery booleanQuery = null; + if (selector.getColumnFamiliesToFetchSize() > 0) { + booleanQueryForFamily = new BooleanQuery(); + for (String familyName : selector.getColumnFamiliesToFetch()) { + booleanQueryForFamily + .add(new TermQuery(new Term(BlurConstants.FAMILY, familyName)), BooleanClause.Occur.SHOULD); + } + booleanQuery = new BooleanQuery(); + booleanQuery.add(new TermQuery(term), BooleanClause.Occur.MUST); + booleanQuery.add(booleanQueryForFamily, BooleanClause.Occur.MUST); + } + Query query = booleanQuery == null ? new TermQuery(term) : booleanQuery; + TopDocs topDocs = indexSearcher.search(query, docFreq); + int totalHits = topDocs.totalHits; + List<Document> docs = new ArrayList<Document>(); + + int start = selector.getStartRecord(); + int end = selector.getMaxRecordsToFetch() + start; + + for (int i = start; i < end; i++) { + if (i >= totalHits) { + break; + } + int doc = topDocs.scoreDocs[i].doc; + indexSearcher.doc(doc, fieldSelector); + Document document = fieldSelector.getDocument(); + try { + document = highlight(doc, document, highlightQuery, analyzer, reader, preTag, postTag); + } catch (InvalidTokenOffsetsException e) { + LOG.error("Unknown error while tring to highlight", e); + } + docs.add(document); + fieldSelector.reset(); + } + return docs; + } /** - * NOTE: This method will not preserve the correct field types. + * NOTE: This method will not preserve the correct field types. + * + * @param preTag + * @param postTag */ - public static Document highlight(int docId, Document document, Query query, BlurAnalyzer analyzer, IndexReader reader) - throws IOException, InvalidTokenOffsetsException { + public static Document highlight(int docId, Document document, Query query, BlurAnalyzer analyzer, + IndexReader reader, String preTag, String postTag) throws IOException, InvalidTokenOffsetsException { - String preTag = "{{"; - String postTag = "}}"; + Query fixedQuery = fixSuperQuery(query); SimpleHTMLFormatter htmlFormatter = new SimpleHTMLFormatter(preTag, postTag); - System.out.println(docId); - System.out.println(document); Document result = new Document(); for (IndexableField f : document) { String name = f.name(); if (FIELDS_NOT_TO_HIGHLIGHT.contains(name)) { + result.add(f); continue; } String text = f.stringValue(); Number numericValue = f.numericValue(); if (numericValue != null) { - if (shouldNumberBeHighlighted(name, numericValue, query)) { + if (shouldNumberBeHighlighted(name, numericValue, fixedQuery)) { String numberHighlight = preTag + text + postTag; result.add(new StringField(name, numberHighlight, Store.YES)); } } else { - Highlighter highlighter = new Highlighter(htmlFormatter, new QueryScorer(query, name)); + Highlighter highlighter = new Highlighter(htmlFormatter, new QueryScorer(fixedQuery, name)); TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, docId, name, analyzer); TextFragment[] frag = highlighter.getBestTextFragments(tokenStream, text, false, 10); for (int j = 0; j < frag.length; j++) { @@ -71,6 +129,21 @@ public class HighlightHelper { return result; } + private static Query fixSuperQuery(Query query) { + if (query instanceof BooleanQuery) { + BooleanQuery bq = (BooleanQuery) query; + for (BooleanClause booleanClause : bq) { + booleanClause.setQuery(fixSuperQuery(booleanClause.getQuery())); + } + return bq; + } else if (query instanceof SuperQuery) { + SuperQuery sq = (SuperQuery) query; + return sq.getQuery(); + } else { + return query; + } + } + public static boolean shouldNumberBeHighlighted(String name, Number numericValue, Query query) { if (query instanceof BooleanQuery) { BooleanQuery booleanQuery = (BooleanQuery) query; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3dd4c2e7/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java index a7461d4..a4aa7be 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java @@ -57,6 +57,7 @@ import org.apache.blur.thrift.generated.Column; import org.apache.blur.thrift.generated.Facet; import org.apache.blur.thrift.generated.FetchRecordResult; import org.apache.blur.thrift.generated.FetchResult; +import org.apache.blur.thrift.generated.HighlightOptions; import org.apache.blur.thrift.generated.Record; import org.apache.blur.thrift.generated.RecordMutation; import org.apache.blur.thrift.generated.Row; @@ -72,9 +73,8 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; - public class IndexManagerTest { - + private static final File TMPDIR = new File("./target/tmp"); private static final String SHARD_NAME = BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, 0); @@ -91,7 +91,7 @@ public class IndexManagerTest { TableContext.clear(); base = new File(TMPDIR, "blur-index-manager-test"); rm(base); - + File file = new File(base, TABLE); file.mkdirs(); @@ -108,97 +108,97 @@ public class IndexManagerTest { indexManager.setIndexServer(server); indexManager.setThreadCount(1); indexManager.setClusterStatus(new ClusterStatus() { - + @Override public void removeTable(String cluster, String table, boolean deleteIndexFiles) { throw new RuntimeException("Not impl"); } - + @Override public boolean isReadOnly(boolean useCache, String cluster, String table) { throw new RuntimeException("Not impl"); } - + @Override public boolean isOpen() { throw new RuntimeException("Not impl"); } - + @Override public boolean isInSafeMode(boolean useCache, String cluster) { throw new RuntimeException("Not impl"); } - + @Override public boolean isEnabled(boolean useCache, String cluster, String table) { throw new RuntimeException("Not impl"); } - + @Override public boolean isBlockCacheEnabled(String cluster, String table) { throw new RuntimeException("Not impl"); } - + @Override public List<String> getTableList(boolean useCache, String cluster) { throw new RuntimeException("Not impl"); } - + @Override public TableDescriptor getTableDescriptor(boolean useCache, String cluster, String table) { return tableDescriptor; } - + @Override public List<String> getShardServerList(String cluster) { throw new RuntimeException("Not impl"); } - + @Override public int getShardCount(boolean useCache, String cluster, String table) { throw new RuntimeException("Not impl"); } - + @Override public List<String> getOnlineShardServers(boolean useCache, String cluster) { throw new RuntimeException("Not impl"); } - + @Override public List<String> getControllerServerList() { throw new RuntimeException("Not impl"); } - + @Override public List<String> getClusterList(boolean useCache) { throw new RuntimeException("Not impl"); } - + @Override public String getCluster(boolean useCache, String table) { return BlurConstants.BLUR_CLUSTER; } - + @Override public Set<String> getBlockCacheFileTypes(String cluster, String table) { throw new RuntimeException("Not impl"); } - + @Override public boolean exists(boolean useCache, String cluster, String table) { throw new RuntimeException("Not impl"); } - + @Override public void enableTable(String cluster, String table) { throw new RuntimeException("Not impl"); } - + @Override public void disableTable(String cluster, String table) { throw new RuntimeException("Not impl"); } - + @Override public void createTable(TableDescriptor tableDescriptor) { throw new RuntimeException("Not impl"); @@ -225,25 +225,43 @@ public class IndexManagerTest { } private void setupData() throws BlurException, IOException { - RowMutation mutation1 = newRowMutation(TABLE, "row-1", - newRecordMutation(FAMILY, "record-1", newColumn("testcol1", "value1"), newColumn("testcol2", "value2"), newColumn("testcol3", "value3"))); - RowMutation mutation2 = newRowMutation(TABLE, "row-2", - newRecordMutation(FAMILY, "record-2", newColumn("testcol1", "value4"), newColumn("testcol2", "value5"), newColumn("testcol3", "value6")), - newRecordMutation(FAMILY, "record-2B", newColumn("testcol2", "value234123"), newColumn("testcol3", "value234123"))); - RowMutation mutation3 = newRowMutation(TABLE, "row-3", - newRecordMutation(FAMILY, "record-3", newColumn("testcol1", "value7"), newColumn("testcol2", "value8"), newColumn("testcol3", "value9"))); - RowMutation mutation4 = newRowMutation(TABLE, "row-4", - newRecordMutation(FAMILY, "record-4", newColumn("testcol1", "value1"), newColumn("testcol2", "value5"), newColumn("testcol3", "value9")), - newRecordMutation(FAMILY, "record-4B", newColumn("testcol2", "value234123"), newColumn("testcol3", "value234123"))); - RowMutation mutation5 = newRowMutation(TABLE,"row-5", - newRecordMutation(FAMILY, "record-5A", newColumn("testcol1", "value13"), newColumn("testcol2", "value14"), newColumn("testcol3", "value15")), - newRecordMutation(FAMILY, "record-5B", newColumn("testcol1", "value16"), newColumn("testcol2", "value17"), newColumn("testcol3", "value18"), newColumn("testcol3", "value19"))); - RowMutation mutation6 = newRowMutation(TABLE, "row-6", - newRecordMutation(FAMILY, "record-6A", newColumn("testcol12", "value110"), newColumn("testcol13", "value102")), - newRecordMutation(FAMILY, "record-6B", newColumn("testcol12", "value101"), newColumn("testcol13", "value104")), + RowMutation mutation1 = newRowMutation( + TABLE, + "row-1", + newRecordMutation(FAMILY, "record-1", newColumn("testcol1", "value1"), newColumn("testcol2", "value2"), + newColumn("testcol3", "value3"))); + RowMutation mutation2 = newRowMutation( + TABLE, + "row-2", + newRecordMutation(FAMILY, "record-2", newColumn("testcol1", "value4"), newColumn("testcol2", "value5"), + newColumn("testcol3", "value6")), + newRecordMutation(FAMILY, "record-2B", newColumn("testcol2", "value234123"), + newColumn("testcol3", "value234123"))); + RowMutation mutation3 = newRowMutation( + TABLE, + "row-3", + newRecordMutation(FAMILY, "record-3", newColumn("testcol1", "value7"), newColumn("testcol2", "value8"), + newColumn("testcol3", "value9"))); + RowMutation mutation4 = newRowMutation( + TABLE, + "row-4", + newRecordMutation(FAMILY, "record-4", newColumn("testcol1", "value1"), newColumn("testcol2", "value5"), + newColumn("testcol3", "value9")), + newRecordMutation(FAMILY, "record-4B", newColumn("testcol2", "value234123"), + newColumn("testcol3", "value234123"))); + RowMutation mutation5 = newRowMutation( + TABLE, + "row-5", + newRecordMutation(FAMILY, "record-5A", newColumn("testcol1", "value13"), newColumn("testcol2", "value14"), + newColumn("testcol3", "value15")), + newRecordMutation(FAMILY, "record-5B", newColumn("testcol1", "value16"), newColumn("testcol2", "value17"), + newColumn("testcol3", "value18"), newColumn("testcol3", "value19"))); + RowMutation mutation6 = newRowMutation(TABLE, "row-6", + newRecordMutation(FAMILY, "record-6A", newColumn("testcol12", "value110"), newColumn("testcol13", "value102")), + newRecordMutation(FAMILY, "record-6B", newColumn("testcol12", "value101"), newColumn("testcol13", "value104")), newRecordMutation(FAMILY2, "record-6C", newColumn("testcol18", "value501"))); - RowMutation mutation7 = newRowMutation(TABLE, "row-7", - newRecordMutation(FAMILY, "record-7A", newColumn("testcol12", "value101"), newColumn("testcol13", "value102")), + RowMutation mutation7 = newRowMutation(TABLE, "row-7", + newRecordMutation(FAMILY, "record-7A", newColumn("testcol12", "value101"), newColumn("testcol13", "value102")), newRecordMutation(FAMILY2, "record-7B", newColumn("testcol18", "value501"))); mutation7.waitToBeVisible = true; indexManager.mutate(mutation1); @@ -256,11 +274,50 @@ public class IndexManagerTest { } @Test + public void testFetchRowByRowIdHighlighting() throws Exception { + Selector selector = new Selector().setRowId("row-6"); + HighlightOptions highlightOptions = new HighlightOptions(); + SimpleQuery simpleQuery = new SimpleQuery(); + simpleQuery.setQueryStr(FAMILY + ".testcol12:value101"); + highlightOptions.setSimpleQuery(simpleQuery); + selector.setHighlightOptions(highlightOptions); + FetchResult fetchResult = new FetchResult(); + indexManager.fetchRow(TABLE, selector, fetchResult); + + assertNotNull(fetchResult.rowResult.row); + Row row = newRow("row-6", newRecord(FAMILY, "record-6B", newColumn("testcol12", "<<<value101>>>"))); + row.recordCount = 3; + assertEquals(row, fetchResult.rowResult.row); + } + + @Test + public void testFetchRecordByLocationIdHighlighting() throws Exception { + Selector selector = new Selector().setLocationId(SHARD_NAME + "/0").setRecordOnly(true); + HighlightOptions highlightOptions = new HighlightOptions(); + SimpleQuery simpleQuery = new SimpleQuery(); + simpleQuery.setQueryStr(FAMILY + ".testcol1:value1"); + simpleQuery.setSuperQueryOn(false); + highlightOptions.setSimpleQuery(simpleQuery); + selector.setHighlightOptions(highlightOptions); + FetchResult fetchResult = new FetchResult(); + indexManager.fetchRow(TABLE, selector, fetchResult); + assertNull(fetchResult.rowResult); + assertNotNull(fetchResult.recordResult.record); + + assertEquals("row-1", fetchResult.recordResult.rowid); + assertEquals("record-1", fetchResult.recordResult.record.recordId); + assertEquals(FAMILY, fetchResult.recordResult.record.family); + + Record record = newRecord(FAMILY, "record-1", newColumn("testcol1", "<<<value1>>>")); + assertEquals(record, fetchResult.recordResult.record); + } + + @Test public void testQueryWithJoinAll() throws Exception { BlurQuery blurQuery = new BlurQuery(); blurQuery.simpleQuery = new SimpleQuery(); blurQuery.simpleQuery.queryStr = "+super:<+test-family.testcol12:value101 +test-family.testcol13:value102> +super:<test-family2.testcol18:value501>"; - + blurQuery.simpleQuery.superQueryOn = true; blurQuery.simpleQuery.type = ScoreType.SUPER; blurQuery.fetch = 10; @@ -336,7 +393,8 @@ public class IndexManagerTest { blurQuery.minimumNumberOfResults = Long.MAX_VALUE; blurQuery.maxQueryTime = Long.MAX_VALUE; blurQuery.uuid = 1; - blurQuery.facets = Arrays.asList(new Facet("test-family.testcol1:value*", Long.MAX_VALUE), new Facet("test-family.testcol1:value-nohit", Long.MAX_VALUE)); + blurQuery.facets = Arrays.asList(new Facet("test-family.testcol1:value*", Long.MAX_VALUE), new Facet( + "test-family.testcol1:value-nohit", Long.MAX_VALUE)); AtomicLongArray facetedCounts = new AtomicLongArray(2); BlurResultIterable iterable = indexManager.query(TABLE, blurQuery, facetedCounts); @@ -363,7 +421,10 @@ public class IndexManagerTest { FetchResult fetchResult = new FetchResult(); indexManager.fetchRow(TABLE, selector, fetchResult); assertNotNull(fetchResult.rowResult.row); - Row row = newRow("row-1", newRecord(FAMILY, "record-1", newColumn("testcol1", "value1"), newColumn("testcol2", "value2"), newColumn("testcol3", "value3"))); + Row row = newRow( + "row-1", + newRecord(FAMILY, "record-1", newColumn("testcol1", "value1"), newColumn("testcol2", "value2"), + newColumn("testcol3", "value3"))); row.recordCount = 1; assertEquals(row, fetchResult.rowResult.row); } @@ -391,7 +452,8 @@ public class IndexManagerTest { assertEquals("record-1", fetchResult.recordResult.record.recordId); assertEquals(FAMILY, fetchResult.recordResult.record.family); - Record record = newRecord(FAMILY, "record-1", newColumn("testcol1", "value1"), newColumn("testcol2", "value2"), newColumn("testcol3", "value3")); + Record record = newRecord(FAMILY, "record-1", newColumn("testcol1", "value1"), newColumn("testcol2", "value2"), + newColumn("testcol3", "value3")); assertEquals(record, fetchResult.recordResult.record); } @@ -401,11 +463,14 @@ public class IndexManagerTest { FetchResult fetchResult = new FetchResult(); indexManager.fetchRow(TABLE, selector, fetchResult); assertNotNull(fetchResult.rowResult.row); - Row row = newRow("row-1", newRecord(FAMILY, "record-1", newColumn("testcol1", "value1"), newColumn("testcol2", "value2"), newColumn("testcol3", "value3"))); + Row row = newRow( + "row-1", + newRecord(FAMILY, "record-1", newColumn("testcol1", "value1"), newColumn("testcol2", "value2"), + newColumn("testcol3", "value3"))); row.recordCount = 1; assertEquals(row, fetchResult.rowResult.row); } - + @Test public void testFetchRowByRowIdPaging() throws Exception { Selector selector = new Selector().setRowId("row-6").setStartRecord(0).setMaxRecordsToFetch(1); @@ -413,28 +478,30 @@ public class IndexManagerTest { indexManager.fetchRow(TABLE, selector, fetchResult); assertNotNull(fetchResult.rowResult.row); - Row row1 = newRow("row-6", newRecord(FAMILY, "record-6A", newColumn("testcol12", "value110"), newColumn("testcol13", "value102"))); + Row row1 = newRow("row-6", + newRecord(FAMILY, "record-6A", newColumn("testcol12", "value110"), newColumn("testcol13", "value102"))); row1.recordCount = 1; assertEquals(row1, fetchResult.rowResult.row); - + selector = new Selector().setRowId("row-6").setStartRecord(1).setMaxRecordsToFetch(1); fetchResult = new FetchResult(); indexManager.fetchRow(TABLE, selector, fetchResult); assertNotNull(fetchResult.rowResult.row); - - Row row2 = newRow("row-6", newRecord(FAMILY, "record-6B", newColumn("testcol12", "value101"), newColumn("testcol13", "value104"))); + + Row row2 = newRow("row-6", + newRecord(FAMILY, "record-6B", newColumn("testcol12", "value101"), newColumn("testcol13", "value104"))); row2.recordCount = 1; assertEquals(row2, fetchResult.rowResult.row); - + selector = new Selector().setRowId("row-6").setStartRecord(2).setMaxRecordsToFetch(1); fetchResult = new FetchResult(); indexManager.fetchRow(TABLE, selector, fetchResult); assertNotNull(fetchResult.rowResult.row); - + Row row3 = newRow("row-6", newRecord(FAMILY2, "record-6C", newColumn("testcol18", "value501"))); row3.recordCount = 1; assertEquals(row3, fetchResult.rowResult.row); - + selector = new Selector().setRowId("row-6").setStartRecord(3).setMaxRecordsToFetch(1); fetchResult = new FetchResult(); indexManager.fetchRow(TABLE, selector, fetchResult); @@ -486,7 +553,8 @@ public class IndexManagerTest { assertEquals("record-1", recordResult.record.recordId); assertEquals("row-1", recordResult.rowid); - Record record = newRecord(FAMILY, "record-1", newColumn("testcol1", "value1"), newColumn("testcol2", "value2"), newColumn("testcol3", "value3")); + Record record = newRecord(FAMILY, "record-1", newColumn("testcol1", "value1"), newColumn("testcol2", "value2"), + newColumn("testcol3", "value3")); assertEquals(record, recordResult.record); } @@ -503,7 +571,8 @@ public class IndexManagerTest { assertEquals(TABLE, schema.table); Map<String, Set<String>> columnFamilies = schema.columnFamilies; assertEquals(new TreeSet<String>(Arrays.asList(FAMILY, FAMILY2)), new TreeSet<String>(columnFamilies.keySet())); - assertEquals(new TreeSet<String>(Arrays.asList("testcol1", "testcol2", "testcol3", "testcol12", "testcol13")), new TreeSet<String>(columnFamilies.get(FAMILY))); + assertEquals(new TreeSet<String>(Arrays.asList("testcol1", "testcol2", "testcol3", "testcol12", "testcol13")), + new TreeSet<String>(columnFamilies.get(FAMILY))); assertEquals(new TreeSet<String>(Arrays.asList("testcol18")), new TreeSet<String>(columnFamilies.get(FAMILY2))); } @@ -654,7 +723,8 @@ public class IndexManagerTest { blurQuery.minimumNumberOfResults = Long.MAX_VALUE; blurQuery.maxQueryTime = Long.MAX_VALUE; blurQuery.uuid = 1; - blurQuery.facets = Arrays.asList(new Facet("test-family.testcol1:value1", Long.MAX_VALUE), new Facet("test-family.testcol1:value-nohit", Long.MAX_VALUE)); + blurQuery.facets = Arrays.asList(new Facet("test-family.testcol1:value1", Long.MAX_VALUE), new Facet( + "test-family.testcol1:value-nohit", Long.MAX_VALUE)); AtomicLongArray facetedCounts = new AtomicLongArray(2); BlurResultIterable iterable = indexManager.query(TABLE, blurQuery, facetedCounts); @@ -683,8 +753,11 @@ public class IndexManagerTest { @Test public void testMutationReplaceRow() throws Exception { - RowMutation mutation = newRowMutation(TABLE, "row-4", - newRecordMutation(FAMILY, "record-4", newColumn("testcol1", "value2"), newColumn("testcol2", "value3"), newColumn("testcol3", "value4"))); + RowMutation mutation = newRowMutation( + TABLE, + "row-4", + newRecordMutation(FAMILY, "record-4", newColumn("testcol1", "value2"), newColumn("testcol2", "value3"), + newColumn("testcol3", "value4"))); mutation.waitToBeVisible = true; indexManager.mutate(mutation); @@ -692,7 +765,10 @@ public class IndexManagerTest { FetchResult fetchResult = new FetchResult(); indexManager.fetchRow(TABLE, selector, fetchResult); assertNotNull(fetchResult.rowResult.row); - Row row = newRow("row-4", newRecord(FAMILY, "record-4", newColumn("testcol1", "value2"), newColumn("testcol2", "value3"), newColumn("testcol3", "value4"))); + Row row = newRow( + "row-4", + newRecord(FAMILY, "record-4", newColumn("testcol1", "value2"), newColumn("testcol2", "value3"), + newColumn("testcol3", "value4"))); row.recordCount = 1; assertEquals(row, fetchResult.rowResult.row); } @@ -713,7 +789,10 @@ public class IndexManagerTest { indexManager.fetchRow(TABLE, selector, fetchResult); Row r = fetchResult.rowResult.row; assertNotNull("new row should exist", r); - Row row = newRow("row-6", newRecord(FAMILY, "record-6", newColumn("testcol1", "value20"), newColumn("testcol2", "value21"), newColumn("testcol3", "value22"))); + Row row = newRow( + "row-6", + newRecord(FAMILY, "record-6", newColumn("testcol1", "value20"), newColumn("testcol2", "value21"), + newColumn("testcol3", "value22"))); row.recordCount = 1; assertEquals("row should match", row, r); } @@ -1020,7 +1099,8 @@ public class IndexManagerTest { updateAndFetchRecord("row-6", rec, rm); } - private Record updateAndFetchRecord(String rowId, String recordId, RecordMutation... recordMutations) throws Exception { + private Record updateAndFetchRecord(String rowId, String recordId, RecordMutation... recordMutations) + throws Exception { RowMutation rowMutation = newRowMutation(UPDATE_ROW, TABLE, rowId, recordMutations); rowMutation.waitToBeVisible = true; indexManager.mutate(rowMutation); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3dd4c2e7/blur-shell/src/main/java/org/apache/blur/shell/Main.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/Main.java b/blur-shell/src/main/java/org/apache/blur/shell/Main.java index 10e9e47..8598f10 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/Main.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/Main.java @@ -51,6 +51,8 @@ public class Main { static boolean debug = false; /** is timing enabled - off by default */ static boolean timed = false; + /** is highlight enabled - off by default */ + static boolean highlight = false; private static Map<String, Command> commands; @@ -97,6 +99,26 @@ public class Main { } } + + private static class HighlightCommand extends Command { + + @Override + public void doit(PrintWriter out, Blur.Iface client, String[] args) throws CommandException, TException, + BlurException { + if (highlight == true) { + highlight = false; + } else { + highlight = true; + } + out.println("highlight of query command is now " + (highlight ? "on" : "off")); + } + + @Override + public String help() { + return "toggle highlight of query output on/off"; + } + + } private static class HelpCommand extends Command { @Override @@ -114,7 +136,7 @@ public class Main { out.println(); out.println(" - Data commands - "); - String[] dataCommands = { "query", "get", "mutate", "delete" }; + String[] dataCommands = { "query", "get", "mutate", "delete", "highlight" }; printCommandAndHelp(out, cmds, dataCommands, bufferLength); out.println(); @@ -193,6 +215,7 @@ public class Main { builder.put("help", new HelpCommand()); builder.put("debug", new DebugCommand()); builder.put("timed", new TimedCommand()); + builder.put("highlight", new HighlightCommand()); builder.put("quit", new QuitCommand()); builder.put("list", new ListTablesCommand()); builder.put("create", new CreateTableCommand()); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3dd4c2e7/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand.java index 28fe909..9637e85 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand.java @@ -31,6 +31,7 @@ import org.apache.blur.thrift.generated.BlurResult; import org.apache.blur.thrift.generated.BlurResults; import org.apache.blur.thrift.generated.FetchResult; import org.apache.blur.thrift.generated.FetchRowResult; +import org.apache.blur.thrift.generated.HighlightOptions; import org.apache.blur.thrift.generated.Row; import org.apache.blur.thrift.generated.Selector; import org.apache.blur.thrift.generated.SimpleQuery; @@ -53,6 +54,10 @@ public class QueryCommand extends Command { simpleQuery.setQueryStr(query); blurQuery.setSimpleQuery(simpleQuery); blurQuery.setSelector(new Selector()); + + if (Main.highlight) { + blurQuery.getSelector().setHighlightOptions(new HighlightOptions()); + } if (Main.debug) { out.println(blurQuery); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3dd4c2e7/blur-thrift/src/main/java/org/apache/blur/thrift/generated/AnalyzerDefinition.java ---------------------------------------------------------------------- diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/generated/AnalyzerDefinition.java b/blur-thrift/src/main/java/org/apache/blur/thrift/generated/AnalyzerDefinition.java index b720b6b..37adc0b 100644 --- a/blur-thrift/src/main/java/org/apache/blur/thrift/generated/AnalyzerDefinition.java +++ b/blur-thrift/src/main/java/org/apache/blur/thrift/generated/AnalyzerDefinition.java @@ -579,16 +579,16 @@ public class AnalyzerDefinition implements org.apache.blur.thirdparty.thrift_0_9 case 3: // COLUMN_FAMILY_DEFINITIONS if (schemeField.type == org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType.MAP) { { - org.apache.blur.thirdparty.thrift_0_9_0.protocol.TMap _map148 = iprot.readMapBegin(); - struct.columnFamilyDefinitions = new HashMap<String,ColumnFamilyDefinition>(2*_map148.size); - for (int _i149 = 0; _i149 < _map148.size; ++_i149) + org.apache.blur.thirdparty.thrift_0_9_0.protocol.TMap _map140 = iprot.readMapBegin(); + struct.columnFamilyDefinitions = new HashMap<String,ColumnFamilyDefinition>(2*_map140.size); + for (int _i141 = 0; _i141 < _map140.size; ++_i141) { - String _key150; // required - ColumnFamilyDefinition _val151; // optional - _key150 = iprot.readString(); - _val151 = new ColumnFamilyDefinition(); - _val151.read(iprot); - struct.columnFamilyDefinitions.put(_key150, _val151); + String _key142; // required + ColumnFamilyDefinition _val143; // optional + _key142 = iprot.readString(); + _val143 = new ColumnFamilyDefinition(); + _val143.read(iprot); + struct.columnFamilyDefinitions.put(_key142, _val143); } iprot.readMapEnd(); } @@ -626,10 +626,10 @@ public class AnalyzerDefinition implements org.apache.blur.thirdparty.thrift_0_9 oprot.writeFieldBegin(COLUMN_FAMILY_DEFINITIONS_FIELD_DESC); { oprot.writeMapBegin(new org.apache.blur.thirdparty.thrift_0_9_0.protocol.TMap(org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType.STRING, org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType.STRUCT, struct.columnFamilyDefinitions.size())); - for (Map.Entry<String, ColumnFamilyDefinition> _iter152 : struct.columnFamilyDefinitions.entrySet()) + for (Map.Entry<String, ColumnFamilyDefinition> _iter144 : struct.columnFamilyDefinitions.entrySet()) { - oprot.writeString(_iter152.getKey()); - _iter152.getValue().write(oprot); + oprot.writeString(_iter144.getKey()); + _iter144.getValue().write(oprot); } oprot.writeMapEnd(); } @@ -672,10 +672,10 @@ public class AnalyzerDefinition implements org.apache.blur.thirdparty.thrift_0_9 if (struct.isSetColumnFamilyDefinitions()) { { oprot.writeI32(struct.columnFamilyDefinitions.size()); - for (Map.Entry<String, ColumnFamilyDefinition> _iter153 : struct.columnFamilyDefinitions.entrySet()) + for (Map.Entry<String, ColumnFamilyDefinition> _iter145 : struct.columnFamilyDefinitions.entrySet()) { - oprot.writeString(_iter153.getKey()); - _iter153.getValue().write(oprot); + oprot.writeString(_iter145.getKey()); + _iter145.getValue().write(oprot); } } } @@ -696,16 +696,16 @@ public class AnalyzerDefinition implements org.apache.blur.thirdparty.thrift_0_9 } if (incoming.get(2)) { { - org.apache.blur.thirdparty.thrift_0_9_0.protocol.TMap _map154 = new org.apache.blur.thirdparty.thrift_0_9_0.protocol.TMap(org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType.STRING, org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType.STRUCT, iprot.readI32()); - struct.columnFamilyDefinitions = new HashMap<String,ColumnFamilyDefinition>(2*_map154.size); - for (int _i155 = 0; _i155 < _map154.size; ++_i155) + org.apache.blur.thirdparty.thrift_0_9_0.protocol.TMap _map146 = new org.apache.blur.thirdparty.thrift_0_9_0.protocol.TMap(org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType.STRING, org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType.STRUCT, iprot.readI32()); + struct.columnFamilyDefinitions = new HashMap<String,ColumnFamilyDefinition>(2*_map146.size); + for (int _i147 = 0; _i147 < _map146.size; ++_i147) { - String _key156; // required - ColumnFamilyDefinition _val157; // optional - _key156 = iprot.readString(); - _val157 = new ColumnFamilyDefinition(); - _val157.read(iprot); - struct.columnFamilyDefinitions.put(_key156, _val157); + String _key148; // required + ColumnFamilyDefinition _val149; // optional + _key148 = iprot.readString(); + _val149 = new ColumnFamilyDefinition(); + _val149.read(iprot); + struct.columnFamilyDefinitions.put(_key148, _val149); } } struct.setColumnFamilyDefinitionsIsSet(true);
