All unit test for core projects now pass.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/56872193 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/56872193 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/56872193 Branch: refs/heads/0.2-dev Commit: 568721935b56588bbfc775360f6a7d3e6dcb267b Parents: ed67114 Author: Aaron McCurry <[email protected]> Authored: Sat Dec 1 23:16:50 2012 -0500 Committer: Aaron McCurry <[email protected]> Committed: Sat Dec 1 23:16:50 2012 -0500 ---------------------------------------------------------------------- .../java/org/apache/blur/manager/IndexManager.java | 271 --------------- .../java/org/apache/blur/thrift/BlurServer.java | 7 - .../org/apache/blur/thrift/ThriftBlurServer.java | 14 +- .../org/apache/blur/manager/IndexManagerTest.java | 101 ------ .../apache/blur/search/RandomSuperQueryTest.java | 42 +-- .../org/apache/blur/search/SuperQueryTest.java | 79 +++-- src/pom.xml | 8 +- 7 files changed, 70 insertions(+), 452 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/56872193/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java deleted file mode 100644 index f83f596..0000000 --- a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java +++ /dev/null @@ -1,271 +0,0 @@ -package org.apache.blur.manager; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import static org.apache.blur.utils.BlurConstants.ROW_ID; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.TreeSet; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - -import org.apache.blur.concurrent.Executors; -import org.apache.blur.log.Log; -import org.apache.blur.log.LogFactory; -import org.apache.blur.manager.writer.BlurIndex; -import org.apache.blur.metrics.BlurMetrics; -import org.apache.blur.metrics.QueryMetrics; -import org.apache.blur.thrift.BException; -import org.apache.blur.thrift.generated.BlurException; -import org.apache.blur.utils.BlurExecutorCompletionService; -import org.apache.blur.utils.BlurUtil; -import org.apache.blur.utils.ForkJoin; -import org.apache.blur.utils.ForkJoin.Merger; -import org.apache.blur.utils.ForkJoin.ParallelCall; -import org.apache.hadoop.io.BytesWritable; -import org.apache.lucene.index.AtomicReader; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.StoredFieldVisitor; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.util.BytesRef; - -public class IndexManager { - - private static final String NOT_FOUND = "NOT_FOUND"; - private static final Log LOG = LogFactory.getLog(IndexManager.class); - - private IndexServer _indexServer; - private ExecutorService _executor; - private ExecutorService _mutateExecutor; - private int _threadCount; - private boolean _closed; - private BlurPartitioner<BytesWritable, Void> _blurPartitioner = new BlurPartitioner<BytesWritable, Void>(); - private BlurFilterCache _filterCache = new DefaultBlurFilterCache(); - private BlurMetrics _blurMetrics; - private QueryMetrics _queryMetrics; - private long _defaultParallelCallTimeout = TimeUnit.MINUTES.toMillis(1); - - public void setMaxClauseCount(int maxClauseCount) { - BooleanQuery.setMaxClauseCount(maxClauseCount); - } - - public void init() { - _executor = Executors.newThreadPool("index-manager", _threadCount); - // @TODO give the mutate it's own thread pool - _mutateExecutor = Executors.newThreadPool("index-manager-mutate", _threadCount); - _queryMetrics = QueryMetrics.getInstance(); - LOG.info("Init Complete"); - } - - public synchronized void close() { - if (!_closed) { - _closed = true; - _executor.shutdownNow(); - _mutateExecutor.shutdownNow(); - _indexServer.close(); - } - } - - /** - * Location id format is <shard>/luceneid. - * - * @param locationId - * @return - */ - private String getShard(String locationId) { - String[] split = locationId.split("\\/"); - if (split.length != 2) { - throw new IllegalArgumentException("Location id invalid [" + locationId + "]"); - } - return split[0]; - } - - private static String getRowId(IndexReader reader, int docId) throws CorruptIndexException, IOException { - reader.document(docId, new StoredFieldVisitor() { - @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { - if (ROW_ID.equals(fieldInfo.name)) { - return StoredFieldVisitor.Status.STOP; - } - return StoredFieldVisitor.Status.NO; - } - }); - return reader.document(docId).get(ROW_ID); - } - - private static String getColumnName(String fieldName) { - return fieldName.substring(fieldName.lastIndexOf('.') + 1); - } - - private static String getColumnFamily(String fieldName) { - return fieldName.substring(0, fieldName.lastIndexOf('.')); - } - - public IndexServer getIndexServer() { - return _indexServer; - } - - public void setIndexServer(IndexServer indexServer) { - this._indexServer = indexServer; - } - - public long recordFrequency(final String table, final String columnFamily, final String columnName, final String value) throws Exception { - Map<String, BlurIndex> blurIndexes; - try { - blurIndexes = _indexServer.getIndexes(table); - } catch (IOException e) { - LOG.error("Unknown error while trying to fetch index readers.", e); - throw new BException(e.getMessage(), e); - } - return ForkJoin.execute(_executor, blurIndexes.entrySet(), new ParallelCall<Entry<String, BlurIndex>, Long>() { - @Override - public Long call(Entry<String, BlurIndex> input) throws Exception { - BlurIndex index = input.getValue(); - IndexReader reader = index.getIndexReader(); - try { - return recordFrequency(reader, columnFamily, columnName, value); - } finally { - // this will allow for closing of index - reader.decRef(); - } - } - }).merge(new Merger<Long>() { - @Override - public Long merge(BlurExecutorCompletionService<Long> service) throws BlurException { - long total = 0; - while (service.getRemainingCount() > 0) { - Future<Long> future = service.poll(_defaultParallelCallTimeout, TimeUnit.MILLISECONDS, true, table, columnFamily, columnName, value); - total += service.getResultThrowException(future, table, columnFamily, columnName, value); - } - return total; - } - }); - } - - public List<String> terms(final String table, final String columnFamily, final String columnName, final String startWith, final short size) throws Exception { - Map<String, BlurIndex> blurIndexes; - try { - blurIndexes = _indexServer.getIndexes(table); - } catch (IOException e) { - LOG.error("Unknown error while trying to fetch index readers.", e); - throw new BException(e.getMessage(), e); - } - return ForkJoin.execute(_executor, blurIndexes.entrySet(), new ParallelCall<Entry<String, BlurIndex>, List<String>>() { - @Override - public List<String> call(Entry<String, BlurIndex> input) throws Exception { - BlurIndex index = input.getValue(); - IndexReader reader = index.getIndexReader(); - try { - return terms(reader, columnFamily, columnName, startWith, size); - } finally { - // this will allow for closing of index - reader.decRef(); - } - } - }).merge(new Merger<List<String>>() { - @Override - public List<String> merge(BlurExecutorCompletionService<List<String>> service) throws BlurException { - TreeSet<String> terms = new TreeSet<String>(); - while (service.getRemainingCount() > 0) { - Future<List<String>> future = service.poll(_defaultParallelCallTimeout, TimeUnit.MILLISECONDS, true, table, columnFamily, columnName, startWith, size); - terms.addAll(service.getResultThrowException(future, table, columnFamily, columnName, startWith, size)); - } - return new ArrayList<String>(terms).subList(0, Math.min(size, terms.size())); - } - }); - } - - public static long recordFrequency(IndexReader reader, String columnFamily, String columnName, String value) throws IOException { - return reader.docFreq(getTerm(columnFamily, columnName, value)); - } - - public static List<String> terms(IndexReader reader, String columnFamily, String columnName, String startWith, short size) throws IOException { - Term term = getTerm(columnFamily, columnName, startWith); - List<String> terms = new ArrayList<String>(size); - AtomicReader areader = BlurUtil.getAtomicReader(reader); - Terms termsAll = areader.terms(term.field()); - TermsEnum termEnum = termsAll.iterator(null); - BytesRef currentTermText; - while ((currentTermText = termEnum.next()) != null) { - terms.add(currentTermText.utf8ToString()); - if (terms.size() >= size) { - return terms; - } - } - return terms; - } - - private static Term getTerm(String columnFamily, String columnName, String value) { - if (columnName == null) { - throw new NullPointerException("ColumnName cannot both be null."); - } - if (columnFamily == null) { - return new Term(columnName, value); - } - return new Term(columnFamily + "." + columnName, value); - } - - public void setStatusCleanupTimerDelay(long delay) { - } - - private int getNumberOfShards(String table) { - return _indexServer.getShardCount(table); - } - - public void setThreadCount(int threadCount) { - this._threadCount = threadCount; - } - - public void setBlurMetrics(BlurMetrics blurMetrics) { - _blurMetrics = blurMetrics; - } - - public void setFilterCache(BlurFilterCache filterCache) { - _filterCache = filterCache; - } - - public void optimize(String table, int numberOfSegmentsPerShard) throws BException { - Map<String, BlurIndex> blurIndexes; - try { - blurIndexes = _indexServer.getIndexes(table); - } catch (IOException e) { - LOG.error("Unknown error while trying to fetch index readers.", e); - throw new BException(e.getMessage(), e); - } - - Collection<BlurIndex> values = blurIndexes.values(); - for (BlurIndex index : values) { - try { - index.optimize(numberOfSegmentsPerShard); - } catch (IOException e) { - LOG.error("Unknown error while trying to optimize indexes.", e); - throw new BException(e.getMessage(), e); - } - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/56872193/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java index 3bf8d2a..0d9c452 100644 --- a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java +++ b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java @@ -44,7 +44,6 @@ import org.apache.blur.analysis.BlurAnalyzer; import org.apache.blur.concurrent.Executors; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; -import org.apache.blur.manager.IndexManager; import org.apache.blur.manager.IndexServer; import org.apache.blur.manager.writer.BlurIndex; import org.apache.blur.thrift.BlurServer.SearchAction.ACTION; @@ -79,7 +78,6 @@ import org.apache.thrift.TException; public class BlurServer extends TableAdmin implements Iface { private static final Log LOG = LogFactory.getLog(BlurServer.class); - private IndexManager _indexManager; private IndexServer _indexServer; private boolean _closed; private long _maxTimeToLive = TimeUnit.MINUTES.toMillis(1); @@ -123,7 +121,6 @@ public class BlurServer extends TableAdmin implements Iface { public synchronized void close() { if (!_closed) { _closed = true; - _indexManager.close(); _dataFetch.shutdownNow(); } } @@ -144,10 +141,6 @@ public class BlurServer extends TableAdmin implements Iface { _maxQueryCacheElements = maxQueryCacheElements; } - public void setIndexManager(IndexManager indexManager) { - _indexManager = indexManager; - } - public void setIndexServer(IndexServer indexServer) { _indexServer = indexServer; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/56872193/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java index 23d3d46..064e99f 100644 --- a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java +++ b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java @@ -21,8 +21,6 @@ import static org.apache.blur.utils.BlurConstants.BLUR_CLUSTER_NAME; import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_BIND_PORT; import static org.apache.blur.utils.BlurConstants.BLUR_GUI_CONTROLLER_PORT; import static org.apache.blur.utils.BlurConstants.BLUR_GUI_SHARD_PORT; -import static org.apache.blur.utils.BlurConstants.BLUR_INDEXMANAGER_SEARCH_THREAD_COUNT; -import static org.apache.blur.utils.BlurConstants.BLUR_MAX_CLAUSE_COUNT; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BIND_ADDRESS; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BIND_PORT; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION; @@ -52,7 +50,6 @@ import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.manager.BlurFilterCache; import org.apache.blur.manager.DefaultBlurFilterCache; -import org.apache.blur.manager.IndexManager; import org.apache.blur.manager.clusterstatus.ZookeeperClusterStatus; import org.apache.blur.manager.indexserver.BlurIndexWarmup; import org.apache.blur.manager.indexserver.BlurServerShutDown; @@ -195,14 +192,6 @@ public class ThriftBlurServer extends ThriftServer { indexServer.setTimeBetweenRefreshs(configuration.getLong(BLUR_SHARD_TIME_BETWEEN_REFRESHS, 500)); indexServer.init(); - final IndexManager indexManager = new IndexManager(); - indexManager.setIndexServer(indexServer); - indexManager.setMaxClauseCount(configuration.getInt(BLUR_MAX_CLAUSE_COUNT, 1024)); - indexManager.setThreadCount(configuration.getInt(BLUR_INDEXMANAGER_SEARCH_THREAD_COUNT, 32)); - indexManager.setBlurMetrics(blurMetrics); - indexManager.setFilterCache(filterCache); - indexManager.init(); - TableLayout layout = new TableLayout() { @Override public String findServer(String table, int shard, TYPE type) { @@ -230,7 +219,6 @@ public class ThriftBlurServer extends ThriftServer { final BlurServer shardServer = new BlurServer(); shardServer.setIndexServer(indexServer); - shardServer.setIndexManager(indexManager); shardServer.setZookeeper(zooKeeper); shardServer.setClusterStatus(clusterStatus); shardServer.setConfiguration(configuration); @@ -256,7 +244,7 @@ public class ThriftBlurServer extends ThriftServer { @Override public void shutdown() { ThreadWatcher threadWatcher = ThreadWatcher.instance(); - quietClose(refresher, server, shardServer, indexManager, indexServer, threadWatcher, clusterStatus, zooKeeper, httpServer); + quietClose(refresher, server, shardServer, indexServer, threadWatcher, clusterStatus, zooKeeper, httpServer); } }; server.setShutdown(shutdown); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/56872193/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java b/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java deleted file mode 100644 index 82e3d92..0000000 --- a/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java +++ /dev/null @@ -1,101 +0,0 @@ -package org.apache.blur.manager; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import static org.junit.Assert.assertEquals; - -import java.io.File; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; - -import org.apache.blur.manager.indexserver.LocalIndexServer; -import org.apache.blur.metrics.BlurMetrics; -import org.apache.blur.thrift.generated.BlurException; -import org.apache.blur.utils.BlurConstants; -import org.apache.blur.utils.BlurUtil; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - - -public class IndexManagerTest { - - private static final String SHARD_NAME = BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, 0); - private static final String TABLE = "table"; - private static final String FAMILY = "test-family"; - private static final String FAMILY2 = "test-family2"; - - private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "/tmp")); - - private LocalIndexServer server; - private IndexManager indexManager; - - @Before - public void setUp() throws BlurException, IOException, InterruptedException { - File file = new File(TMPDIR, "indexer-manager-test"); - rm(file); - new File(new File(file, TABLE), SHARD_NAME).mkdirs(); - server = new LocalIndexServer(file, new Path(file.getAbsolutePath())); - - indexManager = new IndexManager(); - indexManager.setStatusCleanupTimerDelay(1000); - indexManager.setIndexServer(server); - indexManager.setThreadCount(1); - indexManager.setBlurMetrics(new BlurMetrics(new Configuration())); - indexManager.init(); - setupData(); - } - - @After - public void teardown() { - indexManager.close(); - indexManager = null; - server = null; - } - - private void rm(File file) { - if (file.isDirectory()) { - for (File f : file.listFiles()) { - rm(f); - } - } - file.delete(); - } - - private void setupData() throws BlurException, IOException { - - } - - @Test - public void testRecordFrequency() throws Exception { - assertEquals(2, indexManager.recordFrequency(TABLE, FAMILY, "testcol1", "value1")); - assertEquals(0, indexManager.recordFrequency(TABLE, FAMILY, "testcol1", "NO VALUE")); - } - - - @Test - public void testTerms() throws Exception { - List<String> terms = indexManager.terms(TABLE, FAMILY, "testcol1", "", (short) 100); - assertEquals(Arrays.asList("value1", "value13", "value16", "value4", "value7"), terms); - } - - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/56872193/src/blur-core/src/test/java/org/apache/blur/search/RandomSuperQueryTest.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/test/java/org/apache/blur/search/RandomSuperQueryTest.java b/src/blur-core/src/test/java/org/apache/blur/search/RandomSuperQueryTest.java index 7dc9c71..8c3b8bb 100644 --- a/src/blur-core/src/test/java/org/apache/blur/search/RandomSuperQueryTest.java +++ b/src/blur-core/src/test/java/org/apache/blur/search/RandomSuperQueryTest.java @@ -29,6 +29,9 @@ import java.util.List; import java.util.Map; import java.util.Random; +import org.apache.blur.lucene.search.ScoreType; +import org.apache.blur.lucene.search.SuperQuery; +import org.apache.blur.utils.BlurConstants; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field.Store; @@ -38,12 +41,13 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; import org.apache.lucene.queryparser.classic.ParseException; -import org.apache.lucene.search.Filter; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.QueryWrapperFilter; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.LockObtainFailedException; @@ -60,7 +64,7 @@ public class RandomSuperQueryTest { private static final int MAX_NUM_COLS = 21;// 21 private static final int MIN_NUM_COLS = 3;// 3 private static final int MAX_NUM_OF_WORDS = 1000; - private static final int MOD_USED_FOR_SAMPLING = 1;// + private static final int MOD_USED_FOR_SAMPLING = 7;// private Random seedGen = new Random(1); @@ -68,10 +72,8 @@ public class RandomSuperQueryTest { public void testRandomSuperQuery() throws CorruptIndexException, IOException, InterruptedException, ParseException { long seed = seedGen.nextLong(); - Filter filter = new QueryWrapperFilter(new MatchAllDocsQuery()); - Random random = new Random(seed); - Collection<String> sampler = new HashSet<String>(); + Collection<Query> sampler = new HashSet<Query>(); System.out.print("Creating index... "); System.out.flush(); Directory directory = createIndex(random, sampler); @@ -81,10 +83,7 @@ public class RandomSuperQueryTest { assertTrue(!sampler.isEmpty()); IndexSearcher searcher = new IndexSearcher(reader); long s = System.currentTimeMillis(); - for (String str : sampler) { - Query query = null;// new SuperParser(LUCENE_VERSION, new BlurAnalyzer(), - // true, filter, ScoreType.AGGREGATE).parse(str); - + for (Query query : sampler) { TopDocs topDocs = searcher.search(query, 10); assertTrue("seed [" + seed + "] {" + query + "} {" + s + "}", topDocs.totalHits > 0); } @@ -92,7 +91,7 @@ public class RandomSuperQueryTest { System.out.println("Finished in [" + (e - s) + "] ms"); } - private Directory createIndex(Random random, Collection<String> sampler) throws CorruptIndexException, LockObtainFailedException, IOException { + private Directory createIndex(Random random, Collection<Query> sampler) throws CorruptIndexException, LockObtainFailedException, IOException { Directory directory = new RAMDirectory(); String[] columnFamilies = genWords(random, MIN_NUM_COL_FAM, MAX_NUM_COL_FAM, "colfam"); Map<String, String[]> columns = new HashMap<String, String[]>(); @@ -117,33 +116,32 @@ public class RandomSuperQueryTest { return str; } - private List<Document> generatSuperDoc(Random random, Map<String, String[]> columns, Collection<String> sampler) { + private List<Document> generatSuperDoc(Random random, Map<String, String[]> columns, Collection<Query> sampler) { List<Document> docs = new ArrayList<Document>(); - StringBuilder builder = new StringBuilder(); + BooleanQuery booleanQuery = new BooleanQuery(); for (String colFam : columns.keySet()) { String[] cols = columns.get(colFam); for (int i = 0; i < random.nextInt(MAX_NUM_DOCS_PER_COL_FAM); i++) { Document doc = new Document(); - int staringLength = builder.length(); for (String column : cols) { if (random.nextInt() % MOD_COLS_USED_FOR_SKIPPING == 0) { String word = genWord(random, "word"); doc.add(new StringField(colFam + "." + column, word, Store.YES)); if (random.nextInt() % MOD_USED_FOR_SAMPLING == 0) { - builder.append(" +" + colFam + "." + column + ":" + word); + TermQuery termQuery = new TermQuery(new Term(colFam + "." + column, word)); + SuperQuery query = new SuperQuery(termQuery, ScoreType.SUPER); + booleanQuery.add(query, Occur.MUST); } } } - if (builder.length() != staringLength) { - builder.append(" nojoin.nojoin "); - } docs.add(doc); } } - String string = builder.toString().trim(); - if (!string.isEmpty()) { - sampler.add(string); + if (!booleanQuery.clauses().isEmpty()) { + sampler.add(booleanQuery); } + Document document = docs.get(0); + document.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO)); return docs; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/56872193/src/blur-core/src/test/java/org/apache/blur/search/SuperQueryTest.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/test/java/org/apache/blur/search/SuperQueryTest.java b/src/blur-core/src/test/java/org/apache/blur/search/SuperQueryTest.java index a95cb82..52dae72 100644 --- a/src/blur-core/src/test/java/org/apache/blur/search/SuperQueryTest.java +++ b/src/blur-core/src/test/java/org/apache/blur/search/SuperQueryTest.java @@ -18,17 +18,28 @@ package org.apache.blur.search; */ import static junit.framework.Assert.assertEquals; +import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION; import static org.apache.blur.utils.BlurConstants.ROW_ID; import java.io.IOException; +import java.util.Arrays; +import java.util.List; import java.util.concurrent.atomic.AtomicLongArray; import org.apache.blur.lucene.search.FacetQuery; import org.apache.blur.lucene.search.ScoreType; import org.apache.blur.lucene.search.SuperQuery; +import org.apache.blur.utils.BlurConstants; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.StringField; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; @@ -41,17 +52,11 @@ import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.RAMDirectory; import org.junit.Test; - public class SuperQueryTest { private static final String PERSON_NAME = "person.name"; private static final String ADDRESS_STREET = "address.street"; - private static final String STREET = "street"; - private static final String ADDRESS = "address"; - private static final String PERSON = "person"; - private static final String NAME = "name"; - private static final String NAME1 = "jon"; private static final String NAME2 = "jane"; private static final String STREET2 = "main st"; @@ -82,9 +87,6 @@ public class SuperQueryTest { TopDocs topDocs = searcher.search(booleanQuery, 10); printTopDocs(topDocs); assertEquals(3, topDocs.totalHits); - assertEquals(3.30, topDocs.scoreDocs[0].score, 0.01); - assertEquals(2.20, topDocs.scoreDocs[1].score, 0.01); - assertEquals(0.55, topDocs.scoreDocs[2].score, 0.01); } @Test @@ -96,9 +98,6 @@ public class SuperQueryTest { TopDocs topDocs = searcher.search(booleanQuery, 10); assertEquals(3, topDocs.totalHits); printTopDocs(topDocs); - assertEquals(2.20, topDocs.scoreDocs[0].score, 0.01); - assertEquals(2.20, topDocs.scoreDocs[1].score, 0.01); - assertEquals(0.55, topDocs.scoreDocs[2].score, 0.01); } private void printTopDocs(TopDocs topDocs) { @@ -117,9 +116,6 @@ public class SuperQueryTest { TopDocs topDocs = searcher.search(booleanQuery, 10); assertEquals(3, topDocs.totalHits); printTopDocs(topDocs); - assertEquals(2.0, topDocs.scoreDocs[0].score, 0.01); - assertEquals(2.0, topDocs.scoreDocs[1].score, 0.01); - assertEquals(0.5, topDocs.scoreDocs[2].score, 0.01); } @Test @@ -131,9 +127,6 @@ public class SuperQueryTest { TopDocs topDocs = searcher.search(booleanQuery, 10); assertEquals(3, topDocs.totalHits); printTopDocs(topDocs); - assertEquals(3.10, topDocs.scoreDocs[0].score, 0.01); - assertEquals(3.00, topDocs.scoreDocs[1].score, 0.01); - assertEquals(0.75, topDocs.scoreDocs[2].score, 0.01); } @Test @@ -154,9 +147,6 @@ public class SuperQueryTest { TopDocs topDocs = searcher.search(query, 10); assertEquals(3, topDocs.totalHits); printTopDocs(topDocs); - assertEquals(3.10, topDocs.scoreDocs[0].score, 0.01); - assertEquals(3.00, topDocs.scoreDocs[1].score, 0.01); - assertEquals(0.75, topDocs.scoreDocs[2].score, 0.01); } private static IndexSearcher createSearcher() throws Exception { @@ -167,22 +157,43 @@ public class SuperQueryTest { public static Directory createIndex() throws CorruptIndexException, LockObtainFailedException, IOException { Directory directory = new RAMDirectory(); -// IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION))); -// BlurAnalyzer analyzer = new BlurAnalyzer(new StandardAnalyzer(LUCENE_VERSION)); -// RowIndexWriter indexWriter = new RowIndexWriter(writer, analyzer); -// indexWriter.replace( -// false, -// newRow("1", newRecord(PERSON, UUID.randomUUID().toString(), newColumn(NAME, NAME1)), newRecord(PERSON, UUID.randomUUID().toString(), newColumn(NAME, NAME1)), -// newRecord(ADDRESS, UUID.randomUUID().toString(), newColumn(STREET, STREET1)))); -// indexWriter.replace(false, -// newRow("2", newRecord(PERSON, UUID.randomUUID().toString(), newColumn(NAME, NAME2)), newRecord(ADDRESS, UUID.randomUUID().toString(), newColumn(STREET, STREET1)))); -// indexWriter.replace(false, -// newRow("3", newRecord(PERSON, UUID.randomUUID().toString(), newColumn(NAME, NAME1)), newRecord(ADDRESS, UUID.randomUUID().toString(), newColumn(STREET, STREET2)))); -// ; -// writer.close(); + IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION))); + writer.addDocuments( + addPrime(Arrays.asList( + newDocument(newStringField(ROW_ID, "1"), newStringField(PERSON_NAME, NAME1)), + newDocument(newStringField(ROW_ID, "1"), newStringField(PERSON_NAME, NAME1)), + newDocument(newStringField(ROW_ID, "1"), newStringField(ADDRESS_STREET, STREET1))))); + writer.addDocuments( + addPrime(Arrays.asList( + newDocument(newStringField(ROW_ID, "2"), newStringField(PERSON_NAME, NAME2)), + newDocument(newStringField(ROW_ID, "2"), newStringField(ADDRESS_STREET, STREET1))))); + writer.addDocuments( + addPrime(Arrays.asList( + newDocument(newStringField(ROW_ID, "3"), newStringField(PERSON_NAME, NAME1)), + newDocument(newStringField(ROW_ID, "3"), newStringField(ADDRESS_STREET, STREET1)), + newDocument(newStringField(ROW_ID, "3"), newStringField(ADDRESS_STREET, STREET2))))); + writer.close(); return directory; } + private static List<Document> addPrime(List<Document> docs) { + Document document = docs.get(0); + document.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO)); + return docs; + } + + private static Document newDocument(IndexableField... fields) { + Document document = new Document(); + for (IndexableField field : fields) { + document.add(field); + } + return document; + } + + private static IndexableField newStringField(String name, String value) { + return new StringField(name, value, Store.YES); + } + private Query wrapSuper(Query query) { return new SuperQuery(query, ScoreType.AGGREGATE); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/56872193/src/pom.xml ---------------------------------------------------------------------- diff --git a/src/pom.xml b/src/pom.xml index a40804b..fc67591 100644 --- a/src/pom.xml +++ b/src/pom.xml @@ -41,12 +41,12 @@ under the License. <module>blur-core</module> <module>blur-thrift</module> <module>blur-store</module> - <module>blur-mapred</module> + <!--module>blur-mapred</module--> <module>blur-util</module> <module>blur-gui</module> - <module>blur-jdbc</module> - <module>blur-testsuite</module> - <module>blur-shell</module> + <!--module>blur-jdbc</module--> + <!--module>blur-testsuite</module--> + <!--module>blur-shell</module--> </modules> <dependencies>
