Updates to try and get unit tests to pass. IndexManagerTest still has 3 broken tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/bc1e033a Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/bc1e033a Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/bc1e033a Branch: refs/heads/0.1.5 Commit: bc1e033a9b1e6b22ef24719ec14d17e72f8cdfff Parents: c1231e3 Author: Aaron McCurry <[email protected]> Authored: Wed Apr 10 23:09:31 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Wed Apr 10 23:09:31 2013 -0400 ---------------------------------------------------------------------- .../java/org/apache/blur/manager/IndexManager.java | 20 ++- .../apache/blur/thrift/ThriftBlurShardServer.java | 3 +- .../java/org/apache/blur/thrift/ThriftServer.java | 4 + .../src/test/java/org/apache/blur/MiniCluster.java | 35 +++-- .../org/apache/blur/manager/IndexManagerTest.java | 102 ++++++++++++++- .../blur/lucene/search/RandomSuperQueryTest.java | 2 +- .../apache/blur/lucene/search/SuperQueryTest.java | 2 +- .../blur/lucene/search/TestingPagingCollector.java | 2 +- 8 files changed, 143 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/bc1e033a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java index 0e82f57..2d68f99 100644 --- a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java +++ b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java @@ -342,8 +342,8 @@ public class IndexManager { TableContext context = getTableContext(table); if (isSimpleQuery(blurQuery)) { SimpleQuery simpleQuery = blurQuery.simpleQuery; - Filter preFilter = QueryParserUtil - .parseFilter(table, simpleQuery.preSuperFilter, false, analyzer, _filterCache, context); + Filter preFilter = QueryParserUtil.parseFilter(table, simpleQuery.preSuperFilter, false, analyzer, + _filterCache, context); Filter postFilter = QueryParserUtil.parseFilter(table, simpleQuery.postSuperFilter, true, analyzer, _filterCache, context); Query userQuery = QueryParserUtil.parseQuery(simpleQuery.queryStr, simpleQuery.superQueryOn, analyzer, @@ -395,15 +395,16 @@ public class IndexManager { return false; } - private Query getFacetedQuery(BlurQuery blurQuery, Query userQuery, AtomicLongArray counts, BlurAnalyzer analyzer, TableContext context) - throws ParseException { + private Query getFacetedQuery(BlurQuery blurQuery, Query userQuery, AtomicLongArray counts, BlurAnalyzer analyzer, + TableContext context) throws ParseException { if (blurQuery.facets == null) { return userQuery; } return new FacetQuery(userQuery, getFacetQueries(blurQuery, analyzer, context), counts); } - private Query[] getFacetQueries(BlurQuery blurQuery, BlurAnalyzer analyzer, TableContext context) throws ParseException { + private Query[] getFacetQueries(BlurQuery blurQuery, BlurAnalyzer analyzer, TableContext context) + throws ParseException { int size = blurQuery.facets.size(); Query[] queries = new Query[size]; for (int i = 0; i < size; i++) { @@ -985,8 +986,8 @@ public class IndexManager { // IndexSearcher searcher = new IndexSearcher(escapeReader); searcher.setSimilarity(_indexServer.getSimilarity(_table)); Query rewrite = searcher.rewrite((Query) _query.clone()); - - //BlurResultIterableSearcher will close searcher. + + // BlurResultIterableSearcher will close searcher. return new BlurResultIterableSearcher(_running, rewrite, _table, shard, searcher, _selector); } finally { _queriesInternalMeter.mark(); @@ -1022,4 +1023,9 @@ public class IndexManager { } } } + + public void setClusterStatus(ClusterStatus clusterStatus) { + _clusterStatus = clusterStatus; + } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/bc1e033a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java index ab958bf..1beb382 100644 --- a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java +++ b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java @@ -100,8 +100,6 @@ public class ThriftBlurShardServer extends ThriftServer { int bindPort = configuration.getInt(BLUR_SHARD_BIND_PORT, -1); bindPort += serverIndex; -// BlurMetrics blurMetrics = new BlurMetrics(config); - int baseGuiPort = Integer.parseInt(configuration.get(BLUR_GUI_SHARD_PORT)); final HttpJettyServer httpServer; if (baseGuiPort > 0) { @@ -192,6 +190,7 @@ public class ThriftBlurShardServer extends ThriftServer { indexManager.setMaxClauseCount(configuration.getInt(BLUR_MAX_CLAUSE_COUNT, 1024)); indexManager.setThreadCount(configuration.getInt(BLUR_INDEXMANAGER_SEARCH_THREAD_COUNT, 32)); indexManager.setFilterCache(filterCache); + indexManager.setClusterStatus(clusterStatus); indexManager.init(); final BlurShardServer shardServer = new BlurShardServer(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/bc1e033a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java index 787b034..1beba2b 100644 --- a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java +++ b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java @@ -26,6 +26,7 @@ import java.net.UnknownHostException; import java.util.concurrent.ExecutorService; import org.apache.blur.BlurConfiguration; +import org.apache.blur.concurrent.Executors; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.manager.indexserver.BlurServerShutDown.BlurShutdown; @@ -103,13 +104,16 @@ public class ThriftServer { } public void start() throws TTransportException { + _executorService = Executors.newThreadPool("thrift-processors", _threadCount); Blur.Processor<Blur.Iface> processor = new Blur.Processor<Blur.Iface>(_iface); TServerSocket serverTransport = new TServerSocket(getBindInetSocketAddress(_configuration)); TThreadPoolServer.Args args = new TThreadPoolServer.Args(serverTransport); + args.processor(processor); + args.executorService(_executorService); args.transportFactory(new TFramedTransport.Factory()); args.protocolFactory(new TBinaryProtocol.Factory(true, true)); _server = new TThreadPoolServer(args); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/bc1e033a/src/blur-core/src/test/java/org/apache/blur/MiniCluster.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/test/java/org/apache/blur/MiniCluster.java b/src/blur-core/src/test/java/org/apache/blur/MiniCluster.java index 1da7641..94ed435 100644 --- a/src/blur-core/src/test/java/org/apache/blur/MiniCluster.java +++ b/src/blur-core/src/test/java/org/apache/blur/MiniCluster.java @@ -75,7 +75,6 @@ import org.apache.zookeeper.server.ZooKeeperServerMain; import org.apache.zookeeper.server.quorum.QuorumPeerConfig; import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException; - public abstract class MiniCluster { private static Log LOG = LogFactory.getLog(MiniCluster.class); @@ -87,19 +86,20 @@ public abstract class MiniCluster { private static List<ThriftServer> shards = new ArrayList<ThriftServer>(); private static String controllerConnectionStr; - public static void main(String[] args) throws IOException, InterruptedException, KeeperException, BlurException, TException { + public static void main(String[] args) throws IOException, InterruptedException, KeeperException, BlurException, + TException { startDfs("./tmp"); startZooKeeper("./tmp"); startControllers(1); startShards(1); - - //Run the controllers/shards on custom ports. - //BlurConfiguration conf = new BlurConfiguration(false); - //conf.setInt(BLUR_CONTROLLER_BIND_PORT, 40001); - //conf.setInt(BLUR_SHARD_BIND_PORT, 40002); - //startControllers(conf, 1); - //startShards(conf, 1); - + + // Run the controllers/shards on custom ports. + // BlurConfiguration conf = new BlurConfiguration(false); + // conf.setInt(BLUR_CONTROLLER_BIND_PORT, 40001); + // conf.setInt(BLUR_SHARD_BIND_PORT, 40002); + // startControllers(conf, 1); + // startShards(conf, 1); + try { Iface client = BlurClient.getClient(getControllerConnectionStr()); createTable("test", client); @@ -201,13 +201,13 @@ public abstract class MiniCluster { private static BlurConfiguration getBlurConfiguration(BlurConfiguration overrides) { BlurConfiguration conf = getBlurConfiguration(); - - for(Map.Entry<String, String> over: overrides.getProperties().entrySet()) { + + for (Map.Entry<String, String> over : overrides.getProperties().entrySet()) { conf.set(over.getKey().toString(), over.getValue().toString()); } return conf; } - + private static BlurConfiguration getBlurConfiguration() { BlurConfiguration configuration; try { @@ -457,7 +457,8 @@ public abstract class MiniCluster { for (int i = 0; i < enumerate; i++) { Thread thread = threads[i]; if (thread.getName().startsWith("pool")) { - while (thread.isAlive()) { + if (thread.isAlive()) { + thread.interrupt(); LOG.info("Stopping ThreadPoolExecutor [" + thread.getName() + "]"); Object target = getField(thread, "target"); if (target != null) { @@ -466,6 +467,12 @@ public abstract class MiniCluster { e.shutdownNow(); } } + try { + LOG.info("Waiting for thread pool to exit [" + thread.getName() + "]"); + thread.join(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/bc1e033a/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java b/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java index 0056e36..ebd8303 100644 --- a/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java +++ b/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java @@ -45,6 +45,7 @@ import java.util.Set; import java.util.TreeSet; import java.util.concurrent.atomic.AtomicLongArray; +import org.apache.blur.manager.clusterstatus.ClusterStatus; import org.apache.blur.manager.indexserver.LocalIndexServer; import org.apache.blur.manager.results.BlurResultIterable; import org.apache.blur.thrift.generated.AnalyzerDefinition; @@ -92,7 +93,7 @@ public class IndexManagerTest { File file = new File(base, TABLE); file.mkdirs(); - TableDescriptor tableDescriptor = new TableDescriptor(); + final TableDescriptor tableDescriptor = new TableDescriptor(); tableDescriptor.setName(TABLE); tableDescriptor.setTableUri(file.toURI().toString()); tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition()); @@ -104,6 +105,103 @@ public class IndexManagerTest { indexManager.setStatusCleanupTimerDelay(1000); indexManager.setIndexServer(server); indexManager.setThreadCount(1); + indexManager.setClusterStatus(new ClusterStatus() { + + @Override + public void removeTable(String cluster, String table, boolean deleteIndexFiles) { + throw new RuntimeException("Not impl"); + } + + @Override + public boolean isReadOnly(boolean useCache, String cluster, String table) { + throw new RuntimeException("Not impl"); + } + + @Override + public boolean isOpen() { + throw new RuntimeException("Not impl"); + } + + @Override + public boolean isInSafeMode(boolean useCache, String cluster) { + throw new RuntimeException("Not impl"); + } + + @Override + public boolean isEnabled(boolean useCache, String cluster, String table) { + throw new RuntimeException("Not impl"); + } + + @Override + public boolean isBlockCacheEnabled(String cluster, String table) { + throw new RuntimeException("Not impl"); + } + + @Override + public List<String> getTableList(boolean useCache, String cluster) { + throw new RuntimeException("Not impl"); + } + + @Override + public TableDescriptor getTableDescriptor(boolean useCache, String cluster, String table) { + return tableDescriptor; + } + + @Override + public List<String> getShardServerList(String cluster) { + throw new RuntimeException("Not impl"); + } + + @Override + public int getShardCount(boolean useCache, String cluster, String table) { + throw new RuntimeException("Not impl"); + } + + @Override + public List<String> getOnlineShardServers(boolean useCache, String cluster) { + throw new RuntimeException("Not impl"); + } + + @Override + public List<String> getControllerServerList() { + throw new RuntimeException("Not impl"); + } + + @Override + public List<String> getClusterList(boolean useCache) { + throw new RuntimeException("Not impl"); + } + + @Override + public String getCluster(boolean useCache, String table) { + return BlurConstants.BLUR_CLUSTER; + } + + @Override + public Set<String> getBlockCacheFileTypes(String cluster, String table) { + throw new RuntimeException("Not impl"); + } + + @Override + public boolean exists(boolean useCache, String cluster, String table) { + throw new RuntimeException("Not impl"); + } + + @Override + public void enableTable(String cluster, String table) { + throw new RuntimeException("Not impl"); + } + + @Override + public void disableTable(String cluster, String table) { + throw new RuntimeException("Not impl"); + } + + @Override + public void createTable(TableDescriptor tableDescriptor) { + throw new RuntimeException("Not impl"); + } + }); indexManager.init(); setupData(); } @@ -161,6 +259,8 @@ public class IndexManagerTest { BlurQuery blurQuery = new BlurQuery(); blurQuery.simpleQuery = new SimpleQuery(); blurQuery.simpleQuery.queryStr = "+test-family.testcol12:value101 +test-family.testcol13:value102 +test-family2.testcol18:value501"; +// blurQuery.simpleQuery.queryStr = "+super:<+test-family.testcol12:value101 +test-family.testcol13:value102> +test-family2.testcol18:value501"; + blurQuery.simpleQuery.superQueryOn = true; blurQuery.simpleQuery.type = ScoreType.SUPER; blurQuery.fetch = 10; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/bc1e033a/src/blur-query/src/test/java/org/apache/blur/lucene/search/RandomSuperQueryTest.java ---------------------------------------------------------------------- diff --git a/src/blur-query/src/test/java/org/apache/blur/lucene/search/RandomSuperQueryTest.java b/src/blur-query/src/test/java/org/apache/blur/lucene/search/RandomSuperQueryTest.java index 707916c..ceaca80 100644 --- a/src/blur-query/src/test/java/org/apache/blur/lucene/search/RandomSuperQueryTest.java +++ b/src/blur-query/src/test/java/org/apache/blur/lucene/search/RandomSuperQueryTest.java @@ -1,4 +1,4 @@ -package org.apache.blur.search; +package org.apache.blur.lucene.search; /** * Licensed to the Apache Software Foundation (ASF) under one or more http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/bc1e033a/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperQueryTest.java ---------------------------------------------------------------------- diff --git a/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperQueryTest.java b/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperQueryTest.java index fffb348..37346c9 100644 --- a/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperQueryTest.java +++ b/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperQueryTest.java @@ -1,4 +1,4 @@ -package org.apache.blur.search; +package org.apache.blur.lucene.search; /** * Licensed to the Apache Software Foundation (ASF) under one or more http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/bc1e033a/src/blur-query/src/test/java/org/apache/blur/lucene/search/TestingPagingCollector.java ---------------------------------------------------------------------- diff --git a/src/blur-query/src/test/java/org/apache/blur/lucene/search/TestingPagingCollector.java b/src/blur-query/src/test/java/org/apache/blur/lucene/search/TestingPagingCollector.java index 5eaad3f..9358eb2 100644 --- a/src/blur-query/src/test/java/org/apache/blur/lucene/search/TestingPagingCollector.java +++ b/src/blur-query/src/test/java/org/apache/blur/lucene/search/TestingPagingCollector.java @@ -1,4 +1,4 @@ -package org.apache.blur.search; +package org.apache.blur.lucene.search; /** * Licensed to the Apache Software Foundation (ASF) under one or more
