Fixing a double search problem.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/da26c980 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/da26c980 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/da26c980 Branch: refs/heads/master Commit: da26c9804da11aa930e0ee164e43a4f3ee80f431 Parents: eda8a39 Author: Aaron McCurry <[email protected]> Authored: Thu Jan 2 10:45:00 2014 -0500 Committer: Aaron McCurry <[email protected]> Committed: Thu Jan 2 10:46:25 2014 -0500 ---------------------------------------------------------------------- .../blur/thrift/ThriftBlurShardServer.java | 8 ++++- .../java/org/apache/blur/utils/BlurUtil.java | 4 +-- .../org/apache/blur/thrift/BlurClusterTest.java | 33 ++++++++++++++++---- .../src/main/resources/blur-default.properties | 2 +- 4 files changed, 36 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/da26c980/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java index d352dc1..44c3a0e 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java @@ -19,6 +19,7 @@ package org.apache.blur.thrift; import static org.apache.blur.utils.BlurConstants.BLUR_CLUSTER; import static org.apache.blur.utils.BlurConstants.BLUR_CLUSTER_NAME; import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_BIND_PORT; +import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_REMOTE_FETCH_COUNT; import static org.apache.blur.utils.BlurConstants.BLUR_GUI_CONTROLLER_PORT; import static org.apache.blur.utils.BlurConstants.BLUR_GUI_SHARD_PORT; import static org.apache.blur.utils.BlurConstants.BLUR_INDEXMANAGER_FACET_THREAD_COUNT; @@ -212,7 +213,12 @@ public class ThriftBlurShardServer extends ThriftServer { BooleanQuery.setMaxClauseCount(configuration.getInt(BLUR_MAX_CLAUSE_COUNT, 1024)); int maxHeapPerRowFetch = configuration.getInt(BLUR_MAX_HEAP_PER_ROW_FETCH, 10000000); - int fetchCount = configuration.getInt(BLUR_SHARD_FETCHCOUNT, 100); + int remoteFetchCount = configuration.getInt(BLUR_CONTROLLER_REMOTE_FETCH_COUNT, 100); + int fetchCount = configuration.getInt(BLUR_SHARD_FETCHCOUNT, 110); + if (fetchCount + 1 <= remoteFetchCount) { + LOG.warn("[" + BLUR_SHARD_FETCHCOUNT + "] [" + fetchCount + "] is equal to or less than [" + + BLUR_CONTROLLER_REMOTE_FETCH_COUNT + "] [" + remoteFetchCount + "], should be at least 1 greater."); + } int indexManagerThreadCount = configuration.getInt(BLUR_INDEXMANAGER_SEARCH_THREAD_COUNT, 32); int mutateThreadCount = configuration.getInt(BLUR_INDEXMANAGER_MUTATE_THREAD_COUNT, 32); int facetThreadCount = configuration.getInt(BLUR_INDEXMANAGER_FACET_THREAD_COUNT, 16); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/da26c980/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java index ee954a8..1c521df 100644 --- a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java +++ b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java @@ -443,11 +443,9 @@ public class BlurUtil { results.setShardInfo(hitsIterable.getShardInfo()); if (query.minimumNumberOfResults > 0) { hitsIterable.skipTo(query.start); - int count = 0; BlurIterator<BlurResult, BlurException> iterator = hitsIterable.iterator(); - while (iterator.hasNext() && count < query.fetch) { + for (int count = 0; count < query.fetch && iterator.hasNext(); count++) { results.addToResults(iterator.next()); - count++; } } if (results.results == null) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/da26c980/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java index 1055385..66d4372 100644 --- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java +++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java @@ -78,7 +78,7 @@ public class BlurClusterTest { private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "./target/tmp_BlurClusterTest")); private static MiniCluster miniCluster; - + private int numberOfDocs = 1000; @BeforeClass @@ -117,20 +117,39 @@ public class BlurClusterTest { @Test public void runClusterIntegrationTests() throws BlurException, TException, IOException, InterruptedException, KeeperException { + start("testCreateTable"); testCreateTable(); + start("testLoadTable"); testLoadTable(); + start("testForEmptySchema"); testForEmptySchema(); + start("testQueryWithSelector"); testQueryWithSelector(); + start("testQueryWithFacets"); testQueryWithFacets(); + start("testBatchFetch"); testBatchFetch(); + start("testLoadTable"); testQueryCancel(); + start("testBackPressureViaQuery"); testBackPressureViaQuery(); + start("testTestShardFailover"); testTestShardFailover(); + start("testTermsList"); testTermsList(); + start("testCreateDisableAndRemoveTable"); testCreateDisableAndRemoveTable(); + start("testCreateTableWithCustomType"); testCreateTableWithCustomType(); } + private void start(String name) { + System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<"); + System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<"); + System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<"); + System.out.println("Staring [" + name + "]"); + } + private void testForEmptySchema() throws BlurException, TException, IOException { Blur.Iface client = getClient(); Schema schema = client.schema("test"); @@ -379,6 +398,13 @@ public class BlurClusterTest { client.query("test", blurQueryRow); fail.set(true); } catch (BlurException e) { + System.out.println("-------------------"); + System.out.println("-------------------"); + System.out.println("-------------------"); + e.printStackTrace(); + System.out.println("-------------------"); + System.out.println("-------------------"); + System.out.println("-------------------"); error.set(e); } catch (TException e) { e.printStackTrace(); @@ -446,11 +472,6 @@ public class BlurClusterTest { public void testTestShardFailover() throws BlurException, TException, InterruptedException, IOException, KeeperException { - System.out.println("==========================="); - System.out.println("==========================="); - System.out.println("==========================="); - System.out.println("==========================="); - Iface client = getClient(); BlurQuery blurQuery = new BlurQuery(); blurQuery.setUseCacheIfPresent(false); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/da26c980/blur-util/src/main/resources/blur-default.properties ---------------------------------------------------------------------- diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties index 6cd6904..8f09e3b 100644 --- a/blur-util/src/main/resources/blur-default.properties +++ b/blur-util/src/main/resources/blur-default.properties @@ -167,7 +167,7 @@ blur.shard.internal.search.thread.count=8 blur.shard.warmup.thread.count=8 # The fetch count per Lucene search, this fetches pointers to hits -blur.shard.fetchcount=100 +blur.shard.fetchcount=110 # Heap limit on row fetch, once this limit has been reached the request will return blur.max.heap.per.row.fetch=10000000
