Fixed BLUR-260
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/a9cbc374 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/a9cbc374 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/a9cbc374 Branch: refs/heads/master Commit: a9cbc37425587ffac37c7085841cfed4e367d621 Parents: c378790 Author: Aaron McCurry <[email protected]> Authored: Wed Oct 9 13:33:53 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Wed Oct 9 13:38:08 2013 -0400 ---------------------------------------------------------------------- .../clusterstatus/ZookeeperPathConstants.java | 4 ++ .../DistributedLayoutFactoryImpl.java | 42 ++++++++++++++++++ .../MasterBasedDistributedLayoutFactory.java | 45 ++++++++++++-------- .../blur/thrift/BlurControllerServer.java | 38 +++++++++++++---- .../blur/thrift/ThriftBlurControllerServer.java | 1 + .../blur/thrift/ThriftBlurShardServer.java | 33 +++++++------- .../org/apache/blur/thrift/BlurClusterTest.java | 18 ++++++-- .../org/apache/blur/utils/BlurConstants.java | 1 + 8 files changed, 135 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a9cbc374/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java index 9ddf7d8..3e1f619 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java +++ b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java @@ -99,4 +99,8 @@ public class ZookeeperPathConstants { return getTablePath(cluster, table) + "/precache"; } + public static String getShardLayoutPath(String cluster) { + return getClusterPath(cluster) + "/layout"; + } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a9cbc374/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactoryImpl.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactoryImpl.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactoryImpl.java new file mode 100644 index 0000000..96b0771 --- /dev/null +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactoryImpl.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.manager.indexserver; + +import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_DISTRIBUTED_LAYOUT_FACTORY_CLASS; + +import org.apache.blur.BlurConfiguration; +import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants; +import org.apache.zookeeper.ZooKeeper; + +public class DistributedLayoutFactoryImpl { + + public static DistributedLayoutFactory getDistributedLayoutFactory(BlurConfiguration configuration, String cluster, + ZooKeeper zooKeeper) { + String distributedLayoutFactoryClass = configuration.get(BLUR_SHARD_DISTRIBUTED_LAYOUT_FACTORY_CLASS, ""); + if (distributedLayoutFactoryClass.isEmpty()) { + String storagePath = ZookeeperPathConstants.getShardLayoutPath(cluster); + return new MasterBasedDistributedLayoutFactory(zooKeeper, storagePath); + } + try { + Class<?> clazz = Class.forName(distributedLayoutFactoryClass); + return (DistributedLayoutFactory) clazz.newInstance(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a9cbc374/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java index b5fc5ab..e865da7 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.Serializable; +import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -72,9 +73,10 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac public DistributedLayout createDistributedLayout(String table, List<String> shardList, List<String> shardServerList, List<String> offlineShardServers) { MasterBasedDistributedLayout layout = _cachedLayoutMap.get(table); - if (layout == null || layout.isOutOfDate(shardList, shardServerList, offlineShardServers)) { + List<String> onlineShardServerList = getOnlineShardServerList(shardServerList, offlineShardServers); + if (layout == null || layout.isOutOfDate(shardList, onlineShardServerList)) { LOG.info("Layout out of date, recalculating for table [{0}].", table); - MasterBasedDistributedLayout newLayout = newLayout(table, shardList, shardServerList, offlineShardServers); + MasterBasedDistributedLayout newLayout = newLayout(table, shardList, onlineShardServerList); _cachedLayoutMap.put(table, newLayout); return newLayout; } else { @@ -82,8 +84,14 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac } } - private MasterBasedDistributedLayout newLayout(String table, List<String> shardList, List<String> shardServerList, - List<String> offlineShardServers) { + private List<String> getOnlineShardServerList(List<String> shardServerList, List<String> offlineShardServers) { + List<String> list = new ArrayList<String>(shardServerList); + list.removeAll(offlineShardServers); + return list; + } + + private MasterBasedDistributedLayout newLayout(String table, List<String> onlineShardServerList, + List<String> shardServerList) { try { _zooKeeperLockManager.lock(table); String storagePath = getStoragePath(table); @@ -93,7 +101,7 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac byte[] data = _zooKeeper.getData(storagePath, false, stat); if (data != null) { MasterBasedDistributedLayout storedLayout = fromBytes(data); - if (!storedLayout.isOutOfDate(shardList, shardServerList, offlineShardServers)) { + if (!storedLayout.isOutOfDate(onlineShardServerList, shardServerList)) { LOG.info("New layout fetched."); return storedLayout; } @@ -103,9 +111,10 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac } } // recreate - Map<String, String> newCalculatedLayout = calculateNewLayout(table, existingLayout, shardList, shardServerList); - MasterBasedDistributedLayout layout = new MasterBasedDistributedLayout(newCalculatedLayout, shardList, + Map<String, String> newCalculatedLayout = calculateNewLayout(table, existingLayout, onlineShardServerList, shardServerList); + MasterBasedDistributedLayout layout = new MasterBasedDistributedLayout(newCalculatedLayout, + onlineShardServerList, shardServerList); if (_zooKeeper.exists(storagePath, false) == null) { _zooKeeper.create(storagePath, toBytes(layout), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } else { @@ -127,16 +136,16 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac } private Map<String, String> calculateNewLayout(String table, MasterBasedDistributedLayout existingLayout, - List<String> shardList, List<String> shardServerList) { - Set<String> shardServerSet = new TreeSet<String>(shardServerList); + List<String> shardList, List<String> onlineShardServerList) { + Set<String> shardServerSet = new TreeSet<String>(onlineShardServerList); if (existingLayout == null) { // blind setup, basic round robin LOG.info("Blind shard layout."); Map<String, String> newLayoutMap = new TreeMap<String, String>(); - Iterator<String> iterator = shardServerList.iterator(); + Iterator<String> iterator = onlineShardServerList.iterator(); for (String shard : shardList) { if (!iterator.hasNext()) { - iterator = shardServerList.iterator(); + iterator = onlineShardServerList.iterator(); } String server = iterator.next(); newLayoutMap.put(shard, server); @@ -159,7 +168,7 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac LOG.info("Adding in new shard servers for table [{0}]", table); // Add counts for new shard servers - for (String server : shardServerList) { + for (String server : onlineShardServerList) { if (!onlineServerShardCount.containsKey(server)) { LOG.info("New shard server [{0}]", server); onlineServerShardCount.put(server, 0); @@ -179,7 +188,7 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac LOG.info("Leveling any shard hotspots for table [{0}]", table); // Level shards - MasterBasedLeveler.level(shardList.size(), shardServerList.size(), onlineServerShardCount, newLayoutMap); + MasterBasedLeveler.level(shardList.size(), onlineShardServerList.size(), onlineServerShardCount, newLayoutMap); return newLayoutMap; } } @@ -233,13 +242,13 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac static class MasterBasedDistributedLayout implements DistributedLayout, Serializable { private final SortedSet<String> _shardList; - private final SortedSet<String> _shardServerList; + private final SortedSet<String> _onlineShardServerList; private final Map<String, String> _layout; public MasterBasedDistributedLayout(Map<String, String> layout, Collection<String> shardList, - Collection<String> shardServerList) { + Collection<String> onlineShardServerList) { _shardList = new TreeSet<String>(shardList); - _shardServerList = new TreeSet<String>(shardServerList); + _onlineShardServerList = new TreeSet<String>(onlineShardServerList); _layout = layout; } @@ -248,8 +257,8 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac return _layout; } - public boolean isOutOfDate(List<String> shardList, List<String> shardServerList, List<String> offlineShardServers) { - if (!_shardServerList.equals(new TreeSet<String>(shardServerList))) { + public boolean isOutOfDate(List<String> shardList, List<String> onlineShardServerList) { + if (!_onlineShardServerList.equals(new TreeSet<String>(onlineShardServerList))) { return true; } else if (!_shardList.equals(new TreeSet<String>(shardList))) { return true; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a9cbc374/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java index b1b0bf7..1ec7c6a 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java @@ -24,10 +24,10 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; +import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -46,7 +46,9 @@ import org.apache.blur.manager.BlurPartitioner; import org.apache.blur.manager.BlurQueryChecker; import org.apache.blur.manager.IndexManager; import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants; -import org.apache.blur.manager.indexserver.DistributedLayoutManager; +import org.apache.blur.manager.indexserver.DistributedLayout; +import org.apache.blur.manager.indexserver.DistributedLayoutFactory; +import org.apache.blur.manager.indexserver.DistributedLayoutFactoryImpl; import org.apache.blur.manager.results.BlurResultIterable; import org.apache.blur.manager.results.BlurResultIterableClient; import org.apache.blur.manager.results.LazyBlurResult; @@ -125,6 +127,7 @@ public class BlurControllerServer extends TableAdmin implements Iface { private int _remoteFetchCount = 100; private BlurQueryChecker _queryChecker; private AtomicBoolean _running = new AtomicBoolean(); + private Map<String, DistributedLayoutFactory> _distributedLayoutFactoryMap = new ConcurrentHashMap<String, DistributedLayoutFactory>(); private int _maxFetchRetries = 3; private int _maxMutateRetries = 3; @@ -167,6 +170,11 @@ public class BlurControllerServer extends TableAdmin implements Iface { _watchForClusters.watch(new OnChange() { @Override public void action(List<String> children) { + for (String cluster : new HashSet<String>(_distributedLayoutFactoryMap.keySet())) { + if (!children.contains(cluster)) { + _distributedLayoutFactoryMap.remove(cluster); + } + } for (String cluster : children) { try { watchForLayoutChanges(cluster); @@ -233,7 +241,6 @@ public class BlurControllerServer extends TableAdmin implements Iface { List<String> tableList = _clusterStatus.getTableList(false); HashMap<String, Map<String, String>> newLayout = new HashMap<String, Map<String, String>>(); for (String table : tableList) { - DistributedLayoutManager layoutManager = new DistributedLayoutManager(); String cluster = _clusterStatus.getCluster(false, table); if (cluster == null) { continue; @@ -241,16 +248,27 @@ public class BlurControllerServer extends TableAdmin implements Iface { List<String> shardServerList = _clusterStatus.getShardServerList(cluster); List<String> offlineShardServers = _clusterStatus.getOfflineShardServers(false, cluster); List<String> shardList = getShardList(cluster, table); - layoutManager.setNodes(shardServerList); - layoutManager.setNodesOffline(offlineShardServers); - layoutManager.setShards(shardList); - layoutManager.init(); - Map<String, String> layout = layoutManager.getLayout(); - newLayout.put(table, layout); + + DistributedLayoutFactory distributedLayoutFactory = getDistributedLayoutFactory(cluster); + DistributedLayout layout = distributedLayoutFactory.createDistributedLayout(table, shardList, shardServerList, + offlineShardServers); + Map<String, String> map = layout.getLayout(); + LOG.info("New layout for table [{0}] is [{1}]", table, map); + newLayout.put(table, map); } _shardServerLayout.set(newLayout); } + private synchronized DistributedLayoutFactory getDistributedLayoutFactory(String cluster) { + DistributedLayoutFactory distributedLayoutFactory = _distributedLayoutFactoryMap.get(cluster); + if (distributedLayoutFactory == null) { + distributedLayoutFactory = DistributedLayoutFactoryImpl.getDistributedLayoutFactory(_configuration, cluster, + _zookeeper); + _distributedLayoutFactoryMap.put(cluster, distributedLayoutFactory); + } + return distributedLayoutFactory; + } + private List<String> getShardList(String cluster, String table) { List<String> shards = new ArrayList<String>(); TableDescriptor tableDescriptor = _clusterStatus.getTableDescriptor(true, cluster, table); @@ -360,6 +378,8 @@ public class BlurControllerServer extends TableAdmin implements Iface { BlurResults results = convertToBlurResults(hitsIterable, blurQuery, facetCounts, _executor, selector, table); if (!validResults(results, shardCount, blurQuery)) { BlurClientManager.sleep(_defaultDelay, _maxDefaultDelay, retries, _maxDefaultRetries); + Map<String, String> map = _shardServerLayout.get().get(table); + LOG.info("Current layout for table [{0}] is [{1}]", table, map); continue OUTER; } return results; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a9cbc374/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java index 347f1ca..d3eaf80 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java @@ -130,6 +130,7 @@ public class ThriftBlurControllerServer extends ThriftServer { controllerServer.setMaxMutateDelay(configuration.getInt(BLUR_CONTROLLER_RETRY_MAX_MUTATE_DELAY, 2000)); controllerServer.setMaxDefaultDelay(configuration.getInt(BLUR_CONTROLLER_RETRY_MAX_DEFAULT_DELAY, 2000)); controllerServer.setMaxRecordsPerRowFetchRequest(configuration.getInt(BLUR_MAX_RECORDS_PER_ROW_FETCH_REQUEST, 1000)); + controllerServer.setConfiguration(configuration); controllerServer.init(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a9cbc374/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java index 4f6bc43..82cf541 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java @@ -36,7 +36,6 @@ import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_FILTER_CACHE_CLASS; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_HOSTNAME; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WARMUP_CLASS; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WARMUP_THROTTLE; -import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INTERNAL_SEARCH_THREAD_COUNT; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_OPENER_THREAD_COUNT; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_SAFEMODEDELAY; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_SERVER_THRIFT_THREAD_COUNT; @@ -62,7 +61,6 @@ import org.apache.blur.manager.BlurFilterCache; import org.apache.blur.manager.BlurQueryChecker; import org.apache.blur.manager.DefaultBlurFilterCache; import org.apache.blur.manager.IndexManager; -import org.apache.blur.manager.clusterstatus.ClusterStatus; import org.apache.blur.manager.clusterstatus.ZookeeperClusterStatus; import org.apache.blur.manager.indexserver.BlurIndexWarmup; import org.apache.blur.manager.indexserver.BlurServerShutDown; @@ -70,6 +68,7 @@ import org.apache.blur.manager.indexserver.BlurServerShutDown.BlurShutdown; import org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup; import org.apache.blur.manager.indexserver.DistributedIndexServer; import org.apache.blur.manager.indexserver.DistributedLayoutFactory; +import org.apache.blur.manager.indexserver.DistributedLayoutFactoryImpl; import org.apache.blur.manager.writer.BlurIndexRefresher; import org.apache.blur.metrics.JSONReporter; import org.apache.blur.metrics.ReporterSetup; @@ -213,7 +212,8 @@ public class ThriftBlurShardServer extends ThriftServer { final ZooKeeper zooKeeper = ZkUtils.newZooKeeper(zkConnectionStr, sessionTimeout); - BlurUtil.setupZookeeper(zooKeeper, configuration.get(BLUR_CLUSTER_NAME)); + String cluster = configuration.get(BLUR_CLUSTER_NAME, BLUR_CLUSTER); + BlurUtil.setupZookeeper(zooKeeper, cluster); final ZookeeperClusterStatus clusterStatus = new ZookeeperClusterStatus(zooKeeper, configuration); @@ -222,18 +222,18 @@ public class ThriftBlurShardServer extends ThriftServer { BlurFilterCache filterCache = getFilterCache(configuration); BlurIndexWarmup indexWarmup = getIndexWarmup(configuration); - //@todo add in read from config - DistributedLayoutFactory distributedLayoutFactory = null; - String cluster = configuration.get(BLUR_CLUSTER_NAME, BLUR_CLUSTER); + DistributedLayoutFactory distributedLayoutFactory = DistributedLayoutFactoryImpl.getDistributedLayoutFactory( + configuration, cluster, zooKeeper); + long safeModeDelay = configuration.getLong(BLUR_SHARD_SAFEMODEDELAY, 60000); int shardOpenerThreadCount = configuration.getInt(BLUR_SHARD_OPENER_THREAD_COUNT, 16); int internalSearchThreads = configuration.getInt(BLUR_SHARD_WARMUP_THREAD_COUNT, 16); int warmupThreads = configuration.getInt(BLUR_SHARD_WARMUP_THREAD_COUNT, 16); - + final DistributedIndexServer indexServer = new DistributedIndexServer(config, zooKeeper, clusterStatus, - indexWarmup, filterCache, blockCacheDirectoryFactory, distributedLayoutFactory, cluster, nodeName, safeModeDelay, shardOpenerThreadCount, - internalSearchThreads, warmupThreads); + indexWarmup, filterCache, blockCacheDirectoryFactory, distributedLayoutFactory, cluster, nodeName, + safeModeDelay, shardOpenerThreadCount, internalSearchThreads, warmupThreads); final IndexManager indexManager = new IndexManager(); indexManager.setIndexServer(indexServer); @@ -254,6 +254,7 @@ public class ThriftBlurShardServer extends ThriftServer { shardServer.setQueryChecker(queryChecker); shardServer.setConfiguration(configuration); shardServer.setMaxRecordsPerRowFetchRequest(configuration.getInt(BLUR_MAX_RECORDS_PER_ROW_FETCH_REQUEST, 1000)); + shardServer.setConfiguration(configuration); shardServer.init(); Iface iface = BlurUtil.recordMethodCallsAndAverageTimes(shardServer, Iface.class, false); @@ -308,10 +309,10 @@ public class ThriftBlurShardServer extends ThriftServer { } private static BlurFilterCache getFilterCache(BlurConfiguration configuration) { - String _blurFilterCacheClass = configuration.get(BLUR_SHARD_FILTER_CACHE_CLASS); - if (_blurFilterCacheClass != null) { + String blurFilterCacheClass = configuration.get(BLUR_SHARD_FILTER_CACHE_CLASS); + if (blurFilterCacheClass != null) { try { - Class<?> clazz = Class.forName(_blurFilterCacheClass); + Class<?> clazz = Class.forName(blurFilterCacheClass); return (BlurFilterCache) clazz.newInstance(); } catch (Exception e) { throw new RuntimeException(e); @@ -321,11 +322,11 @@ public class ThriftBlurShardServer extends ThriftServer { } private static BlurIndexWarmup getIndexWarmup(BlurConfiguration configuration) { - String _blurFilterCacheClass = configuration.get(BLUR_SHARD_INDEX_WARMUP_CLASS); - if (_blurFilterCacheClass != null && _blurFilterCacheClass.isEmpty()) { - if (!_blurFilterCacheClass.equals("org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup")) { + String blurFilterCacheClass = configuration.get(BLUR_SHARD_INDEX_WARMUP_CLASS); + if (blurFilterCacheClass != null && blurFilterCacheClass.isEmpty()) { + if (!blurFilterCacheClass.equals("org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup")) { try { - Class<?> clazz = Class.forName(_blurFilterCacheClass); + Class<?> clazz = Class.forName(blurFilterCacheClass); return (BlurIndexWarmup) clazz.newInstance(); } catch (Exception e) { throw new RuntimeException(e); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a9cbc374/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java index 39b55bc..7fba16b 100644 --- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java +++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java @@ -293,11 +293,15 @@ public class BlurClusterTest { if (blurException != null) { return blurException; } - Thread.sleep(250); + Thread.sleep(100); if (bufferToPutGcWatcherOverLimitList != null) { - System.out.println("Allocating [" + sizeToAllocate + "] Heap [" + getHeapSize() + "] Max [" + getMaxHeapSize() - + "]"); - bufferToPutGcWatcherOverLimitList.add(new byte[sizeToAllocate]); + if (getHeapSize() < (getMaxHeapSize() * 0.8)) { + System.out.println("Allocating [" + sizeToAllocate + "] Heap [" + getHeapSize() + "] Max [" + + getMaxHeapSize() + "]"); + bufferToPutGcWatcherOverLimitList.add(new byte[sizeToAllocate]); + } else { + System.out.println("Already allocated enough Heap [" + getHeapSize() + "] Max [" + getMaxHeapSize() + "]"); + } } } return null; @@ -313,6 +317,12 @@ public class BlurClusterTest { public void testTestShardFailover() throws BlurException, TException, InterruptedException, IOException, KeeperException { + + System.out.println("==========================="); + System.out.println("==========================="); + System.out.println("==========================="); + System.out.println("==========================="); + Iface client = getClient(); int length = 100; BlurQuery blurQuery = new BlurQuery(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a9cbc374/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java index 2176ec6..1373ad2 100644 --- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java +++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java @@ -75,6 +75,7 @@ public class BlurConstants { public static final String BLUR_SHARD_THRIFT_SELECTOR_THREADS = "blur.shard.thrift.selector.threads"; public static final String BLUR_SHARD_THRIFT_MAX_READ_BUFFER_BYTES = "blur.shard.thrift.max.read.buffer.bytes"; public static final String BLUR_SHARD_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD = "blur.shard.thrift.accept.queue.size.per.thread"; + public static final String BLUR_SHARD_DISTRIBUTED_LAYOUT_FACTORY_CLASS = "blur.shard.distributed.layout.factory.class"; public static final String BLUR_FIELDTYPE = "blur.fieldtype.";
