Updated Branches: refs/heads/master 2bf8cb446 -> 75e52f453
Fixing units, minicluster was broken because changes to the layout was not being picked up by the controller. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/75e52f45 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/75e52f45 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/75e52f45 Branch: refs/heads/master Commit: 75e52f453cfebafab62c01bb488197e927dba29d Parents: 26474c8 Author: Aaron McCurry <[email protected]> Authored: Thu Dec 12 10:25:35 2013 -0500 Committer: Aaron McCurry <[email protected]> Committed: Thu Dec 12 10:27:07 2013 -0500 ---------------------------------------------------------------------- .../clusterstatus/ZookeeperPathConstants.java | 8 ++ .../indexserver/DistributedLayoutFactory.java | 4 +- .../DistributedLayoutFactoryImpl.java | 4 +- .../MasterBasedDistributedLayoutFactory.java | 105 ++++++++++++++----- .../blur/thrift/BlurControllerServer.java | 56 +++------- .../test/java/org/apache/blur/MiniCluster.java | 33 ++++++ ...MasterBasedDistributedLayoutFactoryTest.java | 23 ++-- .../org/apache/blur/utils/BlurConstants.java | 1 + 8 files changed, 150 insertions(+), 84 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/75e52f45/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java index 6c15f17..408bc75 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java +++ b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java @@ -111,4 +111,12 @@ public class ZookeeperPathConstants { return getClusterPath(cluster) + "/layout"; } + public static String getShardLayoutPathTableLayout(String cluster) { + return getShardLayoutPath(cluster) + "/table_layout"; + } + + public static String getShardLayoutPathLocks(String cluster) { + return getShardLayoutPath(cluster) + "/locks"; + } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/75e52f45/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactory.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactory.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactory.java index d5b72bd..a1547e5 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactory.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactory.java @@ -21,6 +21,8 @@ import java.util.List; public interface DistributedLayoutFactory { DistributedLayout createDistributedLayout(String table, List<String> shardList, List<String> shardServerList, - List<String> offlineShardServers, boolean readOnly); + List<String> offlineShardServers); + + DistributedLayout readCurrentLayout(String table); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/75e52f45/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactoryImpl.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactoryImpl.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactoryImpl.java index 96b0771..6c06008 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactoryImpl.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactoryImpl.java @@ -19,7 +19,6 @@ package org.apache.blur.manager.indexserver; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_DISTRIBUTED_LAYOUT_FACTORY_CLASS; import org.apache.blur.BlurConfiguration; -import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants; import org.apache.zookeeper.ZooKeeper; public class DistributedLayoutFactoryImpl { @@ -28,8 +27,7 @@ public class DistributedLayoutFactoryImpl { ZooKeeper zooKeeper) { String distributedLayoutFactoryClass = configuration.get(BLUR_SHARD_DISTRIBUTED_LAYOUT_FACTORY_CLASS, ""); if (distributedLayoutFactoryClass.isEmpty()) { - String storagePath = ZookeeperPathConstants.getShardLayoutPath(cluster); - return new MasterBasedDistributedLayoutFactory(zooKeeper, storagePath); + return new MasterBasedDistributedLayoutFactory(zooKeeper, cluster); } try { Class<?> clazz = Class.forName(distributedLayoutFactoryClass); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/75e52f45/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java index 05189d3..eaa64c3 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java @@ -38,6 +38,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; +import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants; import org.apache.blur.zookeeper.ZkUtils; import org.apache.blur.zookeeper.ZooKeeperLockManager; import org.apache.zookeeper.CreateMode; @@ -48,8 +49,7 @@ import org.apache.zookeeper.data.Stat; public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFactory { - private static final String TABLE_LAYOUT = "table_layout"; - private static final String LOCKS = "locks"; + private static final String SEP = "_"; private static final Log LOG = LogFactory.getLog(MasterBasedDistributedLayoutFactory.class); private final ConcurrentMap<String, MasterBasedDistributedLayout> _cachedLayoutMap = new ConcurrentHashMap<String, MasterBasedDistributedLayout>(); @@ -65,25 +65,48 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac } }; - public MasterBasedDistributedLayoutFactory(ZooKeeper zooKeeper, String storagePath) { + public MasterBasedDistributedLayoutFactory(ZooKeeper zooKeeper, String cluster) { _zooKeeper = zooKeeper; - _storagePath = storagePath; - ZkUtils.mkNodes(_zooKeeper, _storagePath); - ZkUtils.mkNodes(_zooKeeper, _storagePath, LOCKS); - ZkUtils.mkNodes(_zooKeeper, _storagePath, TABLE_LAYOUT); - _tableStoragePath = _storagePath + "/" + TABLE_LAYOUT; - _locksStoragePath = _storagePath + "/" + LOCKS; + _storagePath = ZookeeperPathConstants.getShardLayoutPath(cluster); + _tableStoragePath = ZookeeperPathConstants.getShardLayoutPathTableLayout(cluster); + _locksStoragePath = ZookeeperPathConstants.getShardLayoutPathLocks(cluster); + ZkUtils.mkNodesStr(_zooKeeper, _storagePath); + ZkUtils.mkNodesStr(_zooKeeper, _tableStoragePath); + ZkUtils.mkNodesStr(_zooKeeper, _locksStoragePath); _zooKeeperLockManager = new ZooKeeperLockManager(_zooKeeper, _locksStoragePath); } @Override + public DistributedLayout readCurrentLayout(String table) { + LOG.info("Checking for existing layout for table [{0}]", table); + try { + String existingStoragePath = findExistingStoragePath(table); + if (existingStoragePath == null) { + return null; + } + Stat stat = _zooKeeper.exists(existingStoragePath, false); + if (stat != null) { + LOG.info("Existing layout found for table [{0}]", table); + byte[] data = _zooKeeper.getData(existingStoragePath, false, stat); + if (data != null) { + return fromBytes(data); + } + } + return null; + } catch (Exception e) { + LOG.error("Unknown error during layout read.", e); + throw new RuntimeException(e); + } + } + + @Override public DistributedLayout createDistributedLayout(String table, List<String> shardList, List<String> shardServerList, - List<String> offlineShardServers, boolean readOnly) { + List<String> offlineShardServers) { MasterBasedDistributedLayout layout = _cachedLayoutMap.get(table); List<String> onlineShardServerList = getOnlineShardServerList(shardServerList, offlineShardServers); if (layout == null || layout.isOutOfDate(shardList, onlineShardServerList)) { LOG.info("Layout out of date, recalculating for table [{0}].", table); - MasterBasedDistributedLayout newLayout = newLayout(table, shardList, onlineShardServerList, readOnly); + MasterBasedDistributedLayout newLayout = newLayout(table, shardList, onlineShardServerList); _cachedLayoutMap.put(table, newLayout); return newLayout; } else { @@ -98,16 +121,21 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac } private MasterBasedDistributedLayout newLayout(String table, List<String> onlineShardServerList, - List<String> shardServerList, boolean readOnly) throws LayoutMissingException { + List<String> shardServerList) { try { _zooKeeperLockManager.lock(table); - String storagePath = getStoragePath(table); LOG.info("Checking for existing layout for table [{0}]", table); - Stat stat = _zooKeeper.exists(storagePath, false); + String existingStoragePath = findExistingStoragePath(table); + Stat stat; + if (existingStoragePath == null) { + stat = null; + } else { + stat = _zooKeeper.exists(existingStoragePath, false); + } MasterBasedDistributedLayout existingLayout = null; if (stat != null) { LOG.info("Existing layout found for table [{0}]", table); - byte[] data = _zooKeeper.getData(storagePath, false, stat); + byte[] data = _zooKeeper.getData(existingStoragePath, false, stat); if (data != null) { MasterBasedDistributedLayout storedLayout = fromBytes(data); LOG.info("Checking if layout is out of date for table [{0}]", table); @@ -115,16 +143,10 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac LOG.info("Layout is up-to-date for table [{0}]", table); return storedLayout; } - if (readOnly) { - LOG.info("Using stable layout until update for table [{0}]", table); - return storedLayout; - } // If there was a stored layout, use the stored layout as a // replacement for the existing layout. existingLayout = storedLayout; } - } else if (readOnly) { - throw new LayoutMissingException(); } LOG.info("Calculating new layout for table [{0}]", table); // recreate @@ -133,11 +155,9 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac MasterBasedDistributedLayout layout = new MasterBasedDistributedLayout(newCalculatedLayout, onlineShardServerList, shardServerList); LOG.info("New layout created for table [{0}]", table); - if (_zooKeeper.exists(storagePath, false) == null) { - _zooKeeper.create(storagePath, toBytes(layout), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - } else { - _zooKeeper.setData(storagePath, toBytes(layout), -1); - } + String newPath = _zooKeeper.create(getStoragePath(table) + SEP, toBytes(layout), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT_SEQUENTIAL); + cleanupOldTableLayouts(table, newPath); return layout; } catch (LayoutMissingException e) { throw e; @@ -155,6 +175,38 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac } } + private void cleanupOldTableLayouts(String table, String newPath) throws KeeperException, InterruptedException { + List<String> children = new ArrayList<String>(_zooKeeper.getChildren(_tableStoragePath, false)); + for (String child : children) { + int index = child.lastIndexOf(SEP); + if (index >= 0) { + if (child.substring(0, index).equals(table)) { + String oldPath = _tableStoragePath + "/" + child; + if (!oldPath.equals(newPath)) { + LOG.info("Cleaning up old layouts for table [{0}]", table); + _zooKeeper.delete(oldPath, -1); + } + } + } + } + } + + private String findExistingStoragePath(String table) throws KeeperException, InterruptedException { + List<String> children = new ArrayList<String>(_zooKeeper.getChildren(_tableStoragePath, false)); + String path = null; + for (String child : children) { + int index = child.lastIndexOf(SEP); + if (index >= 0) { + if (child.substring(0, index).equals(table)) { + if (path == null || child.compareTo(path) > 0) { + path = child; + } + } + } + } + return _tableStoragePath + "/" + path; + } + private Map<String, String> calculateNewLayout(String table, MasterBasedDistributedLayout existingLayout, List<String> shardList, List<String> onlineShardServerList) { Set<String> shardServerSet = new TreeSet<String>(onlineShardServerList); @@ -292,4 +344,5 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac return false; } } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/75e52f45/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java index 97ea7de..b57d548 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java @@ -53,7 +53,6 @@ import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants; import org.apache.blur.manager.indexserver.DistributedLayout; import org.apache.blur.manager.indexserver.DistributedLayoutFactory; import org.apache.blur.manager.indexserver.DistributedLayoutFactoryImpl; -import org.apache.blur.manager.indexserver.LayoutMissingException; import org.apache.blur.manager.results.BlurResultIterable; import org.apache.blur.manager.results.BlurResultIterableClient; import org.apache.blur.manager.results.LazyBlurResult; @@ -89,7 +88,6 @@ import org.apache.blur.thrift.generated.User; import org.apache.blur.trace.Trace; import org.apache.blur.trace.Trace.TraceId; import org.apache.blur.trace.Tracer; -import org.apache.blur.utils.BlurConstants; import org.apache.blur.utils.BlurExecutorCompletionService; import org.apache.blur.utils.BlurIterator; import org.apache.blur.utils.BlurUtil; @@ -99,6 +97,7 @@ import org.apache.blur.utils.ForkJoin.ParallelCall; import org.apache.blur.zookeeper.WatchChildren; import org.apache.blur.zookeeper.WatchChildren.OnChange; import org.apache.blur.zookeeper.WatchNodeExistance; +import org.apache.blur.zookeeper.ZkUtils; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs.Ids; @@ -213,44 +212,32 @@ public class BlurControllerServer extends TableAdmin implements Iface { private void watchForLayoutChanges(final String cluster) throws KeeperException, InterruptedException { WatchNodeExistance we1 = new WatchNodeExistance(_zookeeper, ZookeeperPathConstants.getTablesPath(cluster)); + final String shardLayoutPathTableLayoutPath = ZookeeperPathConstants.getShardLayoutPathTableLayout(cluster); + ZkUtils.mkNodesStr(_zookeeper, shardLayoutPathTableLayoutPath); we1.watch(new WatchNodeExistance.OnChange() { @Override public void action(Stat stat) { if (stat != null) { - watch(cluster, ZookeeperPathConstants.getTablesPath(cluster), _watchForTablesPerCluster); + watch(cluster, shardLayoutPathTableLayoutPath, _watchForTablesPerCluster); } } }); if (_watchForTablesPerClusterExistance.putIfAbsent(cluster, we1) != null) { we1.close(); } - - WatchNodeExistance we2 = new WatchNodeExistance(_zookeeper, ZookeeperPathConstants.getTablesPath(cluster)); - we2.watch(new WatchNodeExistance.OnChange() { - @Override - public void action(Stat stat) { - if (stat != null) { - watch(cluster, ZookeeperPathConstants.getOnlineShardsPath(cluster), _watchForOnlineShardsPerCluster); - } - } - }); - if (_watchForOnlineShardsPerClusterExistance.putIfAbsent(cluster, we2) != null) { - we2.close(); - } } - private void watch(String cluster, String path, ConcurrentMap<String, WatchChildren> map) { - WatchChildren watchForTables = new WatchChildren(_zookeeper, path); - watchForTables.watch(new OnChange() { + private void watch(final String cluster, String path, ConcurrentMap<String, WatchChildren> map) { + WatchChildren watchForTableLayoutChanges = new WatchChildren(_zookeeper, path); + watchForTableLayoutChanges.watch(new OnChange() { @Override public void action(List<String> children) { - LOG.info("Layout change."); + LOG.info("Layout change for cluster [{0}].", cluster); updateLayout(); } }); - - if (map.putIfAbsent(cluster, watchForTables) != null) { - watchForTables.close(); + if (map.putIfAbsent(cluster, watchForTableLayoutChanges) != null) { + watchForTableLayoutChanges.close(); } } @@ -266,20 +253,14 @@ public class BlurControllerServer extends TableAdmin implements Iface { if (cluster == null) { continue; } - List<String> shardServerList = _clusterStatus.getShardServerList(cluster); - List<String> offlineShardServers = _clusterStatus.getOfflineShardServers(false, cluster); - List<String> shardList = getShardList(cluster, table); - DistributedLayoutFactory distributedLayoutFactory = getDistributedLayoutFactory(cluster); - try { - DistributedLayout layout = distributedLayoutFactory.createDistributedLayout(table, shardList, shardServerList, - offlineShardServers, true); + DistributedLayout layout = distributedLayoutFactory.readCurrentLayout(table); + if (layout != null) { Map<String, String> map = layout.getLayout(); LOG.info("New layout for table [{0}] is [{1}]", table, map); newLayout.put(table, map); - } catch (LayoutMissingException e) { + } else { LOG.info("Layout missing for table [{0}]", table); - continue; } } _shardServerLayout.set(newLayout); @@ -295,15 +276,6 @@ public class BlurControllerServer extends TableAdmin implements Iface { return distributedLayoutFactory; } - private List<String> getShardList(String cluster, String table) { - List<String> shards = new ArrayList<String>(); - TableDescriptor tableDescriptor = _clusterStatus.getTableDescriptor(true, cluster, table); - for (int i = 0; i < tableDescriptor.shardCount; i++) { - shards.add(BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, i)); - } - return shards; - } - private void registerMyself() { // Register Node try { @@ -954,7 +926,7 @@ public class BlurControllerServer extends TableAdmin implements Iface { int numberOfShards = getShardCount(table); Map<String, String> tableLayout = _shardServerLayout.get().get(table); - if (tableLayout.size() != numberOfShards) { + if (tableLayout == null || tableLayout.size() != numberOfShards) { throw new BException("Cannot update data while shard is missing"); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/75e52f45/blur-core/src/test/java/org/apache/blur/MiniCluster.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/MiniCluster.java b/blur-core/src/test/java/org/apache/blur/MiniCluster.java index 50d290a..58535cb 100644 --- a/blur-core/src/test/java/org/apache/blur/MiniCluster.java +++ b/blur-core/src/test/java/org/apache/blur/MiniCluster.java @@ -38,6 +38,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; @@ -127,6 +128,38 @@ public class MiniCluster { setupBuffers(); startControllers(controllerCount, randomPort); startShards(shardCount, randomPort); + try { + waitForSafeModeToExit(); + } catch (BlurException e) { + throw new RuntimeException(e); + } catch (TException e) { + throw new RuntimeException(e); + } + } + + private void waitForSafeModeToExit() throws BlurException, TException { + String controllerConnectionStr = getControllerConnectionStr(); + Iface client = BlurClient.getClient(controllerConnectionStr); + String clusterName = "default"; + boolean inSafeMode; + boolean isNoLonger = false; + do { + inSafeMode = client.isInSafeMode(clusterName); + if (!inSafeMode) { + if (isNoLonger) { + System.out.println("Cluster " + cluster + " is no longer in safemode."); + } else { + System.out.println("Cluster " + cluster + " is not in safemode."); + } + return; + } + isNoLonger = true; + try { + Thread.sleep(TimeUnit.SECONDS.toMillis(1)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } while (inSafeMode); } private void setupBuffers() { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/75e52f45/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactoryTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactoryTest.java b/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactoryTest.java index bebfcf9..b912b2e 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactoryTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactoryTest.java @@ -16,17 +16,18 @@ */ package org.apache.blur.manager.indexserver; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.TreeMap; import java.util.Map.Entry; +import java.util.TreeMap; import org.apache.blur.MiniCluster; +import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; @@ -45,7 +46,7 @@ public class MasterBasedDistributedLayoutFactoryTest { private static MiniCluster miniCluster; private ZooKeeper _zooKeeper; - private String storagePath = "/MasterBasedDistributedLayoutFactoryTest"; + private String cluster = "default"; @BeforeClass public static void startZooKeeper() throws IOException { @@ -67,7 +68,7 @@ public class MasterBasedDistributedLayoutFactoryTest { } }); - rmr(_zooKeeper, storagePath); + rmr(_zooKeeper, ZookeeperPathConstants.getShardLayoutPath(cluster)); } @After @@ -77,7 +78,7 @@ public class MasterBasedDistributedLayoutFactoryTest { @Test public void testDecreaseInServers() throws IOException, KeeperException, InterruptedException { - MasterBasedDistributedLayoutFactory factory = new MasterBasedDistributedLayoutFactory(_zooKeeper, storagePath); + MasterBasedDistributedLayoutFactory factory = new MasterBasedDistributedLayoutFactory(_zooKeeper, cluster); List<String> shardList = list("shard-0", "shard-1", "shard-2", "shard-3", "shard-4", "shard-5"); List<String> shardServerList = list("server-0", "server-1", "server-2", "server-3", "server-4", "server-5"); @@ -85,8 +86,7 @@ public class MasterBasedDistributedLayoutFactoryTest { String table = "t1"; - DistributedLayout layout1 = factory.createDistributedLayout(table, shardList, shardServerList, offlineShardServers, - false); + DistributedLayout layout1 = factory.createDistributedLayout(table, shardList, shardServerList, offlineShardServers); Map<String, String> expected1 = map(e("shard-0", "server-0"), e("shard-1", "server-1"), e("shard-2", "server-2"), e("shard-3", "server-3"), e("shard-4", "server-4"), e("shard-5", "server-5")); @@ -98,7 +98,7 @@ public class MasterBasedDistributedLayoutFactoryTest { List<String> newOfflineShardServers = list("server-4", "server-5"); DistributedLayout layout2 = factory.createDistributedLayout(table, shardList, newShardServerList, - newOfflineShardServers, false); + newOfflineShardServers); Map<String, String> expected2 = map(e("shard-0", "server-0"), e("shard-1", "server-1"), e("shard-2", "server-2"), e("shard-3", "server-3"), e("shard-4", "server-0"), e("shard-5", "server-1")); @@ -108,7 +108,7 @@ public class MasterBasedDistributedLayoutFactoryTest { @Test public void testIncreaseInServers() throws IOException, KeeperException, InterruptedException { - MasterBasedDistributedLayoutFactory factory = new MasterBasedDistributedLayoutFactory(_zooKeeper, storagePath); + MasterBasedDistributedLayoutFactory factory = new MasterBasedDistributedLayoutFactory(_zooKeeper, cluster); List<String> shardList = list("shard-0", "shard-1", "shard-2", "shard-3", "shard-4", "shard-5"); List<String> shardServerList = list("server-0", "server-1", "server-2", "server-3"); @@ -116,8 +116,7 @@ public class MasterBasedDistributedLayoutFactoryTest { String table = "t1"; - DistributedLayout layout1 = factory.createDistributedLayout(table, shardList, shardServerList, offlineShardServers, - false); + DistributedLayout layout1 = factory.createDistributedLayout(table, shardList, shardServerList, offlineShardServers); Map<String, String> expected1 = map(e("shard-0", "server-0"), e("shard-1", "server-1"), e("shard-2", "server-2"), e("shard-3", "server-3"), e("shard-4", "server-0"), e("shard-5", "server-1")); @@ -129,7 +128,7 @@ public class MasterBasedDistributedLayoutFactoryTest { List<String> newOfflineShardServers = list(); DistributedLayout layout2 = factory.createDistributedLayout(table, shardList, newShardServerList, - newOfflineShardServers, false); + newOfflineShardServers); Map<String, String> expected2 = map(e("shard-0", "server-4"), e("shard-1", "server-5"), e("shard-2", "server-2"), e("shard-3", "server-3"), e("shard-4", "server-0"), e("shard-5", "server-1")); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/75e52f45/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java index 221f96b..82ce116 100644 --- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java +++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java @@ -101,6 +101,7 @@ public class BlurConstants { public static final String BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE_PREFIX = "blur.shard.block.cache.v2.cacheBlockSize."; public static final String BLUR_SHARD_BLOCK_CACHE_V2_FILE_BUFFER_SIZE = "blur.shard.block.cache.v2.fileBufferSize"; public static final String BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE = "blur.shard.block.cache.v2.cacheBlockSize"; + public static final String BLUR_SHARD_BLURINDEX_CLASS = "blur.shard.blurindex.class"; public static final String BLUR_FIELDTYPE = "blur.fieldtype.";
