Updated Branches: refs/heads/apache-blur-0.2 a7e672574 -> 911fee9a7
Fixed BLUR-302 Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/911fee9a Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/911fee9a Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/911fee9a Branch: refs/heads/apache-blur-0.2 Commit: 911fee9a74f1a73b47bf7f96b74cd7fe8392c782 Parents: a7e6725 Author: Aaron McCurry <[email protected]> Authored: Tue Dec 10 08:37:07 2013 -0500 Committer: Aaron McCurry <[email protected]> Committed: Tue Dec 10 08:37:07 2013 -0500 ---------------------------------------------------------------------- .../indexserver/DistributedIndexServer.java | 4 ++-- .../indexserver/DistributedLayoutFactory.java | 2 +- .../indexserver/LayoutMissingException.java | 21 ++++++++++++++++++++ .../MasterBasedDistributedLayoutFactory.java | 14 ++++++++++--- .../blur/thrift/BlurControllerServer.java | 16 ++++++++++----- ...MasterBasedDistributedLayoutFactoryTest.java | 12 ++++++----- 6 files changed, 53 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/911fee9a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java index a0b83de..09019ad 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java @@ -191,7 +191,7 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer { @Override public DistributedLayout createDistributedLayout(String table, List<String> shardList, - List<String> shardServerList, List<String> offlineShardServers) { + List<String> shardServerList, List<String> offlineShardServers, boolean readOnly) { DistributedLayoutManager layoutManager = new DistributedLayoutManager(); layoutManager.setNodes(shardServerList); layoutManager.setNodesOffline(offlineShardServers); @@ -590,7 +590,7 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer { List<String> shardList = getShardList(table); DistributedLayout layoutManager = _distributedLayoutFactory.createDistributedLayout(table, shardList, - shardServerList, offlineShardServers); + shardServerList, offlineShardServers, false); Map<String, String> layout = layoutManager.getLayout(); String nodeName = getNodeName(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/911fee9a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactory.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactory.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactory.java index cb211e2..d5b72bd 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactory.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactory.java @@ -21,6 +21,6 @@ import java.util.List; public interface DistributedLayoutFactory { DistributedLayout createDistributedLayout(String table, List<String> shardList, List<String> shardServerList, - List<String> offlineShardServers); + List<String> offlineShardServers, boolean readOnly); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/911fee9a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LayoutMissingException.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LayoutMissingException.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LayoutMissingException.java new file mode 100644 index 0000000..20ef8cf --- /dev/null +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LayoutMissingException.java @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.manager.indexserver; + +public class LayoutMissingException extends RuntimeException { + private static final long serialVersionUID = 3979582465356840074L; +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/911fee9a/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java index b9faf54..05189d3 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java @@ -78,12 +78,12 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac @Override public DistributedLayout createDistributedLayout(String table, List<String> shardList, List<String> shardServerList, - List<String> offlineShardServers) { + List<String> offlineShardServers, boolean readOnly) { MasterBasedDistributedLayout layout = _cachedLayoutMap.get(table); List<String> onlineShardServerList = getOnlineShardServerList(shardServerList, offlineShardServers); if (layout == null || layout.isOutOfDate(shardList, onlineShardServerList)) { LOG.info("Layout out of date, recalculating for table [{0}].", table); - MasterBasedDistributedLayout newLayout = newLayout(table, shardList, onlineShardServerList); + MasterBasedDistributedLayout newLayout = newLayout(table, shardList, onlineShardServerList, readOnly); _cachedLayoutMap.put(table, newLayout); return newLayout; } else { @@ -98,7 +98,7 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac } private MasterBasedDistributedLayout newLayout(String table, List<String> onlineShardServerList, - List<String> shardServerList) { + List<String> shardServerList, boolean readOnly) throws LayoutMissingException { try { _zooKeeperLockManager.lock(table); String storagePath = getStoragePath(table); @@ -115,10 +115,16 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac LOG.info("Layout is up-to-date for table [{0}]", table); return storedLayout; } + if (readOnly) { + LOG.info("Using stable layout until update for table [{0}]", table); + return storedLayout; + } // If there was a stored layout, use the stored layout as a // replacement for the existing layout. existingLayout = storedLayout; } + } else if (readOnly) { + throw new LayoutMissingException(); } LOG.info("Calculating new layout for table [{0}]", table); // recreate @@ -133,6 +139,8 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac _zooKeeper.setData(storagePath, toBytes(layout), -1); } return layout; + } catch (LayoutMissingException e) { + throw e; } catch (Exception e) { LOG.error("Unknown error during layout update.", e); throw new RuntimeException(e); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/911fee9a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java index 902df6f..b547f10 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java @@ -53,6 +53,7 @@ import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants; import org.apache.blur.manager.indexserver.DistributedLayout; import org.apache.blur.manager.indexserver.DistributedLayoutFactory; import org.apache.blur.manager.indexserver.DistributedLayoutFactoryImpl; +import org.apache.blur.manager.indexserver.LayoutMissingException; import org.apache.blur.manager.results.BlurResultIterable; import org.apache.blur.manager.results.BlurResultIterableClient; import org.apache.blur.manager.results.LazyBlurResult; @@ -270,11 +271,16 @@ public class BlurControllerServer extends TableAdmin implements Iface { List<String> shardList = getShardList(cluster, table); DistributedLayoutFactory distributedLayoutFactory = getDistributedLayoutFactory(cluster); - DistributedLayout layout = distributedLayoutFactory.createDistributedLayout(table, shardList, shardServerList, - offlineShardServers); - Map<String, String> map = layout.getLayout(); - LOG.info("New layout for table [{0}] is [{1}]", table, map); - newLayout.put(table, map); + try { + DistributedLayout layout = distributedLayoutFactory.createDistributedLayout(table, shardList, shardServerList, + offlineShardServers, true); + Map<String, String> map = layout.getLayout(); + LOG.info("New layout for table [{0}] is [{1}]", table, map); + newLayout.put(table, map); + } catch (LayoutMissingException e) { + LOG.info("Layout missing for table [{0}]", table); + continue; + } } _shardServerLayout.set(newLayout); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/911fee9a/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactoryTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactoryTest.java b/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactoryTest.java index 87b95ad..bebfcf9 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactoryTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactoryTest.java @@ -69,7 +69,7 @@ public class MasterBasedDistributedLayoutFactoryTest { }); rmr(_zooKeeper, storagePath); } - + @After public void teardown() throws InterruptedException { _zooKeeper.close(); @@ -85,7 +85,8 @@ public class MasterBasedDistributedLayoutFactoryTest { String table = "t1"; - DistributedLayout layout1 = factory.createDistributedLayout(table, shardList, shardServerList, offlineShardServers); + DistributedLayout layout1 = factory.createDistributedLayout(table, shardList, shardServerList, offlineShardServers, + false); Map<String, String> expected1 = map(e("shard-0", "server-0"), e("shard-1", "server-1"), e("shard-2", "server-2"), e("shard-3", "server-3"), e("shard-4", "server-4"), e("shard-5", "server-5")); @@ -97,7 +98,7 @@ public class MasterBasedDistributedLayoutFactoryTest { List<String> newOfflineShardServers = list("server-4", "server-5"); DistributedLayout layout2 = factory.createDistributedLayout(table, shardList, newShardServerList, - newOfflineShardServers); + newOfflineShardServers, false); Map<String, String> expected2 = map(e("shard-0", "server-0"), e("shard-1", "server-1"), e("shard-2", "server-2"), e("shard-3", "server-3"), e("shard-4", "server-0"), e("shard-5", "server-1")); @@ -115,7 +116,8 @@ public class MasterBasedDistributedLayoutFactoryTest { String table = "t1"; - DistributedLayout layout1 = factory.createDistributedLayout(table, shardList, shardServerList, offlineShardServers); + DistributedLayout layout1 = factory.createDistributedLayout(table, shardList, shardServerList, offlineShardServers, + false); Map<String, String> expected1 = map(e("shard-0", "server-0"), e("shard-1", "server-1"), e("shard-2", "server-2"), e("shard-3", "server-3"), e("shard-4", "server-0"), e("shard-5", "server-1")); @@ -127,7 +129,7 @@ public class MasterBasedDistributedLayoutFactoryTest { List<String> newOfflineShardServers = list(); DistributedLayout layout2 = factory.createDistributedLayout(table, shardList, newShardServerList, - newOfflineShardServers); + newOfflineShardServers, false); Map<String, String> expected2 = map(e("shard-0", "server-4"), e("shard-1", "server-5"), e("shard-2", "server-2"), e("shard-3", "server-3"), e("shard-4", "server-0"), e("shard-5", "server-1"));
