Updated Branches: refs/heads/0.2-dev 57fd000d4 -> 2ef1c1ce5
Added many logging statements for debugging and library updates. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/2ef1c1ce Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/2ef1c1ce Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/2ef1c1ce Branch: refs/heads/0.2-dev Commit: 2ef1c1ce53f4dfb17887879594e96ad688ade003 Parents: 57fd000 Author: Aaron McCurry <[email protected]> Authored: Sun Jan 13 20:45:13 2013 -0500 Committer: Aaron McCurry <[email protected]> Committed: Sun Jan 13 20:45:13 2013 -0500 ---------------------------------------------------------------------- src/blur-core/pom.xml | 4 +- .../manager/clusterstatus/BaseClusterStatus.java | 201 +++++++++++++++ .../clusterstatus/SimpleZKClusterStatus.java | 197 +------------- .../indexserver/DistributedIndexServer.java | 28 ++- .../java/org/apache/blur/thrift/TableContext.java | 2 + .../org/apache/blur/thrift/ThriftBlurServer.java | 7 +- src/blur-mapred/pom.xml | 2 +- src/blur-testsuite/pom.xml | 2 +- src/blur-util/pom.xml | 4 +- src/distribution/pom.xml | 5 + src/distribution/src/main/scripts/bin/blur | 12 +- .../src/main/scripts/bin/stop-server.sh | 20 ++- 12 files changed, 280 insertions(+), 204 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2ef1c1ce/src/blur-core/pom.xml ---------------------------------------------------------------------- diff --git a/src/blur-core/pom.xml b/src/blur-core/pom.xml index 2eb20d5..e99b8b5 100644 --- a/src/blur-core/pom.xml +++ b/src/blur-core/pom.xml @@ -66,13 +66,13 @@ under the License. <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> - <version>3.4.4</version> + <version>3.4.5</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-test</artifactId> - <version>1.0.3</version> + <version>1.0.4</version> <scope>test</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2ef1c1ce/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/BaseClusterStatus.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/BaseClusterStatus.java b/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/BaseClusterStatus.java new file mode 100644 index 0000000..7695dde --- /dev/null +++ b/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/BaseClusterStatus.java @@ -0,0 +1,201 @@ +package org.apache.blur.manager.clusterstatus; + +import java.io.IOException; +import java.util.List; + +import org.apache.blur.analysis.BlurAnalyzer; +import org.apache.blur.log.Log; +import org.apache.blur.log.LogFactory; +import org.apache.blur.thrift.generated.Analyzer; +import org.apache.blur.thrift.generated.TableDescriptor; +import org.apache.blur.utils.BlurUtil; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.Stat; + +public abstract class BaseClusterStatus extends ClusterStatus { + + private static final Log LOG = LogFactory.getLog(SimpleZKClusterStatus.class); + + public abstract ZooKeeper getZooKeeper(); + + + private byte[] getData(String path) throws KeeperException, InterruptedException { + Stat stat = getZooKeeper().exists(path, false); + if (stat == null) { + return null; + } + return getZooKeeper().getData(path, false, stat); + } + + @Override + public TableDescriptor getTableDescriptor(boolean useCache, String table) { + TableDescriptor tableDescriptor = new TableDescriptor(); + try { + String blurTablePath = ZookeeperPathConstants.getTablePath(getClusterName(), table); + byte[] data = getData(blurTablePath); + BlurUtil.write(data, tableDescriptor); + } catch (KeeperException e) { + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return tableDescriptor; + } + + @Override + public void createTable(TableDescriptor tableDescriptor) { + try { + String table = BlurUtil.nullCheck(tableDescriptor.getName(), "Name cannot be null."); + String uri = BlurUtil.nullCheck(tableDescriptor.getStoragePath(), "Storage path cannot be null."); + int shardCount = BlurUtil.zeroCheck(tableDescriptor.shardCount, "ShardCount cannot be less than 1"); + String tablePath = ZookeeperPathConstants.getTablePath(getClusterName(), table); + + Analyzer analyzer = tableDescriptor.getAnalyzer(); + if (analyzer != null) { + // check the analyzer to be valid + new BlurAnalyzer(analyzer); + } + + if (getZooKeeper().exists(tablePath, false) != null) { + throw new IOException("Table [" + table + "] already exists."); + } + BlurUtil.setupFileSystem(uri, shardCount); + BlurUtil.createPath(getZooKeeper(), tablePath, BlurUtil.read(tableDescriptor)); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (KeeperException e) { + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + @Override + public void disableTable(String table) throws IOException { + try { + String tablePath = ZookeeperPathConstants.getTablePath(getClusterName(), table); + if (getZooKeeper().exists(tablePath, false) == null) { + throw new IOException("Table [" + table + "] does not exist."); + } + TableDescriptor tableDescriptor = getTableDescriptor(false, table); + if (!tableDescriptor.isEnabled()) { + throw new IOException("Table [" + table + "] is already disabled."); + } + tableDescriptor.setEnabled(false); + getZooKeeper().setData(tablePath, BlurUtil.read(tableDescriptor), -1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } catch (KeeperException e) { + throw new RuntimeException(e); + } + } + + @Override + public void enableTable(String table) { + try { + String tablePath = ZookeeperPathConstants.getTablePath(getClusterName(), table); + if (getZooKeeper().exists(tablePath, false) == null) { + throw new IOException("Table [" + table + "] does not exist."); + } + TableDescriptor tableDescriptor = getTableDescriptor(false, table); + if (tableDescriptor.isEnabled()) { + LOG.info("Table [" + table + "] is already enabled."); + } + tableDescriptor.setEnabled(true); + getZooKeeper().setData(tablePath, BlurUtil.read(tableDescriptor), -1); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } catch (KeeperException e) { + throw new RuntimeException(e); + } + } + + @Override + public void removeTable(String table, boolean deleteIndexFiles) { + try { + String tablePath = ZookeeperPathConstants.getTablePath(getClusterName(), table); + if (getZooKeeper().exists(tablePath, false) == null) { + throw new IOException("Table [" + table + "] does not exist."); + } + TableDescriptor tableDescriptor = getTableDescriptor(false, table); + if (tableDescriptor.isEnabled()) { + throw new IOException("Table [" + table + "] is NOT disabled."); + } + getZooKeeper().delete(tablePath, -1); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } catch (KeeperException e) { + throw new RuntimeException(e); + } + } + + @Override + public List<String> getOnlineServers(boolean useCache) { + try { + return getZooKeeper().getChildren(ZookeeperPathConstants.getOnlineServersPath(getClusterName()), false); + } catch (KeeperException e) { + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + @Override + public List<String> getServerList(boolean useCache) { + try { + return getZooKeeper().getChildren(ZookeeperPathConstants.getRegisteredServersPath(getClusterName()), false); + } catch (KeeperException e) { + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean exists(boolean useCache, String table) { + List<String> tableList = getTableList(useCache); + return tableList.contains(table); + } + + @Override + public boolean isInSafeMode(boolean useCache) { + try { + String blurSafemodePath = ZookeeperPathConstants.getSafemodePath(getClusterName()); + Stat stat = getZooKeeper().exists(blurSafemodePath, false); + if (stat == null) { + return false; + } + byte[] data = getZooKeeper().getData(blurSafemodePath, false, stat); + if (data == null) { + return false; + } + long timestamp = Long.parseLong(new String(data)); + long waitTime = timestamp - System.currentTimeMillis(); + if (waitTime > 0) { + return true; + } + return false; + } catch (KeeperException e) { + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + @Override + public List<String> getTableList(boolean useCache) { + try { + return getZooKeeper().getChildren(ZookeeperPathConstants.getTablesPath(getClusterName()), false); + } catch (KeeperException e) { + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2ef1c1ce/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/SimpleZKClusterStatus.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/SimpleZKClusterStatus.java b/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/SimpleZKClusterStatus.java index 541e897..e9a021a 100644 --- a/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/SimpleZKClusterStatus.java +++ b/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/SimpleZKClusterStatus.java @@ -1,48 +1,21 @@ package org.apache.blur.manager.clusterstatus; -import java.io.IOException; -import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.blur.analysis.BlurAnalyzer; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; -import org.apache.blur.thrift.generated.Analyzer; -import org.apache.blur.thrift.generated.TableDescriptor; -import org.apache.blur.utils.BlurUtil; -import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.data.Stat; -public class SimpleZKClusterStatus extends ClusterStatus { +public class SimpleZKClusterStatus extends BaseClusterStatus { private static final Log LOG = LogFactory.getLog(SimpleZKClusterStatus.class); private ZooKeeper _zk; private String _cluster; + private AtomicBoolean _running = new AtomicBoolean(); public SimpleZKClusterStatus(String cluster, ZooKeeper zooKeeper) { _cluster = cluster; _zk = zooKeeper; - } - - @Override - public List<String> getOnlineServers(boolean useCache) { - try { - return _zk.getChildren(ZookeeperPathConstants.getOnlineServersPath(_cluster), false); - } catch (KeeperException e) { - throw new RuntimeException(e); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - @Override - public List<String> getServerList(boolean useCache) { - try { - return _zk.getChildren(ZookeeperPathConstants.getRegisteredServersPath(_cluster), false); - } catch (KeeperException e) { - throw new RuntimeException(e); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + _running.set(true); } @Override @@ -51,169 +24,23 @@ public class SimpleZKClusterStatus extends ClusterStatus { } @Override - public boolean exists(boolean useCache, String table) { - List<String> tableList = getTableList(useCache); - return tableList.contains(table); - } - - @Override - public boolean isInSafeMode(boolean useCache) { - try { - String blurSafemodePath = ZookeeperPathConstants.getSafemodePath(_cluster); - Stat stat = _zk.exists(blurSafemodePath, false); - if (stat == null) { - return false; - } - byte[] data = _zk.getData(blurSafemodePath, false, stat); - if (data == null) { - return false; - } - long timestamp = Long.parseLong(new String(data)); - long waitTime = timestamp - System.currentTimeMillis(); - if (waitTime > 0) { - return true; - } - return false; - } catch (KeeperException e) { - throw new RuntimeException(e); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - @Override - public List<String> getTableList(boolean useCache) { - try { - return _zk.getChildren(ZookeeperPathConstants.getTablesPath(_cluster), false); - } catch (KeeperException e) { - throw new RuntimeException(e); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - @Override - public TableDescriptor getTableDescriptor(boolean useCache, String table) { - TableDescriptor tableDescriptor = new TableDescriptor(); - try { - String blurTablePath = ZookeeperPathConstants.getTablePath(getClusterName(), table); - byte[] data = getData(blurTablePath); - BlurUtil.write(data, tableDescriptor); - } catch (KeeperException e) { - throw new RuntimeException(e); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - return tableDescriptor; - } - - @Override - public void createTable(TableDescriptor tableDescriptor) { - try { - String table = BlurUtil.nullCheck(tableDescriptor.getName(), "Name cannot be null."); - String uri = BlurUtil.nullCheck(tableDescriptor.getStoragePath(), "Storage path cannot be null."); - int shardCount = BlurUtil.zeroCheck(tableDescriptor.shardCount, "ShardCount cannot be less than 1"); - String tablePath = ZookeeperPathConstants.getTablePath(getClusterName(), table); - - Analyzer analyzer = tableDescriptor.getAnalyzer(); - if (analyzer != null) { - // check the analyzer to be valid - new BlurAnalyzer(analyzer); - } - - if (_zk.exists(tablePath, false) != null) { - throw new IOException("Table [" + table + "] already exists."); - } - BlurUtil.setupFileSystem(uri, shardCount); - BlurUtil.createPath(_zk, tablePath, BlurUtil.read(tableDescriptor)); - } catch (IOException e) { - throw new RuntimeException(e); - } catch (KeeperException e) { - throw new RuntimeException(e); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - @Override - public void disableTable(String table) throws IOException { - try { - String tablePath = ZookeeperPathConstants.getTablePath(_cluster, table); - if (_zk.exists(tablePath, false) == null) { - throw new IOException("Table [" + table + "] does not exist."); - } - TableDescriptor tableDescriptor = getTableDescriptor(false, table); - if (!tableDescriptor.isEnabled()) { - throw new IOException("Table [" + table + "] is already disabled."); - } - tableDescriptor.setEnabled(false); - _zk.setData(tablePath, BlurUtil.read(tableDescriptor), -1); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } catch (KeeperException e) { - throw new RuntimeException(e); - } - } - - @Override - public void enableTable(String table) { - try { - String tablePath = ZookeeperPathConstants.getTablePath(_cluster, table); - if (_zk.exists(tablePath, false) == null) { - throw new IOException("Table [" + table + "] does not exist."); - } - TableDescriptor tableDescriptor = getTableDescriptor(false, table); - if (tableDescriptor.isEnabled()) { - LOG.info("Table [" + table + "] is already enabled."); - } - tableDescriptor.setEnabled(true); - _zk.setData(tablePath, BlurUtil.read(tableDescriptor), -1); - } catch (IOException e) { - throw new RuntimeException(e); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } catch (KeeperException e) { - throw new RuntimeException(e); - } + public boolean isOpen() { + return _running.get(); } @Override - public void removeTable(String table, boolean deleteIndexFiles) { + public void close() { + _running.set(false); try { - String tablePath = ZookeeperPathConstants.getTablePath(_cluster, table); - if (_zk.exists(tablePath, false) == null) { - throw new IOException("Table [" + table + "] does not exist."); - } - TableDescriptor tableDescriptor = getTableDescriptor(false, table); - if (tableDescriptor.isEnabled()) { - throw new IOException("Table [" + table + "] is NOT disabled."); - } - _zk.delete(tablePath, -1); - } catch (IOException e) { - throw new RuntimeException(e); + _zk.close(); } catch (InterruptedException e) { - throw new RuntimeException(e); - } catch (KeeperException e) { - throw new RuntimeException(e); + LOG.error("Unknwn error during closing.", e); } } @Override - public boolean isOpen() { - return true; - } - - @Override - public void close() { - - } - - private byte[] getData(String path) throws KeeperException, InterruptedException { - Stat stat = _zk.exists(path, false); - if (stat == null) { - return null; - } - return _zk.getData(path, false, stat); + public ZooKeeper getZooKeeper() { + return _zk; } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2ef1c1ce/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java b/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java index 3e1082b..4f2b25d 100644 --- a/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java +++ b/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java @@ -102,7 +102,7 @@ public class DistributedIndexServer extends AbstractIndexServer { // set internally private Timer _timerCacheFlush; private Timer _timerTableWarmer; - + private ExecutorService _openerService; private BlurIndexCloser _closer; private BlurFilterCache _filterCache; @@ -134,7 +134,7 @@ public class DistributedIndexServer extends AbstractIndexServer { waitInSafeModeIfNeeded(); _running.set(true); setupTableWarmer(); -// watchForShardServerChanges(); + watchForShardServerChanges(); } private void watchForShardServerChanges() { @@ -221,6 +221,7 @@ public class DistributedIndexServer extends AbstractIndexServer { private void warmup() { if (_running.get()) { + LOG.debug("Running warmup."); List<String> tableList = _clusterStatus.getTableList(false); _blurMetrics.tableCount.set(tableList.size()); long indexCount = 0; @@ -228,7 +229,9 @@ public class DistributedIndexServer extends AbstractIndexServer { AtomicLong indexMemoryUsage = new AtomicLong(); for (String table : tableList) { try { + LOG.debug("Running warmup for table [{0}].", table); Map<String, BlurIndex> indexes = getIndexes(table); + LOG.debug("Indexes fetched for table [{0}].", table); int count = indexes.size(); indexCount += count; updateMetrics(_blurMetrics, indexes, segmentCount, indexMemoryUsage); @@ -417,14 +420,17 @@ public class DistributedIndexServer extends AbstractIndexServer { @Override public Map<String, BlurIndex> getIndexes(String table) throws IOException { + checkTable(table); Set<String> shardsToServe = getShardsToServe(table); + LOG.debug("Creating missing index map for table [{0}] if missing.", table); synchronized (_indexes) { if (!_indexes.containsKey(table)) { _indexes.putIfAbsent(table, new ConcurrentHashMap<String, BlurIndex>()); } } + Map<String, BlurIndex> tableIndexes = _indexes.get(table); Set<String> shardsBeingServed = new HashSet<String>(tableIndexes.keySet()); if (shardsBeingServed.containsAll(shardsToServe)) { @@ -435,12 +441,13 @@ public class DistributedIndexServer extends AbstractIndexServer { } return result; } else { + LOG.debug("Opening missing shards for table [{0}]", table); return openMissingShards(table, shardsToServe, tableIndexes); } } private BlurIndex openShard(TableContext tableContext, String table, String shard) throws IOException { - LOG.info("Opening shard [{0}] for table [{1}]", shard, table); + LOG.debug("Opening shard [{0}] for table [{1}]", shard, table); ShardContext shardContext = ShardContext.create(tableContext, shard); TableDescriptor tableDescriptor = tableContext.getDescriptor(); Path hdfsDirPath = shardContext.getHdfsDirPath(); @@ -526,13 +533,17 @@ public class DistributedIndexServer extends AbstractIndexServer { } private synchronized Map<String, BlurIndex> openMissingShards(final String table, Set<String> shardsToServe, final Map<String, BlurIndex> tableIndexes) throws IOException { + LOG.debug("Opening missing shards for table [{0}]", table); + LOG.debug("Getting table context for table [{0}]", table); final TableContext tableContext = getTableContext(table); + Map<String, Future<BlurIndex>> opening = new HashMap<String, Future<BlurIndex>>(); for (String s : shardsToServe) { final String shard = s; + LOG.debug("Getting shard [{0}] for table [{1}]", shard, table); BlurIndex blurIndex = tableIndexes.get(shard); if (blurIndex == null) { - LOG.info("Opening missing shard [{0}] from table [{1}]", shard, table); + LOG.debug("Opening missing shard [{0}] from table [{1}]", shard, table); Future<BlurIndex> submit = _openerService.submit(new Callable<BlurIndex>() { @Override public BlurIndex call() throws Exception { @@ -568,6 +579,7 @@ public class DistributedIndexServer extends AbstractIndexServer { } private Set<String> getShardsToServe(String table) { + LOG.debug("Get shards to serve [{0}]", table); TABLE_STATUS tableStatus = getTableStatus(table); if (tableStatus == TABLE_STATUS.DISABLED) { return new HashSet<String>(); @@ -581,6 +593,7 @@ public class DistributedIndexServer extends AbstractIndexServer { } private synchronized Set<String> setupLayoutManager(String table) { + LOG.debug("setupLayoutManager for [{0}]", table); DistributedLayoutManager layoutManager = new DistributedLayoutManager(); String cluster = _clusterStatus.getClusterName(); @@ -588,15 +601,21 @@ public class DistributedIndexServer extends AbstractIndexServer { throw new RuntimeException("Table [" + table + "] is not found."); } + LOG.debug("Getting server list for table [{0}].", table); List<String> shardServerList = _clusterStatus.getServerList(false); + LOG.debug("Getting offline servers for table [{0}].", table); List<String> offlineShardServers = new ArrayList<String>(_clusterStatus.getOfflineServers(false)); + LOG.debug("Getting shard list for table [{0}].", table); List<String> shardList = getShardList(table); layoutManager.setNodes(shardServerList); layoutManager.setNodesOffline(offlineShardServers); layoutManager.setShards(shardList); + + LOG.debug("Initalizing layout for table [{0}].", table); layoutManager.init(); + LOG.debug("Calculating shards to serve for table [{0}]", table); Map<String, String> layout = layoutManager.getLayout(); String nodeName = getNodeName(); Set<String> shardsToServeCache = new TreeSet<String>(); @@ -669,6 +688,7 @@ public class DistributedIndexServer extends AbstractIndexServer { } private void checkTable(String table) { + LOG.debug("Checking table [{0}]", table); if (_clusterStatus.exists(true, table)) { return; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2ef1c1ce/src/blur-core/src/main/java/org/apache/blur/thrift/TableContext.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/TableContext.java b/src/blur-core/src/main/java/org/apache/blur/thrift/TableContext.java index 93db181..5918bfc 100644 --- a/src/blur-core/src/main/java/org/apache/blur/thrift/TableContext.java +++ b/src/blur-core/src/main/java/org/apache/blur/thrift/TableContext.java @@ -20,6 +20,7 @@ import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy; import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.search.similarities.Similarity; +import org.mortbay.log.Log; public class TableContext { @@ -43,6 +44,7 @@ public class TableContext { } public static TableContext create(TableDescriptor tableDescriptor) { + Log.info("Creating table context for table [{0}]", tableDescriptor.getName()); Configuration configuration = new Configuration(); Map<String, String> properties = tableDescriptor.getProperties(); if (properties != null) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2ef1c1ce/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java index a597250..4449908 100644 --- a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java +++ b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java @@ -48,8 +48,7 @@ import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.manager.BlurFilterCache; import org.apache.blur.manager.DefaultBlurFilterCache; -//import org.apache.blur.manager.clusterstatus.ZookeeperClusterStatus; -import org.apache.blur.manager.clusterstatus.SimpleZKClusterStatus; +import org.apache.blur.manager.clusterstatus.ZookeeperClusterStatus; import org.apache.blur.manager.indexserver.BlurIndexWarmup; import org.apache.blur.manager.indexserver.BlurServerShutDown; import org.apache.blur.manager.indexserver.BlurServerShutDown.BlurShutdown; @@ -162,9 +161,9 @@ public class ThriftBlurServer extends AbstractThriftServer { BlurUtil.setupZookeeper(zooKeeper, configuration.get(BLUR_CLUSTER_NAME)); - final SimpleZKClusterStatus clusterStatus = new SimpleZKClusterStatus(BlurConstants.BLUR_CLUSTER, zooKeeper); +// final SimpleZKClusterStatus clusterStatus = new SimpleZKClusterStatus(BlurConstants.BLUR_CLUSTER, zooKeeper); -// final ZookeeperClusterStatus clusterStatus = new ZookeeperClusterStatus(BlurConstants.BLUR_CLUSTER, zooKeeper); + final ZookeeperClusterStatus clusterStatus = new ZookeeperClusterStatus(BlurConstants.BLUR_CLUSTER, zooKeeper); final BlurIndexRefresher refresher = new BlurIndexRefresher(); refresher.init(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2ef1c1ce/src/blur-mapred/pom.xml ---------------------------------------------------------------------- diff --git a/src/blur-mapred/pom.xml b/src/blur-mapred/pom.xml index 87d30fb..ecc0e4e 100644 --- a/src/blur-mapred/pom.xml +++ b/src/blur-mapred/pom.xml @@ -27,7 +27,7 @@ <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> - <version>3.4.4</version> + <version>3.4.5</version> <scope>provided</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2ef1c1ce/src/blur-testsuite/pom.xml ---------------------------------------------------------------------- diff --git a/src/blur-testsuite/pom.xml b/src/blur-testsuite/pom.xml index fed8422..274a976 100644 --- a/src/blur-testsuite/pom.xml +++ b/src/blur-testsuite/pom.xml @@ -56,7 +56,7 @@ under the License. <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> - <version>3.4.4</version> + <version>3.4.5</version> <scope>provided</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2ef1c1ce/src/blur-util/pom.xml ---------------------------------------------------------------------- diff --git a/src/blur-util/pom.xml b/src/blur-util/pom.xml index 663cb6f..defc26f 100644 --- a/src/blur-util/pom.xml +++ b/src/blur-util/pom.xml @@ -35,7 +35,7 @@ under the License. <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> - <version>3.4.4</version> + <version>3.4.5</version> <exclusions> <exclusion> <groupId>javax.mail</groupId> @@ -64,7 +64,7 @@ under the License. <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-core</artifactId> - <version>1.0.3</version> + <version>1.0.4</version> <scope>compile</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2ef1c1ce/src/distribution/pom.xml ---------------------------------------------------------------------- diff --git a/src/distribution/pom.xml b/src/distribution/pom.xml index 39d679e..07b47de 100644 --- a/src/distribution/pom.xml +++ b/src/distribution/pom.xml @@ -22,6 +22,11 @@ <artifactId>blur-mapred</artifactId> <version>0.2.0-SNAPSHOT</version> </dependency> + <dependency> + <groupId>org.apache.blur</groupId> + <artifactId>blur-shell</artifactId> + <version>0.2.0-SNAPSHOT</version> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2ef1c1ce/src/distribution/src/main/scripts/bin/blur ---------------------------------------------------------------------- diff --git a/src/distribution/src/main/scripts/bin/blur b/src/distribution/src/main/scripts/bin/blur index afe5ebb..c2ea254 100755 --- a/src/distribution/src/main/scripts/bin/blur +++ b/src/distribution/src/main/scripts/bin/blur @@ -20,7 +20,17 @@ bin=`cd "$bin"; pwd` . "$bin"/blur-config.sh PROC_NAME=$1 -"$JAVA_HOME"/bin/java -Dblur.name=$PROC_NAME -Djava.library.path=$JAVA_LIBRARY_PATH $BLUR_COMMAND -Dblur.logs.dir=$BLUR_LOGS -Dblur.log.file=$PROC_NAME.log -cp $BLUR_CLASSPATH $@ +if [ $# -eq "0" ]; then + echo "Usage:" + echo "blur shell" + echo " or" + echo "blur CLASSNAME" +elif [ $1 == "shell" ]; then + "$JAVA_HOME"/bin/java -Dblur.name=$PROC_NAME -Djava.library.path=$JAVA_LIBRARY_PATH $BLUR_COMMAND -Dblur.logs.dir=$BLUR_LOGS -Dblur.log.file=$PROC_NAME.log -cp $BLUR_CLASSPATH org.apache.blur.shell.Main $2 +else + "$JAVA_HOME"/bin/java -Dblur.name=$PROC_NAME -Djava.library.path=$JAVA_LIBRARY_PATH $BLUR_COMMAND -Dblur.logs.dir=$BLUR_LOGS -Dblur.log.file=$PROC_NAME.log -cp $BLUR_CLASSPATH $@ +fi + http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2ef1c1ce/src/distribution/src/main/scripts/bin/stop-server.sh ---------------------------------------------------------------------- diff --git a/src/distribution/src/main/scripts/bin/stop-server.sh b/src/distribution/src/main/scripts/bin/stop-server.sh index 42a882f..f0c4be0 100755 --- a/src/distribution/src/main/scripts/bin/stop-server.sh +++ b/src/distribution/src/main/scripts/bin/stop-server.sh @@ -18,16 +18,28 @@ bin=`dirname "$0"` bin=`cd "$bin"; pwd` -. "$bin"/blur-config.sh +. "$bin"/blur-config.sh INSTANCE=0 while [ $INSTANCE -lt $BLUR_NUMBER_OF_SERVER_INSTANCES_PER_MACHINE ]; do PID_FILE=$BLUR_HOME/pids/blur-$INSTANCE.pid if [ -f $PID_FILE ]; then - if kill -0 `cat $PID_FILE` > /dev/null 2>&1; then - echo Stopping Blur Server [$INSTANCE] server with pid [`cat $PID_FILE`]. - kill `cat $PID_FILE` + PID=`cat $PID_FILE` + if kill -0 $PID > /dev/null 2>&1; then + echo Stopping Blur Server [$INSTANCE] server with pid [$PID]. + kill $PID + + if [ $# -eq "1" ]; then + if [ $1 == "-w" ]; then + echo Waiting for Blur Server [$INSTANCE] server with pid [$PID] to end. + while [[ ${?} == 0 ]] + do + sleep 0.1 + ps -p $PID >/dev/null + done + fi + fi else echo No Blur Server [$INSTANCE] server to stop fi
