Adding the management of ZooKeeper by Blur to ease quick setups.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/21ddfded Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/21ddfded Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/21ddfded Branch: refs/heads/master Commit: 21ddfded0875751d592cc62ae8c0905f266e745e Parents: 0bb1028 Author: Aaron McCurry <[email protected]> Authored: Fri Jun 7 13:30:19 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Fri Jun 7 13:30:19 2013 -0400 ---------------------------------------------------------------------- distribution/src/main/scripts/bin/start-all.sh | 3 ++ .../main/scripts/bin/start-zookeeper-server.sh | 39 ++++++++++++++++++++ .../src/main/scripts/bin/start-zookeepers.sh | 25 +++++++++++++ distribution/src/main/scripts/bin/stop-all.sh | 3 ++ .../main/scripts/bin/stop-zookeeper-server.sh | 37 +++++++++++++++++++ .../src/main/scripts/bin/stop-zookeepers.sh | 24 ++++++++++++ distribution/src/main/scripts/bin/zookeepers.sh | 34 +++++++++++++++++ distribution/src/main/scripts/conf/blur-env.sh | 17 +++++++-- 8 files changed, 179 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/21ddfded/distribution/src/main/scripts/bin/start-all.sh ---------------------------------------------------------------------- diff --git a/distribution/src/main/scripts/bin/start-all.sh b/distribution/src/main/scripts/bin/start-all.sh index 7da121a..b951eb5 100755 --- a/distribution/src/main/scripts/bin/start-all.sh +++ b/distribution/src/main/scripts/bin/start-all.sh @@ -20,5 +20,8 @@ bin=`cd "$bin"; pwd` . "$bin"/blur-config.sh +if [ $BLUR_MANAGE_ZK = true ]; then + $BLUR_HOME/bin/start-zookeepers.sh +fi $BLUR_HOME/bin/start-shards.sh $BLUR_HOME/bin/start-controllers.sh \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/21ddfded/distribution/src/main/scripts/bin/start-zookeeper-server.sh ---------------------------------------------------------------------- diff --git a/distribution/src/main/scripts/bin/start-zookeeper-server.sh b/distribution/src/main/scripts/bin/start-zookeeper-server.sh new file mode 100755 index 0000000..2347cc8 --- /dev/null +++ b/distribution/src/main/scripts/bin/start-zookeeper-server.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/blur-config.sh + +if [ $BLUR_MANAGE_ZK = true ]; then + PID_FILE=$BLUR_HOME/pids/zk.pid + + if [ -f $PID_FILE ]; then + if kill -0 `cat $PID_FILE` > /dev/null 2>&1; then + echo ZooKeeper already running as process `cat $PID_FILE`. Stop it first. + exit 0 + fi + fi + + PROC_NAME=blur-zk-$HOSTNAME + nohup "$JAVA_HOME"/bin/java -Dblur.name=$PROC_NAME -Djava.library.path=$JAVA_LIBRARY_PATH -Dblur-zk $BLUR_ZK_JVM_OPTIONS -Dblur.logs.dir=$BLUR_LOGS -Dblur.log.file=$PROC_NAME.log -cp $BLUR_CLASSPATH org.apache.zookeeper.server.quorum.QuorumPeerMain $BLUR_HOME/conf/default_zoo.cfg > "$BLUR_LOGS/$PROC_NAME.out" 2>&1 < /dev/null & + echo $! > $PID_FILE + echo ZooKeeper starting as process `cat $PID_FILE`. +else + echo Blur is not managing ZooKeeper +fi \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/21ddfded/distribution/src/main/scripts/bin/start-zookeepers.sh ---------------------------------------------------------------------- diff --git a/distribution/src/main/scripts/bin/start-zookeepers.sh b/distribution/src/main/scripts/bin/start-zookeepers.sh new file mode 100755 index 0000000..bf18c54 --- /dev/null +++ b/distribution/src/main/scripts/bin/start-zookeepers.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/blur-config.sh + +$BLUR_HOME/bin/zookeepers.sh $BLUR_HOME/bin/start-zookeeper-server.sh + + http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/21ddfded/distribution/src/main/scripts/bin/stop-all.sh ---------------------------------------------------------------------- diff --git a/distribution/src/main/scripts/bin/stop-all.sh b/distribution/src/main/scripts/bin/stop-all.sh index 8c758fb..1672258 100755 --- a/distribution/src/main/scripts/bin/stop-all.sh +++ b/distribution/src/main/scripts/bin/stop-all.sh @@ -22,3 +22,6 @@ bin=`cd "$bin"; pwd` $BLUR_HOME/bin/stop-controllers.sh $BLUR_HOME/bin/stop-shards.sh +if [ $BLUR_MANAGE_ZK = true ]; then + $BLUR_HOME/bin/stop-zookeepers.sh +fi http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/21ddfded/distribution/src/main/scripts/bin/stop-zookeeper-server.sh ---------------------------------------------------------------------- diff --git a/distribution/src/main/scripts/bin/stop-zookeeper-server.sh b/distribution/src/main/scripts/bin/stop-zookeeper-server.sh new file mode 100755 index 0000000..5455988 --- /dev/null +++ b/distribution/src/main/scripts/bin/stop-zookeeper-server.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/blur-config.sh + +if [ $BLUR_MANAGE_ZK = true ]; then + PID_FILE=$BLUR_HOME/pids/zk.pid + if [ -f $PID_FILE ]; then + if kill -0 `cat $PID_FILE` > /dev/null 2>&1; then + echo Stopping ZooKeeper with pid [`cat $PID_FILE`]. + kill `cat $PID_FILE` + else + echo No ZooKeeper to stop + fi + else + echo No ZooKeeper to stop + fi +else + echo Blur is not managing ZooKeeper +fi \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/21ddfded/distribution/src/main/scripts/bin/stop-zookeepers.sh ---------------------------------------------------------------------- diff --git a/distribution/src/main/scripts/bin/stop-zookeepers.sh b/distribution/src/main/scripts/bin/stop-zookeepers.sh new file mode 100755 index 0000000..0bd696e --- /dev/null +++ b/distribution/src/main/scripts/bin/stop-zookeepers.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/blur-config.sh + +$BLUR_HOME/bin/zookeepers.sh $BLUR_HOME/bin/stop-zookeeper-server.sh + http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/21ddfded/distribution/src/main/scripts/bin/zookeepers.sh ---------------------------------------------------------------------- diff --git a/distribution/src/main/scripts/bin/zookeepers.sh b/distribution/src/main/scripts/bin/zookeepers.sh new file mode 100755 index 0000000..275256e --- /dev/null +++ b/distribution/src/main/scripts/bin/zookeepers.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/blur-config.sh + +export HOSTLIST="${BLUR_HOME_CONF}/zookeepers" + +for zookeeper in `cat "$HOSTLIST"|sed "s/#.*$//;/^$/d"`; do + ssh $BLUR_SSH_OPTS $zookeeper $"${@// /\\ }" \ + 2>&1 | sed "s/^/$zookeeper: /" & + if [ "$BLUR_ZK_SLEEP" != "" ]; then + sleep $BLUR_ZK_SLEEP + fi +done + +wait + http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/21ddfded/distribution/src/main/scripts/conf/blur-env.sh ---------------------------------------------------------------------- diff --git a/distribution/src/main/scripts/conf/blur-env.sh b/distribution/src/main/scripts/conf/blur-env.sh index 4d0f9dc..5017102 100755 --- a/distribution/src/main/scripts/conf/blur-env.sh +++ b/distribution/src/main/scripts/conf/blur-env.sh @@ -28,10 +28,18 @@ #-XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:CMSIncrementalDutyCycleMin=10 -XX:CMSIncrementalDutyCycle=50 \ #-XX:ParallelGCThreads=8 -XX:+UseParNewGC -XX:MaxGCPauseMillis=200 -XX:GCTimeRatio=10 -XX:+DisableExplicitGC \ #-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$BLUR_HOME/logs/gc-blur-shard-server_`date +%Y%m%d_%H%M%S`.log -export BLUR_SHARD_JVM_OPTIONS="-Xmx1024m -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=256m -XX:OnOutOfMemoryError=\"kill -9 %p\"" +# Consider adding the -XX:OnOutOfMemoryError="kill -9 %p" option to kill jvms that are failing due to memory issues. +export BLUR_SHARD_JVM_OPTIONS="-Xmx1024m -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=256m " -# JAVA JVM OPTIONS for the shard servers, jvm tuning parameters are placed here. -export BLUR_CONTROLLER_JVM_OPTIONS="-Xmx1024m -Djava.net.preferIPv4Stack=true -XX:OnOutOfMemoryError=\"kill -9 %p\"" +# JAVA JVM OPTIONS for the controller servers, jvm tuning parameters are placed here. +# Consider adding the -XX:OnOutOfMemoryError="kill -9 %p" option to kill jvms that are failing due to memory issues. +export BLUR_CONTROLLER_JVM_OPTIONS="-Xmx1024m -Djava.net.preferIPv4Stack=true " + +# This tells blur to manage the ZooKeeper instances, set this to false if Blur is not going to manage the ZooKeeper cluster +export BLUR_MANAGE_ZK=true + +# JAVA JVM OPTIONS for the zookeepers servers, jvm tuning parameters are placed here. +export BLUR_ZK_JVM_OPTIONS="-Xmx128m -Djava.net.preferIPv4Stack=true " # JAVA JVM OPTIONS for the shard servers, jvm tuning parameters are placed here. export BLUR_COMMAND="-Xmx1024m -Djava.net.preferIPv4Stack=true" @@ -45,6 +53,9 @@ export BLUR_SHARD_SLEEP=0.1 # Time to sleep between controller server commands. export BLUR_CONTROLLER_SLEEP=0.1 +# Time to sleep between zookeeper server commands. +export BLUR_ZK_SLEEP=0.1 + # The of shard servers to spawn per machine. export BLUR_NUMBER_OF_SHARD_SERVER_INSTANCES_PER_MACHINE=1
