http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/assemble/conf/examples/templates/accumulo-site.xml ---------------------------------------------------------------------- diff --git a/assemble/conf/examples/templates/accumulo-site.xml b/assemble/conf/examples/templates/accumulo-site.xml new file mode 100644 index 0000000..a1f4153 --- /dev/null +++ b/assemble/conf/examples/templates/accumulo-site.xml @@ -0,0 +1,175 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + <!-- This is the main configuration file for Apache Accumulo. Available configuration properties + (and their default values) can be found in the user manual (docs/accumulo_user_manual.html). --> + <property> + <name>instance.volumes</name> + <value></value> + <description>comma separated list of URIs for volumes. example: hdfs://localhost:9000/accumulo</description> + </property> + + <property> + <name>instance.zookeeper.host</name> + <value>localhost:2181</value> + <description>comma separated list of zookeeper servers</description> + </property> + + <property> + <name>instance.secret</name> + <value>DEFAULT</value> + <description>A secret unique to a given instance that all servers must know in order to communicate with one another. + Change it before initialization. To + change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret --old [oldpasswd] --new [newpasswd], + and then update this file. + </description> + </property> + + <property> + <name>tserver.memory.maps.max</name> + <value>${memMapMax}</value> + </property> + + <property> + <name>tserver.memory.maps.native.enabled</name> + <value>${nativeEnabled}</value> + </property> + + <property> + <name>tserver.cache.data.size</name> + <value>${cacheDataSize}</value> + </property> + + <property> + <name>tserver.cache.index.size</name> + <value>${cacheIndexSize}</value> + </property> + + <property> + <name>trace.token.property.password</name> + <!-- change this to the root user's password, and/or change the user below --> + <value>secret</value> + </property> + + <!-- Kerberos requirements --> + <property> + <name>instance.rpc.sasl.enabled</name> + <value>true</value> + </property> + + <property> + <name>general.kerberos.keytab</name> + <value>${keytab}</value> + </property> + + <property> + <name>general.kerberos.principal</name> + <value>${principal}</value> + </property> + + <property> + <name>trace.token.type</name> + <value>org.apache.accumulo.core.client.security.tokens.KerberosToken</value> + </property> + + <property> + <name>instance.security.authenticator</name> + <value>org.apache.accumulo.server.security.handler.KerberosAuthenticator</value> + </property> + + <property> + <name>instance.security.authorizor</name> + <value>org.apache.accumulo.server.security.handler.KerberosAuthorizor</value> + </property> + + <property> + <name>instance.security.permissionHandler</name> + <value>org.apache.accumulo.server.security.handler.KerberosPermissionHandler</value> + </property> + <!-- End Kerberos requirements --> + + <property> + <name>trace.user</name> + <value>${traceUser}</value> + </property> + + <property> + <name>tserver.sort.buffer.size</name> + <value>${sortBufferSize}</value> + </property> + + <property> + <name>tserver.walog.max.size</name> + <value>${waLogMaxSize}</value> + </property> +${mvnProjBaseDir} + <property> + <name>general.classpaths</name> + + <value> + <!-- Accumulo requirements --> + $ACCUMULO_HOME/lib/accumulo-server.jar, + $ACCUMULO_HOME/lib/accumulo-core.jar, + $ACCUMULO_HOME/lib/accumulo-start.jar, + $ACCUMULO_HOME/lib/accumulo-fate.jar, + $ACCUMULO_HOME/lib/accumulo-proxy.jar, + $ACCUMULO_HOME/lib/[^.].*.jar, + <!-- ZooKeeper requirements --> + $ZOOKEEPER_HOME/zookeeper[^.].*.jar, + <!-- Common Hadoop requirements --> + $HADOOP_CONF_DIR, + <!-- Hadoop 2 requirements --> + $HADOOP_PREFIX/share/hadoop/common/[^.].*.jar, + $HADOOP_PREFIX/share/hadoop/common/lib/(?!slf4j)[^.].*.jar, + $HADOOP_PREFIX/share/hadoop/hdfs/[^.].*.jar, + $HADOOP_PREFIX/share/hadoop/mapreduce/[^.].*.jar, + $HADOOP_PREFIX/share/hadoop/yarn/[^.].*.jar, + $HADOOP_PREFIX/share/hadoop/yarn/lib/jersey.*.jar, + <!-- End Hadoop 2 requirements --> + <!-- HDP 2.0 requirements --> + /usr/lib/hadoop/[^.].*.jar, + /usr/lib/hadoop/lib/[^.].*.jar, + /usr/lib/hadoop-hdfs/[^.].*.jar, + /usr/lib/hadoop-mapreduce/[^.].*.jar, + /usr/lib/hadoop-yarn/[^.].*.jar, + /usr/lib/hadoop-yarn/lib/jersey.*.jar, + <!-- End HDP 2.0 requirements --> + <!-- HDP 2.2 requirements --> + /usr/hdp/current/hadoop-client/[^.].*.jar, + /usr/hdp/current/hadoop-client/lib/(?!slf4j)[^.].*.jar, + /usr/hdp/current/hadoop-hdfs-client/[^.].*.jar, + /usr/hdp/current/hadoop-mapreduce-client/[^.].*.jar, + /usr/hdp/current/hadoop-yarn-client/[^.].*.jar, + /usr/hdp/current/hadoop-yarn-client/lib/jersey.*.jar, + /usr/hdp/current/hive-client/lib/hive-accumulo-handler.jar + <!-- End HDP 2.2 requirements --> + <!-- IOP 4.1 requirements --> + /usr/iop/current/hadoop-client/[^.].*.jar, + /usr/iop/current/hadoop-client/lib/(?!slf4j)[^.].*.jar, + /usr/iop/current/hadoop-hdfs-client/[^.].*.jar, + /usr/iop/current/hadoop-mapreduce-client/[^.].*.jar, + /usr/iop/current/hadoop-yarn-client/[^.].*.jar, + /usr/iop/current/hadoop-yarn-client/lib/jersey.*.jar, + /usr/iop/current/hive-client/lib/hive-accumulo-handler.jar + <!-- End IOP 4.1 requirements --> + </value> + <description>Classpaths that accumulo checks for updates and class files.</description> + </property> +</configuration>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/assemble/conf/examples/tracers ---------------------------------------------------------------------- diff --git a/assemble/conf/examples/tracers b/assemble/conf/examples/tracers deleted file mode 100644 index 63fb8bb..0000000 --- a/assemble/conf/examples/tracers +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -localhost http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/assemble/conf/examples/tservers ---------------------------------------------------------------------- diff --git a/assemble/conf/examples/tservers b/assemble/conf/examples/tservers deleted file mode 100644 index 63fb8bb..0000000 --- a/assemble/conf/examples/tservers +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -localhost http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/assemble/contrib/bootstrap-hdfs.sh ---------------------------------------------------------------------- diff --git a/assemble/contrib/bootstrap-hdfs.sh b/assemble/contrib/bootstrap-hdfs.sh deleted file mode 100755 index 26f94f4..0000000 --- a/assemble/contrib/bootstrap-hdfs.sh +++ /dev/null @@ -1,91 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Start: Resolve Script Directory -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - contrib=$( cd -P "$( dirname "$SOURCE" )" && pwd ) - SOURCE=$(readlink "$SOURCE") - [[ $SOURCE != /* ]] && SOURCE="$contrib/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -contrib=$( cd -P "$( dirname "$SOURCE" )" && pwd ) -basedir=$( cd -P "${contrib}"/.. && pwd ) -# Stop: Resolve Script Directory - -source "$basedir"/libexec/load-env.sh - -# -# Find the system context directory in HDFS -# -SYSTEM_CONTEXT_HDFS_DIR=$(grep -A1 "general.vfs.classpaths" "$ACCUMULO_CONF_DIR/accumulo-site.xml" | tail -1 | perl -pe 's/\s+<value>//; s/<\/value>//; s/,.+$//; s|[^/]+$||; print $ARGV[1]') - -if [ -z "$SYSTEM_CONTEXT_HDFS_DIR" ] -then - echo "Your accumulo-site.xml file is not set up for the HDFS Classloader. Please add the following to your accumulo-site.xml file where ##CLASSPATH## is one of the following formats:" - echo "A single directory: hdfs://host:port/directory/" - echo "A single directory with a regex: hdfs://host:port/directory/.*.jar" - echo "Multiple directories: hdfs://host:port/directory/.*.jar,hdfs://host:port/directory2/" - echo "" - echo "<property>" - echo " <name>general.vfs.classpaths</name>" - echo " <value>##CLASSPATH##</value>" - echo " <description>location of the jars for the default (system) context</description>" - echo "</property>" - exit 1 -fi - -# -# Create the system context directy in HDFS if it does not exist -# -"$HADOOP_PREFIX/bin/hadoop" fs -ls "$SYSTEM_CONTEXT_HDFS_DIR" > /dev/null -if [[ $? != 0 ]]; then - "$HADOOP_PREFIX/bin/hadoop" fs -mkdir "$SYSTEM_CONTEXT_HDFS_DIR" > /dev/null - if [[ $? != 0 ]]; then - echo "Unable to create classpath directory at $SYSTEM_CONTEXT_HDFS_DIR" - exit 1 - fi -fi - -# -# Replicate to all tservers to avoid network contention on startup -# -TSERVERS=$ACCUMULO_CONF_DIR/tservers -NUM_TSERVERS=$(egrep -v '(^#|^\s*$)' "$TSERVERS" | wc -l) - -#let each datanode service around 50 clients -REP=$(( NUM_TSERVERS / 50 )) -(( REP < 3 )) && REP=3 - -# -# Copy all jars in lib to the system context directory -# -"$HADOOP_PREFIX/bin/hadoop" fs -moveFromLocal "$ACCUMULO_LIB_DIR"/*.jar "$SYSTEM_CONTEXT_HDFS_DIR" > /dev/null -"$HADOOP_PREFIX/bin/hadoop" fs -setrep -R $REP "$SYSTEM_CONTEXT_HDFS_DIR" > /dev/null - -# -# We need some of the jars in lib, copy them back out and remove them from the system context dir -# -"$HADOOP_PREFIX/bin/hadoop" fs -copyToLocal "$SYSTEM_CONTEXT_HDFS_DIR/commons-vfs2.jar" "$ACCUMULO_LIB_DIR/." > /dev/null -"$HADOOP_PREFIX/bin/hadoop" fs -rm "$SYSTEM_CONTEXT_HDFS_DIR/commons-vfs2.jar" > /dev/null -"$HADOOP_PREFIX/bin/hadoop" fs -copyToLocal "$SYSTEM_CONTEXT_HDFS_DIR/accumulo-start.jar" "$ACCUMULO_LIB_DIR/." > /dev/null -"$HADOOP_PREFIX/bin/hadoop" fs -rm "$SYSTEM_CONTEXT_HDFS_DIR/accumulo-start.jar" > /dev/null -"$HADOOP_PREFIX/bin/hadoop" fs -copyToLocal "$SYSTEM_CONTEXT_HDFS_DIR/slf4j*.jar" "$ACCUMULO_LIB_DIR/." > /dev/null -"$HADOOP_PREFIX/bin/hadoop" fs -rm "$SYSTEM_CONTEXT_HDFS_DIR/slf4j*.jar" > /dev/null -for f in $(grep -v '^#' "$ACCUMULO_CONF_DIR/tservers") -do - rsync -ra --delete "$ACCUMULO_HOME" "$(dirname "$ACCUMULO_HOME")" -done http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/assemble/contrib/check-tservers ---------------------------------------------------------------------- diff --git a/assemble/contrib/check-tservers b/assemble/contrib/check-tservers deleted file mode 100755 index 7f9850e..0000000 --- a/assemble/contrib/check-tservers +++ /dev/null @@ -1,199 +0,0 @@ -#! /usr/bin/env python - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script will check the configuration and uniformity of all the nodes in a cluster. -# Checks -# each node is reachable via ssh -# login identity is the same -# the physical memory is the same -# the mounts are the same on each machine -# a set of writable locations (typically different disks) are in fact writable -# -# In order to check for writable partitions, you must configure the WRITABLE variable below. -# - -import subprocess -import time -import select -import os -import sys -import fcntl -import signal -if not sys.platform.startswith('linux'): - sys.stderr.write('This script only works on linux, sorry.\n') - sys.exit(1) - -TIMEOUT = 5 -WRITABLE = [] -#WRITABLE = ['/srv/hdfs1', '/srv/hdfs2', '/srv/hdfs3'] - -def ssh(tserver, *args): - 'execute a command on a remote tserver and return the Popen handle' - handle = subprocess.Popen( ('ssh', '-o', 'StrictHostKeyChecking=no', '-q', '-A', '-n', tserver) + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - handle.tserver = tserver - handle.finished = False - handle.out = '' - return handle - -def wait(handles, seconds): - 'wait for lots of handles simultaneously, and kill anything that doesn\'t return in seconds time\n' - 'Note that stdout will be stored on the handle as the "out" field and "finished" will be set to True' - handles = handles[:] - stop = time.time() + seconds - for h in handles: - fcntl.fcntl(h.stdout, fcntl.F_SETFL, os.O_NONBLOCK) - while handles and time.time() < stop: - wait = min(0, stop - time.time()) - handleMap = dict( [(h.stdout, h) for h in handles] ) - rd, wr, err = select.select(handleMap.keys(), [], [], wait) - for r in rd: - handle = handleMap[r] - while 1: - more = handle.stdout.read(1024) - if more == '': - handles.remove(handle) - handle.poll() - handle.wait() - handle.finished = True - handle.out += more - if len(more) < 1024: - break - for handle in handles: - os.kill(handle.pid, signal.SIGKILL) - handle.poll() - -def runAll(tservers, *cmd): - 'Run the given command on all the tservers, returns Popen handles' - handles = [] - for tserver in tservers: - handles.append(ssh(tserver, *cmd)) - wait(handles, TIMEOUT) - return handles - -def checkIdentity(tservers): - 'Ensure the login identity is consistent across the tservers' - handles = runAll(tservers, 'id', '-u', '-n') - bad = set() - myIdentity = os.popen('id -u -n').read().strip() - for h in handles: - if not h.finished or h.returncode != 0: - print '#', 'cannot look at identity on', h.tserver - bad.add(h.tserver) - else: - identity = h.out.strip() - if identity != myIdentity: - print '#', h.tserver, 'inconsistent identity', identity - bad.add(h.tserver) - return bad - -def checkMemory(tservers): - 'Run free on all tservers and look for weird results' - handles = runAll(tservers, 'free') - bad = set() - mem = {} - swap = {} - for h in handles: - if not h.finished or h.returncode != 0: - print '#', 'cannot look at memory on', h.tserver - bad.add(h.tserver) - else: - if h.out.find('Swap:') < 0: - print '#',h.tserver,'has no swap' - bad.add(h.tserver) - continue - lines = h.out.split('\n') - for line in lines: - if line.startswith('Mem:'): - mem.setdefault(line.split()[1],set()).add(h.tserver) - if line.startswith('Swap:'): - swap.setdefault(line.split()[1],set()).add(h.tserver) - # order memory sizes by most common - mems = sorted([(len(v), k, v) for k, v in mem.items()], reverse=True) - mostCommon = float(mems[0][1]) - for _, size, tservers in mems[1:]: - fract = abs(mostCommon - float(size)) / mostCommon - if fract > 0.05: - print '#',', '.join(tservers), ': unusual memory size', size - bad.update(tservers) - swaps = sorted([(len(v), k, v) for k, v in swap.items()], reverse=True) - mostCommon = float(mems[0][1]) - for _, size, tservers in swaps[1:]: - fract = abs(mostCommon - float(size) / mostCommon) - if fract > 0.05: - print '#',', '.join(tservers), ': unusual swap size', size - bad.update(tservers) - return bad - -def checkWritable(tservers): - 'Touch all the directories that should be writable by this user return any nodes that fail' - if not WRITABLE: - print '# WRITABLE value not configured, not checking partitions' - return [] - handles = runAll(tservers, 'touch', *WRITABLE) - bad = set() - for h in handles: - if not h.finished or h.returncode != 0: - bad.add(h.tserver) - print '#', h.tserver, 'some drives are not writable' - return bad - -def checkMounts(tservers): - 'Check the file systems that are mounted and report any that are unusual' - handles = runAll(tservers, 'mount') - mounts = {} - finished = set() - bad = set() - for handle in handles: - if handle.finished and handle.returncode == 0: - for line in handle.out.split('\n'): - words = line.split() - if len(words) < 5: continue - if words[4] == 'nfs': continue - if words[0].find(':/') >= 0: continue - mount = words[2] - mounts.setdefault(mount, set()).add(handle.tserver) - finished.add(handle.tserver) - else: - bad.add(handle.tserver) - print '#', handle.tserver, 'did not finish' - for m in sorted(mounts.keys()): - diff = finished - mounts[m] - if diff: - bad.update(diff) - print '#', m, 'not mounted on', ', '.join(diff) - return bad - -def main(argv): - if len(argv) < 1: - sys.stderr.write('Usage: check_tservers tservers\n') - sys.exit(1) - sys.stdin.close() - tservers = set() - for tserver in open(argv[0]): - hashPos = tserver.find('#') - if hashPos >= 0: - tserver = tserver[:hashPos] - tserver = tserver.strip() - if not tserver: continue - tservers.add(tserver) - bad = set() - for test in checkIdentity, checkMemory, checkMounts, checkWritable: - bad.update(test(tservers - bad)) - for tserver in sorted(tservers - bad): - print tserver - -main(sys.argv[1:]) http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/assemble/contrib/gen-monitor-cert.sh ---------------------------------------------------------------------- diff --git a/assemble/contrib/gen-monitor-cert.sh b/assemble/contrib/gen-monitor-cert.sh deleted file mode 100755 index e7f313e..0000000 --- a/assemble/contrib/gen-monitor-cert.sh +++ /dev/null @@ -1,85 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Start: Resolve Script Directory -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - contrib=$( cd -P "$( dirname "$SOURCE" )" && pwd ) - SOURCE=$(readlink "$SOURCE") - [[ $SOURCE != /* ]] && SOURCE="$contrib/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -contrib=$( cd -P "$( dirname "$SOURCE" )" && pwd ) -basedir=$( cd -P "${contrib}"/.. && pwd ) -# Stop: Resolve Script Directory - -source "$basedir"/libexec/load-env.sh - -ALIAS="default" -KEYPASS=$(LC_CTYPE=C tr -dc '#-~' < /dev/urandom | tr -d '<>&' | head -c 20) -STOREPASS=$(LC_CTYPE=C tr -dc '#-~' < /dev/urandom | tr -d '<>&' | head -c 20) -KEYSTOREPATH="$ACCUMULO_CONF_DIR/keystore.jks" -TRUSTSTOREPATH="$ACCUMULO_CONF_DIR/conf/cacerts.jks" -CERTPATH="$ACCUMULO_CONF_DIR/server.cer" - -if [[ -e "$KEYSTOREPATH" ]]; then - rm -i "$KEYSTOREPATH" - if [[ -e "$KEYSTOREPATH" ]]; then - echo "KeyStore already exists, exiting" - exit 1 - fi -fi - -if [[ -e "$TRUSTSTOREPATH" ]]; then - rm -i "$TRUSTSTOREPATH" - if [[ -e "$TRUSTSTOREPATH" ]]; then - echo "TrustStore already exists, exiting" - exit 2 - fi -fi - -if [[ -e "$CERTPATH" ]]; then - rm -i "$CERTPATH" - if [[ -e "$CERTPATH" ]]; then - echo "Certificate already exists, exiting" - exit 3 - fi -fi - -"${JAVA_HOME}/bin/keytool" -genkey -alias "$ALIAS" -keyalg RSA -keypass "$KEYPASS" -storepass "$KEYPASS" -keystore "$KEYSTOREPATH" -"${JAVA_HOME}/bin/keytool" -export -alias "$ALIAS" -storepass "$KEYPASS" -file "$CERTPATH" -keystore "$KEYSTOREPATH" -"${JAVA_HOME}/bin/keytool" -import -v -trustcacerts -alias "$ALIAS" -file "$CERTPATH" -keystore "$TRUSTSTOREPATH" -storepass "$STOREPASS" <<< "yes" - -echo -echo "keystore and truststore generated. now add the following to accumulo-site.xml:" -echo -echo " <property>" -echo " <name>monitor.ssl.keyStore</name>" -echo " <value>$KEYSTOREPATH</value>" -echo " </property>" -echo " <property>" -echo " <name>monitor.ssl.keyStorePassword</name>" -echo " <value>$KEYPASS</value>" -echo " </property>" -echo " <property>" -echo " <name>monitor.ssl.trustStore</name>" -echo " <value>$TRUSTSTOREPATH</value>" -echo " </property>" -echo " <property>" -echo " <name>monitor.ssl.trustStorePassword</name>" -echo " <value>$STOREPASS</value>" -echo " </property>" -echo http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/assemble/contrib/tool.sh ---------------------------------------------------------------------- diff --git a/assemble/contrib/tool.sh b/assemble/contrib/tool.sh deleted file mode 100755 index cb8cedc..0000000 --- a/assemble/contrib/tool.sh +++ /dev/null @@ -1,93 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Start: Resolve Script Directory -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - contrib=$( cd -P "$( dirname "$SOURCE" )" && pwd ) - SOURCE=$(readlink "$SOURCE") - [[ $SOURCE != /* ]] && SOURCE="$contrib/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -contrib=$( cd -P "$( dirname "$SOURCE" )" && pwd ) -basedir=$( cd -P "${contrib}"/.. && pwd ) -# Stop: Resolve Script Directory - -source "$basedir"/libexec/load-env.sh - -if [[ -z "$HADOOP_PREFIX" ]] ; then - echo "HADOOP_PREFIX is not set. Please make sure it's set globally or in conf/accumulo-env.sh" - exit 1 -fi -if [[ -z "$ZOOKEEPER_HOME" ]] ; then - echo "ZOOKEEPER_HOME is not set. Please make sure it's set globally or in conf/accumulo-env.sh" - exit 1 -fi - -ZOOKEEPER_CMD="ls -1 $ZOOKEEPER_HOME/zookeeper-[0-9]*[^csn].jar " -if [[ $(eval "$ZOOKEEPER_CMD" | wc -l) -ne 1 ]] ; then - echo "Not exactly one zookeeper jar in $ZOOKEEPER_HOME" - exit 1 -fi -ZOOKEEPER_LIB=$(eval "$ZOOKEEPER_CMD") - -LIB="$ACCUMULO_LIB_DIR" -CORE_LIB="$LIB/accumulo-core.jar" -FATE_LIB="$LIB/accumulo-fate.jar" -THRIFT_LIB="$LIB/libthrift.jar" -JCOMMANDER_LIB="$LIB/jcommander.jar" -COMMONS_VFS_LIB="$LIB/commons-vfs2.jar" -GUAVA_LIB="$LIB/guava.jar" -HTRACE_LIB="$LIB/htrace-core.jar" - -USERJARS=" " -for arg in "$@"; do - if [ "$arg" != "-libjars" -a -z "$TOOLJAR" ]; then - TOOLJAR="$arg" - shift - elif [ "$arg" != "-libjars" -a -z "$CLASSNAME" ]; then - CLASSNAME="$arg" - shift - elif [ -z "$USERJARS" ]; then - USERJARS=$(echo "$arg" | tr "," " ") - shift - elif [ "$arg" = "-libjars" ]; then - USERJARS="" - shift - else - break - fi -done - -LIB_JARS="$THRIFT_LIB,$CORE_LIB,$FATE_LIB,$ZOOKEEPER_LIB,$JCOMMANDER_LIB,$COMMONS_VFS_LIB,$GUAVA_LIB,$HTRACE_LIB" -H_JARS="$THRIFT_LIB:$CORE_LIB:$FATE_LIB:$ZOOKEEPER_LIB:$JCOMMANDER_LIB:$COMMONS_VFS_LIB:$GUAVA_LIB:$HTRACE_LIB" - -for jar in $USERJARS; do - LIB_JARS="$LIB_JARS,$jar" - H_JARS="$H_JARS:$jar" -done -export HADOOP_CLASSPATH="$H_JARS:$HADOOP_CLASSPATH" - -if [[ -z "$CLASSNAME" || -z "$TOOLJAR" ]]; then - echo "Usage: tool.sh path/to/myTool.jar my.tool.class.Name [-libjars my1.jar,my2.jar]" 1>&2 - exit 1 -fi - -#echo USERJARS=$USERJARS -#echo CLASSNAME=$CLASSNAME -#echo HADOOP_CLASSPATH=$HADOOP_CLASSPATH -#echo exec "$HADOOP_PREFIX/bin/hadoop" jar "$TOOLJAR" "$CLASSNAME" -libjars \"$LIB_JARS\" $ARGS -exec "$HADOOP_PREFIX/bin/hadoop" jar "$TOOLJAR" "$CLASSNAME" -libjars "$LIB_JARS" "$@" http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/assemble/libexec/accumulo-watcher.sh ---------------------------------------------------------------------- diff --git a/assemble/libexec/accumulo-watcher.sh b/assemble/libexec/accumulo-watcher.sh deleted file mode 100755 index 1731cdd..0000000 --- a/assemble/libexec/accumulo-watcher.sh +++ /dev/null @@ -1,141 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -LOGHOST=$1 -shift -process=$1 - -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - libexec="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$libexec/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -libexec="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -# Stop: Resolve Script Directory - -source "${libexec}"/load-env.sh - -# Setting for watcher -UNEXPECTED_TIMESPAN=${UNEXPECTED_TIMESPAN:-3600} -UNEXPECTED_RETRIES=${UNEXPECTED_RETRIES:-2} -OOM_TIMESPAN=${OOM_TIMESPAN-3600} -OOM_RETRIES=${OOM_RETRIES-5} -ZKLOCK_TIMESPAN=${ZKLOCK_TIMESPAN-600} -ZKLOCK_RETRIES=${ZKLOCK_RETRIES-5} - -CLEAN_EXIT="Clean Exit" -UNEXPECTED_EXCEPTION="Unexpected exception" -OOM_EXCEPTION="Out of memory exception" -ZKLOCK_LOST="ZKLock lost" -UNKNOWN_ERROR="Unknown error" - -ERRFILE=${ACCUMULO_LOG_DIR}/${process}_${LOGHOST}.err -OUTFILE=${ACCUMULO_LOG_DIR}/${process}_${LOGHOST}.out -DEBUGLOG=${ACCUMULO_LOG_DIR}/${process}_$(hostname).debug.log -COMMAND="${ACCUMULO_BIN_DIR}/accumulo \"\$@\"" - -logger -s "starting process $process at $(date)" -stopRunning="" -while [ -z "$stopRunning" ]; -do - eval "$COMMAND" 2> "$ERRFILE" - exit=$? - unset cause - if [ "$exit" -eq 0 ]; then - potentialStopRunning=$CLEAN_EXIT - elif [ "$exit" -eq 1 ]; then - potentialStopRunning=$UNEXPECTED_EXCEPTION - elif [ "$exit" -eq 130 ]; then - stopRunning="Control C detected, exiting" - elif [ "$exit" -eq 143 ]; then - stopRunning="Process terminated, exiting" - elif [ "$exit" -eq 137 ]; then - potentialStopRunning="Process killed, exiting" - fi - if [ -z "$stopRunning" ]; then - stopRunning=$potentialStopRunning; - - if [ $exit -eq 1 ]; then - source="exit code" - cause=$UNEXPECTED_EXCEPTION - elif tail -n50 "$OUTFILE" | grep "java.lang.OutOfMemoryError:" > /dev/null; then - source="logs" - cause=$OOM_EXCEPTION - elif [ "$process" = "tserver" ]; then - if tail -n50 "$DEBUGLOG" | grep "ERROR: Lost tablet server lock (reason =" > /dev/null ; then - source="logs" - cause=$ZKLOCK_LOST - fi - elif [ "$process" = "master" ]; then - if tail -n50 "$DEBUGLOG" | grep "ERROR: Master lock in zookeeper lost (reason =" > /dev/null ; then - source="logs" - cause=$ZKLOCK_LOST - fi - elif [ "$process" = "gc" ]; then - if tail -n50 "$DEBUGLOG" | grep "FATAL: GC lock in zookeeper lost (reason =" > /dev/null ; then - source="logs" - cause=$ZKLOCK_LOST - fi - elif [ "$process" = "monitor" ]; then - if tail -n50 "$DEBUGLOG" | grep "ERROR: Monitor lock in zookeeper lost (reason =" > /dev/null ; then - source="logs" - cause=$ZKLOCK_LOST - fi - elif [ $exit -ne 0 ]; then - source="exit code" - cause=$UNKNOWN_ERROR - fi - case $cause in - #Unknown exit code - "$UNKNOWN_ERROR") - #window doesn't matter when retries = 0 - RETRIES=0 - ;; - - "$UNEXPECTED_EXCEPTION") - WINDOW=$UNEXPECTED_TIMESPAN - RETRIES=$UNEXPECTED_RETRIES - ;; - - "$OOM_EXCEPTION") - WINDOW=$OOM_TIMESPAN - RETRIES=$OOM_RETRIES - ;; - - "$ZKLOCK_LOST") - WINDOW=$ZKLOCK_TIMESPAN - RETRIES=$ZKLOCK_RETRIES - ;; - esac - - if [ -n "$cause" ]; then - stopRunning="" - declare -i attempts - attempts="$(jobs | grep -c "reason$cause")+1" - if [ "$RETRIES" -le "$attempts" ]; then - stopRunning="$process encountered $cause in $source with exit code $exit- quitting ($attempts/$RETRIES in $WINDOW seconds)" - # kill all sleeps now - for list in $(jobs | cut -b 2-2); do kill %"$list"; done - else - logger -s "$process encountered $cause in $source with exit code $exit- retrying ($attempts/$RETRIES in $WINDOW seconds)" - eval "(sleep $WINDOW ; echo 'reason$cause' >> /dev/null) &" - fi - fi -fi -done -logger -s "$stopRunning" http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/assemble/libexec/cluster.sh ---------------------------------------------------------------------- diff --git a/assemble/libexec/cluster.sh b/assemble/libexec/cluster.sh deleted file mode 100755 index 3ee42c4..0000000 --- a/assemble/libexec/cluster.sh +++ /dev/null @@ -1,325 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -function print_usage { - cat <<EOF -Usage: cluster.sh <command> (<argument> ...) - -Commands: - start-all [--notTservers] Starts all services on cluster - start-tservers Starts all tservers on cluster - start-here Starts all services on this node - start-service <host> <service> Starts <service> on <host> - stop-all Stops all services on cluster - stop-tservers Stops all tservers on cluster - stop-here Stops all services on this node - -EOF - exit 1 -} - -function invalid_args { - echo -e "Invalid arguments: $1\n" - print_usage 1>&2 - exit 1 -} - -function get_ip() { - ip_addr=$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/') - if [[ $? != 0 ]]; then - ip_addr=$(getent ahosts "$(hostname -f)" | grep DGRAM | cut -f 1 -d ' ') - fi - echo "$ip_addr" -} - -function start_service() { - host="$1" - service="$2" - - if [[ $host == "localhost" || $host == $(hostname -f) || $host == $(hostname -s) || $host == $(get_ip) ]]; then - "$libexec/service.sh" start "$host" "$service" - else - $SSH "$host" "bash -c 'ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR} $libexec/service.sh start \"$host\" \"$service\"'" - fi -} - -function start_tservers() { - echo -n "Starting tablet servers ..." - count=1 - for server in $(egrep -v '(^#|^\s*$)' "${ACCUMULO_CONF_DIR}/tservers"); do - echo -n "." - start_service "$server" tserver & - if (( ++count % 72 == 0 )) ; - then - echo - wait - fi - done - echo " done" -} - -function start_all() { - unset DISPLAY - - start_service "$monitor" monitor - - if [ "$1" != "--notTservers" ]; then - start_tservers - fi - - for master in $(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/masters"); do - start_service "$master" master - done - - for gc in $(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/gc"); do - start_service "$gc" gc - done - - for tracer in $(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/tracers"); do - start_service "$tracer" tracer - done -} - -function start_here() { - - local_hosts="$(hostname -a 2> /dev/null) $(hostname) localhost 127.0.0.1 $(get_ip)" - for host in $local_hosts; do - if grep -q "^${host}\$" "$ACCUMULO_CONF_DIR/tservers"; then - start_service "$host" tserver - break - fi - done - - for host in $local_hosts; do - if grep -q "^${host}\$" "$ACCUMULO_CONF_DIR/masters"; then - start_service "$host" master - break - fi - done - - for host in $local_hosts; do - if grep -q "^${host}\$" "$ACCUMULO_CONF_DIR/gc"; then - start_service "$host" gc - break - fi - done - - for host in $local_hosts; do - if [[ $host == "$monitor" ]]; then - start_service "$monitor" monitor - break - fi - done - - for host in $local_hosts; do - if grep -q "^${host}\$" "$ACCUMULO_CONF_DIR/tracers"; then - start_service "$host" tracer - break - fi - done -} - -function stop_service() { - host="$1" - service="$2" - signal="$3" - - # only stop if there's not one already running - if [[ $host == localhost || $host = "$(hostname -s)" || $host = "$(hostname -f)" || $host = $(get_ip) ]] ; then - "$libexec/service.sh" stop "$host" "$service" "$signal" - else - $SSH "$host" "bash -c '$libexec/service.sh stop \"$host\" \"$service\" \"$signal\"'" - fi -} - -function stop_tservers() { - tserver_hosts=$(egrep -v '(^#|^\s*$)' "${ACCUMULO_CONF_DIR}/tservers") - - echo "Stopping unresponsive tablet servers (if any)..." - for server in ${tserver_hosts}; do - # only start if there's not one already running - stop_service "$server" tserver TERM & - done - - sleep 10 - - echo "Stopping unresponsive tablet servers hard (if any)..." - for server in ${tserver_hosts}; do - # only start if there's not one already running - stop_service "$server" tserver KILL & - done - - echo "Cleaning tablet server entries from zookeeper" - ${accumulo_cmd} org.apache.accumulo.server.util.ZooZap -tservers -} - -function stop_all() { - echo "Stopping accumulo services..." - if ! ${accumulo_cmd} admin stopAll - then - echo "Invalid password or unable to connect to the master" - echo "Initiating forced shutdown in 15 seconds (Ctrl-C to abort)" - sleep 10 - echo "Initiating forced shutdown in 5 seconds (Ctrl-C to abort)" - else - echo "Accumulo shut down cleanly" - echo "Utilities and unresponsive servers will shut down in 5 seconds (Ctrl-C to abort)" - fi - - sleep 5 - - #look for master and gc processes not killed by 'admin stopAll' - for signal in TERM KILL ; do - for master in $(grep -v '^#' "$ACCUMULO_CONF_DIR/masters"); do - stop_service "$master" master $signal - done - - for gc in $(grep -v '^#' "$ACCUMULO_CONF_DIR/gc"); do - stop_service "$gc" gc $signal - done - - stop_service "$monitor" monitor $signal - - for tracer in $(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/tracers"); do - stop_service "$tracer" tracer $signal - done - done - - # stop tserver still running - stop_tservers - - echo "Cleaning all server entries in ZooKeeper" - ${accumulo_cmd} org.apache.accumulo.server.util.ZooZap -master -tservers -tracers --site-file "$ACCUMULO_CONF_DIR/accumulo-site.xml" -} - -function stop_here() { - # Determine hostname without errors to user - hosts_to_check=($(hostname -a 2> /dev/null | head -1) $(hostname -f)) - - if egrep -q localhost\|127.0.0.1 "$ACCUMULO_CONF_DIR/tservers"; then - ${accumulo_cmd} admin stop localhost - else - for host in "${hosts_to_check[@]}"; do - if grep -q "$host" "$ACCUMULO_CONF_DIR"/tservers; then - ${accumulo_cmd} admin stop "$host" - fi - done - fi - - for host in "${hosts_to_check[@]}"; do - for signal in TERM KILL; do - for svc in tserver gc master monitor tracer; do - stop_service "$host" $svc $signal - done - done - done -} - -function main() { - # Start: Resolve Script Directory - SOURCE="${BASH_SOURCE[0]}" - while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - libexec="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$libexec/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located - done - libexec="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - # Stop: Resolve Script Directory - - source "$libexec"/load-env.sh - - if [[ -f $ACCUMULO_CONF_DIR/slaves ]]; then - echo "ERROR: A 'slaves' file was found in $ACCUMULO_CONF_DIR/" - echo "Accumulo now reads tablet server hosts from 'tservers' and requires that the 'slaves' file not be present to reduce confusion." - echo "Please rename the 'slaves' file to 'tservers' or remove it if both exist." - exit 1 - fi - - if [[ ! -f $ACCUMULO_CONF_DIR/tservers ]]; then - echo "ERROR: A 'tservers' file was not found at $ACCUMULO_CONF_DIR/tservers" - echo "Please make sure it exists and is configured with tablet server hosts." - exit 1 - fi - - unset master1 - if [[ -f "$ACCUMULO_CONF_DIR/masters" ]]; then - master1=$(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/masters" | head -1) - fi - - if [[ -z "${monitor}" ]] ; then - monitor=$master1 - if [[ -f "$ACCUMULO_CONF_DIR/monitor" ]]; then - monitor=$(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/monitor" | head -1) - fi - if [[ -z "${monitor}" ]] ; then - echo "Could not infer a Monitor role. You need to either define \"${ACCUMULO_CONF_DIR}/monitor\"," - echo "or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." - exit 1 - fi - fi - if [[ ! -f "$ACCUMULO_CONF_DIR/tracers" ]]; then - if [[ -z "${master1}" ]] ; then - echo "Could not find a master node to use as a default for the tracer role." - echo "Either set up \"${ACCUMULO_CONF_DIR}/tracers\" or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." - exit 1 - else - echo "$master1" > "$ACCUMULO_CONF_DIR/tracers" - fi - fi - - if [[ ! -f "$ACCUMULO_CONF_DIR/gc" ]]; then - if [[ -z "${master1}" ]] ; then - echo "Could not infer a GC role. You need to either set up \"${ACCUMULO_CONF_DIR}/gc\" or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." - exit 1 - else - echo "$master1" > "$ACCUMULO_CONF_DIR/gc" - fi - fi - accumulo_cmd="$ACCUMULO_BIN_DIR/accumulo" - - SSH='ssh -qnf -o ConnectTimeout=2' - - if [[ -z $1 ]]; then - invalid_args "<command> cannot be empty" - fi - - case "$1" in - start-all) - start_all "${*:2}" - ;; - start-tservers) - start_tservers - ;; - start-here) - start_here - ;; - stop-all) - stop_all - ;; - stop-tservers) - stop_tservers - ;; - stop-here) - stop_here - ;; - *) - invalid_args "'$1' is an invalid <command>" - ;; - esac -} - -main "$@" http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/assemble/libexec/load-env.sh ---------------------------------------------------------------------- diff --git a/assemble/libexec/load-env.sh b/assemble/libexec/load-env.sh deleted file mode 100755 index 2cc431e..0000000 --- a/assemble/libexec/load-env.sh +++ /dev/null @@ -1,155 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Sources accumulo-env.sh and verifies environment variables - -function verify_env_dir() { - property=$1 - directory=$2 - if [[ -z "$directory" ]]; then - echo "$property is not set. Please make sure it's set globally or in conf/accumulo-env.sh." - exit 1 - fi - if [[ ! -d "$directory" ]]; then - echo "$property=$directory is not a valid directory. Please make sure it's set correctly globally or in conf/accumulo-env.sh." - exit 1 - fi -} - -# Resolve a program to its installation directory -locationByProgram() -{ - RESULT=$( which "$1" ) - if [[ "$?" != 0 && -z "${RESULT}" ]]; then - echo "Cannot find '$1' and '$2' is not set in $ACCUMULO_CONF_DIR/accumulo-env.sh" - exit 1 - fi - while [ -h "${RESULT}" ]; do # resolve $RESULT until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$RESULT" )" && pwd )" - RESULT="$(readlink "${RESULT}")" - [[ "${RESULT}" != /* ]] && RESULT="${DIR}/${RESULT}" # if $RESULT was a relative symlink, we need to resolve it relative to the path where the symlink file was located - done - # find the relative home directory, accounting for an extra bin directory - RESULT=$(dirname "$(dirname "${RESULT}")") - echo "Auto-set ${2} to '${RESULT}'. To suppress this message, set ${2} in conf/accumulo-env.sh" - eval "${2}=${RESULT}" -} - -# Resolve base directory -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do - libexec="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$libexec/$SOURCE" -done -libexec="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -basedir=$( cd -P "${libexec}"/.. && pwd ) - -export ACCUMULO_CONF_DIR="${ACCUMULO_CONF_DIR:-$basedir/conf}" - -if [[ -z $ACCUMULO_CONF_DIR || ! -d $ACCUMULO_CONF_DIR ]]; then - echo "ACCUMULO_CONF_DIR=$ACCUMULO_CONF_DIR is not a valid directory. Please make sure it exists" - exit 1 -fi - -if [[ ! -f $ACCUMULO_CONF_DIR/accumulo-env.sh || ! -f $ACCUMULO_CONF_DIR/accumulo-site.xml ]]; then - echo "The configuration files 'accumulo-env.sh' & 'accumulo-site.xml' must exist in $ACCUMULO_CONF_DIR" - echo "Run 'accumulo create-config' to create them or copy them from $ACCUMULO_CONF_DIR/examples" - echo "Follow the instructions in INSTALL.md to edit them for your environment." - exit 1 -fi - -source "$ACCUMULO_CONF_DIR/accumulo-env.sh" - -## Variables that must be set - -: "${ACCUMULO_TSERVER_OPTS:?"variable is not set in accumulo-env.sh"}" -: "${ACCUMULO_MASTER_OPTS:?"variable is not set in accumulo-env.sh"}" -: "${ACCUMULO_MONITOR_OPTS:?"variable is not set in accumulo-env.sh"}" -: "${ACCUMULO_GC_OPTS:?"variable is not set in accumulo-env.sh"}" -: "${ACCUMULO_SHELL_OPTS:?"variable is not set in accumulo-env.sh"}" -: "${ACCUMULO_GENERAL_OPTS:?"variable is not set in accumulo-env.sh"}" -: "${ACCUMULO_OTHER_OPTS:?"variable is not set in accumulo-env.sh"}" - -### Variables that are derived - -# If not set in accumulo-env.sh, set env variables by program location. -test -z "${JAVA_HOME}" && locationByProgram java JAVA_HOME -test -z "${HADOOP_PREFIX}" && locationByProgram hadoop HADOOP_PREFIX -test -z "${ZOOKEEPER_HOME}" && locationByProgram zkCli.sh ZOOKEEPER_HOME - -export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_PREFIX/etc/hadoop}" -export ACCUMULO_HOME="${ACCUMULO_HOME:-$basedir}" -export ACCUMULO_BIN_DIR="${ACCUMULO_BIN_DIR:-$basedir/bin}" -export ACCUMULO_CONF_DIR="${ACCUMULO_CONF_DIR:-$basedir/conf}" -export ACCUMULO_LIB_DIR="${ACCUMULO_LIB_DIR:-$basedir/lib}" -export ACCUMULO_LIBEXEC_DIR="${ACCUMULO_LIBEXEC_DIR:-$basedir/libexec}" -export ACCUMULO_LOG_DIR="${ACCUMULO_LOG_DIR:-$basedir/logs}" -export ACCUMULO_PID_DIR="${ACCUMULO_PID_DIR:-$basedir/run}" - -# Make directories that may not exist -mkdir -p "${ACCUMULO_LOG_DIR}" 2>/dev/null -mkdir -p "${ACCUMULO_PID_DIR}" 2>/dev/null - -# Verify all directories exist -verify_env_dir "JAVA_HOME" "${JAVA_HOME}" -verify_env_dir "HADOOP_PREFIX" "${HADOOP_PREFIX}" -verify_env_dir "HADOOP_CONF_DIR" "${HADOOP_CONF_DIR}" -verify_env_dir "ZOOKEEPER_HOME" "${ZOOKEEPER_HOME}" -verify_env_dir "ACCUMULO_HOME" "${ACCUMULO_HOME}" -verify_env_dir "ACCUMULO_BIN_DIR" "${ACCUMULO_BIN_DIR}" -verify_env_dir "ACCUMULO_CONF_DIR" "${ACCUMULO_CONF_DIR}" -verify_env_dir "ACCUMULO_LIB_DIR" "${ACCUMULO_LIB_DIR}" -verify_env_dir "ACCUMULO_LIBEXEC_DIR" "${ACCUMULO_LIBEXEC_DIR}" -verify_env_dir "ACCUMULO_LOG_DIR" "${ACCUMULO_LOG_DIR}" -verify_env_dir "ACCUMULO_PID_DIR" "${ACCUMULO_PID_DIR}" - -## Verify Zookeeper installation -ZOOKEEPER_VERSION=$(find -L "$ZOOKEEPER_HOME" -maxdepth 1 -name "zookeeper-[0-9]*.jar" | head -1) -if [ -z "$ZOOKEEPER_VERSION" ]; then - echo "A Zookeeper JAR was not found in $ZOOKEEPER_HOME." - echo "Please check ZOOKEEPER_HOME, either globally or in accumulo-env.sh." - exit 1 -fi -ZOOKEEPER_VERSION=$(basename "${ZOOKEEPER_VERSION##*-}" .jar) - -if [[ "$ZOOKEEPER_VERSION" < "3.4.0" ]]; then - echo "WARN : Using Zookeeper $ZOOKEEPER_VERSION. Use version 3.4.0 or greater. Older versions may not work reliably."; -fi - -## Variables that have a default -export ACCUMULO_KILL_CMD=${ACCUMULO_KILL_CMD:-'kill -9 %p'} -export ACCUMULO_MONITOR_BIND_ALL=${ACCUMULO_MONITOR_BIND_ALL:-"true"} -export ACCUMULO_JAAS_CONF=${ACCUMULO_JAAS_CONF:-${ACCUMULO_CONF_DIR}/jaas.conf} -export ACCUMULO_KRB5_CONF=${ACCUMULO_KRB5_CONF:-${ACCUMULO_CONF_DIR}/krb5.conf} -export ACCUMULO_NUM_OUT_FILES=${ACCUMULO_NUM_OUT_FILES:-5} -export ACCUMULO_WATCHER=${ACCUMULO_WATCHER:-"false"} -export ACCUMULO_NUM_TSERVERS=${ACCUMULO_NUM_TSERVERS:-1} -export ACCUMULO_ENABLE_NUMACTL=${ACCUMULO_ENABLE_NUMACTL:-"false"} -export ACCUMULO_NUMACTL_OPTIONS=${ACCUMULO_NUMACTL_OPTIONS:-"--interleave=all"} - -# Validate that ACCUMULO_NUM_TSERVERS is a positive integer -if ! [[ $ACCUMULO_NUM_TSERVERS =~ ^[0-9]+$ ]]; then - echo "ACCUMULO_NUM_TSERVERS, when defined in accumulo-env.sh, should be a positive number, is '$ACCUMULO_NUM_TSERVERS'" - exit 1 -fi - -export HADOOP_HOME=$HADOOP_PREFIX -export HADOOP_HOME_WARN_SUPPRESS=true - -# See HADOOP-7154 and ACCUMULO-847 -export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-1} http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/assemble/libexec/service.sh ---------------------------------------------------------------------- diff --git a/assemble/libexec/service.sh b/assemble/libexec/service.sh deleted file mode 100755 index 9b47382..0000000 --- a/assemble/libexec/service.sh +++ /dev/null @@ -1,239 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -function print_usage { - cat <<EOF -Usage: service.sh <command> (<argument> ...) - -Commands: - start <host> <service> Starts <service> on local <host> - stop <host> <service> <signal> Stops <service> using <signal> on local <host> - -EOF - exit 1 -} - -function invalid_args { - echo -e "Invalid arguments: $1\n" - print_usage 1>&2 - exit 1 -} - -rotate_log () { - logfile=$1; - max_retained=$2; - if [[ ! "$max_retained" =~ ^[0-9]+$ ]] || [[ "$max_retained" -lt 1 ]] ; then - echo "ACCUMULO_NUM_OUT_FILES should be a positive number, but was '$max_retained'" - exit 1 - fi - - if [ -f "$logfile" ]; then # rotate logs - while [[ "$max_retained" -gt 1 ]]; do - prev=$(( max_retained - 1)) - [ -f "$logfile.$prev" ] && mv -f "$logfile.$prev" "$logfile.$max_retained" - max_retained=$prev - done - mv -f "$logfile" "$logfile.$max_retained"; - fi -} - -function start_service() { - - if [[ $# -ne 2 ]]; then - invalid_args "start command expects these arguments: <host> <service>" - fi - - host="$1" - service="$2" - - address=$host - loghost=$host - - # When the hostname provided is the alias/shortname, try to use the FQDN to make - # sure we send the right address to the Accumulo process. - if [[ "$host" = "$(hostname -s)" ]]; then - host="$(hostname -f)" - address="$host" - fi - - if [[ ${service} == "monitor" && ${ACCUMULO_MONITOR_BIND_ALL} == "true" ]]; then - address="0.0.0.0" - fi - - if [[ $service == "master" ]]; then - "$ACCUMULO_BIN_DIR/accumulo" org.apache.accumulo.master.state.SetGoalState NORMAL - fi - - COMMAND="${ACCUMULO_BIN_DIR}/accumulo" - if [ "${ACCUMULO_WATCHER}" = "true" ]; then - COMMAND="${ACCUMULO_LIBEXEC_DIR}/accumulo-watcher.sh ${loghost}" - fi - - OUTFILE="${ACCUMULO_LOG_DIR}/${service}_${loghost}.out" - ERRFILE="${ACCUMULO_LOG_DIR}/${service}_${loghost}.err" - - # Rotate the .out and .err files - rotate_log "$OUTFILE" "${ACCUMULO_NUM_OUT_FILES}" - rotate_log "$ERRFILE" "${ACCUMULO_NUM_OUT_FILES}" - - # NUMA sanity check - if [[ $ACCUMULO_NUM_TSERVERS -eq 1 && -n $TSERVER_NUMA_OPTIONS ]]; then - echo "TSERVER_NUMA_OPTIONS declared when ACCUMULO_NUM_TSERVERS is 1, use ACCUMULO_NUMACTL_OPTIONS instead" - exit 1 - fi - if [[ $ACCUMULO_NUM_TSERVERS -gt 1 && -n $TSERVER_NUMA_OPTIONS && ${#TSERVER_NUMA_OPTIONS[*]} -ne $ACCUMULO_NUM_TSERVERS ]]; then - echo "TSERVER_NUMA_OPTIONS is declared, but not the same size as ACCUMULO_NUM_TSERVERS" - exit 1 - fi - - if [[ "$service" != "tserver" || $ACCUMULO_NUM_TSERVERS -eq 1 ]]; then - # Check the pid file to figure out if its already running. - PID_FILE="${ACCUMULO_PID_DIR}/accumulo-${ACCUMULO_IDENT_STRING}-${service}.pid" - if [[ -f "${PID_FILE}" ]]; then - PID=$(cat "${PID_FILE}") - if kill -0 "$PID" 2>/dev/null; then - # Starting an already-started service shouldn't be an error per LSB - echo "$host : $service already running (${PID})" - exit 0 - fi - fi - echo "Starting $service on $host" - - ACCUMULO_ENABLE_NUMACTL=${ACCUMULO_ENABLE_NUMACTL:-"true"} - ACCUMULO_NUMACTL_OPTIONS=${ACCUMULO_NUMACTL_OPTIONS:-"--interleave=all"} - NUMA=$(which numactl 2>/dev/null) - NUMACTL_EXISTS="$?" - if [[ ( ${NUMACTL_EXISTS} -eq 0 ) && ( ${ACCUMULO_ENABLE_NUMACTL} == "true" ) ]] ; then - export NUMA_CMD="${NUMA} ${ACCUMULO_NUMACTL_OPTIONS}" - else - export NUMA_CMD="" - fi - - # Fork the process, store the pid - nohup ${NUMA_CMD} "$COMMAND" "${service}" --address "${address}" >"$OUTFILE" 2>"$ERRFILE" < /dev/null & - echo $! > "${PID_FILE}" - - else - - S="$service" - for (( t=1; t<=ACCUMULO_NUM_TSERVERS; t++)); do - - service="$S-$t" - - # Check the pid file to figure out if its already running. - PID_FILE="${ACCUMULO_PID_DIR}/accumulo-${ACCUMULO_IDENT_STRING}-${service}.pid" - if [[ -f "${PID_FILE}" ]]; then - PID=$(cat "${PID_FILE}") - if kill -0 "$PID" 2>/dev/null; then - # Starting an already-started service shouldn't be an error per LSB - echo "$host : $service already running (${PID})" - continue - fi - fi - echo "Starting $service on $host" - - ACCUMULO_NUMACTL_OPTIONS=${ACCUMULO_NUMACTL_OPTIONS:-"--interleave=all"} - ACCUMULO_NUMACTL_OPTIONS=${TSERVER_NUMA_OPTIONS[$t]} - if [[ "$ACCUMULO_ENABLE_NUMACTL" == "true" ]]; then - NUMA=$(which numactl 2>/dev/null) - NUMACTL_EXISTS=$? - if [[ ( ${NUMACTL_EXISTS} -eq 0 ) ]]; then - export NUMA_CMD="${NUMA} ${ACCUMULO_NUMACTL_OPTIONS}" - else - export NUMA_CMD="" - fi - fi - - # We want the files to be consistently named with the log files - # server_identifier_hostname.{out,err}, e.g. tserver_2_fqdn.out - OUTFILE="${ACCUMULO_LOG_DIR}/${S}_${t}_${loghost}.out" - ERRFILE="${ACCUMULO_LOG_DIR}/${S}_${t}_${loghost}.err" - - # Rotate the .out and .err files - rotate_log "$OUTFILE" "${ACCUMULO_NUM_OUT_FILES}" - rotate_log "$ERRFILE" "${ACCUMULO_NUM_OUT_FILES}" - - # Fork the process, store the pid - nohup ${NUMA_CMD} "$COMMAND" "${service}" --address "${address}" >"$OUTFILE" 2>"$ERRFILE" < /dev/null & - echo $! > "${PID_FILE}" - - done - fi - - # Check the max open files limit and selectively warn - MAX_FILES_OPEN=$(ulimit -n) - - if [[ -n $MAX_FILES_OPEN ]] ; then - MAX_FILES_RECOMMENDED=${MAX_FILES_RECOMMENDED:-32768} - if (( MAX_FILES_OPEN < MAX_FILES_RECOMMENDED )) - then - echo "WARN : Max open files on $host is $MAX_FILES_OPEN, recommend $MAX_FILES_RECOMMENDED" >&2 - fi - fi -} - -function stop_service() { - - if [[ $# -ne 3 ]]; then - invalid_args "stop command expects these arguments: <host> <service> <signal>" - fi - - host=$1 - service=$2 - signal=$3 - - for pid_file in ${ACCUMULO_PID_DIR}/accumulo-${ACCUMULO_IDENT_STRING}-${service}*.pid; do - if [[ -f "${pid_file}" ]]; then - echo "Stopping $service on $host"; - kill -s "$signal" "$(cat "${pid_file}")" 2>/dev/null - rm -f "${pid_file}" 2>/dev/null - fi - done -} - -function main() { - # Resolve libexec directory - SOURCE="${BASH_SOURCE[0]}" - while [[ -h "$SOURCE" ]]; do - libexec="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$libexec/$SOURCE" - done - libexec="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - - source "$libexec"/load-env.sh - - ACCUMULO_IDENT_STRING=${ACCUMULO_IDENT_STRING:-$USER} - - if [[ -z $1 ]]; then - invalid_args "<command> cannot be empty" - fi - - case "$1" in - start) - start_service "${@:2}" - ;; - stop) - stop_service "${@:2}" - ;; - *) - invalid_args "'$1' is an invalid <command>" - ;; - esac -} - -main "$@" http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/assemble/libexec/templates/accumulo-env.sh ---------------------------------------------------------------------- diff --git a/assemble/libexec/templates/accumulo-env.sh b/assemble/libexec/templates/accumulo-env.sh deleted file mode 100644 index 078ce6f..0000000 --- a/assemble/libexec/templates/accumulo-env.sh +++ /dev/null @@ -1,90 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -############################ -# Variables that must be set -############################ - -export ACCUMULO_TSERVER_OPTS="${tServerHigh_tServerLow} " -export ACCUMULO_MASTER_OPTS="${masterHigh_masterLow}" -export ACCUMULO_MONITOR_OPTS="${monitorHigh_monitorLow}" -export ACCUMULO_GC_OPTS="${gcHigh_gcLow}" -export ACCUMULO_SHELL_OPTS="${shellHigh_shellLow}" -export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -Djava.net.preferIPv4Stack=true -XX:+CMSClassUnloadingEnabled" -export ACCUMULO_OTHER_OPTS="${otherHigh_otherLow}" - -####################################### -# Variables that are derived if not set -####################################### - -## If not set below, Accumulo will settings in environment or derive -## locations using by looking up java, hadoop, zkiCli.sh on PATH - -# export JAVA_HOME=/path/to/java -# export HADOOP_PREFIX=/path/to/hadoop -# export HADOOP_CONF_DIR=/path/to/hadoop/etc/conf -# export ZOOKEEPER_HOME=/path/to/zookeeper - -# If not set below, Accumulo will derive these locations by determining the root of your -# installation and using the default locations - -# export ACCUMULO_LOG_DIR=/path/to/accumulo/log - -#################################################### -# Variables that have a default. Uncomment to change -#################################################### - -## Specifies what do when the JVM runs out of heap memory -# export ACCUMULO_KILL_CMD='kill -9 %p' -## Should the monitor bind to all network interfaces -- default: true -# export ACCUMULO_MONITOR_BIND_ALL="true" -## Location of jaas.conf file. Needed by JAAS for things like Kerberos based logins -# export ACCUMULO_JAAS_CONF=${ACCUMULO_CONF_DIR}/jaas.conf -## Location of krb5.conf file. Needed by Kerberos subsystems to find login servers -# export ACCUMULO_KRB5_CONF=${ACCUMULO_CONF_DIR}/krb5.conf -## The number of .out and .err files per process to retain -# export ACCUMULO_NUM_OUT_FILES=5 -## Should process be automatically restarted -# export ACCUMULO_WATCHER="false" -## Sets number of tablet servers to run on each node -# export ACCUMULO_NUM_TSERVERS=1 -## Enables NUMA control policy -# export ACCUMULO_ENABLE_NUMACTL="false" -## Sets NUMA control options -# export ACCUMULO_NUMACTL_OPTIONS="--interleave=all" - -############################################### -# Variables that are optional. Uncomment to set -############################################### - -## Example for configuring multiple tservers per host. Note that the ACCUMULO_NUMACTL_OPTIONS -## environment variable is used when ACCUMULO_NUM_TSERVERS is 1 to preserve backwards compatibility. -## If ACCUMULO_NUM_TSERVERS is greater than 2, then the TSERVER_NUMA_OPTIONS array is used if defined. -## If TSERVER_NUMA_OPTIONS is declared but not the correct size, then the service will not start. -# -# export ACCUMULO_NUM_TSERVERS=2 -# declare -a TSERVER_NUMA_OPTIONS -# TSERVER_NUMA_OPTIONS[1]="--cpunodebind 0" -# TSERVER_NUMA_OPTIONS[2]="--cpunodebind 1" - -## Optionally look for hadoop and accumulo native libraries for your -## platform in additional directories. (Use DYLD_LIBRARY_PATH on Mac OS X.) -## May not be necessary for Hadoop 2.x or using an RPM that installs to -## the correct system library directory. -# -# export LD_LIBRARY_PATH=${HADOOP_PREFIX}/lib/native/${PLATFORM}:${LD_LIBRARY_PATH} - http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/assemble/libexec/templates/accumulo-site.xml ---------------------------------------------------------------------- diff --git a/assemble/libexec/templates/accumulo-site.xml b/assemble/libexec/templates/accumulo-site.xml deleted file mode 100644 index a1f4153..0000000 --- a/assemble/libexec/templates/accumulo-site.xml +++ /dev/null @@ -1,175 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> - -<configuration> - <!-- This is the main configuration file for Apache Accumulo. Available configuration properties - (and their default values) can be found in the user manual (docs/accumulo_user_manual.html). --> - <property> - <name>instance.volumes</name> - <value></value> - <description>comma separated list of URIs for volumes. example: hdfs://localhost:9000/accumulo</description> - </property> - - <property> - <name>instance.zookeeper.host</name> - <value>localhost:2181</value> - <description>comma separated list of zookeeper servers</description> - </property> - - <property> - <name>instance.secret</name> - <value>DEFAULT</value> - <description>A secret unique to a given instance that all servers must know in order to communicate with one another. - Change it before initialization. To - change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret --old [oldpasswd] --new [newpasswd], - and then update this file. - </description> - </property> - - <property> - <name>tserver.memory.maps.max</name> - <value>${memMapMax}</value> - </property> - - <property> - <name>tserver.memory.maps.native.enabled</name> - <value>${nativeEnabled}</value> - </property> - - <property> - <name>tserver.cache.data.size</name> - <value>${cacheDataSize}</value> - </property> - - <property> - <name>tserver.cache.index.size</name> - <value>${cacheIndexSize}</value> - </property> - - <property> - <name>trace.token.property.password</name> - <!-- change this to the root user's password, and/or change the user below --> - <value>secret</value> - </property> - - <!-- Kerberos requirements --> - <property> - <name>instance.rpc.sasl.enabled</name> - <value>true</value> - </property> - - <property> - <name>general.kerberos.keytab</name> - <value>${keytab}</value> - </property> - - <property> - <name>general.kerberos.principal</name> - <value>${principal}</value> - </property> - - <property> - <name>trace.token.type</name> - <value>org.apache.accumulo.core.client.security.tokens.KerberosToken</value> - </property> - - <property> - <name>instance.security.authenticator</name> - <value>org.apache.accumulo.server.security.handler.KerberosAuthenticator</value> - </property> - - <property> - <name>instance.security.authorizor</name> - <value>org.apache.accumulo.server.security.handler.KerberosAuthorizor</value> - </property> - - <property> - <name>instance.security.permissionHandler</name> - <value>org.apache.accumulo.server.security.handler.KerberosPermissionHandler</value> - </property> - <!-- End Kerberos requirements --> - - <property> - <name>trace.user</name> - <value>${traceUser}</value> - </property> - - <property> - <name>tserver.sort.buffer.size</name> - <value>${sortBufferSize}</value> - </property> - - <property> - <name>tserver.walog.max.size</name> - <value>${waLogMaxSize}</value> - </property> -${mvnProjBaseDir} - <property> - <name>general.classpaths</name> - - <value> - <!-- Accumulo requirements --> - $ACCUMULO_HOME/lib/accumulo-server.jar, - $ACCUMULO_HOME/lib/accumulo-core.jar, - $ACCUMULO_HOME/lib/accumulo-start.jar, - $ACCUMULO_HOME/lib/accumulo-fate.jar, - $ACCUMULO_HOME/lib/accumulo-proxy.jar, - $ACCUMULO_HOME/lib/[^.].*.jar, - <!-- ZooKeeper requirements --> - $ZOOKEEPER_HOME/zookeeper[^.].*.jar, - <!-- Common Hadoop requirements --> - $HADOOP_CONF_DIR, - <!-- Hadoop 2 requirements --> - $HADOOP_PREFIX/share/hadoop/common/[^.].*.jar, - $HADOOP_PREFIX/share/hadoop/common/lib/(?!slf4j)[^.].*.jar, - $HADOOP_PREFIX/share/hadoop/hdfs/[^.].*.jar, - $HADOOP_PREFIX/share/hadoop/mapreduce/[^.].*.jar, - $HADOOP_PREFIX/share/hadoop/yarn/[^.].*.jar, - $HADOOP_PREFIX/share/hadoop/yarn/lib/jersey.*.jar, - <!-- End Hadoop 2 requirements --> - <!-- HDP 2.0 requirements --> - /usr/lib/hadoop/[^.].*.jar, - /usr/lib/hadoop/lib/[^.].*.jar, - /usr/lib/hadoop-hdfs/[^.].*.jar, - /usr/lib/hadoop-mapreduce/[^.].*.jar, - /usr/lib/hadoop-yarn/[^.].*.jar, - /usr/lib/hadoop-yarn/lib/jersey.*.jar, - <!-- End HDP 2.0 requirements --> - <!-- HDP 2.2 requirements --> - /usr/hdp/current/hadoop-client/[^.].*.jar, - /usr/hdp/current/hadoop-client/lib/(?!slf4j)[^.].*.jar, - /usr/hdp/current/hadoop-hdfs-client/[^.].*.jar, - /usr/hdp/current/hadoop-mapreduce-client/[^.].*.jar, - /usr/hdp/current/hadoop-yarn-client/[^.].*.jar, - /usr/hdp/current/hadoop-yarn-client/lib/jersey.*.jar, - /usr/hdp/current/hive-client/lib/hive-accumulo-handler.jar - <!-- End HDP 2.2 requirements --> - <!-- IOP 4.1 requirements --> - /usr/iop/current/hadoop-client/[^.].*.jar, - /usr/iop/current/hadoop-client/lib/(?!slf4j)[^.].*.jar, - /usr/iop/current/hadoop-hdfs-client/[^.].*.jar, - /usr/iop/current/hadoop-mapreduce-client/[^.].*.jar, - /usr/iop/current/hadoop-yarn-client/[^.].*.jar, - /usr/iop/current/hadoop-yarn-client/lib/jersey.*.jar, - /usr/iop/current/hive-client/lib/hive-accumulo-handler.jar - <!-- End IOP 4.1 requirements --> - </value> - <description>Classpaths that accumulo checks for updates and class files.</description> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/assemble/pom.xml ---------------------------------------------------------------------- diff --git a/assemble/pom.xml b/assemble/pom.xml index e35a3bc..ceef079 100644 --- a/assemble/pom.xml +++ b/assemble/pom.xml @@ -62,6 +62,11 @@ <optional>true</optional> </dependency> <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <optional>true</optional> + </dependency> + <dependency> <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-core</artifactId> <optional>true</optional> @@ -218,7 +223,7 @@ <outputScope>false</outputScope> <sort>true</sort> <!-- this list should match that in src/main/assemblies/component.xml --> - <includeArtifactIds>commons-math3,commons-vfs2,gson,guava,htrace-core,javax.servlet-api,jcommander,jetty-http,jetty-io,jetty-security,jetty-server,jetty-servlet,jetty-util,jline,libthrift,protobuf-java,slf4j-api,slf4j-log4j12</includeArtifactIds> + <includeArtifactIds>commons-math3,commons-vfs2,gson,guava,htrace-core,javax.servlet-api,jcommander,jetty-http,jetty-io,jetty-security,jetty-server,jetty-servlet,jetty-util,jline,libthrift,protobuf-java,slf4j-api,slf4j-log4j12,logj4</includeArtifactIds> <excludeTransitive>true</excludeTransitive> </configuration> </execution> http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/assemble/src/main/assemblies/component.xml ---------------------------------------------------------------------- diff --git a/assemble/src/main/assemblies/component.xml b/assemble/src/main/assemblies/component.xml index e34bccb..3aae23f 100644 --- a/assemble/src/main/assemblies/component.xml +++ b/assemble/src/main/assemblies/component.xml @@ -49,6 +49,7 @@ <include>org.apache.htrace:htrace-core</include> <include>org.slf4j:slf4j-api</include> <include>org.slf4j:slf4j-log4j12</include> + <include>log4j:log4j</include> </includes> <excludes> <exclude>${groupId}:${artifactId}-docs</exclude> @@ -73,32 +74,6 @@ <fileMode>0755</fileMode> </fileSet> <fileSet> - <directory>contrib</directory> - <directoryMode>0755</directoryMode> - <fileMode>0755</fileMode> - <includes> - <include>**/*.sh</include> - <include>check-tservers</include> - </includes> - </fileSet> - <fileSet> - <directory>libexec</directory> - <directoryMode>0755</directoryMode> - <fileMode>0755</fileMode> - <includes> - <include>**/*.sh</include> - </includes> - </fileSet> - <fileSet> - <directory>libexec/templates</directory> - <outputDirectory>libexec/templates</outputDirectory> - <directoryMode>0755</directoryMode> - <fileMode>0644</fileMode> - <includes> - <include>accumulo-site.xml</include> - </includes> - </fileSet> - <fileSet> <directory>./</directory> <outputDirectory>lib/ext</outputDirectory> <directoryMode>0755</directoryMode> @@ -117,36 +92,13 @@ <fileSet> <directory>conf</directory> <directoryMode>0755</directoryMode> - <fileMode>0755</fileMode> - <includes> - <include>**/*.sh</include> - </includes> - </fileSet> - <fileSet> - <directory>conf</directory> - <directoryMode>0755</directoryMode> <fileMode>0644</fileMode> - <excludes> - <exclude>**/*.sh</exclude> - </excludes> - </fileSet> - <fileSet> - <directory>target/example-configs</directory> - <outputDirectory>conf/examples</outputDirectory> - <directoryMode>0755</directoryMode> - <fileMode>0755</fileMode> - <includes> - <include>**/*.sh</include> - </includes> </fileSet> <fileSet> <directory>target/example-configs</directory> <outputDirectory>conf/examples</outputDirectory> <directoryMode>0755</directoryMode> <fileMode>0644</fileMode> - <excludes> - <exclude>**/*.sh</exclude> - </excludes> </fileSet> <!-- Lift generated thrift proxy code into its own directory --> <fileSet> http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/assemble/src/main/scripts/generate-example-configs.sh ---------------------------------------------------------------------- diff --git a/assemble/src/main/scripts/generate-example-configs.sh b/assemble/src/main/scripts/generate-example-configs.sh index 6a8e0a1..facf927 100755 --- a/assemble/src/main/scripts/generate-example-configs.sh +++ b/assemble/src/main/scripts/generate-example-configs.sh @@ -20,4 +20,4 @@ out=target/config.out echo 'Generating example scripts...' > $out -bin/accumulo create-config -o -d target/example-configs -s 2GB -j -v 2 >> $out 2>&1 +bin/accumulo-util create-config -o -d target/example-configs -s 2GB -j -v 2 >> $out 2>&1 http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/contrib/scripts/check-tservers.py ---------------------------------------------------------------------- diff --git a/contrib/scripts/check-tservers.py b/contrib/scripts/check-tservers.py new file mode 100755 index 0000000..7f9850e --- /dev/null +++ b/contrib/scripts/check-tservers.py @@ -0,0 +1,199 @@ +#! /usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script will check the configuration and uniformity of all the nodes in a cluster. +# Checks +# each node is reachable via ssh +# login identity is the same +# the physical memory is the same +# the mounts are the same on each machine +# a set of writable locations (typically different disks) are in fact writable +# +# In order to check for writable partitions, you must configure the WRITABLE variable below. +# + +import subprocess +import time +import select +import os +import sys +import fcntl +import signal +if not sys.platform.startswith('linux'): + sys.stderr.write('This script only works on linux, sorry.\n') + sys.exit(1) + +TIMEOUT = 5 +WRITABLE = [] +#WRITABLE = ['/srv/hdfs1', '/srv/hdfs2', '/srv/hdfs3'] + +def ssh(tserver, *args): + 'execute a command on a remote tserver and return the Popen handle' + handle = subprocess.Popen( ('ssh', '-o', 'StrictHostKeyChecking=no', '-q', '-A', '-n', tserver) + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + handle.tserver = tserver + handle.finished = False + handle.out = '' + return handle + +def wait(handles, seconds): + 'wait for lots of handles simultaneously, and kill anything that doesn\'t return in seconds time\n' + 'Note that stdout will be stored on the handle as the "out" field and "finished" will be set to True' + handles = handles[:] + stop = time.time() + seconds + for h in handles: + fcntl.fcntl(h.stdout, fcntl.F_SETFL, os.O_NONBLOCK) + while handles and time.time() < stop: + wait = min(0, stop - time.time()) + handleMap = dict( [(h.stdout, h) for h in handles] ) + rd, wr, err = select.select(handleMap.keys(), [], [], wait) + for r in rd: + handle = handleMap[r] + while 1: + more = handle.stdout.read(1024) + if more == '': + handles.remove(handle) + handle.poll() + handle.wait() + handle.finished = True + handle.out += more + if len(more) < 1024: + break + for handle in handles: + os.kill(handle.pid, signal.SIGKILL) + handle.poll() + +def runAll(tservers, *cmd): + 'Run the given command on all the tservers, returns Popen handles' + handles = [] + for tserver in tservers: + handles.append(ssh(tserver, *cmd)) + wait(handles, TIMEOUT) + return handles + +def checkIdentity(tservers): + 'Ensure the login identity is consistent across the tservers' + handles = runAll(tservers, 'id', '-u', '-n') + bad = set() + myIdentity = os.popen('id -u -n').read().strip() + for h in handles: + if not h.finished or h.returncode != 0: + print '#', 'cannot look at identity on', h.tserver + bad.add(h.tserver) + else: + identity = h.out.strip() + if identity != myIdentity: + print '#', h.tserver, 'inconsistent identity', identity + bad.add(h.tserver) + return bad + +def checkMemory(tservers): + 'Run free on all tservers and look for weird results' + handles = runAll(tservers, 'free') + bad = set() + mem = {} + swap = {} + for h in handles: + if not h.finished or h.returncode != 0: + print '#', 'cannot look at memory on', h.tserver + bad.add(h.tserver) + else: + if h.out.find('Swap:') < 0: + print '#',h.tserver,'has no swap' + bad.add(h.tserver) + continue + lines = h.out.split('\n') + for line in lines: + if line.startswith('Mem:'): + mem.setdefault(line.split()[1],set()).add(h.tserver) + if line.startswith('Swap:'): + swap.setdefault(line.split()[1],set()).add(h.tserver) + # order memory sizes by most common + mems = sorted([(len(v), k, v) for k, v in mem.items()], reverse=True) + mostCommon = float(mems[0][1]) + for _, size, tservers in mems[1:]: + fract = abs(mostCommon - float(size)) / mostCommon + if fract > 0.05: + print '#',', '.join(tservers), ': unusual memory size', size + bad.update(tservers) + swaps = sorted([(len(v), k, v) for k, v in swap.items()], reverse=True) + mostCommon = float(mems[0][1]) + for _, size, tservers in swaps[1:]: + fract = abs(mostCommon - float(size) / mostCommon) + if fract > 0.05: + print '#',', '.join(tservers), ': unusual swap size', size + bad.update(tservers) + return bad + +def checkWritable(tservers): + 'Touch all the directories that should be writable by this user return any nodes that fail' + if not WRITABLE: + print '# WRITABLE value not configured, not checking partitions' + return [] + handles = runAll(tservers, 'touch', *WRITABLE) + bad = set() + for h in handles: + if not h.finished or h.returncode != 0: + bad.add(h.tserver) + print '#', h.tserver, 'some drives are not writable' + return bad + +def checkMounts(tservers): + 'Check the file systems that are mounted and report any that are unusual' + handles = runAll(tservers, 'mount') + mounts = {} + finished = set() + bad = set() + for handle in handles: + if handle.finished and handle.returncode == 0: + for line in handle.out.split('\n'): + words = line.split() + if len(words) < 5: continue + if words[4] == 'nfs': continue + if words[0].find(':/') >= 0: continue + mount = words[2] + mounts.setdefault(mount, set()).add(handle.tserver) + finished.add(handle.tserver) + else: + bad.add(handle.tserver) + print '#', handle.tserver, 'did not finish' + for m in sorted(mounts.keys()): + diff = finished - mounts[m] + if diff: + bad.update(diff) + print '#', m, 'not mounted on', ', '.join(diff) + return bad + +def main(argv): + if len(argv) < 1: + sys.stderr.write('Usage: check_tservers tservers\n') + sys.exit(1) + sys.stdin.close() + tservers = set() + for tserver in open(argv[0]): + hashPos = tserver.find('#') + if hashPos >= 0: + tserver = tserver[:hashPos] + tserver = tserver.strip() + if not tserver: continue + tservers.add(tserver) + bad = set() + for test in checkIdentity, checkMemory, checkMounts, checkWritable: + bad.update(test(tservers - bad)) + for tserver in sorted(tservers - bad): + print tserver + +main(sys.argv[1:]) http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java index c128c80..f724936 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java @@ -222,7 +222,7 @@ class ConfigurationDocGen { } /** - * Generates documentation for conf/accumulo-site.xml file usage. Arguments are: "--generate-asciidoc filename" + * Generates documentation for accumulo-site.xml file usage. Arguments are: "--generate-asciidoc filename" * * @param args * command-line arguments http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/core/src/main/java/org/apache/accumulo/core/conf/Property.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index 2f8b4fa..faa67f9 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -131,7 +131,7 @@ public enum Property { INSTANCE_SECRET("instance.secret", "DEFAULT", PropertyType.STRING, "A secret unique to a given instance that all servers must know in order to communicate with one another." + "It should be changed prior to the initialization of Accumulo. To change it after Accumulo has been initialized, use the ChangeSecret tool " - + "and then update conf/accumulo-site.xml everywhere. Before using the ChangeSecret tool, make sure Accumulo is not running and you are logged " + + "and then update accumulo-site.xml everywhere. Before using the ChangeSecret tool, make sure Accumulo is not running and you are logged " + "in as the user that controls Accumulo files in HDFS. To use the ChangeSecret tool, run the command: " + "./bin/accumulo org.apache.accumulo.server.util.ChangeSecret"), INSTANCE_VOLUMES("instance.volumes", "", PropertyType.STRING,
