sijie closed pull request #1407: Refactor bookkeeper bash scripts and move dlog script to root bin directory URL: https://github.com/apache/bookkeeper/pull/1407
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/bin/bookkeeper b/bin/bookkeeper index e2bd30eed..47c4e3f0b 100755 --- a/bin/bookkeeper +++ b/bin/bookkeeper @@ -18,111 +18,23 @@ # * limitations under the License. # */ -if [ -f /sbin/sysctl ]; then - # check if net.ipv6.bindv6only is set to 1 - bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null) - if [ -n "$bindv6only" ] && [ "$bindv6only" -eq "1" ] - then - echo "Error: \"net.ipv6.bindv6only\" is set to 1 - Java networking could be broken" - echo "For more info (the following page also applies to bookkeeper): http://wiki.apache.org/hadoop/HadoopIPv6"; - exit 1 - fi -fi - -# See the following page for extensive details on setting -# up the JVM to accept JMX remote management: -# http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html -# by default we allow local JMX connections -if [ "x$JMXLOCALONLY" = "x" ] -then - JMXLOCALONLY=false -fi - -if [ "x$JMXDISABLE" = "x" ] -then - # for some reason these two options are necessary on jdk6 on Ubuntu - # accord to the docs they are not necessary, but otw jconsole cannot - # do a local attach - JMX_ARGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY" -else - echo "JMX disabled by user request" >&2 -fi - BINDIR=`dirname "$0"` -BK_HOME=`cd $BINDIR/..;pwd` - -DEFAULT_CONF=$BK_HOME/conf/bk_server.conf -DEFAULT_LOG_CONF=$BK_HOME/conf/log4j.properties -SERVER_HOME=${BK_HOME}/bookkeeper-server - -source $BK_HOME/conf/bkenv.sh - -LOCALBOOKIES_CONFIG_DIR="${LOCALBOOKIES_CONFIG_DIR:-/tmp/localbookies-config}" - -# Check for the java to use -if [[ -z $JAVA_HOME ]]; then - JAVA=$(which java) - if [ $? = 0 ]; then - echo "JAVA_HOME not set, using java from PATH. ($JAVA)" - else - echo "Error: JAVA_HOME not set, and no java executable found in $PATH." 1>&2 - exit 1 - fi -else - JAVA=$JAVA_HOME/bin/java -fi +BK_HOME=`cd ${BINDIR}/..;pwd` -find-server-jar() { - DIR=$1 - if [ -d $DIR ]; then - cd $DIR - for f in *.jar; do - if [[ $f =~ ^(org.apache.bookkeeper-)?bookkeeper-server-[0-9\\.]*(-SNAPSHOT)?.jar$ ]]; then - echo $DIR/$f - return - fi - done - fi -} +source ${BK_HOME}/bin/common.sh -RELEASE_JAR=$(find-server-jar ${BK_HOME}) -if [ -n "${RELEASE_JAR}" ]; then - BOOKIE_JAR=${RELEASE_JAR} -else - RELEASE_JAR=$(find-server-jar ${BK_HOME}/lib) - if [ -n "${RELEASE_JAR}" ]; then - BOOKIE_JAR=${RELEASE_JAR} - fi -fi - -BUILT_JAR=$(find-server-jar ${SERVER_HOME}/target) +BOOKIE_MODULE_PATH=bookkeeper-server +BOOKIE_MODULE_NAME="(org.apache.bookkeeper-)?bookkeeper-server" +BOOKIE_MODULE_HOME=${BK_HOME}/${BOOKIE_MODULE_PATH} -if [ -z "${BUILT_JAR}" ] && [ -z "${BOOKIE_JAR}" ]; then - echo "Couldn't find bookkeeper jar." - read -p "Do you want me to run \`mvn package -DskiptTests\` for you ? " answer - case "${answer:0:1}" in - y|Y ) - mvn package -DskipTests - ;; - * ) - exit 1 - ;; - esac +# find the module jar +BOOKIE_JAR=$(find_module_jar ${BOOKIE_MODULE_PATH} ${BOOKIE_MODULE_NAME}) - BUILT_JAR=$(find-server-jar ${SERVER_HOME}/target) - if [ -n "${BUILT_JAR}" ]; then - BOOKIE_JAR=$BUILT_JAR - fi -fi +# set up the classpath +BOOKIE_CLASSPATH=$(set_module_classpath ${BOOKIE_MODULE_PATH}) -if [ -e "${BUILT_JAR}" ]; then - BOOKIE_JAR="${BUILT_JAR}" -fi - -if [ ! -e "${BOOKIE_JAR}" ]; then - echo "Could not find bookkeeper jar." - exit 1 -fi +# default variables +DEFAULT_CONF=${BK_HOME}/conf/bk_server.conf bookkeeper_help() { cat <<EOF @@ -138,8 +50,8 @@ where command is one of: or command is the full name of a class with a defined main() method. Environment variables: - BOOKIE_LOG_CONF Log4j configuration file (default $DEFAULT_LOG_CONF) - BOOKIE_CONF Configuration file (default: $DEFAULT_CONF) + BOOKIE_LOG_CONF Log4j configuration file (default ${DEFAULT_LOG_CONF}) + BOOKIE_CONF Configuration file (default: ${DEFAULT_CONF}) BOOKIE_EXTRA_OPTS Extra options to be passed to the jvm BOOKIE_EXTRA_CLASSPATH Add extra paths to the bookkeeper classpath ENTRY_FORMATTER_CLASS Entry formatter class to format entries. @@ -150,31 +62,6 @@ These variable can also be set in conf/bkenv.sh EOF } -add_maven_deps_to_classpath() { - MVN="mvn" - if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn - fi - - # Need to generate classpath from maven pom. This is costly so generate it - # and cache it. Save the file into our target dir so a mvn clean will get - # clean it up and force us create a new one. - f="${SERVER_HOME}/target/cached_classpath.txt" - if [ ! -f "${f}" ] - then - ${MVN} -f "${SERVER_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null - fi - BOOKIE_CLASSPATH=${CLASSPATH}:`cat "${f}"` -} - -if [ -d "$BK_HOME/lib" ]; then - for i in $BK_HOME/lib/*.jar; do - BOOKIE_CLASSPATH=$BOOKIE_CLASSPATH:$i - done -else - add_maven_deps_to_classpath -fi - # if no args specified, show usage if [ $# = 0 ]; then bookkeeper_help; @@ -185,103 +72,68 @@ fi COMMAND=$1 shift -if [ $COMMAND == "shell" ]; then - DEFAULT_LOG_CONF=$BK_HOME/conf/log4j.shell.properties - if [[ $1 == "-localbookie" ]]; then - if [[ $2 == *:* ]]; - then - BOOKIE_CONF=$LOCALBOOKIES_CONFIG_DIR/$2.conf - shift 2 - else - BOOKIE_CONF=$LOCALBOOKIES_CONFIG_DIR/baseconf.conf - shift - fi +LOCALBOOKIES_CONFIG_DIR="${LOCALBOOKIES_CONFIG_DIR:-/tmp/localbookies-config}" +if [ ${COMMAND} == "shell" ]; then + DEFAULT_LOG_CONF=${BK_HOME}/conf/log4j.shell.properties + if [[ $1 == "-localbookie" ]]; then + if [[ $2 == *:* ]]; + then + BOOKIE_CONF=${LOCALBOOKIES_CONFIG_DIR}/$2.conf + shift 2 + else + BOOKIE_CONF=${LOCALBOOKIES_CONFIG_DIR}/baseconf.conf + shift fi + fi fi if [ -z "$BOOKIE_CONF" ]; then - BOOKIE_CONF=$DEFAULT_CONF + BOOKIE_CONF=${DEFAULT_CONF} fi +# Configure logging if [ -z "$BOOKIE_LOG_CONF" ]; then - BOOKIE_LOG_CONF=$DEFAULT_LOG_CONF + BOOKIE_LOG_CONF=${DEFAULT_LOG_CONF} fi - BOOKIE_LOG_DIR=${BOOKIE_LOG_DIR:-"$BK_HOME/logs"} -if [ ! -d ${BOOKIE_LOG_DIR} ]; then - mkdir ${BOOKIE_LOG_DIR} -fi - -# Configure JVM settings -DEFAULT_BOOKIE_GC_OPTS="-XX:+UseG1GC \ - -XX:MaxGCPauseMillis=10 \ - -XX:+ParallelRefProcEnabled \ - -XX:+UnlockExperimentalVMOptions \ - -XX:+AggressiveOpts \ - -XX:+DoEscapeAnalysis \ - -XX:ParallelGCThreads=32 \ - -XX:ConcGCThreads=32 \ - -XX:G1NewSizePercent=50 \ - -XX:+DisableExplicitGC \ - -XX:-ResizePLAB" -DEFAULT_BOOKIE_GC_LOGGING_OPTS="-XX:+PrintGCDetails \ - -XX:+PrintGCApplicationStoppedTime \ - -XX:+UseGCLogFileRotation \ - -XX:NumberOfGCLogFiles=5 \ - -XX:GCLogFileSize=64m \ - -Xloggc:${BOOKIE_LOG_DIR}/gc_%p.log" -BOOKIE_MAX_HEAP_MEMORY=${BOOKIE_MAX_HEAP_MEMORY:-"1g"} -BOOKIE_MIN_HEAP_MEMORY=${BOOKIE_MIN_HEAP_MEMORY:-"1g"} -BOOKIE_MAX_DIRECT_MEMORY=${BOOKIE_MAX_DIRECT_MEMORY:-"2g"} -BOOKIE_MEM_OPTS=${BOOKIE_MEM_OPTS:-"-Xms${BOOKIE_MAX_HEAP_MEMORY} -Xmx${BOOKIE_MIN_HEAP_MEMORY} -XX:MaxDirectMemorySize=${BOOKIE_MAX_DIRECT_MEMORY}"} -BOOKIE_GC_OPTS=${BOOKIE_GC_OPTS:-"${DEFAULT_BOOKIE_GC_OPTS}"} -BOOKIE_GC_LOGGING_OPTS=${BOOKIE_GC_LOGGING_OPTS:-"${DEFAULT_BOOKIE_GC_LOGGING_OPTS}"} +BOOKIE_LOG_FILE=${BOOKIE_LOG_FILE:-"bookkeeper-server.log"} +BOOKIE_ROOT_LOGGER=${BOOKIE_ROOT_LOGGER:-"INFO,CONSOLE"} +# Configure the classpath BOOKIE_CLASSPATH="$BOOKIE_JAR:$BOOKIE_CLASSPATH:$BOOKIE_EXTRA_CLASSPATH" BOOKIE_CLASSPATH="`dirname $BOOKIE_LOG_CONF`:$BOOKIE_CLASSPATH" -OPTS="$OPTS -Dlog4j.configuration=`basename $BOOKIE_LOG_CONF`" - -OPTS="-cp $BOOKIE_CLASSPATH $OPTS" -# Disable ipv6 as it can cause issues -OPTS="$OPTS -Djava.net.preferIPv4Stack=true" -# Disable netty leak detection -NETTY_LEAK_DETECTION_LEVEL=${NETTY_LEAK_DETECTION_LEVEL:-"disabled"} -OPTS="$OPTS -Dio.netty.leakDetectionLevel=${NETTY_LEAK_DETECTION_LEVEL}" -NETTY_RECYCLER_MAXCAPACITY=${NETTY_RECYCLER_MAXCAPACITY:-"1000"} -OPTS="$OPTS -Dio.netty.recycler.maxCapacity.default=${NETTY_RECYCLER_MAXCAPACITY}" -NETTY_RECYCLER_LINKCAPACITY=${NETTY_RECYCLER_LINKCAPACITY:-"1024"} -OPTS="$OPTS -Dio.netty.recycler.linkCapacity=${NETTY_RECYCLER_LINKCAPACITY}" +# Build the OPTS +BOOKIE_OPTS=$(build_bookie_opts) +GC_OPTS=$(build_bookie_jvm_opts ${BOOKIE_LOG_DIR} "gc_%p.log") +NETTY_OPTS=$(build_netty_opts) +LOGGING_OPTS=$(build_logging_opts ${BOOKIE_LOG_CONF} ${BOOKIE_LOG_DIR} ${BOOKIE_LOG_FILE} ${BOOKIE_ROOT_LOGGER}) -OPTS="$OPTS $BOOKIE_MEM_OPTS $BOOKIE_GC_OPTS $BOOKIE_GC_LOGGING_OPTS $BOOKIE_PERF_OPTS $BOOKIE_EXTRA_OPTS" +OPTS="${OPTS} -cp ${BOOKIE_CLASSPATH} ${BOOKIE_OPTS} ${GC_OPTS} ${NETTY_OPTS} ${LOGGING_OPTS} ${BOOKIE_EXTRA_OPTS}" -# log directory & file -BOOKIE_ROOT_LOGGER=${BOOKIE_ROOT_LOGGER:-"INFO,CONSOLE"} -BOOKIE_LOG_FILE=${BOOKIE_LOG_FILE:-"bookkeeper-server.log"} - -#Configure log configuration system properties -OPTS="$OPTS -Dbookkeeper.root.logger=$BOOKIE_ROOT_LOGGER" -OPTS="$OPTS -Dbookkeeper.log.dir=$BOOKIE_LOG_DIR" -OPTS="$OPTS -Dbookkeeper.log.file=$BOOKIE_LOG_FILE" +# Create log dir if it doesn't exist +if [ ! -d ${BOOKIE_LOG_DIR} ]; then + mkdir ${BOOKIE_LOG_DIR} +fi #Change to BK_HOME to support relative paths cd "$BK_HOME" -if [ $COMMAND == "bookie" ]; then - exec $JAVA $OPTS $JMX_ARGS org.apache.bookkeeper.server.Main --conf $BOOKIE_CONF $@ -elif [ $COMMAND == "autorecovery" ]; then - exec $JAVA $OPTS $JMX_ARGS org.apache.bookkeeper.replication.AutoRecoveryMain --conf $BOOKIE_CONF $@ -elif [ $COMMAND == "localbookie" ]; then +if [ ${COMMAND} == "bookie" ]; then + exec ${JAVA} ${OPTS} ${JMX_ARGS} org.apache.bookkeeper.server.Main --conf ${BOOKIE_CONF} $@ +elif [ ${COMMAND} == "autorecovery" ]; then + exec ${JAVA} ${OPTS} ${JMX_ARGS} org.apache.bookkeeper.replication.AutoRecoveryMain --conf ${BOOKIE_CONF} $@ +elif [ ${COMMAND} == "localbookie" ]; then NUMBER=$1 shift - exec $JAVA $OPTS $JMX_ARGS -Dzookeeper.4lw.commands.whitelist='*' org.apache.bookkeeper.util.LocalBookKeeper $NUMBER $BOOKIE_CONF $@ -elif [ $COMMAND == "upgrade" ]; then - exec $JAVA $OPTS org.apache.bookkeeper.bookie.FileSystemUpgrade --conf $BOOKIE_CONF $@ -elif [ $COMMAND == "shell" ]; then + exec ${JAVA} ${OPTS} ${JMX_ARGS} -Dzookeeper.4lw.commands.whitelist='*' org.apache.bookkeeper.util.LocalBookKeeper ${NUMBER} ${BOOKIE_CONF} $@ +elif [ ${COMMAND} == "upgrade" ]; then + exec ${JAVA} ${OPTS} org.apache.bookkeeper.bookie.FileSystemUpgrade --conf ${BOOKIE_CONF} $@ +elif [ ${COMMAND} == "shell" ]; then ENTRY_FORMATTER_ARG="-DentryFormatterClass=${ENTRY_FORMATTER_CLASS:-org.apache.bookkeeper.util.StringEntryFormatter}" - exec $JAVA $OPTS $ENTRY_FORMATTER_ARG org.apache.bookkeeper.bookie.BookieShell -conf $BOOKIE_CONF $@ -elif [ $COMMAND == "help" ]; then + exec ${JAVA} ${OPTS} ${ENTRY_FORMATTER_ARG} org.apache.bookkeeper.bookie.BookieShell -conf ${BOOKIE_CONF} $@ +elif [ ${COMMAND} == "help" ]; then bookkeeper_help; else - exec $JAVA $OPTS $COMMAND $@ + exec ${JAVA} ${OPTS} ${COMMAND} $@ fi diff --git a/bin/bookkeeper-cli b/bin/bookkeeper-cli index 31d9f6027..2e53be057 100755 --- a/bin/bookkeeper-cli +++ b/bin/bookkeeper-cli @@ -21,133 +21,46 @@ # BookKeeper CLI (experimental) BINDIR=`dirname "$0"` -BK_HOME=`cd $BINDIR/..;pwd` +BK_HOME=`cd ${BINDIR}/..;pwd` -DEFAULT_CONF=$BK_HOME/conf/bk_server.conf -DEFAULT_LOG_CONF=$BK_HOME/conf/log4j.cli.properties -TOOLS_HOME=${BK_HOME}/bookkeeper-tools +source ${BK_HOME}/bin/common.sh +source ${BK_HOME}/conf/bk_cli_env.sh -source $BK_HOME/conf/bk_cli_env.sh +CLI_MODULE_PATH=bookkeeper-tools +CLI_MODULE_NAME="(org.apache.bookkeeper-)?bookkeeper-tools" +CLI_MODULE_HOME=${BK_HOME}/${CLI_MODULE_PATH} -# Check for the java to use -if [[ -z $JAVA_HOME ]]; then - JAVA=$(which java) - if [ $? = 0 ]; then - echo "JAVA_HOME not set, using java from PATH. ($JAVA)" - else - echo "Error: JAVA_HOME not set, and no java executable found in $PATH." 1>&2 - exit 1 - fi -else - JAVA=$JAVA_HOME/bin/java -fi - -find_cli_jar() { - DIR=$1 - if [ -d $DIR ]; then - cd $DIR - for f in *.jar; do - if [[ $f =~ ^(org.apache.bookkeeper-)?bookkeeper-tools-[0-9\\.]*(-SNAPSHOT)?.jar$ ]]; then - echo $DIR/$f - return - fi - done - fi -} - -RELEASE_JAR=$(find_cli_jar ${BK_HOME}) -if [ -n "${RELEASE_JAR}" ]; then - CLI_JAR=${RELEASE_JAR} -else - RELEASE_JAR=$(find_cli_jar ${BK_HOME}/lib) - if [ -n "${RELEASE_JAR}" ]; then - CLI_JAR=${RELEASE_JAR} - fi -fi - -BUILT_JAR=$(find_cli_jar ${TOOLS_HOME}/target) - -if [ -z "${BUILT_JAR}" ] && [ -z "${CLI_JAR}" ]; then - echo "Couldn't find bookkeeper jar." - read -p "Do you want me to run \`mvn package -DskiptTests\` for you ? " answer - case "${answer:0:1}" in - y|Y ) - mvn package -DskipTests - ;; - * ) - exit 1 - ;; - esac - - BUILT_JAR=$(find_cli_jar ${TOOLS_HOME}/target) - if [ -n "${BUILT_JAR}" ]; then - CLI_JAR=$BUILT_JAR - fi -fi - -if [ -e "${BUILT_JAR}" ]; then - CLI_JAR="${BUILT_JAR}" -fi - -if [ ! -e "${CLI_JAR}" ]; then - echo "Could not find bookkeeper cli jar." - exit 1 -fi +# find the module jar +CLI_JAR=$(find_module_jar ${CLI_MODULE_PATH} ${CLI_MODULE_NAME}) -add_maven_deps_to_classpath() { - MVN="mvn" - if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn - fi +# set up the classpath +CLI_CLASSPATH=$(set_module_classpath ${CLI_MODULE_PATH}) - # Need to generate classpath from maven pom. This is costly so generate it - # and cache it. Save the file into our target dir so a mvn clean will get - # clean it up and force us create a new one. - f="${TOOLS_HOME}/target/cached_classpath.txt" - if [ ! -f "${f}" ] - then - ${MVN} -f "${TOOLS_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null - fi - CLI_CLASSPATH=${CLASSPATH}:`cat "${f}"` -} - -if [ -d "$BK_HOME/lib" ]; then - for i in $BK_HOME/lib/*.jar; do - CLI_CLASSPATH=$CLI_CLASSPATH:$i - done -else - add_maven_deps_to_classpath -fi - -if [ -z "$CLI_CONF" ]; then - CLI_CONF=$DEFAULT_CONF +DEFAULT_CONF=${BK_HOME}/conf/bk_server.conf +if [ -z "${CLI_CONF}" ]; then + CLI_CONF=${DEFAULT_CONF} fi -if [ -z "$CLI_LOG_CONF" ]; then - CLI_LOG_CONF=$DEFAULT_LOG_CONF +DEFAULT_LOG_CONF=${BK_HOME}/conf/log4j.cli.properties +if [ -z "${CLI_LOG_CONF}" ]; then + CLI_LOG_CONF=${DEFAULT_LOG_CONF} fi +CLI_LOG_DIR=${CLI_LOG_DIR:-"$BK_HOME/logs"} +CLI_LOG_FILE=${CLI_LOG_FILE:-"bookkeeper-cli.log"} +CLI_ROOT_LOGGER=${CLI_ROOT_LOGGER:-"INFO,CONSOLE"} +# Configure the classpath CLI_CLASSPATH="$CLI_JAR:$CLI_CLASSPATH:$CLI_EXTRA_CLASSPATH" CLI_CLASSPATH="`dirname $CLI_LOG_CONF`:$CLI_CLASSPATH" -OPTS="$OPTS -Dlog4j.configuration=`basename $CLI_LOG_CONF`" -OPTS="-cp $CLI_CLASSPATH $OPTS" - -OPTS="$OPTS $CLI_EXTRA_OPTS" - -# Disable ipv6 as it can cause issues -OPTS="$OPTS -Djava.net.preferIPv4Stack=true" - -# log directory & file -CLI_ROOT_LOGGER=${CLI_ROOT_LOGGER:-"INFO,CONSOLE"} -CLI_LOG_DIR=${CLI_LOG_DIR:-"$BK_HOME/logs"} -CLI_LOG_FILE=${CLI_LOG_FILE:-"bookkeeper-cli.log"} +# Build the OPTs +BOOKIE_OPTS=$(build_bookie_opts) +GC_OPTS=$(build_cli_jvm_opts ${CLI_LOG_DIR} "bookkeeper-cli-gc.log") +NETTY_OPTS=$(build_netty_opts) +LOGGING_OPTS=$(build_logging_opts ${CLI_LOG_CONF} ${CLI_LOG_DIR} ${CLI_LOG_FILE} ${CLI_ROOT_LOGGER}) -#Configure log configuration system properties -OPTS="$OPTS -Dbookkeeper.cli.root.logger=$CLI_ROOT_LOGGER" -OPTS="$OPTS -Dbookkeeper.cli.log.dir=$CLI_LOG_DIR" -OPTS="$OPTS -Dbookkeeper.cli.log.file=$CLI_LOG_FILE" +OPTS="${OPTS} -cp ${CLI_CLASSPATH} ${BOOKIE_OPTS} ${GC_OPTS} ${NETTY_OPTS} ${LOGGING_OPTS} ${CLI_EXTRA_OPTS}" #Change to BK_HOME to support relative paths cd "$BK_HOME" -exec $JAVA $OPTS org.apache.bookkeeper.tools.cli.BookieShell --conf $CLI_CONF $@ +exec ${JAVA} ${OPTS} org.apache.bookkeeper.tools.cli.BookKeeperCLI --conf ${CLI_CONF} $@ diff --git a/bin/common.sh b/bin/common.sh new file mode 100755 index 000000000..0c048c9c6 --- /dev/null +++ b/bin/common.sh @@ -0,0 +1,251 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Check net.ipv6.bindv6only +if [ -f /sbin/sysctl ]; then + # check if net.ipv6.bindv6only is set to 1 + bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null) + if [ -n "$bindv6only" ] && [ "$bindv6only" -eq "1" ] + then + echo "Error: \"net.ipv6.bindv6only\" is set to 1 - Java networking could be broken" + echo "For more info (the following page also applies to bookkeeper): http://wiki.apache.org/hadoop/HadoopIPv6"; + exit 1 + fi +fi + +# See the following page for extensive details on setting +# up the JVM to accept JMX remote management: +# http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# by default we allow local JMX connections +if [ "x$JMXLOCALONLY" = "x" ] +then + JMXLOCALONLY=false +fi + +if [ "x$JMXDISABLE" = "x" ] +then + # for some reason these two options are necessary on jdk6 on Ubuntu + # accord to the docs they are not necessary, but otw jconsole cannot + # do a local attach + JMX_ARGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY" +else + echo "JMX disabled by user request" >&2 +fi + +# Check for the java to use +if [[ -z ${JAVA_HOME} ]]; then + JAVA=$(which java) + if [ $? = 0 ]; then + echo "JAVA_HOME not set, using java from PATH. ($JAVA)" + else + echo "Error: JAVA_HOME not set, and no java executable found in $PATH." 1>&2 + exit 1 + fi +else + JAVA=${JAVA_HOME}/bin/java +fi + +BINDIR=${BK_BINDIR:-"`dirname "$0"`"} +BK_HOME=${BK_HOME:-"`cd ${BINDIR}/..;pwd`"} +BK_CONFDIR=${BK_HOME}/conf +DEFAULT_LOG_CONF=${BK_CONFDIR}/log4j.properties + +source ${BK_CONFDIR}/nettyenv.sh +source ${BK_CONFDIR}/bkenv.sh +source ${BK_CONFDIR}/bk_cli_env.sh + +# default netty settings +NETTY_LEAK_DETECTION_LEVEL=${NETTY_LEAK_DETECTION_LEVEL:-"disabled"} +NETTY_RECYCLER_MAXCAPACITY=${NETTY_RECYCLER_MAXCAPACITY:-"1000"} +NETTY_RECYCLER_LINKCAPACITY=${NETTY_RECYCLER_LINKCAPACITY:-"1024"} + +# default bookie JVM settings +DEFAULT_BOOKIE_GC_OPTS="-XX:+UseG1GC \ + -XX:MaxGCPauseMillis=10 \ + -XX:+ParallelRefProcEnabled \ + -XX:+UnlockExperimentalVMOptions \ + -XX:+AggressiveOpts \ + -XX:+DoEscapeAnalysis \ + -XX:ParallelGCThreads=32 \ + -XX:ConcGCThreads=32 \ + -XX:G1NewSizePercent=50 \ + -XX:+DisableExplicitGC \ + -XX:-ResizePLAB" +DEFAULT_BOOKIE_GC_LOGGING_OPTS="-XX:+PrintGCDetails \ + -XX:+PrintGCApplicationStoppedTime \ + -XX:+UseGCLogFileRotation \ + -XX:NumberOfGCLogFiles=5 \ + -XX:GCLogFileSize=64m" +BOOKIE_MAX_HEAP_MEMORY=${BOOKIE_MAX_HEAP_MEMORY:-"1g"} +BOOKIE_MIN_HEAP_MEMORY=${BOOKIE_MIN_HEAP_MEMORY:-"1g"} +BOOKIE_MAX_DIRECT_MEMORY=${BOOKIE_MAX_DIRECT_MEMORY:-"2g"} +BOOKIE_MEM_OPTS=${BOOKIE_MEM_OPTS:-"-Xms${BOOKIE_MIN_HEAP_MEMORY} -Xmx${BOOKIE_MAX_HEAP_MEMORY} -XX:MaxDirectMemorySize=${BOOKIE_MAX_DIRECT_MEMORY}"} +BOOKIE_GC_OPTS=${BOOKIE_GC_OPTS:-"${DEFAULT_BOOKIE_GC_OPTS}"} +BOOKIE_GC_LOGGING_OPTS=${BOOKIE_GC_LOGGING_OPTS:-"${DEFAULT_BOOKIE_GC_LOGGING_OPTS}"} + +# default CLI JVM settings +DEFAULT_CLI_GC_OPTS="-XX:+UseG1GC \ + -XX:MaxGCPauseMillis=10" +DEFAULT_CLI_GC_LOGGING_OPTS="-XX:+PrintGCDetails \ + -XX:+PrintGCApplicationStoppedTime \ + -XX:+UseGCLogFileRotation \ + -XX:NumberOfGCLogFiles=5 \ + -XX:GCLogFileSize=64m" +CLI_MAX_HEAP_MEMORY=${CLI_MAX_HEAP_MEMORY:-"512M"} +CLI_MIN_HEAP_MEMORY=${CLI_MIN_HEAP_MEMORY:-"256M"} +CLI_MEM_OPTS=${CLI_MEM_OPTS:-"-Xms${CLI_MIN_HEAP_MEMORY} -Xmx${CLI_MAX_HEAP_MEMORY}"} +CLI_GC_OPTS=${CLI_GC_OPTS:-"${DEFAULT_CLI_GC_OPTS}"} +CLI_GC_LOGGING_OPTS=${CLI_GC_LOGGING_OPTS:-"${DEFAULT_CLI_GC_LOGGING_OPTS}"} + +find_module_jar_at() { + DIR=$1 + MODULE=$2 + REGEX="^${MODULE}-[0-9\\.]*(-SNAPSHOT)?.jar$" + if [ -d ${DIR} ]; then + cd ${DIR} + for f in *.jar; do + if [[ ${f} =~ ${REGEX} ]]; then + echo ${DIR}/${f} + return + fi + done + fi +} + +find_module_jar() { + MODULE_PATH=$1 + MODULE_NAME=$2 + RELEASE_JAR=$(find_module_jar_at ${BK_HOME} ${MODULE_NAME}) + if [ -n "${RELEASE_JAR}" ]; then + MODULE_JAR=${RELEASE_JAR} + else + RELEASE_JAR=$(find_module_jar_at ${BK_HOME}/lib ${MODULE_NAME}) + if [ -n "${RELEASE_JAR}" ]; then + MODULE_JAR=${RELEASE_JAR} + fi + fi + + if [ -z "${MODULE_JAR}" ]; then + BUILT_JAR=$(find_module_jar_at ${BK_HOME}/${MODULE_PATH}/target ${MODULE_NAME}) + if [ -z "${BUILT_JAR}" ]; then + echo "Couldn't find module '${MODULE_NAME}' jar." >&2 + read -p "Do you want me to run \`mvn package -DskiptTests\` for you ? " answer + case "${answer:0:1}" in + y|Y ) + mvn package -DskipTests + ;; + * ) + exit 1 + ;; + esac + + BUILT_JAR=$(find_module_jar_at ${BK_HOME}/${MODULE_PATH}/target ${MODULE_NAME}) + fi + if [ -n "${BUILT_JAR}" ]; then + MODULE_JAR=${BUILT_JAR} + fi + fi + + if [ ! -e "${MODULE_JAR}" ]; then + echo "Could not find module '${MODULE_JAR}' jar." >&2 + exit 1 + fi + echo ${MODULE_JAR} + return +} + +add_maven_deps_to_classpath() { + MODULE_PATH=$1 + MVN="mvn" + if [ "$MAVEN_HOME" != "" ]; then + MVN=${MAVEN_HOME}/bin/mvn + fi + + # Need to generate classpath from maven pom. This is costly so generate it + # and cache it. Save the file into our target dir so a mvn clean will get + # clean it up and force us create a new one. + f="${MODULE_PATH}/target/cached_classpath.txt" + if [ ! -f ${f} ]; then + ${MVN} -f "${MODULE_PATH}/pom.xml" dependency:build-classpath -Dmdep.outputFile="target/cached_classpath.txt" &> /dev/null + fi +} + +set_module_classpath() { + MODULE_PATH=$1 + if [ -d "${BK_HOME}/lib" ]; then + BK_CLASSPATH="" + for i in ${BK_HOME}/lib/*.jar; do + BK_CLASSPATH=${BK_CLASSPATH}:${i} + done + echo ${BK_CLASSPATH} + else + add_maven_deps_to_classpath ${MODULE_PATH} >&2 + cat ${MODULE_PATH}/target/cached_classpath.txt + fi + return +} + +build_bookie_jvm_opts() { + LOG_DIR=$1 + GC_LOG_FILENAME=$2 + + echo "$BOOKIE_MEM_OPTS $BOOKIE_GC_OPTS $BOOKIE_GC_LOGGING_OPTS $BOOKIE_PERF_OPTS -Xloggc:${LOG_DIR}/${GC_LOG_FILENAME}" +} + +build_cli_jvm_opts() { + LOG_DIR=$1 + GC_LOG_FILENAME=$2 + + echo "$CLI_MEM_OPTS $CLI_GC_OPTS $CLI_GC_LOGGING_OPTS -Xloggc:${LOG_DIR}/${GC_LOG_FILENAME}" +} + +build_netty_opts() { + echo "-Dio.netty.leakDetectionLevel=${NETTY_LEAK_DETECTION_LEVEL} \ + -Dio.netty.recycler.maxCapacity.default=${NETTY_RECYCLER_MAXCAPACITY} \ + -Dio.netty.recycler.linkCapacity=${NETTY_RECYCLER_LINKCAPACITY}" +} + +build_logging_opts() { + CONF_FILE=$1 + LOG_DIR=$2 + LOG_FILE=$3 + LOGGER=$4 + + echo "-Dlog4j.configuration=`basename ${CONF_FILE}` \ + -Dbookkeeper.root.logger=${LOGGER} \ + -Dbookkeeper.log.dir=${LOG_DIR} \ + -Dbookkeeper.log.file=${LOG_FILE}" +} + +build_cli_logging_opts() { + CONF_FILE=$1 + LOG_DIR=$2 + LOG_FILE=$3 + LOGGER=$4 + + echo "-Dlog4j.configuration=`basename ${CONF_FILE}` \ + -Dbookkeeper.cli.root.logger=${LOGGER} \ + -Dbookkeeper.cli.log.dir=${LOG_DIR} \ + -Dbookkeeper.cli.log.file=${LOG_FILE}" +} + +build_bookie_opts() { + echo "-Djava.net.preferIPv4Stack=true" +} diff --git a/bin/dlog b/bin/dlog new file mode 100755 index 000000000..9a9b8f161 --- /dev/null +++ b/bin/dlog @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# +# vim:et:ft=sh:sts=2:sw=2 +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +BINDIR=`dirname "$0"` +BK_HOME=`cd ${BINDIR}/..;pwd` + +source ${BK_HOME}/bin/common.sh + +DLOG_MODULE_NAME="(org.apache.distributedlog-)?distributedlog-core" +DLOG_MODULE_PATH=stream/distributedlog/core +DLOG_MODULE_HOME=${BK_HOME}/${DLOG_MODULE_PATH} + +# find the module jar +DLOG_JAR=$(find_module_jar ${DLOG_MODULE_PATH} ${DLOG_MODULE_NAME}) + +# set up the classpath +DLOG_CLASSPATH=$(set_module_classpath ${DLOG_MODULE_PATH}) + +# default variables +DEFAULT_CONF=${BK_HOME}/conf/bk_server.conf +DEFAULT_CLI_CONF=${BK_HOME}/conf/bk_server.conf +DEFAULT_LOG_CONF=${BK_HOME}/conf/log4j.properties +DEFAULT_CLI_LOG_CONF=${BK_HOME}/conf/log4j.cli.properties + +dlog_help() { + cat <<EOF +Usage: dlog <command> +where command is one of: + local Run distributedlog sandbox + tool Run distributedlog tool + admin Run distributedlog admin tool + help This help message + +or command is the full name of a class with a defined main() method. + +Environment variables: + + BOOKIE_CONF Bookie configuration file (default: ${DEFAULT_CONF}) + BOOKIE_EXTRA_OPTS Extra options to be passed to the bookie jvm + BOOKIE_EXTRA_CLASSPATH Add extra paths to the bookie classpath + CLI_CONF CLI configuration file (default: ${DEFAULT_CLI_CONF}) + CLI_EXTRA_OPTS Extra options to be passed to the CLI jvm + CLI_EXTRA_CLASSPATH Add extra paths to the CLI classpath + +These variable can also be set in conf/bkenv.sh & conf/bk_cli_env.sh +EOF +} + +# if no args specified, show usage +if [ $# = 0 ]; then + dlog_help; + exit 1; +fi + +# get arguments +COMMAND=$1 +shift + +if [ ${COMMAND} == "local" ]; then + DLOG_CONF=${BOOKIE_CONF:-"${DEFAULT_CONF}"} + DLOG_LOG_CONF=${BOOKIE_LOG_CONF:-"${DEFAULT_LOG_CONF}"} + DLOG_LOG_DIR=${BOOKIE_LOG_DIR:-"${BK_HOME}/logs"} + DLOG_LOG_FILE=${BOOKIE_LOG_FILE:-"dlog.log"} + DLOG_ROOT_LOGGER=${BOOKIE_ROOT_LOGGER:-"INFO,CONSOLE"} + DLOG_EXTRA_CLASSPATH=${BOOKIE_EXTRA_CLASSPATH} + DLOG_GC_OPTS=$(build_bookie_jvm_opts ${DLOG_LOG_DIR} "dlog_gc_%p.log") + DLOG_EXTRA_OPTS=${BOOKIE_EXTRA_OPTS} + DLOG_LOGGING_OPTS=$(build_logging_opts ${DLOG_LOG_CONF} ${DLOG_LOG_DIR} ${DLOG_LOG_FILE} ${DLOG_ROOT_LOGGER}) +else + DLOG_CONF=${CLI_CONF:-"${DEFAULT_CLI_CONF}"} + DLOG_LOG_CONF=${CLI_LOG_CONF:-"${DEFAULT_CLI_LOG_CONF}"} + DLOG_LOG_DIR=${CLI_LOG_DIR:-"${BK_HOME}/logs"} + DLOG_LOG_FILE=${CLI_LOG_FILE:-"dlog-cli.log"} + DLOG_ROOT_LOGGER=${CLI_ROOT_LOGGER:-"INFO,ROLLINGFILE"} + DLOG_EXTRA_CLASSPATH=${CLI_EXTRA_CLASSPATH} + DLOG_GC_OPTS=$(build_cli_jvm_opts ${DLOG_LOG_DIR} "dlog-cli-gc.log") + DLOG_EXTRA_OPTS=${CLI_EXTRA_OPTS} + DLOG_LOGGING_OPTS=$(build_cli_logging_opts ${DLOG_LOG_CONF} ${DLOG_LOG_DIR} ${DLOG_LOG_FILE} ${DLOG_ROOT_LOGGER}) +fi + +DLOG_CLASSPATH="$DLOG_JAR:$DLOG_CLASSPATH:$DLOG_EXTRA_CLASSPATH" +DLOG_CLASSPATH="`dirname $DLOG_LOG_CONF`:$DLOG_CLASSPATH" + +# Build the OPTS +BOOKIE_OPTS=$(build_bookie_opts) +NETTY_OPTS=$(build_netty_opts) +OPTS="${OPTS} -cp ${DLOG_CLASSPATH} ${BOOKIE_OPTS} ${DLOG_GC_OPTS} ${NETTY_OPTS} ${DLOG_LOGGING_OPTS} ${DLOG_EXTRA_OPTS}" + +if [ ! -d ${DLOG_LOG_DIR} ]; then + mkdir ${DLOG_LOG_DIR} +fi + +#Change to BK_HOME to support relative paths +cd "$BK_HOME" +case "${COMMAND}" in + local) + exec ${JAVA} ${OPTS} ${JMX_ARGS} -Dzookeeper.4lw.commands.whitelist='*' org.apache.distributedlog.LocalDLMEmulator $@ + ;; + tool) + exec ${JAVA} ${OPTS} org.apache.distributedlog.tools.Tool org.apache.distributedlog.tools.DistributedLogTool $@ + ;; + admin) + exec ${JAVA} ${OPTS} org.apache.distributedlog.tools.Tool org.apache.distributedlog.admin.DistributedLogAdmin $@ + ;; + help) + dlog_help + ;; + *) + exec ${JAVA} ${OPTS} ${COMMAND} $@ + ;; +esac + diff --git a/bookkeeper-dist/all/pom.xml b/bookkeeper-dist/all/pom.xml index 806c9ad2b..665db8e71 100644 --- a/bookkeeper-dist/all/pom.xml +++ b/bookkeeper-dist/all/pom.xml @@ -78,6 +78,21 @@ <version>${project.version}</version> </dependency> + <!-- bookkeeper.tools (new CLI) --> + <dependency> + <groupId>org.apache.bookkeeper</groupId> + <artifactId>bookkeeper-tools</artifactId> + <version>${project.version}</version> + </dependency> + + <!-- dlog --> + <dependency> + <groupId>org.apache.distributedlog</groupId> + <artifactId>distributedlog-core</artifactId> + <version>${project.version}</version> + </dependency> + + <!-- bookkeeper benchmark --> <dependency> <groupId>org.apache.bookkeeper</groupId> <artifactId>bookkeeper-benchmark</artifactId> diff --git a/bookkeeper-dist/server/pom.xml b/bookkeeper-dist/server/pom.xml index 6e273f0ca..c6300a1f0 100644 --- a/bookkeeper-dist/server/pom.xml +++ b/bookkeeper-dist/server/pom.xml @@ -62,6 +62,20 @@ <version>${project.version}</version> </dependency> + <!-- bookkeeper.tools (new CLI) --> + <dependency> + <groupId>org.apache.bookkeeper</groupId> + <artifactId>bookkeeper-tools</artifactId> + <version>${project.version}</version> + </dependency> + + <!-- dlog --> + <dependency> + <groupId>org.apache.distributedlog</groupId> + <artifactId>distributedlog-core</artifactId> + <version>${project.version}</version> + </dependency> + <!-- slf4j binding --> <dependency> <groupId>org.slf4j</groupId> diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index e4bb88b08..b0ba7e9f8 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -273,6 +273,7 @@ Apache Software License, Version 2. - lib/com.beust-jcommander-1.48.jar [36] - lib/com.yahoo.datasketches-memory-0.8.3.jar [37] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [37] +- lib/net.jpountz.lz4-lz4-1.3.0.jar [38] [1] Source available at https://github.com/FasterXML/jackson-annotations/tree/jackson-annotations-2.8.9 [2] Source available at https://github.com/FasterXML/jackson-core/tree/jackson-core-2.8.9 @@ -310,6 +311,7 @@ Apache Software License, Version 2. [35] Source available at https://github.com/facebook/rocksdb/tree/v5.8.6 [36] Source available at https://github.com/cbeust/jcommander/tree/jcommander-1.48 [37] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3 +[38] Source available at https://github.com/lz4/lz4-java/tree/1.3.0 ------------------------------------------------------------------------------------ lib/io.netty-netty-3.10.1.Final.jar contains the extensions to Java Collections Framework which has diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index bd0dba266..9f9ccc648 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -238,6 +238,7 @@ Apache Software License, Version 2. - lib/com.beust-jcommander-1.48.jar [23] - lib/com.yahoo.datasketches-memory-0.8.3.jar [24] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [24] +- lib/net.jpountz.lz4-lz4-1.3.0.jar [25] [1] Source available at https://github.com/FasterXML/jackson-annotations/tree/jackson-annotations-2.8.9 [2] Source available at https://github.com/FasterXML/jackson-core/tree/jackson-core-2.8.9 @@ -263,6 +264,7 @@ Apache Software License, Version 2. [22] Source available at https://github.com/facebook/rocksdb/tree/v5.8.6 [23] Source available at https://github.com/cbeust/jcommander/tree/jcommander-1.48 [24] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3 +[25] Source available at https://github.com/lz4/lz4-java/tree/1.3.0 ------------------------------------------------------------------------------------ lib/io.netty-netty-all-4.1.12.Final.jar bundles some 3rd party dependencies diff --git a/conf/bk_cli_env.sh b/conf/bk_cli_env.sh index 2f57061d1..8faa03d36 100644 --- a/conf/bk_cli_env.sh +++ b/conf/bk_cli_env.sh @@ -23,7 +23,9 @@ # Set JAVA_HOME here to override the environment setting # JAVA_HOME= +######################################## # default settings for bookkeeper cli +######################################## # Configuration file of settings used in bookkeeper cli # CLI_CONF= @@ -34,9 +36,9 @@ # Add extra paths to the bookkeeper classpath # CLI_EXTRA_CLASSPATH= -# +################################# # CLI Logging Options -# +################################# # Log4j configuration file # CLI_LOG_CONF= @@ -49,3 +51,17 @@ # Log level & appender # CLI_ROOT_LOGGER="INFO,CONSOLE" + +################################# +# JVM memory options +################################# + +# CLI_MAX_HEAP_MEMORY=512M +# CLI_MIN_HEAP_MEMORY=256M +# CLI_MEM_OPTS= + +# JVM GC options +# CLI_GC_OPTS= + +# JVM GC logging options +# CLI_GC_LOGGING_OPTS= diff --git a/conf/bkenv.sh b/conf/bkenv.sh index 0cb39ecad..eebc4ff18 100644 --- a/conf/bkenv.sh +++ b/conf/bkenv.sh @@ -21,18 +21,50 @@ # Set JAVA_HOME here to override the environment setting # JAVA_HOME= -# default settings for starting bookkeeper +######################################## +# default settings for bookkeeper +######################################## # Configuration file of settings used in bookie server # BOOKIE_CONF= +# Extra options to be passed to the jvm +# BOOKIE_EXTRA_OPTS= + +# Add extra paths to the bookkeeper classpath +# BOOKIE_EXTRA_CLASSPATH= + +# Folder where the Bookie server PID file should be stored +# BOOKIE_PID_DIR= + +# Wait time before forcefully kill the Bookie server instance, if the stop is not successful +# BOOKIE_STOP_TIMEOUT= + +# Entry formatter class to format entries. +# ENTRY_FORMATTER_CLASS= + +# this default config dir should match the 'localBookiesConfigDirectory' config value in the conf file of LocalBookKeeper +# LOCALBOOKIES_CONFIG_DIR=/tmp/localbookies-config + +################################# +# BookKeeper Logging Options +################################# + # Log4j configuration file # BOOKIE_LOG_CONF= # Logs location # BOOKIE_LOG_DIR= -# JVM memory options +# Log file name +# BOOKIE_LOG_FILE="bookkeeper.log" + +# Log level & appender +# BOOKIE_ROOT_LOGGER="INFO,CONSOLE" + +################################# +# BookKeeper JVM memory options +################################# # BOOKIE_MAX_HEAP_MEMORY=1g # BOOKIE_MIN_HEAP_MEMORY=1g @@ -47,32 +79,3 @@ # JVM performance options # BOOKIE_PERF_OPTS="-XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch -XX:-UseBiasedLocking" - -# Netty options - -# netty buffer leak detection level - {@link http://netty.io/wiki/reference-counted-objects.html#wiki-h3-11} -# NETTY_LEAK_DETECTION_LEVEL= - -# netty recycler max capacity -# NETTY_RECYCLER_MAXCAPACITY= - -# netty recycler link capacity -# NETTY_RECYCLER_LINKCAPACITY= - -# Extra options to be passed to the jvm -# BOOKIE_EXTRA_OPTS= - -# Add extra paths to the bookkeeper classpath -# BOOKIE_EXTRA_CLASSPATH= - -# Folder where the Bookie server PID file should be stored -# BOOKIE_PID_DIR= - -# Wait time before forcefully kill the Bookie server instance, if the stop is not successful -# BOOKIE_STOP_TIMEOUT= - -# Entry formatter class to format entries. -# ENTRY_FORMATTER_CLASS= - -# this default config dir should match the 'localBookiesConfigDirectory' config value in the conf file of LocalBookKeeper -# LOCALBOOKIES_CONFIG_DIR=/tmp/localbookies-config diff --git a/conf/nettyenv.sh b/conf/nettyenv.sh new file mode 100644 index 000000000..c4702b023 --- /dev/null +++ b/conf/nettyenv.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Netty options + +# netty buffer leak detection level - {@link http://netty.io/wiki/reference-counted-objects.html#wiki-h3-11} +# NETTY_LEAK_DETECTION_LEVEL= + +# netty recycler max capacity +# NETTY_RECYCLER_MAXCAPACITY= + +# netty recycler link capacity +# NETTY_RECYCLER_LINKCAPACITY= diff --git a/dev/check-binary-license b/dev/check-binary-license index a150ba827..b56cbbf79 100755 --- a/dev/check-binary-license +++ b/dev/check-binary-license @@ -59,6 +59,11 @@ for J in $JARS; do continue fi + echo $J | grep -q "org.apache.distributedlog" + if [ $? == 0 ]; then + continue + fi + echo "$LICENSE" | grep -q $J if [ $? != 0 ]; then echo $J unaccounted for in LICENSE diff --git a/docker/scripts/entrypoint.sh b/docker/scripts/entrypoint.sh index 91e63c305..7ade924bc 100755 --- a/docker/scripts/entrypoint.sh +++ b/docker/scripts/entrypoint.sh @@ -37,12 +37,14 @@ export BK_journalDirectory=${BK_journalDirectory:-${BK_DATA_DIR}/journal} export BK_ledgerDirectories=${BK_ledgerDirectories:-${BK_DATA_DIR}/ledgers} export BK_indexDirectories=${BK_indexDirectories:-${BK_DATA_DIR}/index} export BK_metadataServiceUri=${BK_metadataServiceUri:-"zk://${BK_zkServers}${BK_zkLedgersRootPath}"} +export BK_dlogRootPath=${BK_dlogRootPath:-"${BK_CLUSTER_ROOT_PATH}/distributedlog"} echo "BK_bookiePort bookie service port is $BK_bookiePort" echo "BK_zkServers is $BK_zkServers" echo "BK_DATA_DIR is $BK_DATA_DIR" echo "BK_CLUSTER_ROOT_PATH is $BK_CLUSTER_ROOT_PATH" echo "BK_metadataServiceUri is $BK_metadataServiceUri" +echo "BK_dlogRootPath is $BK_dlogRootPath" mkdir -p "${BK_journalDirectory}" "${BK_ledgerDirectories}" "${BK_indexDirectories}" # -------------- # @@ -109,6 +111,48 @@ else fi fi +# Create default dlog namespace +# Use ephemeral zk node as lock to keep initialize atomic. +/opt/bookkeeper/bin/bookkeeper org.apache.zookeeper.ZooKeeperMain -server ${BK_zkServers} stat ${BK_dlogRootPath} +if [ $? -eq 0 ]; then + echo "Dlog namespace already created, no need to create another one" +else + # create ephemeral zk node dlogInitLock, initiator who this node, then do init; other initiators will wait. + /opt/bookkeeper/bin/bookkeeper org.apache.zookeeper.ZooKeeperMain -server ${BK_zkServers} create -e ${BK_CLUSTER_ROOT_PATH}/dlogInitLock + if [ $? -eq 0 ]; then + # dlogInitLock created success, this is the successor to do znode init + echo "Dlog namespace not exist, do the init to create them." + /opt/bookkeeper/bin/dlog admin bind -l ${BK_zkLedgersRootPath} -s ${BK_zkServers} -c distributedlog://${BK_zkServers}${BK_dlogRootPath} + if [ $? -eq 0 ]; then + echo "Dlog namespace is created successfully." + else + echo "Failed to create dlog namespace ${BK_dlogRootPath}. please check the reason." + exit + fi + else + echo "Other docker instance is doing initialize at the same time, will wait in this instance." + tenSeconds=1 + while [ ${tenSeconds} -lt 10 ] + do + sleep 10 + /opt/bookkeeper/bin/bookkeeper org.apache.zookeeper.ZooKeeperMain -server ${BK_zkServers} stat ${BK_dlogRootPath} + if [ $? -eq 0 ]; then + echo "Waited $tenSeconds * 10 seconds, dlog namespace created" + break + else + echo "Waited $tenSeconds * 10 seconds, dlog namespace still not created" + (( tenSeconds++ )) + continue + fi + done + + if [ ${tenSeconds} -eq 10 ]; then + echo "Waited 100 seconds for creating dlog namespace, something wrong, please check" + exit + fi + fi +fi + echo "run command by exec" exec "$@" diff --git a/stream/distributedlog/core/bin/common.sh b/stream/distributedlog/core/bin/common.sh deleted file mode 100755 index 2b1315766..000000000 --- a/stream/distributedlog/core/bin/common.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -if [ $(uname) == "Linux" ]; then - # check if net.ipv6.bindv6only is set to 1 - bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null) - if [ -n "${bindv6only}" ] && [ "${bindv6only}" -eq "1" ]; then - echo "Error: \"net.ipv6.bindv6only\" is set to 1 - Java networking could be broken" - echo "For more info (the following page also applies to DistributedLog): http://wiki.apache.org/hadoop/HadoopIPv6"; - exit 1 - fi -fi - -# See the following page for extensive details on setting -# up the JVM to accept JMX remote management: -# http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html -# by default we allow local JMX connections -if [ -z "${JMXLOCALONLY}" ]; then - JMXLOCALONLY=false -fi - -if [ -z "${JMXDISABLE}" ]; then - echo "JMX enabled by default" >&2 - # for some reason these two options are necessary on jdk6 on Ubuntu - # accord to the docs they are not necessary, but otw jconsole cannot - # do a local attach - JMX_ARGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=${JMXLOCALONLY}" -else - echo "JMX disabled by user request" >&2 -fi - -echo "DLOG_HOME => ${DLOG_HOME}" -DEFAULT_LOG_CONF="${DLOG_HOME}/conf/log4j.properties" - -[ -f "${DLOG_HOME}/conf/dlogenv.sh" ] && source "${DLOG_HOME}/conf/dlogenv.sh" - -# exclude tests jar -RELEASE_JAR=$(ls ${DLOG_HOME}/distributedlog-*.jar 2> /dev/null | grep -v 'tests\|javadoc\|sources' | tail -1) -if [ $? == 0 ]; then - DLOG_JAR="${RELEASE_JAR}" -fi - -# exclude tests jar -BUILT_JAR=$(ls ${DLOG_HOME}/target/distributedlog-*.jar 2> /dev/null | grep -v 'tests\|javadoc\|sources' | tail -1) - -if [ -e "${BUILD_JAR}" ] && [ -e "${DLOG_JAR}" ]; then - echo "\nCouldn't find dlog jar."; - echo "Make sure you've run 'mvn package'\n"; - exit 1; -elif [ -e "${BUILT_JAR}" ]; then - DLOG_JAR="${BUILT_JAR}" -fi - -add_maven_deps_to_classpath() { - MVN="mvn" - if [ -n "${MAVEN_HOME}" ]; then - MVN="${MAVEN_HOME}/bin/mvn" - fi - - # Need to generate classpath from maven pom. This is costly so generate it - # and cache it. Save the file into our target dir so a mvn clean will get - # clean it up and force us create a new one. - f="${PWD}/${DLOG_HOME}/target/cached_classpath.txt" - if [ ! -f "${f}" ]; then - "${MVN}" -f "${DLOG_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null - fi - DLOG_CLASSPATH="${CLASSPATH}":$(cat "${f}") -} - -if [ -d "${DLOG_HOME}/lib" ]; then - for i in ${DLOG_HOME}/lib/*.jar; do - DLOG_CLASSPATH="${DLOG_CLASSPATH}:${i}" - done -else - add_maven_deps_to_classpath -fi - -# if no args specified, exit -if [ $# = 0 ]; then - exit 1 -fi - -if [ -z "${DLOG_LOG_CONF}" ]; then - DLOG_LOG_CONF="${DEFAULT_LOG_CONF}" -fi - -DLOG_CLASSPATH="${DLOG_JAR}:${DLOG_CLASSPATH}:${DLOG_EXTRA_CLASSPATH}" -if [ -n "${DLOG_LOG_CONF}" ]; then - DLOG_CLASSPATH="$(dirname ${DLOG_LOG_CONF}):${DLOG_CLASSPATH}" - OPTS="${OPTS} -Dlog4j.configuration=$(basename ${DLOG_LOG_CONF})" -fi -OPTS="-cp ${DLOG_CLASSPATH} ${OPTS} ${DLOG_EXTRA_OPTS}" - -OPTS="${OPTS} ${DLOG_EXTRA_OPTS}" - -# Disable ipv6 as it can cause issues -OPTS="${OPTS} -Djava.net.preferIPv4Stack=true" - -# log directory & file -DLOG_ROOT_LOGGER=${DLOG_ROOT_LOGGER:-"INFO,R"} -DLOG_LOG_DIR=${DLOG_LOG_DIR:-"$DLOG_HOME/logs"} -DLOG_LOG_FILE=${DLOG_LOG_FILE:-"dlog.log"} - -#Configure log configuration system properties -OPTS="$OPTS -Ddlog.root.logger=${DLOG_ROOT_LOGGER}" -OPTS="$OPTS -Ddlog.log.dir=${DLOG_LOG_DIR}" -OPTS="$OPTS -Ddlog.log.file=${DLOG_LOG_FILE}" diff --git a/stream/distributedlog/core/bin/dlog b/stream/distributedlog/core/bin/dlog deleted file mode 100755 index 01f8ee344..000000000 --- a/stream/distributedlog/core/bin/dlog +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -set -e - -BASEDIR=$(dirname "$0") - -DLOG_HOME="${BASEDIR}/.." - -usage() { - cat <<EOF -Usage: runner <command> -where command is one of: - local Run distributedlog sandbox - example Run distributedlog example - tool Run distributedlog tool - proxy_tool Run distributedlog proxy tool to interact with proxies - balancer Run distributedlog balancer - admin Run distributedlog admin tool - help This help message - -or command is the full name of a class with a defined main() method. - -Environment variables: - DLOG_LOG_CONF Log4j configuration file (default $DEFAULT_LOG_CONF) - DLOG_EXTRA_OPTS Extra options to be passed to the jvm - DLOG_EXTRA_CLASSPATH Add extra paths to the dlog classpath - -These variable can also be set in conf/dlogenv.sh -EOF -} - -source "${DLOG_HOME}"/bin/common.sh - -# get arguments -COMMAND=$1 -shift - -case "${COMMAND}" in - local) - exec java $OPTS $JMX_ARGS -Dzookeeper.4lw.commands.whitelist='*' org.apache.distributedlog.LocalDLMEmulator $@ - ;; - tool) - exec java $OPTS org.apache.distributedlog.tools.Tool org.apache.distributedlog.tools.DistributedLogTool $@ - ;; - admin) - exec java $OPTS org.apache.distributedlog.tools.Tool org.apache.distributedlog.admin.DistributedLogAdmin $@ - ;; - help) - usage - ;; - *) - exec java $OPTS $COMMAND $@ - ;; -esac - diff --git a/stream/distributedlog/core/conf/bookie.conf.template b/stream/distributedlog/core/conf/bookie.conf.template deleted file mode 100644 index 5ca89d072..000000000 --- a/stream/distributedlog/core/conf/bookie.conf.template +++ /dev/null @@ -1,183 +0,0 @@ -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -## Bookie settings - -# Port that bookie server listen on -bookiePort=3181 - -# TODO: change the journal directory -# Directory Bookkeeper outputs its write ahead log -journalDirectory=/tmp/data/bk/journal - -# TODO: change the ledgers directory -# Directory Bookkeeper outputs ledger snapshots -ledgerDirectories=/tmp/data/bk/ledgers - -# TODO: change the index directory -# Directory in which index files will be stored. -indexDirectories=/tmp/data/bk/ledgers - -# Ledger Manager Class -# What kind of ledger manager is used to manage how ledgers are stored, managed -# and garbage collected. Try to read 'BookKeeper Internals' for detail info. -ledgerManagerType=hierarchical - -# Root zookeeper path to store ledger metadata -# This parameter is used by zookeeper-based ledger manager as a root znode to -# store all ledgers. -zkLedgersRootPath=/messaging/bookkeeper/ledgers - -# Max file size of entry logger, in bytes -# A new entry log file will be created when the old one reaches the file size limitation -logSizeLimit=1073741823 - -# Max file size of journal file, in mega bytes -# A new journal file will be created when the old one reaches the file size limitation -# -journalMaxSizeMB=2048 - -# Max number of old journal file to kept -# Keep a number of old journal files would help data recovery in specia case -# -journalMaxBackups=5 - -# How long the interval to trigger next garbage collection, in milliseconds -# Since garbage collection is running in background, too frequent gc -# will heart performance. It is better to give a higher number of gc -# interval if there is enough disk capacity. -# gc per 1 hour (aligning with most DL rolling interval) -gcInitialWaitTime=600000 -gcWaitTime=3600000 -# do minor compaction per 2 hours -minorCompactionInterval=7200 -minorCompactionThreshold=0.2 -# disable major compaction -majorCompactionInterval=0 -# reduce major compaction threshold to a low value to prevent bad force compaction behavior -majorCompactionThreshold=0.3 -# Compaction Rate & Max Outstanding -compactionRate=10737418 -compactionMaxOutstandingRequests=10737418 - -# How long the interval to flush ledger index pages to disk, in milliseconds -# Flushing index files will introduce much random disk I/O. -# If separating journal dir and ledger dirs each on different devices, -# flushing would not affect performance. But if putting journal dir -# and ledger dirs on same device, performance degrade significantly -# on too frequent flushing. You can consider increment flush interval -# to get better performance, but you need to pay more time on bookie -# server restart after failure. -# -flushInterval=1000 - -# Interval to watch whether bookie is dead or not, in milliseconds -# -# bookieDeathWatchInterval=1000 - -## zookeeper client settings - -# A list of one of more servers on which zookeeper is running. -# The server list can be comma separated values, for example: -# zkServers=zk1:2181,zk2:2181,zk3:2181 -zkServers=localhost:2181 - -# ZooKeeper client session timeout in milliseconds -# Bookie server will exit if it received SESSION_EXPIRED because it -# was partitioned off from ZooKeeper for more than the session timeout -# JVM garbage collection, disk I/O will cause SESSION_EXPIRED. -# Increment this value could help avoiding this issue -zkTimeout=30000 - -## NIO Server settings - -# This settings is used to enabled/disabled Nagle's algorithm, which is a means of -# improving the efficiency of TCP/IP networks by reducing the number of packets -# that need to be sent over the network. -# If you are sending many small messages, such that more than one can fit in -# a single IP packet, setting server.tcpnodelay to false to enable Nagle algorithm -# can provide better performance. -# Default value is true. -# -serverTcpNoDelay=true - -## ledger cache settings - -# Max number of ledger index files could be opened in bookie server -# If number of ledger index files reaches this limitation, bookie -# server started to swap some ledgers from memory to disk. -# Too frequent swap will affect performance. You can tune this number -# to gain performance according your requirements. -openFileLimit=20000 - -# Size of a index page in ledger cache, in bytes -# A larger index page can improve performance writing page to disk, -# which is efficent when you have small number of ledgers and these -# ledgers have similar number of entries. -# If you have large number of ledgers and each ledger has fewer entries, -# smaller index page would improve memory usage. -pageSize=8192 - -# How many index pages provided in ledger cache -# If number of index pages reaches this limitation, bookie server -# starts to swap some ledgers from memory to disk. You can increment -# this value when you found swap became more frequent. But make sure -# pageLimit*pageSize should not more than JVM max memory limitation, -# otherwise you would got OutOfMemoryException. -# In general, incrementing pageLimit, using smaller index page would -# gain bettern performance in lager number of ledgers with fewer entries case -# If pageLimit is -1, bookie server will use 1/3 of JVM memory to compute -# the limitation of number of index pages. -pageLimit=131072 - -#If all ledger directories configured are full, then support only read requests for clients. -#If "readOnlyModeEnabled=true" then on all ledger disks full, bookie will be converted -#to read-only mode and serve only read requests. Otherwise the bookie will be shutdown. -readOnlyModeEnabled=true - -# Bookie Journal Settings -writeBufferSizeBytes=262144 -journalFlushWhenQueueEmpty=false -journalRemoveFromPageCache=true -journalAdaptiveGroupWrites=true -journalMaxGroupWaitMSec=4 -journalBufferedEntriesThreshold=180 -journalBufferedWritesThreshold=131072 -journalMaxGroupedEntriesToCommit=200 -journalPreAllocSizeMB=4 - -# Sorted Ledger Storage Settings -sortedLedgerStorageEnabled=true -skipListSizeLimit=67108864 -skipListArenaChunkSize=2097152 -skipListArenaMaxAllocSize=131072 -fileInfoCacheInitialCapacity=10000 -fileInfoMaxIdleTime=3600 - -# Bookie Threads Settings (NOTE: change this to align the cpu cores) -numAddWorkerThreads=4 -numJournalCallbackThreads=4 -numReadWorkerThreads=4 -numLongPollWorkerThreads=4 - -# stats -statsProviderClass=org.apache.bookkeeper.stats.CodahaleMetricsServletProvider -# Exporting codahale stats -codahaleStatsHttpPort=9001 -useHostNameAsBookieID=true -allowLoopback=true diff --git a/stream/distributedlog/core/conf/distributedlog.conf b/stream/distributedlog/core/conf/distributedlog.conf deleted file mode 100644 index dac71acdd..000000000 --- a/stream/distributedlog/core/conf/distributedlog.conf +++ /dev/null @@ -1,125 +0,0 @@ -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -######################## -# ZooKeeper Client Settings -######################## - -# zookeeper settings -zkSessionTimeoutSeconds=30 -zkNumRetries=0 -zkRetryStartBackoffMillis=100 -zkRetryMaxBackoffMillis=200 -# bkc zookeeper settings -bkcZKSessionTimeoutSeconds=60 -bkcZKNumRetries=20 -bkcZKRetryStartBackoffMillis=100 -bkcZKRetryMaxBackoffMillis=200 - -######################## -# BookKeeper Client Settings -######################## - -# bookkeeper client timeouts -bkcWriteTimeoutSeconds=10 -bkcReadTimeoutSeconds=1 -bkcNumWorkerThreads=16 -# bkcNumIOThreads=16 -bkc.numChannelsPerBookie=1 -bkc.enableTaskExecutionStats=true -bkc.connectTimeoutMillis=1000 -bkc.enablePerHostStats=true - -######################## -# DL Settings -######################## - -# lock timeout -lockTimeoutSeconds=0 -# dl worker threads -numWorkerThreads=16 - -### Recovery Related Settings - -# recover log segments in background -recoverLogSegmentsInBackground=true -# disable max id in proxy -maxIdSanityCheck=true -# use allocator pool for proxy -enableLedgerAllocatorPool=false -# ledger allocator pool size -ledgerAllocatorPoolCoreSize=20 -# check stream exists or not -createStreamIfNotExists=true -# encode dc id in version -encodeDCIDInVersion=true -# logSegmentNameVersion -logSegmentNameVersion=1 - -### Write Performance Related Settings - -# ensemble size -ensemble-size=3 -write-quorum-size=3 -ack-quorum-size=2 -bkc.ensemblePlacementPolicy=org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy -bkc.delayEnsembleChange=true - -# sync settings -# buffer size is large because when we rewrite we perform a very large write to persist -# all queue state at once (up to max queue memory size, ex. 16MB). the write will be -# throttled if it takes too long, which can hurt performance, so important to optimize -# for this case. -output-buffer-size=512000 -enableImmediateFlush=false -periodicFlushFrequencyMilliSeconds=6 -logFlushTimeoutSeconds=120 - -### Ledger Rolling Related Settings - -# retention policy -retention-size=0 -# rolling ledgers (disable time rolling/enable size rolling) -rolling-interval=0 - -# max logsegment bytes=2GB -# much larger than max journal size, effectively never roll and let drpc do it -maxLogSegmentBytes=2147483648 - -# rolling concurrency -logSegmentRollingConcurrency=1 -# disable sanityCheckDelete -sanityCheckDelete=false -ledgerAllocatorPoolName=drpc-alloc-pool - -### Readahead settings - -enableReadAhead=true -ReadAheadBatchSize=10 -ReadAheadMaxEntries=100 -ReadAheadWaitTime=10 - -### Rate limit - -rpsSoftWriteLimit=1 -rpsHardWriteLimit=5 -rpsHardServiceLimit=15 - -### Config - -dynamicConfigReloadIntervalSec=5 diff --git a/stream/distributedlog/core/conf/dlogenv.sh b/stream/distributedlog/core/conf/dlogenv.sh deleted file mode 100644 index 345e60f1f..000000000 --- a/stream/distributedlog/core/conf/dlogenv.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/sh -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -################## -# General -################## - -# Log4j configuration file -# DLOG_LOG_CONF= - -# Extra options to be passed to the jvm -# DLOG_EXTRA_OPTS= - -# Add extra paths to the dlog classpath -# DLOG_EXTRA_CLASSPATH= - -# Configure the root logger -# DLOG_ROOT_LOGGER= - -# Configure the log dir -# DLOG_LOG_DIR= - -# Configure the log file -# DLOG_LOG_FILE= - -################# -# ZooKeeper -################# - -# Configure zookeeper root logger -# ZK_ROOT_LOGGER= - -################# -# Bookie -################# - -# Configure bookie root logger -# BK_ROOT_LOGGER= - -################# -# Write Proxy -################# - -# Configure write proxy root logger -# WP_ROOT_LOGGER= - -# write proxy configuration file -# WP_CONF_FILE=${DL_HOME}/conf/write_proxy.conf - -# port and stats port -# WP_SERVICE_PORT=4181 -# WP_STATS_PORT=9000 - -# shard id -# WP_SHARD_ID=0 - -# write proxy namespace -# WP_NAMESPACE=distributedlog://127.0.0.1:2181/messaging/distributedlog/mynamespace diff --git a/stream/distributedlog/core/conf/log4j.properties b/stream/distributedlog/core/conf/log4j.properties deleted file mode 100644 index af1cf5f73..000000000 --- a/stream/distributedlog/core/conf/log4j.properties +++ /dev/null @@ -1,56 +0,0 @@ -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# -# DistributedLog Logging Configuration -# - -# Default values -dlog.root.logger=INFO, R -dlog.log.dir=logs -dlog.log.file=dlog.log - -log4j.rootLogger=${dlog.root.logger} -log4j.logger.org.apache.zookeeper=INFO -log4j.logger.org.apache.bookkeeper=INFO - -# redirect executor output to executors.log since slow op warnings can be quite verbose -log4j.logger.org.apache.bookkeeper.util.SafeRunnable=INFO, Executors -log4j.additivity.org.apache.bookkeeper.util.SafeRunnable=false - -log4j.appender.Executors=org.apache.log4j.RollingFileAppender -log4j.appender.Executors.Threshold=INFO -log4j.appender.Executors.File=${dlog.log.dir}/executors.log -log4j.appender.Executors.MaxFileSize=20MB -log4j.appender.Executors.MaxBackupIndex=5 -log4j.appender.Executors.layout=org.apache.log4j.PatternLayout -log4j.appender.Executors.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n - -log4j.appender.R=org.apache.log4j.RollingFileAppender -log4j.appender.R.Threshold=INFO -log4j.appender.R.File=${dlog.log.dir}/${dlog.log.file} -log4j.appender.R.MaxFileSize=20MB -log4j.appender.R.MaxBackupIndex=50 -log4j.appender.R.layout=org.apache.log4j.PatternLayout -log4j.appender.R.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n - -log4j.appender.stderr=org.apache.log4j.ConsoleAppender -log4j.appender.stderr.Target=System.err -log4j.appender.stderr.Threshold=INFO -log4j.appender.stderr.layout=org.apache.log4j.PatternLayout -log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n diff --git a/stream/distributedlog/core/conf/write_proxy.conf b/stream/distributedlog/core/conf/write_proxy.conf deleted file mode 100644 index 7f5351ab5..000000000 --- a/stream/distributedlog/core/conf/write_proxy.conf +++ /dev/null @@ -1,143 +0,0 @@ -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -######################## -# ZooKeeper Client Settings -######################## - -# zookeeper settings -zkSessionTimeoutSeconds=1 -zkNumRetries=0 -zkRetryStartBackoffMillis=100 -zkRetryMaxBackoffMillis=200 -# bkc zookeeper settings -bkcZKSessionTimeoutSeconds=60 -bkcZKNumRetries=20 -bkcZKRetryStartBackoffMillis=100 -bkcZKRetryMaxBackoffMillis=200 - -######################## -# BookKeeper Client Settings -######################## - -# bookkeeper client timeouts -bkcWriteTimeoutSeconds=2 -bkcReadTimeoutSeconds=2 -bkcNumWorkerThreads=32 -bkc.numChannelsPerBookie=1 -bkc.enableTaskExecutionStats=true -bkc.connectTimeoutMillis=200 -bkc.enableParallelRecoveryRead=true -bkc.recoveryReadBatchSize=5 -bkc.enablePerHostStats=true - -######################## -# DL Settings -######################## - -# Metadata Settings - -# ledger metadata version that supports sequence id -ledger-metadata-layout=5 - -# lock timeout -lockTimeoutSeconds=0 -# dl worker threads -numWorkerThreads=32 - -### Recovery Related Settings - -# recover log segments in background -recoverLogSegmentsInBackground=false -# disable max id in proxy -maxIdSanityCheck=false -# use allocator pool for proxy -enableLedgerAllocatorPool=true -# ledger allocator pool path -ledgerAllocatorPoolPath=.write_proxy_allocation_pool -# ledger allocator pool size -ledgerAllocatorPoolCoreSize=40 -# check stream exists or not -createStreamIfNotExists=true -# encode dc id in version -encodeDCIDInVersion=true -# logSegmentNameVersion -logSegmentNameVersion=1 - -### Write Performance Related Settings - -# ensemble size -ensemble-size=3 -write-quorum-size=3 -ack-quorum-size=2 -bkc.ensemblePlacementPolicy=org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy -bkc.delayEnsembleChange=true -bkc.writeRequestToChannelAsync=true - -# enable immediate flush -enableImmediateFlush=true -# 0k output buffer -output-buffer-size=0 -# disable periodical flush -periodicFlushFrequencyMilliSeconds=0 -enableTaskExecutionStats=true -taskExecutionWarnTimeMicros=100000 - -### Ledger Rolling Related Settings - -# retention policy -retention-size=4 -# rolling ledgers (enable time rolling): 120 minutes = 2 hours -rolling-interval=120 -# max logsegment bytes : 2GB -maxLogSegmentBytes=2147483648 -# rolling concurrency -logSegmentRollingConcurrency=1 -# disable sanityCheckDelete -sanityCheckDelete=false -# compression codec -compressionType=lz4 - -### Per Stream Stats -enablePerStreamStat=true - -######################## -# DL Settings -######################## - -# proxy server settings -server_mode=DURABLE -serviceTimeoutMs=60000 -streamProbationTimeoutMs=120000 -server_threads=16 -server_dlsn_version=1 -server_enable_perstream_stat=true -server_graceful_shutdown_period_ms=20000 - -# write limits -perWriterOutstandingWriteLimit=-1 -globalOutstandingWriteLimit=-1 -outstandingWriteLimitDarkmode=false - -# bytes per second limit applied at the host level (50MBps on 1Gib machines) -bpsHardServiceLimit=52428800 -# bytes per second limit after which no new streams may be acquired (65MBps on 1Gib machines) -bpsStreamAcquireServiceLimit=47185920 - -# limit the maximum number of streams -maxAcquiredPartitionsPerProxy=-1 diff --git a/stream/distributedlog/core/conf/zookeeper.conf.dynamic.template b/stream/distributedlog/core/conf/zookeeper.conf.dynamic.template deleted file mode 100644 index b397c50df..000000000 --- a/stream/distributedlog/core/conf/zookeeper.conf.dynamic.template +++ /dev/null @@ -1 +0,0 @@ -#/** # * Copyright 2017 The Apache Software Foundation # * # * Licensed to the Apache Software Foundation (ASF) under one # * or more contributor license agreements. See the NOTICE file # * distributed with this work for additional information # * regarding copyright ownership. The ASF licenses this file # * to you under the Apache License, Version 2.0 (the # * "License"); you may not use this file except in compliance # * with the License. You may obtain a copy of the License at # * # * http://www.apache.org/licenses/LICENSE-2.0 # * # * Unless required by applicable law or agreed to in writing, software # * distributed under the License is distributed on an "AS IS" BASIS, # * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # * See the License for the specific language governing permissions and # * limitations under the License. # */ server.1=127.0.0.1:2710:3710:participant;0.0.0.0:2181 diff --git a/stream/distributedlog/core/conf/zookeeper.conf.template b/stream/distributedlog/core/conf/zookeeper.conf.template deleted file mode 100644 index 3c0546e22..000000000 --- a/stream/distributedlog/core/conf/zookeeper.conf.template +++ /dev/null @@ -1,82 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The number of milliseconds of each tick -tickTime=2000 - -# the port at which the clients will connect -clientPort=2181 - -# The number of ticks that the initial -# synchronization phase can take -initLimit=10 - -# The number of ticks that can pass between -# sending a request and getting an acknowledgement -syncLimit=30 - -# the directory where the snapshot is stored. -dataDir=/tmp/data/zookeeper - -# where txlog are written -dataLogDir=/tmp/data/zookeeper/txlog - -# the port at which the admin will listen -adminPort=9990 -zookeeper.admin.enableServer=true - -# limit on queued clients - default: 1000 -globalOutstandingLimit=1000 - -# number of transactions before snapshots are taken - default: 100000 -snapCount=100000 - -# max # of clients - 0==unlimited -maxClientCnxns=25 - -# Election implementation to use. A value of "0" corresponds to the original -# UDP-based version, "1" corresponds to the non-authenticated UDP-based -# version of fast leader election, "2" corresponds to the authenticated -# UDP-based version of fast leader election, and "3" corresponds to TCP-based -# version of fast leader election. Currently, only 0 and 3 are supported, -# 3 being the default -electionAlg=3 - -# Leader accepts client connections. Default value is "yes". The leader -# machine coordinates updates. For higher update throughput at thes slight -# expense of read throughput the leader can be configured to not accept -# clients and focus on coordination. -leaderServes=yes - -# Skips ACL checks. This results in a boost in throughput, but opens up full -# access to the data tree to everyone. -skipACL=no - -# Purge txn logs every hour. Before 3.4.x this was done with an external cron -# job, now we can do it internally. -autopurge.purgeInterval=1 - -# Prior to version 3.4 ZooKeeper has always used NIO directly, however in -# versions 3.4 and later Netty is supported as an option to NIO (replaces). -# serverCnxnFactory=org.apache.zookeeper.server.NIOServerCnxnFactory - -standaloneEnabled=false -# ZooKeeper Dynamic Reconfiguration -# See: https://zookeeper.apache.org/doc/trunk/zookeeperReconfig.html -# -# standaloneEnabled=false -# dynamicConfigFile=/path/to/zoo.cfg.dynamic -# -server.1=127.0.0.1:2710:3710:participant;0.0.0.0:2181 diff --git a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/BookKeeperClusterUtils.java b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/BookKeeperClusterUtils.java index a7f9f8c03..3cd370f3f 100644 --- a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/BookKeeperClusterUtils.java +++ b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/BookKeeperClusterUtils.java @@ -92,6 +92,26 @@ public static boolean metadataFormatIfNeeded(DockerClient docker, String version } } + public static String createDlogNamespaceIfNeeded(DockerClient docker, + String version, + String namespace) throws Exception { + String zkServers = BookKeeperClusterUtils.zookeeperConnectString(docker); + String dlogUri = "distributedlog://" + zkServers + namespace; + try (ZooKeeper zk = BookKeeperClusterUtils.zookeeperClient(docker)) { + if (zk.exists(namespace, false) == null) { + String dlog = "/opt/bookkeeper/" + version + "/bin/dlog"; + + runOnAnyBookie(docker, dlog, + "admin", + "bind", + "-l", "/ledgers", + "-s", zkServers, + "-c", dlogUri); + } + } + return dlogUri; + } + public static void formatAllBookies(DockerClient docker, String version) throws Exception { String bookkeeper = "/opt/bookkeeper/" + version + "/bin/bookkeeper"; BookKeeperClusterUtils.runOnAllBookies(docker, bookkeeper, "shell", "bookieformat", "-nonInteractive"); @@ -123,6 +143,15 @@ public static boolean runOnAnyBookie(DockerClient docker, String... cmds) throws } } + public static String getAnyBookie() throws Exception { + Optional<String> bookie = DockerUtils.cubeIdsMatching("bookkeeper").stream().findAny(); + if (bookie.isPresent()) { + return bookie.get(); + } else { + throw new Exception("No bookie is available"); + } + } + public static void runOnAllBookies(DockerClient docker, String... cmds) throws Exception { for (String b : DockerUtils.cubeIdsMatching("bookkeeper")) { DockerUtils.runCommand(docker, b, cmds); diff --git a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/DockerUtils.java b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/DockerUtils.java index 11959afe7..53d6228a6 100644 --- a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/DockerUtils.java +++ b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/DockerUtils.java @@ -18,6 +18,8 @@ */ package org.apache.bookkeeper.tests; +import static java.nio.charset.StandardCharsets.UTF_8; + import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.async.ResultCallback; import com.github.dockerjava.api.command.InspectExecResponse; @@ -131,10 +133,21 @@ public static String getContainerIP(DockerClient docker, String containerId) { throw new IllegalArgumentException("Container " + containerId + " has no networks"); } - public static void runCommand(DockerClient docker, String containerId, String... cmd) throws Exception { + public static String runCommand(DockerClient docker, String containerId, String... cmd) throws Exception { + return runCommand(docker, containerId, false, cmd); + } + + public static String runCommand(DockerClient docker, String containerId, boolean ignoreError, String... cmd) + throws Exception { CompletableFuture<Boolean> future = new CompletableFuture<>(); - String execid = docker.execCreateCmd(containerId).withCmd(cmd).exec().getId(); + String execid = docker.execCreateCmd(containerId) + .withCmd(cmd) + .withAttachStderr(true) + .withAttachStdout(true) + .exec() + .getId(); String cmdString = Arrays.stream(cmd).collect(Collectors.joining(" ")); + StringBuffer output = new StringBuffer(); docker.execStartCmd(execid).withDetach(false).exec(new ResultCallback<Frame>() { @Override public void close() {} @@ -147,6 +160,7 @@ public void onStart(Closeable closeable) { @Override public void onNext(Frame object) { LOG.info("DOCKER.exec({}:{}): {}", containerId, cmdString, object); + output.append(new String(object.getPayload(), UTF_8)); } @Override @@ -169,10 +183,15 @@ public void onComplete() { } int retCode = resp.getExitCode(); if (retCode != 0) { - throw new Exception( - String.format("cmd(%s) failed on %s with exitcode %d", - cmdString, containerId, retCode)); + LOG.error("DOCKER.exec({}:{}): failed with {} : {}", containerId, cmdString, retCode, output); + if (!ignoreError) { + throw new Exception(String.format("cmd(%s) failed on %s with exitcode %d", + cmdString, containerId, retCode)); + } + } else { + LOG.info("DOCKER.exec({}:{}): completed with {}", containerId, cmdString, retCode); } + return output.toString(); } public static Set<String> cubeIdsMatching(String needle) { diff --git a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/BookieShellTestBase.java b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/BookieShellTestBase.java new file mode 100644 index 000000000..f1a58ed10 --- /dev/null +++ b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/BookieShellTestBase.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.bookkeeper.tests.integration; + +import static org.junit.Assert.assertTrue; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Before; +import org.junit.Test; + +@Slf4j +public abstract class BookieShellTestBase { + + private String currentVersion = System.getProperty("currentVersion"); + private String bkScript; + + @Before + public void setup() { + bkScript = "/opt/bookkeeper/" + currentVersion + "/bin/bookkeeper"; + } + + @Test + public abstract void test000_Setup() throws Exception; + + @Test + public abstract void test999_Teardown(); + + protected abstract String runCommandInAnyContainer(String... cmd) throws Exception; + + @Test + public void test001_SimpleTest() throws Exception { + assertTrue(runCommandInAnyContainer( + bkScript, + "shell", + "simpletest", + "-ensemble", "3", + "-writeQuorum", "3", + "-ackQuorum", "2", + "-numEntries", "100" + ).contains("100 entries written to ledger")); + } + + @Test + public void test002_ListROBookies() throws Exception { + assertTrue(runCommandInAnyContainer( + bkScript, + "shell", + "listbookies", + "-ro" + ).contains("No bookie exists!")); + } + + @Test + public void test003_ListRWBookies() throws Exception { + assertTrue(runCommandInAnyContainer( + bkScript, + "shell", + "listbookies", + "-rw" + ).contains("ReadWrite Bookies :")); + } + +} diff --git a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestBookieShellCluster.java b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestBookieShellCluster.java new file mode 100644 index 000000000..04b950a41 --- /dev/null +++ b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestBookieShellCluster.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.bookkeeper.tests.integration; + +import static org.junit.Assert.assertTrue; + +import com.github.dockerjava.api.DockerClient; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.tests.BookKeeperClusterUtils; +import org.apache.bookkeeper.tests.DockerUtils; +import org.jboss.arquillian.junit.Arquillian; +import org.jboss.arquillian.test.api.ArquillianResource; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.MethodSorters; + +@Slf4j +@RunWith(Arquillian.class) +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class TestBookieShellCluster extends BookieShellTestBase { + + @ArquillianResource + private DockerClient docker; + + private String currentVersion = System.getProperty("currentVersion"); + + @Test + @Override + public void test000_Setup() throws Exception { + // First test to run, formats metadata and bookies + if (BookKeeperClusterUtils.metadataFormatIfNeeded(docker, currentVersion)) { + BookKeeperClusterUtils.formatAllBookies(docker, currentVersion); + } + assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, currentVersion)); + } + + @Test + @Override + public void test999_Teardown() { + assertTrue(BookKeeperClusterUtils.stopAllBookies(docker)); + } + + @Override + protected String runCommandInAnyContainer(String... cmds) throws Exception { + String bookie = BookKeeperClusterUtils.getAnyBookie(); + return DockerUtils.runCommand(docker, bookie, cmds); + } + + @Test + @Override + public void test001_SimpleTest() throws Exception { + super.test001_SimpleTest(); + } + + @Test + @Override + public void test002_ListROBookies() throws Exception { + super.test002_ListROBookies(); + } + + @Test + @Override + public void test003_ListRWBookies() throws Exception { + super.test003_ListRWBookies(); + } +} diff --git a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestCLI.java b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestCLI.java new file mode 100644 index 000000000..77618088c --- /dev/null +++ b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestCLI.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.bookkeeper.tests.integration; + +import static org.junit.Assert.assertTrue; + +import com.github.dockerjava.api.DockerClient; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.tests.BookKeeperClusterUtils; +import org.apache.bookkeeper.tests.DockerUtils; +import org.jboss.arquillian.junit.Arquillian; +import org.jboss.arquillian.test.api.ArquillianResource; +import org.junit.Before; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.MethodSorters; + +@Slf4j +@RunWith(Arquillian.class) +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class TestCLI { + + @ArquillianResource + private DockerClient docker; + + private String currentVersion = System.getProperty("currentVersion"); + private String bkCLI; + + @Before + public void setup() { + bkCLI = "/opt/bookkeeper/" + currentVersion + "/bin/bookkeeper-cli"; + } + + @Test + public void test000_Setup() throws Exception { + // First test to run, formats metadata and bookies + if (BookKeeperClusterUtils.metadataFormatIfNeeded(docker, currentVersion)) { + BookKeeperClusterUtils.formatAllBookies(docker, currentVersion); + } + assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, currentVersion)); + + } + + @Test + public void test999_Teardown() throws Exception { + assertTrue(BookKeeperClusterUtils.stopAllBookies(docker)); + } + + @Test + public void test001_SimpleTest() throws Exception { + String bookie = BookKeeperClusterUtils.getAnyBookie(); + assertTrue(DockerUtils.runCommand(docker, bookie, + bkCLI, + "client", + "simpletest", + "--ensemble-size", "3", + "--write-quorum-size", "3", + "--ack-quorum-size", "2", + "--num-entries", "100" + ).contains("100 entries written to ledger")); + } + + @Test + public void test002_ListROBookies() throws Exception { + String bookie = BookKeeperClusterUtils.getAnyBookie(); + assertTrue(DockerUtils.runCommand(docker, bookie, + bkCLI, + "cluster", + "listbookies", + "-ro" + ).contains("No bookie exists!")); + } + + @Test + public void test003_ListRWBookies() throws Exception { + String bookie = BookKeeperClusterUtils.getAnyBookie(); + assertTrue(DockerUtils.runCommand(docker, bookie, + bkCLI, + "cluster", + "listbookies", + "-rw" + ).contains("ReadWrite Bookies :")); + } + +} diff --git a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestDlogCLI.java b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestDlogCLI.java new file mode 100644 index 000000000..83ecde61f --- /dev/null +++ b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestDlogCLI.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.bookkeeper.tests.integration; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.github.dockerjava.api.DockerClient; +import java.util.HashSet; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.tests.BookKeeperClusterUtils; +import org.apache.bookkeeper.tests.DockerUtils; +import org.jboss.arquillian.junit.Arquillian; +import org.jboss.arquillian.test.api.ArquillianResource; +import org.junit.Before; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.MethodSorters; + +@Slf4j +@RunWith(Arquillian.class) +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class TestDlogCLI { + + private static final String DLOG_STREAM_PREFIX = "stream-"; + private static final String STREAMS_REGEX = "0-99"; + + @ArquillianResource + private DockerClient docker; + + private String currentVersion = System.getProperty("currentVersion"); + private String dlogCLI; + private String dlogUri; + + @Before + public void setup() { + dlogCLI = "/opt/bookkeeper/" + currentVersion + "/bin/dlog"; + dlogUri = "distributedlog://" + BookKeeperClusterUtils.zookeeperConnectString(docker) + "/distributedlog"; + } + + @Test + public void test000_Setup() throws Exception { + // First test to run, formats metadata and bookies, then create a dlog namespace + if (BookKeeperClusterUtils.metadataFormatIfNeeded(docker, currentVersion)) { + BookKeeperClusterUtils.formatAllBookies(docker, currentVersion); + } + BookKeeperClusterUtils.createDlogNamespaceIfNeeded(docker, currentVersion, "/distributedlog"); + } + + @Test + public void test999_Teardown() throws Exception { + assertTrue(BookKeeperClusterUtils.stopAllBookies(docker)); + } + + @Test + public void test001_CreateStreams() throws Exception { + String bookie = BookKeeperClusterUtils.getAnyBookie(); + assertTrue(DockerUtils.runCommand(docker, bookie, + dlogCLI, + "tool", + "create", + "--prefix", DLOG_STREAM_PREFIX, + "--expression", STREAMS_REGEX, + "--uri", dlogUri, + "-f" + ).isEmpty()); + } + + @Test + public void test002_ListStreams() throws Exception { + String bookie = BookKeeperClusterUtils.getAnyBookie(); + String output = DockerUtils.runCommand(docker, bookie, + dlogCLI, + "tool", + "list", + "--uri", dlogUri, + "-f" + ); + String[] lines = output.split("\\r?\\n"); + Set<String> streams = new HashSet<>(); + for (String string : lines) { + if (string.startsWith(DLOG_STREAM_PREFIX)) { + streams.add(string); + } + } + assertEquals(100, streams.size()); + } + + @Test + public void test003_ShowStream() throws Exception { + String bookie = BookKeeperClusterUtils.getAnyBookie(); + String output = DockerUtils.runCommand(docker, bookie, true, + dlogCLI, + "tool", + "show", + "--uri", dlogUri, + "--stream", "stream-99", + "-f"); + assertTrue(output.contains("Log stream-99:<default> has no records")); + } + + @Test + public void test004_DeleteStream() throws Exception { + String bookie = BookKeeperClusterUtils.getAnyBookie(); + String output = DockerUtils.runCommand(docker, bookie, + dlogCLI, + "tool", + "delete", + "--uri", dlogUri, + "--stream", "stream-99", + "-f"); + assertTrue(output.isEmpty()); + } + + @Test + public void test005_CheckStreamDeleted() throws Exception { + String bookie = BookKeeperClusterUtils.getAnyBookie(); + String output = DockerUtils.runCommand(docker, bookie, true, + dlogCLI, + "tool", + "show", + "--uri", dlogUri, + "--stream", "stream-99", + "-f"); + assertTrue(output.contains("Log stream-99 does not exist or has been deleted")); + } + +} diff --git a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestSmoke.java b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestSmoke.java index f2eff854e..8ecf28be8 100644 --- a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestSmoke.java +++ b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/TestSmoke.java @@ -44,14 +44,15 @@ import org.apache.bookkeeper.tests.BookKeeperClusterUtils; import org.jboss.arquillian.junit.Arquillian; import org.jboss.arquillian.test.api.ArquillianResource; -import org.junit.After; import org.junit.Assert; -import org.junit.Before; +import org.junit.FixMethodOrder; import org.junit.Test; import org.junit.runner.RunWith; +import org.junit.runners.MethodSorters; @Slf4j @RunWith(Arquillian.class) +@FixMethodOrder(MethodSorters.NAME_ASCENDING) public class TestSmoke { private static byte[] PASSWD = "foobar".getBytes(); @@ -60,8 +61,8 @@ private String currentVersion = System.getProperty("currentVersion"); - @Before - public void setup() throws Exception { + @Test + public void test000_Setup() throws Exception { // First test to run, formats metadata and bookies if (BookKeeperClusterUtils.metadataFormatIfNeeded(docker, currentVersion)) { BookKeeperClusterUtils.formatAllBookies(docker, currentVersion); @@ -69,13 +70,13 @@ public void setup() throws Exception { Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, currentVersion)); } - @After - public void teardown() throws Exception { + @Test + public void tear999_Teardown() { Assert.assertTrue(BookKeeperClusterUtils.stopAllBookies(docker)); } @Test - public void testReadWrite() throws Exception { + public void test001_ReadWrite() throws Exception { String zookeeper = BookKeeperClusterUtils.zookeeperConnectString(docker); int numEntries = 100; try (BookKeeper bk = new BookKeeper(zookeeper)) { @@ -92,7 +93,7 @@ public void testReadWrite() throws Exception { } @Test - public void testReadWriteAdv() throws Exception { + public void test002_ReadWriteAdv() throws Exception { String zookeeper = BookKeeperClusterUtils.zookeeperConnectString(docker); int numEntries = 100; try (BookKeeper bk = new BookKeeper(zookeeper)) { @@ -125,12 +126,12 @@ private static void readEntries(BookKeeper bk, } @Test - public void testTailingReadsWithoutExplicitLac() throws Exception { + public void test003_TailingReadsWithoutExplicitLac() throws Exception { testTailingReads(100, 98, 0); } @Test - public void testTailingReadsWithExplicitLac() throws Exception { + public void test004_TailingReadsWithExplicitLac() throws Exception { testTailingReads(100, 99, 100); } @@ -229,12 +230,12 @@ private static void writeEntries(int numEntries, } @Test - public void testLongTailingReadsWithoutExplicitLac() throws Exception { + public void test005_LongTailingReadsWithoutExplicitLac() throws Exception { testLongPollTailingReads(100, 98, 0); } @Test - public void testLongTailingReadsWithExplicitLac() throws Exception { + public void test006_LongTailingReadsWithExplicitLac() throws Exception { testLongPollTailingReads(100, 99, 100); } diff --git a/tests/pom.xml b/tests/pom.xml index 8ddc84cd5..91b5947e9 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -40,6 +40,7 @@ <module>integration-tests-topologies</module> <module>backward-compat</module> <module>integration</module> + <module>scripts</module> </modules> <build> <plugins> diff --git a/tests/scripts/pom.xml b/tests/scripts/pom.xml new file mode 100644 index 000000000..e100ea81e --- /dev/null +++ b/tests/scripts/pom.xml @@ -0,0 +1,75 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation=" http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.bookkeeper.tests</groupId> + <artifactId>tests-parent</artifactId> + <version>4.8.0-SNAPSHOT</version> + </parent> + + <groupId>org.apache.bookkeeper.tests</groupId> + <artifactId>scripts</artifactId> + <packaging>jar</packaging> + <name>Apache BookKeeper :: Tests :: Bash Scripts Test</name> + + <dependencies> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>com.googlecode.maven-download-plugin</groupId> + <artifactId>download-maven-plugin</artifactId> + <executions> + <execution> + <id>install-shunit2</id> + <phase>integration-test</phase> + <goals> + <goal>wget</goal> + </goals> + <configuration> + <url>https://github.com/kward/shunit2/archive/v2.1.7.zip</url> + <unpack>true</unpack> + <outputDirectory>${project.basedir}/target/lib</outputDirectory> + <skip>${skipTests}</skip> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <executions> + <execution> + <id>bash-tests</id> + <phase>integration-test</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <skip>${skipTests}</skip> + <workingDirectory>${project.basedir}/src/test/bash</workingDirectory> + <executable>${project.basedir}/src/test/bash/bk_test.sh</executable> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> diff --git a/tests/scripts/src/test/bash/bk_test.sh b/tests/scripts/src/test/bash/bk_test.sh new file mode 100755 index 000000000..3c7b02e6f --- /dev/null +++ b/tests/scripts/src/test/bash/bk_test.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# +# vim:et:ft=sh:sts=2:sw=2 +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +ARGV0=`basename "$0"` +PREFIX="bk_test_" +SHELLS="/bin/sh /bin/bash" + +find_tests_at() { + DIR=$1 + PREF=$2 + REGEX="^${PREF}[a-z_]*.sh$" + RESULTS="" + if [ -d ${DIR} ]; then + cd ${DIR} + for f in *.sh; do + if [[ ${f} =~ ${REGEX} ]]; then + RESULTS="${RESULTS} ${f}" + fi + done + fi + echo ${RESULTS} +} + +TESTS=$(find_tests_at "." ${PREFIX}) + +# load common unit test functions +source ./versions +source ./bk_test_helpers + +usage() { + echo "usage: ${ARGV0} [-e key=val ...] [-s shell(s)] [-t test(s)]" +} + +env='' + +# process command line flags +while getopts 'e:hs:t:' opt; do + case ${opt} in + e) # set an environment variable + key=`expr "${OPTARG}" : '\([^=]*\)='` + val=`expr "${OPTARG}" : '[^=]*=\(.*\)'` + if [ -z "${key}" -o -z "${val}" ]; then + usage + exit 1 + fi + eval "${key}='${val}'" + export ${key} + env="${env:+${env} }${key}" + ;; + h) usage; exit 0 ;; # output help + s) shells=${OPTARG} ;; # list of shells to run + t) tests=${OPTARG} ;; # list of tests to run + *) usage; exit 1 ;; + esac +done +shift `expr ${OPTIND} - 1` + +# fill shells and/or tests +shells=${shells:-${SHELLS}} +tests=${tests:-${TESTS}} + +# error checking +if [ -z "${tests}" ]; then + bk_info 'no tests found to run; exiting' + exit 0 +fi + +# print run info +cat <<EOF +#------------------------------------------------------------------------------ +# System data +# +# test run info +shells="${shells}" +tests="${tests}" +EOF +for key in ${env}; do + eval "echo \"${key}=\$${key}\"" +done +echo + +# output system data +echo "# system info" +echo "$ date" +date + +echo "$ uname -mprsv" +uname -mprsv + +# +# run tests +# + +for shell in ${shells}; do + echo + + # check for existance of shell + if [ ! -x ${shell} ]; then + bk_warn "unable to run tests with the ${shell} shell" + continue + fi + + cat <<EOF +#------------------------------------------------------------------------------ +# Running the test suite with ${shell} +# +EOF + + shell_name=`basename ${shell}` + shell_version=`versions_shellVersion "${shell}"` + + echo "shell name: ${shell_name}" + echo "shell version: ${shell_version}" + + # execute the tests + for suite in ${tests}; do + suiteName=`expr "${suite}" : "${PREFIX}\(.*\).sh"` + echo + echo "--- Executing the '${suiteName}' test suite ---" + ( exec ${shell} ./${suite} 2>&1; ) + done +done diff --git a/tests/scripts/src/test/bash/bk_test_bin_common.sh b/tests/scripts/src/test/bash/bk_test_bin_common.sh new file mode 100644 index 000000000..0c726ea97 --- /dev/null +++ b/tests/scripts/src/test/bash/bk_test_bin_common.sh @@ -0,0 +1,256 @@ +#!/usr/bin/env bash +# +# vim:et:ft=sh:sts=2:sw=2 +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# load test helpers +. ./bk_test_helpers + +#------------------------------------------------------------------------------ +# suite tests +# + +testDefaultVariables() { + source ${BK_BINDIR}/common.sh + assertEquals "BINDIR is not set correctly" "${BK_BINDIR}" "${BINDIR}" + assertEquals "BK_HOME is not set correctly" "${BK_HOMEDIR}" "${BK_HOME}" + assertEquals "DEFAULT_LOG_CONF is not set correctly" "${BK_CONFDIR}/log4j.properties" "${DEFAULT_LOG_CONF}" + assertEquals "NETTY_LEAK_DETECTION_LEVEL is not set correctly" "disabled" "${NETTY_LEAK_DETECTION_LEVEL}" + assertEquals "NETTY_RECYCLER_MAXCAPACITY is not set correctly" "1000" "${NETTY_RECYCLER_MAXCAPACITY}" + assertEquals "NETTY_RECYCLER_LINKCAPACITY is not set correctly" "1024" "${NETTY_RECYCLER_LINKCAPACITY}" + assertEquals "BOOKIE_MAX_HEAP_MEMORY is not set correctly" "1g" "${BOOKIE_MAX_HEAP_MEMORY}" + assertEquals "BOOKIE_MIN_HEAP_MEMORY is not set correctly" "1g" "${BOOKIE_MIN_HEAP_MEMORY}" + assertEquals "BOOKIE_MAX_DIRECT_MEMORY is not set correctly" "2g" "${BOOKIE_MAX_DIRECT_MEMORY}" + assertEquals "BOOKIE_MEM_OPTS is not set correctly" "-Xms1g -Xmx1g -XX:MaxDirectMemorySize=2g" "${BOOKIE_MEM_OPTS}" + assertEquals "BOOKIE_GC_OPTS is not set correctly" "${DEFAULT_BOOKIE_GC_OPTS}" "${BOOKIE_GC_OPTS}" + assertEquals "BOOKIE_GC_LOGGING_OPTS is not set correctly" "${DEFAULT_BOOKIE_GC_LOGGING_OPTS}" "${BOOKIE_GC_LOGGING_OPTS}" + assertEquals "CLI_MAX_HEAP_MEMORY is not set correctly" "512M" "${CLI_MAX_HEAP_MEMORY}" + assertEquals "CLI_MIN_HEAP_MEMORY is not set correctly" "256M" "${CLI_MIN_HEAP_MEMORY}" + assertEquals "CLI_MEM_OPTS is not set correctly" "-Xms256M -Xmx512M" "${CLI_MEM_OPTS}" + assertEquals "CLI_GC_OPTS is not set correctly" "${DEFAULT_CLI_GC_OPTS}" "${CLI_GC_OPTS}" + assertEquals "CLI_GC_LOGGING_OPTS is not set correctly" "${DEFAULT_CLI_GC_LOGGING_OPTS}" "${CLI_GC_LOGGING_OPTS}" +} + +testFindModuleJarAt() { + source ${BK_BINDIR}/common.sh + + MODULE="test-module" + + # case 1: empty dir + TEST_DIR1=${BK_TMPDIR}/testdir1 + mkdir -p ${TEST_DIR1} + MODULE_JAR1=$(find_module_jar_at ${TEST_DIR1} ${MODULE}) + assertEquals "No module jar should be found at empty dir" "" "${MODULE_JAR1}" + + # case 2: SNAPSHOT jar + TEST_FILES=( + "invalid-${MODULE}.jar" + "invalid-${MODULE}-4.8.0.jar" + "invalid-${MODULE}-4.8.0-SNAPSHOT.jar" + "${MODULE}.jar.invalid" + "${MODULE}-4.8.0.jar.invalid" + "${MODULE}-4.8.0-SNAPSHOT.jar.invalid" + "${MODULE}.jar" + "${MODULE}-4.8.0-SNAPSHOT.jar" + ) + + TEST_DIR2=${BK_TMPDIR}/testdir2 + mkdir -p ${TEST_DIR2} + count=0 + while [ "x${TEST_FILES[count]}" != "x" ] + do + touch ${TEST_DIR2}/${TEST_FILES[count]} + count=$(( $count + 1 )) + done + MODULE_JAR2=$(find_module_jar_at ${TEST_DIR2} ${MODULE}) + assertEquals "${MODULE}-4.8.0-SNAPSHOT.jar is not found" "${TEST_DIR2}/${MODULE}-4.8.0-SNAPSHOT.jar" "${MODULE_JAR2}" + + # case 3: release jar + TEST_FILES=( + "invalid-${MODULE}.jar" + "invalid-${MODULE}-4.8.0.jar" + "invalid-${MODULE}-4.8.0-SNAPSHOT.jar" + "${MODULE}.jar.invalid" + "${MODULE}-4.8.0.jar.invalid" + "${MODULE}-4.8.0-SNAPSHOT.jar.invalid" + "${MODULE}.jar" + "${MODULE}-4.8.0.jar" + ) + + TEST_DIR3=${BK_TMPDIR}/testdir3 + mkdir -p ${TEST_DIR3} + count=0 + while [ "x${TEST_FILES[count]}" != "x" ] + do + touch ${TEST_DIR3}/${TEST_FILES[count]} + count=$(( $count + 1 )) + done + MODULE_JAR3=$(find_module_jar_at ${TEST_DIR3} ${MODULE}) + assertEquals "${MODULE}-4.8.0.jar is not found" "${TEST_DIR3}/${MODULE}-4.8.0.jar" "${MODULE_JAR3}" +} + +testFindModuleJar() { + BK_HOME=${BK_TMPDIR} + # prepare the env files + mkdir -p ${BK_HOME}/conf + echo "" > ${BK_HOME}/conf/nettyenv.sh + echo "" > ${BK_HOME}/conf/bkenv.sh + echo "" > ${BK_HOME}/conf/bk_cli_env.sh + + source ${BK_BINDIR}/common.sh + + MODULE="test-module" + MODULE_PATH="testmodule" + VERSION="4.8.0" + + TEST_FILES=( + "${MODULE}-${VERSION}.jar" + "lib/${MODULE}-${VERSION}.jar" + "${MODULE_PATH}/target/${MODULE}-${VERSION}.jar" + ) + count=0 + while [ "x${TEST_FILES[count]}" != "x" ] + do + DIR=`dirname ${BK_TMPDIR}/${TEST_FILES[count]}` + mkdir -p ${DIR} + touch ${BK_TMPDIR}/${TEST_FILES[count]} + count=$(( $count + 1 )) + done + + count=0 + while [ "x${TEST_FILES[count]}" != "x" ] + do + FILE="${BK_TMPDIR}/${TEST_FILES[count]}" + ACTUAL_FILE=$(find_module_jar ${MODULE_PATH} ${MODULE}) + + assertEquals "Module file is not found" "${FILE}" "${ACTUAL_FILE}" + + # delete the file + rm ${FILE} + count=$(( $count + 1 )) + done + + unset BK_HOME +} + +testLoadEnvfiles() { + BK_HOME=${BK_TMPDIR} + + # prepare the env files + mkdir -p ${BK_HOME}/conf + echo "NETTY_LEAK_DETECTION_LEVEL=enabled" > ${BK_HOME}/conf/nettyenv.sh + echo "BOOKIE_MAX_HEAP_MEMORY=2048M" > ${BK_HOME}/conf/bkenv.sh + echo "CLI_MAX_HEAP_MEMORY=2048M" > ${BK_HOME}/conf/bk_cli_env.sh + + # load the common.sh + source ${BK_BINDIR}/common.sh + + assertEquals "NETTY_LEAK_DETECTION_LEVEL is not set correctly" "enabled" "${NETTY_LEAK_DETECTION_LEVEL}" + assertEquals "BOOKIE_MAX_HEAP_MEMORY is not set correctly" "2048M" "${BOOKIE_MAX_HEAP_MEMORY}" + assertEquals "CLI_MAX_HEAP_MEMORY is not set correctly" "2048M" "${CLI_MAX_HEAP_MEMORY}" + + unset NETTY_LEAK_DETECTION_LEVEL + unset BOOKIE_MAX_HEAP_MEMORY + unset CLI_MAX_HEAP_MEMORY + unset BK_HOME +} + +testBuildBookieJVMOpts() { + source ${BK_BINDIR}/common.sh + + TEST_LOG_DIR=${BK_TMPDIR}/logdir + TEST_GC_LOG_FILENAME="test-gc.log" + ACTUAL_JVM_OPTS=$(build_bookie_jvm_opts ${TEST_LOG_DIR} ${TEST_GC_LOG_FILENAME}) + EXPECTED_JVM_OPTS="-Xms1g -Xmx1g -XX:MaxDirectMemorySize=2g ${DEFAULT_BOOKIE_GC_OPTS} ${DEFAULT_BOOKIE_GC_LOGGING_OPTS} -Xloggc:${TEST_LOG_DIR}/${TEST_GC_LOG_FILENAME}" + + assertEquals "JVM OPTS is not set correctly" "${EXPECTED_JVM_OPTS}" "${ACTUAL_JVM_OPTS}" +} + +testBuildCLIJVMOpts() { + source ${BK_BINDIR}/common.sh + + TEST_LOG_DIR=${BK_TMPDIR}/logdir + TEST_GC_LOG_FILENAME="test-gc.log" + ACTUAL_JVM_OPTS=$(build_cli_jvm_opts ${TEST_LOG_DIR} ${TEST_GC_LOG_FILENAME}) + EXPECTED_JVM_OPTS="-Xms256M -Xmx512M ${DEFAULT_CLI_GC_OPTS} ${DEFAULT_CLI_GC_LOGGING_OPTS} -Xloggc:${TEST_LOG_DIR}/${TEST_GC_LOG_FILENAME}" + + assertEquals "JVM OPTS is not set correctly" "${EXPECTED_JVM_OPTS}" "${ACTUAL_JVM_OPTS}" +} + +testBuildNettyOpts() { + source ${BK_BINDIR}/common.sh + + ACTUAL_NETTY_OPTS=$(build_netty_opts) + EXPECTED_NETTY_OPTS="-Dio.netty.leakDetectionLevel=disabled \ + -Dio.netty.recycler.maxCapacity.default=1000 \ + -Dio.netty.recycler.linkCapacity=1024" + + assertEquals "Netty OPTS is not set correctly" "${EXPECTED_NETTY_OPTS}" "${ACTUAL_NETTY_OPTS}" +} + +testBuildBookieOpts() { + source ${BK_BINDIR}/common.sh + + ACTUAL_OPTS=$(build_bookie_opts) + EXPECTED_OPTS="-Djava.net.preferIPv4Stack=true" + + assertEquals "Bookie OPTS is not set correctly" "${EXPECTED_OPTS}" "${ACTUAL_OPTS}" +} + +testBuildLoggingOpts() { + TEST_CONF_FILE="test.conf" + TEST_LOG_DIR="test_log_dir" + TEST_LOG_FILE="test_log_file" + TEST_LOGGER="INFO,TEST" + + EXPECTED_OPTS="-Dlog4j.configuration=${TEST_CONF_FILE} \ + -Dbookkeeper.root.logger=${TEST_LOGGER} \ + -Dbookkeeper.log.dir=${TEST_LOG_DIR} \ + -Dbookkeeper.log.file=${TEST_LOG_FILE}" + ACTUAL_OPTS=$(build_logging_opts ${TEST_CONF_FILE} ${TEST_LOG_DIR} ${TEST_LOG_FILE} ${TEST_LOGGER}) + + assertEquals "Logging OPTS is not set correctly" "${EXPECTED_OPTS}" "${ACTUAL_OPTS}" +} + +testBuildCLILoggingOpts() { + TEST_CONF_FILE="test.conf" + TEST_LOG_DIR="test_log_dir" + TEST_LOG_FILE="test_log_file" + TEST_LOGGER="INFO,TEST" + + EXPECTED_OPTS="-Dlog4j.configuration=${TEST_CONF_FILE} \ + -Dbookkeeper.cli.root.logger=${TEST_LOGGER} \ + -Dbookkeeper.cli.log.dir=${TEST_LOG_DIR} \ + -Dbookkeeper.cli.log.file=${TEST_LOG_FILE}" + ACTUAL_OPTS=$(build_cli_logging_opts ${TEST_CONF_FILE} ${TEST_LOG_DIR} ${TEST_LOG_FILE} ${TEST_LOGGER}) + + assertEquals "Logging OPTS is not set correctly" "${EXPECTED_OPTS}" "${ACTUAL_OPTS}" +} + +#------------------------------------------------------------------------------ +# suite functions +# + +oneTimeSetUp() { + bk_oneTimeSetUp +} + +# load and run shUnit2 +. ${BK_SHUNIT} diff --git a/tests/scripts/src/test/bash/bk_test_helpers b/tests/scripts/src/test/bash/bk_test_helpers new file mode 100644 index 000000000..6f43e79d1 --- /dev/null +++ b/tests/scripts/src/test/bash/bk_test_helpers @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# +# vim:et:ft=sh:sts=2:sw=2 +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# name of script +BK_ARGV0=`basename "$0"` + +# path to shUnit2 library. can be overridden by setting BK_SHUNIT_INC. +BK_SHUNIT=${BK_SHUNIT_INC:-../../../target/lib/shunit2-2.1.7/shunit2} + +# path to bk bin directory. +TESTDIR=`dirname "$0"` +BK_BINDIR=`cd ${TESTDIR}/../../../../../bin;pwd` +BK_HOMEDIR=`cd ${TESTDIR}/../../../../..;pwd` +BK_CONFDIR=`cd ${TESTDIR}/../../../../../conf;pwd` + +# +# test helper functions +# + +# message functions +bk_trace() { echo "bk_test:TRACE $@" >&2; } +bk_debug() { echo "bk_test:DEBUG $@" >&2; } +bk_info() { echo "bk_test:INFO $@" >&2; } +bk_warn() { echo "bk_test:WARN $@" >&2; } +bk_error() { echo "bk_test:ERROR $@" >&2; } +bk_fatal() { echo "bk_test:FATAL $@" >&2; } + +bk_oneTimeSetUp() { + # these will be cleaned up automatically by shunit2 + BK_TMPDIR=${SHUNIT_TMPDIR} + stdoutF="${BK_TMPDIR}/stdout" + stderrF="${BK_TMPDIR}/stderr" + expectedF="${BK_TMPDIR}/expected" +} + +# Assert the success of an operation. +# +# If an operation is not successful (i.e. it returns a non-zero return code) +# dump the output of the stderrF to the screen. +# +# Args: +# message: string: message to output [optional] +# result: integer: operation result +assertSuccess() { + if [ $# -eq 2 ]; then + bk_message_=$1 + shift + else + bk_message_='' + fi + bk_result_=$1 + + assertEquals "${bk_message_}" ${SHUNIT_TRUE} ${bk_result_} + [ ${bk_result_} -eq ${SHUNIT_TRUE} ] || cat "${stderrF}" + + unset bk_message_ bk_result_ +} + +assertError() { + if [ $# -eq 2 ]; then + bk_message_="$1: " + shift + else + bk_message_='' + fi + bk_error_=$1 + + bk_file_=${stderrF} + grep "^bk_test:ERROR.*${bk_error_}" "${bk_file_}" >/dev/null + bk_result_=$? + assertTrue "${bk_message_}missing '${bk_error_}' error" ${bk_result_} + [ ${bk_result_} -eq 0 ] || cat "${bk_file_}" + + unset bk_file_ bk_error_ bk_message_ bk_result_ +} diff --git a/tests/scripts/src/test/bash/versions b/tests/scripts/src/test/bash/versions new file mode 100644 index 000000000..9b868f509 --- /dev/null +++ b/tests/scripts/src/test/bash/versions @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# +# vim:et:ft=sh:sts=2:sw=2 +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +ARGV0=`basename "$0"` +LSB_RELEASE="/etc/lsb-release" +VERSIONS_SHELLS="/bin/bash /bin/sh" + +true; TRUE=$? +false; FALSE=$? +ERROR=2 + +UNAME_R=`uname -r` +UNAME_S=`uname -s` + +__versions_haveStrings=${ERROR} + +versions_osName() { + os_name_='unrecognized' + os_system_=${UNAME_S} + os_release_=${UNAME_R} + case ${os_system_} in + CYGWIN_NT-*) os_name_='Cygwin' ;; + Darwin) + os_name_=`/usr/bin/sw_vers -productName` + os_version_=`versions_osVersion` + case ${os_version_} in + 10.4|10.4.[0-9]*) os_name_='Mac OS X Tiger' ;; + 10.5|10.5.[0-9]*) os_name_='Mac OS X Leopard' ;; + 10.6|10.6.[0-9]*) os_name_='Mac OS X Snow Leopard' ;; + 10.7|10.7.[0-9]*) os_name_='Mac OS X Lion' ;; + 10.8|10.8.[0-9]*) os_name_='Mac OS X Mountain Lion' ;; + 10.9|10.9.[0-9]*) os_name_='Mac OS X Mavericks' ;; + 10.10|10.10.[0-9]*) os_name_='Mac OS X Yosemite' ;; + 10.11|10.11.[0-9]*) os_name_='Mac OS X El Capitan' ;; + 10.12|10.12.[0-9]*) os_name_='macOS Sierra' ;; + 10.13|10.13.[0-9]*) os_name_='macOS High Sierra' ;; + *) os_name_='macOS' ;; + esac + ;; + FreeBSD) os_name_='FreeBSD' ;; + Linux) os_name_='Linux' ;; + SunOS) + os_name_='SunOS' + if [ -r '/etc/release' ]; then + if grep 'OpenSolaris' /etc/release >/dev/null; then + os_name_='OpenSolaris' + else + os_name_='Solaris' + fi + fi + ;; + esac + + echo ${os_name_} + unset os_name_ os_system_ os_release_ os_version_ +} + +versions_osVersion() { + os_version_='unrecognized' + os_system_=${UNAME_S} + os_release_=${UNAME_R} + case ${os_system_} in + CYGWIN_NT-*) + os_version_=`expr "${os_release_}" : '\([0-9]*\.[0-9]\.[0-9]*\).*'` + ;; + Darwin) + os_version_=`/usr/bin/sw_vers -productVersion` + ;; + FreeBSD) + os_version_=`expr "${os_release_}" : '\([0-9]*\.[0-9]*\)-.*'` + ;; + Linux) + if [ -r '/etc/os-release' ]; then + os_version_=`awk -F= '$1~/PRETTY_NAME/{print $2}' /etc/os-release \ + |sed 's/"//g'` + elif [ -r '/etc/redhat-release' ]; then + os_version_=`cat /etc/redhat-release` + elif [ -r '/etc/SuSE-release' ]; then + os_version_=`head -n 1 /etc/SuSE-release` + elif [ -r "${LSB_RELEASE}" ]; then + if grep -q 'DISTRIB_ID=Ubuntu' "${LSB_RELEASE}"; then + # shellcheck disable=SC2002 + os_version_=`cat "${LSB_RELEASE}" \ + |awk -F= '$1~/DISTRIB_DESCRIPTION/{print $2}' \ + |sed 's/"//g;s/ /-/g'` + fi + fi + ;; + SunOS) + if [ -r '/etc/release' ]; then + if grep 'OpenSolaris' /etc/release >/dev/null; then # OpenSolaris + os_version_=`grep 'OpenSolaris' /etc/release |awk '{print $2"("$3")"}'` + else # Solaris + major_=`echo "${os_release_}" |sed 's/[0-9]*\.\([0-9]*\)/\1/'` + minor_=`grep Solaris /etc/release |sed 's/[^u]*\(u[0-9]*\).*/\1/'` + os_version_="${major_}${minor_}" + fi + fi + ;; + esac + + echo "${os_version_}" + unset os_release_ os_system_ os_version_ major_ minor_ +} + +versions_shellVersion() { + shell_=$1 + + shell_present_=${FALSE} + case "${shell_}" in + ash) [ -x '/bin/busybox' ] && shell_present_=${TRUE} ;; + *) [ -x "${shell_}" ] && shell_present_=${TRUE} ;; + esac + if [ ${shell_present_} -eq ${FALSE} ]; then + echo 'not installed' + return ${FALSE} + fi + + version_='' + case ${shell_} in + */sh) + # This could be one of any number of shells. Try until one fits. + version_='' + [ -z "${version_}" ] && version_=`versions_shell_bash "${shell_}"` + ;; + */bash) version_=`versions_shell_bash "${shell_}"` ;; + *) version_='invalid' + esac + + echo "${version_:-unknown}" + unset shell_ version_ +} + +versions_shell_bash() { + $1 --version : 2>&1 |grep 'GNU bash' |sed 's/.*version \([^ ]*\).*/\1/' +} + +versions_main() { + # Treat unset variables as an error. + set -u + + os_name=`versions_osName` + os_version=`versions_osVersion` + echo "os: ${os_name} version: ${os_version}" + + for shell in ${VERSIONS_SHELLS}; do + shell_version=`versions_shellVersion "${shell}"` + echo "shell: ${shell} version: ${shell_version}" + done +} + +if [ "${ARGV0}" = 'versions' ]; then + versions_main "$@" +fi ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
