This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push: new e223e7997a Run shfmt and shellcheck on scripts (#2677) e223e7997a is described below commit e223e7997a574c80388f3c707c142e5498bec98f Author: Christopher Tubbs <ctubb...@apache.org> AuthorDate: Mon May 9 15:12:22 2022 -0400 Run shfmt and shellcheck on scripts (#2677) * Run shfmt to format scripts * Run shellcheck to check all scripts for bugs * Add automated check for shfmt and shellcheck during CI --- .github/workflows/scripts.yaml | 58 ++++++ assemble/bin/accumulo | 21 ++- assemble/bin/accumulo-cluster | 53 +++--- assemble/bin/accumulo-service | 51 ++--- assemble/bin/accumulo-util | 72 +++---- assemble/build.sh | 209 +++++++++++++-------- assemble/conf/accumulo-env.sh | 29 ++- assemble/src/main/scripts/create-jshell.sh | 19 +- contrib/ci/find-unapproved-chars.sh | 4 +- contrib/ci/find-unapproved-junit.sh | 5 +- .../ci/install-shfmt.sh | 12 +- contrib/ci/it-matrix.sh | 3 +- .../ci/run-shellcheck.sh | 10 +- .../generate-thrift.sh => contrib/ci/run-shfmt.sh | 9 +- core/src/main/scripts/generate-thrift.sh | 50 ++--- server/base/src/main/scripts/generate-protobuf.sh | 17 +- start/src/test/shell/makeHelloWorldJars.sh | 18 +- start/src/test/shell/makeTestJars.sh | 19 +- test/src/main/scripts/generate-thrift.sh | 2 +- test/src/test/scripts/run-test.sh | 21 +-- 20 files changed, 405 insertions(+), 277 deletions(-) diff --git a/.github/workflows/scripts.yaml b/.github/workflows/scripts.yaml new file mode 100644 index 0000000000..6f78957c31 --- /dev/null +++ b/.github/workflows/scripts.yaml @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# This workflow will build a Java project with Maven +# See also: +# https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven + +name: ScriptQA + +on: + push: + branches: [ '*' ] + pull_request: + branches: [ '*' ] + +jobs: + shfmt: + name: shfmt + timeout-minutes: 3 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Show the first log message + run: git log -n1 + - name: Install shfmt + run: contrib/ci/install-shfmt.sh + - name: Checking formatting of all scripts + run: contrib/ci/run-shfmt.sh + + shellcheck: + name: ShellCheck + timeout-minutes: 3 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Show the first log message + run: git log -n1 + - name: Install shfmt + run: contrib/ci/install-shfmt.sh + - name: Running shellcheck on all scripts + run: contrib/ci/run-shellcheck.sh + diff --git a/assemble/bin/accumulo b/assemble/bin/accumulo index f82f848b19..dc20572406 100755 --- a/assemble/bin/accumulo +++ b/assemble/bin/accumulo @@ -21,14 +21,16 @@ function main() { SOURCE="${BASH_SOURCE[0]}" - while [[ -h "${SOURCE}" ]]; do - bin="$( cd -P "$( dirname "${SOURCE}" )" && pwd )" - SOURCE="$(readlink "${SOURCE}")" - [[ "${SOURCE}" != /* ]] && SOURCE="${bin}/${SOURCE}" + while [[ -L ${SOURCE} ]]; do + bin="$(cd -P "$(dirname "${SOURCE}")" && pwd)" + SOURCE="$(readlink "${SOURCE}")" + [[ ${SOURCE} != /* ]] && SOURCE="${bin}/${SOURCE}" done # Set up variables needed by accumulo-env.sh - bin="$( cd -P "$( dirname "${SOURCE}" )" && pwd )"; export bin - basedir=$( cd -P "${bin}"/.. && pwd ); export basedir + bin="$(cd -P "$(dirname "${SOURCE}")" && pwd)" + export bin + basedir=$(cd -P "${bin}"/.. && pwd) + export basedir export conf="${ACCUMULO_CONF_DIR:-${basedir}/conf}" export lib="${basedir}/lib" export cmd="$1" @@ -41,6 +43,7 @@ function main() { echo "accumulo-env.sh must exist in $conf" exit 1 fi + #shellcheck source=../conf/accumulo-env.sh source "$conf/accumulo-env.sh" # Accumulo is moving away from these variables but they still might be needed @@ -63,7 +66,7 @@ function main() { local jShellPath="$conf/jshell-init.jsh" if [[ $cmd == "jshell" ]]; then shift - if [[ -f "$jShellPath" ]]; then + if [[ -f $jShellPath ]]; then exec "$cmd" --startup DEFAULT --startup "$jShellPath" "$@" else exec "$cmd" "$@" @@ -75,11 +78,11 @@ function main() { else JAVA=$(type -P java) fi - if [[ ! -x "$JAVA" ]]; then + if [[ ! -x $JAVA ]]; then echo "Could not find any executable java binary. Please set java on your PATH or set JAVA_HOME" exit 1 fi - if [[ -n "$ACCUMULO_JAVA_PREFIX" ]]; then + if [[ -n $ACCUMULO_JAVA_PREFIX ]]; then JAVA=("${ACCUMULO_JAVA_PREFIX[*]}" "$JAVA") fi diff --git a/assemble/bin/accumulo-cluster b/assemble/bin/accumulo-cluster index c0b34b0cd1..b4b5eab722 100755 --- a/assemble/bin/accumulo-cluster +++ b/assemble/bin/accumulo-cluster @@ -66,7 +66,8 @@ function parse_config { trap 'rm -f "$CONFIG_FILE"' EXIT CONFIG_FILE=$(mktemp) || exit 1 - ${accumulo_cmd} org.apache.accumulo.core.conf.cluster.ClusterConfigParser "${conf}"/cluster.yaml > "$CONFIG_FILE" || parse_fail + ${accumulo_cmd} org.apache.accumulo.core.conf.cluster.ClusterConfigParser "${conf}"/cluster.yaml >"$CONFIG_FILE" || parse_fail + #shellcheck source=/dev/null . "$CONFIG_FILE" rm -f "$CONFIG_FILE" @@ -86,7 +87,7 @@ function parse_config { for queue in $COMPACTION_QUEUES; do Q="COMPACTOR_HOSTS_${queue}" - if [[ -z "${!Q}" ]]; then + if [[ -z ${!Q} ]]; then echo "WARN: External compaction queue $queue configured, but no compactors configured for it" fi done @@ -107,8 +108,9 @@ function parse_config { } function get_ip() { - ip_addr=$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/') - if [[ $? != 0 ]]; then + local ip_addr + ip_addr=$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/') + if [[ -z $ip_addr ]]; then ip_addr=$(getent ahosts "$(hostname -f)" | grep DGRAM | cut -f 1 -d ' ') fi echo "$ip_addr" @@ -119,16 +121,16 @@ function control_service() { host="$2" service="$3" - local last_instance_id; last_instance_id=1 - [[ "$service" == "tserver" ]] && last_instance_id=${NUM_TSERVERS:-1} + local last_instance_id + last_instance_id=1 + [[ $service == "tserver" ]] && last_instance_id=${NUM_TSERVERS:-1} - for (( inst_id=1; inst_id<=last_instance_id; inst_id++ )) - do + for ((inst_id = 1; inst_id <= last_instance_id; inst_id++)); do ACCUMULO_SERVICE_INSTANCE="" - [[ "$service" == "tserver" && ${NUM_TSERVERS:-1} -gt 1 ]] && ACCUMULO_SERVICE_INSTANCE=${inst_id} - [[ "$service" == "compactor" ]] && ACCUMULO_SERVICE_INSTANCE="${inst_id}_${5}" + [[ $service == "tserver" && ${NUM_TSERVERS:-1} -gt 1 ]] && ACCUMULO_SERVICE_INSTANCE=${inst_id} + [[ $service == "compactor" ]] && ACCUMULO_SERVICE_INSTANCE="${inst_id}_${5}" - if [[ $host == localhost || $host == "$(hostname -s)" || $host == "$(hostname -f)" || $host == $(get_ip) ]] ; then + if [[ $host == localhost || $host == "$(hostname -s)" || $host == "$(hostname -f)" || $host == $(get_ip) ]]; then # # The server processes take arguments (e.g. -p, -o, -q [in the case of the Compactor]). Always add the -a argument # using the value of $host @@ -140,7 +142,7 @@ function control_service() { fi else if [[ $# -gt 3 ]]; then - EXTRA_ARGS="${@:4}" + EXTRA_ARGS="${*:4}" $SSH "$host" "bash -c 'ACCUMULO_SERVICE_INSTANCE=${ACCUMULO_SERVICE_INSTANCE} ${bin}/accumulo-service \"$service\" \"$control_cmd\" \"-a\" \"$host\" $EXTRA_ARGS '" else $SSH "$host" "bash -c 'ACCUMULO_SERVICE_INSTANCE=${ACCUMULO_SERVICE_INSTANCE} ${bin}/accumulo-service \"$service\" \"$control_cmd\" \"-a\" \"$host\"'" @@ -159,8 +161,7 @@ function start_tservers() { for server in $TSERVER_HOSTS; do echo -n "." start_service "$server" tserver & - if (( ++count % 72 == 0 )) ; - then + if ((++count % 72 == 0)); then echo wait fi @@ -171,7 +172,7 @@ function start_tservers() { function start_all() { unset DISPLAY - if [[ "$1" != "--no-tservers" ]]; then + if [[ $1 != "--no-tservers" ]]; then start_tservers fi @@ -202,7 +203,7 @@ function start_all() { function start_here() { - local_hosts="$(hostname -a 2> /dev/null) $(hostname) localhost 127.0.0.1 $(get_ip)" + local_hosts="$(hostname -a 2>/dev/null) $(hostname) localhost 127.0.0.1 $(get_ip)" for host in $local_hosts; do for tserver in $TSERVER_HOSTS; do @@ -325,11 +326,9 @@ function kill_all() { ${accumulo_cmd} org.apache.accumulo.server.util.ZooZap -manager -tservers -compaction-coordinators -compactors } - function stop_all() { echo "Stopping Accumulo cluster..." - if ! ${accumulo_cmd} admin stopAll - then + if ! ${accumulo_cmd} admin stopAll; then echo "Invalid password or unable to connect to the manager" echo "Initiating forced shutdown in 15 seconds (Ctrl-C to abort)" sleep 10 @@ -343,7 +342,7 @@ function stop_all() { # Look for processes not killed by 'admin stopAll' - for end_cmd in "stop" "kill" ; do + for end_cmd in "stop" "kill"; do for manager in $MANAGER_HOSTS; do end_service $end_cmd "$manager" manager @@ -379,7 +378,7 @@ function stop_all() { function stop_here() { # Determine hostname without errors to user - hosts_to_check=("$(hostname -a 2> /dev/null | head -1)" "$(hostname -f)") + hosts_to_check=("$(hostname -a 2>/dev/null | head -1)" "$(hostname -f)") if echo "${TSERVER_HOSTS}" | grep -Eq 'localhost|127[.]0[.]0[.]1'; then ${accumulo_cmd} admin stop localhost @@ -417,12 +416,12 @@ function main() { # Resolve base directory SOURCE="${BASH_SOURCE[0]}" while [ -h "${SOURCE}" ]; do - bin="$( cd -P "$( dirname "${SOURCE}" )" && pwd )" - SOURCE="$(readlink "${SOURCE}")" - [[ "${SOURCE}" != /* ]] && SOURCE="${bin}/${SOURCE}" + bin="$(cd -P "$(dirname "${SOURCE}")" && pwd)" + SOURCE="$(readlink "${SOURCE}")" + [[ ${SOURCE} != /* ]] && SOURCE="${bin}/${SOURCE}" done - bin="$( cd -P "$( dirname "${SOURCE}" )" && pwd )" - basedir=$( cd -P "${bin}"/.. && pwd ) + bin="$(cd -P "$(dirname "${SOURCE}")" && pwd)" + basedir=$(cd -P "${bin}"/.. && pwd) conf="${ACCUMULO_CONF_DIR:-${basedir}/conf}" accumulo_cmd="${bin}/accumulo" @@ -430,7 +429,7 @@ function main() { case "$1" in create-config) - cat <<EOF > "$conf"/cluster.yaml + cat <<EOF >"$conf"/cluster.yaml manager: - localhost diff --git a/assemble/bin/accumulo-service b/assemble/bin/accumulo-service index 78e4462be8..86ac440885 100755 --- a/assemble/bin/accumulo-service +++ b/assemble/bin/accumulo-service @@ -46,21 +46,21 @@ function invalid_args { exit 1 } -function rotate_log () { +function rotate_log() { logfile="$1" max_retained="5" - if [[ -f "$logfile" ]]; then - while [[ "$max_retained" -gt 1 ]]; do - prev=$(( max_retained - 1)) + if [[ -f $logfile ]]; then + while [[ $max_retained -gt 1 ]]; do + prev=$((max_retained - 1)) [ -f "$logfile.$prev" ] && mv -f "$logfile.$prev" "$logfile.$max_retained" max_retained=$prev done - mv -f "$logfile" "$logfile.$max_retained"; + mv -f "$logfile" "$logfile.$max_retained" fi } function start_service() { - if [[ -f "$pid_file" ]]; then + if [[ -f $pid_file ]]; then pid=$(cat "$pid_file") if kill -0 "$pid" 2>/dev/null; then echo "$host : $service already running (${pid})" @@ -78,30 +78,30 @@ function start_service() { rotate_log "$outfile" rotate_log "$errfile" - nohup "${bin}/accumulo" "$service" "$@" >"$outfile" 2>"$errfile" < /dev/null & - echo "$!" > "${pid_file}" + nohup "${bin}/accumulo" "$service" "$@" >"$outfile" 2>"$errfile" </dev/null & + echo "$!" >"${pid_file}" # Check the max open files limit and selectively warn max_files_open=$(ulimit -n) - if [[ -n $max_files_open ]] ; then + if [[ -n $max_files_open ]]; then max_files_recommended=32768 - if (( max_files_open < max_files_recommended )); then + if ((max_files_open < max_files_recommended)); then echo "WARN : Max open files on $host is $max_files_open, recommend $max_files_recommended" >&2 fi fi } function stop_service() { - if [[ -f "$pid_file" ]]; then - echo "Stopping $service on $host"; + if [[ -f $pid_file ]]; then + echo "Stopping $service on $host" kill -s TERM "$(cat "$pid_file")" 2>/dev/null rm -f "${pid_file}" 2>/dev/null fi } function kill_service() { - if [[ -f "$pid_file" ]]; then - echo "Killing $service on $host"; + if [[ -f $pid_file ]]; then + echo "Killing $service on $host" kill -s KILL "$(cat "$pid_file")" 2>/dev/null rm -f "${pid_file}" 2>/dev/null fi @@ -115,17 +115,20 @@ function main() { # Resolve base directory SOURCE="${BASH_SOURCE[0]}" while [ -h "${SOURCE}" ]; do - bin="$( cd -P "$( dirname "${SOURCE}" )" && pwd )" - SOURCE="$(readlink "${SOURCE}")" - [[ "${SOURCE}" != /* ]] && SOURCE="${bin}/${SOURCE}" + bin="$(cd -P "$(dirname "${SOURCE}")" && pwd)" + SOURCE="$(readlink "${SOURCE}")" + [[ ${SOURCE} != /* ]] && SOURCE="${bin}/${SOURCE}" done # Set up variables needed by accumulo-env.sh - export bin="$( cd -P "$( dirname "${SOURCE}" )" && pwd )" - export basedir=$( cd -P "${bin}"/.. && pwd ) + bin="$(cd -P "$(dirname "${SOURCE}")" && pwd)" + export bin + basedir=$(cd -P "${bin}"/.. && pwd) + export basedir export conf="${ACCUMULO_CONF_DIR:-${basedir}/conf}" export lib="${basedir}/lib" if [[ -f "${conf}/accumulo-env.sh" ]]; then + #shellcheck source=../conf/accumulo-env.sh source "${conf}/accumulo-env.sh" fi ACCUMULO_LOG_DIR="${ACCUMULO_LOG_DIR:-${basedir}/logs}" @@ -135,9 +138,9 @@ function main() { mkdir -p "$ACCUMULO_PID_DIR" 2>/dev/null host="$(hostname)" - if [[ -z "$host" ]]; then - host=$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/') - fi + if [[ -z $host ]]; then + host=$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/') + fi service="$1" if [[ $service == "master" ]]; then @@ -147,11 +150,11 @@ function main() { pid_file="${ACCUMULO_PID_DIR}/accumulo-${service}${ACCUMULO_SERVICE_INSTANCE}.pid" case "$service" in - gc|manager|monitor|tserver|compaction-coordinator|compactor) + gc | manager | monitor | tserver | compaction-coordinator | compactor) if [[ -z $2 ]]; then invalid_args "<command> cannot be empty" fi - case "$2" in + case "$2" in start) start_service "${@:3}" ;; diff --git a/assemble/bin/accumulo-util b/assemble/bin/accumulo-util index 723c6903d8..1a2dc3c11f 100755 --- a/assemble/bin/accumulo-util +++ b/assemble/bin/accumulo-util @@ -39,7 +39,7 @@ function build_native() { fi native_tarballs=("$basedir"/lib/accumulo-native-*.tar.gz) - if (( ${#native_tarballs[@]} > 1 )); then + if ((${#native_tarballs[@]} > 1)); then echo "Found multiple native tar.gz files: ${native_tarballs[*]}" exit 1 fi @@ -56,10 +56,9 @@ function build_native() { TMP_DIR=$(mktemp -d /tmp/accumulo-native.XXXXXX) || exit 1 # Unpack the tarball to our temp directory - if ! tar xf "${native_tarballs[0]}" -C "${TMP_DIR}" - then - echo "Failed to unpack native tarball to ${TMP_DIR}" - exit 1 + if ! tar xf "${native_tarballs[0]}" -C "${TMP_DIR}"; then + echo "Failed to unpack native tarball to ${TMP_DIR}" + exit 1 fi # Move to the first (only) directory in our unpacked tarball @@ -69,7 +68,10 @@ function build_native() { # Make the native library export USERFLAGS="$*" - make || { echo 'Make failed!'; exit 1; } + make || { + echo 'Make failed!' + exit 1 + } # "install" the artifact cp libaccumulo.* "${final_native_target}" || exit 1 @@ -81,43 +83,43 @@ function build_native() { } function gen_monitor_cert() { - if [[ -z "$JAVA_HOME" || ! -d "$JAVA_HOME" ]]; then + if [[ -z $JAVA_HOME || ! -d $JAVA_HOME ]]; then echo "JAVA_HOME=${JAVA_HOME} must be set and exist" exit 1 fi ALIAS="default" - KEYPASS=$(LC_CTYPE=C tr -dc '#-~' < /dev/urandom | tr -d '<>&' | head -c 20) - STOREPASS=$(LC_CTYPE=C tr -dc '#-~' < /dev/urandom | tr -d '<>&' | head -c 20) + KEYPASS=$(LC_CTYPE=C tr -dc '#-~' </dev/urandom | tr -d '<>&' | head -c 20) + STOREPASS=$(LC_CTYPE=C tr -dc '#-~' </dev/urandom | tr -d '<>&' | head -c 20) KEYSTOREPATH="${conf}/keystore.jks" TRUSTSTOREPATH="${conf}/cacerts.jks" CERTPATH="${conf}/server.cer" - if [[ -e "$KEYSTOREPATH" ]]; then - rm -i "$KEYSTOREPATH" - if [[ -e "$KEYSTOREPATH" ]]; then - echo "KeyStore already exists, exiting" - exit 1 - fi + if [[ -e $KEYSTOREPATH ]]; then + rm -i "$KEYSTOREPATH" + if [[ -e $KEYSTOREPATH ]]; then + echo "KeyStore already exists, exiting" + exit 1 + fi fi - if [[ -e "$TRUSTSTOREPATH" ]]; then - rm -i "$TRUSTSTOREPATH" - if [[ -e "$TRUSTSTOREPATH" ]]; then - echo "TrustStore already exists, exiting" - exit 2 - fi + if [[ -e $TRUSTSTOREPATH ]]; then + rm -i "$TRUSTSTOREPATH" + if [[ -e $TRUSTSTOREPATH ]]; then + echo "TrustStore already exists, exiting" + exit 2 + fi fi - if [[ -e "$CERTPATH" ]]; then - rm -i "$CERTPATH" - if [[ -e "$CERTPATH" ]]; then - echo "Certificate already exists, exiting" - exit 3 + if [[ -e $CERTPATH ]]; then + rm -i "$CERTPATH" + if [[ -e $CERTPATH ]]; then + echo "Certificate already exists, exiting" + exit 3 fi fi "${JAVA_HOME}/bin/keytool" -genkey -alias "$ALIAS" -keyalg RSA -keypass "$KEYPASS" -storepass "$KEYPASS" -keystore "$KEYSTOREPATH" "${JAVA_HOME}/bin/keytool" -export -alias "$ALIAS" -storepass "$KEYPASS" -file "$CERTPATH" -keystore "$KEYSTOREPATH" - "${JAVA_HOME}/bin/keytool" -import -v -trustcacerts -alias "$ALIAS" -file "$CERTPATH" -keystore "$TRUSTSTOREPATH" -storepass "$STOREPASS" <<< "yes" + "${JAVA_HOME}/bin/keytool" -import -v -trustcacerts -alias "$ALIAS" -file "$CERTPATH" -keystore "$TRUSTSTOREPATH" -storepass "$STOREPASS" <<<"yes" echo echo "keystore and truststore generated. now add the following to accumulo.properties:" @@ -132,14 +134,16 @@ function gen_monitor_cert() { function main() { SOURCE="${BASH_SOURCE[0]}" while [ -h "${SOURCE}" ]; do - bin="$( cd -P "$( dirname "${SOURCE}" )" && pwd )" - SOURCE="$(readlink "${SOURCE}")" - [[ "${SOURCE}" != /* ]] && SOURCE="${bin}/${SOURCE}" + bin="$(cd -P "$(dirname "${SOURCE}")" && pwd)" + SOURCE="$(readlink "${SOURCE}")" + [[ ${SOURCE} != /* ]] && SOURCE="${bin}/${SOURCE}" done - bin="$( cd -P "$( dirname "${SOURCE}" )" && pwd )" - basedir=$( cd -P "${bin}"/.. && pwd ) - conf="${ACCUMULO_CONF_DIR:-${basedir}/conf}" - lib="${basedir}/lib" + bin="$(cd -P "$(dirname "${SOURCE}")" && pwd)" + export bin + basedir=$(cd -P "${bin}"/.. && pwd) + export basedir + export conf="${ACCUMULO_CONF_DIR:-${basedir}/conf}" + export lib="${basedir}/lib" case "$1" in build-native) diff --git a/assemble/build.sh b/assemble/build.sh index 5bbca61372..9678baf964 100755 --- a/assemble/build.sh +++ b/assemble/build.sh @@ -30,21 +30,36 @@ export tagPrefix="rel/" # check if running in a color terminal terminalSupportsColor() { - local c; c=$(tput colors 2>/dev/null) || c=-1 + local c + c=$(tput colors 2>/dev/null) || c=-1 [[ -t 1 ]] && [[ $c -ge 8 ]] } terminalSupportsColor && doColor=1 || doColor=0 -color() { local c; c=$1; shift; [[ $doColor -eq 1 ]] && echo -e "\\e[0;${c}m${*}\\e[0m" || echo "$@"; } +color() { + local c + c=$1 + shift + [[ $doColor -eq 1 ]] && echo -e "\\e[0;${c}m${*}\\e[0m" || echo "$@" +} red() { color 31 "$@"; } green() { color 32 "$@"; } yellow() { color 33 "$@"; } -fail() { echo -e ' ' "$@"; exit 1; } -runLog() { local o; o=$1 && shift && echo "$(green Running) $(yellow "$@" '>>' "$o")" && echo Running "$@" >> "$o" && eval "$@" >> "$o"; } +fail() { + echo -e ' ' "$@" + exit 1 +} +runLog() { + local o + o=$1 && shift && echo "$(green Running) $(yellow "$@" '>>' "$o")" && echo Running "$@" >>"$o" && eval "$@" >>"$o" +} run() { echo "$(green Running) $(yellow "$@")" && eval "$@"; } -currentBranch() { local b; b=$(git symbolic-ref -q HEAD) && echo "${b##refs/heads/}"; } +currentBranch() { + local b + b=$(git symbolic-ref -q HEAD) && echo "${b##refs/heads/}" +} cacheGPG() { # make sure gpg agent has key cached @@ -54,7 +69,8 @@ cacheGPG() { until selectFingerprint; do red 'ERROR - Invalid selection' done - local TESTFILE; TESTFILE=$(mktemp --tmpdir "${USER}-gpgTestFile-XXXXXXXX.txt") + local TESTFILE + TESTFILE=$(mktemp --tmpdir "${USER}-gpgTestFile-XXXXXXXX.txt") [[ -r $TESTFILE ]] && gpg --local-user "$SELECTED_FINGERPRINT" --sign "$TESTFILE" && rm -f "$TESTFILE" "$TESTFILE.gpg" } @@ -69,9 +85,14 @@ prompter() { echo "$x" } -pretty() { local f; f=$1; shift; git log "--pretty=tformat:$f" "$@"; } +pretty() { + local f + f=$1 + shift + git log "--pretty=tformat:$f" "$@" +} gitCommits() { pretty %H "$@"; } -gitCommit() { gitCommits -n1 "$@"; } +gitCommit() { gitCommits -n1 "$@"; } gitSubject() { pretty %s "$@"; } selectFingerprint() { @@ -100,42 +121,50 @@ selectFingerprint() { createEmail() { echo - yellow "IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!!" + yellow "IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!!" echo - echo " Release candidate will be staged at:" - echo " $(yellow 'https://repository.apache.org/#stagingRepositories')" + echo " Release candidate will be staged at:" + echo " $(yellow 'https://repository.apache.org/#stagingRepositories')" echo - echo " $(green 'DO') click $(green 'Close') to complete the staging process!" - echo " $(red 'DO *NOT*') click $(red 'Release') until after the vote has been approved!" + echo " $(green 'DO') click $(green 'Close') to complete the staging process!" + echo " $(red 'DO *NOT*') click $(red 'Release') until after the vote has been approved!" echo - yellow "IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!!" + yellow "IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!!" echo # $1 version (optional); $2 rc sequence num (optional); $3 staging repo num (optional) - local ver; [[ -n "$1" ]] && ver=$1 || ver=$(prompter 'version to be released (eg. x.y.z)' '[0-9]+[.][0-9]+[.][0-9]+') - local rc; [[ -n "$2" ]] && rc=$2 || rc=$(prompter 'release candidate sequence number (eg. 1, 2, etc.)' '[0-9]+') - local stagingrepo; [[ -n "$3" ]] && stagingrepo=$3 || stagingrepo=$(prompter 'staging repository number from https://repository.apache.org/#stagingRepositories' '[0-9]+') - local srcSha; [[ -n "$4" ]] && srcSha=$4 || srcSha=$(prompter 'SHA512 for source tarball' '[0-9a-f]{128}') - local binSha; [[ -n "$5" ]] && binSha=$5 || binSha=$(prompter 'SHA512 for binary tarball' '[0-9a-f]{128}') - - local branch; branch=$ver-rc$rc - local commit; commit=$(gitCommit "$branch") || exit 1 - local tag; tag=$tagPrefix$ver + local ver + [[ -n $1 ]] && ver=$1 || ver=$(prompter 'version to be released (eg. x.y.z)' '[0-9]+[.][0-9]+[.][0-9]+') + local rc + [[ -n $2 ]] && rc=$2 || rc=$(prompter 'release candidate sequence number (eg. 1, 2, etc.)' '[0-9]+') + local stagingrepo + [[ -n $3 ]] && stagingrepo=$3 || stagingrepo=$(prompter 'staging repository number from https://repository.apache.org/#stagingRepositories' '[0-9]+') + local srcSha + [[ -n $4 ]] && srcSha=$4 || srcSha=$(prompter 'SHA512 for source tarball' '[0-9a-f]{128}') + local binSha + [[ -n $5 ]] && binSha=$5 || binSha=$(prompter 'SHA512 for binary tarball' '[0-9a-f]{128}') + + local branch + branch=$ver-rc$rc + local commit + commit=$(gitCommit "$branch") || exit 1 + local tag + tag=$tagPrefix$ver echo - yellow "IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!!" + yellow "IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!!" echo - echo " Don't forget to make the staged branch available for review by" - echo " pushing a branch named $(yellow "$branch") with its head at" - echo " $(yellow "$commit"):" + echo " Don't forget to make the staged branch available for review by" + echo " pushing a branch named $(yellow "$branch") with its head at" + echo " $(yellow "$commit"):" echo - echo " # replace $(yellow "\$origin") with your upstream remote name " - echo " $(green "git push") $(yellow "\$origin") $(green "$commit:refs/heads/$branch")" + echo " # replace $(yellow 'ORIGIN') with your upstream remote name " + echo " $(green "git push") $(yellow 'ORIGIN') $(green "$commit:refs/heads/$branch")" echo - echo " Remember, $(red DO NOT PUSH) the $(red "$tag") tag until after the vote" - echo " passes and the tag is re-made with a gpg signature using:" - echo " $(red "git tag -f -s -m '$projNameLong $ver' $tag") \\" - echo " $(red "$commit")" + echo " Remember, $(red DO NOT PUSH) the $(red "$tag") tag until after the vote" + echo " passes and the tag is re-made with a gpg signature using:" + echo " $(red "git tag -f -s -m '$projNameLong $ver' $tag") \\" + echo " $(red "$commit")" echo - yellow "IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!!" + yellow "IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!!" echo read -r -s -p 'Press Enter to generate the [VOTE] email...' echo 1>&2 @@ -146,14 +175,18 @@ createEmail() { done # compute the date with a buffer of 30 minutes - local votedate; votedate=$(date -d "+3 days 30 minutes" "+%s") + local votedate + votedate=$(date -d "+3 days 30 minutes" "+%s") # round back to the previous half-hour - local halfhour; halfhour=$((votedate - (votedate % 1800))) + local halfhour + halfhour=$((votedate - (votedate % 1800))) votedate=$(date -u -d"1970-01-01 $halfhour seconds UTC") export TZ="America/New_York" - local edtvotedate; edtvotedate=$(date -d"1970-01-01 $halfhour seconds UTC") + local edtvotedate + edtvotedate=$(date -d"1970-01-01 $halfhour seconds UTC") export TZ="America/Los_Angeles" - local pdtvotedate; pdtvotedate=$(date -d"1970-01-01 $halfhour seconds UTC") + local pdtvotedate + pdtvotedate=$(date -d"1970-01-01 $halfhour seconds UTC") cat <<EOF $(yellow '============================================================') @@ -224,9 +257,12 @@ cleanUpAndFail() { run git checkout "$3" # pre-populate branches with expected next branch; de-duplicate later - local branches; branches=("$4") - local tags; tags=() - local x; local y + local branches + branches=("$4") + local tags + tags=() + local x + local y for x in $(gitCommits "${cBranch}..${nBranch}"); do for y in $(git branch --contains "$x" | cut -c3-); do branches=("${branches[@]}" "$y") @@ -238,7 +274,8 @@ cleanUpAndFail() { # de-duplicate branches local a - local tmpArray; tmpArray=("${branches[@]}") + local tmpArray + tmpArray=("${branches[@]}") IFS=$'\n' read -d '' -r -a branches < <(printf '%s\n' "${tmpArray[@]}" | sort -u) for x in "${branches[@]}"; do echo "Do you wish to clean up (delete) the branch $(yellow "$x")?" @@ -277,21 +314,21 @@ selectRemote() { createReleaseCandidate() { echo - yellow "WARNING!! WARNING!! WARNING!! WARNING!! WARNING!! WARNING!!" + yellow "WARNING!! WARNING!! WARNING!! WARNING!! WARNING!! WARNING!!" echo - echo " Don't forget to $(yellow 'Set up your development environment')!" - echo " For details, see the section by that name at:" - echo " $(green 'https://infra.apache.org/publishing-maven-artifacts.html')" + echo " Don't forget to $(yellow 'Set up your development environment')!" + echo " For details, see the section by that name at:" + echo " $(green 'https://infra.apache.org/publishing-maven-artifacts.html')" echo - echo " This will modify your local git repository by creating" - echo " branches and tags. Afterwards, you may need to perform" - echo " some manual steps to complete the release or to rollback" - echo " in the case of failure." + echo " This will modify your local git repository by creating" + echo " branches and tags. Afterwards, you may need to perform" + echo " some manual steps to complete the release or to rollback" + echo " in the case of failure." echo - echo " Release candidate will be staged at:" - echo " $(yellow 'https://repository.apache.org/#stagingRepositories')" + echo " Release candidate will be staged at:" + echo " $(yellow 'https://repository.apache.org/#stagingRepositories')" echo - yellow "WARNING!! WARNING!! WARNING!! WARNING!! WARNING!! WARNING!!" + yellow "WARNING!! WARNING!! WARNING!! WARNING!! WARNING!! WARNING!!" echo if [[ ${#@} -ne 0 ]]; then @@ -303,30 +340,38 @@ createReleaseCandidate() { ver=$(xmllint --shell pom.xml <<<'xpath /*[local-name()="project"]/*[local-name()="version"]/text()' | grep content= | cut -f2 -d=) ver=${ver%%-SNAPSHOT} echo "Building release candidate for version: $(green "$ver")" - local tag; tag=$tagPrefix$ver - - local cBranch; cBranch=$(currentBranch) || fail "$(red Failure)" to get current branch from git - local rc; rc=$(prompter 'release candidate sequence number (eg. 1, 2, etc.)' '[0-9]+') - local tmpNextVer; tmpNextVer="${ver%.*}.$((${ver##*.}+1))" - local nextVer; nextVer=$(prompter "next snapshot version to be released [$tmpNextVer]" '([0-9]+[.][0-9]+[.][0-9]+)?') + local tag + tag=$tagPrefix$ver + + local cBranch + cBranch=$(currentBranch) || fail "$(red Failure)" to get current branch from git + local rc + rc=$(prompter 'release candidate sequence number (eg. 1, 2, etc.)' '[0-9]+') + local tmpNextVer + tmpNextVer="${ver%.*}.$((${ver##*.} + 1))" + local nextVer + nextVer=$(prompter "next snapshot version to be released [$tmpNextVer]" '([0-9]+[.][0-9]+[.][0-9]+)?') [[ -n $nextVer ]] || nextVer=$tmpNextVer - local rcBranch; rcBranch=$ver-rc$rc - local nBranch; nBranch=$rcBranch-next + local rcBranch + rcBranch=$ver-rc$rc + local nBranch + nBranch=$rcBranch-next cacheGPG || fail "Unable to cache GPG credentials into gpg-agent" # create working branch - run git checkout -b "$nBranch" "$cBranch" || \ + run git checkout -b "$nBranch" "$cBranch" || fail "Unable to create working branch $(red "$nBranch") from $(red "$cBranch")!" # create a release candidate from a branch - local oFile; oFile=$(mktemp --tmpdir "$projName-build-$rcBranch-XXXXXXXX.log") + local oFile + oFile=$(mktemp --tmpdir "$projName-build-$rcBranch-XXXXXXXX.log") { [[ -w $oFile ]] && runLog "$oFile" mvn clean release:clean } || cleanUpAndFail 'mvn clean release:clean' "$oFile" "$cBranch" "$nBranch" - runLog "$oFile" mvn -B release:prepare -DdevelopmentVersion="${nextVer}-SNAPSHOT" "-Dgpg.keyname=$SELECTED_FINGERPRINT" || \ + runLog "$oFile" mvn -B release:prepare -DdevelopmentVersion="${nextVer}-SNAPSHOT" "-Dgpg.keyname=$SELECTED_FINGERPRINT" || cleanUpAndFail "mvn -B release:prepare -DdevelopmentVersion=${nextVer}-SNAPSHOT" "$oFile" "$cBranch" "$nBranch" - runLog "$oFile" mvn release:perform "-Dgpg.keyname=$SELECTED_FINGERPRINT" || \ + runLog "$oFile" mvn release:perform "-Dgpg.keyname=$SELECTED_FINGERPRINT" || cleanUpAndFail "mvn release:perform" "$oFile" "$cBranch" "$nBranch" # switch back to original branch @@ -334,22 +379,22 @@ createReleaseCandidate() { # verify the next branch contains both expected log messages and no more { - [[ $(gitCommits "${cBranch}..${nBranch}" | wc -l) -eq 2 ]] && \ - [[ $(gitCommit "${nBranch}~2") == $(gitCommit "${cBranch}") ]] && \ - [[ $(gitSubject "${nBranch}") =~ ^\[maven-release-plugin\]\ prepare\ for\ next ]] && \ + [[ $(gitCommits "${cBranch}..${nBranch}" | wc -l) -eq 2 ]] && + [[ $(gitCommit "${nBranch}~2") == $(gitCommit "${cBranch}") ]] && + [[ $(gitSubject "${nBranch}") =~ ^\[maven-release-plugin\]\ prepare\ for\ next ]] && [[ $(gitSubject "${nBranch}~1") =~ ^\[maven-release-plugin\]\ prepare\ release\ rel[/] ]] } || cleanUpAndFail "verifying that $nBranch contains only logs from release plugin" # verify the tag is one behind $nBranch and one ahead of $cBranch - [[ $(gitCommit "${nBranch}~1") == $(gitCommit "refs/tags/$tag") ]] || \ + [[ $(gitCommit "${nBranch}~1") == $(gitCommit "refs/tags/$tag") ]] || cleanUpAndFail "verifying that ${nBranch}~1 == refs/tags/$tag" # remove tag which was created - run git tag -d "$tag" || \ + run git tag -d "$tag" || cleanUpAndFail "removing unused git tag $tag" # create release candidate branch to vote on - run git branch "$rcBranch" "${nBranch}~1" || \ + run git branch "$rcBranch" "${nBranch}~1" || cleanUpAndFail "creating branch $rcBranch" # determine remote to use @@ -364,20 +409,25 @@ createReleaseCandidate() { echo "Do you wish to push the following branches to $SELECTED_REMOTE ($(green "$SELECTED_REMOTE_URL"))?" echo " $(yellow "$rcBranch") (for others to examine for the vote)" echo " $(yellow "$nBranch") (for merging into $cBranch if vote passes)" - local a; a=$(prompter "letter 'y' or 'n'" '[yn]') + local a + a=$(prompter "letter 'y' or 'n'" '[yn]') { - [[ $a == 'y' ]] && \ + [[ $a == 'y' ]] && run git push -u "$SELECTED_REMOTE" "refs/heads/$nBranch" "refs/heads/$rcBranch" } || red "Did not push branches; you'll need to perform this step manually." fi - local numSrc; numSrc=$(find target/checkout/ -type f -name "$projName-$ver-source-release.tar.gz" | wc -l) - local numBin; numBin=$(find target/checkout/ -type f -name "$projName-$ver-bin.tar.gz" | wc -l) + local numSrc + numSrc=$(find target/checkout/ -type f -name "$projName-$ver-source-release.tar.gz" | wc -l) + local numBin + numBin=$(find target/checkout/ -type f -name "$projName-$ver-bin.tar.gz" | wc -l) shopt -s globstar - local srcSha; srcSha="" - local binSha; binSha="" - [[ $numSrc = "1" ]] && srcSha=$(sha512sum target/checkout/**/"$projName-$ver-source-release.tar.gz" | cut -f1 -d" ") - [[ $numBin = "1" ]] && binSha=$(sha512sum target/checkout/**/"$projName-$ver-bin.tar.gz" | cut -f1 -d" ") + local srcSha + srcSha="" + local binSha + binSha="" + [[ $numSrc == "1" ]] && srcSha=$(sha512sum target/checkout/**/"$projName-$ver-source-release.tar.gz" | cut -f1 -d" ") + [[ $numBin == "1" ]] && binSha=$(sha512sum target/checkout/**/"$projName-$ver-bin.tar.gz" | cut -f1 -d" ") # continue to creating email notification echo "$(red Running)" "$(yellow "$scriptname" --create-email "$ver" "$rc")" @@ -394,4 +444,3 @@ elif [[ $1 == '--create-email' ]]; then else fail "Missing one of: $(red --create-release-candidate), $(red --create-email)" fi - diff --git a/assemble/conf/accumulo-env.sh b/assemble/conf/accumulo-env.sh index 2ce716077f..7286d05a8b 100644 --- a/assemble/conf/accumulo-env.sh +++ b/assemble/conf/accumulo-env.sh @@ -44,22 +44,26 @@ ZOOKEEPER_HOME="${ZOOKEEPER_HOME:-/path/to/zookeeper}" ########################## ## Verify that Hadoop & Zookeeper installation directories exist -if [[ ! -d "$ZOOKEEPER_HOME" ]]; then +if [[ ! -d $ZOOKEEPER_HOME ]]; then echo "ZOOKEEPER_HOME=$ZOOKEEPER_HOME is not set to a valid directory in accumulo-env.sh" exit 1 fi -if [[ ! -d "$HADOOP_HOME" ]]; then +if [[ ! -d $HADOOP_HOME ]]; then echo "HADOOP_HOME=$HADOOP_HOME is not set to a valid directory in accumulo-env.sh" exit 1 fi ## Build using existing CLASSPATH, conf/ directory, dependencies in lib/, and external Hadoop & Zookeeper dependencies -if [[ -n "$CLASSPATH" ]]; then +if [[ -n $CLASSPATH ]]; then + # conf is set by calling script that sources this env file + #shellcheck disable=SC2154 CLASSPATH="${CLASSPATH}:${conf}" else CLASSPATH="${conf}" fi ZK_JARS=$(find "$ZOOKEEPER_HOME/lib/" -maxdepth 1 -name '*.jar' -not -name '*slf4j*' -not -name '*log4j*' | paste -sd:) +# lib is set by calling script that sources this env file +#shellcheck disable=SC2154 CLASSPATH="${CLASSPATH}:${lib}/*:${HADOOP_CONF_DIR}:${ZOOKEEPER_HOME}/*:${ZK_JARS}:${HADOOP_HOME}/share/hadoop/client/*" export CLASSPATH @@ -68,24 +72,29 @@ export CLASSPATH ################################################################## ## JVM options set for all processes. Extra options can be passed in by setting ACCUMULO_JAVA_OPTS to an array of options. -JAVA_OPTS=($ACCUMULO_JAVA_OPTS +read -r -a accumulo_initial_opts < <(echo "$ACCUMULO_JAVA_OPTS") +JAVA_OPTS=("${accumulo_initial_opts[@]}" '-XX:OnOutOfMemoryError=kill -9 %p' '-XX:-OmitStackTraceInFastThrow' '-Djava.net.preferIPv4Stack=true' "-Daccumulo.native.lib.path=${lib}/native") ## Make sure Accumulo native libraries are built since they are enabled by default -"${bin}"/accumulo-util build-native &> /dev/null +# bin is set by calling script that sources this env file +#shellcheck disable=SC2154 +"${bin}"/accumulo-util build-native &>/dev/null ## JVM options set for individual applications +# cmd is set by calling script that sources this env file +#shellcheck disable=SC2154 case "$cmd" in - manager|master) JAVA_OPTS=("${JAVA_OPTS[@]}" '-Xmx512m' '-Xms512m') ;; + manager | master) JAVA_OPTS=("${JAVA_OPTS[@]}" '-Xmx512m' '-Xms512m') ;; monitor) JAVA_OPTS=("${JAVA_OPTS[@]}" '-Xmx256m' '-Xms256m') ;; - gc) JAVA_OPTS=("${JAVA_OPTS[@]}" '-Xmx256m' '-Xms256m') ;; + gc) JAVA_OPTS=("${JAVA_OPTS[@]}" '-Xmx256m' '-Xms256m') ;; tserver) JAVA_OPTS=("${JAVA_OPTS[@]}" '-Xmx768m' '-Xms768m') ;; compaction-coordinator) JAVA_OPTS=("${JAVA_OPTS[@]}" '-Xmx512m' '-Xms512m') ;; compactor) JAVA_OPTS=("${JAVA_OPTS[@]}" '-Xmx256m' '-Xms256m') ;; - *) JAVA_OPTS=("${JAVA_OPTS[@]}" '-Xmx256m' '-Xms64m') ;; + *) JAVA_OPTS=("${JAVA_OPTS[@]}" '-Xmx256m' '-Xms64m') ;; esac ## JVM options set for logging. Review log4j2.properties file to see how they are used. @@ -106,7 +115,7 @@ JAVA_OPTS=("${JAVA_OPTS[@]}" #JAVA_OPTS=("${JAVA_OPTS[@]}" "-javaagent:path/to/opentelemetry-javaagent-all.jar") case "$cmd" in - monitor|gc|manager|master|tserver|compaction-coordinator|compactor) + monitor | gc | manager | master | tserver | compaction-coordinator | compactor) JAVA_OPTS=("${JAVA_OPTS[@]}" "-Dlog4j.configurationFile=log4j2-service.properties") ;; *) @@ -123,7 +132,7 @@ export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-1} ## Add Hadoop native libraries to shared library paths given operating system case "$(uname)" in Darwin) export DYLD_LIBRARY_PATH="${HADOOP_HOME}/lib/native:${DYLD_LIBRARY_PATH}" ;; - *) export LD_LIBRARY_PATH="${HADOOP_HOME}/lib/native:${LD_LIBRARY_PATH}" ;; + *) export LD_LIBRARY_PATH="${HADOOP_HOME}/lib/native:${LD_LIBRARY_PATH}" ;; esac ############################################### diff --git a/assemble/src/main/scripts/create-jshell.sh b/assemble/src/main/scripts/create-jshell.sh index 5197ce05e9..b550439ae6 100755 --- a/assemble/src/main/scripts/create-jshell.sh +++ b/assemble/src/main/scripts/create-jshell.sh @@ -21,7 +21,7 @@ function addAccumuloAPI() { local srcDir="$1" api apiPath # Validate each source directory before populating JShell-Init file - if [[ ! -d "$srcDir" ]]; then + if [[ ! -d $srcDir ]]; then echo "$srcDir is not a valid directory. Please make sure it exists." exit 1 fi @@ -37,7 +37,7 @@ function addAccumuloAPI() { } function addClientBuild() { -cat <<EOF + cat <<EOF URL clientPropUrl = AccumuloClient.class.getClassLoader().getResource("accumulo-client.properties"); AccumuloClient client = null; @@ -58,15 +58,15 @@ function main() { local SOURCE bin scriptPath mainBase corePath # Establish Accumulo's main base directory SOURCE="${BASH_SOURCE[0]}" - while [[ -h "${SOURCE}" ]]; do - bin="$( cd -P "$( dirname "${SOURCE}" )" && pwd )" + while [[ -L ${SOURCE} ]]; do + bin="$(cd -P "$(dirname "${SOURCE}")" && pwd)" SOURCE="$(readlink "${SOURCE}")" - [[ "${SOURCE}" != /* ]] && SOURCE="${bin}/${SOURCE}" + [[ ${SOURCE} != /* ]] && SOURCE="${bin}/${SOURCE}" done # Establish file and folder paths for JShell config - scriptPath="$( cd -P "$( dirname "${SOURCE}" )" && pwd )" - mainBase="$( cd -P "${scriptPath}"/../../../.. && pwd )" + scriptPath="$(cd -P "$(dirname "${SOURCE}")" && pwd)" + mainBase="$(cd -P "${scriptPath}"/../../../.. && pwd)" corePath="$mainBase/core/src/main/java/org/apache/accumulo/core" # Create new jshell-init file @@ -87,11 +87,12 @@ function main() { echo 'import org.apache.hadoop.io.Text;' echo echo '// Initialization Code' - echo 'System.out.println("Preparing JShell for Apache Accumulo\n");' + echo 'System.out.println("Preparing JShell for Apache Accumulo");' + echo 'System.out.println();' echo echo '// Accumulo Client Build' addClientBuild - } > "$mainBase/assemble/target/jshell-init.jsh" + } >"$mainBase/assemble/target/jshell-init.jsh" } main "$@" diff --git a/contrib/ci/find-unapproved-chars.sh b/contrib/ci/find-unapproved-chars.sh index fa48b307e7..2558319497 100755 --- a/contrib/ci/find-unapproved-chars.sh +++ b/contrib/ci/find-unapproved-chars.sh @@ -40,7 +40,8 @@ function findallnonascii() { } function comparecounts() { - local count; count=$(findallnonascii | wc -l) + local count + count=$(findallnonascii | wc -l) if [[ $NUM_EXPECTED -ne $count ]]; then echo "Expected $NUM_EXPECTED, but found $count unapproved non-ASCII characters:" findallnonascii 'print' @@ -49,4 +50,3 @@ function comparecounts() { } comparecounts && echo "Found exactly $NUM_EXPECTED unapproved non-ASCII characters, as expected" - diff --git a/contrib/ci/find-unapproved-junit.sh b/contrib/ci/find-unapproved-junit.sh index a10bea4499..45e12d86a8 100755 --- a/contrib/ci/find-unapproved-junit.sh +++ b/contrib/ci/find-unapproved-junit.sh @@ -28,7 +28,7 @@ ALLOWED=( start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java ) -ALLOWED_PIPE_SEP=$({ for x in "${ALLOWED[@]}"; do echo "$x" ; done; } | paste -sd'|') +ALLOWED_PIPE_SEP=$({ for x in "${ALLOWED[@]}"; do echo "$x"; done; } | paste -sd'|') function findalljunitproblems() { # -P for perl matching, -R for recursive, -l for matching files @@ -44,7 +44,8 @@ function findalljunitproblems() { } function comparecounts() { - local count; count=$(findalljunitproblems | wc -l) + local count + count=$(findalljunitproblems | wc -l) if [[ $NUM_EXPECTED -ne $count ]]; then echo "Expected $NUM_EXPECTED, but found $count classes using the wrong JUnit APIs:" findalljunitproblems 'print' diff --git a/test/src/main/scripts/generate-thrift.sh b/contrib/ci/install-shfmt.sh similarity index 69% copy from test/src/main/scripts/generate-thrift.sh copy to contrib/ci/install-shfmt.sh index 3cd1b8beed..53eb76d863 100755 --- a/test/src/main/scripts/generate-thrift.sh +++ b/contrib/ci/install-shfmt.sh @@ -18,10 +18,12 @@ # under the License. # -# This script will regenerate the thrift code for accumulo-trace. -INCLUDED_MODULES=(-) -BASE_OUTPUT_PACKAGE='org.apache.accumulo' -PACKAGES_TO_GENERATE=(test.rpc) +# Install shfmt tool to search for and optionally format bash scripts +# This is useful for other CI tools to run ShellCheck and shfmt to format -. ../core/src/main/scripts/generate-thrift.sh +set -e +set -x +shfmt_version=3.4.3 +sudo wget "https://github.com/mvdan/sh/releases/download/v${shfmt_version}/shfmt_v${shfmt_version}_linux_amd64" -O /usr/local/bin/shfmt && + sudo chmod +x /usr/local/bin/shfmt diff --git a/contrib/ci/it-matrix.sh b/contrib/ci/it-matrix.sh index 89a887dbf3..4fa9990959 100755 --- a/contrib/ci/it-matrix.sh +++ b/contrib/ci/it-matrix.sh @@ -31,8 +31,7 @@ echo -n '::set-output name=matrix::{"profile":[' for x in $(find "$gitRootDir" -name '*IT.java' -exec basename '{}' .java \; | sort -u | xargs -n "$testsPerJob" | tr ' ' ','); do [[ $count -gt 0 ]] && echo -n ',' echo -n "{\"name\":\"task_$count\",\"its\":\"$x\"}" - ((count=count+1)) + ((count = count + 1)) done echo ']}' echo "Finished creating matrix ($count tasks)" - diff --git a/test/src/main/scripts/generate-thrift.sh b/contrib/ci/run-shellcheck.sh similarity index 79% copy from test/src/main/scripts/generate-thrift.sh copy to contrib/ci/run-shellcheck.sh index 3cd1b8beed..f42122feb5 100755 --- a/test/src/main/scripts/generate-thrift.sh +++ b/contrib/ci/run-shellcheck.sh @@ -18,10 +18,10 @@ # under the License. # -# This script will regenerate the thrift code for accumulo-trace. -INCLUDED_MODULES=(-) -BASE_OUTPUT_PACKAGE='org.apache.accumulo' -PACKAGES_TO_GENERATE=(test.rpc) +# Run ShellCheck on all bash scripts -. ../core/src/main/scripts/generate-thrift.sh +set -e +set -x +mapfile -t filestocheck < <(shfmt -f .) +shellcheck -P SCRIPTDIR -x "${filestocheck[@]}" diff --git a/test/src/main/scripts/generate-thrift.sh b/contrib/ci/run-shfmt.sh similarity index 79% copy from test/src/main/scripts/generate-thrift.sh copy to contrib/ci/run-shfmt.sh index 3cd1b8beed..5ecbefca09 100755 --- a/test/src/main/scripts/generate-thrift.sh +++ b/contrib/ci/run-shfmt.sh @@ -18,10 +18,9 @@ # under the License. # -# This script will regenerate the thrift code for accumulo-trace. -INCLUDED_MODULES=(-) -BASE_OUTPUT_PACKAGE='org.apache.accumulo' -PACKAGES_TO_GENERATE=(test.rpc) +# Check formatting of all bash scripts -. ../core/src/main/scripts/generate-thrift.sh +set -e +set -x +shfmt -ln bash -l -d -i 2 -ci -s . diff --git a/core/src/main/scripts/generate-thrift.sh b/core/src/main/scripts/generate-thrift.sh index a582da1caa..1fc01c470f 100755 --- a/core/src/main/scripts/generate-thrift.sh +++ b/core/src/main/scripts/generate-thrift.sh @@ -30,12 +30,12 @@ # Use INCLUDED_MODULES=(-) in calling scripts that require no other modules # ======================================================================================================================== [[ -z $REQUIRED_THRIFT_VERSION ]] && REQUIRED_THRIFT_VERSION='0.15.0' -[[ -z $INCLUDED_MODULES ]] && INCLUDED_MODULES=() -[[ -z $BASE_OUTPUT_PACKAGE ]] && BASE_OUTPUT_PACKAGE='org.apache.accumulo.core' -[[ -z $PACKAGES_TO_GENERATE ]] && PACKAGES_TO_GENERATE=(gc master manager tabletserver securityImpl clientImpl dataImpl replication trace compaction) -[[ -z $BUILD_DIR ]] && BUILD_DIR='target' -[[ -z $LANGUAGES_TO_GENERATE ]] && LANGUAGES_TO_GENERATE=(java) -[[ -z $FINAL_DIR ]] && FINAL_DIR='src/main' +[[ -z $INCLUDED_MODULES ]] && INCLUDED_MODULES=() +[[ -z $BASE_OUTPUT_PACKAGE ]] && BASE_OUTPUT_PACKAGE='org.apache.accumulo.core' +[[ -z $PACKAGES_TO_GENERATE ]] && PACKAGES_TO_GENERATE=(gc master manager tabletserver securityImpl clientImpl dataImpl replication trace compaction) +[[ -z $BUILD_DIR ]] && BUILD_DIR='target' +[[ -z $LANGUAGES_TO_GENERATE ]] && LANGUAGES_TO_GENERATE=(java) +[[ -z $FINAL_DIR ]] && FINAL_DIR='src/main' # ======================================================================================================================== fail() { @@ -44,8 +44,7 @@ fail() { } # Test to see if we have thrift installed -VERSION=$(thrift -version 2>/dev/null | grep -F "${REQUIRED_THRIFT_VERSION}" | wc -l | sed -e 's/^ *//' -e 's/ *$//') -if [[ "${VERSION}" != '1' ]] ; then +if ! thrift -version 2>/dev/null | grep -qF "${REQUIRED_THRIFT_VERSION}"; then # Nope: bail echo "****************************************************" echo "*** thrift is not available" @@ -55,29 +54,30 @@ if [[ "${VERSION}" != '1' ]] ; then fi # Include thrift sources from additional modules -THRIFT_ARGS='' +THRIFT_ARGS=() for i in "${INCLUDED_MODULES[@]}"; do - if [ ${i} != '-' ]; then - test -d ${i} || fail missing required included module ${i} - THRIFT_ARGS="${THRIFT_ARGS} -I ${i}/src/main/thrift" + if [[ $i != '-' ]]; then + test -d "$i" || fail missing required included module "$i" + THRIFT_ARGS=("${THRIFT_ARGS[@]}" -I "$i/src/main/thrift") fi done # Ensure output directories are created -THRIFT_ARGS="${THRIFT_ARGS} -o $BUILD_DIR" -mkdir -p $BUILD_DIR -rm -rf $BUILD_DIR/gen-java +THRIFT_ARGS=("${THRIFT_ARGS[@]}" -o "$BUILD_DIR") +mkdir -p "$BUILD_DIR" +rm -rf "$BUILD_DIR"/gen-java for f in src/main/thrift/*.thrift; do - thrift ${THRIFT_ARGS} --gen java:generated_annotations=suppress "$f" || fail unable to generate java thrift classes - thrift ${THRIFT_ARGS} --gen py "$f" || fail unable to generate python thrift classes - thrift ${THRIFT_ARGS} --gen rb "$f" || fail unable to generate ruby thrift classes - thrift ${THRIFT_ARGS} --gen cpp "$f" || fail unable to generate cpp thrift classes + thrift "${THRIFT_ARGS[@]}" --gen java:generated_annotations=suppress "$f" || fail unable to generate java thrift classes + thrift "${THRIFT_ARGS[@]}" --gen py "$f" || fail unable to generate python thrift classes + thrift "${THRIFT_ARGS[@]}" --gen rb "$f" || fail unable to generate ruby thrift classes + thrift "${THRIFT_ARGS[@]}" --gen cpp "$f" || fail unable to generate cpp thrift classes done # For all generated thrift code, get rid of all warnings and add the LICENSE header # add dummy method to suppress "unnecessary suppress warnings" for classes which don't have any unused variables # this only affects classes, enums aren't affected +#shellcheck disable=SC1004 find $BUILD_DIR/gen-java -name '*.java' -exec grep -Zl '^public class ' {} + | xargs -0 sed -i -e 's/^[}]$/ private static void unusedMethod() {}\ }/' @@ -117,8 +117,9 @@ for lang in "${LANGUAGES_TO_GENERATE[@]}"; do esac for file in "${FILE_SUFFIX[@]}"; do - for f in $(find $BUILD_DIR/gen-$lang -name "*$file"); do - cat - "$f" > "${f}-with-license" <<EOF + mapfile -t ALL_FILES_TO_LICENSE < <(find "$BUILD_DIR/gen-$lang" -name "*$file") + for f in "${ALL_FILES_TO_LICENSE[@]}"; do + cat - "$f" >"${f}-with-license" <<EOF ${PREFIX}${LINE_NOTATION} Licensed to the Apache Software Foundation (ASF) under one ${LINE_NOTATION} or more contributor license agreements. See the NOTICE file ${LINE_NOTATION} distributed with this work for additional information @@ -170,9 +171,10 @@ for d in "${PACKAGES_TO_GENERATE[@]}"; do esac mkdir -p "$DDIR" for file in "${FILE_SUFFIX[@]}"; do - for f in $(find $SDIR -name *$file); do - DEST="$DDIR/$(basename $f)" - if ! cmp -s "${f}-with-license" "${DEST}" ; then + mapfile -t ALL_LICENSE_FILES_TO_COPY < <(find "$SDIR" -name "*$file") + for f in "${ALL_LICENSE_FILES_TO_COPY[@]}"; do + DEST="$DDIR/$(basename "$f")" + if ! cmp -s "${f}-with-license" "${DEST}"; then echo cp -f "${f}-with-license" "${DEST}" cp -f "${f}-with-license" "${DEST}" || fail unable to copy files to java workspace fi diff --git a/server/base/src/main/scripts/generate-protobuf.sh b/server/base/src/main/scripts/generate-protobuf.sh index e182a835d2..842b26d876 100755 --- a/server/base/src/main/scripts/generate-protobuf.sh +++ b/server/base/src/main/scripts/generate-protobuf.sh @@ -26,8 +26,8 @@ # Leave the BUILD_DIR and FINAL_DIR alone for Maven builds. # ======================================================================================================================== [[ -z $REQUIRED_PROTOC_VERSION ]] && REQUIRED_PROTOC_VERSION='libprotoc 3.19.2' -[[ -z $BUILD_DIR ]] && BUILD_DIR='target/proto' -[[ -z $FINAL_DIR ]] && FINAL_DIR='src/main' +[[ -z $BUILD_DIR ]] && BUILD_DIR='target/proto' +[[ -z $FINAL_DIR ]] && FINAL_DIR='src/main' # ======================================================================================================================== fail() { @@ -36,8 +36,7 @@ fail() { } # Test to see if we have protoc installed -VERSION=$(protoc --version 2>/dev/null | grep -F "${REQUIRED_PROTOC_VERSION}" | wc -l) -if [[ $VERSION -ne 1 ]] ; then +if ! protoc --version 2>/dev/null | grep -qF "${REQUIRED_PROTOC_VERSION}"; then # Nope: bail echo "****************************************************" echo "*** protoc is not available" @@ -65,8 +64,9 @@ SUFFIX=" FILE_SUFFIX=(.java) for file in "${FILE_SUFFIX[@]}"; do - for f in $(find $BUILD_DIR/ -name "*$file"); do - cat - "$f" > "${f}-with-license" <<EOF + mapfile -t ALL_FILES_TO_LICENSE < <(find "$BUILD_DIR/" -name "*$file") + for f in "${ALL_FILES_TO_LICENSE[@]}"; do + cat - "$f" >"${f}-with-license" <<EOF ${PREFIX}${LINE_NOTATION} Licensed to the Apache Software Foundation (ASF) under one or more ${LINE_NOTATION} contributor license agreements. See the NOTICE file distributed with ${LINE_NOTATION} this work for additional information regarding copyright ownership. @@ -91,9 +91,10 @@ DDIR="${FINAL_DIR}/java/org/apache/accumulo/server/replication/proto" FILE_SUFFIX=(.java) mkdir -p "$DDIR" for file in "${FILE_SUFFIX[@]}"; do - for f in $(find $SDIR -name *$file); do + mapfile -t ALL_LICENSE_FILES_TO_COPY < <(find "$SDIR" -name "*$file") + for f in "${ALL_LICENSE_FILES_TO_COPY[@]}"; do DEST=$DDIR/$(basename "$f") - if ! cmp -s "${f}-with-license" "${DEST}" ; then + if ! cmp -s "${f}-with-license" "${DEST}"; then echo cp -f "${f}-with-license" "${DEST}" cp -f "${f}-with-license" "${DEST}" || fail unable to copy files to java workspace fi diff --git a/start/src/test/shell/makeHelloWorldJars.sh b/start/src/test/shell/makeHelloWorldJars.sh index 2a24a261ca..878ec5dbc7 100755 --- a/start/src/test/shell/makeHelloWorldJars.sh +++ b/start/src/test/shell/makeHelloWorldJars.sh @@ -19,17 +19,17 @@ # if [ -z "$JAVA_HOME" ]; then - echo "JAVA_HOME is not set. Java is required to proceed" - exit 1 + echo "JAVA_HOME is not set. Java is required to proceed" + exit 1 fi mkdir -p target/generated-sources/HelloWorld/test -sed "s/%%/Hello World\!/" < src/test/java/test/HelloWorldTemplate > target/generated-sources/HelloWorld/test/HelloWorld.java -$JAVA_HOME/bin/javac target/generated-sources/HelloWorld/test/HelloWorld.java -d target/generated-sources/HelloWorld -$JAVA_HOME/bin/jar -cf target/test-classes/HelloWorld.jar -C target/generated-sources/HelloWorld test/HelloWorld.class +sed "s/%%/Hello World\!/" <src/test/java/test/HelloWorldTemplate >target/generated-sources/HelloWorld/test/HelloWorld.java +"$JAVA_HOME"/bin/javac target/generated-sources/HelloWorld/test/HelloWorld.java -d target/generated-sources/HelloWorld +"$JAVA_HOME"/bin/jar -cf target/test-classes/HelloWorld.jar -C target/generated-sources/HelloWorld test/HelloWorld.class rm -r target/generated-sources/HelloWorld/test mkdir -p target/generated-sources/HalloWelt/test -sed "s/%%/Hallo Welt/" < src/test/java/test/HelloWorldTemplate > target/generated-sources/HalloWelt/test/HelloWorld.java -$JAVA_HOME/bin/javac target/generated-sources/HalloWelt/test/HelloWorld.java -d target/generated-sources/HalloWelt -$JAVA_HOME/bin/jar -cf target/test-classes/HelloWorld2.jar -C target/generated-sources/HalloWelt test/HelloWorld.class -rm -r target/generated-sources/HalloWelt/test \ No newline at end of file +sed "s/%%/Hallo Welt/" <src/test/java/test/HelloWorldTemplate >target/generated-sources/HalloWelt/test/HelloWorld.java +"$JAVA_HOME"/bin/javac target/generated-sources/HalloWelt/test/HelloWorld.java -d target/generated-sources/HalloWelt +"$JAVA_HOME"/bin/jar -cf target/test-classes/HelloWorld2.jar -C target/generated-sources/HalloWelt test/HelloWorld.class +rm -r target/generated-sources/HalloWelt/test diff --git a/start/src/test/shell/makeTestJars.sh b/start/src/test/shell/makeTestJars.sh index f6781e3b96..19a00ce8fa 100755 --- a/start/src/test/shell/makeTestJars.sh +++ b/start/src/test/shell/makeTestJars.sh @@ -19,16 +19,15 @@ # if [ -z "$JAVA_HOME" ]; then - echo "JAVA_HOME is not set. Java is required to proceed" - exit 1 + echo "JAVA_HOME is not set. Java is required to proceed" + exit 1 fi -for x in A B C -do - mkdir -p target/generated-sources/$x/test target/test-classes/ClassLoaderTest$x - sed "s/testX/test$x/" < src/test/java/test/TestTemplate > target/generated-sources/$x/test/TestObject.java - export CLASSPATH=target/test-classes - $JAVA_HOME/bin/javac target/generated-sources/$x/test/TestObject.java -d target/generated-sources/$x - $JAVA_HOME/bin/jar -cf target/test-classes/ClassLoaderTest$x/Test.jar -C target/generated-sources/$x test/TestObject.class - rm -r target/generated-sources/$x +for x in A B C; do + mkdir -p target/generated-sources/$x/test target/test-classes/ClassLoaderTest$x + sed "s/testX/test$x/" <src/test/java/test/TestTemplate >target/generated-sources/$x/test/TestObject.java + export CLASSPATH=target/test-classes + "$JAVA_HOME"/bin/javac target/generated-sources/$x/test/TestObject.java -d target/generated-sources/$x + "$JAVA_HOME"/bin/jar -cf target/test-classes/ClassLoaderTest$x/Test.jar -C target/generated-sources/$x test/TestObject.class + rm -r target/generated-sources/$x done diff --git a/test/src/main/scripts/generate-thrift.sh b/test/src/main/scripts/generate-thrift.sh index 3cd1b8beed..c01241b82a 100755 --- a/test/src/main/scripts/generate-thrift.sh +++ b/test/src/main/scripts/generate-thrift.sh @@ -23,5 +23,5 @@ INCLUDED_MODULES=(-) BASE_OUTPUT_PACKAGE='org.apache.accumulo' PACKAGES_TO_GENERATE=(test.rpc) +#shellcheck source=../../../../core/src/main/scripts/generate-thrift.sh . ../core/src/main/scripts/generate-thrift.sh - diff --git a/test/src/test/scripts/run-test.sh b/test/src/test/scripts/run-test.sh index 9c1daec97e..5d09a97cdb 100755 --- a/test/src/test/scripts/run-test.sh +++ b/test/src/test/scripts/run-test.sh @@ -18,25 +18,24 @@ # under the License. # - if [[ -z $1 ]]; then - echo "Usage: $(basename $0) TestClass1[,TestClass2,TestClass3] ..." - echo " $(basename $0) \"Prefix*IT[,Prefix2*IT]\" ..." - echo " $(basename $0) \"MyIT#method1+method2\" ..." - exit 1 + echo "Usage: $(basename "$0") TestClass1[,TestClass2,TestClass3] ..." + echo " $(basename "$0") \"Prefix*IT[,Prefix2*IT]\" ..." + echo " $(basename "$0") \"MyIT#method1+method2\" ..." + exit 1 fi # Start: Resolve Script Directory SOURCE="${BASH_SOURCE[0]}" -while [[ -h "${SOURCE}" ]]; do # resolve $SOURCE until the file is no longer a symlink - bin=$( cd -P "$( dirname "${SOURCE}" )" && pwd ) - SOURCE=$(readlink "${SOURCE}") - [[ "${SOURCE}" != /* ]] && SOURCE="${bin}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +while [[ -L ${SOURCE} ]]; do # resolve $SOURCE until the file is no longer a symlink + bin=$(cd -P "$(dirname "${SOURCE}")" && pwd) + SOURCE=$(readlink "${SOURCE}") + [[ ${SOURCE} != /* ]] && SOURCE="${bin}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located done -bin=$( cd -P "$( dirname "${SOURCE}" )" && pwd ) +bin=$(cd -P "$(dirname "${SOURCE}")" && pwd) # Stop: Resolve Script Directory -cd "$bin/.." +cd "$bin/.." || exit 1 tests=$1 shift