This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push: new e74a1f5 Replace use of `pssh` with `parallel-ssh` (#179) e74a1f5 is described below commit e74a1f5904bc165d4054ea1a0fd93d8eacc82606 Author: Dom G <47725857+domgargu...@users.noreply.github.com> AuthorDate: Thu Dec 16 14:39:28 2021 -0500 Replace use of `pssh` with `parallel-ssh` (#179) * Replaced pssh and pscp * Fall back to parallel-ssh when pssh unavailable * Fix quoting in sudo commands in agitator * Preserve PSSH environment over sudo commands for agitator * Properly set PSSH in perl agitator code from process environment * Fix a couple typos in READMEs Co-authored-by: Christopher Tubbs <ctubb...@apache.org> --- bin/agitator | 15 +++++++++++---- libexec/master-agitator.pl | 14 ++++++++++---- test/bench/README.md | 2 +- test/compat/japi-compliance/README.md | 2 +- test/scalability/README.md | 2 +- test/stress/README.md | 7 ++++--- test/stress/start-readers.sh | 4 ++-- test/stress/start-writers.sh | 4 ++-- test/stress/stop-readers.sh | 2 +- test/stress/stop-writers.sh | 2 +- test/stress/stress-env.sh | 12 ++++++++++++ 11 files changed, 46 insertions(+), 20 deletions(-) diff --git a/bin/agitator b/bin/agitator index 8d15f94..cf1dd55 100755 --- a/bin/agitator +++ b/bin/agitator @@ -31,7 +31,14 @@ EOF } function start_agitator() { - hash pssh 2>/dev/null || { echo >&2 "The agitator requires pssh to be installed. Aborting."; exit 1; } + ## check that pssh is installed, falling back to parallel-ssh if needed + if hash pssh 2>/dev/null; then + PSSH=pssh + elif hash parallel-ssh 2>/dev/null; then + PSSH=parallel-ssh + else + echo >&2 "The agitator requires pssh/parallel-ssh to be installed. Aborting."; exit 1; + fi mkdir -p "${at_home}/logs" log_base="${at_home}/logs/$(date +%Y%m%d%H%M%S)_$(hostname)" @@ -50,15 +57,15 @@ function start_agitator() { $tserver_cmd > "${tserver_log}.out" 2> "${tserver_log}.err" & else echo "Running master-agitator and tserver-agitator as $AGTR_ACCUMULO_USER using sudo." - sudo -u "$AGTR_ACCUMULO_USER" "$master_cmd" > "${master_log}.out" 2> "${master_log}.err" & - sudo -u "$AGTR_ACCUMULO_USER" "$tserver_cmd" > "${tserver_log}.out" 2> "${tserver_log}.err" & + sudo --preserve-env=PSSH -u "$AGTR_ACCUMULO_USER" $master_cmd > "${master_log}.out" 2> "${master_log}.err" & + sudo -u "$AGTR_ACCUMULO_USER" $tserver_cmd > "${tserver_log}.out" 2> "${tserver_log}.err" & fi if [[ $AGITATOR_USER == "$AGTR_HDFS_USER" ]]; then echo "Running datanode-agitator as $AGITATOR_USER" $datanode_cmd > "${datanode_log}.out" 2> "${datanode_log}.err" & else echo "Running datanode-agitator as $AGTR_HDFS_USER using sudo." - sudo -u "$AGTR_HDFS_USER" "$datanode_cmd" > "${datanode_log}.out" 2> "${datanode_log}.err" & + sudo -u "$AGTR_HDFS_USER" $datanode_cmd > "${datanode_log}.out" 2> "${datanode_log}.err" & fi if ${AGTR_HDFS:-false} ; then diff --git a/libexec/master-agitator.pl b/libexec/master-agitator.pl index e90c70d..dfc2155 100755 --- a/libexec/master-agitator.pl +++ b/libexec/master-agitator.pl @@ -30,6 +30,12 @@ if( defined $ENV{'ACCUMULO_HOME'} ){ print "ERROR: ACCUMULO_HOME needs to be set!"; exit(1); } +if( defined $ENV{'PSSH'} ){ + $PSSH = $ENV{'PSSH'}; +} else { + print "ERROR: PSSH needs to be set!"; + exit(1); +} $accumuloConfDir = $accumuloHome . '/conf'; @@ -67,11 +73,11 @@ while(1){ system($cmd); }else{ print STDERR "$t Killing all masters\n"; - $cmd = "pssh -h $accumuloConfDir/masters \"pkill -f '[ ]org.apache.accumulo.start.*master'\" < /dev/null"; + $cmd = "$PSSH -h $accumuloConfDir/masters \"pkill -f '[ ]org.apache.accumulo.start.*master'\" < /dev/null"; print "$t $cmd\n"; system($cmd); - $cmd = "pssh -h $accumuloConfDir/$gcfile \"pkill -f '[ ]org.apache.accumulo.start.*gc'\" < /dev/null"; + $cmd = "$PSSH -h $accumuloConfDir/$gcfile \"pkill -f '[ ]org.apache.accumulo.start.*gc'\" < /dev/null"; print "$t $cmd\n"; system($cmd); } @@ -80,11 +86,11 @@ while(1){ $t = strftime "%Y%m%d %H:%M:%S", localtime; print STDERR "$t Running start-all\n"; - $cmd = "pssh -h $accumuloConfDir/masters \"$accumuloHome/bin/accumulo-service master start\" < /dev/null"; + $cmd = "$PSSH -h $accumuloConfDir/masters \"$accumuloHome/bin/accumulo-service master start\" < /dev/null"; print "$t $cmd\n"; system($cmd); - $cmd = "pssh -h $accumuloConfDir/$gcfile \"$accumuloHome/bin/accumulo-service gc start\" < /dev/null"; + $cmd = "$PSSH -h $accumuloConfDir/$gcfile \"$accumuloHome/bin/accumulo-service gc start\" < /dev/null"; print "$t $cmd\n"; system($cmd); } diff --git a/test/bench/README.md b/test/bench/README.md index 0929bc3..2e35fa9 100644 --- a/test/bench/README.md +++ b/test/bench/README.md @@ -55,7 +55,7 @@ The 4th Benchmark is Terasort. Run the benchmarks with speed 'slow' to do a ful Misc ---- -These benchmarks create tables in accumulo named 'test_ingest' and 'CloudIngestTest'. These tables are deleted +These benchmarks create tables in accumulo named `test_ingest` and 'CloudIngestTest'. These tables are deleted at the end of the benchmarks. The benchmarks will also alter user auths while it runs. It is recommended that a benchmark user is created. diff --git a/test/compat/japi-compliance/README.md b/test/compat/japi-compliance/README.md index dfd4314..ed2b841 100644 --- a/test/compat/japi-compliance/README.md +++ b/test/compat/japi-compliance/README.md @@ -26,7 +26,7 @@ following command. japi-compliance-checker.pl -skip-deprecated -old japi-accumulo-1.5.xml -new japi-accumulo-1.6.xml -l accumulo ``` -Optionally, you can use the --skip-classes argument with the provided exclude_classes.txt file to skip classes from +Optionally, you can use the --skip-classes argument with the provided `exclude_classes.txt` file to skip classes from org.apache.accumulo.core.data that aren't in the public API. This directory should have a library configuration file for each release on supported lines as well as an in-progress diff --git a/test/scalability/README.md b/test/scalability/README.md index a677b0b..6110031 100644 --- a/test/scalability/README.md +++ b/test/scalability/README.md @@ -26,7 +26,7 @@ your Accumulo instance. Do not run these tests on a cluster holding essential data. 1. Make sure you have both `ACCUMULO_HOME` and `HADOOP_HOME` set in your - `$ACCUMULO_CONF_DIR/accumulo-env.sh.` + `$ACCUMULO_CONF_DIR/accumulo-env.sh` 2. Edit the 'site.conf' file in the `conf` directory containing settings needed by test nodes to connect to Accumulo, and to guide the tests. diff --git a/test/stress/README.md b/test/stress/README.md index 19fb08a..f16bd79 100644 --- a/test/stress/README.md +++ b/test/stress/README.md @@ -48,10 +48,11 @@ across hosts. ## Prerequisites The utilities are run on a normal, initialized Accumulo cluster. By default, -they only work with a table named "stress_test". +they only work with a table named `stress_test`. -The start and stop scripts rely on pssh. Before running them, you will need -to install pssh on the machines that will be controlled by them. +The start and stop scripts rely on pssh/parallel-ssh. Before running them, +you will need to install pssh/parallel-ssh on the machines that will be +controlled by them. ## Running diff --git a/test/stress/start-readers.sh b/test/stress/start-readers.sh index 6f651f4..4558e52 100755 --- a/test/stress/start-readers.sh +++ b/test/stress/start-readers.sh @@ -35,6 +35,6 @@ if [[ ! -f ${DIR}/readers ]]; then fi # Copy environment out -pscp -h "${DIR}/readers" "${DIR}/stress-env.sh" "${DIR}" +$PSCP -h "${DIR}/readers" "${DIR}/stress-env.sh" "${DIR}" -pssh -h "${DIR}/readers" "nohup ${DIR}/reader.sh >${DIR}/reader.out 2>${DIR}/reader.err < /dev/null &" +$PSSH -h "${DIR}/readers" "nohup ${DIR}/reader.sh >${DIR}/reader.out 2>${DIR}/reader.err < /dev/null &" diff --git a/test/stress/start-writers.sh b/test/stress/start-writers.sh index 474117c..718a38a 100755 --- a/test/stress/start-writers.sh +++ b/test/stress/start-writers.sh @@ -35,6 +35,6 @@ if [[ ! -f ${DIR}/writers ]]; then fi # Copy environment out -pscp -h "${DIR}/writers" "${DIR}/stress-env.sh" "${DIR}" +$PSCP -h "${DIR}/writers" "${DIR}/stress-env.sh" "${DIR}" -pssh -h "${DIR}/writers" "nohup ${DIR}/writer.sh >${DIR}/writer.out 2>${DIR}/writer.err < /dev/null &" +$PSSH -h "${DIR}/writers" "nohup ${DIR}/writer.sh >${DIR}/writer.out 2>${DIR}/writer.err < /dev/null &" diff --git a/test/stress/stop-readers.sh b/test/stress/stop-readers.sh index 8bd4a6f..8d37fa7 100755 --- a/test/stress/stop-readers.sh +++ b/test/stress/stop-readers.sh @@ -33,4 +33,4 @@ if [[ ! -f "${DIR}/readers" ]]; then echo readers file is missing exit 1 fi -pssh -h "${DIR}/readers" "pkill -f '[o]rg.apache.accumulo.test.stress.random.Scan'" < /dev/null +$PSSH -h "${DIR}/readers" "pkill -f '[o]rg.apache.accumulo.test.stress.random.Scan'" < /dev/null diff --git a/test/stress/stop-writers.sh b/test/stress/stop-writers.sh index 5c9e8d7..248574f 100755 --- a/test/stress/stop-writers.sh +++ b/test/stress/stop-writers.sh @@ -33,4 +33,4 @@ if [[ ! -f "${DIR}/writers" ]]; then echo writers file is missing exit 1 fi -pssh -h "${DIR}/writers" "pkill -f '[o]rg.apache.accumulo.test.stress.random.Write'" < /dev/null +$PSSH -h "${DIR}/writers" "pkill -f '[o]rg.apache.accumulo.test.stress.random.Write'" < /dev/null diff --git a/test/stress/stress-env.sh b/test/stress/stress-env.sh index 6efb24f..c142cc7 100644 --- a/test/stress/stress-env.sh +++ b/test/stress/stress-env.sh @@ -15,6 +15,18 @@ ACCUMULO_HOME=${ACCUMULO_HOME:-/opt/accumulo} +## check that pssh and pscp is installed, falling back to parallel-ssh/scp if needed +if hash pssh 2>/dev/null; then + PSSH=pssh + PSCP=pscp + hash pscp.pssh 2>/dev/null && PSCP=pscp.pssh +elif hash parallel-ssh 2>/dev/null; then + PSSH=parallel-ssh + PSCP=parallel-scp +else + echo >&2 "The stress test requires pssh/parallel-ssh and pscp/parallel-scp to be installed. Aborting."; exit 1; +fi + # Edit the credentials to match your system USERPASS='-u root -p secret' INSTANCE='-z localhost -i inst'