This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new e74a1f5 Replace use of `pssh` with `parallel-ssh` (#179)
e74a1f5 is described below
commit e74a1f5904bc165d4054ea1a0fd93d8eacc82606
Author: Dom G <47725857+domgargu...@users.noreply.github.com>
AuthorDate: Thu Dec 16 14:39:28 2021 -0500
Replace use of `pssh` with `parallel-ssh` (#179)
* Replaced pssh and pscp
* Fall back to parallel-ssh when pssh unavailable
* Fix quoting in sudo commands in agitator
* Preserve PSSH environment over sudo commands for agitator
* Properly set PSSH in perl agitator code from process environment
* Fix a couple typos in READMEs
Co-authored-by: Christopher Tubbs <ctubb...@apache.org>
---
bin/agitator | 15 +++++++++++----
libexec/master-agitator.pl | 14 ++++++++++----
test/bench/README.md | 2 +-
test/compat/japi-compliance/README.md | 2 +-
test/scalability/README.md | 2 +-
test/stress/README.md | 7 ++++---
test/stress/start-readers.sh | 4 ++--
test/stress/start-writers.sh | 4 ++--
test/stress/stop-readers.sh | 2 +-
test/stress/stop-writers.sh | 2 +-
test/stress/stress-env.sh | 12 ++++++++++++
11 files changed, 46 insertions(+), 20 deletions(-)
diff --git a/bin/agitator b/bin/agitator
index 8d15f94..cf1dd55 100755
--- a/bin/agitator
+++ b/bin/agitator
@@ -31,7 +31,14 @@ EOF
}
function start_agitator() {
- hash pssh 2>/dev/null || { echo >&2 "The agitator requires pssh to be
installed. Aborting."; exit 1; }
+ ## check that pssh is installed, falling back to parallel-ssh if needed
+ if hash pssh 2>/dev/null; then
+ PSSH=pssh
+ elif hash parallel-ssh 2>/dev/null; then
+ PSSH=parallel-ssh
+ else
+ echo >&2 "The agitator requires pssh/parallel-ssh to be installed.
Aborting."; exit 1;
+ fi
mkdir -p "${at_home}/logs"
log_base="${at_home}/logs/$(date +%Y%m%d%H%M%S)_$(hostname)"
@@ -50,15 +57,15 @@ function start_agitator() {
$tserver_cmd > "${tserver_log}.out" 2> "${tserver_log}.err" &
else
echo "Running master-agitator and tserver-agitator as $AGTR_ACCUMULO_USER
using sudo."
- sudo -u "$AGTR_ACCUMULO_USER" "$master_cmd" > "${master_log}.out" 2>
"${master_log}.err" &
- sudo -u "$AGTR_ACCUMULO_USER" "$tserver_cmd" > "${tserver_log}.out" 2>
"${tserver_log}.err" &
+ sudo --preserve-env=PSSH -u "$AGTR_ACCUMULO_USER" $master_cmd >
"${master_log}.out" 2> "${master_log}.err" &
+ sudo -u "$AGTR_ACCUMULO_USER" $tserver_cmd > "${tserver_log}.out" 2>
"${tserver_log}.err" &
fi
if [[ $AGITATOR_USER == "$AGTR_HDFS_USER" ]]; then
echo "Running datanode-agitator as $AGITATOR_USER"
$datanode_cmd > "${datanode_log}.out" 2> "${datanode_log}.err" &
else
echo "Running datanode-agitator as $AGTR_HDFS_USER using sudo."
- sudo -u "$AGTR_HDFS_USER" "$datanode_cmd" > "${datanode_log}.out" 2>
"${datanode_log}.err" &
+ sudo -u "$AGTR_HDFS_USER" $datanode_cmd > "${datanode_log}.out" 2>
"${datanode_log}.err" &
fi
if ${AGTR_HDFS:-false} ; then
diff --git a/libexec/master-agitator.pl b/libexec/master-agitator.pl
index e90c70d..dfc2155 100755
--- a/libexec/master-agitator.pl
+++ b/libexec/master-agitator.pl
@@ -30,6 +30,12 @@ if( defined $ENV{'ACCUMULO_HOME'} ){
print "ERROR: ACCUMULO_HOME needs to be set!";
exit(1);
}
+if( defined $ENV{'PSSH'} ){
+ $PSSH = $ENV{'PSSH'};
+} else {
+ print "ERROR: PSSH needs to be set!";
+ exit(1);
+}
$accumuloConfDir = $accumuloHome . '/conf';
@@ -67,11 +73,11 @@ while(1){
system($cmd);
}else{
print STDERR "$t Killing all masters\n";
- $cmd = "pssh -h $accumuloConfDir/masters \"pkill -f '[
]org.apache.accumulo.start.*master'\" < /dev/null";
+ $cmd = "$PSSH -h $accumuloConfDir/masters \"pkill -f '[
]org.apache.accumulo.start.*master'\" < /dev/null";
print "$t $cmd\n";
system($cmd);
- $cmd = "pssh -h $accumuloConfDir/$gcfile \"pkill -f '[
]org.apache.accumulo.start.*gc'\" < /dev/null";
+ $cmd = "$PSSH -h $accumuloConfDir/$gcfile \"pkill -f '[
]org.apache.accumulo.start.*gc'\" < /dev/null";
print "$t $cmd\n";
system($cmd);
}
@@ -80,11 +86,11 @@ while(1){
$t = strftime "%Y%m%d %H:%M:%S", localtime;
print STDERR "$t Running start-all\n";
- $cmd = "pssh -h $accumuloConfDir/masters
\"$accumuloHome/bin/accumulo-service master start\" < /dev/null";
+ $cmd = "$PSSH -h $accumuloConfDir/masters
\"$accumuloHome/bin/accumulo-service master start\" < /dev/null";
print "$t $cmd\n";
system($cmd);
- $cmd = "pssh -h $accumuloConfDir/$gcfile
\"$accumuloHome/bin/accumulo-service gc start\" < /dev/null";
+ $cmd = "$PSSH -h $accumuloConfDir/$gcfile
\"$accumuloHome/bin/accumulo-service gc start\" < /dev/null";
print "$t $cmd\n";
system($cmd);
}
diff --git a/test/bench/README.md b/test/bench/README.md
index 0929bc3..2e35fa9 100644
--- a/test/bench/README.md
+++ b/test/bench/README.md
@@ -55,7 +55,7 @@ The 4th Benchmark is Terasort. Run the benchmarks with speed
'slow' to do a ful
Misc
----
-These benchmarks create tables in accumulo named 'test_ingest' and
'CloudIngestTest'. These tables are deleted
+These benchmarks create tables in accumulo named `test_ingest` and
'CloudIngestTest'. These tables are deleted
at the end of the benchmarks. The benchmarks will also alter user auths while
it runs. It is recommended that
a benchmark user is created.
diff --git a/test/compat/japi-compliance/README.md
b/test/compat/japi-compliance/README.md
index dfd4314..ed2b841 100644
--- a/test/compat/japi-compliance/README.md
+++ b/test/compat/japi-compliance/README.md
@@ -26,7 +26,7 @@ following command.
japi-compliance-checker.pl -skip-deprecated -old japi-accumulo-1.5.xml -new
japi-accumulo-1.6.xml -l accumulo
```
-Optionally, you can use the --skip-classes argument with the provided
exclude_classes.txt file to skip classes from
+Optionally, you can use the --skip-classes argument with the provided
`exclude_classes.txt` file to skip classes from
org.apache.accumulo.core.data that aren't in the public API.
This directory should have a library configuration file for each release on
supported lines as well as an in-progress
diff --git a/test/scalability/README.md b/test/scalability/README.md
index a677b0b..6110031 100644
--- a/test/scalability/README.md
+++ b/test/scalability/README.md
@@ -26,7 +26,7 @@ your Accumulo instance. Do not run these tests on a cluster
holding essential
data.
1. Make sure you have both `ACCUMULO_HOME` and `HADOOP_HOME` set in your
- `$ACCUMULO_CONF_DIR/accumulo-env.sh.`
+ `$ACCUMULO_CONF_DIR/accumulo-env.sh`
2. Edit the 'site.conf' file in the `conf` directory containing settings
needed by test nodes to connect to Accumulo, and to guide the tests.
diff --git a/test/stress/README.md b/test/stress/README.md
index 19fb08a..f16bd79 100644
--- a/test/stress/README.md
+++ b/test/stress/README.md
@@ -48,10 +48,11 @@ across hosts.
## Prerequisites
The utilities are run on a normal, initialized Accumulo cluster. By default,
-they only work with a table named "stress_test".
+they only work with a table named `stress_test`.
-The start and stop scripts rely on pssh. Before running them, you will need
-to install pssh on the machines that will be controlled by them.
+The start and stop scripts rely on pssh/parallel-ssh. Before running them,
+you will need to install pssh/parallel-ssh on the machines that will be
+controlled by them.
## Running
diff --git a/test/stress/start-readers.sh b/test/stress/start-readers.sh
index 6f651f4..4558e52 100755
--- a/test/stress/start-readers.sh
+++ b/test/stress/start-readers.sh
@@ -35,6 +35,6 @@ if [[ ! -f ${DIR}/readers ]]; then
fi
# Copy environment out
-pscp -h "${DIR}/readers" "${DIR}/stress-env.sh" "${DIR}"
+$PSCP -h "${DIR}/readers" "${DIR}/stress-env.sh" "${DIR}"
-pssh -h "${DIR}/readers" "nohup ${DIR}/reader.sh >${DIR}/reader.out
2>${DIR}/reader.err < /dev/null &"
+$PSSH -h "${DIR}/readers" "nohup ${DIR}/reader.sh >${DIR}/reader.out
2>${DIR}/reader.err < /dev/null &"
diff --git a/test/stress/start-writers.sh b/test/stress/start-writers.sh
index 474117c..718a38a 100755
--- a/test/stress/start-writers.sh
+++ b/test/stress/start-writers.sh
@@ -35,6 +35,6 @@ if [[ ! -f ${DIR}/writers ]]; then
fi
# Copy environment out
-pscp -h "${DIR}/writers" "${DIR}/stress-env.sh" "${DIR}"
+$PSCP -h "${DIR}/writers" "${DIR}/stress-env.sh" "${DIR}"
-pssh -h "${DIR}/writers" "nohup ${DIR}/writer.sh >${DIR}/writer.out
2>${DIR}/writer.err < /dev/null &"
+$PSSH -h "${DIR}/writers" "nohup ${DIR}/writer.sh >${DIR}/writer.out
2>${DIR}/writer.err < /dev/null &"
diff --git a/test/stress/stop-readers.sh b/test/stress/stop-readers.sh
index 8bd4a6f..8d37fa7 100755
--- a/test/stress/stop-readers.sh
+++ b/test/stress/stop-readers.sh
@@ -33,4 +33,4 @@ if [[ ! -f "${DIR}/readers" ]]; then
echo readers file is missing
exit 1
fi
-pssh -h "${DIR}/readers" "pkill -f
'[o]rg.apache.accumulo.test.stress.random.Scan'" < /dev/null
+$PSSH -h "${DIR}/readers" "pkill -f
'[o]rg.apache.accumulo.test.stress.random.Scan'" < /dev/null
diff --git a/test/stress/stop-writers.sh b/test/stress/stop-writers.sh
index 5c9e8d7..248574f 100755
--- a/test/stress/stop-writers.sh
+++ b/test/stress/stop-writers.sh
@@ -33,4 +33,4 @@ if [[ ! -f "${DIR}/writers" ]]; then
echo writers file is missing
exit 1
fi
-pssh -h "${DIR}/writers" "pkill -f
'[o]rg.apache.accumulo.test.stress.random.Write'" < /dev/null
+$PSSH -h "${DIR}/writers" "pkill -f
'[o]rg.apache.accumulo.test.stress.random.Write'" < /dev/null
diff --git a/test/stress/stress-env.sh b/test/stress/stress-env.sh
index 6efb24f..c142cc7 100644
--- a/test/stress/stress-env.sh
+++ b/test/stress/stress-env.sh
@@ -15,6 +15,18 @@
ACCUMULO_HOME=${ACCUMULO_HOME:-/opt/accumulo}
+## check that pssh and pscp is installed, falling back to parallel-ssh/scp if
needed
+if hash pssh 2>/dev/null; then
+ PSSH=pssh
+ PSCP=pscp
+ hash pscp.pssh 2>/dev/null && PSCP=pscp.pssh
+elif hash parallel-ssh 2>/dev/null; then
+ PSSH=parallel-ssh
+ PSCP=parallel-scp
+else
+ echo >&2 "The stress test requires pssh/parallel-ssh and pscp/parallel-scp
to be installed. Aborting."; exit 1;
+fi
+
# Edit the credentials to match your system
USERPASS='-u root -p secret'
INSTANCE='-z localhost -i inst'
