[accumulo-testing] branch main updated: Fix broken agitator scripts (#184)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new 267dc84 Fix broken agitator scripts (#184)
267dc84 is described below
commit 267dc84971d6fa37af6183dc5a0f3706f9762911
Author: Mike Miller
AuthorDate: Wed Jan 12 15:28:01 2022 -0500
Fix broken agitator scripts (#184)
* Create bash function to start agitators
* Drop broken perl scripts in favor of bash functions
* Add function to call ClusterConfigParser to read cluster.yaml
* Make agitators only log to one file
* Rename env variables from master to manager
Co-authored-by: Christopher Tubbs
Co-authored-by: Dom G.
---
bin/agitator | 175 ++-
conf/env.sh | 8 +-
libexec/datanode-agitator.pl | 136 -
libexec/master-agitator.pl | 98
libexec/tserver-agitator.pl | 130
5 files changed, 143 insertions(+), 404 deletions(-)
diff --git a/bin/agitator b/bin/agitator
index 91e456c..3f532a7 100755
--- a/bin/agitator
+++ b/bin/agitator
@@ -30,6 +30,85 @@ Possible commands:
EOF
}
+# Starts a app specific agitator
+# usage: start_app_agitator app_name kill_sleep_time restart_sleep_time
min_kill max_kill start_cmd kill_cmd
+# Requires that a list of hosts to be set in $APP_HOSTS
+function start_app_agitator() {
+ local app_name=$1
+ local kill_sleep_time=$2
+ local restart_sleep_time=$3
+ local min_kill=$4
+ local max_kill=$5
+ local start_cmd=$6
+ local kill_cmd=$7
+ local hosts_array; readarray -td' ' hosts_array < <(get_app_hosts
"$app_name")
+ local num_hosts=${#hosts_array[@]}
+ local node_to_kill
+ nodes_to_kill_array=()
+ local T
+ local ENV_VARS="ACCUMULO_HOME=$ACCUMULO_HOME ZOOKEEPER_HOME=$ZOOKEEPER_HOME
HADOOP_HOME=$HADOOP_HOME JAVA_HOME=$JAVA_HOME"
+
+ if (( num_hosts == 0 )); then
+echo "ERROR: No hosts were found in env for $app_name"
+exit 1
+ fi
+ if (( max_kill > num_hosts )); then
+echo "ERROR: Max kill $max_kill is greater than number of hosts $num_hosts"
+exit 1
+ fi
+ if (( max_kill < min_kill )); then
+echo "ERROR: Max kill $max_kill is less than min kill $min_kill"
+exit 1
+ fi
+
+ T="$(date +'%Y%m%d %H:%M:%S')"
+ echo "$T Starting $app_name agitation. Kill every $kill_sleep_time minutes,
restart every $restart_sleep_time minutes."
+ echo "$T Will randomly kill between $min_kill and $max_kill of the
following: ${hosts_array[*]}"
+ while true; do
+echo "$T Sleeping for $kill_sleep_time minutes"
+sleep $((kill_sleep_time * 60))
+
+T="$(date +'%Y%m%d %H:%M:%S')"
+if ((max_kill == 1)) ; then
+ node_to_kill=${hosts_array[0]}
+ echo "$T Killing $app_name at $node_to_kill"
+ ssh "$node_to_kill" "$kill_cmd"
+else
+ local num_to_kill=$((min_kill + RANDOM % max_kill))
+ # get the random nodes to kill
+ local count=0
+ while [[ $count -lt $num_to_kill ]]; do
+randomHostIndex=$((1 + RANDOM % num_hosts))
+node_to_kill=${hosts_array[randomHostIndex]}
+# only add host to the array if its not already there
+if [[ ! " ${nodes_to_kill_array[*]} " =~ $node_to_kill ]]; then
+ nodes_to_kill_array[count]=$node_to_kill
+fi
+count=${#nodes_to_kill_array[@]}
+ done
+ echo "$T Killing $count $app_name nodes"
+ for i in "${nodes_to_kill_array[@]}"; do
+ssh "$i" "$kill_cmd"
+ done
+fi
+
+T="$(date +'%Y%m%d %H:%M:%S')"
+echo "$T Sleeping for $restart_sleep_time minutes."
+sleep $((restart_sleep_time * 60))
+
+T="$(date +'%Y%m%d %H:%M:%S')"
+if ((max_kill == 1)) ; then
+ echo "$T Restarting $app_name at $node_to_kill"
+ ssh "$node_to_kill" "bash -c '${ENV_VARS} $start_cmd'"
+else
+ for i in "${nodes_to_kill_array[@]}"; do
+echo "$T Restarting $app_name node at ${i}"
+ssh "$i" "bash -c '${ENV_VARS} $start_cmd'"
+ done
+fi
+ done
+}
+
function start_agitator() {
## check that pssh is installed, falling back to parallel-ssh if needed
## make sure to export it, so it can be seen inside the agitator perl script
@@ -40,58 +119,82 @@ function start_agitator() {
else
echo >&2 "The agitator requires pssh/parallel-ssh to be installed.
Aborting."; exit 1;
fi
+ ## read configuration into env variables
+ read_cluster_conf
mkdir -p "${at_home}/logs"
log_base="${at_home}/logs/$(date +%Y%m%d%H%M%S)_$(hostname)"
- libexec="${at_home}/libexec"
- master_log="${log_base}_master-agitator"
- tserver_log="${log_base}_tserver-agitator"
- datanode_log="${log_base}_datanode-agitator"
- master_cmd="nohup ${libexec}/master-agitator.pl $AGTR_MASTER_KILL_SLEEP_TIME
$AGTR_MASTER_RESTART_SLEEP_TIME"
-
[accumulo] branch main updated: Update protobuf to 3.19.2 (#2412)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 30c49ef Update protobuf to 3.19.2 (#2412)
30c49ef is described below
commit 30c49ef1a193bd6712efe4c28e304e107b47a1c3
Author: Mike Miller
AuthorDate: Wed Jan 12 07:48:34 2022 -0500
Update protobuf to 3.19.2 (#2412)
---
.../accumulo/core/protobuf/ProtobufUtil.java | 6 +-
pom.xml| 2 +-
.../server/replication/proto/Replication.java | 682 ++---
server/base/src/main/protobuf/replication.proto| 1 +
server/base/src/main/scripts/generate-protobuf.sh | 2 +-
5 files changed, 452 insertions(+), 241 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/protobuf/ProtobufUtil.java
b/core/src/main/java/org/apache/accumulo/core/protobuf/ProtobufUtil.java
index 000916b..57baf5a 100644
--- a/core/src/main/java/org/apache/accumulo/core/protobuf/ProtobufUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/protobuf/ProtobufUtil.java
@@ -20,7 +20,7 @@ package org.apache.accumulo.core.protobuf;
import org.apache.accumulo.core.data.Value;
-import com.google.protobuf.GeneratedMessage;
+import com.google.protobuf.GeneratedMessageV3;
import com.google.protobuf.TextFormat;
/**
@@ -29,11 +29,11 @@ import com.google.protobuf.TextFormat;
public class ProtobufUtil {
private static final char LEFT_BRACKET = '[', RIGHT_BRACKET = ']';
- public static Value toValue(GeneratedMessage msg) {
+ public static Value toValue(GeneratedMessageV3 msg) {
return new Value(msg.toByteArray());
}
- public static String toString(GeneratedMessage msg) {
+ public static String toString(GeneratedMessageV3 msg) {
// Too much typing
return LEFT_BRACKET + TextFormat.shortDebugString(msg) + RIGHT_BRACKET;
}
diff --git a/pom.xml b/pom.xml
index 2aaa691..12112c1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -286,7 +286,7 @@
com.google.protobuf
protobuf-java
-3.7.1
+3.19.2
com.lmax
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/replication/proto/Replication.java
b/server/base/src/main/java/org/apache/accumulo/server/replication/proto/Replication.java
index 4a20d87..043ff72 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/replication/proto/Replication.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/replication/proto/Replication.java
@@ -21,102 +21,113 @@
package org.apache.accumulo.server.replication.proto;
-@SuppressWarnings({"unused"})
-public final class Replication {
+@SuppressWarnings({"unused"}) public final class Replication {
private Replication() {}
public static void registerAllExtensions(
+ com.google.protobuf.ExtensionRegistryLite registry) {
+ }
+
+ public static void registerAllExtensions(
com.google.protobuf.ExtensionRegistry registry) {
+registerAllExtensions(
+(com.google.protobuf.ExtensionRegistryLite) registry);
}
- public interface StatusOrBuilder
- extends com.google.protobuf.MessageOrBuilder {
+ public interface StatusOrBuilder extends
+ // @@protoc_insertion_point(interface_extends:Status)
+ com.google.protobuf.MessageOrBuilder {
-// optional int64 begin = 1 [default = 0];
/**
- * optional int64 begin = 1 [default = 0];
- *
*
* offset where replication should start
*
+ *
+ * optional int64 begin = 1 [default = 0];
+ * @return Whether the begin field is set.
*/
boolean hasBegin();
/**
- * optional int64 begin = 1 [default = 0];
- *
*
* offset where replication should start
*
+ *
+ * optional int64 begin = 1 [default = 0];
+ * @return The begin.
*/
long getBegin();
-// optional int64 end = 2 [default = 0];
/**
- * optional int64 end = 2 [default = 0];
- *
*
* offset where data is ready for replication
*
+ *
+ * optional int64 end = 2 [default = 0];
+ * @return Whether the end field is set.
*/
boolean hasEnd();
/**
- * optional int64 end = 2 [default = 0];
- *
*
* offset where data is ready for replication
*
+ *
+ * optional int64 end = 2 [default = 0];
+ * @return The end.
*/
long getEnd();
-// optional bool infiniteEnd = 3 [default = false];
/**
- * optional bool infiniteEnd = 3 [default = false];
- *
*
* do we have a discrete 'end'
*
+ *
+ * optional bool infiniteEnd = 3 [default = false];
+ * @return Whether the infiniteEnd field is set.
*/
boolean hasInfiniteEnd();
/**
- * optional bool infiniteEnd = 3 [default = false];
- *
*
* do we have a discret
