Author: arkurth
Date: Mon May 18 18:47:49 2009
New Revision: 776052
URL: http://svn.apache.org/viewvc?rev=776052&view=rev
Log:
VCL-118
Added delay after a failed attempt to utils.pm:run_ssh_command() and
run_scp_command(). Also added ability to pass a hash reference to
run_ssh_command(). The list of arguments it accepts is somewhat lengthy and
this will make it easier to keep the arguments straight when calling it. It
will also make it easier to add additional arguments later on.
Added 'max_attempts' argument to run_ssh_command allowing caller to specify how
many times the command will be attempted if it fails. The default is 3. This
argument allows caller to control the number of attempts. For example, only 1
attempt may be desired if the caller only wants to know if SSH is responding.
Changed call in Windows_mod.pm::wait_for_ssh() to call run_ssh_command using a
hash reference argument. It specifies max_attempts to be 1.
Modified:
incubator/vcl/trunk/managementnode/lib/VCL/Module/OS/Windows_mod.pm
incubator/vcl/trunk/managementnode/lib/VCL/utils.pm
Modified: incubator/vcl/trunk/managementnode/lib/VCL/Module/OS/Windows_mod.pm
URL:
http://svn.apache.org/viewvc/incubator/vcl/trunk/managementnode/lib/VCL/Module/OS/Windows_mod.pm?rev=776052&r1=776051&r2=776052&view=diff
==============================================================================
--- incubator/vcl/trunk/managementnode/lib/VCL/Module/OS/Windows_mod.pm
(original)
+++ incubator/vcl/trunk/managementnode/lib/VCL/Module/OS/Windows_mod.pm Mon May
18 18:47:49 2009
@@ -2524,8 +2524,13 @@
}
# Run a test SSH command
- my ($exit_status, $output) =
run_ssh_command($computer_node_name, $management_node_keys, "echo testing ssh
on $computer_node_name", '', '', 1);
-
+ my ($exit_status, $output) = run_ssh_command({
+ node => $computer_node_name,
+ command => "echo testing ssh on $computer_node_name",
+ max_attempts => 1,
+ output_level => 0,
+ });
+
# The exit status will be 0 if the command succeeded
if (defined($exit_status) && $exit_status == 0) {
notify($ERRORS{'OK'}, 0, "$computer_node_name is
responding to ssh");
Modified: incubator/vcl/trunk/managementnode/lib/VCL/utils.pm
URL:
http://svn.apache.org/viewvc/incubator/vcl/trunk/managementnode/lib/VCL/utils.pm?rev=776052&r1=776051&r2=776052&view=diff
==============================================================================
--- incubator/vcl/trunk/managementnode/lib/VCL/utils.pm (original)
+++ incubator/vcl/trunk/managementnode/lib/VCL/utils.pm Mon May 18 18:47:49 2009
@@ -6651,23 +6651,55 @@
=head2 run_ssh_command
Parameters : $node, $identity_path, $command, $user, $port
- Returns : If ssh command completed successfully, an array containing
- the exit status and a reference to an array containing the
- lines of output of the command
specified is returned.
- $array[0] = the exit status of the command
- $array[1] = reference to array
containing lines of output
- generated by the command
- If the ssh command fails, an empty
array is returned. You can
- therefore use "if (run_ssh_command())",
true means the command
- executed, false means it failed.
+ -or-
+ Hash reference with the following keys:
+ node - node name (required)
+ command - command to be executed
remotely (required)
+ identity_paths - string containing
paths to identity key files separated by commas (optional)
+ user - user to run remote command as
(optional, default is 'root')
+ port - SSH port number (optional,
default is 22)
+ output_level - allows the amount of
output to be controlled: 0, 1, or 2 (optional)
+ max_attempts - maximum number of SSH
attempts to make
+ Returns : If successful: array:
+ $array[0] = the exit status of the command
+ $array[1] = reference to array
containing lines of output
+ If failed: false
Description : Runs an SSH command on the specified node.
=cut
sub run_ssh_command {
- my ($node, $identity_paths, $command, $user, $port, $no_output) = @_;
- my ($package, $filename, $line, $sub) = caller(0);
-
+ my ($node, $identity_paths, $command, $user, $port, $output_level) = @_;
+
+ my $max_attempts = 3;
+
+ if (ref($_[0]) eq 'HASH') {
+ my $arguments = shift;
+
+ $node = $arguments->{node};
+ $command = $arguments->{command};
+ $identity_paths = $arguments->{identity_paths} || '';
+ $user = $arguments->{user} || 'root';
+ $port = $arguments->{port} || '22';
+ $output_level = $arguments->{output_level};
+ $max_attempts = $arguments->{max_attempts} || 3;
+ }
+
+ # Determine the output level if it was specified
+ # Set $output_level to 0, 1, or 2
+ if (!defined($output_level)) {
+ $output_level = 2;
+ }
+ elsif ($output_level =~ /0|none/i) {
+ $output_level = 0;
+ }
+ elsif ($output_level =~ /1|min/i) {
+ $output_level = 1;
+ }
+ else {
+ $output_level = 2;
+ }
+
# Check the arguments
if (!defined($node) || !$node) {
notify($ERRORS{'WARNING'}, 0, "computer node was not
specified");
@@ -6679,8 +6711,8 @@
}
# Set default values if not passed as an argument
- $user = "root" if (!defined($user) || !$user);
- $port = 22 if (!defined($port) || !$port);
+ $user = "root" if (!$user);
+ $port = 22 if (!$port);
# TODO: Add ssh path to config file and set global variable
# Locate the path to the ssh binary
@@ -6701,12 +6733,21 @@
# Format the identity path string
if ($identity_paths) {
- $identity_paths =~ s/^\s*/-i /;
+ # Add -i to beginning of string
+ $identity_paths = "-i $identity_paths";
+
+ # Split string on commas, add -i to each value after a comma
$identity_paths =~ s/\s*,\s*/ -i /g;
+
+ # Add a space to the end of the string
$identity_paths .= ' ';
}
+ else {
+ $identity_paths = '';
+ }
- #notify($ERRORS{'DEBUG'}, 0, "node: $node, identity file path:
$identity_path, user: $user, port: $port");
+ #notify($ERRORS{'DEBUG'}, 0, "ssh path: $ssh_path");
+ #notify($ERRORS{'DEBUG'}, 0, "node: $node, identity file paths:
$identity_paths, user: $user, port: $port");
#notify($ERRORS{'DEBUG'}, 0, "command: $command");
# Assemble the SSH command
@@ -6724,22 +6765,35 @@
my $ssh_output;
my $ssh_output_formatted;
my $attempts = 0;
- my $max_attempts = 3;
my $exit_status = 255;
# Make multiple attempts if failure occurs
while ($attempts < $max_attempts) {
$attempts++;
+
+ # Delay performing next attempt if this isn't the first attempt
+ if ($attempts > 1) {
+ my $delay;
+ if ($attempts == 2) {
+ $delay = 2;
+ }
+ else {
+ # Progressively increase the delay
+ $delay = (5 * $attempts);
+ }
+ notify($ERRORS{'DEBUG'}, 0, "sleeping for $delay
seconds before making next SSH attempt") if $output_level;
+ sleep $delay;
+ }
## Add -v (verbose) argument to command if this is the 2nd
attempt
#$ssh_command =~ s/$ssh_path/$ssh_path -v/ if $attempts == 2;
# Print the SSH command, only display the attempt # if > 1
if ($attempts == 1) {
- notify($ERRORS{'DEBUG'}, 0, "executing SSH command on
$node:\n$ssh_command");
+ notify($ERRORS{'DEBUG'}, 0, "executing SSH command on
$node:\n$ssh_command") if $output_level;
}
else {
- notify($ERRORS{'DEBUG'}, 0, "attempt
$attempts/$max_attempts: executing SSH command on $node:\n$ssh_command");
+ notify($ERRORS{'DEBUG'}, 0, "attempt
$attempts/$max_attempts: executing SSH command on $node:\n$ssh_command") if
$output_level;
}
# Execute the command
@@ -6758,7 +6812,7 @@
# This likely means a Perl bug was encountered
# Assume command was successful
if ($? == -1) {
- notify($ERRORS{'DEBUG'}, 0, "exit status changed from
$exit_status to 0, Perl bug likely encountered");
+ notify($ERRORS{'DEBUG'}, 0, "exit status changed from
$exit_status to 0, Perl bug likely encountered") if $output_level;
$exit_status = 0;
}
@@ -6772,7 +6826,7 @@
$ssh_output =~ s/Warning: Permanently added .+ to the list of
known hosts\.//igs;
# Remove any spaces from the beginning and end of the output
- $ssh_output =~ s/^\s+|\s+$//g;
+ $ssh_output =~ s/(^\s+)|(\s+$)//g;
# Set the output string to none if no output was produced
$ssh_output = 'none' if !$ssh_output;
@@ -6799,14 +6853,7 @@
# Check the exit status
# ssh exits with the exit status of the remote command or with
255 if an error occurred.
if ($exit_status == 255 || $ssh_output_formatted =~ /lost
connection|reset by peer|no route to host|connection refused|connection timed
out/i) {
- # Temporary fix for problem of nodes using different
ports
- if ($attempts == 3) {
- $max_attempts++;
- notify($ERRORS{'OK'}, 0, "making 1 more attempt
using port 24");
- $ssh_command = "$ssh_path $identity_paths -l
$user -p 24 -x $node '$command 2>&1' 2>&1";
- }
-
- notify($ERRORS{'WARNING'}, 0, "attempt
$attempts/$max_attempts: failed to execute SSH command on $node: $command, exit
status: $exit_status, SSH exits with the exit status of the remote command or
with 255 if an error occurred, output:\n$ssh_output_formatted");
+ notify($ERRORS{'WARNING'}, 0, "attempt
$attempts/$max_attempts: failed to execute SSH command on $node: $command, exit
status: $exit_status, SSH exits with the exit status of the remote command or
with 255 if an error occurred, output:\n$ssh_output_formatted") if
$output_level;
next;
}
else {
@@ -6816,7 +6863,7 @@
my @output_lines = split(/\n/, $ssh_output);
# Print the output unless no_output is set
- notify($ERRORS{'DEBUG'}, 0, "run_ssh_command
output:\n$ssh_output") unless $no_output;
+ notify($ERRORS{'DEBUG'}, 0, "run_ssh_command output:\n"
. join("\n", @output_lines)) if $output_level > 1;
# Print the command and exit status
(my $ssh_output_summary = $ssh_output) =~ s/\s+/ /gs;
@@ -6826,11 +6873,11 @@
}
# Display the full ssh command if the exit status is
not 0
- if ($exit_status && !$no_output) {
- notify($ERRORS{'OK'}, 0, "SSH command executed
on $node, command:\n$ssh_command\nreturning ($exit_status,
\"$ssh_output_summary\")");
+ if ($exit_status) {
+ notify($ERRORS{'OK'}, 0, "SSH command executed
on $node, command:\n$ssh_command\nreturning ($exit_status,
\"$ssh_output_summary\")") if $output_level > 1;
}
- elsif (!$no_output) {
- notify($ERRORS{'DEBUG'}, 0, "SSH command
executed on $node, returning ($exit_status, \"$ssh_output_summary\")");
+ else {
+ notify($ERRORS{'DEBUG'}, 0, "SSH command
executed on $node, returning ($exit_status, \"$ssh_output_summary\")") if
$output_level > 1;
}
# Return the exit status and output
@@ -6839,7 +6886,7 @@
} ## end while ($attempts < $max_attempts)
# Failure, SSH command did not run at all
- notify($ERRORS{'WARNING'}, 0, "failed to run SSH command after
$attempts attempts, command: $ssh_command, exit status: $exit_status,
output:\n$ssh_output_formatted");
+ notify($ERRORS{'WARNING'}, 0, "failed to run SSH command after
$attempts attempts, command: $ssh_command, exit status: $exit_status,
output:\n$ssh_output_formatted") if $output_level;
return ();
} ## end sub run_ssh_command
@@ -6939,21 +6986,35 @@
# Make multiple attempts if failure occurs
while ($attempts < $max_attempts) {
$attempts++;
-
+
+ # Delay performing next attempt if this isn't the first attempt
+ if ($attempts > 1) {
+ my $delay;
+ if ($attempts == 2) {
+ $delay = 2;
+ }
+ else {
+ # Progressively increase the delay
+ $delay = (5 * $attempts);
+ }
+ notify($ERRORS{'DEBUG'}, 0, "sleeping for $delay
seconds before making next SCP attempt");
+ sleep $delay;
+ }
+
## Add -v (verbose) argument to command if this is the 2nd
attempt
#$scp_command =~ s/$scp_path/$scp_path -v/ if $attempts == 2;
-
+
notify($ERRORS{'DEBUG'}, 0, "attempt $attempts/$max_attempts:
executing SCP command: $scp_command");
-
+
$scp_output = `$scp_command`;
-
+
# Save the exit status
$scp_exit_status = $?;
-
+
# Strip out the key warning message
$scp_output =~ s/\...@{10,}.*man-in-the-middle attacks\.//igs;
$scp_output =~ s/^\s+|\s+$//g;
-
+
if ($scp_output && length($scp_output) > 0) {
# Add a newline to the beginning of the output if
something was generated
# This is to make multi-line output more readable
@@ -6963,7 +7024,7 @@
# Indicate there was no output if it is blank
$scp_output = 'none';
}
-
+
# Get a slice of the SCP output if there are many lines
my @scp_output_formatted_lines = split("\n", $scp_output);
my $scp_output_formatted_line_count = scalar
@scp_output_formatted_lines;
@@ -6972,7 +7033,7 @@
push(@scp_output_formatted_lines, "displayed first 50
of $scp_output_formatted_line_count SCP output lines");
$scp_output = join("\n", @scp_output_formatted_lines);
}
-
+
# Check the output for known error messages
# Check the exit status
# scp exits with 0 on success or >0 if an error occurred
@@ -6993,7 +7054,7 @@
return 1;
}
} ## end while ($attempts < $max_attempts)
-
+
# Failure
notify($ERRORS{'WARNING'}, 0, "failed to copy files using scp after
$attempts attempts, command: $scp_command, exit status: $scp_exit_status,
output: $scp_output");
return 0;
