Author: arkurth
Date: Mon May 18 18:47:49 2009
New Revision: 776052

URL: http://svn.apache.org/viewvc?rev=776052&view=rev
Log:
VCL-118
Added delay after a failed attempt to utils.pm:run_ssh_command() and 
run_scp_command(). Also added ability to pass a hash reference to 
run_ssh_command(). The list of arguments it accepts is somewhat lengthy and 
this will make it easier to keep the arguments straight when calling it. It 
will also make it easier to add additional arguments later on.

Added 'max_attempts' argument to run_ssh_command allowing caller to specify how 
many times the command will be attempted if it fails. The default is 3. This 
argument allows caller to control the number of attempts. For example, only 1 
attempt may be desired if the caller only wants to know if SSH is responding.

Changed call in Windows_mod.pm::wait_for_ssh() to call run_ssh_command using a 
hash reference argument. It specifies max_attempts to be 1.

Modified:
    incubator/vcl/trunk/managementnode/lib/VCL/Module/OS/Windows_mod.pm
    incubator/vcl/trunk/managementnode/lib/VCL/utils.pm

Modified: incubator/vcl/trunk/managementnode/lib/VCL/Module/OS/Windows_mod.pm
URL: 
http://svn.apache.org/viewvc/incubator/vcl/trunk/managementnode/lib/VCL/Module/OS/Windows_mod.pm?rev=776052&r1=776051&r2=776052&view=diff
==============================================================================
--- incubator/vcl/trunk/managementnode/lib/VCL/Module/OS/Windows_mod.pm 
(original)
+++ incubator/vcl/trunk/managementnode/lib/VCL/Module/OS/Windows_mod.pm Mon May 
18 18:47:49 2009
@@ -2524,8 +2524,13 @@
                }
 
                # Run a test SSH command
-               my ($exit_status, $output) = 
run_ssh_command($computer_node_name, $management_node_keys, "echo testing ssh 
on $computer_node_name", '', '', 1);
-
+               my ($exit_status, $output) = run_ssh_command({
+                       node => $computer_node_name,
+                       command => "echo testing ssh on $computer_node_name",
+                       max_attempts => 1,
+                       output_level => 0,
+               });
+               
                # The exit status will be 0 if the command succeeded
                if (defined($exit_status) && $exit_status == 0) {
                        notify($ERRORS{'OK'}, 0, "$computer_node_name is 
responding to ssh");

Modified: incubator/vcl/trunk/managementnode/lib/VCL/utils.pm
URL: 
http://svn.apache.org/viewvc/incubator/vcl/trunk/managementnode/lib/VCL/utils.pm?rev=776052&r1=776051&r2=776052&view=diff
==============================================================================
--- incubator/vcl/trunk/managementnode/lib/VCL/utils.pm (original)
+++ incubator/vcl/trunk/managementnode/lib/VCL/utils.pm Mon May 18 18:47:49 2009
@@ -6651,23 +6651,55 @@
 =head2 run_ssh_command
 
  Parameters  : $node, $identity_path, $command, $user, $port
- Returns     : If ssh command completed successfully, an array containing
-               the exit status and a reference to an array containing the
-                                       lines of output of the command 
specified is returned.
-               $array[0] = the exit status of the command
-                                       $array[1] = reference to array 
containing lines of output
-                                       generated by the command
-                                       If the ssh command fails, an empty 
array is returned. You can
-                                       therefore use "if (run_ssh_command())", 
true means the command
-                                       executed, false means it failed.
+                                       -or-
+                                       Hash reference with the following keys:
+                                       node - node name (required)
+                                       command - command to be executed 
remotely (required)
+                                       identity_paths - string containing 
paths to identity key files separated by commas (optional)
+                                       user - user to run remote command as 
(optional, default is 'root')
+                                       port - SSH port number (optional, 
default is 22)
+                                       output_level - allows the amount of 
output to be controlled: 0, 1, or 2 (optional)
+                                       max_attempts - maximum number of SSH 
attempts to make
+ Returns     : If successful: array:
+                  $array[0] = the exit status of the command
+                                          $array[1] = reference to array 
containing lines of output
+                                       If failed: false
  Description : Runs an SSH command on the specified node.
 
 =cut
 
 sub run_ssh_command {
-       my ($node, $identity_paths, $command, $user, $port, $no_output) = @_;
-       my ($package, $filename, $line, $sub) = caller(0);
-
+       my ($node, $identity_paths, $command, $user, $port, $output_level) = @_;
+       
+       my $max_attempts = 3;
+       
+       if (ref($_[0]) eq 'HASH') {
+               my $arguments = shift;
+               
+               $node = $arguments->{node};
+               $command = $arguments->{command};
+               $identity_paths = $arguments->{identity_paths} || '';
+               $user = $arguments->{user} || 'root';
+               $port = $arguments->{port} || '22';
+               $output_level = $arguments->{output_level};
+               $max_attempts = $arguments->{max_attempts} || 3;
+       }
+       
+       # Determine the output level if it was specified
+       # Set $output_level to 0, 1, or 2
+       if (!defined($output_level)) {
+               $output_level = 2;
+       }
+       elsif ($output_level =~ /0|none/i) {
+               $output_level = 0;
+       }
+       elsif ($output_level =~ /1|min/i) {
+               $output_level = 1;
+       }
+       else {
+               $output_level = 2;
+       }
+       
        # Check the arguments
        if (!defined($node) || !$node) {
                notify($ERRORS{'WARNING'}, 0, "computer node was not 
specified");
@@ -6679,8 +6711,8 @@
        }
 
        # Set default values if not passed as an argument
-       $user = "root" if (!defined($user) || !$user);
-       $port = 22     if (!defined($port) || !$port);
+       $user = "root" if (!$user);
+       $port = 22     if (!$port);
 
        # TODO: Add ssh path to config file and set global variable
        # Locate the path to the ssh binary
@@ -6701,12 +6733,21 @@
 
        # Format the identity path string
        if ($identity_paths) {
-               $identity_paths =~ s/^\s*/-i /;
+               # Add -i to beginning of string
+               $identity_paths = "-i $identity_paths";
+               
+               # Split string on commas, add -i to each value after a comma
                $identity_paths =~ s/\s*,\s*/ -i /g;
+               
+               # Add a space to the end of the string
                $identity_paths .= ' ';
        }
+       else {
+               $identity_paths = '';
+       }
 
-       #notify($ERRORS{'DEBUG'}, 0, "node: $node, identity file path: 
$identity_path, user: $user, port: $port");
+       #notify($ERRORS{'DEBUG'}, 0, "ssh path: $ssh_path");
+       #notify($ERRORS{'DEBUG'}, 0, "node: $node, identity file paths: 
$identity_paths, user: $user, port: $port");
        #notify($ERRORS{'DEBUG'}, 0, "command: $command");
 
        # Assemble the SSH command
@@ -6724,22 +6765,35 @@
        my $ssh_output;
        my $ssh_output_formatted;
        my $attempts        = 0;
-       my $max_attempts    = 3;
        my $exit_status = 255;
 
        # Make multiple attempts if failure occurs
        while ($attempts < $max_attempts) {
                $attempts++;
+               
+               # Delay performing next attempt if this isn't the first attempt
+               if ($attempts > 1) {
+                       my $delay;
+                       if ($attempts == 2) {
+                               $delay = 2;
+                       }
+                       else {
+                               # Progressively increase the delay
+                               $delay = (5 * $attempts);
+                       }
+                       notify($ERRORS{'DEBUG'}, 0, "sleeping for $delay 
seconds before making next SSH attempt") if $output_level;
+                       sleep $delay;
+               }
 
                ## Add -v (verbose) argument to command if this is the 2nd 
attempt
                #$ssh_command =~ s/$ssh_path/$ssh_path -v/ if $attempts == 2;
 
                # Print the SSH command, only display the attempt # if > 1
                if ($attempts == 1) {
-                       notify($ERRORS{'DEBUG'}, 0, "executing SSH command on 
$node:\n$ssh_command");
+                       notify($ERRORS{'DEBUG'}, 0, "executing SSH command on 
$node:\n$ssh_command") if $output_level;
                }
                else {
-                       notify($ERRORS{'DEBUG'}, 0, "attempt 
$attempts/$max_attempts: executing SSH command on $node:\n$ssh_command");
+                       notify($ERRORS{'DEBUG'}, 0, "attempt 
$attempts/$max_attempts: executing SSH command on $node:\n$ssh_command") if 
$output_level;
                }
 
                # Execute the command
@@ -6758,7 +6812,7 @@
                # This likely means a Perl bug was encountered
                # Assume command was successful
                if ($? == -1) {
-                       notify($ERRORS{'DEBUG'}, 0, "exit status changed from 
$exit_status to 0, Perl bug likely encountered");
+                       notify($ERRORS{'DEBUG'}, 0, "exit status changed from 
$exit_status to 0, Perl bug likely encountered") if $output_level;
                        $exit_status = 0;
                }
                
@@ -6772,7 +6826,7 @@
                $ssh_output =~ s/Warning: Permanently added .+ to the list of 
known hosts\.//igs;
                
                # Remove any spaces from the beginning and end of the output
-               $ssh_output =~ s/^\s+|\s+$//g;
+               $ssh_output =~ s/(^\s+)|(\s+$)//g;
 
                # Set the output string to none if no output was produced
                $ssh_output = 'none' if !$ssh_output;
@@ -6799,14 +6853,7 @@
                # Check the exit status
                # ssh exits with the exit status of the remote command or with 
255 if an error occurred.
                if ($exit_status == 255 || $ssh_output_formatted =~ /lost 
connection|reset by peer|no route to host|connection refused|connection timed 
out/i) {
-                       # Temporary fix for problem of nodes using different 
ports
-                       if ($attempts == 3) {
-                               $max_attempts++;
-                               notify($ERRORS{'OK'}, 0, "making 1 more attempt 
using port 24");
-                               $ssh_command = "$ssh_path $identity_paths -l 
$user -p 24 -x $node '$command 2>&1' 2>&1";
-                       }
-                       
-                       notify($ERRORS{'WARNING'}, 0, "attempt 
$attempts/$max_attempts: failed to execute SSH command on $node: $command, exit 
status: $exit_status, SSH exits with the exit status of the remote command or 
with 255 if an error occurred, output:\n$ssh_output_formatted");
+                       notify($ERRORS{'WARNING'}, 0, "attempt 
$attempts/$max_attempts: failed to execute SSH command on $node: $command, exit 
status: $exit_status, SSH exits with the exit status of the remote command or 
with 255 if an error occurred, output:\n$ssh_output_formatted") if 
$output_level;
                        next;
                }
                else {
@@ -6816,7 +6863,7 @@
                        my @output_lines = split(/\n/, $ssh_output);
                        
                        # Print the output unless no_output is set
-                       notify($ERRORS{'DEBUG'}, 0, "run_ssh_command 
output:\n$ssh_output") unless $no_output;
+                       notify($ERRORS{'DEBUG'}, 0, "run_ssh_command output:\n" 
. join("\n", @output_lines)) if $output_level > 1;
                        
                        # Print the command and exit status
                        (my $ssh_output_summary = $ssh_output) =~ s/\s+/ /gs;
@@ -6826,11 +6873,11 @@
                        }
                        
                        # Display the full ssh command if the exit status is 
not 0
-                       if ($exit_status && !$no_output) {
-                               notify($ERRORS{'OK'}, 0, "SSH command executed 
on $node, command:\n$ssh_command\nreturning ($exit_status, 
\"$ssh_output_summary\")");
+                       if ($exit_status) {
+                               notify($ERRORS{'OK'}, 0, "SSH command executed 
on $node, command:\n$ssh_command\nreturning ($exit_status, 
\"$ssh_output_summary\")") if $output_level > 1;
                        }
-                       elsif (!$no_output) {
-                               notify($ERRORS{'DEBUG'}, 0, "SSH command 
executed on $node, returning ($exit_status, \"$ssh_output_summary\")");
+                       else {
+                               notify($ERRORS{'DEBUG'}, 0, "SSH command 
executed on $node, returning ($exit_status, \"$ssh_output_summary\")") if 
$output_level > 1;
                        }
                        
                        # Return the exit status and output
@@ -6839,7 +6886,7 @@
        } ## end while ($attempts < $max_attempts)
 
        # Failure, SSH command did not run at all
-       notify($ERRORS{'WARNING'}, 0, "failed to run SSH command after 
$attempts attempts, command: $ssh_command, exit status: $exit_status, 
output:\n$ssh_output_formatted");
+       notify($ERRORS{'WARNING'}, 0, "failed to run SSH command after 
$attempts attempts, command: $ssh_command, exit status: $exit_status, 
output:\n$ssh_output_formatted") if $output_level;
        return ();
 
 } ## end sub run_ssh_command
@@ -6939,21 +6986,35 @@
        # Make multiple attempts if failure occurs
        while ($attempts < $max_attempts) {
                $attempts++;
-
+               
+               # Delay performing next attempt if this isn't the first attempt
+               if ($attempts > 1) {
+                       my $delay;
+                       if ($attempts == 2) {
+                               $delay = 2;
+                       }
+                       else {
+                               # Progressively increase the delay
+                               $delay = (5 * $attempts);
+                       }
+                       notify($ERRORS{'DEBUG'}, 0, "sleeping for $delay 
seconds before making next SCP attempt");
+                       sleep $delay;
+               }
+               
                ## Add -v (verbose) argument to command if this is the 2nd 
attempt
                #$scp_command =~ s/$scp_path/$scp_path -v/ if $attempts == 2;
-
+               
                notify($ERRORS{'DEBUG'}, 0, "attempt $attempts/$max_attempts: 
executing SCP command: $scp_command");
-
+               
                $scp_output = `$scp_command`;
-
+               
                # Save the exit status
                $scp_exit_status = $?;
-
+               
                # Strip out the key warning message
                $scp_output =~ s/\...@{10,}.*man-in-the-middle attacks\.//igs;
                $scp_output =~ s/^\s+|\s+$//g;
-
+               
                if ($scp_output && length($scp_output) > 0) {
                        # Add a newline to the beginning of the output if 
something was generated
                        # This is to make multi-line output more readable
@@ -6963,7 +7024,7 @@
                        # Indicate there was no output if it is blank
                        $scp_output = 'none';
                }
-
+               
                # Get a slice of the SCP output if there are many lines
                my @scp_output_formatted_lines = split("\n", $scp_output);
                my $scp_output_formatted_line_count = scalar 
@scp_output_formatted_lines;
@@ -6972,7 +7033,7 @@
                        push(@scp_output_formatted_lines, "displayed first 50 
of $scp_output_formatted_line_count SCP output lines");
                        $scp_output = join("\n", @scp_output_formatted_lines);
                }
-
+               
                # Check the output for known error messages
                # Check the exit status
                # scp exits with 0 on success or >0 if an error occurred
@@ -6993,7 +7054,7 @@
                        return 1;
                }
        } ## end while ($attempts < $max_attempts)
-
+       
        # Failure
        notify($ERRORS{'WARNING'}, 0, "failed to copy files using scp after 
$attempts attempts, command: $scp_command, exit status: $scp_exit_status, 
output: $scp_output");
        return 0;


Reply via email to