Author: arkurth Date: Mon May 18 18:47:49 2009 New Revision: 776052 URL: http://svn.apache.org/viewvc?rev=776052&view=rev Log: VCL-118 Added delay after a failed attempt to utils.pm:run_ssh_command() and run_scp_command(). Also added ability to pass a hash reference to run_ssh_command(). The list of arguments it accepts is somewhat lengthy and this will make it easier to keep the arguments straight when calling it. It will also make it easier to add additional arguments later on.
Added 'max_attempts' argument to run_ssh_command allowing caller to specify how many times the command will be attempted if it fails. The default is 3. This argument allows caller to control the number of attempts. For example, only 1 attempt may be desired if the caller only wants to know if SSH is responding. Changed call in Windows_mod.pm::wait_for_ssh() to call run_ssh_command using a hash reference argument. It specifies max_attempts to be 1. Modified: incubator/vcl/trunk/managementnode/lib/VCL/Module/OS/Windows_mod.pm incubator/vcl/trunk/managementnode/lib/VCL/utils.pm Modified: incubator/vcl/trunk/managementnode/lib/VCL/Module/OS/Windows_mod.pm URL: http://svn.apache.org/viewvc/incubator/vcl/trunk/managementnode/lib/VCL/Module/OS/Windows_mod.pm?rev=776052&r1=776051&r2=776052&view=diff ============================================================================== --- incubator/vcl/trunk/managementnode/lib/VCL/Module/OS/Windows_mod.pm (original) +++ incubator/vcl/trunk/managementnode/lib/VCL/Module/OS/Windows_mod.pm Mon May 18 18:47:49 2009 @@ -2524,8 +2524,13 @@ } # Run a test SSH command - my ($exit_status, $output) = run_ssh_command($computer_node_name, $management_node_keys, "echo testing ssh on $computer_node_name", '', '', 1); - + my ($exit_status, $output) = run_ssh_command({ + node => $computer_node_name, + command => "echo testing ssh on $computer_node_name", + max_attempts => 1, + output_level => 0, + }); + # The exit status will be 0 if the command succeeded if (defined($exit_status) && $exit_status == 0) { notify($ERRORS{'OK'}, 0, "$computer_node_name is responding to ssh"); Modified: incubator/vcl/trunk/managementnode/lib/VCL/utils.pm URL: http://svn.apache.org/viewvc/incubator/vcl/trunk/managementnode/lib/VCL/utils.pm?rev=776052&r1=776051&r2=776052&view=diff ============================================================================== --- incubator/vcl/trunk/managementnode/lib/VCL/utils.pm (original) +++ incubator/vcl/trunk/managementnode/lib/VCL/utils.pm Mon May 18 18:47:49 2009 @@ -6651,23 +6651,55 @@ =head2 run_ssh_command Parameters : $node, $identity_path, $command, $user, $port - Returns : If ssh command completed successfully, an array containing - the exit status and a reference to an array containing the - lines of output of the command specified is returned. - $array[0] = the exit status of the command - $array[1] = reference to array containing lines of output - generated by the command - If the ssh command fails, an empty array is returned. You can - therefore use "if (run_ssh_command())", true means the command - executed, false means it failed. + -or- + Hash reference with the following keys: + node - node name (required) + command - command to be executed remotely (required) + identity_paths - string containing paths to identity key files separated by commas (optional) + user - user to run remote command as (optional, default is 'root') + port - SSH port number (optional, default is 22) + output_level - allows the amount of output to be controlled: 0, 1, or 2 (optional) + max_attempts - maximum number of SSH attempts to make + Returns : If successful: array: + $array[0] = the exit status of the command + $array[1] = reference to array containing lines of output + If failed: false Description : Runs an SSH command on the specified node. =cut sub run_ssh_command { - my ($node, $identity_paths, $command, $user, $port, $no_output) = @_; - my ($package, $filename, $line, $sub) = caller(0); - + my ($node, $identity_paths, $command, $user, $port, $output_level) = @_; + + my $max_attempts = 3; + + if (ref($_[0]) eq 'HASH') { + my $arguments = shift; + + $node = $arguments->{node}; + $command = $arguments->{command}; + $identity_paths = $arguments->{identity_paths} || ''; + $user = $arguments->{user} || 'root'; + $port = $arguments->{port} || '22'; + $output_level = $arguments->{output_level}; + $max_attempts = $arguments->{max_attempts} || 3; + } + + # Determine the output level if it was specified + # Set $output_level to 0, 1, or 2 + if (!defined($output_level)) { + $output_level = 2; + } + elsif ($output_level =~ /0|none/i) { + $output_level = 0; + } + elsif ($output_level =~ /1|min/i) { + $output_level = 1; + } + else { + $output_level = 2; + } + # Check the arguments if (!defined($node) || !$node) { notify($ERRORS{'WARNING'}, 0, "computer node was not specified"); @@ -6679,8 +6711,8 @@ } # Set default values if not passed as an argument - $user = "root" if (!defined($user) || !$user); - $port = 22 if (!defined($port) || !$port); + $user = "root" if (!$user); + $port = 22 if (!$port); # TODO: Add ssh path to config file and set global variable # Locate the path to the ssh binary @@ -6701,12 +6733,21 @@ # Format the identity path string if ($identity_paths) { - $identity_paths =~ s/^\s*/-i /; + # Add -i to beginning of string + $identity_paths = "-i $identity_paths"; + + # Split string on commas, add -i to each value after a comma $identity_paths =~ s/\s*,\s*/ -i /g; + + # Add a space to the end of the string $identity_paths .= ' '; } + else { + $identity_paths = ''; + } - #notify($ERRORS{'DEBUG'}, 0, "node: $node, identity file path: $identity_path, user: $user, port: $port"); + #notify($ERRORS{'DEBUG'}, 0, "ssh path: $ssh_path"); + #notify($ERRORS{'DEBUG'}, 0, "node: $node, identity file paths: $identity_paths, user: $user, port: $port"); #notify($ERRORS{'DEBUG'}, 0, "command: $command"); # Assemble the SSH command @@ -6724,22 +6765,35 @@ my $ssh_output; my $ssh_output_formatted; my $attempts = 0; - my $max_attempts = 3; my $exit_status = 255; # Make multiple attempts if failure occurs while ($attempts < $max_attempts) { $attempts++; + + # Delay performing next attempt if this isn't the first attempt + if ($attempts > 1) { + my $delay; + if ($attempts == 2) { + $delay = 2; + } + else { + # Progressively increase the delay + $delay = (5 * $attempts); + } + notify($ERRORS{'DEBUG'}, 0, "sleeping for $delay seconds before making next SSH attempt") if $output_level; + sleep $delay; + } ## Add -v (verbose) argument to command if this is the 2nd attempt #$ssh_command =~ s/$ssh_path/$ssh_path -v/ if $attempts == 2; # Print the SSH command, only display the attempt # if > 1 if ($attempts == 1) { - notify($ERRORS{'DEBUG'}, 0, "executing SSH command on $node:\n$ssh_command"); + notify($ERRORS{'DEBUG'}, 0, "executing SSH command on $node:\n$ssh_command") if $output_level; } else { - notify($ERRORS{'DEBUG'}, 0, "attempt $attempts/$max_attempts: executing SSH command on $node:\n$ssh_command"); + notify($ERRORS{'DEBUG'}, 0, "attempt $attempts/$max_attempts: executing SSH command on $node:\n$ssh_command") if $output_level; } # Execute the command @@ -6758,7 +6812,7 @@ # This likely means a Perl bug was encountered # Assume command was successful if ($? == -1) { - notify($ERRORS{'DEBUG'}, 0, "exit status changed from $exit_status to 0, Perl bug likely encountered"); + notify($ERRORS{'DEBUG'}, 0, "exit status changed from $exit_status to 0, Perl bug likely encountered") if $output_level; $exit_status = 0; } @@ -6772,7 +6826,7 @@ $ssh_output =~ s/Warning: Permanently added .+ to the list of known hosts\.//igs; # Remove any spaces from the beginning and end of the output - $ssh_output =~ s/^\s+|\s+$//g; + $ssh_output =~ s/(^\s+)|(\s+$)//g; # Set the output string to none if no output was produced $ssh_output = 'none' if !$ssh_output; @@ -6799,14 +6853,7 @@ # Check the exit status # ssh exits with the exit status of the remote command or with 255 if an error occurred. if ($exit_status == 255 || $ssh_output_formatted =~ /lost connection|reset by peer|no route to host|connection refused|connection timed out/i) { - # Temporary fix for problem of nodes using different ports - if ($attempts == 3) { - $max_attempts++; - notify($ERRORS{'OK'}, 0, "making 1 more attempt using port 24"); - $ssh_command = "$ssh_path $identity_paths -l $user -p 24 -x $node '$command 2>&1' 2>&1"; - } - - notify($ERRORS{'WARNING'}, 0, "attempt $attempts/$max_attempts: failed to execute SSH command on $node: $command, exit status: $exit_status, SSH exits with the exit status of the remote command or with 255 if an error occurred, output:\n$ssh_output_formatted"); + notify($ERRORS{'WARNING'}, 0, "attempt $attempts/$max_attempts: failed to execute SSH command on $node: $command, exit status: $exit_status, SSH exits with the exit status of the remote command or with 255 if an error occurred, output:\n$ssh_output_formatted") if $output_level; next; } else { @@ -6816,7 +6863,7 @@ my @output_lines = split(/\n/, $ssh_output); # Print the output unless no_output is set - notify($ERRORS{'DEBUG'}, 0, "run_ssh_command output:\n$ssh_output") unless $no_output; + notify($ERRORS{'DEBUG'}, 0, "run_ssh_command output:\n" . join("\n", @output_lines)) if $output_level > 1; # Print the command and exit status (my $ssh_output_summary = $ssh_output) =~ s/\s+/ /gs; @@ -6826,11 +6873,11 @@ } # Display the full ssh command if the exit status is not 0 - if ($exit_status && !$no_output) { - notify($ERRORS{'OK'}, 0, "SSH command executed on $node, command:\n$ssh_command\nreturning ($exit_status, \"$ssh_output_summary\")"); + if ($exit_status) { + notify($ERRORS{'OK'}, 0, "SSH command executed on $node, command:\n$ssh_command\nreturning ($exit_status, \"$ssh_output_summary\")") if $output_level > 1; } - elsif (!$no_output) { - notify($ERRORS{'DEBUG'}, 0, "SSH command executed on $node, returning ($exit_status, \"$ssh_output_summary\")"); + else { + notify($ERRORS{'DEBUG'}, 0, "SSH command executed on $node, returning ($exit_status, \"$ssh_output_summary\")") if $output_level > 1; } # Return the exit status and output @@ -6839,7 +6886,7 @@ } ## end while ($attempts < $max_attempts) # Failure, SSH command did not run at all - notify($ERRORS{'WARNING'}, 0, "failed to run SSH command after $attempts attempts, command: $ssh_command, exit status: $exit_status, output:\n$ssh_output_formatted"); + notify($ERRORS{'WARNING'}, 0, "failed to run SSH command after $attempts attempts, command: $ssh_command, exit status: $exit_status, output:\n$ssh_output_formatted") if $output_level; return (); } ## end sub run_ssh_command @@ -6939,21 +6986,35 @@ # Make multiple attempts if failure occurs while ($attempts < $max_attempts) { $attempts++; - + + # Delay performing next attempt if this isn't the first attempt + if ($attempts > 1) { + my $delay; + if ($attempts == 2) { + $delay = 2; + } + else { + # Progressively increase the delay + $delay = (5 * $attempts); + } + notify($ERRORS{'DEBUG'}, 0, "sleeping for $delay seconds before making next SCP attempt"); + sleep $delay; + } + ## Add -v (verbose) argument to command if this is the 2nd attempt #$scp_command =~ s/$scp_path/$scp_path -v/ if $attempts == 2; - + notify($ERRORS{'DEBUG'}, 0, "attempt $attempts/$max_attempts: executing SCP command: $scp_command"); - + $scp_output = `$scp_command`; - + # Save the exit status $scp_exit_status = $?; - + # Strip out the key warning message $scp_output =~ s/\...@{10,}.*man-in-the-middle attacks\.//igs; $scp_output =~ s/^\s+|\s+$//g; - + if ($scp_output && length($scp_output) > 0) { # Add a newline to the beginning of the output if something was generated # This is to make multi-line output more readable @@ -6963,7 +7024,7 @@ # Indicate there was no output if it is blank $scp_output = 'none'; } - + # Get a slice of the SCP output if there are many lines my @scp_output_formatted_lines = split("\n", $scp_output); my $scp_output_formatted_line_count = scalar @scp_output_formatted_lines; @@ -6972,7 +7033,7 @@ push(@scp_output_formatted_lines, "displayed first 50 of $scp_output_formatted_line_count SCP output lines"); $scp_output = join("\n", @scp_output_formatted_lines); } - + # Check the output for known error messages # Check the exit status # scp exits with 0 on success or >0 if an error occurred @@ -6993,7 +7054,7 @@ return 1; } } ## end while ($attempts < $max_attempts) - + # Failure notify($ERRORS{'WARNING'}, 0, "failed to copy files using scp after $attempts attempts, command: $scp_command, exit status: $scp_exit_status, output: $scp_output"); return 0;