Author: arkurth
Date: Fri Oct 17 17:41:20 2014
New Revision: 1632640
URL: http://svn.apache.org/r1632640
Log:
VCL-685
Updated VIM_SSH.pm::_run_vim_cmd to detect if another process is attempting to
run 'services.sh restart' to prevent multiple failed attempts. It does this by
checking if the semaphore exists after a failed attempt. Added
Module.pm::does_semaphore_exist. This calls Semaphore.pm::semaphore_exists.
Added utils.pm::set_reservation_lastcheck. This is called to set
reservation.lastcheck to 20 minutes prior to the end of the reservation when
VIM_SSH.pm detects uncorrectable problems. Updated
utils.pm:update_reservation_lastcheck to not change the value if the current
value is in the future.
Other
Added $no_cache=1 argument to calls to get_computer_info and get_vmhost_info in
DataStructure.pm. This should always get fresh data.
Removed requirement to be called as an object method from
Module.pm::create_mn_os_object. The '$self' variable wasn't being used anywhere.
Updated Module.pm::new to not create a new management node OS object if the
'mn_os' argument was specified. It had been needlessly creating multiple
objects.
Modified:
vcl/trunk/managementnode/lib/VCL/DataStructure.pm
vcl/trunk/managementnode/lib/VCL/Module.pm
vcl/trunk/managementnode/lib/VCL/Module/Provisioning/VMware/VIM_SSH.pm
vcl/trunk/managementnode/lib/VCL/Module/Semaphore.pm
vcl/trunk/managementnode/lib/VCL/utils.pm
Modified: vcl/trunk/managementnode/lib/VCL/DataStructure.pm
URL:
http://svn.apache.org/viewvc/vcl/trunk/managementnode/lib/VCL/DataStructure.pm?rev=1632640&r1=1632639&r2=1632640&view=diff
==============================================================================
--- vcl/trunk/managementnode/lib/VCL/DataStructure.pm (original)
+++ vcl/trunk/managementnode/lib/VCL/DataStructure.pm Fri Oct 17 17:41:20 2014
@@ -657,7 +657,7 @@ sub _initialize : Init {
# Get the computer info if the computer_identifier argument was
specified and add it to this object
if ($computer_identifier) {
notify($ERRORS{'DEBUG'}, 0, "computer identifier argument was
specified, retrieving data for computer: $computer_identifier");
- my $computer_info = get_computer_info($computer_identifier);
+ my $computer_info = get_computer_info($computer_identifier, 1);
if (!$computer_info) {
notify($ERRORS{'WARNING'}, 0, "DataStructure object
could not be initialized, failed to retrieve data for computer:
$computer_identifier");
@@ -672,7 +672,7 @@ sub _initialize : Init {
# Get the VM host info if the vmhost_id argument was specified and add
it to this object
if ($vmhost_id) {
notify($ERRORS{'DEBUG'}, 0, "VM host identifier argument was
specified, retrieving data for VM host: $vmhost_id");
- my $vmhost_info = get_vmhost_info($vmhost_id);
+ my $vmhost_info = get_vmhost_info($vmhost_id, 1);
if (!$vmhost_info) {
notify($ERRORS{'WARNING'}, 0, "DataStructure object
could not be initialized, failed to retrieve data for VM host: $vmhost_id");
Modified: vcl/trunk/managementnode/lib/VCL/Module.pm
URL:
http://svn.apache.org/viewvc/vcl/trunk/managementnode/lib/VCL/Module.pm?rev=1632640&r1=1632639&r2=1632640&view=diff
==============================================================================
--- vcl/trunk/managementnode/lib/VCL/Module.pm (original)
+++ vcl/trunk/managementnode/lib/VCL/Module.pm Fri Oct 17 17:41:20 2014
@@ -198,7 +198,16 @@ sub new {
# Create a management node OS object
# Check to make sure the object currently being created is not a MN OS
object to avoid endless loop
if (!$self->isa('VCL::Module::OS::Linux::ManagementNode')) {
- if (my $mn_os = $self->create_mn_os_object()) {
+ my $mn_os;
+ # Check if the mn_os argument was provided
+ if ($args->{mn_os}) {
+ $mn_os = $args->{mn_os};
+ }
+ else {
+ $mn_os = $self->create_mn_os_object()
+ }
+
+ if ($mn_os) {
$self->set_mn_os($mn_os);
$self->data->set_mn_os($mn_os);
}
@@ -432,26 +441,6 @@ sub create_os_object {
=cut
sub create_mn_os_object {
- my $self = shift;
- unless (ref($self) && $self->isa('VCL::Module')) {
- notify($ERRORS{'CRITICAL'}, 0, "subroutine was called as a
function, it must be called as a class method");
- return;
- }
-
- # Check if an OS object has already been stored in the calling object
- if (my $mn_os = $self->mn_os(0)) {
- return $mn_os;
- }
-
- # Make sure calling object isn't an OS module to avoid an infinite loop
- if ($self->isa('VCL::Module::OS::Linux::ManagementNode')) {
- notify($ERRORS{'WARNING'}, 0, "this subroutine cannot be called
from an existing management node OS module: " . ref($self));
- return;
- }
-
- my $request_data = $self->data->get_request_data();
- my $reservation_id = $self->data->get_reservation_id();
-
# Create a DataStructure object containing computer data for the
management node
my $mn_data;
eval {
@@ -627,7 +616,7 @@ sub create_provisioning_object {
return 0;
}
notify($ERRORS{'DEBUG'}, 0, "$provisioning_perl_package module loaded");
-
+
# Attempt to provisioner the object, pass it the mn_os object if it has
already been created
my $constructor_arguments = {};
$constructor_arguments->{data_structure} = $self->data();
@@ -1276,7 +1265,7 @@ sub get_semaphore {
}
# Attempt to create a new semaphore object
- my $semaphore = VCL::Module::Semaphore->new({'data_structure' =>
$self->data});
+ my $semaphore = VCL::Module::Semaphore->new({'data_structure' =>
$self->data, mn_os => $self->mn_os});
if (!$semaphore) {
notify($ERRORS{'WARNING'}, 0, "failed to create semaphore
object");
return;
@@ -1297,6 +1286,40 @@ sub get_semaphore {
#/////////////////////////////////////////////////////////////////////////////
+=head2 does_semaphore_exist
+
+ Parameters : $semaphore_id
+ Returns : boolean
+ Description : Determines if an open Semaphore exists on this management node
+ matching the $semaphore_id.
+
+=cut
+
+sub does_semaphore_exist {
+ my $self = shift;
+ unless (ref($self) && $self->isa('VCL::Module')) {
+ notify($ERRORS{'CRITICAL'}, 0, "subroutine was called as a
function, it must be called as a class method");
+ return;
+ }
+
+ my ($semaphore_id) = @_;
+ if (!$semaphore_id) {
+ notify($ERRORS{'WARNING'}, 0, "semaphore ID argument was not
supplied");
+ return;
+ }
+
+ # Attempt to create a new semaphore object
+ my $semaphore = VCL::Module::Semaphore->new({'data_structure' =>
$self->data, mn_os => $self->mn_os});
+ if (!$semaphore) {
+ notify($ERRORS{'WARNING'}, 0, "failed to create semaphore
object");
+ return;
+ }
+
+ return $semaphore->semaphore_exists($semaphore_id);
+}
+
+#/////////////////////////////////////////////////////////////////////////////
+
=head2 setup_get_menu
Parameters : none
Modified: vcl/trunk/managementnode/lib/VCL/Module/Provisioning/VMware/VIM_SSH.pm
URL:
http://svn.apache.org/viewvc/vcl/trunk/managementnode/lib/VCL/Module/Provisioning/VMware/VIM_SSH.pm?rev=1632640&r1=1632639&r2=1632640&view=diff
==============================================================================
--- vcl/trunk/managementnode/lib/VCL/Module/Provisioning/VMware/VIM_SSH.pm
(original)
+++ vcl/trunk/managementnode/lib/VCL/Module/Provisioning/VMware/VIM_SSH.pm Fri
Oct 17 17:41:20 2014
@@ -90,7 +90,7 @@ sub initialize {
notify($ERRORS{'WARNING'}, 0, "required 'vmhost_os'
argument was not passed");
return;
}
-
+
#
if (ref $args->{vmhost_os} !~ /VCL::Module::OS/) {
notify($ERRORS{'CRITICAL'}, 0, "'vmhost_os' argument
passed is not a reference to a VCL::Module::OS object, type: " .
ref($args->{vmhost_os}));
@@ -177,12 +177,21 @@ sub _run_vim_cmd {
my $wait_seconds = 2;
my $connection_reset_errors = 0;
-
- while ($attempt++ < $attempt_limit) {
+ my $services_restart_detected = 0;
+ ATTEMPT: while ($attempt++ < $attempt_limit) {
if ($attempt > 1) {
# Wait before making next attempt
notify($ERRORS{'OK'}, 0, "sleeping $wait_seconds
seconds before making attempt $attempt/$attempt_limit");
sleep $wait_seconds;
+
+ my $semaphore_id =
"$vmhost_computer_name-vmware_services_restart";
+ if ($self->does_semaphore_exist($semaphore_id)) {
+ $services_restart_detected = 1;
+ notify($ERRORS{'DEBUG'}, 0, "detected another
process is restarting VMware services, sleeping for 10 seconds");
+ sleep_uninterrupted(10);
+ my $wait_message = "another process is
restarting VMware services on $vmhost_computer_name";
+
$self->code_loop_timeout(sub{!$self->does_semaphore_exist(@_)},
[$semaphore_id], $wait_message, 140, 5);
+ }
}
# The following error is somewhat common if several processes
are adding/removing VMs at the same time:
@@ -212,11 +221,36 @@ sub _run_vim_cmd {
$connection_reset_errors++;
notify($ERRORS{'OK'}, 0, "attempt
$attempt/$attempt_limit: connection reset while attempting to run command on VM
host $vmhost_computer_name: $command, output:\n" . join("\n", @$output));
- # If 3 connection reset errors occured, attempt to run
services.sh restart
- if ($connection_reset_errors == 3) {
- notify($ERRORS{'CRITICAL'}, 0, "calling
'services.sh restart', encountered $connection_reset_errors connection reset on
VM host $vmhost_computer_name, output:\n" . join("\n", @$output));
- $self->_services_restart();
+ # If 2 connection reset errors occured, attempt to run
services.sh restart
+ if ($connection_reset_errors == 2) {
+ if ($services_restart_detected) {
+ notify($ERRORS{'WARNING'}, 0,
"encountered $connection_reset_errors connection reset errors on VM host
$vmhost_computer_name, not calling 'services.sh restart', another process
already attempted it");
+ }
+ else {
+ notify($ERRORS{'OK'}, 0, "calling
'services.sh restart', encountered $connection_reset_errors connection reset
errors on VM host $vmhost_computer_name");
+ $self->_services_restart();
+ next ATTEMPT;
+ }
}
+ elsif ($connection_reset_errors > 2) {
+ notify($ERRORS{'WARNING'}, 0, "encountered
$connection_reset_errors connection reset errors on VM host
$vmhost_computer_name");
+ }
+ else {
+ next ATTEMPT;
+ }
+
+ # Problem probably won't correct itself
+ # If request state is 'inuse', set the
reservation.lastcheck value to 20 minutes before request.end
+ # This avoids 'inuse' processes from being created over
and over again which will fail
+ my $request_state_name =
$self->data->get_request_state_name();
+ #if ($request_state_name eq 'inuse') {
+ my $reservation_id =
$self->data->get_reservation_id();
+ my $request_end_time_epoch =
convert_to_epoch_seconds($self->data->get_request_end_time());
+ my $current_time_epoch = time;
+ my $reservation_lastcheck_epoch =
($request_end_time_epoch-(20*60));
+ set_reservation_lastcheck($reservation_id,
$reservation_lastcheck_epoch);
+ #}
+ return;
}
else {
# VIM command command was executed
@@ -279,9 +313,8 @@ sub _services_restart {
}
my $services_command = "services.sh restart";
-
notify($ERRORS{'DEBUG'}, 0, "restarting VMware services on
$vmhost_computer_name");
- my ($services_exit_status, $services_output) =
$self->vmhost_os->execute($services_command, 1, 120);
+ my ($services_exit_status, $services_output) =
$self->vmhost_os->execute($services_command, 0, 120);
if (!defined($services_output)) {
notify($ERRORS{'WARNING'}, 0, "failed to run command on VM host
$vmhost_computer_name: $services_command");
return;
@@ -289,7 +322,6 @@ sub _services_restart {
else {
notify($ERRORS{'OK'}, 0, "executed command to restart VMware
services on $vmhost_computer_name, command: '$services_command', output:\n" .
join("\n", @$services_output));
}
-
return 1;
}
@@ -337,61 +369,74 @@ sub _check_service_pid {
($running_pid) = "@$ps_output" =~ /(\d+)/g;
if (!$running_pid) {
notify($ERRORS{'DEBUG'}, 0, "parent $process_name PID
is not running");
- return;
}
elsif ($running_pid > 1) {
notify($ERRORS{'DEBUG'}, 0, "retrieved parent
$process_name PID: $running_pid");
}
else {
notify($ERRORS{'WARNING'}, 0, "parent $process_name PID
not valid: $running_pid, command: '$ps_command', output:\n" . join("\n",
@$ps_output));
- return;
+ $running_pid = '';
}
}
- # Retrieve the PID stored in the PID file
- my $file_pid;
- my @pid_file_contents =
$self->vmhost_os->get_file_contents($pid_file_path);
- if (@pid_file_contents) {
- ($file_pid) = "@pid_file_contents" =~ /(\d+)/g;
- if ($file_pid) {
- notify($ERRORS{'DEBUG'}, 0, "retrieved PID stored in
$pid_file_path: $file_pid");
+ # Check if the .pid file exists
+ my $pid_file_exists = $self->vmhost_os->file_exists($pid_file_path);
+ if (!$running_pid) {
+ if ($pid_file_exists) {
+ notify($ERRORS{'DEBUG'}, 0, "running $process_name
process was not detected but PID file exists: $pid_file_path, deleting file");
+ if ($self->vmhost_os->delete_file($pid_file_path)) {
+ notify($ERRORS{'DEBUG'}, 0, "deleted file on
$vmhost_computer_name: $pid_file_path");
+ }
+ else {
+ notify($ERRORS{'WARNING'}, 0, "failed to delete
file on $vmhost_computer_name: $pid_file_path");
+ }
+ return 1;
}
else {
- notify($ERRORS{'WARNING'}, 0, "unable to determine PID
stored in $pid_file_path, contents:\n" . join("\n", @pid_file_contents));
+ notify($ERRORS{'DEBUG'}, 0, "running $process_name
process was not detected and PID file does not exist: $pid_file_path");
+ return 1;
}
}
else {
- notify($ERRORS{'WARNING'}, 0, "failed to retrieve contents of
$pid_file_path");
- }
-
- # Check if it is necessary to update the PID file
- if ($file_pid) {
- if ($file_pid eq $running_pid) {
- notify($ERRORS{'OK'}, 0, "PID in $pid_file_path
($file_pid) matches PID of parent $process_name process ($running_pid), update
not necessary");
- return 1;
+ if ($pid_file_exists) {
+ # Retrieve the PID stored in the PID file
+ my @pid_file_contents =
$self->vmhost_os->get_file_contents($pid_file_path);
+ if (@pid_file_contents) {
+ my ($file_pid) = "@pid_file_contents" =~
/(\d+)/g;
+ if ($file_pid) {
+ notify($ERRORS{'DEBUG'}, 0, "retrieved
PID stored in $pid_file_path: $file_pid");
+ if ($file_pid eq $running_pid) {
+ notify($ERRORS{'OK'}, 0, "PID
in $pid_file_path ($file_pid) matches PID of parent $process_name process
($running_pid), update not necessary");
+ return 1;
+ }
+ else {
+ notify($ERRORS{'OK'}, 0, "PID
in $pid_file_path ($file_pid) does not match PID of parent $process_name
process ($running_pid), updating $pid_file_path to contain $running_pid");
+ }
+ }
+ else {
+ notify($ERRORS{'WARNING'}, 0, "unable
to determine PID stored in $pid_file_path, contents:\n" . join("\n",
@pid_file_contents));
+ }
+ }
+ else {
+ notify($ERRORS{'WARNING'}, 0, "failed to
retrieve contents of $pid_file_path");
+ }
+ }
+
+ # Update the PID file with the correct PID
+ my $echo_command = "echo -n $running_pid > $pid_file_path";
+ my ($echo_exit_status, $echo_output) =
$self->vmhost_os->execute($echo_command);
+ if (!defined($echo_output)) {
+ notify($ERRORS{'WARNING'}, 0, "failed to run command to
update $pid_file_path on $vmhost_computer_name");
+ return;
+ }
+ elsif (grep(/(ash:|echo:)/, @$echo_output)) {
+ notify($ERRORS{'WARNING'}, 0, "error occurred updating
$pid_file_path on $vmhost_computer_name, command: '$echo_command', output:\n" .
joini("\n", @$echo_output));
+ return;
}
else {
- notify($ERRORS{'OK'}, 0, "PID in $pid_file_path
($file_pid) does not match PID of parent $process_name process ($running_pid),
updating $pid_file_path to contain $running_pid");
+ notify($ERRORS{'OK'}, 0, "updated $pid_file_path on
$vmhost_computer_name to contain the correct PID: $running_pid");
}
}
- else {
- notify($ERRORS{'OK'}, 0, "PID in $pid_file_path could not be
determined, updating $pid_file_path to contain $running_pid");
- }
-
- # Update the PID file with the correct PID
- my $echo_command = "echo -n $running_pid > $pid_file_path";
- my ($echo_exit_status, $echo_output) =
$self->vmhost_os->execute($echo_command);
- if (!defined($echo_output)) {
- notify($ERRORS{'WARNING'}, 0, "failed to run command to update
$pid_file_path on $vmhost_computer_name");
- return;
- }
- elsif (grep(/(ash:|echo:)/, @$echo_output)) {
- notify($ERRORS{'WARNING'}, 0, "error occurred updating
$pid_file_path on $vmhost_computer_name, command: '$echo_command', output:\n" .
joini("\n", @$echo_output));
- return;
- }
- else {
- notify($ERRORS{'OK'}, 0, "updated $pid_file_path on
$vmhost_computer_name to contain the correct PID: $running_pid");
- }
return 1;
}
Modified: vcl/trunk/managementnode/lib/VCL/Module/Semaphore.pm
URL:
http://svn.apache.org/viewvc/vcl/trunk/managementnode/lib/VCL/Module/Semaphore.pm?rev=1632640&r1=1632639&r2=1632640&view=diff
==============================================================================
--- vcl/trunk/managementnode/lib/VCL/Module/Semaphore.pm (original)
+++ vcl/trunk/managementnode/lib/VCL/Module/Semaphore.pm Fri Oct 17 17:41:20
2014
@@ -194,8 +194,61 @@ sub open_lockfile {
return;
}
+ # Determine which process is locking the file
+ my @locking_pids = $self->get_lockfile_owning_pid($file_path);
+ if (@locking_pids) {
+ if (grep { $_ eq $PID } @locking_pids) {
+ # The current process already has an exclusive lock on
the file
+ # This could happen if open_lockfile is called more
than once for the same file in the same scope
+ notify($ERRORS{'WARNING'}, 0, "file is already locked
by this process: @locking_pids");
+ return;
+ }
+ else {
+ # Attempt to retrieve the names of the locking
process(es)
+ my ($ps_exit_status, $ps_output) = run_command("ps -o
pid=,cmd= @locking_pids", 1);
+ if (defined($ps_output) && !grep(/(ps:)/, @$ps_output))
{
+ notify($ERRORS{'DEBUG'}, 0, "file is locked by
another process: @locking_pids\n" . join("\n", @$ps_output));
+ }
+ else {
+ notify($ERRORS{'DEBUG'}, 0, "file is locked by
another process: @locking_pids");
+ }
+ return;
+ }
+ }
+ else {
+ notify($ERRORS{'DEBUG'}, 0, "unable to determine PIDs of
processes which prevented an exclusive lock to be obtained on $file_path, lock
may have been released before lsof command was executed");
+ }
+
+ return;
+}
+
+#/////////////////////////////////////////////////////////////////////////////
+
+=head2 get_lockfile_owning_pid
+
+ Parameters : $file_path
+ Returns : integer
+ Description : Runs lsof to determine if a process has an exclusive lock on the
+ lockfile.
+
+=cut
+
+sub get_lockfile_owning_pid {
+ my $self = shift;
+ unless (ref($self) && $self->isa('VCL::Module')) {
+ notify($ERRORS{'CRITICAL'}, 0, "subroutine was called as a
function, it must be called as a class method");
+ return;
+ }
+
+ # Get the file path argument
+ my ($file_path) = @_;
+ if (!$file_path) {
+ notify($ERRORS{'WARNING'}, 0, "file path argument was not
supplied");
+ return;
+ }
+
# Run lsof to determine which process is locking the file
- my ($exit_status, $output) = run_command("/usr/sbin/lsof -Fp
$file_path", 1);
+ my ($exit_status, $output) = $self->mn_os->execute("/usr/sbin/lsof -Fp
$file_path", 0);
if (!defined($output)) {
notify($ERRORS{'WARNING'}, 0, "failed to run losf command to
determine which process is locking the file: $file_path");
return;
@@ -207,31 +260,19 @@ sub open_lockfile {
# Parse the lsof output to determine the PID
my @locking_pids = map { /^p(\d+)/ } @$output;
- if (@locking_pids && grep { $_ eq $PID } @locking_pids) {
- # The current process already has an exclusive lock on the file
- # This could happen if open_lockfile is called more than once
for the same file in the same scope
- notify($ERRORS{'WARNING'}, 0, "file is already locked by this
process: @locking_pids");
- return;
- }
- elsif (@locking_pids) {
- # Attempt to retrieve the names of the locking process(es)
- my ($ps_exit_status, $ps_output) = run_command("ps -o pid=,cmd=
@locking_pids", 1);
- if (defined($ps_output) && !grep(/(ps:)/, @$ps_output)) {
- notify($ERRORS{'DEBUG'}, 0, "file is locked by another
process: @locking_pids\n" . join("\n", @$ps_output));
- }
- else {
- notify($ERRORS{'DEBUG'}, 0, "file is locked by another
process: @locking_pids");
- }
- return;
+ my $locking_pid_count = scalar(@locking_pids);
+ if (@locking_pids) {
+ notify($ERRORS{'DEBUG'}, 0, "$file_path is locked by process" .
($locking_pid_count == 1 ? '' : 'es') . ": @locking_pids");
+ return @locking_pids;
}
elsif (grep(/\w/, @$output)) {
- notify($ERRORS{'WARNING'}, 0, "unable to determine PIDs from
lsof output\n:" . join("\n", @$output));
+ notify($ERRORS{'WARNING'}, 0, "failed to determine owning PID
of lockfile: $file_path, unable to determine PIDs from lsof output\n:" .
join("\n", @$output));
+ return;
}
else {
- notify($ERRORS{'DEBUG'}, 0, "lsof did not return any PIDs of
processes which prevented an exclusive lock to be obtained, lock may have been
released before lsof command was executed");
+ notify($ERRORS{'DEBUG'}, 0, "file is not locked of lockfile:
$file_path");
+ return ();
}
-
- return 0;
}
#/////////////////////////////////////////////////////////////////////////////
@@ -290,6 +331,83 @@ sub release_lockfile {
#/////////////////////////////////////////////////////////////////////////////
+=head2 get_lockfile_paths
+
+ Parameters : none
+ Returns : array
+ Description : Returns the paths to all lockfiles.
+
+=cut
+
+sub get_lockfile_paths {
+ my $self = shift;
+ unless (ref($self) && $self->isa('VCL::Module')) {
+ notify($ERRORS{'CRITICAL'}, 0, "subroutine was called as a
function, it must be called as a class method");
+ return;
+ }
+
+ my @lockfile_paths = $self->mn_os->find_files($LOCKFILE_DIRECTORY_PATH,
"*.$LOCKFILE_EXTENSION");
+
+ my $lockfile_path_count = scalar(@lockfile_paths);
+ notify($ERRORS{'DEBUG'}, 0, "retreived $lockfile_path_count lockfile
path" . ($lockfile_path_count == 1 ? '' : 's') . ":\n" . join("\n",
@lockfile_paths));
+ return @lockfile_paths;
+}
+
+#/////////////////////////////////////////////////////////////////////////////
+
+=head2 semaphore_exists
+
+ Parameters : $semaphore_id
+ Returns : boolean
+ Description : Determines if an open Semaphore exists on this management node
+ matching the $semaphore_id.
+
+=cut
+
+sub semaphore_exists {
+ my $self = shift;
+ unless (ref($self) && $self->isa('VCL::Module')) {
+ notify($ERRORS{'CRITICAL'}, 0, "subroutine was called as a
function, it must be called as a class method");
+ return;
+ }
+
+ my ($semaphore_id) = @_;
+ if (!$semaphore_id) {
+ notify($ERRORS{'WARNING'}, 0, "semaphore ID argument was not
supplied");
+ return;
+ }
+
+ my @lockfile_paths = $self->get_lockfile_paths();
+ if (!@lockfile_paths) {
+ notify($ERRORS{'DEBUG'}, 0, "did not find any lockfiles on this
management node");
+ return ();
+ }
+
+ for my $lockfile_path (@lockfile_paths) {
+ my ($lockfile_semaphore_id) = $lockfile_path =~
/([^\/]+)\.$LOCKFILE_EXTENSION/;
+ if ($lockfile_semaphore_id ne $semaphore_id) {
+ next;
+ }
+
+ # Check if the lockfile is actually locked by another process
+ # It may have been released or deleted
+ my @lockfile_owning_pids =
$self->get_lockfile_owning_pid($lockfile_path);
+ if (@lockfile_owning_pids) {
+ notify($ERRORS{'DEBUG'}, 0, "'$semaphore_id' semaphore
exists, lockfile path: $lockfile_path, owning PID: @lockfile_owning_pids");
+ return 1;
+ }
+ else {
+ notify($ERRORS{'DEBUG'}, 0, "ignoring lockfile not
locked by another process: $lockfile_path");
+ next;
+ }
+ }
+
+ notify($ERRORS{'DEBUG'}, 0, "'$semaphore_id' semaphore does not exist");
+ return 0;
+}
+
+#/////////////////////////////////////////////////////////////////////////////
+
=head2 get_reservation_semaphore_ids
Parameters : $reservation_id
@@ -313,14 +431,13 @@ sub get_reservation_semaphore_ids {
return;
}
- my @lockfile_paths = $self->mn_os->find_files($LOCKFILE_DIRECTORY_PATH,
"*.$LOCKFILE_EXTENSION");
+ my @lockfile_paths = $self->get_lockfile_paths();
if (!@lockfile_paths) {
notify($ERRORS{'DEBUG'}, 0, "did not find any lockfiles on this
management node");
return ();
}
my @reservation_semaphore_ids;
-
for my $lockfile_path (@lockfile_paths) {
my ($semaphore_id) = $lockfile_path =~
/([^\/]+)\.$LOCKFILE_EXTENSION/;
@@ -377,7 +494,7 @@ sub get_process_semaphore_ids {
return;
}
- my @lockfile_paths = $self->mn_os->find_files($LOCKFILE_DIRECTORY_PATH,
"*.$LOCKFILE_EXTENSION");
+ my @lockfile_paths = $self->get_lockfile_paths();
if (!@lockfile_paths) {
notify($ERRORS{'DEBUG'}, 0, "did not find any lockfiles on this
management node");
return ();
Modified: vcl/trunk/managementnode/lib/VCL/utils.pm
URL:
http://svn.apache.org/viewvc/vcl/trunk/managementnode/lib/VCL/utils.pm?rev=1632640&r1=1632639&r2=1632640&view=diff
==============================================================================
--- vcl/trunk/managementnode/lib/VCL/utils.pm (original)
+++ vcl/trunk/managementnode/lib/VCL/utils.pm Fri Oct 17 17:41:20 2014
@@ -206,6 +206,7 @@ our @EXPORT = qw(
set_hash_process_id
set_logfile_path
set_managementnode_state
+ set_reservation_lastcheck
set_variable
setnextimage
setup_confirm
@@ -5265,6 +5266,7 @@ SET
reservation.lastcheck = '$lastcheck'
WHERE
reservation.id IN ($reservation_id_string)
+AND reservation.lastcheck <= NOW()
EOF
# Call the database execute subroutine
@@ -5280,6 +5282,63 @@ EOF
#/////////////////////////////////////////////////////////////////////////////
+=head2 set_reservation_lastcheck
+
+ Parameters : $reservation_id, $lastcheck
+ Returns : string
+ Description : Updates reservation.lastcheck to the time specified.
+
+=cut
+
+sub set_reservation_lastcheck {
+ my ($reservation_id, $reservation_lastcheck) = @_;
+
+ # Check the passed parameter
+ if (!$reservation_id || !$reservation_lastcheck) {
+ notify($ERRORS{'WARNING'}, 0, "reservation ID and last check
datetime was not specified");
+ return;
+ }
+
+ if ($reservation_lastcheck !~ /:/) {
+ $reservation_lastcheck =
convert_to_datetime($reservation_lastcheck);
+ }
+ my $reservation_lastcheck_epoch =
convert_to_epoch_seconds($reservation_lastcheck);
+
+ # Only allow the lastcheck time to be set in the future
+ my $current_time_epoch = time;
+ my $duration_seconds =
($reservation_lastcheck_epoch-$current_time_epoch);
+ if ($duration_seconds < 0) {
+ notify($ERRORS{'WARNING'}, 0, "reservation.lastcheck not set to
$reservation_lastcheck for reservation ID $reservation_id, time is in the
past");
+ return;
+ }
+ elsif ($duration_seconds < (20*60)) {
+ notify($ERRORS{'WARNING'}, 0, "reservation.lastcheck not set to
$reservation_lastcheck for reservation ID $reservation_id, time is too close to
the current time");
+ return;
+ }
+
+ # Construct the update statement
+ my $update_statement = <<EOF;
+UPDATE
+reservation
+SET
+reservation.lastcheck = '$reservation_lastcheck'
+WHERE
+reservation.id = '$reservation_id'
+EOF
+
+ # Call the database execute subroutine
+ if (database_execute($update_statement)) {
+ notify($ERRORS{'DEBUG'}, 0, "reservation.lastcheck set to
'$reservation_lastcheck' for reservation ID $reservation_id");
+ return 1;
+ }
+ else {
+ notify($ERRORS{'WARNING'}, 0, "failed to set
reservation.lastcheck to '$reservation_lastcheck' for reservation ID
$reservation_id");
+ return;
+ }
+}
+
+#/////////////////////////////////////////////////////////////////////////////
+
=head2 update_log_loaded_time
Parameters : $request_logid
