--- configure.ac | 8 + fence/agents/scsi/Makefile.am | 20 +- fence/agents/scsi/fence_scsi.8 | 119 ----- fence/agents/scsi/fence_scsi.pl | 875 ---------------------------------- fence/agents/scsi/fence_scsi.py | 444 +++++++++++++++++ fence/agents/scsi/fence_scsi_check.pl | 170 ------- make/fencebuild.mk | 4 + 7 files changed, 467 insertions(+), 1173 deletions(-) delete mode 100644 fence/agents/scsi/fence_scsi.8 delete mode 100644 fence/agents/scsi/fence_scsi.pl create mode 100644 fence/agents/scsi/fence_scsi.py delete mode 100644 fence/agents/scsi/fence_scsi_check.pl
diff --git a/configure.ac b/configure.ac index c24198c..c4e38f1 100644 --- a/configure.ac +++ b/configure.ac @@ -163,6 +163,10 @@ CLUSTERDATA=${datadir}/cluster AC_PATH_PROG([IPMITOOL_PATH], [ipmitool], [/usr/bin/ipmitool]) AC_PATH_PROG([AMTTOOL_PATH], [amttool], [/usr/bin/amttool]) AC_PATH_PROG([GNUTLSCLI_PATH], [gnutlscli], [/usr/bin/gnutls-cli]) +AC_PATH_PROG([COROSYNC_CMAPCTL_PATH], [corosync-cmapctl], [/usr/sbin/corosync-cmapctl]) +AC_PATH_PROG([SG_PERSIST_PATH], [sg_persist], [/usr/bin/sg_persist]) +AC_PATH_PROG([SG_TURS_PATH], [sg_turs], [/usr/bin/sg_turs]) +AC_PATH_PROG([VGS_PATH], [vgs], [/usr/sbin/vgs]) ## do subst AC_SUBST([DEFAULT_CONFIG_DIR]) @@ -189,6 +193,10 @@ AM_CONDITIONAL(BUILD_XENAPILIB, test $XENAPILIB -eq 1) AC_SUBST([IPMITOOL_PATH]) AC_SUBST([AMTTOOL_PATH]) +AC_SUBST([COROSYNC_CMAPCTL_PATH]) +AC_SUBST([SG_PERSIST_PATH]) +AC_SUBST([SG_TURS_PATH]) +AC_SUBST([VGS_PATH]) ## *FLAGS handling diff --git a/fence/agents/scsi/Makefile.am b/fence/agents/scsi/Makefile.am index 5652bda..896406a 100644 --- a/fence/agents/scsi/Makefile.am +++ b/fence/agents/scsi/Makefile.am @@ -2,20 +2,22 @@ MAINTAINERCLEANFILES = Makefile.in TARGET = fence_scsi -SRC = $(TARGET).pl +SYMTARGET = fence_scsi_check -EXTRA_DIST = $(SRC) \ - $(TARGET)_check.pl +SRC = $(TARGET).py -scsidatadir = $(CLUSTERDATA) +EXTRA_DIST = $(SRC) -scsidata_SCRIPTS = $(TARGET)_check.pl +sbin_SCRIPTS = $(TARGET) $(SYMTARGET) -sbin_SCRIPTS = $(TARGET) +man_MANS = $(TARGET).8 -dist_man_MANS = $(TARGET).8 +$(SYMTARGET): $(TARGET) + ln -s $^ $@ include $(top_srcdir)/make/fencebuild.mk +include $(top_srcdir)/make/fenceman.mk +include $(top_srcdir)/make/agentpycheck.mk -clean-local: - rm -f $(TARGET) +clean-local: clean-man + rm -f $(TARGET) $(SYMTARGET) diff --git a/fence/agents/scsi/fence_scsi.8 b/fence/agents/scsi/fence_scsi.8 deleted file mode 100644 index 180de4f..0000000 --- a/fence/agents/scsi/fence_scsi.8 +++ /dev/null @@ -1,119 +0,0 @@ -.TH fence_scsi 8 - -.SH NAME -fence_scsi - I/O fencing agent for SCSI persistent reservations - -.SH SYNOPSIS -.B -fence_scsi -[\fIOPTION\fR]... - -.SH DESCRIPTION -fence_scsi is an I/O fencing agent that uses SCSI-3 persistent -reservations to control access to shared storage devices. These -devices must support SCSI-3 persistent reservations (SPC-3 or greater) -as well as the "preempt-and-abort" subcommand. - -The fence_scsi agent works by having each node in the cluster register -a unique key with the SCSI devive(s). Once registered, a single node -will become the reservation holder by creating a "write exclusive, -registrants only" reservation on the device(s). The result is that -only registered nodes may write to the device(s). When a node failure -occurs, the fence_scsi agent will remove the key belonging to the -failed node from the device(s). The failed node will no longer be able -to write to the device(s). A manual reboot is required. In the cluster -environment unfence action should be configured also. - -Keys are either be specified manually (see -k option) or generated -automatically (see -n option). Automatic key generation requires that -cman be running. Keys will then be generated using the cluster ID and -node ID such that each node has a unique key that can be determined by -any other node in the cluster. - -Devices can either be specified manually (see -d option) or discovered -automatically. Multiple devices can be specified manually by using a -comma-separated list. If no devices are specified, the fence_scsi -agent will attempt to discover devices by looking for cluster volumes -and extracting the underlying devices. Devices may be device-mapper -multipath devices or raw devices. If using a device-mapper multipath -device, the fence_scsi agent will find the underlying devices (paths) -and created registrations for each path. - -.SH OPTIONS -.TP -\fB-o\fP \fIaction\fR -Fencing action. This value can be "on", "off", "status", or -"metadata". The "on", "off", and "status" actions require either a key -(see -k option) or node name (see -n option). For "on", the agent will -attempt to register with the device(s) and create a reservation if -none exists. The "off" action will attempt to remove a node's key from -the device(s). The "status" action will report whether or not a node's -key is currently register with one or more of the devices. The -"metadata" action will display the XML metadata. The default action if -"off". -.TP -\fB-d\fP \fIdevices\fR -List of devices to use for current operation. Devices can be -comma-separated list of raw device (eg. /dev/sdc) or device-mapper -multipath devices (eg. /dev/dm-3). Each device must support SCSI-3 -persistent reservations. -.TP -\fB-f\fP \fIlogfile\fR -Log output to file. -.TP -\fB-n\fP \fInodename\fR -Name of the node to be fenced. The node name is used to generate the -key value used for the current operation. This option will be ignored -when used with the -k option. -.TP -\fB-k\fP \fIkey\fR -Key to use for the current operation. This key should be unique to a -node. For the "on" action, the key specifies the key use to register -the local node. For the "off" action, this key specifies the key to be -removed from the device(s). -.TP -\fB-H\fP \fIdelay\fR -Wait X seconds before fencing is started (Default Value: 0) - -.TP -\fB-a\fP -Use the APTPL flag for registrations. This option is only used for the -"on" action. -.TP -\fB-h\fP -Print out a help message describing available options, then exit. -.TP -\fB-v\fP -Verbose output. -.TP -\fB-V\fP -Print out a version message, then exit. - -.SH STDIN PARAMETERS -.TP -\fIagent = "param"\fR -This option is used by fence_node(8) and is ignored by fence_scsi. -.TP -\fInodename = "param"\fR -Same as -n option. -.TP -\fIaction = "param" \fR -Same as -o option. -.TP -\fIdevices = "param"\fR -Same as -d option. -.TP -\fIlogfile = "param"\fR -Same as -f option -.TP -\fIkey = "param"\fR -Same as -k option. -.TP -\fIdelay = "param"\fR -Same as -H option. -.TP -\fIaptpl = "1" -Enable the APTPL flag. Default is 0 (disable). - -.SH SEE ALSO -fence(8), fence_node(8), sg_persist(8), vgs(8), cman_tool(8), cman(5) diff --git a/fence/agents/scsi/fence_scsi.pl b/fence/agents/scsi/fence_scsi.pl deleted file mode 100644 index 6808ff5..0000000 --- a/fence/agents/scsi/fence_scsi.pl +++ /dev/null @@ -1,875 +0,0 @@ -#!/usr/bin/perl - -use Cwd 'realpath'; -use File::Basename; -use File::Path; -use Getopt::Std; -use POSIX; -use B; - -#BEGIN_VERSION_GENERATION -$RELEASE_VERSION=""; -$REDHAT_COPYRIGHT=""; -$BUILD_DATE=""; -#END_VERSION_GENERATION - -my $ME = fileparse ($0, ".pl"); - -################################################################################ - -sub log_debug ($) -{ - my $time = strftime "%b %e %T", localtime; - my ($msg) = @_; - - print STDOUT "$time $ME: [debug] $msg\n" unless defined ($opt_q); - - return; -} - -sub log_error ($) -{ - my $time = strftime "%b %e %T", localtime; - my ($msg) = @_; - - print STDERR "$time $ME: [error] $msg\n" unless defined ($opt_q); - - exit (1); -} - -sub do_action_on ($@) -{ - my $self = (caller(0))[3]; - my ($node_key, @devices) = @_; - - key_write ($node_key); - - foreach $dev (@devices) { - log_error ("device $dev does not exist") if (! -e $dev); - log_error ("device $dev is not a block device") if (! -b $dev); - - if (do_register_ignore ($node_key, $dev) != 0) { - log_error ("failed to create registration (key=$node_key, device=$dev)"); - } - - if (!get_reservation_key ($dev)) { - if (do_reserve ($node_key, $dev) != 0) { - if (!get_reservation_key ($dev)) { - log_error ("failed to create reservation (key=$node_key, device=$dev)"); - } - } - } - } - - return; -} - -sub do_action_off ($@) -{ - my $self = (caller(0))[3]; - my ($node_key, @devices) = @_; - - my $host_key = key_read (); - - if ($host_key eq $node_key) { - log_error ($self); - } - - foreach $dev (@devices) { - log_error ("device $dev does not exist") if (! -e $dev); - log_error ("device $dev is not a block device") if (! -b $dev); - - my @keys = grep { /^$node_key$/i } get_registration_keys ($dev); - - if (scalar (@keys) != 0) { - do_preempt_abort ($host_key, $node_key, $dev); - } - } - - return; -} - -sub do_action_status ($@) -{ - my $self = (caller(0))[3]; - my ($node_key, @devices) = @_; - - my $dev_count = 0; - my $key_count = 0; - - foreach $dev (@devices) { - log_error ("device $dev does not exist") if (! -e $dev); - log_error ("device $dev is not a block device") if (! -b $dev); - - do_reset ($dev); - - my @keys = grep { /^$node_key$/i } get_registration_keys ($dev); - - if (scalar (@keys) != 0) { - $dev_count++; - } - } - - if ($dev_count != 0) { - exit (0); - } else { - exit (2); - } -} - -sub do_verify_on ($@) -{ - my $self = (caller(0))[3]; - my ($node_key, @devices) = @_; - my $count = 0; - - for $dev (@devices) { - my @keys = grep { /^$node_key$/i } get_registration_keys ($dev); - - ## check that our key is registered - if (scalar (@keys) == 0) { - log_debug ("failed to register key $node_key on device $dev"); - $count++; - next; - } - - ## write dev to device file once registration is verified - dev_write ($dev); - - ## check that a reservation exists - if (!get_reservation_key ($dev)) { - log_debug ("no reservation exists on device $dev"); - $count++; - } - } - - if ($count != 0) { - log_error ("$self: failed to verify $count devices"); - } -} - -sub do_verify_off ($@) -{ - my $self = (caller(0))[3]; - my ($node_key, @devices) = @_; - my $count = 0; - - for $dev (@devices) { - my @keys = grep { /^$node_key$/i } get_registration_keys ($dev); - - ## check that our key is not registered - if (scalar (@keys) != 0) { - log_debug ("failed to remove key $node_key from device $dev"); - $count++; - next; - } - - ## check that a reservation exists - if (!get_reservation_key ($dev)) { - log_debug ("no reservation exists on device $dev"); - $count++; - } - } - - if ($count != 0) { - log_error ("$self: failed to verify $count devices"); - } -} - -sub do_register ($$$) -{ - my $self = (caller(0))[3]; - my ($host_key, $node_key, $dev) = @_; - - $dev = realpath ($dev); - - if (substr ($dev, 5) =~ /^dm/) { - my @slaves = get_mpath_slaves ($dev); - foreach (@slaves) { - do_register ($node_key, $_); - } - return; - } - - log_debug ("$self (host_key=$host_key, node_key=$node_key, dev=$dev)"); - - my $cmd; - my $out; - my $err; - - do_reset ($dev); - - $cmd = "sg_persist -n -o -G -K $host_key -S $node_key -d $dev"; - $cmd .= " -Z" if (defined $opt_a); - $out = qx { $cmd 2> /dev/null }; - $err = ($?>>8); - - # if ($err != 0) { - # log_error ("$self (err=$err)"); - # } - - log_debug ("$self (err=$err)"); - - return ($err); -} - -sub do_register_ignore ($$) -{ - my $self = (caller(0))[3]; - my ($node_key, $dev) = @_; - - $dev = realpath ($dev); - - if (substr ($dev, 5) =~ /^dm/) { - my @slaves = get_mpath_slaves ($dev); - foreach (@slaves) { - do_register_ignore ($node_key, $_); - } - return; - } - - log_debug ("$self (node_key=$node_key, dev=$dev)"); - - my $cmd; - my $out; - my $err; - - do_reset ($dev); - - $cmd = "sg_persist -n -o -I -S $node_key -d $dev"; - $cmd .= " -Z" if (defined $opt_a); - $out = qx { $cmd 2> /dev/null }; - $err = ($?>>8); - - # if ($err != 0) { - # log_error ("$self (err=$err)"); - # } - - log_debug ("$self (err=$err)"); - - return ($err); -} - -sub do_reserve ($$) -{ - my $self = (caller(0))[3]; - my ($host_key, $dev) = @_; - - log_debug ("$self (host_key=$host_key, dev=$dev)"); - - my $cmd = "sg_persist -n -o -R -T 5 -K $host_key -d $dev"; - my $out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - # if ($err != 0) { - # log_error ("$self (err=$err)"); - # } - - log_debug ("$self (err=$err)"); - - return ($err); -} - -sub do_release ($$) -{ - my $self = (caller(0))[3]; - my ($host_key, $dev) = @_; - - log_debug ("$self (host_key=$host_key, dev=$dev)"); - - my $cmd = "sg_persist -n -o -L -T 5 -K $host_key -d $dev"; - my $out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - # if ($err != 0) { - # log_error ("$self (err=$err)"); - # } - - log_debug ("$self (err=$err)"); - - return ($err); -} - -sub do_preempt ($$$) -{ - my $self = (caller(0))[3]; - my ($host_key, $node_key, $dev) = @_; - - log_debug ("$self (host_key=$host_key, node_key=$node_key, dev=$dev)"); - - my $cmd = "sg_persist -n -o -P -T 5 -K $host_key -S $node_key -d $dev"; - my $out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - # if ($err != 0) { - # log_error ("$self (err=$err)"); - # } - - log_debug ("$self (err=$err)"); - - return ($err); -} - -sub do_preempt_abort ($$$) -{ - my $self = (caller(0))[3]; - my ($host_key, $node_key, $dev) = @_; - - log_debug ("$self (host_key=$host_key, node_key=$node_key, dev=$dev)"); - - my $cmd = "sg_persist -n -o -A -T 5 -K $host_key -S $node_key -d $dev"; - my $out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - # if ($err != 0) { - # log_error ("$self (err=$err)"); - # } - - log_debug ("$self (err=$err)"); - - return ($err); -} - -sub do_reset (S) -{ - my $self = (caller(0))[3]; - my ($dev) = @_; - - my $cmd = "sg_turs $dev"; - my @out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - ## note that it is not necessarily an error is $err is non-zero, - ## so just log the device and status and continue. - - log_debug ("$self (dev=$dev, status=$err)"); - - return ($err); -} - -sub dev_unlink () -{ - my $self = (caller(0))[3]; - my $file = "/var/run/cluster/fence_scsi.dev"; - - if (-e $file) { - unlink ($file) or die "$!\n"; - } - - return; -} - -sub dev_write ($) -{ - my $self = (caller(0))[3]; - my $file = "/var/run/cluster/fence_scsi.dev"; - my $dev = shift; - - if (! -d "/var/run/cluster") { - mkpath ("/var/run/cluster"); - } - - open (\*FILE, "+>>$file") or die "$!\n"; - - ## since the file is opened for read, write and append, - ## we need to seek to the beginning of the file before grep. - - seek (FILE, 0, 0); - - if (! grep { /^$dev$/ } <FILE>) { - print FILE "$dev\n"; - } - - close (FILE); - - return; -} - -sub key_read () -{ - my $self = (caller(0))[3]; - my $file = "/var/run/cluster/fence_scsi.key"; - my $key; - - open (\*FILE, "<$file") or die "$!\n"; - chomp ($key = <FILE>); - close (FILE); - - return ($key); -} - -sub key_write ($) -{ - my $self = (caller(0))[3]; - my $file = "/var/run/cluster/fence_scsi.key"; - my $key = shift; - - if (! -d "/var/run/cluster") { - mkpath ("/var/run/cluster"); - } - - open (\*FILE, ">$file") or die "$!\n"; - print FILE "$key\n"; - close (FILE); - - return; -} - -sub get_key ($) -{ - my $self = (caller(0))[3]; - - my $key = sprintf ("%.4x%.4x", get_cluster_id (), get_node_id ($_[0])); - - return ($key); -} - -sub get_node_id ($) -{ - my $self = (caller(0))[3]; - my $node = $_[0]; - - my $cmd = "/usr/sbin/corosync-cmapctl nodelist."; - my @out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - if ($err != 0) { - log_error ("$self (err=$err)"); - } - - # die "[error]: $self\n" if ($?>>8); - - foreach my $line (@out) { - chomp($line); - if ($line =~ /.(\d+?).ring._addr \(str\) = ${node}$/) { - return $1; - } - } - - log_error("$self (unable to parse output of corosync-cmapctl or node does not exist)"); -} - -sub get_cluster_id () -{ - my $self = (caller(0))[3]; - my $cluster_id; - - my $cmd = "/usr/sbin/corosync-cmapctl totem.cluster_name"; - my $out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - if ($err != 0) { - log_error ("$self (err=$err)"); - } - - # die "[error]: $self\n" if ($?>>8); - - chomp($out); - - if ($out =~ /=\s(.*?)$/) { - my $cluster_name = $1; - # tranform string to a number - $cluster_id = (hex B::hash($cluster_name)) % 65536; - } else { - log_error("$self (unable to parse output of corosync-cmapctl)"); - } - - return ($cluster_id); -} - -sub get_devices_clvm () -{ - my $self = (caller(0))[3]; - my @devices; - - my $cmd = "vgs --noheadings " . - " --separator : " . - " --sort pv_uuid " . - " --options vg_attr,pv_name " . - " --config 'global { locking_type = 0 } " . - " devices { preferred_names = [ \"^/dev/dm\" ] }'"; - - my @out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - if ($err != 0) { - log_error ("$self (err=$err)"); - } - - # die "[error]: $self\n" if ($?>>8); - - foreach (@out) { - chomp; - my ($vg_attr, $pv_name) = split (/:/, $_); - if ($vg_attr =~ /c$/) { - push (@devices, $pv_name); - } - } - - return (@devices); -} - -sub get_devices_scsi () -{ - my $self = (caller(0))[3]; - my @devices; - - opendir (\*DIR, "/sys/block/") or die "$!\n"; - @devices = grep { /^sd/ } readdir (DIR); - closedir (DIR); - - return (@devices); -} - -sub get_mpath_name ($) -{ - my $self = (caller(0))[3]; - my ($dev) = @_; - my $name; - - if ($dev =~ /^\/dev\//) { - $dev = substr ($dev, 5); - } - - open (\*FILE, "/sys/block/$dev/dm/name") or die "$!\n"; - chomp ($name = <FILE>); - close (FILE); - - return ($name); -} - -sub get_mpath_uuid ($) -{ - my $self = (caller(0))[3]; - my ($dev) = @_; - my $uuid; - - if ($dev =~ /^\/dev\//) { - $dev = substr ($dev, 5); - } - - open (\*FILE, "/sys/block/$dev/dm/uuid") or die "$!\n"; - chomp ($uuid = <FILE>); - close (FILE); - - return ($name); -} - -sub get_mpath_slaves ($) -{ - my $self = (caller(0))[3]; - my ($dev) = @_; - my @slaves; - - if ($dev =~ /^\/dev\//) { - $dev = substr ($dev, 5); - } - - opendir (\*DIR, "/sys/block/$dev/slaves/") or die "$!\n"; - - @slaves = grep { !/^\./ } readdir (DIR); - if ($slaves[0] =~ /^dm/) { - @slaves = get_mpath_slaves ($slaves[0]); - } else { - @slaves = map { "/dev/$_" } @slaves; - } - - closedir (DIR); - - return (@slaves); -} - -sub get_registration_keys ($) -{ - my $self = (caller(0))[3]; - my ($dev) = @_; - my @keys; - - my $cmd = "sg_persist -n -i -k -d $dev"; - my @out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - if ($err != 0) { - log_error ("$self (err=$err)"); - } - - # die "[error]: $self\n" if ($?>>8); - - foreach (@out) { - chomp; - if ($_ =~ s/^\s+0x//i) { - push (@keys, $_); - } - } - - return (@keys); -} - -sub get_reservation_key ($) -{ - my $self = (caller(0))[3]; - my ($dev) = @_; - my $key; - - my $cmd = "sg_persist -n -i -r -d $dev"; - my @out = qx { $cmd 2> /dev/null }; - my $err = ($?>>8); - - if ($err != 0) { - log_error ("$self (err=$err)"); - } - - # die "[error]: $self\n" if ($?>>8); - - foreach (@out) { - chomp; - if ($_ =~ s/^\s+key=0x//i) { - $key = $_; - last; - } - } - - return ($key) -} - -sub get_options_stdin () -{ - my $num = 0; - - while (<STDIN>) { - chomp; - s/^\s*//; - s/\s*$//; - - next if (/^#/); - - $num++; - - next unless ($_); - - my ($opt, $arg) = split (/\s*=\s*/, $_); - - if ($opt eq "") { - exit (1); - } - elsif ($opt eq "aptpl") { - $opt_a = $arg; - } - elsif ($opt eq "devices") { - $opt_d = $arg; - } - elsif ($opt eq "logfile") { - $opt_f = $arg; - } - elsif ($opt eq "key") { - $opt_k = $arg; - } - elsif ($opt eq "nodename") { - $opt_n = $arg; - } - elsif ($opt eq "action") { - $opt_o = $arg; - } - elsif ($opt eq "delay") { - $opt_H = $arg; - } - } -} - -sub print_usage () -{ - print "Usage:\n"; - print "\n"; - print "$ME [options]\n"; - print "\n"; - print "Options:\n"; - print " -a Use APTPL flag\n"; - print " -d <devices> Devices to be used for action\n"; - print " -f <logfile> File to write debug/error output\n"; - print " -H <timeout> Wait X seconds before fencing is started\n"; - print " -h Usage\n"; - print " -k <key> Key to be used for current action\n"; - print " -n <nodename> Name of node to operate on\n"; - print " -o <action> Action: off (default), on, or status\n"; - print " -q Quiet mode\n"; - print " -V Version\n"; - - exit (0); -} - -sub print_version () -{ - print "$ME $RELEASE_VERSION $BUILD_DATE\n"; - print "$REDHAT_COPYRIGHT\n" if ( $REDHAT_COPYRIGHT ); - - exit (0); -} - -sub print_metadata () -{ - print "<?xml version=\"1.0\" ?>\n"; - print "<resource-agent name=\"fence_scsi\"" . - " shortdesc=\"fence agent for SCSI-3 persistent reservations\">\n"; - print "<longdesc>fence_scsi</longdesc>\n"; - print "<vendor-url>http://www.t10.org</vendor-url>\n"; - print "<parameters>\n"; - print "\t<parameter name=\"aptpl\" unique=\"0\" required=\"0\">\n"; - print "\t\t<getopt mixed=\"-a\"/>\n"; - print "\t\t<content type=\"boolean\"/>\n"; - print "\t\t<shortdesc lang=\"en\">" . - "Use APTPL flag for registrations" . - "</shortdesc>\n"; - print "\t</parameter>\n"; - print "\t<parameter name=\"devices\" unique=\"0\" required=\"0\">\n"; - print "\t\t<getopt mixed=\"-d\"/>\n"; - print "\t\t<content type=\"string\"/>\n"; - print "\t\t<shortdesc lang=\"en\">" . - "List of devices to be used for fencing action" . - "</shortdesc>\n"; - print "\t</parameter>\n"; - print "\t<parameter name=\"logfile\" unique=\"0\" required=\"0\">\n"; - print "\t\t<getopt mixed=\"-f\"/>\n"; - print "\t\t<content type=\"string\"/>\n"; - print "\t\t<shortdesc lang=\"en\">" . - "File to write error/debug messages" . - "</shortdesc>\n"; - print "\t</parameter>\n"; - print "\t<parameter name=\"delay\" unique=\"0\" required=\"0\">\n"; - print "\t\t<getopt mixed=\"-H\"/>\n"; - print "\t\t<content type=\"string\"/>\n"; - print "\t\t<shortdesc lang=\"en\">" . - "Wait X seconds before fencing is started" . - "</shortdesc>\n"; - print "\t</parameter>\n"; - print "\t<parameter name=\"key\" unique=\"0\" required=\"0\">\n"; - print "\t\t<getopt mixed=\"-k\"/>\n"; - print "\t\t<content type=\"string\"/>\n"; - print "\t\t<shortdesc lang=\"en\">" . - "Key value to be used for fencing action" . - "</shortdesc>\n"; - print "\t</parameter>\n"; - print "\t<parameter name=\"action\" unique=\"0\" required=\"0\">\n"; - print "\t\t<getopt mixed=\"-o\"/>\n"; - print "\t\t<content type=\"string\" default=\"off\"/>\n"; - print "\t\t<shortdesc lang=\"en\">" . - "Fencing action" . - "</shortdesc>\n"; - print "\t</parameter>\n"; - print "\t<parameter name=\"nodename\" unique=\"0\" required=\"0\">\n"; - print "\t\t<getopt mixed=\"-n\"/>\n"; - print "\t\t<content type=\"string\"/>\n"; - print "\t\t<shortdesc lang=\"en\">" . - "Name of node" . - "</shortdesc>\n"; - print "\t</parameter>\n"; - print "</parameters>\n"; - print "<actions>\n"; - print "\t<action name=\"on\" on_target=\"1\" automatic=\"1\"/>\n"; - print "\t<action name=\"off\"/>\n"; - print "\t<action name=\"status\"/>\n"; - print "\t<action name=\"metadata\"/>\n"; - print "</actions>\n"; - print "</resource-agent>\n"; - - exit (0); -} - -################################################################################ - -if (@ARGV > 0) { - getopts ("ad:f:H:hk:n:o:qV") or print_usage; - print_usage if (defined $opt_h); - print_version if (defined $opt_V); -} else { - get_options_stdin (); -} - -## handle the metadata action here to avoid other parameter checks -## -if ($opt_o =~ /^metadata$/i) { - print_metadata; -} - -## if the logfile (-f) parameter was specified, open the logfile -## and redirect STDOUT and STDERR to the logfile. -## -if (defined $opt_f) { - open (LOG, ">>$opt_f") or die "$!\n"; - open (STDOUT, ">&LOG"); - open (STDERR, ">&LOG"); -} - -## verify that either key or nodename have been specified -## -if ((!defined $opt_n) && (!defined $opt_k)) { - print_usage (); -} - -## determine key value -## -if (defined $opt_k) { - $key = $opt_k; -} else { - $key = get_key ($opt_n); -} - -## verify that key is not zero -## -if (hex($key) == 0) { - log_error ("key cannot be zero"); -} - -## remove any leading zeros from key -## -if ($key =~ /^0/) { - $key =~ s/^0+//; -} - -## get devices -## -if (defined $opt_d) { - @devices = split (/\s*,\s*/, $opt_d); -} else { - @devices = get_devices_clvm (); -} - -## verify that device list is not empty -## -if (scalar (@devices) == 0) { - log_error ("no devices found"); -} - -## default action is "off" -## -if (!defined $opt_o) { - $opt_o = "off"; -} - -## Wait for defined period (-H / delay= ) -## -if ((defined $opt_H) && ($opt_H =~ /^[0-9]+/)) { - sleep($opt_H); -} - -## determine the action to perform -## -if ($opt_o =~ /^on$/i) { - do_action_on ($key, @devices); - do_verify_on ($key, @devices); -} -elsif ($opt_o =~ /^off$/i) { - do_action_off ($key, @devices); - do_verify_off ($key, @devices); -} -elsif ($opt_o =~ /^status/i) { - do_action_status ($key, @devices); -} else { - log_error ("unknown action '$opt_o'"); - exit (1); -} - -## close the logfile -## -if (defined $opt_f) { - close (LOG); -} diff --git a/fence/agents/scsi/fence_scsi.py b/fence/agents/scsi/fence_scsi.py new file mode 100644 index 0000000..e1d9f89 --- /dev/null +++ b/fence/agents/scsi/fence_scsi.py @@ -0,0 +1,444 @@ +#!/usr/bin/python -tt + +import sys +import stat +import re +import os +import time +import logging +import atexit +import hashlib +sys.path.append("@FENCEAGENTSLIBDIR@") +from fencing import fail_usage, run_command, atexit_handler, check_input, process_input, show_docs, fence_action, all_opt + +#BEGIN_VERSION_GENERATION +RELEASE_VERSION="" +REDHAT_COPYRIGHT="" +BUILD_DATE="" +#END_VERSION_GENERATION + +STORE_PATH = "/var/run/cluster/fence_scsi" + + +def get_status(conn, options): + del conn + status = "off" + for dev in options["devices"]: + is_block_device(dev) + reset_dev(options, dev) + if options["--key"] in get_registration_keys(options, dev): + status = "on" + else: + logging.debug("No registration for key "\ + + options["--key"] + " on device " + dev + "\n") + return status + + +def set_status(conn, options): + del conn + count = 0 + if options["--action"] == "on": + set_key(options) + for dev in options["devices"]: + is_block_device(dev) + + register_dev(options, dev) + if options["--key"] not in get_registration_keys(options, dev): + count += 1 + logging.debug("Failed to register key "\ + + options["--key"] + "on device " + dev + "\n") + continue + dev_write(dev, options) + + if get_reservation_key(options, dev) is None \ + and not reserve_dev(options, dev) \ + and get_reservation_key(options, dev) is None: + count += 1 + logging.debug("Failed to create reservation (key="\ + + options["--key"] + ", device=" + dev + ")\n") + + else: + host_key = get_key() + if host_key == options["--key"].lower(): + fail_usage("Failed: keys cannot be same. You can not fence yourself.") + for dev in options["devices"]: + is_block_device(dev) + + if options["--key"] in get_registration_keys(options, dev): + preempt_abort(options, host_key, dev) + + for dev in options["devices"]: + if options["--key"] in get_registration_keys(options, dev): + count += 1 + logging.debug("Failed to remove key "\ + + options["--key"] + " on device " + dev + "\n") + continue + + if not get_reservation_key(options, dev): + count += 1 + logging.debug("No reservation exists on device " + dev + "\n") + if count: + logging.error("Failed to verify " + str(count) + " device(s)") + sys.exit(1) + + +#run command, returns dict, ret["err"] = exit code; ret["out"] = output +def run_cmd(options, cmd): + ret = {} + (ret["err"], ret["out"], _) = run_command(options, cmd) + ret["out"] = "".join([i for i in ret["out"] if i is not None]) + return ret + + +# check if device exist and is block device +def is_block_device(dev): + if not os.path.exists(dev): + fail_usage("Failed: device \"" + dev + "\" does not exist") + if not stat.S_ISBLK(os.stat(dev).st_mode): + fail_usage("Failed: device \"" + dev + "\" is not a block device") + + +# cancel registration +def preempt_abort(options, host, dev): + cmd = options["--sg_persist-path"] + " -n -o -A -T 5 -K " + host + " -S " + options["--key"] + " -d " + dev + return not bool(run_cmd(options, cmd)["err"]) + + +def reset_dev(options, dev): + return run_cmd(options, options["--sg_turs-path"] + " " + dev)["err"] + + +def register_dev(options, dev): + dev = os.path.realpath(dev) + if re.search("^dm", dev[5:]): + register_dev(i for i in get_mpath_slaves(dev)) + return True + reset_dev(options, dev) + cmd = options["--sg_persist-path"] + " -n -o -I -S " + options["--key"] + " -d " + dev + cmd += " -Z" if "--aptpl" in options else "" + #cmd return code != 0 but registration can be successful + return not bool(run_cmd(options, cmd)["err"]) + + +def reserve_dev(options, dev): + cmd = options["--sg_persist-path"] + " -n -o -R -T 5 -K " + options["--key"] + " -d " + dev + return not bool(run_cmd(options, cmd)["err"]) + + +def get_reservation_key(options, dev): + cmd = options["--sg_persist-path"] + " -n -i -r -d " + dev + out = run_cmd(options, cmd) + if out["err"]: + fail_usage("Cannot get reservation key") + match = re.search("\s+key=0x(\S+)\s+", out["out"], re.IGNORECASE) + return match.group(1) if match else None + + +def get_registration_keys(options, dev): + keys = [] + cmd = options["--sg_persist-path"] + " -n -i -k -d " + dev + out = run_cmd(options, cmd) + if out["err"]: + fail_usage("Cannot get registration keys") + for line in out["out"].split("\n"): + match = re.search("\s+0x(\S+)\s*", line) + if match: + keys.append(match.group(1)) + return keys + + +def get_cluster_id(options): + cmd = options["--corosync-cmap-path"] + " totem.cluster_name" + + match = re.search("\(str\) = (\S+)\n", run_cmd(options, cmd)["out"]) + return hashlib.md5(match.group(1)).hexdigest() if match else fail_usage("Failed: cannot get cluster name") + + +def get_node_id(options): + cmd = options["--corosync-cmap-path"] + " nodelist." + + match = re.search(".(\d).ring._addr \(str\) = " + options["--nodename"] + "\n", run_cmd(options, cmd)["out"]) + return match.group(1) if match else fail_usage("Failed: unable to parse output of corosync-cmapctl or node does not exist") + + +def generate_key(options): + return "%.4s%.4d" % (get_cluster_id(options), int(get_node_id(options))) + + +# save node key to file +def set_key(options): + file_path = options["store_path"] + ".key" + if not os.path.isdir(os.path.dirname(options["store_path"])): + os.makedirs(os.path.dirname(options["store_path"])) + try: + f = open(file_path, "w") + except IOError: + fail_usage("Failed: Cannot open file \""+ file_path + "\"") + f.write(options["--key"].lower() + "\n") + f.close() + + +# read node key from file +def get_key(): + file_path = STORE_PATH + ".key" + try: + f = open(file_path, "r") + except IOError: + fail_usage("Failed: Cannot open file \""+ file_path + "\"") + return f.readline().strip().lower() + + +def dev_write(dev, options): + file_path = options["store_path"] + ".dev" + if not os.path.isdir(os.path.dirname(options["store_path"])): + os.makedirs(os.path.dirname(options["store_path"])) + try: + f = open(file_path, "a+") + except IOError: + fail_usage("Failed: Cannot open file \""+ file_path + "\"") + out = f.read() + if not re.search("^" + dev + "\s+", out): + f.write(dev + "\n") + f.close() + + +def dev_read(): + file_path = STORE_PATH + ".dev" + try: + f = open(file_path, "r") + except IOError: + fail_usage("Failed: Cannot open file \"" + file_path + "\"") + # get not empty lines from file + devs = filter(None, [line.strip() for line in f]) + f.close() + return devs + + +def dev_delete(options): + file_path = options["store_path"] + ".dev" + os.remove(file_path) if os.path.exists(file_path) else None + + +def get_clvm_devices(options): + devs = [] + cmd = options["--vgs-path"] + " " +\ + "--noheadings " +\ + "--separator : " +\ + "--sort pv_uuid " +\ + "--options vg_attr,pv_name "+\ + "--config 'global { locking_type = 0 } devices { preferred_names = [ \"^/dev/dm\" ] }'" + out = run_cmd(options, cmd) + if out["err"]: + fail_usage("Failed: Cannot get clvm devices") + for line in out["out"].split("\n"): + if 'c' in line.split(":")[0]: + devs.append(line.split(":")[1]) + return devs + + +def get_mpath_slaves(dev): + if dev[:5] == "/dev/": + dev = dev[5:] + slaves = [i for i in os.listdir("/sys/block/" + dev + "/slaves/") if i[:1] != "."] + if slaves[0][:2] == "dm": + slaves = get_mpath_slaves(slaves[0]) + else: + slaves = ["/dev/" + x for x in slaves] + return slaves + + +def define_new_opts(): + all_opt["devices"] = { + "getopt" : "d:", + "longopt" : "devices", + "help" : "-d, --devices=[devices] List of devices to use for current operation", + "required" : "0", + "shortdesc" : "List of devices to use for current operation. Devices can be comma-separated list of raw device (eg. /dev/sdc) or device-mapper multipath devices (eg. /dev/dm-3). Each device must support SCSI-3 persistent reservations.", + "order": 1 + } + all_opt["nodename"] = { + "getopt" : "n:", + "longopt" : "nodename", + "help" : "-n, --nodename=[nodename] Name of the node to be fenced", + "required" : "0", + "shortdesc" : "Name of the node to be fenced. The node name is used to generate the key value used for the current operation. This option will be ignored when used with the -k option.", + "order": 1 + } + all_opt["key"] = { + "getopt" : "k:", + "longopt" : "key", + "help" : "-k, --key=[key] Key to use for the current operation", + "required" : "0", + "shortdesc" : "Key to use for the current operation. This key should be unique to a node. For the \"on\" action, the key specifies the key use to register the local node. For the \"off\" action, this key specifies the key to be removed from the device(s).", + "order": 1 + } + all_opt["aptpl"] = { + "getopt" : "a", + "longopt" : "aptpl", + "help" : "-a, --aptpl Use the APTPL flag for registrations", + "required" : "0", + "shortdesc" : "Use the APTPL flag for registrations. This option is only used for the 'on' action.", + "order": 1 + } + all_opt["logfile"] = { + "getopt" : "f:", + "longopt" : "logfile", + "help" : "-a, --logfile Log output (stdout and stderr) to file", + "required" : "0", + "shortdesc" : "Log output (stdout and stderr) to file", + "order": 5 + } + all_opt["corosync-cmap_path"] = { + "getopt" : "Z:", + "longopt" : "corosync-cmap-path", + "help" : "--corosync-cmap-path=[path] Path to corosync-cmapctl binary", + "required" : "0", + "shortdesc" : "Path to corosync-cmapctl binary", + "default" : "@COROSYNC_CMAPCTL_PATH@", + "order": 200 + } + all_opt["sg_persist_path"] = { + "getopt" : "X:", + "longopt" : "sg_persist-path", + "help" : "--sg_persist-path=[path] Path to sg_persist binary", + "required" : "0", + "shortdesc" : "Path to sg_persist binary", + "default" : "@SG_PERSIST_PATH@", + "order": 200 + } + all_opt["sg_turs_path"] = { + "getopt" : "I:", + "longopt" : "sg_turs-path", + "help" : "--sg_turs-path=[path] Path to sg_turs binary", + "required" : "0", + "shortdesc" : "Path to sg_turs binary", + "default" : "@SG_TURS_PATH@", + "order": 200 + } + all_opt["vgs_path"] = { + "getopt" : "J:", + "longopt" : "vgs-path", + "help" : "--vgs-path=[path] Path to vgs binary", + "required" : "0", + "shortdesc" : "Path to vgs binary", + "default" : "@VGS_PATH@", + "order": 200 + } + + +def scsi_check_get_verbose(): + try: + f = open("/etc/sysconfig/watchdog", "r") + except IOError: + return False + match = re.search("^\s*verbose=yes", "".join(f.readlines()), re.MULTILINE) + f.close() + return bool(match) + + +def scsi_check(): + if len(sys.argv) >= 3 and sys.argv[1] == "repair": + return int(sys.argv[2]) + options = {} + options["--sg_turs-path"] = "@SG_TURS_PATH@" + options["--sg_persist-path"] = "@SG_PERSIST_PATH@" + options["--power-timeout"] = "5" + if scsi_check_get_verbose(): + logging.getLogger().setLevel(logging.DEBUG) + devs = dev_read() + if not devs: + logging.error("No devices found") + return 0 + key = get_key() + if not key: + logging.error("Key not found") + return 0 + for dev in devs: + if key in get_registration_keys(options, dev): + logging.debug("key " + key + " registered with device " + dev) + return 0 + else: + logging.debug("key " + key + " not registered with device " + dev) + logging.debug("key " + key + " registered with any devices") + return 2 + + +def main(): + + atexit.register(atexit_handler) + + device_opt = ["no_login", "no_password", "devices", "nodename", "key",\ + "aptpl", "fabric_fencing", "on_target", "corosync-cmap_path",\ + "sg_persist_path", "sg_turs_path", "logfile", "vgs_path"] + + define_new_opts() + + all_opt["action"]["help"] = "-o, --action=[action] Action: status, off (default) or on" + all_opt["action"]["default"] = "off" + all_opt["delay"]["getopt"] = "H:" + + #fence_scsi_check + if os.path.basename(sys.argv[0]) == "fence_scsi_check": + sys.exit(scsi_check()) + + options = check_input(device_opt, process_input(device_opt)) + + docs = { } + docs["shortdesc"] = "Fence agent for SCSI persistentl reservation" + docs["longdesc"] = "fence_scsi is an I/O fencing agent that uses SCSI-3 \ +persistent reservations to control access to shared storage devices. These \ +devices must support SCSI-3 persistent reservations (SPC-3 or greater) as \ +well as the \"preempt-and-abort\" subcommand.\nThe fence_scsi agent works by \ +having each node in the cluster register a unique key with the SCSI \ +devive(s). Once registered, a single node will become the reservation holder \ +by creating a \"write exclusive, registrants only\" reservation on the \ +device(s). The result is that only registered nodes may write to the \ +device(s). When a node failure occurs, the fence_scsi agent will remove the \ +key belonging to the failed node from the device(s). The failed node will no \ +longer be able to write to the device(s). A manual reboot is required." + docs["vendorurl"] = "" + show_docs(options, docs) + + # backward compatibility layer BEGIN + if "--logfile" in options: + try: + logfile = open(options["--logfile"], 'w') + sys.stderr = logfile + sys.stdout = logfile + except IOError: + fail_usage("Failed: Unable to create file " + options["--logfile"]) + # backward compatibility layer END + + options["store_path"] = STORE_PATH + + # Input control BEGIN + if not (("--nodename" in options and options["--nodename"])\ + or ("--key" in options and options["--key"])): + fail_usage("Failed: nodename or key is required") + + if not ("--key" in options and options["--key"]): + options["--key"] = generate_key(options) + + if options["--key"] == "0" or not options["--key"]: + fail_usage("Failed: key cannot be 0") + + options["--key"] = options["--key"].lstrip('0') + + if not ("--devices" in options and options["--devices"].split(",")): + options["devices"] = get_clvm_devices(options) + else: + options["devices"] = options["--devices"].split(",") + + if not options["devices"]: + fail_usage("Failed: No devices found") + # Input control END + + # backward compatibility layer + if "--delay" in options and options["--delay"].isdigit(): + time.sleep(int(options["--delay"])) + + result = fence_action(None, options, set_status, get_status) + sys.exit(result) + +if __name__ == "__main__": + main() diff --git a/fence/agents/scsi/fence_scsi_check.pl b/fence/agents/scsi/fence_scsi_check.pl deleted file mode 100644 index 9ecd7a5..0000000 --- a/fence/agents/scsi/fence_scsi_check.pl +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/perl - -use POSIX; - -################################################################################ - -my $dev_file = "/var/run/cluster/fence_scsi.dev"; -my $key_file = "/var/run/cluster/fence_scsi.key"; - -################################################################################ - -sub log_debug ($) -{ - my $time = strftime ("%b %e %T", localtime); - my $msg = shift; - - print STDOUT "$time [$0] debug: $msg\n" if ($verbose); - - return; -} - -sub log_error ($) -{ - my $time = strftime ("%b %e %T", localtime); - my $msg = shift; - - print STDERR "$time [$0] error: $msg\n"; - - return; -} - -sub do_reset ($) -{ - my $dev = shift; - - my $cmd = "sg_turs $dev"; - my @out = qx { $cmd 2> /dev/null }; - - return; -} - -sub get_registration_keys ($) -{ - my $dev = shift; - my @keys = (); - - do_reset ($dev); - - my $cmd = "sg_persist -n -i -k -d $dev"; - my @out = qx { $cmd 2> /dev/null }; - - if ($?>>8 != 0) { - log_error ("$cmd"); - exit (0); - } - - foreach (@out) { - chomp; - if (s/^\s+0x//i) { - push (@keys, $_); - } - } - - return (@keys); -} - -sub get_reservation_keys ($) -{ - my $dev = shift; - my @keys = (); - - do_reset ($dev); - - my $cmd = "sg_persist -n -i -r -d $dev"; - my @out = qx { $cmd 2> /dev/null }; - - if ($?>>8 != 0) { - log_error ("$cmd"); - exit (0); - } - - foreach (@out) { - chomp; - if (s/^\s+key=0x//i) { - push (@keys, $_); - } - } - - return (@keys); -} - -sub get_verbose () -{ - open (\*FILE, "</etc/sysconfig/watchdog") or return; - chomp (my @opt = <FILE>); - close (FILE); - - foreach (@opt) { - next if (/^#/); - next unless ($_); - - if (/^verbose=yes$/i) { - return (1); - } - } - - return (0); -} - -sub key_read () -{ - open (\*FILE, "<$key_file") or exit (0); - chomp (my $key = <FILE>); - close (FILE); - - return ($key); -} - -sub dev_read () -{ - open (\*FILE, "<$dev_file") or exit (0); - chomp (my @dev = <FILE>); - close (FILE); - - return (@dev); -} - -################################################################################ - -if ($ARGV[0] =~ /^repair$/i) { - exit ($ARGV[1]); -} - -if (-e "/etc/sysconfig/watchdog") { - $verbose = get_verbose (); -} - -if (! -e $dev_file) { - log_debug ("$dev_file does not exit"); - exit (0); -} elsif (-z $dev_file) { - log_debug ("$dev_file is empty"); - exit (0); -} - -if (! -e $key_file) { - log_debug ("$key_file does not exist"); - exit (0); -} elsif (-z $key_file) { - log_debug ("$key_file is empty"); - exit (0); -} - -my $key = key_read (); -my @dev = dev_read (); - -foreach (@dev) { - my @keys = grep { /^$key$/i } get_registration_keys ($_); - - if (scalar (@keys) != 0) { - log_debug ("key $key registered with device $_"); - exit (0); - } else { - log_debug ("key $key not registered with device $_"); - } -} - -log_debug ("key $key not registered with any devices"); - -exit (2); diff --git a/make/fencebuild.mk b/make/fencebuild.mk index 819ac36..1c4be6b 100644 --- a/make/fencebuild.mk +++ b/make/fencebuild.mk @@ -12,6 +12,10 @@ $(TARGET): $(SRC) -e 's#@''IPMITOOL_PATH@#${IPMITOOL_PATH}#g' \ -e 's#@''AMTTOOL_PATH@#${AMTTOOL_PATH}#g' \ -e 's#@''GNUTLSCLI_PATH@#${GNUTLSCLI_PATH}#g' \ + -e 's#@''COROSYNC_CMAPCTL_PATH@#${COROSYNC_CMAPCTL_PATH}#g' \ + -e 's#@''SG_PERSIST_PATH@#${SG_PERSIST_PATH}#g' \ + -e 's#@''SG_TURS_PATH@#${SG_TURS_PATH}#g' \ + -e 's#@''VGS_PATH@#${VGS_PATH}#g' \ > $@ if [ 0 -eq `echo "$(SRC)" | grep fence_ &> /dev/null; echo $$?` ]; then \ -- 1.8.3.1