hi,
Here is a complete patch for opkg-torque that does the following:
1/ handle both torque-oscar and distro torque (handle both path /opt/pbs and
/usr)
2/ fix update_mom_config (handle multiple config path, fix psb_server config)
3/ api-post-deploy
- if pbs_server is not initialized, it creates the initial config
- if pbs_server is not running it starts it (restart fails)
- doesn't fail if workq is already defined (step 7 ran again)
TODO: torque is built with munge support on RHEL6 (need to handle munge config
if pbs_server --about|grep enable-munge-auth is non-empty)
TODO: since 2.5.4, torque is able to handle GPU nodes.
=> need to update Nodes table to store gpu_num (just like cpu_num)
=> need to populate Node table cpu_num and gpu_num fields.
=> need to update node creation with gpus=xxx from oscar database.
Best regards.
--
Olivier LAHAYE
CEA DRT/LIST/DCSI/DIR
Index: update_mom_config
===================================================================
--- update_mom_config (révision 9483)
+++ update_mom_config (copie de travail)
@@ -5,35 +5,46 @@
# Author: Jeremy Enos je...@ncsa.uiuc.edu
# Bernard Li <b...@bcgsc.ca>
+# Olivier Lahaye <olivier.lah...@cea.fr>
# Updates mom config file with real TORQUE server hostname and pushes it out.
#
+# Try to find the correct mom config file.
+mom_config_files="/etc/torque/mom/config /var/lib/torque/mom_priv/config /var/spool/pbs/mom_priv/config"
. /etc/profile
-mom_config=/var/spool/pbs/mom_priv/config
-sed_tmp=/tmp/mom_config_tmp
-if [ -f $mom_config ] ; then
- cat $mom_config |sed s/pbs_oscar/$HOSTNAME/g > $sed_tmp
- /bin/cp -f $sed_tmp $mom_config
- /bin/rm -f $sed_tmp
- echo "TORQUE mom config file updated with clienthost: $HOSTNAME"
- echo "Pushing config file to clients..."
- if [ -x `which cpush` ] ; then
- cpush $mom_config $mom_config
- else
- echo "Error: Can't find cpush utility."
- exit 1
+for mom_config in $mom_config_files
+do
+ if [ -f $mom_config ] ; then
+ # update the $pbsserver full line instead of replacing pbs_oscar
+ # this permits the use of distro torque package.
+ # if there is no $pbsserver line in the file, it doesn't fail and
+ # the $PBS_SERVER_HOME/server_name hostname is used instead.
+ sed -i -e "s/\$pbsserver.*$/\$pbsserver $HOSTNAME/g" $mom_config
+ echo "TORQUE mom config file $mom_config"
+ echo "updated with clienthost: $HOSTNAME"
+ echo "Pushing config file to clients..."
+ if [ -x `which cpush` ] ; then
+ cpush $mom_config $mom_config
+ else
+ echo "Error: Can't find cpush utility."
+ exit 1
+ fi
+ echo "Sending SIGHUP to all moms..."
+ if [ -x `which cexec` ] ; then
+ cexec killall -1 pbs_mom
+ else
+ echo "Error: Can't find cexec utility."
+ exit 1
+ fi
+ exit 0 # Config done, we can exit now
fi
- echo "Sending SIGHUP to all moms..."
- if [ -x `which cexec` ] ; then
- cexec killall -1 pbs_mom
- else
- echo "Error: Can't find cexec utility."
- exit 1
- fi
-else
- echo "Error: Not updating TORQUE mom config file."
- echo "$mom_config not found."
- exit 1
-fi
+done
+
+# Still here: we did not find any config file to update.
+echo "Error: Not updating TORQUE mom config file."
+echo "None of the following config file found:"
+echo "$mom_config_files"
+exit 1
+
Index: api-post-deploy
===================================================================
--- api-post-deploy (révision 9483)
+++ api-post-deploy (copie de travail)
@@ -33,6 +33,18 @@
my $default; # Did we type --default on the command line?
#########################################################################
+# Guess TORQUE home is it /usr or /opt/pbs ?
+#########################################################################
+
+my $torque_home;
+if (-f '/usr/bin/pbsnodes') {
+ $torque_home='/usr';
+} else {
+ $torque_home='/opt/pbs';
+}
+
+
+#########################################################################
# compactSpaces strips off the leading and trailing spaces from a #
# string. If you also pass in $compact=1, then it compresses multiple #
# spaces within the string down to 1 space. Also, you can pass in #
@@ -72,7 +84,7 @@
open(CMD,"/etc/init.d/pbs_server status |");
my $result = <CMD>;
close(CMD);
- system('/etc/init.d/pbs_server restart')
+ system('/etc/init.d/pbs_server start')
unless ($result =~ /is running/);
}
@@ -82,6 +94,21 @@
}
######################################################################
+# Check to see if workq is defined #
+######################################################################
+sub isWorkqDefined
+{
+ open(CMD,$torque_home."/bin/qmgr -c 'l q workq' |");
+ my $result = <CMD>;
+ close(CMD);
+ if($result =~ /^Queue workq/) {
+ return 1;
+ } else {
+ return 0;
+ }
+}
+
+######################################################################
# Check to see if 'Maui' is running. If not, then start it. #
######################################################################
sub restartMaui
@@ -130,7 +157,7 @@
opkg_print("Updating pbs_server nodes\n");
# Read in the current nodes as shown by 'pbsnodes -a'
-open (CMD,'/opt/pbs/bin/pbsnodes -a |');
+open (CMD,$torque_home.'/bin/pbsnodes -a |');
my $inp = "";
my $server = "";
my $opt = "";
@@ -189,9 +216,9 @@
$hostname = $nodes{$node}{HOST};
if ($pbsnodes{$hostname})
{
- system("/opt/pbs/bin/pbsnodes -o $hostname");
+ system($torque_home."/bin/pbsnodes -o $hostname");
# Reset the number of processors for this node
- system('/opt/pbs/bin/qmgr -a -e -c "set node ' . $hostname .
+ system($torque_home.'/bin/qmgr -a -e -c "set node ' . $hostname .
' np = ' . $nodes{$node}{NUM_PROCS} . '"');
# Search the properties for 'all' and add it if not found
my(@props) = split(/,/,$pbsnodes{$hostname}{'properties '});
@@ -200,15 +227,15 @@
{
$alldefined = 1 if $prop eq 'all';
}
- system('/opt/pbs/bin/qmgr -a -e -c "set node ' . $hostname .
+ system($torque_home.'/bin/qmgr -a -e -c "set node ' . $hostname .
' properties += all"') unless $alldefined;
- system("/opt/pbs/bin/pbsnodes -c $hostname");
+ system($torque_home."/bin/pbsnodes -c $hostname");
# Remove from pbsnodes hash
delete $pbsnodes{$nodes{$node}{HOST}};
}
else
{ # Didn't find a match -> ADD the node
- system('/opt/pbs/bin/qmgr -a -e -c "create node ' . $hostname .
+ system($torque_home.'/bin/qmgr -a -e -c "create node ' . $hostname .
' np = ' . $nodes{$node}{NUM_PROCS} . ' , properties = all"');
}
@@ -221,102 +248,107 @@
# Now go through the remaining pbsnodes hash and delete these nodes
foreach my $node (sortnodes( keys %pbsnodes ))
{
- system('/opt/pbs/bin/qmgr -a -e -c "delete node ' . $node . '"');
+ system($torque_home.'/bin/qmgr -a -e -c "delete node ' . $node . '"');
}
restartPBSserver();
-# Next, use qmgr command to set up the values for workq
-opkg_print("Creating TORQUE workq queue...\n");
+if(isWorkqDefined()) {
+ opkg_print("workq already defined...Skipping queue creation...\n");
+} else {
+ # Next, use qmgr command to set up the values for workq
+ opkg_print("Creating TORQUE workq queue...\n");
-# These are default values set only when not present or when --default
-my @default_params = (
- 'create queue workq',
- 'set queue workq queue_type = Execution',
- 'set queue workq resources_max.cput = 10000:00:00',
- 'set queue workq resources_max.walltime = 10000:00:00',
- 'set queue workq resources_min.cput = 00:00:01',
- 'set queue workq resources_min.ncpus = 1',
- 'set queue workq resources_min.nodect = 1',
- 'set queue workq resources_min.walltime = 00:00:01',
- 'set queue workq resources_default.cput = 10000:00:00',
- 'set queue workq resources_default.ncpus = 1',
- 'set queue workq resources_default.nodect = 1',
- 'set queue workq resources_default.walltime = 10000:00:00',
- 'set queue workq enabled = True',
- 'set queue workq started = True',
- 'set server scheduling = True',
- 'set server default_queue = workq',
- 'set server mail_from = adm',
- 'set server query_other_jobs = True',
-);
+ # These are default values set only when not present or when --default
+ my @default_params = (
+ 'create queue workq',
+ 'set queue workq queue_type = Execution',
+ 'set queue workq resources_max.cput = 10000:00:00',
+ 'set queue workq resources_max.walltime = 10000:00:00',
+ 'set queue workq resources_min.cput = 00:00:01',
+ 'set queue workq resources_min.ncpus = 1',
+ 'set queue workq resources_min.nodect = 1',
+ 'set queue workq resources_min.walltime = 00:00:01',
+ 'set queue workq resources_default.cput = 10000:00:00',
+ 'set queue workq resources_default.ncpus = 1',
+ 'set queue workq resources_default.nodect = 1',
+ 'set queue workq resources_default.walltime = 10000:00:00',
+ 'set queue workq enabled = True',
+ 'set queue workq started = True',
+ 'set server scheduling = True',
+ 'set server default_queue = workq',
+ 'set server mail_from = adm',
+ 'set server query_other_jobs = True',
+ );
-# These are variable parameters that are set everytime
-my @variable_params = (
- "set queue workq resources_max.ncpus = $TOT_NP",
- "set queue workq resources_max.nodect = $TOT_NODES",
- "set queue workq resources_available.nodect = $TOT_NODES",
- "set server resources_available.ncpus = $TOT_NP",
- "set server resources_available.nodect = $TOT_NODES",
- "set server resources_available.nodes = $TOT_NODES",
- "set server resources_max.ncpus = $TOT_NP",
- "set server resources_max.nodes = $TOT_NODES",
- "set server scheduler_iteration = 60",
- "set server log_events = $loglevel",
-);
+ # These are variable parameters that are set everytime
+ my @variable_params = (
+ "set queue workq resources_max.ncpus = $TOT_NP",
+ "set queue workq resources_max.nodect = $TOT_NODES",
+ "set queue workq resources_available.nodect = $TOT_NODES",
+ "set server resources_available.ncpus = $TOT_NP",
+ "set server resources_available.nodect = $TOT_NODES",
+ "set server resources_available.nodes = $TOT_NODES",
+ "set server resources_max.ncpus = $TOT_NP",
+ "set server resources_max.nodes = $TOT_NODES",
+ "set server scheduler_iteration = 60",
+ "set server log_events = $loglevel",
+ );
-# First, read in the current values for the localhost
-undef $/; # Suck in the entire files into one big string
-open (CMD,'/opt/pbs/bin/qmgr -a -e -c "print server @localhost" |');
-my $cmd_output = <CMD>;
-close (CMD);
+ # First, read in the current values for the localhost
+ undef $/; # Suck in the entire files into one big string
+ open (CMD,$torque_home.'/bin/qmgr -a -e -c "print server @localhost" |');
+ my $cmd_output = <CMD>;
+ close (CMD);
-# Next, build up a command string to pass back to the qmgr command.
-my $line = "";
-my $key = "";
-my $value = "";
-my $cmd_input = "";
+ # Next, build up a command string to pass back to the qmgr command.
+ my $line = "";
+ my $key = "";
+ my $value = "";
+ my $cmd_input = "";
-# Search through the default_params for anything missing.
-foreach $line (@default_params)
- {
- ($key,$value) = split /=/, $line;
- $key = compactSpaces($key);
- $value = compactSpaces($value);
- if ($value eq "")
- { # Search for just the $key since there's no value for it.
- $cmd_input .= "$key\n" if
- ($cmd_output !~ /$key/m) ||
- (defined($default) && $key !~ /create/);
- }
- else
- { # Search for "$key =". Output key and value if needed.
- $cmd_input .= "$key = $value\n" if
- ($cmd_output !~ /$key\s*=/m) || defined($default);
- }
- }
+ # Search through the default_params for anything missing.
+ foreach $line (@default_params)
+ {
+ ($key,$value) = split /=/, $line;
+ $key = compactSpaces($key);
+ $value = compactSpaces($value);
+ if ($value eq "")
+ { # Search for just the $key since there's no value for it.
+ $cmd_input .= "$key\n" if
+ ($cmd_output !~ /$key/m) ||
+ (defined($default) && $key !~ /create/);
+ }
+ else
+ { # Search for "$key =". Output key and value if needed.
+ $cmd_input .= "$key = $value\n" if
+ ($cmd_output !~ /$key\s*=/m) || defined($default);
+ }
+ }
-# Then, force the setting of the of the variable_params.
-foreach $line (@variable_params)
- {
- ($key,$value) = split /=/, $line;
- $key = compactSpaces($key);
- $value = compactSpaces($value);
- $cmd_input .= "$key";
- $cmd_input .= " = $value"
- if ($value ne "");
- $cmd_input .= "\n";
- }
+ # Then, force the setting of the of the variable_params.
+ foreach $line (@variable_params)
+ {
+ ($key,$value) = split /=/, $line;
+ $key = compactSpaces($key);
+ $value = compactSpaces($value);
+ $cmd_input .= "$key";
+ $cmd_input .= " = $value"
+ if ($value ne "");
+ $cmd_input .= "\n";
+ }
-# Run the qmgr command if necessary
-if ($cmd_input ne "")
- {
- my $pbs_spool = "/var/spool/pbs";
- open (CMD,"|/opt/pbs/bin/qmgr -a -e ");
- print CMD $cmd_input;
- close (CMD)
- or croak("ERROR: Impossible to Configure TORQUE queues, ".
- "check the logs at $pbs_spool");
- }
+ # Run the qmgr command if necessary
+ if ($cmd_input ne "")
+ {
+ my $pbs_server_logs = "/var/log/torque/server_logs";
+ open (CMD,"|".$torque_home."/bin/qmgr -a -e ");
+ print CMD $cmd_input;
+ close (CMD)
+ or croak("ERROR: Impossible to Configure TORQUE queues, ".
+ "check the logs at $pbs_server_logs");
+ }
+}
+
restartMaui();
# Get current state of server's mom
Index: server-post-install
===================================================================
--- server-post-install (révision 9483)
+++ server-post-install (copie de travail)
@@ -37,6 +37,9 @@
#######################################################
sub startPBSserver
{
+ if(!-f '/var/lib/torque/server_priv/serverdb' && !-f '/var/spool/pbs/server_priv/serverdb') {
+ system('/etc/init.d/pbs_server create'); # no config file: need to initialise config.
+ }
system('/etc/init.d/pbs_server restart');
}
@@ -68,10 +71,20 @@
my $xpbsmonrc;
# Attention: More sophistication needed here.
+# Check for 32/64 bit architecture AND
+# Check if using /opt located oscar torque package or distro torque package
if ($arch eq "x86_64") {
- $xpbsmonrc = "/opt/pbs/lib64/xpbsmon/xpbsmonrc";
+ if (-f "/usr/lib64/xpbsmon/xpbsmonrc") {
+ $xpbsmonrc = "/usr/lib64/xpbsmon/xpbsmonrc";
+ } else {
+ $xpbsmonrc = "/opt/pbs/lib64/xpbsmon/xpbsmonrc";
+ }
} else {
- $xpbsmonrc = "/opt/pbs/lib/xpbsmon/xpbsmonrc";
+ if (-f "/usr/lib/xpbsmon/xpbsmonrc") {
+ $xpbsmonrc = "/usr/lib/xpbsmon/xpbsmonrc";
+ } else {
+ $xpbsmonrc = "/opt/pbs/lib/xpbsmon/xpbsmonrc";
+ }
}
# Eventually 'torque_gui' is not installed, so we cannot configure xpbsmonrc
------------------------------------------------------------------------------
Everyone hates slow websites. So do we.
Make your web apps faster with AppDynamics
Download AppDynamics Lite for free today:
http://p.sf.net/sfu/appdyn_sfd2d_oct
_______________________________________________
Oscar-devel mailing list
Oscar-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/oscar-devel
