Repository: airavata Updated Branches: refs/heads/develop a9d277167 -> cd230e275
adding queueSpecific macro support Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/cd230e27 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/cd230e27 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/cd230e27 Branch: refs/heads/develop Commit: cd230e275fe69b0ff936324ef658f6eedc796a49 Parents: a9d2771 Author: scnakandala <[email protected]> Authored: Mon Jun 26 13:11:10 2017 -0400 Committer: scnakandala <[email protected]> Committed: Mon Jun 26 13:11:10 2017 -0400 ---------------------------------------------------------------------- .../src/main/resources/PBS_Groovy.template | 1 + .../src/main/resources/SLURM_Groovy.template | 1 + .../apache/airavata/gfac/core/GFacUtils.java | 75 +++----------------- .../org/apache/airavata/gfac/core/Script.java | 1 + .../gfac/core/context/ProcessContext.java | 26 ++++--- 5 files changed, 29 insertions(+), 75 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/cd230e27/modules/configuration/server/src/main/resources/PBS_Groovy.template ---------------------------------------------------------------------- diff --git a/modules/configuration/server/src/main/resources/PBS_Groovy.template b/modules/configuration/server/src/main/resources/PBS_Groovy.template index 97c49eb..8e3f277 100644 --- a/modules/configuration/server/src/main/resources/PBS_Groovy.template +++ b/modules/configuration/server/src/main/resources/PBS_Groovy.template @@ -14,6 +14,7 @@ if (usedMem != null && usedMem != "") out.print '#PBS -l vmem=' + usedMem + 'M\n' if (nodes != null && nodes != "" && processPerNode != null && processPerNode != "") out.print '#PBS -l nodes=' + nodes + ':ppn=' + processPerNode + '\n' + if (queueSpecificMacros != null) for(queueMacro in queueSpecificMacros) out.print queueMacro +'\n' %> #PBS -m abe http://git-wip-us.apache.org/repos/asf/airavata/blob/cd230e27/modules/configuration/server/src/main/resources/SLURM_Groovy.template ---------------------------------------------------------------------- diff --git a/modules/configuration/server/src/main/resources/SLURM_Groovy.template b/modules/configuration/server/src/main/resources/SLURM_Groovy.template index bae646e..95596c7 100644 --- a/modules/configuration/server/src/main/resources/SLURM_Groovy.template +++ b/modules/configuration/server/src/main/resources/SLURM_Groovy.template @@ -13,6 +13,7 @@ if (queueName != null && queueName != "") out.print '#SBATCH -p ' + queueName + if (standardErrorFile != null && standardErrorFile != "") out.print '#SBATCH -e ' + standardErrorFile + '\n' if (qualityOfService != null && qualityOfService != "") out.print '#SBATCH --qos=' + qualityOfService + '\n' if (reservation != null && reservation != "") out.print '#SBATCH --reservation=' + reservation + '\n' + if (queueSpecificMacros != null) for(queueMacro in queueSpecificMacros) out.print queueMacro +'\n' %> #SBATCH --mail-type=ALL http://git-wip-us.apache.org/repos/asf/airavata/blob/cd230e27/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java index 358d6a1..1bb9e6f 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java @@ -23,12 +23,7 @@ import groovy.lang.Writable; import groovy.text.GStringTemplateEngine; import groovy.text.TemplateEngine; import org.apache.airavata.common.exception.ApplicationSettingsException; -import org.apache.airavata.common.utils.AiravataUtils; -import org.apache.airavata.common.utils.AiravataZKUtils; -import org.apache.airavata.common.utils.ApplicationSettings; -import org.apache.airavata.common.utils.DBUtil; -import org.apache.airavata.common.utils.ServerSettings; -import org.apache.airavata.common.utils.ZkConstants; +import org.apache.airavata.common.utils.*; import org.apache.airavata.credential.store.store.CredentialReader; import org.apache.airavata.credential.store.store.impl.CredentialReaderImpl; import org.apache.airavata.gfac.core.context.ProcessContext; @@ -36,54 +31,23 @@ import org.apache.airavata.gfac.core.context.TaskContext; import org.apache.airavata.messaging.core.MessageContext; import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription; import org.apache.airavata.model.appcatalog.appdeployment.CommandObject; -import org.apache.airavata.model.appcatalog.computeresource.CloudJobSubmission; -import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription; -import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionInterface; -import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol; -import org.apache.airavata.model.appcatalog.computeresource.LOCALSubmission; -import org.apache.airavata.model.appcatalog.computeresource.MonitorMode; -import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManager; -import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManagerType; -import org.apache.airavata.model.appcatalog.computeresource.SSHJobSubmission; -import org.apache.airavata.model.appcatalog.computeresource.UnicoreJobSubmission; +import org.apache.airavata.model.appcatalog.computeresource.*; import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference; import org.apache.airavata.model.application.io.DataType; import org.apache.airavata.model.application.io.InputDataObjectType; import org.apache.airavata.model.application.io.OutputDataObjectType; import org.apache.airavata.model.commons.ErrorModel; -import org.apache.airavata.model.data.replica.DataProductModel; -import org.apache.airavata.model.data.replica.DataProductType; -import org.apache.airavata.model.data.replica.DataReplicaLocationModel; -import org.apache.airavata.model.data.replica.ReplicaLocationCategory; -import org.apache.airavata.model.data.replica.ReplicaPersistentType; +import org.apache.airavata.model.data.replica.*; import org.apache.airavata.model.experiment.ExperimentModel; import org.apache.airavata.model.job.JobModel; -import org.apache.airavata.model.messaging.event.JobIdentifier; -import org.apache.airavata.model.messaging.event.JobStatusChangeEvent; -import org.apache.airavata.model.messaging.event.MessageType; -import org.apache.airavata.model.messaging.event.ProcessIdentifier; -import org.apache.airavata.model.messaging.event.ProcessStatusChangeEvent; -import org.apache.airavata.model.messaging.event.TaskIdentifier; -import org.apache.airavata.model.messaging.event.TaskStatusChangeEvent; +import org.apache.airavata.model.messaging.event.*; import org.apache.airavata.model.parallelism.ApplicationParallelismType; import org.apache.airavata.model.process.ProcessModel; import org.apache.airavata.model.scheduling.ComputationalResourceSchedulingModel; -import org.apache.airavata.model.status.JobStatus; -import org.apache.airavata.model.status.ProcessState; -import org.apache.airavata.model.status.ProcessStatus; -import org.apache.airavata.model.status.TaskState; -import org.apache.airavata.model.status.TaskStatus; +import org.apache.airavata.model.status.*; import org.apache.airavata.model.task.JobSubmissionTaskModel; import org.apache.airavata.registry.core.experiment.catalog.impl.RegistryFactory; -import org.apache.airavata.registry.cpi.AppCatalog; -import org.apache.airavata.registry.cpi.AppCatalogException; -import org.apache.airavata.registry.cpi.CompositeIdentifier; -import org.apache.airavata.registry.cpi.ExpCatChildDataType; -import org.apache.airavata.registry.cpi.ExperimentCatalog; -import org.apache.airavata.registry.cpi.ExperimentCatalogModelType; -import org.apache.airavata.registry.cpi.GwyResourceProfile; -import org.apache.airavata.registry.cpi.RegistryException; -import org.apache.airavata.registry.cpi.ReplicaCatalog; +import org.apache.airavata.registry.cpi.*; import org.apache.airavata.registry.cpi.utils.Constants; import org.apache.commons.io.FileUtils; import org.apache.curator.framework.CuratorFramework; @@ -100,35 +64,15 @@ import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathExpressionException; -import javax.xml.xpath.XPathFactory; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; +import javax.xml.xpath.*; +import java.io.*; import java.net.InetAddress; import java.net.URISyntaxException; import java.net.URL; import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.security.SecureRandom; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.TreeSet; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -501,6 +445,7 @@ public class GFacUtils { groovyMap.add(Script.GATEWAY_ID, processContext.getGatewayId()); groovyMap.add(Script.GATEWAY_USER_NAME, processContext.getProcessModel().getUserName()); groovyMap.add(Script.APPLICATION_NAME, processContext.getApplicationInterfaceDescription().getApplicationName()); + groovyMap.add(Script.QUEUE_SPECIFIC_MACROS, processContext.getQueueSpecificMacros()); groovyMap.add(Script.ACCOUNT_STRING, processContext.getAllocationProjectNumber()); groovyMap.add(Script.RESERVATION, processContext.getReservation()); http://git-wip-us.apache.org/repos/asf/airavata/blob/cd230e27/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java index 3ac5797..0937c7e 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java @@ -70,6 +70,7 @@ public enum Script { GATEWAY_ID("gatewayId"), GATEWAY_USER_NAME("gatewayUserName"), APPLICATION_NAME("applicationName"), + QUEUE_SPECIFIC_MACROS("queueSpecificMacros") ; String name; http://git-wip-us.apache.org/repos/asf/airavata/blob/cd230e27/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java index 5dd06db..6ca29e3 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java @@ -27,10 +27,7 @@ import org.apache.airavata.gfac.core.cluster.ServerInfo; import org.apache.airavata.messaging.core.Publisher; import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription; import org.apache.airavata.model.appcatalog.appinterface.ApplicationInterfaceDescription; -import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription; -import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol; -import org.apache.airavata.model.appcatalog.computeresource.MonitorMode; -import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManager; +import org.apache.airavata.model.appcatalog.computeresource.*; import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference; import org.apache.airavata.model.appcatalog.gatewayprofile.GatewayResourceProfile; import org.apache.airavata.model.appcatalog.gatewayprofile.StoragePreference; @@ -51,11 +48,7 @@ import org.apache.curator.framework.CuratorFramework; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; public class ProcessContext { @@ -108,6 +101,7 @@ public class ProcessContext { private SSHKeyAuthentication sshKeyAuthentication; private boolean recoveryWithCancel = false; private String usageReportingGatewayId; + private List<String> queueSpecificMacros; /** * Note: process context property use lazy loading approach. In runtime you will see some properties as null @@ -696,7 +690,19 @@ public class ProcessContext { } } - public static class ProcessContextBuilder{ + public List<String> getQueueSpecificMacros() { + String queueName = getProcessCRSchedule().getQueueName(); + Optional<BatchQueue> queue = getComputeResourceDescription().getBatchQueues().stream() + .filter(x->x.getQueueName().equals(queueName)).findFirst(); + if(queue.isPresent()){ + if(queue.get().getQueueSpecificMacros() != null && !queue.get().getQueueSpecificMacros().equals("")){ + return Arrays.asList(queue.get().getQueueSpecificMacros().split(",")); + } + } + return null; + } + + public static class ProcessContextBuilder{ private final String processId; private final String gatewayId; private final String tokenId;
