Repository: airavata Updated Branches: refs/heads/master 532f2691e -> c57378bda
More changes to slurm implementation - AIRAVATA-1057 Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/0315de48 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/0315de48 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/0315de48 Branch: refs/heads/master Commit: 0315de482a079dc97d1a128f08d554629ad47721 Parents: 532f269 Author: lahiru <[email protected]> Authored: Thu Mar 6 22:27:38 2014 -0500 Committer: lahiru <[email protected]> Committed: Thu Mar 6 22:27:38 2014 -0500 ---------------------------------------------------------------------- .../client/samples/CreateLaunchExperiment.java | 12 ++-- .../airavata/job/monitor/AMQPMonitorTest.java | 4 +- .../airavata/job/monitor/QstatMonitorTest.java | 3 +- .../src/main/assembly/bin-assembly.xml | 1 + .../src/main/resources/SLURMTemplate.xslt | 69 ++++++++++++++++++++ .../org/apache/airavata/gfac/cpi/GFacImpl.java | 16 +++-- .../core/gfac/services/impl/BigRed2Test.java | 7 +- .../gfac/services/impl/GSISSHProviderTest.java | 3 +- 8 files changed, 98 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/0315de48/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java ---------------------------------------------------------------------- diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java index 63184d3..3bad86b 100644 --- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java +++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java @@ -123,7 +123,7 @@ public class CreateLaunchExperiment { DocumentCreator documentCreator = new DocumentCreator(getAiravataAPI()); documentCreator.createLocalHostDocs(); documentCreator.createGramDocs(); - documentCreator.createGSISSHDocs(); + documentCreator.createSlurmDocs(); } catch (AiravataAPIInvocationException e) { logger.error("Unable to create airavata API", e.getMessage()); throw new AiravataAPIInvocationException(e); @@ -165,11 +165,13 @@ public class CreateLaunchExperiment { output.setValue(""); exOut.add(output); - Experiment simpleExperiment = ExperimentModelUtil.createSimpleExperiment("project1", "admin", "echoExperiment", "SimpleEcho2", "SimpleEcho2", exInputs); + Experiment simpleExperiment = + ExperimentModelUtil.createSimpleExperiment("project1", "admin", "echoExperiment", "SimpleEcho2", "SimpleEcho2", exInputs); simpleExperiment.setExperimentOutputs(exOut); - ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling("trestles.sdsc.edu", 1, 1, 1, "normal", 0, 0, 1, "sds128"); - scheduling.setResourceHostId("gsissh-trestles"); + ComputationalResourceScheduling scheduling = + ExperimentModelUtil.createComputationResourceScheduling("stampede.tacc.xsede.org", 1, 1, 1, "normal", 0, 0, 1, "TG-STA110014S"); + scheduling.setResourceHostId("stampede-host"); UserConfigurationData userConfigurationData = new UserConfigurationData(); userConfigurationData.setAiravataAutoSchedule(false); userConfigurationData.setOverrideManualScheduledParams(false); @@ -192,7 +194,7 @@ public class CreateLaunchExperiment { } public static void launchExperiment (Airavata.Client client, String expId) - throws ExperimentNotFoundException, AiravataSystemException, InvalidRequestException,AiravataClientException, TException{ + throws TException{ try { client.launchExperiment(expId, "testToken"); } catch (ExperimentNotFoundException e) { http://git-wip-us.apache.org/repos/asf/airavata/blob/0315de48/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/AMQPMonitorTest.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/AMQPMonitorTest.java b/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/AMQPMonitorTest.java index 8f920c1..a6ef7ea 100644 --- a/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/AMQPMonitorTest.java +++ b/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/AMQPMonitorTest.java @@ -31,6 +31,7 @@ import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo import org.apache.airavata.job.monitor.exception.AiravataMonitorException; import org.apache.airavata.job.monitor.impl.push.amqp.AMQPMonitor; import org.apache.airavata.schemas.gfac.GsisshHostType; +import org.apache.axiom.om.util.CommonUtils; import org.junit.Before; import org.junit.Test; @@ -93,7 +94,8 @@ public class AMQPMonitorTest { ServerInfo serverInfo = new ServerInfo("ogce", "trestles.sdsc.edu"); - Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, "/opt/torque/bin/"); + Cluster pbsCluster = new + PBSCluster(serverInfo, authenticationInfo, org.apache.airavata.gsi.ssh.util.CommonUtils.getPBSJobManager("/opt/torque/bin/")); // Execute command http://git-wip-us.apache.org/repos/asf/airavata/blob/0315de48/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/QstatMonitorTest.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/QstatMonitorTest.java b/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/QstatMonitorTest.java index 6b1074a..aabc0d7 100644 --- a/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/QstatMonitorTest.java +++ b/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/QstatMonitorTest.java @@ -28,6 +28,7 @@ import org.apache.airavata.gsi.ssh.api.authentication.GSIAuthenticationInfo; import org.apache.airavata.gsi.ssh.api.job.JobDescriptor; import org.apache.airavata.gsi.ssh.impl.PBSCluster; import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo; +import org.apache.airavata.gsi.ssh.util.CommonUtils; import org.apache.airavata.job.monitor.exception.AiravataMonitorException; import org.apache.airavata.job.monitor.impl.pull.qstat.QstatMonitor; import org.apache.airavata.schemas.gfac.GsisshHostType; @@ -92,7 +93,7 @@ public class QstatMonitorTest { ServerInfo serverInfo = new ServerInfo("ogce", hostDescription.getType().getHostAddress()); - Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, "/opt/torque/bin/"); + Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, CommonUtils.getPBSJobManager("/opt/torque/bin/")); // Execute command http://git-wip-us.apache.org/repos/asf/airavata/blob/0315de48/modules/distribution/airavata-server/src/main/assembly/bin-assembly.xml ---------------------------------------------------------------------- diff --git a/modules/distribution/airavata-server/src/main/assembly/bin-assembly.xml b/modules/distribution/airavata-server/src/main/assembly/bin-assembly.xml index 9b9f67c..8509c77 100644 --- a/modules/distribution/airavata-server/src/main/assembly/bin-assembly.xml +++ b/modules/distribution/airavata-server/src/main/assembly/bin-assembly.xml @@ -126,6 +126,7 @@ <include>provenance.sql</include> <include>gfac-config.xml</include> <include>PBSTemplate.xslt</include> + <include>SLURMTemplate.xslt</include> <include>gsissh.properties</include> </includes> </fileSet> http://git-wip-us.apache.org/repos/asf/airavata/blob/0315de48/modules/distribution/airavata-server/src/main/resources/SLURMTemplate.xslt ---------------------------------------------------------------------- diff --git a/modules/distribution/airavata-server/src/main/resources/SLURMTemplate.xslt b/modules/distribution/airavata-server/src/main/resources/SLURMTemplate.xslt new file mode 100644 index 0000000..169dbbf --- /dev/null +++ b/modules/distribution/airavata-server/src/main/resources/SLURMTemplate.xslt @@ -0,0 +1,69 @@ +<!--Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file + distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under + the Apache License, Version 2.0 (theà "License"); you may not use this file except in compliance with the License. You may + obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to + in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF + ANY ~ KIND, either express or implied. See the License for the specific language governing permissions and limitations under + the License. --> +<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"; xmlns:ns="http://airavata.apache.org/gsi/ssh/2012/12";> +<xsl:output method="text" /> +<xsl:template match="/ns:JobDescriptor"> +<xsl:choose> +<xsl:when test="ns:shellName"> +#!<xsl:value-of select="ns:shellName"/> + </xsl:when> + </xsl:choose> +<xsl:choose> + <xsl:when test="ns:queueName"> +#SBATCH -p <xsl:value-of select="ns:queueName"/> + </xsl:when> + </xsl:choose> +<xsl:choose> + <xsl:when test="ns:nodes"> +#SBATCH -n <xsl:value-of select="ns:nodes"/> + </xsl:when> + </xsl:choose> + <xsl:choose> + <xsl:when test="ns:mailAddress"> +#SBATCH -mail-user=<xsl:value-of select="ns:mailAddress"/> + </xsl:when> + </xsl:choose> + <xsl:choose> + <xsl:when test="ns:mailType"> +#SBATCH -mail-type=<xsl:value-of select="ns:mailType"/> + </xsl:when> + </xsl:choose> + <xsl:choose> +<xsl:when test="ns:acountString"> +#SBATCH -A <xsl:value-of select="ns:acountString"/> + </xsl:when> + </xsl:choose> + <xsl:choose> + <xsl:when test="ns:maxWallTime"> +#SBATCH -t <xsl:value-of select="ns:maxWallTime"/> + </xsl:when> + </xsl:choose> + <xsl:choose> + <xsl:when test="ns:jobName"> +#SBATCH -J <xsl:value-of select="ns:jobName"/> + </xsl:when> + </xsl:choose> + <xsl:choose> + <xsl:when test="ns:standardOutFile"> +#SBATCH -o <xsl:value-of select="ns:standardOutFile"/> + </xsl:when> + </xsl:choose> + <xsl:choose> + <xsl:when test="ns:standardOutFile"> +#SBATCH -e <xsl:value-of select="ns:standardErrorFile"/> + </xsl:when> + </xsl:choose> +cd <xsl:text> </xsl:text><xsl:value-of select="ns:workingDirectory"/><xsl:text>
</xsl:text> + <xsl:choose><xsl:when test="ns:jobSubmitterCommand"> +<xsl:value-of select="ns:jobSubmitterCommand"/><xsl:text> </xsl:text></xsl:when></xsl:choose><xsl:value-of select="ns:executablePath"/><xsl:text> </xsl:text> +<xsl:for-each select="ns:inputs/ns:input"> + <xsl:value-of select="."/><xsl:text> </xsl:text> + </xsl:for-each> +</xsl:template> + +</xsl:stylesheet> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/0315de48/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/cpi/GFacImpl.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/cpi/GFacImpl.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/cpi/GFacImpl.java index 8988554..47dee57 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/cpi/GFacImpl.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/cpi/GFacImpl.java @@ -375,7 +375,9 @@ public class GFacImpl implements GFac { GSIAuthenticationInfo authenticationInfo = new MyProxyAuthenticationInfo(requestData.getMyProxyUserName(), requestData.getMyProxyPassword(), requestData.getMyProxyServerUrl(), requestData.getMyProxyPort(), requestData.getMyProxyLifeTime(), System.getProperty(Constants.TRUSTED_CERTIFICATE_SYSTEM_PROPERTY)); - ServerInfo serverInfo = new ServerInfo(requestData.getMyProxyUserName(), registeredHost.getType().getHostAddress()); + GsisshHostType gsisshHostType = (GsisshHostType)registeredHost.getType(); + ServerInfo serverInfo = new ServerInfo(requestData.getMyProxyUserName(), registeredHost.getType().getHostAddress(), + gsisshHostType.getPort()); Cluster pbsCluster = null; try { @@ -383,17 +385,17 @@ public class GFacImpl implements GFac { String installedParentPath = ((HpcApplicationDeploymentType) jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType()).getInstalledParentPath(); String jobManager = ((GsisshHostType) registeredHost.getType()).getJobManager(); - if(jobManager == null){ - log.error("No Job Manager is configured, so we are picking pbs as the default job manager"); + if (jobManager == null) { + log.error("No Job Manager is configured, so we are picking pbs as the default job manager"); jConfig = CommonUtils.getPBSJobManager(installedParentPath); - }else{ - if("pbs".equalsIgnoreCase(jobManager)){ + } else { + if ("pbs".equalsIgnoreCase(jobManager)) { jConfig = CommonUtils.getPBSJobManager(installedParentPath); - }else if("slurm".equalsIgnoreCase(jobManager)){ + } else if ("slurm".equalsIgnoreCase(jobManager)) { jConfig = CommonUtils.getSLURMJobManager(installedParentPath); } } - pbsCluster = new PBSCluster(serverInfo, authenticationInfo,jConfig); + pbsCluster = new PBSCluster(serverInfo, authenticationInfo, jConfig); } catch (SSHApiException e) { e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. } http://git-wip-us.apache.org/repos/asf/airavata/blob/0315de48/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2Test.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2Test.java b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2Test.java index bca3b2b..feeefe2 100644 --- a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2Test.java +++ b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2Test.java @@ -36,8 +36,11 @@ import org.apache.airavata.gsi.ssh.api.Cluster; import org.apache.airavata.gsi.ssh.api.SSHApiException; import org.apache.airavata.gsi.ssh.api.ServerInfo; import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo; +import org.apache.airavata.gsi.ssh.api.job.JobManagerConfiguration; +import org.apache.airavata.gsi.ssh.api.job.PBSJobConfiguration; import org.apache.airavata.gsi.ssh.impl.PBSCluster; import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPasswordAuthenticationInfo; +import org.apache.airavata.gsi.ssh.util.CommonUtils; import org.apache.airavata.schemas.gfac.*; import org.junit.Before; import org.junit.Test; @@ -189,8 +192,8 @@ public class BigRed2Test { Cluster pbsCluster = null; try { - pbsCluster = new PBSCluster(serverInfo, authenticationInfo, - (app.getInstalledParentPath())); + JobManagerConfiguration pbsJobManager = CommonUtils.getPBSJobManager(app.getInstalledParentPath()); + pbsCluster = new PBSCluster(serverInfo, authenticationInfo,pbsJobManager); } catch (SSHApiException e) { e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. } http://git-wip-us.apache.org/repos/asf/airavata/blob/0315de48/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTest.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTest.java b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTest.java index abc41b9..1041540 100644 --- a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTest.java +++ b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTest.java @@ -39,6 +39,7 @@ import org.apache.airavata.gsi.ssh.api.ServerInfo; import org.apache.airavata.gsi.ssh.api.authentication.GSIAuthenticationInfo; import org.apache.airavata.gsi.ssh.impl.PBSCluster; import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo; +import org.apache.airavata.gsi.ssh.util.CommonUtils; import org.apache.airavata.schemas.gfac.*; import org.junit.Before; import org.junit.Test; @@ -191,7 +192,7 @@ public class GSISSHProviderTest { ServerInfo serverInfo = new ServerInfo("ogce", "trestles.sdsc.edu"); Cluster pbsCluster = null; try { - pbsCluster = new PBSCluster(serverInfo, authenticationInfo, app.getInstalledParentPath()); + pbsCluster = new PBSCluster(serverInfo, authenticationInfo, CommonUtils.getPBSJobManager(app.getInstalledParentPath())); } catch (SSHApiException e) { e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. }
