Repository: airavata Updated Branches: refs/heads/master 03ecceab1 -> 0f10016dd
Adding samples for ultrascan application. Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/0f10016d Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/0f10016d Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/0f10016d Branch: refs/heads/master Commit: 0f10016ddd1e61870f7b32de12f7c0278b2bbf15 Parents: 03eccea Author: raminder <[email protected]> Authored: Fri Jul 25 16:40:51 2014 -0400 Committer: raminder <[email protected]> Committed: Fri Jul 25 16:40:51 2014 -0400 ---------------------------------------------------------------------- .../samples/CreateLaunchExperimentUS3.java | 72 +++--- .../client/tools/RegisterUS3Application.java | 244 +++++++++++++++++++ 2 files changed, 279 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/0f10016d/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperimentUS3.java ---------------------------------------------------------------------- diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperimentUS3.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperimentUS3.java index ac1e7f5..25f27dc 100644 --- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperimentUS3.java +++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperimentUS3.java @@ -48,7 +48,7 @@ import org.slf4j.LoggerFactory; public class CreateLaunchExperimentUS3 { //FIXME: Read from a config file - public static final String THRIFT_SERVER_HOST = "gw111.iu.xsede.org"; + public static final String THRIFT_SERVER_HOST = "gridfarm005.ucs.indiana.edu"; public static final int THRIFT_SERVER_PORT = 8930; private final static Logger logger = LoggerFactory.getLogger(CreateLaunchExperiment.class); private static final String DEFAULT_USER = "default.registry.user"; @@ -59,10 +59,8 @@ public class CreateLaunchExperimentUS3 { final Airavata.Client airavata = AiravataClientFactory.createAiravataClient(THRIFT_SERVER_HOST, THRIFT_SERVER_PORT); System.out.println("API version is " + airavata.getAPIVersion()); // addDescriptors(); -// final String expId = createExperimentForTrestles(airavata); - final String expId = createUS3ExperimentForTrestles(airavata); -// final String expId = createExperimentForStampede(airavata); -// final String expId = createUS3ExperimentForStampede(airavata); +// final String expId = createUS3ExperimentForTrestles(airavata); + final String expId = createUS3ExperimentForStampede(airavata); System.out.println("Experiment ID : " + expId); launchExperiment(airavata, expId); System.out.println("Launched successfully"); @@ -202,30 +200,30 @@ public class CreateLaunchExperimentUS3 { List<DataObjectType> exOut = new ArrayList<DataObjectType>(); DataObjectType output = new DataObjectType(); output.setKey("output"); - output.setType(DataType.URI); - output.setValue(""); - DataObjectType output1 = new DataObjectType(); - output1.setKey("stdout"); - output1.setType(DataType.STDOUT); - output1.setValue(""); - DataObjectType output2 = new DataObjectType(); - output2.setKey("stderr"); - output2.setType(DataType.STDERR); - output2.setValue(""); +// output.setType(DataType.URI); + output.setValue(""); +// DataObjectType output1 = new DataObjectType(); +// output1.setKey("stdout"); +// output1.setType(DataType.STDOUT); +// output1.setValue(""); +// DataObjectType output2 = new DataObjectType(); +// output2.setKey("stderr"); +// output2.setType(DataType.STDERR); +// output2.setValue(""); exOut.add(output); - exOut.add(output1); - exOut.add(output2); +// exOut.add(output1); +// exOut.add(output2); - Project project = ProjectModelUtil.createProject("project1", "admin", "test project"); - String projectId = client.createProject(project); + // Project project = ProjectModelUtil.createProject("ultrascan", "ultrascan", "test project"); + //String projectId = client.createProject(project); - Experiment simpleExperiment = ExperimentModelUtil.createSimpleExperiment(projectId, "admin", "US3ExperimentTrestles", "US3AppTrestles", "US3AppTrestles", exInputs); + Experiment simpleExperiment = ExperimentModelUtil.createSimpleExperiment("ultrascan_41574ef5-b054-4d03-ab20-2cfe768d5096", "ultrascan", "US3ExperimentTrestles", "US3AppTrestles", "ultrascan_68d397d9-ffc2-470e-bdf7-8d7b4f1cab2e", exInputs); simpleExperiment.setExperimentOutputs(exOut); - ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling("trestles.sdsc.edu", 2, 32, 0, "shared", 0, 0, 0, "uot111"); + ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling("trestles.sdsc.xsede.org_fcf21cd7-d8ce-4359-bd7e-49062ce80265", 32, 2, 0, "shared", 30, 0, 0, "uot111"); UserConfigurationData userConfigurationData = new UserConfigurationData(); - scheduling.setResourceHostId("gsissh-trestles"); + scheduling.setResourceHostId("trestles.sdsc.xsede.org_fcf21cd7-d8ce-4359-bd7e-49062ce80265"); userConfigurationData.setAiravataAutoSchedule(false); userConfigurationData.setOverrideManualScheduledParams(false); @@ -264,27 +262,27 @@ public class CreateLaunchExperimentUS3 { output.setKey("output"); output.setType(DataType.URI); output.setValue(""); - DataObjectType output1 = new DataObjectType(); - output1.setKey("stdout"); - output1.setType(DataType.STDOUT); - output1.setValue(""); - DataObjectType output2 = new DataObjectType(); - output2.setKey("stderr"); - output2.setType(DataType.STDERR); - output2.setValue(""); +// DataObjectType output1 = new DataObjectType(); +// output1.setKey("stdout"); +// output1.setType(DataType.STDOUT); +// output1.setValue(""); +// DataObjectType output2 = new DataObjectType(); +// output2.setKey("stderr"); +// output2.setType(DataType.STDERR); +// output2.setValue(""); exOut.add(output); - exOut.add(output1); - exOut.add(output2); +// exOut.add(output1); +// exOut.add(output2); - Project project = ProjectModelUtil.createProject("project1", "admin", "test project"); - String projectId = client.createProject(project); +// Project project = ProjectModelUtil.createProject("project1", "admin", "test project"); +// String projectId = client.createProject(project); - Experiment simpleExperiment = ExperimentModelUtil.createSimpleExperiment(projectId, "admin", "US3ExperimentStampede", "US3AppStampede", "US3AppStampede", exInputs); + Experiment simpleExperiment = ExperimentModelUtil.createSimpleExperiment("ultrascan_41574ef5-b054-4d03-ab20-2cfe768d5096", "ultrascan", "US3ExperimentStampede", "US3AppStampede", "ultrascan_68d397d9-ffc2-470e-bdf7-8d7b4f1cab2e", exInputs); simpleExperiment.setExperimentOutputs(exOut); - ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling("stampede.tacc.xsede.org", 2, 32, 0, "normal", 0, 0, 0, "TG-MCB070039N"); + ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling("stampede.tacc.xsede.org_7e291aa8-319a-4c70-a3b5-b2e6f91c8f5d", 16, 2, 0, "normal", 30, 0, 0, "TG-MCB070039N"); - scheduling.setResourceHostId("gsissh-stampede"); + scheduling.setResourceHostId("stampede.tacc.xsede.org_7e291aa8-319a-4c70-a3b5-b2e6f91c8f5d"); UserConfigurationData userConfigurationData = new UserConfigurationData(); userConfigurationData.setAiravataAutoSchedule(false); http://git-wip-us.apache.org/repos/asf/airavata/blob/0f10016d/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterUS3Application.java ---------------------------------------------------------------------- diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterUS3Application.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterUS3Application.java new file mode 100644 index 0000000..1339e29 --- /dev/null +++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterUS3Application.java @@ -0,0 +1,244 @@ +package org.apache.airavata.client.tools; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.airavata.api.Airavata; +import org.apache.airavata.api.client.AiravataClientFactory; +import org.apache.airavata.model.appcatalog.appdeployment.ApplicationParallelismType; +import org.apache.airavata.model.appcatalog.appinterface.DataType; +import org.apache.airavata.model.appcatalog.appinterface.InputDataObjectType; +import org.apache.airavata.model.appcatalog.appinterface.OutputDataObjectType; +import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription; +import org.apache.airavata.model.appcatalog.computeresource.JobManagerCommand; +import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManager; +import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManagerType; +import org.apache.airavata.model.appcatalog.computeresource.SCPDataMovement; +import org.apache.airavata.model.appcatalog.computeresource.SSHJobSubmission; +import org.apache.airavata.model.appcatalog.computeresource.SecurityProtocol; +import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference; +import org.apache.airavata.model.appcatalog.gatewayprofile.GatewayResourceProfile; +import org.apache.airavata.model.error.AiravataClientException; +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RegisterUS3Application { + + private final static Logger logger = LoggerFactory.getLogger(RegisterSampleApplications.class); + + public static final String THRIFT_SERVER_HOST = "gridfarm005.ucs.indiana.edu"; + public static final int THRIFT_SERVER_PORT = 8930; + private static final String DEFAULT_GATEWAY = "default"; + private static Airavata.Client airavataClient; + + //Host Id's + private static String stampedeResourceId = "stampede.tacc.xsede.org_7e291aa8-319a-4c70-a3b5-b2e6f91c8f5d"; + private static String trestlesResourceId = "trestles.sdsc.xsede.org_fcf21cd7-d8ce-4359-bd7e-49062ce80265"; + private static String lonestarResourceId = "lonestar.tacc.teragrid.org_fe9b698a-4f68-459b-acc4-09c457a769b2"; + private static String alamoResourceId = "alamo.uthscsa.edu_2ff1f99d-565a-48c6-9f7d-a2dc5019d238"; + + private static String ultrascanModuleId = "ultrascan_c393abae-f75d-442f-8ad3-f0b59376131a"; + private static String ultrascanAppId = "ultrascan_68d397d9-ffc2-470e-bdf7-8d7b4f1cab2e"; + + public static void main(String[] args) { + try { + airavataClient = AiravataClientFactory.createAiravataClient(THRIFT_SERVER_HOST, THRIFT_SERVER_PORT); + System.out.println("API version is " + airavataClient.getAPIVersion()); + + //Register all compute hosts +// registerXSEDEHosts(); + + //Register module +// registerAppModules(); + + //Register Application +// registerUltrascanInterface(); + + //Register Deployment +// registerApplicationDeployment(); + registerGatewayResourceProfile(); + + } catch (Exception e) { + logger.error("Error while connecting with server", e.getMessage()); + e.printStackTrace(); + } + + } + public static void registerXSEDEHosts() { + try { + System.out.println("\n #### Registering XSEDE Computational Resources #### \n"); + + //Register Stampede + stampedeResourceId = registerComputeHost("stampede.tacc.xsede.org", "TACC Stampede Cluster", + ResourceJobManagerType.SLURM, "push", "/usr/bin", SecurityProtocol.GSI, 2222, null); + System.out.println("Stampede Resource Id is " + stampedeResourceId); + + //Register Trestles + trestlesResourceId = registerComputeHost("trestles.sdsc.xsede.org", "SDSC Trestles Cluster", + ResourceJobManagerType.PBS, "push", "/opt/torque/bin/", SecurityProtocol.GSI, 22, null); + System.out.println("Trestles Resource Id is " + trestlesResourceId); + + //Register Lonestar + lonestarResourceId = registerComputeHost("lonestar.tacc.teragrid.org", "TACC Lonestar Cluster", + ResourceJobManagerType.UGE, "push", "/opt/torque/bin/", SecurityProtocol.GSI, 22, null); + System.out.println("Lonestar Resource Id is " + lonestarResourceId); + + //Register Alamo + alamoResourceId = registerComputeHost("alamo.uthscsa.edu", "Alamo Cluster", + ResourceJobManagerType.PBS, "push", "/opt/torque/bin/", SecurityProtocol.SSH_KEYS, 22, null); + System.out.println("Alamo Cluster " + alamoResourceId); + + } catch (TException e) { + e.printStackTrace(); + } + + } + public static void registerAppModules() { + try { + System.out.println("\n #### Registering Application Modules #### \n"); + + //Register Echo + ultrascanModuleId = airavataClient.registerApplicationModule( + RegisterSampleApplicationsUtils.createApplicationModule( + "ultrascan", "1.0", "ultrascan application")); + System.out.println("Ultrascan Module Id " + ultrascanModuleId); + }catch(Exception e){ + e.getLocalizedMessage(); + } + } + public static void registerUltrascanInterface() { + try { + System.out.println("#### Registering WRF Interface #### \n"); + + List<String> appModules = new ArrayList<String>(); + appModules.add(ultrascanModuleId); + + InputDataObjectType input1 = RegisterSampleApplicationsUtils.createAppInput("input", null, + DataType.URI, null, false, "input tar file", null); + + List<InputDataObjectType> applicationInputs = new ArrayList<InputDataObjectType>(); + applicationInputs.add(input1); + + OutputDataObjectType output1 = RegisterSampleApplicationsUtils.createAppOutput("output", + "", DataType.URI); + +// OutputDataObjectType output2 = RegisterSampleApplicationsUtils.createAppOutput("stdout", +// "", DataType.URI); +// OutputDataObjectType output3 = RegisterSampleApplicationsUtils.createAppOutput("stderr", +// "", DataType.URI); + + List<OutputDataObjectType> applicationOutputs = new ArrayList<OutputDataObjectType>(); + applicationOutputs.add(output1); + + ultrascanAppId = airavataClient.registerApplicationInterface( + RegisterSampleApplicationsUtils.createApplicationInterfaceDescription("ultrascan", "ultrascan application", + appModules, applicationInputs, applicationOutputs)); + System.out.println("Ultrascan Application Interface Id " + ultrascanAppId); + + } catch (TException e) { + e.printStackTrace(); + } + } + + public static void registerApplicationDeployment() { + try { + System.out.println("#### Registering Application Deployments on Stampede #### \n"); + + // Register Stampede + String ultascanStamplede = airavataClient.registerApplicationDeployment(RegisterSampleApplicationsUtils.createApplicationDeployment(ultrascanModuleId, + stampedeResourceId, "/home1/01623/us3/bin/us_mpi_analysis", ApplicationParallelismType.MPI, "ultrascan application")); + System.out.println("Ultrascan on stampede deployment Id " + ultascanStamplede); + + String ultascanTrestles = airavataClient.registerApplicationDeployment(RegisterSampleApplicationsUtils.createApplicationDeployment(ultrascanModuleId, + trestlesResourceId, "/home/us3/trestles/bin/us_mpi_analysis", ApplicationParallelismType.MPI, "ultrascan application")); + System.out.println("Ultrascan on trestles deployment Id " + ultascanTrestles); + String ultascanLonestar = airavataClient.registerApplicationDeployment(RegisterSampleApplicationsUtils.createApplicationDeployment(ultrascanModuleId, + lonestarResourceId, "/home1/01623/us3/bin/us_mpi_analysis", ApplicationParallelismType.MPI, "ultrascan application")); + System.out.println("Ultrascan on lonestar deployment Id " + ultascanLonestar); + String ultascanAlamo = airavataClient.registerApplicationDeployment(RegisterSampleApplicationsUtils.createApplicationDeployment(ultrascanModuleId, + alamoResourceId, "/home/us3/bin/us_mpi_analysis.sh", ApplicationParallelismType.MPI, "ultrascan application")); + System.out.println("Ultrascan on alamo deployment Id " + ultascanAlamo); + } catch (Exception e) { + e.printStackTrace(); + } + } + public static void registerGatewayResourceProfile() { + + try { + System.out.println("#### Registering Gateway proflie #### \n"); + + + ComputeResourcePreference stampedeResourcePreferences = RegisterSampleApplicationsUtils. + createComputeResourcePreference(stampedeResourceId, "TG-MCB070039N", false, null, null, null, + "/scratch/01623/us3/jobs/"); + + ComputeResourcePreference trestlesResourcePreferences = RegisterSampleApplicationsUtils. + createComputeResourcePreference(trestlesResourceId, "uot111", false, null, null, null, + "/oasis/projects/nsf/uot111/us3/airavata-workdirs/"); + + ComputeResourcePreference lonestarResourcePreferences = RegisterSampleApplicationsUtils. + createComputeResourcePreference(lonestarResourceId, "ULTRASCAN", false, null, null, null, + "/scratch/01623/us3/airavata/"); + + ComputeResourcePreference alamoResourcePreferences = RegisterSampleApplicationsUtils. + createComputeResourcePreference(lonestarResourceId, null, false, null, null, null, + "/mnt/glusterfs/work/"); + + GatewayResourceProfile gatewayResourceProfile = new GatewayResourceProfile(); + gatewayResourceProfile.setGatewayID(DEFAULT_GATEWAY); + gatewayResourceProfile.setGatewayName(DEFAULT_GATEWAY); + gatewayResourceProfile.addToComputeResourcePreferences(stampedeResourcePreferences); + gatewayResourceProfile.addToComputeResourcePreferences(trestlesResourcePreferences); + gatewayResourceProfile.addToComputeResourcePreferences(lonestarResourcePreferences); + gatewayResourceProfile.addToComputeResourcePreferences(alamoResourcePreferences); + + String gatewayProfile = airavataClient.registerGatewayResourceProfile(gatewayResourceProfile); + System.out.println("Gateway Profile is registered with Id " + gatewayProfile); + + } catch (TException e) { + e.printStackTrace(); + } + } + public static String registerComputeHost(String hostName, String hostDesc, ResourceJobManagerType resourceJobManagerType, String monitoringEndPoint, + String jobMangerBinPath, SecurityProtocol securityProtocol, int portNumber, String jobManagerCommand) throws TException { + + ComputeResourceDescription computeResourceDescription = RegisterSampleApplicationsUtils + .createComputeResourceDescription(hostName, hostDesc, null, null); + + String computeResourceId = airavataClient.registerComputeResource(computeResourceDescription); + + if (computeResourceId.isEmpty()) + throw new AiravataClientException(); + + ResourceJobManager resourceJobManager = RegisterSampleApplicationsUtils.createResourceJobManager(resourceJobManagerType, monitoringEndPoint, + jobMangerBinPath, null); + + if (jobManagerCommand != null) { + Map<JobManagerCommand, String> jobManagerCommandStringMap = new HashMap<JobManagerCommand, String>(); + jobManagerCommandStringMap.put(JobManagerCommand.SUBMISSION, jobManagerCommand); + resourceJobManager.setJobManagerCommands(jobManagerCommandStringMap); + } + + SSHJobSubmission sshJobSubmission = new SSHJobSubmission(); + sshJobSubmission.setResourceJobManager(resourceJobManager); + sshJobSubmission.setSecurityProtocol(securityProtocol); + sshJobSubmission.setSshPort(portNumber); + boolean sshAddStatus = airavataClient.addSSHJobSubmissionDetails(computeResourceId, 1, sshJobSubmission); + + if (!sshAddStatus) + throw new AiravataClientException(); + + SCPDataMovement scpDataMovement = new SCPDataMovement(); + scpDataMovement.setSecurityProtocol(securityProtocol); + scpDataMovement.setSshPort(portNumber); + boolean scpAddStatus = airavataClient.addSCPDataMovementDetails(computeResourceId, 1, scpDataMovement); + + if (!scpAddStatus) + throw new AiravataClientException(); + + return computeResourceId; + } +}
