Repository: airavata Updated Branches: refs/heads/master f5a2eac62 -> e0df621b6
Create and launch sample now independent; better file output handling; some bits of us3 and juropa testing Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/e0df621b Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/e0df621b Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/e0df621b Branch: refs/heads/master Commit: e0df621b6f4af5147824d4c5d61d155b134828ff Parents: f5a2eac Author: msmemon <[email protected]> Authored: Mon Mar 30 12:16:20 2015 +0200 Committer: msmemon <[email protected]> Committed: Mon Mar 30 12:16:20 2015 +0200 ---------------------------------------------------------------------- .../client/samples/CreateLaunchBES.java | 487 +++++++++++++++++++ .../tools/RegisterSampleApplications.java | 38 +- .../tools/RegisterSampleApplicationsUtils.java | 3 +- .../gfac/bes/utils/ApplicationProcessor.java | 58 ++- .../airavata/gfac/bes/utils/BESConstants.java | 4 + .../gfac/bes/utils/DataTransferrer.java | 7 +- .../gfac/bes/utils/ResourceProcessor.java | 8 +- .../gfac/bes/utils/UASDataStagingProcessor.java | 5 +- 8 files changed, 588 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/e0df621b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchBES.java ---------------------------------------------------------------------- diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchBES.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchBES.java new file mode 100644 index 0000000..44ebfc7 --- /dev/null +++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchBES.java @@ -0,0 +1,487 @@ +package org.apache.airavata.client.samples; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.apache.airavata.api.Airavata; +import org.apache.airavata.api.client.AiravataClientFactory; +import org.apache.airavata.client.tools.RegisterSampleApplications; +import org.apache.airavata.client.tools.RegisterSampleApplicationsUtils; +import org.apache.airavata.model.appcatalog.appinterface.InputDataObjectType; +import org.apache.airavata.model.appcatalog.appinterface.OutputDataObjectType; +import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription; +import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionInterface; +import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol; +import org.apache.airavata.model.appcatalog.computeresource.SecurityProtocol; +import org.apache.airavata.model.appcatalog.computeresource.UnicoreJobSubmission; +import org.apache.airavata.model.error.AiravataClientException; +import org.apache.airavata.model.error.AiravataErrorType; +import org.apache.airavata.model.error.AiravataSystemException; +import org.apache.airavata.model.error.ExperimentNotFoundException; +import org.apache.airavata.model.error.InvalidRequestException; +import org.apache.airavata.model.util.ExperimentModelUtil; +import org.apache.airavata.model.workspace.Gateway; +import org.apache.airavata.model.workspace.Project; +import org.apache.airavata.model.workspace.experiment.AdvancedOutputDataHandling; +import org.apache.airavata.model.workspace.experiment.ComputationalResourceScheduling; +import org.apache.airavata.model.workspace.experiment.ErrorDetails; +import org.apache.airavata.model.workspace.experiment.Experiment; +import org.apache.airavata.model.workspace.experiment.ExperimentSummary; +import org.apache.airavata.model.workspace.experiment.UserConfigurationData; +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CreateLaunchBES { + + public static final String THRIFT_SERVER_HOST = "localhost"; + public static final int THRIFT_SERVER_PORT = 8930; + + private final static Logger logger = LoggerFactory.getLogger(CreateLaunchExperiment.class); + private static final String DEFAULT_USER = "default.registry.user"; + private static final String DEFAULT_GATEWAY = "php_reference_gateway"; + private static Airavata.Client airavataClient; + + private static String echoAppId = "Echo_5dd52cd4-f9a0-459f-9baf-f8e715e44548"; + private static String mpiAppId = "HelloMPI_f0bb3b56-914e-4752-bb7b-dd18ef9dce00"; + + private static String unicoreHostName = "fsd-cloud15.zam.kfa-juelich.de"; + + private static String gatewayId; + + // unicore service endpoint url + private static final String unicoreEndPointURL = "https://deisa-unic.fz-juelich.de:9111/FZJ_JUROPA/services/BESFactory?res=default_bes_factory";; +// private static final String unicoreEndPointURL = "https://fsd-cloud15.zam.kfa-juelich.de:7000/INTEROP1/services/BESFactory?res=default_bes_factory";; + + public static void main(String[] args) throws Exception { + airavataClient = AiravataClientFactory.createAiravataClient(THRIFT_SERVER_HOST, THRIFT_SERVER_PORT); + System.out.println("API version is " + airavataClient.getAPIVersion()); +// createGateway(); +// getGateway("testGatewayId"); +// registerApplications(); // run this only the first time + createAndLaunchExp(); + } + + private static String fsdResourceId; + + + public static void getAvailableAppInterfaceComputeResources(String appInterfaceId) { + try { + Map<String, String> availableAppInterfaceComputeResources = airavataClient.getAvailableAppInterfaceComputeResources(appInterfaceId); + for (String key : availableAppInterfaceComputeResources.keySet()) { + System.out.println("id : " + key); + System.out.println("name : " + availableAppInterfaceComputeResources.get(key)); + } + } catch (AiravataSystemException e) { + e.printStackTrace(); + } catch (InvalidRequestException e) { + e.printStackTrace(); + } catch (AiravataClientException e) { + e.printStackTrace(); + } catch (TException e) { + e.printStackTrace(); + } + + } + + + public static void createGateway() { + try { + Gateway gateway = new Gateway(); + gateway.setGatewayId("testGatewayId2"); + gateway.setGatewayName("testGateway2"); + gatewayId = airavataClient.addGateway(gateway); + System.out.println(gatewayId); + } catch (AiravataSystemException e) { + e.printStackTrace(); + } catch (InvalidRequestException e) { + e.printStackTrace(); + } catch (AiravataClientException e) { + e.printStackTrace(); + } catch (TException e) { + e.printStackTrace(); + } + + } + + public static void getGateway(String gatewayId) { + try { + Gateway gateway = airavataClient.getGateway(gatewayId); + gateway.setDomain("testDomain"); + airavataClient.updateGateway(gatewayId, gateway); + List<Gateway> allGateways = airavataClient.getAllGateways(); + System.out.println(allGateways.size()); + if (airavataClient.isGatewayExist(gatewayId)) { + Gateway gateway1 = airavataClient.getGateway(gatewayId); + System.out.println(gateway1.getGatewayName()); + } + boolean b = airavataClient.deleteGateway("testGatewayId2"); + System.out.println(b); + } catch (AiravataSystemException e) { + e.printStackTrace(); + } catch (InvalidRequestException e) { + e.printStackTrace(); + } catch (AiravataClientException e) { + e.printStackTrace(); + } catch (TException e) { + e.printStackTrace(); + } + + } + + + public static void createAndLaunchExp() throws TException { + List<String> experimentIds = new ArrayList<String>(); + try { + for (int i = 0; i < 1; i++) { +// final String expId = createEchoExperimentForFSD(airavataClient); + final String expId = createMPIExperimentForFSD(airavataClient); + experimentIds.add(expId); + System.out.println("Experiment ID : " + expId); +// updateExperiment(airavata, expId); + launchExperiment(airavataClient, expId); + } + + Thread.sleep(10000); + for (String exId : experimentIds) { + Experiment experiment = airavataClient.getExperiment(exId); + System.out.println(experiment.getExperimentID() + " " + experiment.getExperimentStatus().getExperimentState().name()); + } + + + } catch (Exception e) { + logger.error("Error while connecting with server", e.getMessage()); + e.printStackTrace(); + + } + } + + public static void launchExperiment(Airavata.Client client, String expId) + throws TException { + try { + String tokenId = "-0bbb-403b-a88a-42b6dbe198e9"; + client.launchExperiment(expId, tokenId); + } catch (ExperimentNotFoundException e) { + logger.error("Error occured while launching the experiment...", e.getMessage()); + throw new ExperimentNotFoundException(e); + } catch (AiravataSystemException e) { + logger.error("Error occured while launching the experiment...", e.getMessage()); + throw new AiravataSystemException(e); + } catch (InvalidRequestException e) { + logger.error("Error occured while launching the experiment...", e.getMessage()); + throw new InvalidRequestException(e); + } catch (AiravataClientException e) { + logger.error("Error occured while launching the experiment...", e.getMessage()); + throw new AiravataClientException(e); + } catch (TException e) { + logger.error("Error occured while launching the experiment...", e.getMessage()); + throw new TException(e); + } + } + + public static void registerApplications() { + RegisterSampleApplications registerSampleApplications = new RegisterSampleApplications(airavataClient); + + //Register all compute hosts + registerSampleApplications.registerXSEDEHosts(); + + + //Register Gateway Resource Preferences + registerSampleApplications.registerGatewayResourceProfile(); + + //Register all application modules + registerSampleApplications.registerAppModules(); + + //Register all application deployments + registerSampleApplications.registerAppDeployments(); + + //Register all application interfaces + registerSampleApplications.registerAppInterfaces(); + } + + public static String registerUnicoreEndpoint(String hostName, String hostDesc, JobSubmissionProtocol protocol, SecurityProtocol securityProtocol) throws TException { + + ComputeResourceDescription computeResourceDescription = RegisterSampleApplicationsUtils + .createComputeResourceDescription(hostName, hostDesc, null, null); + + fsdResourceId = airavataClient.registerComputeResource(computeResourceDescription); + + if (fsdResourceId.isEmpty()) + throw new AiravataClientException(); + + System.out.println("FSD Compute ResourceID: " + fsdResourceId); + + JobSubmissionInterface jobSubmission = RegisterSampleApplicationsUtils.createJobSubmissionInterface(fsdResourceId, protocol, 2); + UnicoreJobSubmission ucrJobSubmission = new UnicoreJobSubmission(); + ucrJobSubmission.setSecurityProtocol(securityProtocol); + ucrJobSubmission.setUnicoreEndPointURL(unicoreEndPointURL); + + return jobSubmission.getJobSubmissionInterfaceId(); + } + + public static String createEchoExperimentForFSD(Airavata.Client client) throws TException { + try { + List<InputDataObjectType> exInputs = client.getApplicationInputs(echoAppId); + for (InputDataObjectType inputDataObjectType : exInputs) { + if (inputDataObjectType.getName().equalsIgnoreCase("Input_to_Echo")) { + inputDataObjectType.setValue("Hello World"); + } else if (inputDataObjectType.getName().equalsIgnoreCase("Input_to_Echo2")) { + inputDataObjectType.setValue("http://www.textfiles.com/100/ad.txt";); + } else if (inputDataObjectType.getName().equalsIgnoreCase("Input_to_Echo3")) { + inputDataObjectType.setValue("file:///tmp/test.txt"); + } + } + List<OutputDataObjectType> exOut = client.getApplicationOutputs(echoAppId); + + Experiment simpleExperiment = + ExperimentModelUtil.createSimpleExperiment("default", "admin", "echoExperiment", "SimpleEcho2", echoAppId, exInputs); + simpleExperiment.setExperimentOutputs(exOut); + + + Map<String, String> computeResources = airavataClient.getAvailableAppInterfaceComputeResources(echoAppId); + if (computeResources != null && computeResources.size() != 0) { + for (String id : computeResources.keySet()) { + String resourceName = computeResources.get(id); + if (resourceName.equals(unicoreHostName)) { + ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling(id, 1, 1, 1, "normal", 30, 0, 1048576, "sds128"); + UserConfigurationData userConfigurationData = new UserConfigurationData(); + userConfigurationData.setAiravataAutoSchedule(false); + userConfigurationData.setOverrideManualScheduledParams(false); + userConfigurationData.setComputationalResourceScheduling(scheduling); + + userConfigurationData.setGenerateCert(false); + userConfigurationData.setUserDN(""); + + // set output directory + AdvancedOutputDataHandling dataHandling = new AdvancedOutputDataHandling(); + dataHandling.setOutputDataDir("/tmp/airavata/output/" + UUID.randomUUID().toString() + "/"); + userConfigurationData.setAdvanceOutputDataHandling(dataHandling); + simpleExperiment.setUserConfigurationData(userConfigurationData); + + return client.createExperiment(DEFAULT_GATEWAY, simpleExperiment); + } + } + } + } catch (AiravataSystemException e) { + logger.error("Error occured while creating the experiment...", e.getMessage()); + throw new AiravataSystemException(e); + } catch (InvalidRequestException e) { + logger.error("Error occured while creating the experiment...", e.getMessage()); + throw new InvalidRequestException(e); + } catch (AiravataClientException e) { + logger.error("Error occured while creating the experiment...", e.getMessage()); + throw new AiravataClientException(e); + } catch (TException e) { + logger.error("Error occured while creating the experiment...", e.getMessage()); + throw new TException(e); + } + return null; + } + + + public static String createMPIExperimentForFSD(Airavata.Client client) throws TException { + try { + List<InputDataObjectType> exInputs = client.getApplicationInputs(mpiAppId); + for (InputDataObjectType inputDataObjectType : exInputs) { +// if (inputDataObjectType.getName().equalsIgnoreCase("Sample_Input")) { +// inputDataObjectType.setValue(""); +// } + if (inputDataObjectType.getName().equalsIgnoreCase("NumberOfProcesses")) { + inputDataObjectType.setValue("32"); + } + if (inputDataObjectType.getName().equalsIgnoreCase("US3INPUT")) { + inputDataObjectType.setValue("file://home/m.memon/us3input/smallerdata/hpcinput-uslims3.uthscsa.edu-uslims3_cauma3-01594.tar"); + } + if (inputDataObjectType.getName().equalsIgnoreCase("US3INPUTARG")) { + inputDataObjectType.setValue("hpcinput-uslims3.uthscsa.edu-uslims3_cauma3-01594.tar"); + } + } + + List<OutputDataObjectType> exOut = client.getApplicationOutputs(mpiAppId); + + for (OutputDataObjectType outputDataObjectType : exOut) { + if(outputDataObjectType.getName().equals("US3OUT")){ + outputDataObjectType.setValue("output/analysis-results.tar"); + } + } + + Experiment simpleExperiment = + ExperimentModelUtil.createSimpleExperiment("default", "admin", "mpiExperiment", "HelloMPI", mpiAppId, exInputs); + simpleExperiment.setExperimentOutputs(exOut); + + + Map<String, String> computeResources = airavataClient.getAvailableAppInterfaceComputeResources(mpiAppId); + if (computeResources != null && computeResources.size() != 0) { + for (String id : computeResources.keySet()) { + String resourceName = computeResources.get(id); + if (resourceName.equals(unicoreHostName)) { + ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling(id, 0, 4, 0, null, 10, 0, 0, null); + UserConfigurationData userConfigurationData = new UserConfigurationData(); + userConfigurationData.setAiravataAutoSchedule(false); + userConfigurationData.setOverrideManualScheduledParams(false); + userConfigurationData.setComputationalResourceScheduling(scheduling); + + userConfigurationData.setGenerateCert(true); + userConfigurationData.setUserDN("CN=m.memon, O=Ultrascan Gateway, C=DE"); + + // set output directory + AdvancedOutputDataHandling dataHandling = new AdvancedOutputDataHandling(); + dataHandling.setOutputDataDir("/tmp/airavata/output/" + UUID.randomUUID().toString() + "/"); + userConfigurationData.setAdvanceOutputDataHandling(dataHandling); + simpleExperiment.setUserConfigurationData(userConfigurationData); + + return client.createExperiment(DEFAULT_GATEWAY, simpleExperiment); + } + } + } + } catch (AiravataSystemException e) { + logger.error("Error occured while creating the experiment...", e.getMessage()); + throw new AiravataSystemException(e); + } catch (InvalidRequestException e) { + logger.error("Error occured while creating the experiment...", e.getMessage()); + throw new InvalidRequestException(e); + } catch (AiravataClientException e) { + logger.error("Error occured while creating the experiment...", e.getMessage()); + throw new AiravataClientException(e); + } catch (TException e) { + logger.error("Error occured while creating the experiment...", e.getMessage()); + throw new TException(e); + } + return null; + } + + + public static List<Experiment> getExperimentsForUser(Airavata.Client client, String user) { + try { + return client.getAllUserExperiments(DEFAULT_GATEWAY, user); + } catch (AiravataSystemException e) { + e.printStackTrace(); + } catch (InvalidRequestException e) { + e.printStackTrace(); + } catch (AiravataClientException e) { + e.printStackTrace(); + } catch (TException e) { + e.printStackTrace(); + } + return null; + } + + public static List<Project> getAllUserProject(Airavata.Client client, String user) { + try { + return client.getAllUserProjects(DEFAULT_GATEWAY, user); + } catch (AiravataSystemException e) { + e.printStackTrace(); + } catch (InvalidRequestException e) { + e.printStackTrace(); + } catch (AiravataClientException e) { + e.printStackTrace(); + } catch (TException e) { + e.printStackTrace(); + } + return null; + } + + public static List<Project> searchProjectsByProjectName(Airavata.Client client, String user, String projectName) { + try { + return client.searchProjectsByProjectName(DEFAULT_GATEWAY, user, projectName); + } catch (AiravataSystemException e) { + e.printStackTrace(); + } catch (InvalidRequestException e) { + e.printStackTrace(); + } catch (AiravataClientException e) { + e.printStackTrace(); + } catch (TException e) { + e.printStackTrace(); + } + return null; + } + + public static List<Project> searchProjectsByProjectDesc(Airavata.Client client, String user, String desc) { + try { + return client.searchProjectsByProjectDesc(DEFAULT_GATEWAY, user, desc); + } catch (AiravataSystemException e) { + e.printStackTrace(); + } catch (InvalidRequestException e) { + e.printStackTrace(); + } catch (AiravataClientException e) { + e.printStackTrace(); + } catch (TException e) { + e.printStackTrace(); + } + return null; + } + + + public static List<ExperimentSummary> searchExperimentsByName(Airavata.Client client, String user, String expName) { + try { + return client.searchExperimentsByName(DEFAULT_GATEWAY, user, expName); + } catch (AiravataSystemException e) { + e.printStackTrace(); + } catch (InvalidRequestException e) { + e.printStackTrace(); + } catch (AiravataClientException e) { + e.printStackTrace(); + } catch (TException e) { + e.printStackTrace(); + } + return null; + } + + public static List<ExperimentSummary> searchExperimentsByDesc(Airavata.Client client, String user, String desc) { + try { + return client.searchExperimentsByDesc(DEFAULT_GATEWAY, user, desc); + } catch (AiravataSystemException e) { + e.printStackTrace(); + } catch (InvalidRequestException e) { + e.printStackTrace(); + } catch (AiravataClientException e) { + e.printStackTrace(); + } catch (TException e) { + e.printStackTrace(); + } + return null; + } + + public static List<ExperimentSummary> searchExperimentsByApplication(Airavata.Client client, String user, String app) { + try { + return client.searchExperimentsByApplication(DEFAULT_GATEWAY, user, app); + } catch (AiravataSystemException e) { + e.printStackTrace(); + } catch (InvalidRequestException e) { + e.printStackTrace(); + } catch (AiravataClientException e) { + e.printStackTrace(); + } catch (TException e) { + e.printStackTrace(); + } + return null; + } + + public static void getExperiment(Airavata.Client client, String expId) throws Exception { + try { + Experiment experiment = client.getExperiment(expId); + List<ErrorDetails> errors = experiment.getErrors(); + if (errors != null && !errors.isEmpty()) { + for (ErrorDetails error : errors) { + System.out.println("ERROR MESSAGE : " + error.getActualErrorMessage()); + } + } + + } catch (ExperimentNotFoundException e) { + logger.error("Experiment does not exist", e); + throw new ExperimentNotFoundException("Experiment does not exist"); + } catch (AiravataSystemException e) { + logger.error("Error while retrieving experiment", e); + throw new AiravataSystemException(AiravataErrorType.INTERNAL_ERROR); + } catch (InvalidRequestException e) { + logger.error("Error while retrieving experiment", e); + throw new InvalidRequestException("Error while retrieving experiment"); + } catch (AiravataClientException e) { + logger.error("Error while retrieving experiment", e); + throw new AiravataClientException(AiravataErrorType.INTERNAL_ERROR); + } + } + +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e0df621b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java ---------------------------------------------------------------------- diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java index e10dde2..19999e0 100644 --- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java +++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java @@ -29,6 +29,7 @@ import java.util.*; import org.apache.airavata.api.Airavata; import org.apache.airavata.api.client.AiravataClientFactory; import org.apache.airavata.model.appcatalog.appdeployment.ApplicationParallelismType; +import org.apache.airavata.model.appcatalog.appdeployment.SetEnvPaths; import org.apache.airavata.model.appcatalog.appinterface.*; import org.apache.airavata.model.appcatalog.computeresource.*; import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference; @@ -60,7 +61,8 @@ public class RegisterSampleApplications { private static String fsdResourceId; // unicore service endpoint url - private static final String unicoreEndPointURL = "https://fsd-cloud15.zam.kfa-juelich.de:7000/INTEROP1/services/BESFactory?res=default_bes_factory";; +// private static final String unicoreEndPointURL = "https://fsd-cloud15.zam.kfa-juelich.de:7000/INTEROP1/services/BESFactory?res=default_bes_factory";; + private static final String unicoreEndPointURL = "https://deisa-unic.fz-juelich.de:9111/FZJ_JUROPA/services/BESFactory?res=default_bes_factory";; //Appplication Names private static final String echoName = "Echo"; @@ -565,17 +567,40 @@ public class RegisterSampleApplications { InputDataObjectType input1 = RegisterSampleApplicationsUtils.createAppInput("Sample_Input", "", DataType.STRING, null, 1,true, false, false, "An optional MPI source file", null); + InputDataObjectType input11 = RegisterSampleApplicationsUtils.createAppInput("US3INPUT", "", + DataType.URI, null, 1,true, false, false, "Input US3 file", null); + + InputDataObjectType input12 = RegisterSampleApplicationsUtils.createAppInput("US3INPUTARG", "", + DataType.STRING, null, 1,true, false, false, "Input US3 Arg", null); + + + InputDataObjectType input2 = RegisterSampleApplicationsUtils.createAppInput("NumberOfProcesses", "", + DataType.INTEGER, null, 2,false, true, false, "Number Of Processes", null); + + + InputDataObjectType input3 = RegisterSampleApplicationsUtils.createAppInput("ProcessesPerHost", "", + DataType.INTEGER, null, 3,false, true, false, "Processes per host", null); + + List<InputDataObjectType> applicationInputs = new ArrayList<InputDataObjectType>(); applicationInputs.add(input1); + applicationInputs.add(input2); + applicationInputs.add(input3); + applicationInputs.add(input11); + applicationInputs.add(input12); OutputDataObjectType output1 = RegisterSampleApplicationsUtils.createAppOutput("STDOutput", "", DataType.STDOUT, true, true); OutputDataObjectType output2 = RegisterSampleApplicationsUtils.createAppOutput("STDErr", "", DataType.STDERR, true, true); + OutputDataObjectType output3 = RegisterSampleApplicationsUtils.createAppOutput("US3OUT", + "", DataType.STRING, true, false); + List<OutputDataObjectType> applicationOutputs = new ArrayList<OutputDataObjectType>(); applicationOutputs.add(output1); applicationOutputs.add(output2); + applicationOutputs.add(output3); mpiInterfaceId = airavataClient.registerApplicationInterface(DEFAULT_GATEWAY, @@ -1263,11 +1288,18 @@ public class RegisterSampleApplications { RegisterSampleApplicationsUtils.createApplicationDeployment(echoModuleId, fsdResourceId, "/bin/echo", ApplicationParallelismType.SERIAL, echoDescription, null, null, null)); System.out.println("Echo on FSD deployment Id: " + echoAppDeployId); - + //Register MPI +// String mpiAppDeployId = airavataClient.registerApplicationDeployment(DEFAULT_GATEWAY, +// RegisterSampleApplicationsUtils.createApplicationDeployment(mpiModuleId, fsdResourceId, +// "/home/bes/hellompi", ApplicationParallelismType.OPENMP_MPI, mpiDescription, null, null, null)); + + String mpiAppDeployId = airavataClient.registerApplicationDeployment(DEFAULT_GATEWAY, RegisterSampleApplicationsUtils.createApplicationDeployment(mpiModuleId, fsdResourceId, - "/home/bes/hellompi", ApplicationParallelismType.OPENMP_MPI, mpiDescription, null, null, null)); + "us_mpi_analysis", ApplicationParallelismType.MPI, mpiDescription, null, null, null)); + + System.out.println("MPI on FSD deployment Id: " + mpiAppDeployId); } catch (TException e) { http://git-wip-us.apache.org/repos/asf/airavata/blob/e0df621b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplicationsUtils.java ---------------------------------------------------------------------- diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplicationsUtils.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplicationsUtils.java index 86f7d1a..5ef1eae 100644 --- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplicationsUtils.java +++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplicationsUtils.java @@ -24,6 +24,7 @@ package org.apache.airavata.client.tools; import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription; import org.apache.airavata.model.appcatalog.appdeployment.ApplicationModule; import org.apache.airavata.model.appcatalog.appdeployment.ApplicationParallelismType; +import org.apache.airavata.model.appcatalog.appdeployment.SetEnvPaths; import org.apache.airavata.model.appcatalog.appinterface.*; import org.apache.airavata.model.appcatalog.computeresource.*; import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference; @@ -73,7 +74,7 @@ public class RegisterSampleApplicationsUtils { return deployment; } - public static ApplicationModule createApplicationModule(String appModuleName, + public static ApplicationModule createApplicationModule(String appModuleName, String appModuleVersion, String appModuleDescription) { ApplicationModule module = new ApplicationModule(); module.setAppModuleDescription(appModuleDescription); http://git-wip-us.apache.org/repos/asf/airavata/blob/e0df621b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java index 7c7a59e..bc91e49 100644 --- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java +++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java @@ -21,9 +21,13 @@ package org.apache.airavata.gfac.bes.utils; +import java.util.Iterator; +import java.util.List; + import org.apache.airavata.gfac.core.context.JobExecutionContext; import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription; import org.apache.airavata.model.appcatalog.appdeployment.ApplicationParallelismType; +import org.apache.airavata.model.appcatalog.appinterface.InputDataObjectType; import org.ggf.schemas.jsdl.x2005.x11.jsdl.ApplicationType; import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType; import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.FileNameType; @@ -77,24 +81,35 @@ public class ApplicationProcessor { }else if (parallelism.equals(ApplicationParallelismType.MPI)){ JSDLUtils.getSPMDApplication(value).setSPMDVariation(SPMDVariations.MPI.value()); } + + // setting number of processes + try { + String np = getInputAsString(context, BESConstants.NUMBER_OF_PROCESSES); + if((np != null) && (Integer.parseInt(np) > 0)){ + NumberOfProcessesType num = NumberOfProcessesType.Factory.newInstance(); + num.setStringValue(np); + JSDLUtils.getSPMDApplication(value).setNumberOfProcesses(num); + } + + }catch(RuntimeException np) { + // do nothing + } - int totalCPUCount = context.getTaskData().getTaskScheduling().getTotalCPUCount(); - if(totalCPUCount > 0){ - NumberOfProcessesType num = NumberOfProcessesType.Factory.newInstance(); - num.setStringValue(String.valueOf(totalCPUCount)); - JSDLUtils.getSPMDApplication(value).setNumberOfProcesses(num); - } - int totalNodeCount = context.getTaskData().getTaskScheduling().getNodeCount(); - if(totalNodeCount > 0){ - int ppn = totalCPUCount / totalNodeCount; - ProcessesPerHostType pph = ProcessesPerHostType.Factory.newInstance(); - pph.setStringValue(String.valueOf(ppn)); - JSDLUtils.getSPMDApplication(value).setProcessesPerHost(pph); + try { + // setting processes per host + String pphost = getInputAsString(context, BESConstants.PROCESSES_PER_HOST); + if((pphost != null) && (Integer.parseInt(pphost) > 0)){ + ProcessesPerHostType pph = ProcessesPerHostType.Factory.newInstance(); + pph.setStringValue(String.valueOf(pphost)); + JSDLUtils.getSPMDApplication(value).setProcessesPerHost(pph); + } + }catch(RuntimeException np) { + // do nothing } - + int totalThreadCount = context.getTaskData().getTaskScheduling().getNumberOfThreads(); - + // we take it as threads per processes if(totalThreadCount > 0){ ThreadsPerProcessType tpp = ThreadsPerProcessType.Factory.newInstance(); tpp.setStringValue(String.valueOf(totalThreadCount)); @@ -187,5 +202,20 @@ public class ApplicationProcessor { return isParallel; } + private static String getInputAsString(JobExecutionContext context, String name) { + List<InputDataObjectType> inputList = context.getTaskData().getApplicationInputs(); + String value = null; + for (Iterator<InputDataObjectType> iterator = inputList.iterator(); iterator.hasNext();) { + InputDataObjectType inputDataObjectType = iterator + .next(); + if (inputDataObjectType.getName().equals(name)) { + value = inputDataObjectType.getValue(); + break; + } + } + return value; + } + + } http://git-wip-us.apache.org/repos/asf/airavata/blob/e0df621b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/BESConstants.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/BESConstants.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/BESConstants.java index ffbf62e..6e87e28 100644 --- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/BESConstants.java +++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/BESConstants.java @@ -36,6 +36,10 @@ public interface BESConstants { public static final String PROP_CA_KEY_PATH = "bes.ca.key.path"; public static final String PROP_CA_KEY_PASS = "bes.ca.key.pass"; + + public static final String NUMBER_OF_PROCESSES = "NumberOfProcesses"; + + public static final String PROCESSES_PER_HOST = "ProcessesPerHost"; } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/e0df621b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java index cb036fa..d7f3244 100644 --- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java +++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java @@ -37,6 +37,8 @@ import org.apache.airavata.model.appcatalog.appinterface.DataType; import org.apache.airavata.model.appcatalog.appinterface.InputDataObjectType; import org.apache.airavata.model.appcatalog.appinterface.OutputDataObjectType; import org.apache.airavata.model.workspace.experiment.TaskDetails; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,9 +93,10 @@ public class DataTransferrer { file.mkdirs(); } List<String> outPrms = extractOutParams(jobContext); + for (String outPrm : outPrms) { - String outputLocation = downloadLocation+File.separator+outPrm; - FileDownloader fileDownloader = new FileDownloader(outPrm,outputLocation, Mode.overwrite); + if("".equals(outPrm)) continue; + FileDownloader fileDownloader = new FileDownloader(outPrm,downloadLocation, Mode.overwrite); try { fileDownloader.perform(storageClient); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/airavata/blob/e0df621b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java index fce0c31..cc3ad8b 100644 --- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java +++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java @@ -34,10 +34,13 @@ public class ResourceProcessor { TaskDetails taskData = context.getTaskData(); + + if(taskData != null && taskData.isSetTaskScheduling()){ try { - ComputationalResourceScheduling crs = taskData.getTaskScheduling(); + ComputationalResourceScheduling crs = taskData.getTaskScheduling(); + if (crs.getTotalPhysicalMemory() > 0) { RangeValueType rangeType = new RangeValueType(); rangeType.setLowerBound(Double.NaN); @@ -52,6 +55,8 @@ public class ResourceProcessor { rangeType.setUpperBound(Double.NaN); rangeType.setExact(crs.getNodeCount()); JSDLUtils.setTotalResourceCountRequirements(value, rangeType); + // set totalcpu count to -1 as we dont need that + crs.setTotalCPUCount(0); } if(crs.getWallTimeLimit() > 0) { @@ -70,6 +75,7 @@ public class ResourceProcessor { rangeType.setExact(crs.getTotalCPUCount()); JSDLUtils.setTotalCPUCountRequirements(value, rangeType); } + } catch (NullPointerException npe) { new GFacProviderException("No value set for resource requirements.",npe); } http://git-wip-us.apache.org/repos/asf/airavata/blob/e0df621b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java index e82edef..09c6e11 100644 --- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java +++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java @@ -136,7 +136,10 @@ public class UASDataStagingProcessor { ApplicationProcessor.addApplicationArgument(value, context, input.getValue()); } else if (input.getType().equals(DataType.FLOAT) || input.getType().equals(DataType.INTEGER)){ - ApplicationProcessor.addApplicationArgument(value, context, String.valueOf(input.getValue())); + if(! (input.getName().equals(BESConstants.NUMBER_OF_PROCESSES) || input.getName().equals(BESConstants.PROCESSES_PER_HOST))) { + // temp avoid environ going to app args + ApplicationProcessor.addApplicationArgument(value, context, String.valueOf(input.getValue())); + } } } }
