adding application layer
Project: http://git-wip-us.apache.org/repos/asf/airavata-sandbox/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata-sandbox/commit/50df93aa Tree: http://git-wip-us.apache.org/repos/asf/airavata-sandbox/tree/50df93aa Diff: http://git-wip-us.apache.org/repos/asf/airavata-sandbox/diff/50df93aa Branch: refs/heads/master Commit: 50df93aa7da2dbef4c77a4c45122c8374d7c4a8b Parents: 8b284d8 Author: scnakandala <supun.nakand...@gmail.com> Authored: Tue Aug 8 14:54:01 2017 -0400 Committer: scnakandala <supun.nakand...@gmail.com> Committed: Tue Aug 8 14:54:01 2017 -0400 ---------------------------------------------------------------------- airavata-layered-architecture/.DS_Store | Bin 0 -> 6148 bytes .../applications/hpc/HPCApplication.java | 58 +++++++++ .../hpc/HPCApplicationExecutor.java | 52 ++++++++ .../airavata/applications/hpc/TensorFlow.java | 67 +++++++++++ .../org/apache/airavata/models/Constants.java | 30 +++++ .../models/resources/Authentication.java | 30 +++++ .../models/resources/CommandOutput.java | 57 +++++++++ .../airavata/models/resources/JobStatus.java | 28 +++++ .../models/resources/JobSubmissionOutput.java | 88 ++++++++++++++ .../airavata/models/resources/ServerInfo.java | 49 ++++++++ .../models/resources/hpc/GroovyMap.java | 93 ++++++++++++++ .../resources/hpc/JobManagerConfiguration.java | 49 ++++++++ .../models/resources/hpc/OutputParser.java | 66 ++++++++++ .../resources/hpc/PBSJobConfiguration.java | 119 ++++++++++++++++++ .../models/resources/hpc/RawCommandInfo.java | 41 +++++++ .../airavata/models/resources/hpc/Script.java | 61 ++++++++++ .../models/runners/ssh/SSHApiException.java | 35 ++++++ .../runners/ssh/SSHKeyAuthentication.java | 92 ++++++++++++++ .../models/runners/ssh/SSHServerInfo.java | 52 ++++++++ .../models/runners/ssh/SSHUserInfo.java | 66 ++++++++++ .../airavata/resources/Authentication.java | 30 ----- .../apache/airavata/resources/OutputParser.java | 66 ---------- .../apache/airavata/resources/ServerInfo.java | 49 -------- .../resources/batch/BatchJobOutputParser.java | 3 +- .../airavata/resources/batch/BigRed2.java | 70 +++++++++++ .../airavata/resources/batch/GroovyMap.java | 93 -------------- .../resources/batch/HPCBatchResource.java | 27 +++-- .../batch/JobManagerConfiguration.java | 51 -------- .../airavata/resources/batch/JobStatus.java | 28 ----- .../resources/batch/JobSubmissionOutput.java | 88 -------------- .../resources/batch/PBSJobConfiguration.java | 120 ------------------- .../resources/batch/RawCommandInfo.java | 41 ------- .../apache/airavata/resources/batch/Script.java | 61 ---------- .../apache/airavata/runners/CommandOutput.java | 57 --------- .../airavata/runners/ssh/SSHApiException.java | 35 ------ .../runners/ssh/SSHKeyAuthentication.java | 92 -------------- .../apache/airavata/runners/ssh/SSHRunner.java | 4 + .../airavata/runners/ssh/SSHServerInfo.java | 52 -------- .../airavata/runners/ssh/SSHUserInfo.java | 66 ---------- .../src/main/resources/code_tf.py | 33 +++++ .../src/main/resources/job_tf.pbs | 18 +++ .../src/main/resources/ssh/id_rsa | 30 +++++ .../src/main/resources/ssh/id_rsa.pub | 1 + .../src/main/resources/ssh/known_hosts | 0 .../java/org/apache/airavata/Constants.java | 30 ----- .../applications/hpc/HPCApplicationTest.java | 46 +++++++ .../resources/batch/HPCBatchResourceTest.java | 32 ++++- .../airavata/runners/ssh/SSHRunnerTest.java | 4 +- .../src/test/resources/code_tf.py | 30 ----- .../src/test/resources/job_tf.pbs | 18 --- .../src/test/resources/ssh/id_rsa.pub | 1 - .../src/test/resources/ssh/known_hosts | 0 52 files changed, 1382 insertions(+), 1027 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/.DS_Store ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/.DS_Store b/airavata-layered-architecture/.DS_Store new file mode 100644 index 0000000..81229e8 Binary files /dev/null and b/airavata-layered-architecture/.DS_Store differ http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/applications/hpc/HPCApplication.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/applications/hpc/HPCApplication.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/applications/hpc/HPCApplication.java new file mode 100644 index 0000000..c6d4517 --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/applications/hpc/HPCApplication.java @@ -0,0 +1,58 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.applications.hpc; + +import org.apache.airavata.models.resources.hpc.GroovyMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +public class HPCApplication { + private final static Logger logger = LoggerFactory.getLogger(HPCApplication.class); + + private String applicationName; + + private Map<String, String> applicationInputs; + + public HPCApplication(String applicationName, Map<String, String> applicationInputs) { + this.applicationName = applicationName; + this.applicationInputs = applicationInputs; + } + + public String getApplicationName() { + return applicationName; + } + + public void setApplicationName(String applicationName) { + this.applicationName = applicationName; + } + + public Map<String, String> getApplicationInputs() { + return applicationInputs; + } + + public void setApplicationInputs(Map<String, String> applicationInputs) { + this.applicationInputs = applicationInputs; + } + + public GroovyMap getJobMap(String computeResource){return null;} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/applications/hpc/HPCApplicationExecutor.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/applications/hpc/HPCApplicationExecutor.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/applications/hpc/HPCApplicationExecutor.java new file mode 100644 index 0000000..fe5c1b3 --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/applications/hpc/HPCApplicationExecutor.java @@ -0,0 +1,52 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.applications.hpc; + +import org.apache.airavata.models.resources.hpc.GroovyMap; +import org.apache.airavata.models.resources.hpc.Script; +import org.apache.airavata.resources.batch.BigRed2; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +public class HPCApplicationExecutor { + private final static Logger logger = LoggerFactory.getLogger(HPCApplicationExecutor.class); + + public void executeApplication(String routingKey, HPCApplication application, String computeResource, + GroovyMap jobSpecification) throws Exception { + if(computeResource.equals("bigred2.uits.iu.edu")){ + //Even though here I use different class specific for BigRed2 there should be a generic class (HPCBatchResource) + //which will be configured with BigRed2 specific config values read from a database + BigRed2 bigRed2 = new BigRed2(); + String workingDirectory = jobSpecification.get(Script.WORKING_DIR).toString(); + bigRed2.makeDirectory(routingKey, workingDirectory); + + for(Map.Entry<String, String> entry: application.getApplicationInputs().entrySet()){ + bigRed2.copyTo(routingKey, entry.getValue(), workingDirectory + "/" + entry.getKey()); + } + + bigRed2.submitBatchJob(routingKey, jobSpecification); + }else{ + throw new Exception("Unsupported compute resource..."); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/applications/hpc/TensorFlow.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/applications/hpc/TensorFlow.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/applications/hpc/TensorFlow.java new file mode 100644 index 0000000..d4a1042 --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/applications/hpc/TensorFlow.java @@ -0,0 +1,67 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.applications.hpc; + +import org.apache.airavata.models.resources.hpc.GroovyMap; +import org.apache.airavata.models.resources.hpc.Script; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class TensorFlow extends HPCApplication { + private final static Logger logger = LoggerFactory.getLogger(TensorFlow.class); + + public TensorFlow(String applicationName, Map<String, String> applicationInputs) { + super(applicationName, applicationInputs); + } + + public GroovyMap getJobSpecification(String computeResource){ + GroovyMap jobSpecification = new GroovyMap(); + + jobSpecification.add(Script.NODES, 1); + jobSpecification.add(Script.PROCESS_PER_NODE, 16); + jobSpecification.add(Script.MAX_WALL_TIME, "00:30:00"); + + jobSpecification.add(Script.QUEUE_NAME, "debug_gpu"); + + jobSpecification.add(Script.MAIL_ADDRESS, "supun.nakand...@gmail.com"); + + List<String> moduleLoads = new ArrayList<>(); + moduleLoads.add("module load ccm"); + moduleLoads.add("module load singularity"); + jobSpecification.add(Script.MODULE_COMMANDS, moduleLoads); + + jobSpecification.add(Script.WORKING_DIR, "/N/dc2/scratch/snakanda/work-dirs"); + List<java.lang.String> inputs = new ArrayList<>(); + inputs.add("~/airavata/code_tf.py"); + jobSpecification.add(Script.INPUTS, inputs); + jobSpecification.add(Script.EXECUTABLE_PATH, "singularity exec /N/soft/cle5/singularity/images/tensorflow1.1-ubuntu-py2.7.11-test.img python"); + jobSpecification.add(Script.JOB_SUBMITTER_COMMAND,"ccmrun"); + + jobSpecification.add(Script.STANDARD_OUT_FILE, "STDOUT.txt"); + jobSpecification.add(Script.STANDARD_ERROR_FILE, "STDERR.txt"); + + return jobSpecification; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/models/Constants.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/models/Constants.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/Constants.java new file mode 100644 index 0000000..f2c8b0f --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/Constants.java @@ -0,0 +1,30 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.models; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Constants { + private final static Logger logger = LoggerFactory.getLogger(Constants.class); + + public static final String loginUserName = "snakanda"; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/Authentication.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/Authentication.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/Authentication.java new file mode 100644 index 0000000..fb4a60c --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/Authentication.java @@ -0,0 +1,30 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.models.resources; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Authentication { + private final static Logger logger = LoggerFactory.getLogger(Authentication.class); + + protected String userName; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/CommandOutput.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/CommandOutput.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/CommandOutput.java new file mode 100644 index 0000000..d994298 --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/CommandOutput.java @@ -0,0 +1,57 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.models.resources; + + +import com.jcraft.jsch.Channel; + +import java.io.OutputStream; + +/** + * Output of a certain command. + */ +public interface CommandOutput { + + /** + * Gets the output of the command as a stream. + * @param channel Command output as a stream. + */ + void onOutput(Channel channel); + + /** + * Gets standard error as a output stream. + * @return Command error as a stream. + */ + OutputStream getStandardError(); + + /** + * The command exit code. + * @param code The program exit code + */ + void exitCode(int code); + + /** + * Return the exit code of the command execution. + * @return exit code + */ + int getExitCode(); + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/JobStatus.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/JobStatus.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/JobStatus.java new file mode 100644 index 0000000..a07a846 --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/JobStatus.java @@ -0,0 +1,28 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.models.resources; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class JobStatus { + private final static Logger logger = LoggerFactory.getLogger(JobStatus.class); +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/JobSubmissionOutput.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/JobSubmissionOutput.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/JobSubmissionOutput.java new file mode 100644 index 0000000..b918c10 --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/JobSubmissionOutput.java @@ -0,0 +1,88 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.models.resources; + +public class JobSubmissionOutput { + + private int exitCode = Integer.MIN_VALUE; + private String stdOut; + private String stdErr; + private String command; + private String jobId; + private boolean isJobSubmissionFailed; + private String failureReason; + + public int getExitCode() { + return exitCode; + } + + public void setExitCode(int exitCode) { + this.exitCode = exitCode; + } + + public String getStdOut() { + return stdOut; + } + + public void setStdOut(String stdOut) { + this.stdOut = stdOut; + } + + public String getStdErr() { + return stdErr; + } + + public void setStdErr(String stdErr) { + this.stdErr = stdErr; + } + + public String getCommand() { + return command; + } + + public void setCommand(String command) { + this.command = command; + } + + public String getJobId() { + return jobId; + } + + public void setJobId(String jobId) { + this.jobId = jobId; + } + + public boolean isJobSubmissionFailed() { + return isJobSubmissionFailed; + } + + public void setJobSubmissionFailed(boolean jobSubmissionFailed) { + isJobSubmissionFailed = jobSubmissionFailed; + } + + public String getFailureReason() { + return failureReason; + } + + public void setFailureReason(String failureReason) { + this.failureReason = failureReason; + } +} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/ServerInfo.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/ServerInfo.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/ServerInfo.java new file mode 100644 index 0000000..436ae44 --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/ServerInfo.java @@ -0,0 +1,49 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.models.resources; + +public class ServerInfo { + + public static enum ComProtocol {SSH, LOCAL} + + protected String host; + protected String userName; + protected int port; + protected ComProtocol comProtocol; + + public ServerInfo(){} + + public ServerInfo(String userName, String host, ComProtocol comProtocol, int port) { + this.userName = userName; + this.host = host; + this.comProtocol = comProtocol; + this.port = port; + } + + public String getHost() { + return host; + } + + public String getUserName() { + return userName; + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/GroovyMap.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/GroovyMap.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/GroovyMap.java new file mode 100644 index 0000000..8b13e3a --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/GroovyMap.java @@ -0,0 +1,93 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.models.resources.hpc; + +import java.util.HashMap; +import java.util.Optional; + +public class GroovyMap extends HashMap<String, Object> { + + + public GroovyMap() { + super(); + // to mitigate groovy exception groovy.lang.MissingPropertyException: No such property: <name> for class: groovy.lang.Binding + addDefaultValues(); + } + + public GroovyMap add(Script name, Object value){ + put(name.name, value); + return this; + } + + @Override + public Object get(Object key) { + return super.getOrDefault(key, null); + } + + public Object get(Script script) { + return get(script.name); + } + + public Optional<String> getStringValue(Script script) { + Object obj = get(script); + if (obj instanceof String) { + return Optional.of((String) obj); + } else if (obj == null) { + return Optional.empty(); + } else { + throw new IllegalArgumentException("Value is not String type"); + } + } + + private void addDefaultValues() { + this.add(Script.SHELL_NAME, null) + .add(Script.QUEUE_NAME, null) + .add(Script.NODES, null) + .add(Script.CPU_COUNT, null) + .add(Script.MAIL_ADDRESS, null) + .add(Script.ACCOUNT_STRING, null) + .add(Script.MAX_WALL_TIME, null) + .add(Script.JOB_NAME, null) + .add(Script.STANDARD_OUT_FILE, null) + .add(Script.STANDARD_ERROR_FILE, null) + .add(Script.QUALITY_OF_SERVICE, null) + .add(Script.RESERVATION, null) + .add(Script.EXPORTS, null) + .add(Script.MODULE_COMMANDS, null) + .add(Script.SCRATCH_LOCATION, null) + .add(Script.WORKING_DIR, null) + .add(Script.PRE_JOB_COMMANDS, null) + .add(Script.JOB_SUBMITTER_COMMAND, null) + .add(Script.EXECUTABLE_PATH, null) + .add(Script.INPUTS, null) + .add(Script.POST_JOB_COMMANDS, null) + .add(Script.USED_MEM, null) + .add(Script.PROCESS_PER_NODE, null) + .add(Script.CHASSIS_NAME, null) + .add(Script.INPUT_DIR, null) + .add(Script.OUTPUT_DIR, null) + .add(Script.USER_NAME, null) + .add(Script.GATEWAY_ID, null) + .add(Script.GATEWAY_USER_NAME, null) + .add(Script.APPLICATION_NAME, null); + } + +} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/JobManagerConfiguration.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/JobManagerConfiguration.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/JobManagerConfiguration.java new file mode 100644 index 0000000..abe55f6 --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/JobManagerConfiguration.java @@ -0,0 +1,49 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.models.resources.hpc; + +public interface JobManagerConfiguration { + public static enum JobManagerCommand{JOB_MONITORING, SUBMISSION, DELETION}; + + public RawCommandInfo getCancelCommand(String jobID); + + public String getJobDescriptionTemplateName(); + + public RawCommandInfo getMonitorCommand(String jobID); + + public RawCommandInfo getUserBasedMonitorCommand(String userName); + + public RawCommandInfo getJobIdMonitorCommand(String jobName , String userName); + + public String getScriptExtension(); + + public RawCommandInfo getSubmitCommand(String workingDirectory, String pbsFilePath); + + public OutputParser getParser(); + + public String getInstalledPath(); + + public String getBaseCancelCommand(); + + public String getBaseMonitorCommand(); + + public String getBaseSubmitCommand(); +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/OutputParser.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/OutputParser.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/OutputParser.java new file mode 100644 index 0000000..90919b2 --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/OutputParser.java @@ -0,0 +1,66 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.models.resources.hpc; + +import org.apache.airavata.models.resources.JobStatus; + +import java.util.Map; + +public interface OutputParser { + /** + * This can be used to parseSingleJob the result of a job submission to get the JobID + * @param rawOutput + * @return the job id as a String, or null if no job id found + */ + public String parseJobSubmission(String rawOutput)throws Exception; + + + /** + * Parse output return by job submission task and identify jobSubmission failures. + * @param rawOutput + * @return true if job submission has been failed, false otherwise. + */ + public boolean isJobSubmissionFailed(String rawOutput); + + + /** + * This can be used to get the job status from the output + * @param jobID + * @param rawOutput + */ + public JobStatus parseJobStatus(String jobID, String rawOutput)throws Exception; + + /** + * This can be used to parseSingleJob a big output and get multipleJob statuses + * @param statusMap list of status map will return and key will be the job ID + * @param rawOutput + */ + public void parseJobStatuses(String userName, Map<String, JobStatus> statusMap, String rawOutput)throws Exception; + + /** + * filter the jobId value of given JobName from rawOutput + * @param jobName + * @param rawOutput + * @return + * @throws Exception + */ + public String parseJobId(String jobName, String rawOutput) throws Exception; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/PBSJobConfiguration.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/PBSJobConfiguration.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/PBSJobConfiguration.java new file mode 100644 index 0000000..47dc333 --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/PBSJobConfiguration.java @@ -0,0 +1,119 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.models.resources.hpc; + +import org.apache.commons.io.FilenameUtils; + +import java.io.File; +import java.util.Map; + +public class PBSJobConfiguration implements JobManagerConfiguration { + + private final Map<JobManagerCommand, String> jobManagerCommands; + private String jobDescriptionTemplateName; + private String scriptExtension; + private String installedPath; + private OutputParser parser; + + public PBSJobConfiguration(String jobDescriptionTemplateName, String scriptExtension, String installedPath, + Map<JobManagerCommand, String> jobManagerCommands, OutputParser parser) { + this.jobDescriptionTemplateName = jobDescriptionTemplateName; + this.scriptExtension = scriptExtension; + this.parser = parser; + installedPath = installedPath.trim(); + if (installedPath.endsWith("/")) { + this.installedPath = installedPath; + } else { + this.installedPath = installedPath + "/"; + } + this.jobManagerCommands = jobManagerCommands; + } + + public RawCommandInfo getCancelCommand(String jobID) { + return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.DELETION).trim() + " " + + jobID); + } + + public String getJobDescriptionTemplateName() { + return jobDescriptionTemplateName; + } + + public void setJobDescriptionTemplateName(String jobDescriptionTemplateName) { + this.jobDescriptionTemplateName = jobDescriptionTemplateName; + } + + public RawCommandInfo getMonitorCommand(String jobID) { + return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.JOB_MONITORING).trim() + + " -f " + jobID); + } + + public String getScriptExtension() { + return scriptExtension; + } + + public RawCommandInfo getSubmitCommand(String workingDirectory, String pbsFilePath) { + return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.SUBMISSION).trim() + " " + + workingDirectory + File.separator + FilenameUtils.getName(pbsFilePath)); + } + + public String getInstalledPath() { + return installedPath; + } + + public void setInstalledPath(String installedPath) { + this.installedPath = installedPath; + } + + public OutputParser getParser() { + return parser; + } + + public void setParser(OutputParser parser) { + this.parser = parser; + } + + public RawCommandInfo getUserBasedMonitorCommand(String userName) { + return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.JOB_MONITORING).trim() + + " -u " + userName); + } + + @Override + public RawCommandInfo getJobIdMonitorCommand(String jobName, String userName) { + // For PBS there is no option to get jobDetails by JobName, so we search with userName + return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.JOB_MONITORING).trim() + + " -u " + userName + " -f | grep \"Job_Name = " + jobName + "\" -B1"); + } + + @Override + public String getBaseCancelCommand() { + return jobManagerCommands.get(JobManagerCommand.DELETION).trim(); + } + + @Override + public String getBaseMonitorCommand() { + return jobManagerCommands.get(JobManagerCommand.JOB_MONITORING).trim(); + } + + @Override + public String getBaseSubmitCommand() { + return jobManagerCommands.get(JobManagerCommand.SUBMISSION).trim(); + } +} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/RawCommandInfo.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/RawCommandInfo.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/RawCommandInfo.java new file mode 100644 index 0000000..c96a7e7 --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/RawCommandInfo.java @@ -0,0 +1,41 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.models.resources.hpc; + +public class RawCommandInfo { + private String rawCommand; + + public RawCommandInfo(String cmd) { + this.rawCommand = cmd; + } + + public String getCommand() { + return this.rawCommand; + } + + public String getRawCommand() { + return rawCommand; + } + + public void setRawCommand(String rawCommand) { + this.rawCommand = rawCommand; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/Script.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/Script.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/Script.java new file mode 100644 index 0000000..b873a52 --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/resources/hpc/Script.java @@ -0,0 +1,61 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.models.resources.hpc; + +public enum Script { + + SHELL_NAME("shellName"), + QUEUE_NAME("queueName"), + NODES("nodes"), + CPU_COUNT("cpuCount"), + MAIL_ADDRESS("mailAddress"), + ACCOUNT_STRING("accountString"), + MAX_WALL_TIME("maxWallTime"), + JOB_NAME("jobName"), + STANDARD_OUT_FILE("standardOutFile"), + STANDARD_ERROR_FILE("standardErrorFile"), + QUALITY_OF_SERVICE("qualityOfService"), + RESERVATION("reservation"), + EXPORTS("exports"), + MODULE_COMMANDS("moduleCommands"), + SCRATCH_LOCATION("scratchLocation"), + WORKING_DIR("workingDirectory"), + PRE_JOB_COMMANDS("preJobCommands"), + JOB_SUBMITTER_COMMAND("jobSubmitterCommand"), + EXECUTABLE_PATH("executablePath"), + INPUTS("inputs"), + POST_JOB_COMMANDS("postJobCommands"), + USED_MEM("usedMem"), + PROCESS_PER_NODE("processPerNode"), + CHASSIS_NAME("chassisName"), + INPUT_DIR("inputDir"), + OUTPUT_DIR("outputDir"), + USER_NAME("userName"), + GATEWAY_ID("gatewayId"), + GATEWAY_USER_NAME("gatewayUserName"), + APPLICATION_NAME("applicationName"), + ; + + String name; + Script(String name) { + this.name = name; + } +} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/models/runners/ssh/SSHApiException.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/models/runners/ssh/SSHApiException.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/runners/ssh/SSHApiException.java new file mode 100644 index 0000000..78a0de0 --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/runners/ssh/SSHApiException.java @@ -0,0 +1,35 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.models.runners.ssh; + +/** + * An exception class to wrap SSH command execution related errors. + */ +public class SSHApiException extends Exception { + + public SSHApiException(String message) { + super(message); + } + + public SSHApiException(String message, Exception e) { + super(message, e); + } +} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/models/runners/ssh/SSHKeyAuthentication.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/models/runners/ssh/SSHKeyAuthentication.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/runners/ssh/SSHKeyAuthentication.java new file mode 100644 index 0000000..6187485 --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/runners/ssh/SSHKeyAuthentication.java @@ -0,0 +1,92 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.models.runners.ssh; + +import org.apache.airavata.models.resources.Authentication; + +public class SSHKeyAuthentication extends Authentication { + private byte[] privateKey; + private byte[] publicKey; + private String passphrase; + private String knownHostsFilePath; + private String strictHostKeyChecking; // yes or no + + public SSHKeyAuthentication(String userName, byte[] privateKey, byte[] publicKey, String passphrase, String knownHostsFilePath, boolean strictHostKeyChecking) { + this.userName = userName; + this.privateKey = privateKey; + this.publicKey = publicKey; + this.passphrase = passphrase; + this.knownHostsFilePath = knownHostsFilePath; + if(strictHostKeyChecking){ + this.strictHostKeyChecking = "yes"; + }else{ + this.strictHostKeyChecking = "no"; + } + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public byte[] getPrivateKey() { + return privateKey; + } + + public void setPrivateKey(byte[] privateKey) { + this.privateKey = privateKey; + } + + public byte[] getPublicKey() { + return publicKey; + } + + public void setPublicKey(byte[] publicKey) { + this.publicKey = publicKey; + } + + public String getPassphrase() { + return passphrase; + } + + public void setPassphrase(String passphrase) { + this.passphrase = passphrase; + } + + public String getKnownHostsFilePath() { + return knownHostsFilePath; + } + + public void setKnownHostsFilePath(String knownHostsFilePath) { + this.knownHostsFilePath = knownHostsFilePath; + } + + public String getStrictHostKeyChecking() { + return strictHostKeyChecking; + } + + public void setStrictHostKeyChecking(String strictHostKeyChecking) { + this.strictHostKeyChecking = strictHostKeyChecking; + } +} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/models/runners/ssh/SSHServerInfo.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/models/runners/ssh/SSHServerInfo.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/runners/ssh/SSHServerInfo.java new file mode 100644 index 0000000..050ce29 --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/runners/ssh/SSHServerInfo.java @@ -0,0 +1,52 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.models.runners.ssh; + +import org.apache.airavata.models.resources.ServerInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SSHServerInfo extends ServerInfo { + private final static Logger logger = LoggerFactory.getLogger(SSHServerInfo.class); + + SSHKeyAuthentication authentication; + int sshPort; + + public SSHServerInfo(String userName, String host, SSHKeyAuthentication authentication, int port){ + super(userName, host, ComProtocol.SSH, port); + this.authentication = authentication; + this.sshPort = port; + } + + public SSHServerInfo(String userName, String host, SSHKeyAuthentication authentication){ + super(userName, host, ComProtocol.SSH, 22); + this.authentication = authentication; + this.sshPort = 22; + } + + public SSHKeyAuthentication getAuthentication() { + return authentication; + } + + public int getSshPort() { + return sshPort; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/models/runners/ssh/SSHUserInfo.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/models/runners/ssh/SSHUserInfo.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/runners/ssh/SSHUserInfo.java new file mode 100644 index 0000000..3f39725 --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/models/runners/ssh/SSHUserInfo.java @@ -0,0 +1,66 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.models.runners.ssh; + +import com.jcraft.jsch.UserInfo; + +public class SSHUserInfo implements UserInfo { + + private String userName; + private String password; + private String passphrase; + + public SSHUserInfo(String userName, String password, String passphrase) { + this.userName = userName; + this.password = password; + this.passphrase = passphrase; + } + + @Override + public String getPassphrase() { + return null; + } + + @Override + public String getPassword() { + return null; + } + + @Override + public boolean promptPassword(String s) { + return false; + } + + @Override + public boolean promptPassphrase(String s) { + return false; + } + + @Override + public boolean promptYesNo(String s) { + return false; + } + + @Override + public void showMessage(String s) { + + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/Authentication.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/Authentication.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/Authentication.java deleted file mode 100644 index 426f55e..0000000 --- a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/Authentication.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.resources; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class Authentication { - private final static Logger logger = LoggerFactory.getLogger(Authentication.class); - - protected String userName; -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/OutputParser.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/OutputParser.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/OutputParser.java deleted file mode 100644 index de523e4..0000000 --- a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/OutputParser.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.resources; - -import org.apache.airavata.resources.batch.JobStatus; - -import java.util.Map; - -public interface OutputParser { - /** - * This can be used to parseSingleJob the result of a job submission to get the JobID - * @param rawOutput - * @return the job id as a String, or null if no job id found - */ - public String parseJobSubmission(String rawOutput)throws Exception; - - - /** - * Parse output return by job submission task and identify jobSubmission failures. - * @param rawOutput - * @return true if job submission has been failed, false otherwise. - */ - public boolean isJobSubmissionFailed(String rawOutput); - - - /** - * This can be used to get the job status from the output - * @param jobID - * @param rawOutput - */ - public JobStatus parseJobStatus(String jobID, String rawOutput)throws Exception; - - /** - * This can be used to parseSingleJob a big output and get multipleJob statuses - * @param statusMap list of status map will return and key will be the job ID - * @param rawOutput - */ - public void parseJobStatuses(String userName, Map<String, JobStatus> statusMap, String rawOutput)throws Exception; - - /** - * filter the jobId value of given JobName from rawOutput - * @param jobName - * @param rawOutput - * @return - * @throws Exception - */ - public String parseJobId(String jobName, String rawOutput) throws Exception; -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/ServerInfo.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/ServerInfo.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/ServerInfo.java deleted file mode 100644 index 0bb46f5..0000000 --- a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/ServerInfo.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.resources; - -public class ServerInfo { - - public static enum ComProtocol {SSH, LOCAL} - - protected String host; - protected String userName; - protected int port; - protected ComProtocol comProtocol; - - public ServerInfo(){} - - public ServerInfo(String userName, String host, ComProtocol comProtocol, int port) { - this.userName = userName; - this.host = host; - this.comProtocol = comProtocol; - this.port = port; - } - - public String getHost() { - return host; - } - - public String getUserName() { - return userName; - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/BatchJobOutputParser.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/BatchJobOutputParser.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/BatchJobOutputParser.java index 2278663..eab3ca7 100644 --- a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/BatchJobOutputParser.java +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/BatchJobOutputParser.java @@ -20,7 +20,8 @@ */ package org.apache.airavata.resources.batch; -import org.apache.airavata.resources.OutputParser; +import org.apache.airavata.models.resources.JobStatus; +import org.apache.airavata.models.resources.hpc.OutputParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/BigRed2.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/BigRed2.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/BigRed2.java new file mode 100644 index 0000000..aac31b6 --- /dev/null +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/BigRed2.java @@ -0,0 +1,70 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.resources.batch; + +import org.apache.airavata.models.*; +import org.apache.airavata.models.resources.Authentication; +import org.apache.airavata.models.resources.ServerInfo; +import org.apache.airavata.models.resources.hpc.JobManagerConfiguration; +import org.apache.airavata.models.resources.hpc.PBSJobConfiguration; +import org.apache.airavata.models.runners.ssh.SSHKeyAuthentication; +import org.apache.airavata.models.runners.ssh.SSHServerInfo; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; + +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +public class BigRed2 extends HPCBatchResource { + private final static Logger logger = LoggerFactory.getLogger(BigRed2.class); + + public BigRed2(ServerInfo serverInfo, JobManagerConfiguration jobManagerConfiguration, Authentication authentication) throws Exception { + super(serverInfo, jobManagerConfiguration, authentication); + } + + public BigRed2() throws Exception { + //These should be read from a database. For simplicity I have hardcoded them + SSHKeyAuthentication br2SshAuthentication = new SSHKeyAuthentication( + Constants.loginUserName, + IOUtils.toByteArray(BigRed2.class.getClassLoader().getResourceAsStream("ssh/id_rsa")), + IOUtils.toByteArray(BigRed2.class.getClassLoader().getResourceAsStream("ssh/id_rsa.pub")), + "dummy", + BigRed2.class.getClassLoader().getResource("ssh/known_hosts").getPath(), + false + ); + SSHServerInfo br2 = new SSHServerInfo(Constants.loginUserName, "bigred2.uits.iu.edu", br2SshAuthentication,22); + Map<JobManagerConfiguration.JobManagerCommand, String> jobManagerCommands = new HashMap<>(); + jobManagerCommands.put(JobManagerConfiguration.JobManagerCommand.SUBMISSION, "qsub"); + jobManagerCommands.put(JobManagerConfiguration.JobManagerCommand.JOB_MONITORING, "qstat"); + jobManagerCommands.put(JobManagerConfiguration.JobManagerCommand.DELETION, "qdel"); + + JobManagerConfiguration pbsJobConfiguration = new PBSJobConfiguration(PBSJobConfiguration.class.getClassLoader(). + getResource("resources/batch/PBS_Groovy.template").getPath(), ".pbs", + "/opt/torque/torque-5.0.1/bin", jobManagerCommands, new BatchJobOutputParser()); + + this.serverInfo = (SSHServerInfo) br2; + this.authentication = (SSHKeyAuthentication) br2SshAuthentication; + this.jobManagerConfiguration = pbsJobConfiguration; + this.outputParser = jobManagerConfiguration.getParser(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/GroovyMap.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/GroovyMap.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/GroovyMap.java deleted file mode 100644 index 8709e8f..0000000 --- a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/GroovyMap.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.resources.batch; - -import java.util.HashMap; -import java.util.Optional; - -public class GroovyMap extends HashMap<String, Object> { - - - public GroovyMap() { - super(); - // to mitigate groovy exception groovy.lang.MissingPropertyException: No such property: <name> for class: groovy.lang.Binding - addDefaultValues(); - } - - public GroovyMap add(Script name, Object value){ - put(name.name, value); - return this; - } - - @Override - public Object get(Object key) { - return super.getOrDefault(key, null); - } - - public Object get(Script script) { - return get(script.name); - } - - public Optional<String> getStringValue(Script script) { - Object obj = get(script); - if (obj instanceof String) { - return Optional.of((String) obj); - } else if (obj == null) { - return Optional.empty(); - } else { - throw new IllegalArgumentException("Value is not String type"); - } - } - - private void addDefaultValues() { - this.add(Script.SHELL_NAME, null) - .add(Script.QUEUE_NAME, null) - .add(Script.NODES, null) - .add(Script.CPU_COUNT, null) - .add(Script.MAIL_ADDRESS, null) - .add(Script.ACCOUNT_STRING, null) - .add(Script.MAX_WALL_TIME, null) - .add(Script.JOB_NAME, null) - .add(Script.STANDARD_OUT_FILE, null) - .add(Script.STANDARD_ERROR_FILE, null) - .add(Script.QUALITY_OF_SERVICE, null) - .add(Script.RESERVATION, null) - .add(Script.EXPORTS, null) - .add(Script.MODULE_COMMANDS, null) - .add(Script.SCRATCH_LOCATION, null) - .add(Script.WORKING_DIR, null) - .add(Script.PRE_JOB_COMMANDS, null) - .add(Script.JOB_SUBMITTER_COMMAND, null) - .add(Script.EXECUTABLE_PATH, null) - .add(Script.INPUTS, null) - .add(Script.POST_JOB_COMMANDS, null) - .add(Script.USED_MEM, null) - .add(Script.PROCESS_PER_NODE, null) - .add(Script.CHASSIS_NAME, null) - .add(Script.INPUT_DIR, null) - .add(Script.OUTPUT_DIR, null) - .add(Script.USER_NAME, null) - .add(Script.GATEWAY_ID, null) - .add(Script.GATEWAY_USER_NAME, null) - .add(Script.APPLICATION_NAME, null); - } - -} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/HPCBatchResource.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/HPCBatchResource.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/HPCBatchResource.java index 46f2a48..2572de0 100644 --- a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/HPCBatchResource.java +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/HPCBatchResource.java @@ -21,16 +21,18 @@ package org.apache.airavata.resources.batch; import com.jcraft.jsch.JSchException; -import groovy.lang.Writable; +import groovy.lang.*; import groovy.text.GStringTemplateEngine; import groovy.text.TemplateEngine; -import org.apache.airavata.resources.Authentication; -import org.apache.airavata.resources.OutputParser; -import org.apache.airavata.resources.ServerInfo; +import org.apache.airavata.models.resources.Authentication; +import org.apache.airavata.models.resources.JobSubmissionOutput; +import org.apache.airavata.models.resources.ServerInfo; +import org.apache.airavata.models.resources.hpc.*; +import org.apache.airavata.models.resources.hpc.Script; +import org.apache.airavata.models.runners.ssh.SSHKeyAuthentication; +import org.apache.airavata.models.runners.ssh.SSHServerInfo; import org.apache.airavata.runners.ssh.SSHCommandOutputReader; -import org.apache.airavata.runners.ssh.SSHKeyAuthentication; import org.apache.airavata.runners.ssh.SSHRunner; -import org.apache.airavata.runners.ssh.SSHServerInfo; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,10 +46,10 @@ public class HPCBatchResource { private static final int MAX_RETRY_COUNT = 3; - private ServerInfo serverInfo; - private Authentication authentication; - private JobManagerConfiguration jobManagerConfiguration; - private OutputParser outputParser; + protected ServerInfo serverInfo; + protected Authentication authentication; + protected JobManagerConfiguration jobManagerConfiguration; + protected OutputParser outputParser; public HPCBatchResource(ServerInfo serverInfo, JobManagerConfiguration jobManagerConfiguration, Authentication authentication) throws Exception { @@ -64,9 +66,12 @@ public class HPCBatchResource { this.outputParser = jobManagerConfiguration.getParser(); } - public JobSubmissionOutput submitBatchJob(String routingKey, GroovyMap groovyMap, String workingDirectory) throws Exception { + public HPCBatchResource() {} + + public JobSubmissionOutput submitBatchJob(String routingKey, GroovyMap groovyMap) throws Exception { File tempJobFile = File.createTempFile("temp_job", jobManagerConfiguration.getScriptExtension()); FileUtils.writeStringToFile(tempJobFile, generateScript(groovyMap)); + String workingDirectory = groovyMap.get(Script.WORKING_DIR).toString(); String jobScriptFilePath = tempJobFile.getPath(); JobSubmissionOutput jsoutput = new JobSubmissionOutput(); http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobManagerConfiguration.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobManagerConfiguration.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobManagerConfiguration.java deleted file mode 100644 index bc0bfcd..0000000 --- a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobManagerConfiguration.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.resources.batch; - -import org.apache.airavata.resources.OutputParser; - -public interface JobManagerConfiguration { - public static enum JobManagerCommand{JOB_MONITORING, SUBMISSION, DELETION}; - - public RawCommandInfo getCancelCommand(String jobID); - - public String getJobDescriptionTemplateName(); - - public RawCommandInfo getMonitorCommand(String jobID); - - public RawCommandInfo getUserBasedMonitorCommand(String userName); - - public RawCommandInfo getJobIdMonitorCommand(String jobName , String userName); - - public String getScriptExtension(); - - public RawCommandInfo getSubmitCommand(String workingDirectory, String pbsFilePath); - - public OutputParser getParser(); - - public String getInstalledPath(); - - public String getBaseCancelCommand(); - - public String getBaseMonitorCommand(); - - public String getBaseSubmitCommand(); -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobStatus.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobStatus.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobStatus.java deleted file mode 100644 index faa17e2..0000000 --- a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobStatus.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.resources.batch; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class JobStatus { - private final static Logger logger = LoggerFactory.getLogger(JobStatus.class); -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobSubmissionOutput.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobSubmissionOutput.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobSubmissionOutput.java deleted file mode 100644 index 44bec6b..0000000 --- a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobSubmissionOutput.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.resources.batch; - -public class JobSubmissionOutput { - - private int exitCode = Integer.MIN_VALUE; - private String stdOut; - private String stdErr; - private String command; - private String jobId; - private boolean isJobSubmissionFailed; - private String failureReason; - - public int getExitCode() { - return exitCode; - } - - public void setExitCode(int exitCode) { - this.exitCode = exitCode; - } - - public String getStdOut() { - return stdOut; - } - - public void setStdOut(String stdOut) { - this.stdOut = stdOut; - } - - public String getStdErr() { - return stdErr; - } - - public void setStdErr(String stdErr) { - this.stdErr = stdErr; - } - - public String getCommand() { - return command; - } - - public void setCommand(String command) { - this.command = command; - } - - public String getJobId() { - return jobId; - } - - public void setJobId(String jobId) { - this.jobId = jobId; - } - - public boolean isJobSubmissionFailed() { - return isJobSubmissionFailed; - } - - public void setJobSubmissionFailed(boolean jobSubmissionFailed) { - isJobSubmissionFailed = jobSubmissionFailed; - } - - public String getFailureReason() { - return failureReason; - } - - public void setFailureReason(String failureReason) { - this.failureReason = failureReason; - } -} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/PBSJobConfiguration.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/PBSJobConfiguration.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/PBSJobConfiguration.java deleted file mode 100644 index 07bccc0..0000000 --- a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/PBSJobConfiguration.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.resources.batch; - -import org.apache.airavata.resources.OutputParser; -import org.apache.commons.io.FilenameUtils; - -import java.io.File; -import java.util.Map; - -public class PBSJobConfiguration implements JobManagerConfiguration { - - private final Map<JobManagerCommand, String> jobManagerCommands; - private String jobDescriptionTemplateName; - private String scriptExtension; - private String installedPath; - private OutputParser parser; - - public PBSJobConfiguration(String jobDescriptionTemplateName, String scriptExtension, String installedPath, - Map<JobManagerCommand, String> jobManagerCommands, OutputParser parser) { - this.jobDescriptionTemplateName = jobDescriptionTemplateName; - this.scriptExtension = scriptExtension; - this.parser = parser; - installedPath = installedPath.trim(); - if (installedPath.endsWith("/")) { - this.installedPath = installedPath; - } else { - this.installedPath = installedPath + "/"; - } - this.jobManagerCommands = jobManagerCommands; - } - - public RawCommandInfo getCancelCommand(String jobID) { - return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.DELETION).trim() + " " + - jobID); - } - - public String getJobDescriptionTemplateName() { - return jobDescriptionTemplateName; - } - - public void setJobDescriptionTemplateName(String jobDescriptionTemplateName) { - this.jobDescriptionTemplateName = jobDescriptionTemplateName; - } - - public RawCommandInfo getMonitorCommand(String jobID) { - return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.JOB_MONITORING).trim() - + " -f " + jobID); - } - - public String getScriptExtension() { - return scriptExtension; - } - - public RawCommandInfo getSubmitCommand(String workingDirectory, String pbsFilePath) { - return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.SUBMISSION).trim() + " " + - workingDirectory + File.separator + FilenameUtils.getName(pbsFilePath)); - } - - public String getInstalledPath() { - return installedPath; - } - - public void setInstalledPath(String installedPath) { - this.installedPath = installedPath; - } - - public OutputParser getParser() { - return parser; - } - - public void setParser(OutputParser parser) { - this.parser = parser; - } - - public RawCommandInfo getUserBasedMonitorCommand(String userName) { - return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.JOB_MONITORING).trim() - + " -u " + userName); - } - - @Override - public RawCommandInfo getJobIdMonitorCommand(String jobName, String userName) { - // For PBS there is no option to get jobDetails by JobName, so we search with userName - return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.JOB_MONITORING).trim() - + " -u " + userName + " -f | grep \"Job_Name = " + jobName + "\" -B1"); - } - - @Override - public String getBaseCancelCommand() { - return jobManagerCommands.get(JobManagerCommand.DELETION).trim(); - } - - @Override - public String getBaseMonitorCommand() { - return jobManagerCommands.get(JobManagerCommand.JOB_MONITORING).trim(); - } - - @Override - public String getBaseSubmitCommand() { - return jobManagerCommands.get(JobManagerCommand.SUBMISSION).trim(); - } -} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/RawCommandInfo.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/RawCommandInfo.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/RawCommandInfo.java deleted file mode 100644 index 3193505..0000000 --- a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/RawCommandInfo.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.resources.batch; - -public class RawCommandInfo { - private String rawCommand; - - public RawCommandInfo(String cmd) { - this.rawCommand = cmd; - } - - public String getCommand() { - return this.rawCommand; - } - - public String getRawCommand() { - return rawCommand; - } - - public void setRawCommand(String rawCommand) { - this.rawCommand = rawCommand; - } -} \ No newline at end of file