Repository: airavata-sandbox Updated Branches: refs/heads/master 2e56d5ce3 -> e76ace3f9
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/Script.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/Script.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/Script.java deleted file mode 100644 index 7f49753..0000000 --- a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/Script.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.resources.batch; - -public enum Script { - - SHELL_NAME("shellName"), - QUEUE_NAME("queueName"), - NODES("nodes"), - CPU_COUNT("cpuCount"), - MAIL_ADDRESS("mailAddress"), - ACCOUNT_STRING("accountString"), - MAX_WALL_TIME("maxWallTime"), - JOB_NAME("jobName"), - STANDARD_OUT_FILE("standardOutFile"), - STANDARD_ERROR_FILE("standardErrorFile"), - QUALITY_OF_SERVICE("qualityOfService"), - RESERVATION("reservation"), - EXPORTS("exports"), - MODULE_COMMANDS("moduleCommands"), - SCRATCH_LOCATION("scratchLocation"), - WORKING_DIR("workingDirectory"), - PRE_JOB_COMMANDS("preJobCommands"), - JOB_SUBMITTER_COMMAND("jobSubmitterCommand"), - EXECUTABLE_PATH("executablePath"), - INPUTS("inputs"), - POST_JOB_COMMANDS("postJobCommands"), - USED_MEM("usedMem"), - PROCESS_PER_NODE("processPerNode"), - CHASSIS_NAME("chassisName"), - INPUT_DIR("inputDir"), - OUTPUT_DIR("outputDir"), - USER_NAME("userName"), - GATEWAY_ID("gatewayId"), - GATEWAY_USER_NAME("gatewayUserName"), - APPLICATION_NAME("applicationName"), - ; - - String name; - Script(String name) { - this.name = name; - } -} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/CommandOutput.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/CommandOutput.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/CommandOutput.java deleted file mode 100644 index c3f8877..0000000 --- a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/CommandOutput.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.runners; - - -import com.jcraft.jsch.Channel; - -import java.io.OutputStream; - -/** - * Output of a certain command. - */ -public interface CommandOutput { - - /** - * Gets the output of the command as a stream. - * @param channel Command output as a stream. - */ - void onOutput(Channel channel); - - /** - * Gets standard error as a output stream. - * @return Command error as a stream. - */ - OutputStream getStandardError(); - - /** - * The command exit code. - * @param code The program exit code - */ - void exitCode(int code); - - /** - * Return the exit code of the command execution. - * @return exit code - */ - int getExitCode(); - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHApiException.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHApiException.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHApiException.java deleted file mode 100644 index e117351..0000000 --- a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHApiException.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.runners.ssh; - -/** - * An exception class to wrap SSH command execution related errors. - */ -public class SSHApiException extends Exception { - - public SSHApiException(String message) { - super(message); - } - - public SSHApiException(String message, Exception e) { - super(message, e); - } -} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHKeyAuthentication.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHKeyAuthentication.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHKeyAuthentication.java deleted file mode 100644 index 0acb357..0000000 --- a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHKeyAuthentication.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.runners.ssh; - -import org.apache.airavata.resources.Authentication; - -public class SSHKeyAuthentication extends Authentication{ - private byte[] privateKey; - private byte[] publicKey; - private String passphrase; - private String knownHostsFilePath; - private String strictHostKeyChecking; // yes or no - - public SSHKeyAuthentication(String userName, byte[] privateKey, byte[] publicKey, String passphrase, String knownHostsFilePath, boolean strictHostKeyChecking) { - this.userName = userName; - this.privateKey = privateKey; - this.publicKey = publicKey; - this.passphrase = passphrase; - this.knownHostsFilePath = knownHostsFilePath; - if(strictHostKeyChecking){ - this.strictHostKeyChecking = "yes"; - }else{ - this.strictHostKeyChecking = "no"; - } - } - - public String getUserName() { - return userName; - } - - public void setUserName(String userName) { - this.userName = userName; - } - - public byte[] getPrivateKey() { - return privateKey; - } - - public void setPrivateKey(byte[] privateKey) { - this.privateKey = privateKey; - } - - public byte[] getPublicKey() { - return publicKey; - } - - public void setPublicKey(byte[] publicKey) { - this.publicKey = publicKey; - } - - public String getPassphrase() { - return passphrase; - } - - public void setPassphrase(String passphrase) { - this.passphrase = passphrase; - } - - public String getKnownHostsFilePath() { - return knownHostsFilePath; - } - - public void setKnownHostsFilePath(String knownHostsFilePath) { - this.knownHostsFilePath = knownHostsFilePath; - } - - public String getStrictHostKeyChecking() { - return strictHostKeyChecking; - } - - public void setStrictHostKeyChecking(String strictHostKeyChecking) { - this.strictHostKeyChecking = strictHostKeyChecking; - } -} http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHRunner.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHRunner.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHRunner.java index 6bb29a9..6e7bda2 100644 --- a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHRunner.java +++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHRunner.java @@ -21,6 +21,10 @@ package org.apache.airavata.runners.ssh; import com.jcraft.jsch.*; +import org.apache.airavata.models.runners.ssh.SSHApiException; +import org.apache.airavata.models.runners.ssh.SSHKeyAuthentication; +import org.apache.airavata.models.runners.ssh.SSHServerInfo; +import org.apache.airavata.models.runners.ssh.SSHUserInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHServerInfo.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHServerInfo.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHServerInfo.java deleted file mode 100644 index 6a99bc1..0000000 --- a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHServerInfo.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.runners.ssh; - -import org.apache.airavata.resources.ServerInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SSHServerInfo extends ServerInfo { - private final static Logger logger = LoggerFactory.getLogger(SSHServerInfo.class); - - SSHKeyAuthentication authentication; - int sshPort; - - public SSHServerInfo(String userName, String host, SSHKeyAuthentication authentication, int port){ - super(userName, host, ComProtocol.SSH, port); - this.authentication = authentication; - this.sshPort = port; - } - - public SSHServerInfo(String userName, String host, SSHKeyAuthentication authentication){ - super(userName, host, ComProtocol.SSH, 22); - this.authentication = authentication; - this.sshPort = 22; - } - - public SSHKeyAuthentication getAuthentication() { - return authentication; - } - - public int getSshPort() { - return sshPort; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHUserInfo.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHUserInfo.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHUserInfo.java deleted file mode 100644 index 9f2792b..0000000 --- a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHUserInfo.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.runners.ssh; - -import com.jcraft.jsch.UserInfo; - -public class SSHUserInfo implements UserInfo { - - private String userName; - private String password; - private String passphrase; - - public SSHUserInfo(String userName, String password, String passphrase) { - this.userName = userName; - this.password = password; - this.passphrase = passphrase; - } - - @Override - public String getPassphrase() { - return null; - } - - @Override - public String getPassword() { - return null; - } - - @Override - public boolean promptPassword(String s) { - return false; - } - - @Override - public boolean promptPassphrase(String s) { - return false; - } - - @Override - public boolean promptYesNo(String s) { - return false; - } - - @Override - public void showMessage(String s) { - - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/resources/code_tf.py ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/resources/code_tf.py b/airavata-layered-architecture/src/main/resources/code_tf.py new file mode 100644 index 0000000..104e246 --- /dev/null +++ b/airavata-layered-architecture/src/main/resources/code_tf.py @@ -0,0 +1,33 @@ +import tensorflow as tf +import numpy as np +import numpy.polynomial.polynomial as poly +import time + +from random import randint + +tf.reset_default_graph() +num_neurons = 5000 + +indices = [] +values = [] +for i in range(num_neurons): + for j in range(num_neurons): + x = 3 + if i != j: + number = randint(0, 99) + if number < 5: + indices.append([i, j]) + values.append(1.0/5) + +connections = tf.SparseTensor(indices=indices, values=values, dense_shape=[num_neurons, num_neurons]) + +neuron_values = tf.Variable(np.ones(num_neurons), dtype=tf.float32) + +mul_product = tf.sparse_tensor_dense_matmul(connections, tf.reshape(neuron_values, shape=(num_neurons, 1))) + +sess = tf.Session() +sess.run(tf.global_variables_initializer()) + +temp = time.time() +output = sess.run(mul_product) +print(time.time()-temp) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/resources/job_tf.pbs ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/resources/job_tf.pbs b/airavata-layered-architecture/src/main/resources/job_tf.pbs new file mode 100644 index 0000000..a4d3485 --- /dev/null +++ b/airavata-layered-architecture/src/main/resources/job_tf.pbs @@ -0,0 +1,18 @@ +#!/bin/bash +# file to submit non interactive jobs to bigred2 + +#PBS -l nodes=1:ppn=16 +#PBS -l gres=ccm +#PBS -q debug_gpu +#PBS -l walltime=00:30:00 + +#PBS -o STDOUT.txt +#PBS -e STDERR.txt + +#PBS -m bea +#PBS -M supun.nakand...@gmail.com + +module load ccm +module load anaconda2/4.2.0 +source activate tensorflow_env +ccmrun python ~/airavata/code_tf.py \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/resources/ssh/id_rsa ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/resources/ssh/id_rsa b/airavata-layered-architecture/src/main/resources/ssh/id_rsa new file mode 100644 index 0000000..96430a2 --- /dev/null +++ b/airavata-layered-architecture/src/main/resources/ssh/id_rsa @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-128-CBC,3AF46750EC6C202FA7E19DD44994DCBC + +T1iV/k7DS8Li4pwYyySEBpPv8OWbd5Z1h5Z1a4FNScLZGsLm8CvduJxvDrQWNOfd +qC5M8VNXvATN2K3+vj82aqLQ9qpYoQAC1yggHWEfqemlolKk89ICLK4LVjMZX2dw +l3kHwNgIa0rDZtE8rxV+8M9u+IQ7/j88FcLgqVzPjBHLlnrWveZQ+IhuxFTg8E5I +1iolp/gaAkE9cfnEPpje3vf+m0Dkb0Td2vO7V2keBvyIG8s4CwvKWQiifRYu4jQJ +Vit6/UgySbEzd+SxYBPOruOCM8IEeLpUX7xE6Zt2fgA9CwMqTpuIFPKmCfj2ON5h +bgJkLA3pLgs+imzBOYEZLBB5UO2iNuDbYTfF7nYvSUXaSCn3ekzleGgtJ6L2+03U +D5bntrVDwIV510dRAN2EiyB744p+tzOm5mbTPFHtrefiZdfumsj2er4rPy/9FA09 +4aBlPPS0yCGTV87Edtuve4UcFxkpc5f3QgdxUOAZQI1KHILluZjQI93yMAt++KeF +Fvf7dEtT8VVrwzsCGb9Pc1lyboGJxohm63Xe7os2Rn2i4LHXVX6NhoPC4L+U8YJC +53q68G3ExNjeIu/N7y2QKtrXYLnqzvxZ0Tm1jDVp2UyG2yVzOeiITrXCG9qPBmrs +59fcIXsWb3xFQMOeKN6WzOaMEGPnCI+SJJwIc2PzR0OKeBB/rBy8zNaIOb8gqHb7 +uSd0UJxASa5801TlJtdhtGyVGZJtG7vyHIn8vF/GT+VQY7ehbRRPGRRij3J+ruvS +rMZU8oTntvMUdDLkI/JeD6yInzX4n1rWGLoHeck75nA4CHuHBYF+Pgr5q8I/dts+ +gFUgqFq2s9vn3haP/pnJfQikuEWfd62S1IU7x+8Ia96LomitbRBwZM1fG3Tphtv0 +7i+bLIUx+voRn/ayPRnasnQVoQgYe7g1CTm6zE1N8d8OFY9P6AoUhOmSxyrC6W8M +ZttxFntaHwF1s07bfl+Yr1KJaodMfvP5JBpCjK+dMyk0temC8Y3dzC4Fu0CLEwrD +p25iD0xvY0QYa4oRgZMD6gGmww6olPoda7/txZp+gu0GOcgYogozT8mcUSq0RZor +cWL/lrjcjZHbeDbyWQehChdV4i6y7CGjQhry1f3V+TQtSDRxX5344UpQalhKLjsf +XdMMJYuD2u/1BV4Lg+O6aFwRwGQtNkF5AMTq3FDluptWdAcAU3dsO9sNTCTmgCwx +lD4Pew4lBYZe0WZ+2o+0X1efyGJJ6/lKjfpVF0eNwQqFQ8SxSFZvUrwJsK+NujjH +nhjobs64n4MRAPnckVqO+vlY408uUDUoNOzROAC9zDl216ZXT8zTmWeoNkQS6DKb +JXtbXPDHWdo9VLik5h08FjmwdNoMS18mkSDpkwkP3hGCQnWhKZDjlU6kfksRbeee +t6WI4pKIbzHQJV+CLm1uzC1o7EfsD4dbSpefpJUD5D+mTH8MBkjb7p5+E2/TER62 +rIdXjpHEA/s8W3bmdTSam1mqA3VUBGVQryIfAv2qOyVjQx2Y28+pDTAtMT6FnYUZ +sVUucxUp+TE7WqIQX8AzjNAudu25R7YPQvgQXjSrA6G6bxNQNC/TQeT+vgO4g8Ah +-----END RSA PRIVATE KEY----- \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/resources/ssh/id_rsa.pub ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/resources/ssh/id_rsa.pub b/airavata-layered-architecture/src/main/resources/ssh/id_rsa.pub new file mode 100644 index 0000000..e378be0 --- /dev/null +++ b/airavata-layered-architecture/src/main/resources/ssh/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCr9VP22p0I+2W5o/klPv/OvfTihvcBQwZXKPrSLFF+OB9nVNtIfDDETIwwex7mknn3Kks1jFvEdKMrvRjOFeFInDv3N40LjohHu4v2tiawAON7MOLpz/iX5dWp0wteixlDKfGe7PAEMAk054kLSDiB3em2zBK4d9ApedA5k2JG1dmAsNK0KkbfgFPd5+iXrzgTg4XiefHQoaCSUyS7w6t8645djbYOP+b+SJtgslaf2RqeoBVvrA6YQJE1pUYjcm9yL4KwyqaPo+N/2XZ6xys5+WN8svtL3uRduENU1MQSTpdFq+GLCY4SgLMFgLJKoxHjcjPRfKyE/eYk1gQA7b/Z snaka...@149-161-141-51.dhcp-bl.indiana.edu http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/main/resources/ssh/known_hosts ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/main/resources/ssh/known_hosts b/airavata-layered-architecture/src/main/resources/ssh/known_hosts new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/test/java/org/apache/airavata/Constants.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/test/java/org/apache/airavata/Constants.java b/airavata-layered-architecture/src/test/java/org/apache/airavata/Constants.java deleted file mode 100644 index 539cfb5..0000000 --- a/airavata-layered-architecture/src/test/java/org/apache/airavata/Constants.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class Constants { - private final static Logger logger = LoggerFactory.getLogger(Constants.class); - - public static final String loginUserName = "snakanda"; -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/test/java/org/apache/airavata/applications/hpc/HPCApplicationTest.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/test/java/org/apache/airavata/applications/hpc/HPCApplicationTest.java b/airavata-layered-architecture/src/test/java/org/apache/airavata/applications/hpc/HPCApplicationTest.java new file mode 100644 index 0000000..34640d8 --- /dev/null +++ b/airavata-layered-architecture/src/test/java/org/apache/airavata/applications/hpc/HPCApplicationTest.java @@ -0,0 +1,46 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.applications.hpc; + +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +public class HPCApplicationTest { + private final static Logger logger = LoggerFactory.getLogger(HPCApplicationTest.class); + + @Test + public void testTensorFlow() throws Exception { + Map<String, String> inputs = new HashMap<>(); + inputs.put("code_tf.py", HPCApplicationTest.class.getClassLoader().getResource("code_tf.py").getPath()); + TensorFlow tensorflow = new TensorFlow("tensorflow", inputs); + + String routingKey = UUID.randomUUID().toString(); + HPCApplicationExecutor hpcApplicationExecutor = new HPCApplicationExecutor(); + String computerResource = "bigred2.uits.iu.edu"; + hpcApplicationExecutor.executeApplication(routingKey, tensorflow, computerResource, + tensorflow.getJobSpecification(computerResource)); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/test/java/org/apache/airavata/resources/batch/HPCBatchResourceTest.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/test/java/org/apache/airavata/resources/batch/HPCBatchResourceTest.java b/airavata-layered-architecture/src/test/java/org/apache/airavata/resources/batch/HPCBatchResourceTest.java index 525e28f..19b12ef 100644 --- a/airavata-layered-architecture/src/test/java/org/apache/airavata/resources/batch/HPCBatchResourceTest.java +++ b/airavata-layered-architecture/src/test/java/org/apache/airavata/resources/batch/HPCBatchResourceTest.java @@ -20,10 +20,14 @@ */ package org.apache.airavata.resources.batch; -import org.apache.airavata.Constants; -import org.apache.airavata.runners.ssh.SSHKeyAuthentication; +import org.apache.airavata.models.*; +import org.apache.airavata.models.resources.hpc.GroovyMap; +import org.apache.airavata.models.resources.hpc.JobManagerConfiguration; +import org.apache.airavata.models.resources.hpc.PBSJobConfiguration; +import org.apache.airavata.models.resources.hpc.Script; +import org.apache.airavata.models.runners.ssh.SSHKeyAuthentication; +import org.apache.airavata.models.runners.ssh.SSHServerInfo; import org.apache.airavata.runners.ssh.SSHRunnerTest; -import org.apache.airavata.runners.ssh.SSHServerInfo; import org.apache.commons.io.IOUtils; import org.junit.Test; import org.slf4j.Logger; @@ -35,7 +39,7 @@ public class HPCBatchResourceTest { private final static Logger logger = LoggerFactory.getLogger(HPCBatchResourceTest.class); @Test - public void test() throws Exception { + public void testGenericResource() throws Exception { SSHKeyAuthentication br2SshAuthentication = new SSHKeyAuthentication( Constants.loginUserName, IOUtils.toByteArray(SSHRunnerTest.class.getClassLoader().getResourceAsStream("ssh/id_rsa")), @@ -55,6 +59,23 @@ public class HPCBatchResourceTest { "/opt/torque/torque-5.0.1/bin", jobManagerCommands, new BatchJobOutputParser()); HPCBatchResource hpcBatchResource = new HPCBatchResource(br2, pbsJobConfiguration, br2SshAuthentication); + + String routingKey = UUID.randomUUID().toString(); + + //Defining all the configurations + hpcBatchResource.submitBatchJob(routingKey, getJobSpecification()); + } + + @Test + public void testBR2() throws Exception { + String routingKey = UUID.randomUUID().toString(); + + //Defining only which resource to use and the implementation handles the details + BigRed2 bigRed2 = new BigRed2(); + bigRed2.submitBatchJob(routingKey, getJobSpecification()); + } + + private GroovyMap getJobSpecification(){ GroovyMap jobSpecification = new GroovyMap(); jobSpecification.add(Script.NODES, 1); @@ -80,7 +101,6 @@ public class HPCBatchResourceTest { jobSpecification.add(Script.STANDARD_OUT_FILE, "STDOUT.txt"); jobSpecification.add(Script.STANDARD_ERROR_FILE, "STDERR.txt"); - String routingKey = UUID.randomUUID().toString(); - hpcBatchResource.submitBatchJob(routingKey, jobSpecification, "~/airavata"); + return jobSpecification; } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/test/java/org/apache/airavata/runners/ssh/SSHRunnerTest.java ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/test/java/org/apache/airavata/runners/ssh/SSHRunnerTest.java b/airavata-layered-architecture/src/test/java/org/apache/airavata/runners/ssh/SSHRunnerTest.java index 39bb55c..6e9ae50 100644 --- a/airavata-layered-architecture/src/test/java/org/apache/airavata/runners/ssh/SSHRunnerTest.java +++ b/airavata-layered-architecture/src/test/java/org/apache/airavata/runners/ssh/SSHRunnerTest.java @@ -20,7 +20,9 @@ */ package org.apache.airavata.runners.ssh; -import org.apache.airavata.Constants; +import org.apache.airavata.models.Constants; +import org.apache.airavata.models.runners.ssh.SSHKeyAuthentication; +import org.apache.airavata.models.runners.ssh.SSHServerInfo; import org.apache.commons.io.IOUtils; import org.junit.Assert; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/test/resources/code_tf.py ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/test/resources/code_tf.py b/airavata-layered-architecture/src/test/resources/code_tf.py deleted file mode 100644 index 221e960..0000000 --- a/airavata-layered-architecture/src/test/resources/code_tf.py +++ /dev/null @@ -1,30 +0,0 @@ -import tensorflow as tf -import numpy as np -import numpy.polynomial.polynomial as poly - -from random import randint - -tf.reset_default_graph() -num_neurons = 5000 - -indices = [] -values = [] -for i in range(num_neurons): - for j in range(num_neurons): - x = 3 - if i != j: - number = randint(0, 99) - if number < 5: - indices.append([i, j]) - values.append(1.0/5) - -connections = tf.SparseTensor(indices=indices, values=values, dense_shape=[num_neurons, num_neurons]) - -neuron_values = tf.Variable(np.ones(num_neurons), dtype=tf.float32) - -mul_product = tf.sparse_tensor_dense_matmul(connections, tf.reshape(neuron_values, shape=(num_neurons, 1))) - -sess = tf.Session() -sess.run(tf.global_variables_initializer()) - -output = sess.run(mul_product) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/test/resources/job_tf.pbs ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/test/resources/job_tf.pbs b/airavata-layered-architecture/src/test/resources/job_tf.pbs deleted file mode 100644 index a4d3485..0000000 --- a/airavata-layered-architecture/src/test/resources/job_tf.pbs +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# file to submit non interactive jobs to bigred2 - -#PBS -l nodes=1:ppn=16 -#PBS -l gres=ccm -#PBS -q debug_gpu -#PBS -l walltime=00:30:00 - -#PBS -o STDOUT.txt -#PBS -e STDERR.txt - -#PBS -m bea -#PBS -M supun.nakand...@gmail.com - -module load ccm -module load anaconda2/4.2.0 -source activate tensorflow_env -ccmrun python ~/airavata/code_tf.py \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/test/resources/ssh/id_rsa.pub ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/test/resources/ssh/id_rsa.pub b/airavata-layered-architecture/src/test/resources/ssh/id_rsa.pub deleted file mode 100644 index e378be0..0000000 --- a/airavata-layered-architecture/src/test/resources/ssh/id_rsa.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCr9VP22p0I+2W5o/klPv/OvfTihvcBQwZXKPrSLFF+OB9nVNtIfDDETIwwex7mknn3Kks1jFvEdKMrvRjOFeFInDv3N40LjohHu4v2tiawAON7MOLpz/iX5dWp0wteixlDKfGe7PAEMAk054kLSDiB3em2zBK4d9ApedA5k2JG1dmAsNK0KkbfgFPd5+iXrzgTg4XiefHQoaCSUyS7w6t8645djbYOP+b+SJtgslaf2RqeoBVvrA6YQJE1pUYjcm9yL4KwyqaPo+N/2XZ6xys5+WN8svtL3uRduENU1MQSTpdFq+GLCY4SgLMFgLJKoxHjcjPRfKyE/eYk1gQA7b/Z snaka...@149-161-141-51.dhcp-bl.indiana.edu http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/50df93aa/airavata-layered-architecture/src/test/resources/ssh/known_hosts ---------------------------------------------------------------------- diff --git a/airavata-layered-architecture/src/test/resources/ssh/known_hosts b/airavata-layered-architecture/src/test/resources/ssh/known_hosts deleted file mode 100644 index e69de29..0000000
