http://git-wip-us.apache.org/repos/asf/airavata/blob/9bf53d2c/modules/data-manager/src/main/java/org/apache/airavata/data/manager/utils/ssh/SSHUtils.java ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/main/java/org/apache/airavata/data/manager/utils/ssh/SSHUtils.java b/modules/data-manager/src/main/java/org/apache/airavata/data/manager/utils/ssh/SSHUtils.java new file mode 100644 index 0000000..343d55e --- /dev/null +++ b/modules/data-manager/src/main/java/org/apache/airavata/data/manager/utils/ssh/SSHUtils.java @@ -0,0 +1,500 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.data.manager.utils.ssh; + +import com.jcraft.jsch.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.util.Arrays; +import java.util.List; + +/** + * Utility class to do all ssh and scp related things. + */ +public class SSHUtils { + private static final Logger log = LoggerFactory.getLogger(SSHUtils.class); + + + /** + * This will copy a local file to a remote location + * + * @param remoteFile remote location you want to transfer the file, this cannot be a directory, if user pass + * a dirctory we do copy it to that directory but we simply return the directory name + * todo handle the directory name as input and return the proper final output file name + * @param localFile Local file to transfer, this can be a directory + * @return returns the final remote file path, so that users can use the new file location + */ + public static String scpTo(String localFile, String remoteFile, Session session) throws IOException, + JSchException, SSHApiException { + FileInputStream fis = null; + String prefix = null; + if (new File(localFile).isDirectory()) { + prefix = localFile + File.separator; + } + boolean ptimestamp = true; + + // exec 'scp -t rfile' remotely + String command = "scp " + (ptimestamp ? "-p" : "") + " -t " + remoteFile; + Channel channel = session.openChannel("exec"); + + SSHStandardOutReader stdOutReader = new SSHStandardOutReader(); + ((ChannelExec) channel).setErrStream(stdOutReader.getStandardError()); + ((ChannelExec) channel).setCommand(command); + + // get I/O streams for remote scp + OutputStream out = channel.getOutputStream(); + InputStream in = channel.getInputStream(); + + channel.connect(); + + if (checkAck(in) != 0) { + String error = "Error Reading input Stream"; + log.error(error); + throw new SSHApiException(error); + } + + File _lfile = new File(localFile); + + if (ptimestamp) { + command = "T" + (_lfile.lastModified() / 1000) + " 0"; + // The access time should be sent here, + // but it is not accessible with JavaAPI ;-< + command += (" " + (_lfile.lastModified() / 1000) + " 0\n"); + out.write(command.getBytes()); + out.flush(); + if (checkAck(in) != 0) { + String error = "Error Reading input Stream"; + log.error(error); + throw new SSHApiException(error); + } + } + + // send "C0644 filesize filename", where filename should not include '/' + long filesize = _lfile.length(); + command = "C0644 " + filesize + " "; + if (localFile.lastIndexOf('/') > 0) { + command += localFile.substring(localFile.lastIndexOf('/') + 1); + } else { + command += localFile; + } + command += "\n"; + out.write(command.getBytes()); + out.flush(); + if (checkAck(in) != 0) { + String error = "Error Reading input Stream"; + log.error(error); + throw new SSHApiException(error); + } + + // send a content of localFile + fis = new FileInputStream(localFile); + byte[] buf = new byte[1024]; + while (true) { + int len = fis.read(buf, 0, buf.length); + if (len <= 0) break; + out.write(buf, 0, len); //out.flush(); + } + fis.close(); + fis = null; + // send '\0' + buf[0] = 0; + out.write(buf, 0, 1); + out.flush(); + if (checkAck(in) != 0) { + String error = "Error Reading input Stream"; + log.error(error); + throw new SSHApiException(error); + } + out.close(); + stdOutReader.onOutput(channel); + + + channel.disconnect(); + if (stdOutReader.getStdErrorString().contains("scp:")) { + throw new SSHApiException(stdOutReader.getStdErrorString()); + } + //since remote file is always a file we just return the file + return remoteFile; + } + + /** + * This method will copy a remote file to a local directory + * + * @param remoteFile remote file path, this has to be a full qualified path + * @param localFile This is the local file to copy, this can be a directory too + * @return returns the final local file path of the new file came from the remote resource + */ + public static void scpFrom(String remoteFile, String localFile, Session session) throws IOException, + JSchException, SSHApiException { + FileOutputStream fos = null; + try { + String prefix = null; + if (new File(localFile).isDirectory()) { + prefix = localFile + File.separator; + } + + // exec 'scp -f remotefile' remotely + String command = "scp -f " + remoteFile; + Channel channel = session.openChannel("exec"); + ((ChannelExec) channel).setCommand(command); + + SSHStandardOutReader stdOutReader = new SSHStandardOutReader(); + ((ChannelExec) channel).setErrStream(stdOutReader.getStandardError()); + // get I/O streams for remote scp + OutputStream out = channel.getOutputStream(); + InputStream in = channel.getInputStream(); + + if (!channel.isClosed()){ + channel.connect(); + } + + byte[] buf = new byte[1024]; + + // send '\0' + buf[0] = 0; + out.write(buf, 0, 1); + out.flush(); + + while (true) { + int c = checkAck(in); + if (c != 'C') { + break; + } + + // read '0644 ' + in.read(buf, 0, 5); + + long filesize = 0L; + while (true) { + if (in.read(buf, 0, 1) < 0) { + // error + break; + } + if (buf[0] == ' ') break; + filesize = filesize * 10L + (long) (buf[0] - '0'); + } + + String file = null; + for (int i = 0; ; i++) { + in.read(buf, i, 1); + if (buf[i] == (byte) 0x0a) { + file = new String(buf, 0, i); + break; + } + } + + //System.out.println("filesize="+filesize+", file="+file); + + // send '\0' + buf[0] = 0; + out.write(buf, 0, 1); + out.flush(); + + // read a content of lfile + fos = new FileOutputStream(prefix == null ? localFile : prefix + file); + int foo; + while (true) { + if (buf.length < filesize) foo = buf.length; + else foo = (int) filesize; + foo = in.read(buf, 0, foo); + if (foo < 0) { + // error + break; + } + fos.write(buf, 0, foo); + filesize -= foo; + if (filesize == 0L) break; + } + fos.close(); + fos = null; + + if (checkAck(in) != 0) { + String error = "Error transfering the file content"; + log.error(error); + throw new SSHApiException(error); + } + + // send '\0' + buf[0] = 0; + out.write(buf, 0, 1); + out.flush(); + } + stdOutReader.onOutput(channel); + if (stdOutReader.getStdErrorString().contains("scp:")) { + throw new SSHApiException(stdOutReader.getStdErrorString()); + } + + } catch (Exception e) { + log.error(e.getMessage(), e); + } finally { + try { + if (fos != null) fos.close(); + } catch (Exception ee) { + } + } + } + + /** + * This method will copy a remote file to a local directory + * + * @param sourceFile remote file path, this has to be a full qualified path + * @param sourceSession JSch session for source + * @param destinationFile This is the local file to copy, this can be a directory too + * @param destinationSession JSch Session for target + * @return returns the final local file path of the new file came from the remote resource + */ + public static void scpThirdParty(String sourceFile, Session sourceSession, String destinationFile, Session destinationSession) throws + IOException, JSchException { + OutputStream sout = null; + InputStream sin = null; + OutputStream dout = null; + InputStream din = null; + try { + String prefix = null; + + // exec 'scp -f sourceFile' + String sourceCommand = "scp -f " + sourceFile; + Channel sourceChannel = sourceSession.openChannel("exec"); + ((ChannelExec) sourceChannel).setCommand(sourceCommand); + SSHStandardOutReader sourceStdOutReader = new SSHStandardOutReader(); + ((ChannelExec) sourceChannel).setErrStream(sourceStdOutReader.getStandardError()); + // get I/O streams for remote scp + sout = sourceChannel.getOutputStream(); + sin = sourceChannel.getInputStream(); + sourceChannel.connect(); + + + boolean ptimestamp = true; + // exec 'scp -t destinationFile' + String command = "scp " + (ptimestamp ? "-p" : "") + " -t " + destinationFile; + Channel targetChannel = destinationSession.openChannel("exec"); + SSHStandardOutReader targetStdOutReader = new SSHStandardOutReader(); + ((ChannelExec) targetChannel).setErrStream(targetStdOutReader.getStandardError()); + ((ChannelExec) targetChannel).setCommand(command); + // get I/O streams for remote scp + dout = targetChannel.getOutputStream(); + din = targetChannel.getInputStream(); + targetChannel.connect(); + + if (checkAck(din) != 0) { + String error = "Error Reading input Stream"; + log.error(error); + throw new Exception(error); + } + + + byte[] buf = new byte[1024]; + + // send '\0' + buf[0] = 0; + sout.write(buf, 0, 1); + sout.flush(); + + while (true) { + int c = checkAck(sin); + if (c != 'C') { + break; + } + + // read '0644 ' + sin.read(buf, 0, 5); + + long fileSize = 0L; + while (true) { + if (sin.read(buf, 0, 1) < 0) { + // error + break; + } + if (buf[0] == ' ') break; + fileSize = fileSize * 10L + (long) (buf[0] - '0'); + } + + String fileName = null; + for (int i = 0; ; i++) { + sin.read(buf, i, 1); + if (buf[i] == (byte) 0x0a) { + fileName = new String(buf, 0, i); + break; + } + } + String initData = "C0644 " + fileSize + " " + fileName + "\n"; + assert dout != null; + dout.write(initData.getBytes()); + dout.flush(); + + // send '\0' to source + buf[0] = 0; + sout.write(buf, 0, 1); + sout.flush(); + + int rLength; + while (true) { + if (buf.length < fileSize) rLength = buf.length; + else rLength = (int) fileSize; + rLength = sin.read(buf, 0, rLength); // read content of the source File + if (rLength < 0) { + // error + break; + } + dout.write(buf, 0, rLength); // write to destination file + fileSize -= rLength; + if (fileSize == 0L) break; + } + + // send '\0' to target + buf[0] = 0; + dout.write(buf, 0, 1); + dout.flush(); + if (checkAck(din) != 0) { + String error = "Error Reading input Stream"; + log.error(error); + throw new Exception(error); + } + dout.close(); + dout = null; + + if (checkAck(sin) != 0) { + String error = "Error transfering the file content"; + log.error(error); + throw new Exception(error); + } + + // send '\0' + buf[0] = 0; + sout.write(buf, 0, 1); + sout.flush(); + } + + } catch (Exception e) { + log.error(e.getMessage(), e); + } finally { + try { + if (dout != null) dout.close(); + } catch (Exception ee) { + log.error("", ee); + } + try { + if (din != null) din.close(); + } catch (Exception ee) { + log.error("", ee); + } + try { + if (sout != null) sout.close(); + } catch (Exception ee) { + log.error("", ee); + } + try { + if (din != null) din.close(); + } catch (Exception ee) { + log.error("", ee); + } + } + } + + public static void makeDirectory(String path, Session session) throws IOException, JSchException, SSHApiException { + + // exec 'scp -t rfile' remotely + String command = "mkdir -p " + path; + Channel channel = session.openChannel("exec"); + SSHStandardOutReader stdOutReader = new SSHStandardOutReader(); + + ((ChannelExec) channel).setCommand(command); + + + ((ChannelExec) channel).setErrStream(stdOutReader.getStandardError()); + try { + channel.connect(); + } catch (JSchException e) { + + channel.disconnect(); + + throw new SSHApiException("Unable to retrieve command output. Command - " + command + + " on server - " + session.getHost() + ":" + session.getPort() + + " connecting user name - " + + session.getUserName(), e); + } + stdOutReader.onOutput(channel); + if (stdOutReader.getStdErrorString().contains("mkdir:")) { + throw new SSHApiException(stdOutReader.getStdErrorString()); + } + + channel.disconnect(); + } + + public static List<String> listDirectory(String path, Session session) throws IOException, JSchException, + SSHApiException { + + // exec 'scp -t rfile' remotely + String command = "ls " + path; + Channel channel = session.openChannel("exec"); + SSHStandardOutReader stdOutReader = new SSHStandardOutReader(); + + ((ChannelExec) channel).setCommand(command); + + + ((ChannelExec) channel).setErrStream(stdOutReader.getStandardError()); + try { + channel.connect(); + } catch (JSchException e) { + + channel.disconnect(); + + throw new SSHApiException("Unable to retrieve command output. Command - " + command + + " on server - " + session.getHost() + ":" + session.getPort() + + " connecting user name - " + + session.getUserName(), e); + } + stdOutReader.onOutput(channel); + stdOutReader.getStdOutputString(); + if (stdOutReader.getStdErrorString().contains("ls:")) { + throw new SSHApiException(stdOutReader.getStdErrorString()); + } + channel.disconnect(); + return Arrays.asList(stdOutReader.getStdOutputString().split("\n")); + } + + + static int checkAck(InputStream in) throws IOException { + int b = in.read(); + if (b == 0) return b; + if (b == -1) return b; + + if (b == 1 || b == 2) { + StringBuffer sb = new StringBuffer(); + int c; + do { + c = in.read(); + sb.append((char) c); + } + while (c != '\n'); + if (b == 1) { // error + System.out.print(sb.toString()); + } + if (b == 2) { // fatal error + System.out.print(sb.toString()); + } + } + return b; + } + +}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bf53d2c/modules/data-manager/src/test/java/org/apache/airavata/data/manager/DataManagerImplTest.java ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/test/java/org/apache/airavata/data/manager/DataManagerImplTest.java b/modules/data-manager/src/test/java/org/apache/airavata/data/manager/DataManagerImplTest.java index d8ecafd..ecab2af 100644 --- a/modules/data-manager/src/test/java/org/apache/airavata/data/manager/DataManagerImplTest.java +++ b/modules/data-manager/src/test/java/org/apache/airavata/data/manager/DataManagerImplTest.java @@ -20,10 +20,12 @@ */ package org.apache.airavata.data.manager; -import org.apache.airavata.data.manager.utils.AppCatInit; import org.apache.airavata.data.manager.utils.DataCatInit; +import org.apache.airavata.data.manager.utils.ssh.SSHKeyAuthentication; import org.apache.airavata.model.data.resource.DataReplicaLocationModel; import org.apache.airavata.model.data.resource.DataResourceModel; +import org.apache.airavata.registry.core.experiment.catalog.impl.RegistryFactory; +import org.apache.airavata.registry.cpi.DataCatalog; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -31,26 +33,45 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.net.URI; import java.util.ArrayList; import java.util.List; import java.util.UUID; public class DataManagerImplTest { private final static Logger logger = LoggerFactory.getLogger(DataManagerImplTest.class); - private static AppCatInit appCatInit; private static DataCatInit dataCatInit; private static DataManager dataManager; private static DataResourceModel dataResourceModel; private static DataReplicaLocationModel dataReplicaLocationModel; + @BeforeClass public static void setUp() { try { System.out.println("********** SET UP ************"); - appCatInit = new AppCatInit("appcatalog-derby.sql"); - appCatInit.initializeDB(); dataCatInit = new DataCatInit("datacatalog-derby.sql"); dataCatInit.initializeDB(); - dataManager = DataManagerFactory.getDataManager(); + DataCatalog dataCatalog = RegistryFactory.getDataCatalog(); + SSHKeyAuthentication sshKeyAuthentication = new SSHKeyAuthentication(); + ClassLoader classLoader = DataManagerImplTest.class.getClassLoader(); + File privateKey = new File(classLoader.getResource("id_rsa").getFile()); + File publicKey = new File(classLoader.getResource("id_rsa.pub").getFile()); + File knownHosts = new File(classLoader.getResource("known_hosts").getFile()); + sshKeyAuthentication.setUserName("airavata"); + sshKeyAuthentication.setPrivateKeyFilePath(privateKey.getAbsolutePath()); + sshKeyAuthentication.setPublicKeyFilePath(publicKey.getAbsolutePath()); + sshKeyAuthentication.setKnownHostsFilePath(knownHosts.getAbsolutePath()); + sshKeyAuthentication.setPassphrase("airavata_2015"); + sshKeyAuthentication.setStrictHostKeyChecking("no"); + +// sshKeyAuthentication.setUserName("pga"); +// sshKeyAuthentication.setKnownHostsFilePath("/Users/supun/.ssh/known_hosts"); +// sshKeyAuthentication.setPublicKeyFilePath("/Users/supun/.ssh/id_rsa.pub"); +// sshKeyAuthentication.setPrivateKeyFilePath("/Users/supun/.ssh/id_rsa"); +// sshKeyAuthentication.setPassphrase(""); + + dataManager = new DataManagerImpl(dataCatalog, sshKeyAuthentication); dataResourceModel = new DataResourceModel(); dataResourceModel.setResourceName("test-file.txt"); dataReplicaLocationModel = new DataReplicaLocationModel(); @@ -58,7 +79,7 @@ public class DataManagerImplTest { ArrayList<String> dataLocations = new ArrayList<>(); dataLocations.add("scp://g75.iu.xsede.org:/var/www/portal/experimentData/test-file.txt"); dataReplicaLocationModel.setDataLocations(dataLocations); - } catch (DataManagerException e) { + } catch (Exception e) { logger.error(e.getMessage(), e); } } @@ -200,4 +221,53 @@ public class DataManagerImplTest { Assert.fail(); } } + + @Test + public void testCopyResource(){ + try { + String resourceId = dataManager.publishResource(dataResourceModel); + File temp = File.createTempFile("temp-file-1", ".tmp"); + dataReplicaLocationModel.setResourceId(resourceId); + ArrayList<String> dataLocations = new ArrayList<>(); + dataLocations.add(DataManagerConstants.LOCAL_URI_SCHEME+"://"+temp.getAbsolutePath()); + dataReplicaLocationModel.setDataLocations(dataLocations); + String replicaId = dataManager.publishReplicaLocation(dataReplicaLocationModel); + String destPath = DataManagerConstants.LOCAL_URI_SCHEME+"://" + System.getProperty("java.io.tmpdir") + + File.separator + "temp-file-2"; + dataManager.copyResource(resourceId,replicaId, destPath); + File newFile = new File((new URI(destPath)).getPath()); + Assert.assertTrue(newFile.exists()); + DataReplicaLocationModel newReplicaLocation = new DataReplicaLocationModel(); + newReplicaLocation.setReplicaName("new replica location"); + ArrayList<String> newDataLocations = new ArrayList<>(); + newDataLocations.add(DataManagerConstants.LOCAL_URI_SCHEME+"://"+newFile.getAbsolutePath()); + newReplicaLocation.setDataLocations(newDataLocations); + newReplicaLocation.setResourceId(resourceId); + dataManager.publishReplicaLocation(newReplicaLocation); + List<DataReplicaLocationModel> replicaLocationModelList = dataManager.getAllReplicaLocations(resourceId); + Assert.assertTrue(replicaLocationModelList.size()==2); + +// String scpDestLocation = DataManagerConstants.SCP_URI_SCHEME+"://gw75.iu.xsede.org:/var/www/portal" + +// "/experimentData/scnakandala/temp-file"; +// boolean result = dataManager.copyResource(resourceId, replicaId, scpDestLocation); +// Assert.assertTrue(result); +// newDataLocations = new ArrayList<>(); +// newDataLocations.add(scpDestLocation); +// newReplicaLocation.setDataLocations(newDataLocations); +// newReplicaLocation.setResourceId(resourceId); +// replicaId = dataManager.publishReplicaLocation(newReplicaLocation); +// String scpDestLocationNew = DataManagerConstants.SCP_URI_SCHEME+"://gw75.iu.xsede.org:22/var/www/portal" + +// "/experimentData/scnakandala/temp-file-new"; +// result = dataManager.copyResource(resourceId, replicaId, scpDestLocationNew); +// Assert.assertTrue(result); +// String localDestLocation = DataManagerConstants.LOCAL_URI_SCHEME+"://" + System.getProperty("java.io.tmpdir") +// + File.separator + "temp-file-sup" + System.currentTimeMillis(); +// result = dataManager.copyResource(resourceId, replicaId, localDestLocation); +// Assert.assertTrue(result); +// Assert.assertTrue((new File((new URI(localDestLocation)).getPath())).exists()); + } catch (Exception e) { + e.printStackTrace(); + Assert.fail(); + } + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/9bf53d2c/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/AppCatInit.java ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/AppCatInit.java b/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/AppCatInit.java deleted file mode 100644 index c2f66c7..0000000 --- a/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/AppCatInit.java +++ /dev/null @@ -1,320 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -package org.apache.airavata.data.manager.utils; - -import org.apache.airavata.common.exception.ApplicationSettingsException; -import org.apache.airavata.common.utils.ServerSettings; -import org.apache.derby.drda.NetworkServerControl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.InetAddress; -import java.net.URI; -import java.sql.*; -import java.util.StringTokenizer; - -public class AppCatInit { - private static final Logger logger = LoggerFactory.getLogger(AppCatInit.class); - public static final String DERBY_SERVER_MODE_SYS_PROPERTY = "derby.drda.startNetworkServer"; - public String scriptName = "appcatalog-derby.sql"; - private NetworkServerControl server; - private static final String delimiter = ";"; - public static final String COMPUTE_RESOURCE_TABLE = "COMPUTE_RESOURCE"; - private String jdbcUrl = null; - private String jdbcDriver = null; - private String jdbcUser = null; - private String jdbcPassword = null; - - public AppCatInit(String scriptName) { - this.scriptName = scriptName; - } - - public static boolean checkStringBufferEndsWith(StringBuffer buffer, String suffix) { - if (suffix.length() > buffer.length()) { - return false; - } - // this loop is done on purpose to avoid memory allocation performance - // problems on various JDKs - // StringBuffer.lastIndexOf() was introduced in jdk 1.4 and - // implementation is ok though does allocation/copying - // StringBuffer.toString().endsWith() does massive memory - // allocation/copying on JDK 1.5 - // See http://issues.apache.org/bugzilla/show_bug.cgi?id=37169 - int endIndex = suffix.length() - 1; - int bufferIndex = buffer.length() - 1; - while (endIndex >= 0) { - if (buffer.charAt(bufferIndex) != suffix.charAt(endIndex)) { - return false; - } - bufferIndex--; - endIndex--; - } - return true; - } - - private static boolean isServerStarted(NetworkServerControl server, int ntries) - { - for (int i = 1; i <= ntries; i ++) - { - try { - Thread.sleep(500); - server.ping(); - return true; - } - catch (Exception e) { - if (i == ntries) - return false; - } - } - return false; - } - - public void initializeDB() { - - try{ - jdbcDriver = ServerSettings.getSetting("appcatalog.jdbc.driver"); - jdbcUrl = ServerSettings.getSetting("appcatalog.jdbc.url"); - jdbcUser = ServerSettings.getSetting("appcatalog.jdbc.user"); - jdbcPassword = ServerSettings.getSetting("appcatalog.jdbc.password"); - jdbcUrl = jdbcUrl + "?" + "user=" + jdbcUser + "&" + "password=" + jdbcPassword; - } catch (ApplicationSettingsException e) { - logger.error("Unable to read properties", e); - } - - startDerbyInServerMode(); - if(!isServerStarted(server, 20)){ - throw new RuntimeException("Derby server cound not started within five seconds..."); - } -// startDerbyInEmbeddedMode(); - - Connection conn = null; - try { - Class.forName(jdbcDriver).newInstance(); - conn = DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPassword); - if (!isDatabaseStructureCreated(COMPUTE_RESOURCE_TABLE, conn)) { - executeSQLScript(conn); - logger.info("New Database created for App Catalog !!!"); - } else { - logger.debug("Database already created for App Catalog!"); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - throw new RuntimeException("Database failure", e); - } finally { - try { - if (conn != null){ - if (!conn.getAutoCommit()) { - conn.commit(); - } - conn.close(); - } - } catch (SQLException e) { - logger.error(e.getMessage(), e); - } - } - } - - public static boolean isDatabaseStructureCreated(String tableName, Connection conn) { - try { - System.out.println("Running a query to test the database tables existence."); - // check whether the tables are already created with a query - Statement statement = null; - try { - statement = conn.createStatement(); - ResultSet rs = statement.executeQuery("select * from " + tableName); - if (rs != null) { - rs.close(); - } - } finally { - try { - if (statement != null) { - statement.close(); - } - } catch (SQLException e) { - return false; - } - } - } catch (SQLException e) { - return false; - } - - return true; - } - - private void executeSQLScript(Connection conn) throws Exception { - StringBuffer sql = new StringBuffer(); - BufferedReader reader = null; - try{ - - InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(scriptName); - reader = new BufferedReader(new InputStreamReader(inputStream)); - String line; - while ((line = reader.readLine()) != null) { - line = line.trim(); - if (line.startsWith("//")) { - continue; - } - if (line.startsWith("--")) { - continue; - } - StringTokenizer st = new StringTokenizer(line); - if (st.hasMoreTokens()) { - String token = st.nextToken(); - if ("REM".equalsIgnoreCase(token)) { - continue; - } - } - sql.append(" ").append(line); - - // SQL defines "--" as a comment to EOL - // and in Oracle it may contain a hint - // so we cannot just remove it, instead we must end it - if (line.indexOf("--") >= 0) { - sql.append("\n"); - } - if ((checkStringBufferEndsWith(sql, delimiter))) { - executeSQL(sql.substring(0, sql.length() - delimiter.length()), conn); - sql.replace(0, sql.length(), ""); - } - } - // Catch any statements not followed by ; - if (sql.length() > 0) { - executeSQL(sql.toString(), conn); - } - }catch (IOException e){ - logger.error("Error occurred while executing SQL script for creating Airavata database", e); - throw new Exception("Error occurred while executing SQL script for creating Airavata database", e); - }finally { - if (reader != null) { - reader.close(); - } - - } - - } - - private static void executeSQL(String sql, Connection conn) throws Exception { - // Check and ignore empty statements - if ("".equals(sql.trim())) { - return; - } - - Statement statement = null; - try { - logger.debug("SQL : " + sql); - - boolean ret; - int updateCount = 0, updateCountTotal = 0; - statement = conn.createStatement(); - ret = statement.execute(sql); - updateCount = statement.getUpdateCount(); - do { - if (!ret) { - if (updateCount != -1) { - updateCountTotal += updateCount; - } - } - ret = statement.getMoreResults(); - if (ret) { - updateCount = statement.getUpdateCount(); - } - } while (ret); - - logger.debug(sql + " : " + updateCountTotal + " rows affected"); - - SQLWarning warning = conn.getWarnings(); - while (warning != null) { - logger.warn(warning + " sql warning"); - warning = warning.getNextWarning(); - } - conn.clearWarnings(); - } catch (SQLException e) { - if (e.getSQLState().equals("X0Y32")) { - // eliminating the table already exception for the derby - // database - logger.info("Table Already Exists", e); - } else { - throw new Exception("Error occurred while executing : " + sql, e); - } - } finally { - if (statement != null) { - try { - statement.close(); - } catch (SQLException e) { - logger.error("Error occurred while closing result set.", e); - } - } - } - } - - private void startDerbyInServerMode() { - try { - System.setProperty(DERBY_SERVER_MODE_SYS_PROPERTY, "true"); - server = new NetworkServerControl(InetAddress.getByName("0.0.0.0"), - 20000, - jdbcUser, jdbcPassword); - java.io.PrintWriter consoleWriter = new java.io.PrintWriter(System.out, true); - server.start(consoleWriter); - } catch (IOException e) { - logger.error("Unable to start Apache derby in the server mode! Check whether " + - "specified port is available"); - } catch (Exception e) { - logger.error("Unable to start Apache derby in the server mode! Check whether " + - "specified port is available"); - } - - } - - public static int getPort(String jdbcURL){ - try{ - String cleanURI = jdbcURL.substring(5); - URI uri = URI.create(cleanURI); - return uri.getPort(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - return -1; - } - } - - private void startDerbyInEmbeddedMode(){ - try { - Class.forName("org.apache.derby.jdbc.EmbeddedDriver"); - DriverManager.getConnection("jdbc:derby:memory:unit-testing-jpa;create=true").close(); - } catch (ClassNotFoundException e) { - logger.error(e.getMessage(), e); - } catch (SQLException e) { - logger.error(e.getMessage(), e); - } - } - - public void stopDerbyServer() { - try { - server.shutdown(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } -} http://git-wip-us.apache.org/repos/asf/airavata/blob/9bf53d2c/modules/data-manager/src/test/resources/id_rsa ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/test/resources/id_rsa b/modules/data-manager/src/test/resources/id_rsa new file mode 100644 index 0000000..4527347 --- /dev/null +++ b/modules/data-manager/src/test/resources/id_rsa @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-128-CBC,4882DC230357A4388E4A1690AB275C75 + +/REcchTscfYjoYsJ8W6C1o0elvwpEpz/orX7+uoRTv0Iar9KxCOIg+Ms24cfO2Jn +0PLHbB373XWTCtLMWojY0EgOJvQlzP3T4rzllGW/4nm+xhioxmxHY5nV3hi1JeFg +H4BS9OJH1bfZvlLZiIgjmDQEKM2hMRjm2SXv9wrreZfiAgF1D7XfIYd15Tkaitzh +bGkeAAFucC1Se3dw8aDvou5o9hLSyq/o6qg3WUcUMTSl8Cwh/8ubVjKpdonB0JO8 +hekvaeMkKCe4lh4ITsvQ09pyC3OpK+KzOVHbJeFZNGZ3uSNDnegG1I4AOpd+ypAt +ZF5NZDQBW/LS85pghRS/LEjPJhW1M2Z05lpDWNpN4B2T51BomOvHKZKcuZk97V45 +1s/BI8I3sUsEyUqHumZXdSpnsSY10GfE+agY+0ltLRNtC+UMyDOZnQXepG7AZsAp +xRlzuNvCt21oWdZPUuc8eHStAjzaxnbOlEG1AO1FMS11uXKSH8MfMSk0yYBUvQSv +Ug5GZv9jkuSlHxBNGZqZU/xPNzkgixeRiBGyQAg75UcxXjTgQGawSL196baR3kS7 +66o+13Aq3hEtnUqePb1iD0XqxDKRdNe8oZiI+eTzjqNL6m0dWQeVLUO8nzRqR37S ++/fpY/NQWkEubeRmunVq074rdjvZOk9oBXBTe5eMDhUbqM3yKN9GYx9MYDjAK06Y +nvZmtl6So979JyGlAFXuc9YqxYiS/BGmOZ/iYLeSC0Bn566c0+nHydfnsxdZWicO +YzvZrqte/LywJWZkcjHv9St8hCram346M9QTBDAp9id2RuQiIW0cx4PUOeEzN+AC +AEQM5qIkqZYyx9z3eTZJIP/aT7bwQdRlZ7lJ8AIwX8AaLAwlDNDdwAjXojiPuqe3 +AWnfrWSHsdiojgoSxbmbDH8lZz110tk3TId2yjOZC5Fsy4eOW68/GQZMKeWD3ylT +ajpNYZyYUlERUJDsFDoJA877lbaGii/D5vAU5HSZUnFJeiCXTGWfBlouG/GnNhrk +I+sfajxagJBHTa1Gamo12h/0xUftxuWqp1Sh3uTInoGowmWYEazNw9FFm/fjT6Sr +sYpw0UD77vPsziWqaZyLyvc0hVEyupnvKwz0B2IdvTHOgF//XEezjy3Ivu6c/cJs +qt4z8WACX+yeepzyEkJ5aVluHUtjNG+EfkFhnhg9dGZ5pOVVck6M6fo4b8dw15NF +e0OPgLU8hfwqUwalG3QVrlZjhdPlDk0rAKTDWiQuvozkLcrT862tYkxrKnAcMdZN +uEgbbKkuF32csp9zbgJtGBf/yvI1BaomIN/ZSp02h0onlAYC4u/ybA/bjDEWy8Qv +u1m9YKpQYQtXiY5bB3++FzVVOBpkjyBPd/S9m5apPw4FP5IECSszcGFz7ReAG2AY +u4wTnDcL19tGn0k1Z03VVgESAh04SxVZBKzC6BtTEYwwYkP2HoG11XEEfIQ5JY7W +IowHL1WcPx69C2TbGt3JGkIjPatXR67sFKGiRceJzTEjsoUbjaqsRJO8QZN6vq31 +8iuI0VFM8VAG4bTc10s/18QaPd3lpKIo/OIIxUJc+c5GrppcHib9CEaCD0qn7vEd +-----END RSA PRIVATE KEY----- http://git-wip-us.apache.org/repos/asf/airavata/blob/9bf53d2c/modules/data-manager/src/test/resources/id_rsa.pub ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/test/resources/id_rsa.pub b/modules/data-manager/src/test/resources/id_rsa.pub new file mode 100644 index 0000000..c7eaaf3 --- /dev/null +++ b/modules/data-manager/src/test/resources/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDH5ggPJf0yYW39EEW2PsEojMLILZX/EvaMnghsj9/0HPY3C+DpFx5PAeYND4ITxZWnRjQmKsYN6Z7tL4wYbWfhMPUk8urKEB8ncdQK7tZZHdJuUmprfIi02LgeE2JmM13eimF2fEMRrCFqqMnCM63ihE/jzcyhwosRjlKfzYv5C23a8orGZh9g1V2LxjA7lkUyga7B/nsJBk75SA/uJZ4TcfeWS3okk2Pv8mKLmN23WKfAqvQadDkUphYPITa7MpdaUoVIDkUllzE82V6KQ4tXcbh1xSktv6k8sR9e7BsZSJVHFHoDitV9dcQGZIOdzmpNw2ZfmNW6IdrfrrVER7Cb [email protected] http://git-wip-us.apache.org/repos/asf/airavata/blob/9bf53d2c/modules/data-manager/src/test/resources/known_hosts ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/test/resources/known_hosts b/modules/data-manager/src/test/resources/known_hosts new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/airavata/blob/9bf53d2c/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/model/DataResource.java ---------------------------------------------------------------------- diff --git a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/model/DataResource.java b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/model/DataResource.java index 392f2f3..fef50c9 100644 --- a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/model/DataResource.java +++ b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/model/DataResource.java @@ -34,6 +34,7 @@ public class DataResource { private String resourceId; private String resourceName; private String resourceDescription; + private String ownerName; private int resourceSize; private Timestamp creationTime; private Timestamp lastModifiedTime; @@ -68,6 +69,15 @@ public class DataResource { this.resourceDescription = resourceDescription; } + @Column(name = "OWNER_NAME") + public String getOwnerName() { + return ownerName; + } + + public void setOwnerName(String ownerName) { + this.ownerName = ownerName; + } + @Column(name = "RESOURCE_SIZE") public int getResourceSize() { return resourceSize; http://git-wip-us.apache.org/repos/asf/airavata/blob/9bf53d2c/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/ThriftDataModelConversion.java ---------------------------------------------------------------------- diff --git a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/ThriftDataModelConversion.java b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/ThriftDataModelConversion.java index 6be55c3..1b3a995 100644 --- a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/ThriftDataModelConversion.java +++ b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/ThriftDataModelConversion.java @@ -41,6 +41,7 @@ public class ThriftDataModelConversion { dataResourceModel.setResourceId(dataResource.getResourceId()); dataResourceModel.setResourceName(dataResource.getResourceName()); dataResourceModel.setResourceDescription(dataResource.getResourceDescription()); + dataResourceModel.setOwnerName(dataResource.getOwnerName()); dataResourceModel.setResourceSize(dataResource.getResourceSize()); dataResourceModel.setCreationTime(dataResource.getCreationTime().getTime()); dataResourceModel.setLastModifiedTime(dataResource.getLastModifiedTime().getTime()); @@ -61,6 +62,7 @@ public class ThriftDataModelConversion { dataResource.setResourceId(dataResourceModel.getResourceId()); dataResource.setResourceName(dataResourceModel.getResourceName()); dataResource.setResourceDescription(dataResourceModel.getResourceDescription()); + dataResource.setOwnerName(dataResourceModel.getOwnerName()); dataResource.setResourceSize(dataResourceModel.getResourceSize()); dataResource.setCreationTime(new Timestamp(dataResourceModel.getCreationTime())); dataResource.setLastModifiedTime(new Timestamp(dataResourceModel.getLastModifiedTime())); http://git-wip-us.apache.org/repos/asf/airavata/blob/9bf53d2c/modules/registry/registry-core/src/main/resources/datacatalog-derby.sql ---------------------------------------------------------------------- diff --git a/modules/registry/registry-core/src/main/resources/datacatalog-derby.sql b/modules/registry/registry-core/src/main/resources/datacatalog-derby.sql index ef4a777..22e5a70 100644 --- a/modules/registry/registry-core/src/main/resources/datacatalog-derby.sql +++ b/modules/registry/registry-core/src/main/resources/datacatalog-derby.sql @@ -24,6 +24,7 @@ CREATE TABLE DATA_RESOURCE RESOURCE_ID VARCHAR (255), RESOURCE_NAME VARCHAR (255), RESOURCE_DESCRIPTION VARCHAR (1024), + OWNER_NAME VARCHAR (255), RESOURCE_SIZE INTEGER , CREATION_TIME TIMESTAMP DEFAULT CURRENT_TIMESTAMP, LAST_MODIFIED_TIME TIMESTAMP DEFAULT '0000-00-00 00:00:00', http://git-wip-us.apache.org/repos/asf/airavata/blob/9bf53d2c/modules/registry/registry-core/src/test/resources/datacatalog-derby.sql ---------------------------------------------------------------------- diff --git a/modules/registry/registry-core/src/test/resources/datacatalog-derby.sql b/modules/registry/registry-core/src/test/resources/datacatalog-derby.sql index ef4a777..22e5a70 100644 --- a/modules/registry/registry-core/src/test/resources/datacatalog-derby.sql +++ b/modules/registry/registry-core/src/test/resources/datacatalog-derby.sql @@ -24,6 +24,7 @@ CREATE TABLE DATA_RESOURCE RESOURCE_ID VARCHAR (255), RESOURCE_NAME VARCHAR (255), RESOURCE_DESCRIPTION VARCHAR (1024), + OWNER_NAME VARCHAR (255), RESOURCE_SIZE INTEGER , CREATION_TIME TIMESTAMP DEFAULT CURRENT_TIMESTAMP, LAST_MODIFIED_TIME TIMESTAMP DEFAULT '0000-00-00 00:00:00', http://git-wip-us.apache.org/repos/asf/airavata/blob/9bf53d2c/thrift-interface-descriptions/airavata-api/airavata_api.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/airavata-api/airavata_api.thrift b/thrift-interface-descriptions/airavata-api/airavata_api.thrift index 858acd4..47a8389 100644 --- a/thrift-interface-descriptions/airavata-api/airavata_api.thrift +++ b/thrift-interface-descriptions/airavata-api/airavata_api.thrift @@ -505,7 +505,7 @@ service Airavata { /** * Create an experiment for the specified user belonging to the gateway. The gateway identity is not explicitly passed - * but inferred from the authentication header. This experiment is just a persistent place holder. The client + * but inferred from the sshKeyAuthentication header. This experiment is just a persistent place holder. The client * has to subsequently configure and launch the created experiment. No action is taken on Airavata Server except * registering the experiment in a persistent store. * http://git-wip-us.apache.org/repos/asf/airavata/blob/9bf53d2c/thrift-interface-descriptions/airavata-api/airavata_errors.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/airavata-api/airavata_errors.thrift b/thrift-interface-descriptions/airavata-api/airavata_errors.thrift index e0849a1..870e9a0 100644 --- a/thrift-interface-descriptions/airavata-api/airavata_errors.thrift +++ b/thrift-interface-descriptions/airavata-api/airavata_errors.thrift @@ -101,7 +101,7 @@ exception TimedOutException { } /** -* This exception is thrown for invalid authentication requests. +* This exception is thrown for invalid sshKeyAuthentication requests. * * message: contains the cause of the authorization failure. */ http://git-wip-us.apache.org/repos/asf/airavata/blob/9bf53d2c/thrift-interface-descriptions/airavata-api/data_movement_models.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/airavata-api/data_movement_models.thrift b/thrift-interface-descriptions/airavata-api/data_movement_models.thrift index 677d790..af449e6 100644 --- a/thrift-interface-descriptions/airavata-api/data_movement_models.thrift +++ b/thrift-interface-descriptions/airavata-api/data_movement_models.thrift @@ -31,7 +31,7 @@ enum DMType { } /** - * Enumeration of security authentication and authorization mechanisms supported by Airavata. This enumeration just + * Enumeration of security sshKeyAuthentication and authorization mechanisms supported by Airavata. This enumeration just * describes the supported mechanism. The corresponding security credentials are registered with Airavata Credential * store. * http://git-wip-us.apache.org/repos/asf/airavata/blob/9bf53d2c/thrift-interface-descriptions/airavata-api/data_resource_models.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/airavata-api/data_resource_models.thrift b/thrift-interface-descriptions/airavata-api/data_resource_models.thrift index 96b6757..187b0b8 100644 --- a/thrift-interface-descriptions/airavata-api/data_resource_models.thrift +++ b/thrift-interface-descriptions/airavata-api/data_resource_models.thrift @@ -27,9 +27,10 @@ struct DataResourceModel { 1: optional string resourceId, 2: optional string resourceName, 3: optional string resourceDescription, - 4: optional i32 resourceSize, - 5: optional i64 creationTime, - 6: optional i64 lastModifiedTime + 4: optional string ownerName, + 5: optional i32 resourceSize, + 6: optional i64 creationTime, + 7: optional i64 lastModifiedTime } struct DataReplicaLocationModel { http://git-wip-us.apache.org/repos/asf/airavata/blob/9bf53d2c/thrift-interface-descriptions/orchestrator-cpi/orchestrator.cpi.service.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/orchestrator-cpi/orchestrator.cpi.service.thrift b/thrift-interface-descriptions/orchestrator-cpi/orchestrator.cpi.service.thrift index f775306..2212b4d 100644 --- a/thrift-interface-descriptions/orchestrator-cpi/orchestrator.cpi.service.thrift +++ b/thrift-interface-descriptions/orchestrator-cpi/orchestrator.cpi.service.thrift @@ -48,7 +48,7 @@ service OrchestratorService { /** * In order to run single applications users should create an associating * process and hand it over for execution - * along with a credential store token for authentication + * along with a credential store token for sshKeyAuthentication * * @param processId * @param airavataCredStoreToken
