Repository: airavata Updated Branches: refs/heads/develop dceabc83e -> a7ff42e47
http://git-wip-us.apache.org/repos/asf/airavata/blob/9fd6be25/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/db/dao/FileTransferRequestDaoTest.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/db/dao/FileTransferRequestDaoTest.java b/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/db/dao/FileTransferRequestDaoTest.java new file mode 100644 index 0000000..780c2b6 --- /dev/null +++ b/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/db/dao/FileTransferRequestDaoTest.java @@ -0,0 +1,45 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.db.dao; + +import org.apache.airavata.model.file.transfer.FileTransferRequestModel; +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +@Ignore +public class FileTransferRequestDaoTest { + private final static Logger logger = LoggerFactory.getLogger(FileTransferRequestDaoTest.class); + + @Test + public void testFileTransferRequestDao() throws IOException { + FileTransferRequestModel fileTransferRequestModel = new FileTransferRequestModel(); + fileTransferRequestModel.setSrcHostCredToken("djkalbsbdaslfbalsfbslf"); + fileTransferRequestModel.setSrcFilePath("test-file-path"); + FileTransferRequestDao fileTransferRequestDao = new FileTransferRequestDao(); + String transferId = fileTransferRequestDao.createFileTransferRequest(fileTransferRequestModel); + fileTransferRequestModel = fileTransferRequestDao.getFileTransferRequest(transferId); + System.out.println("Transfer Id:" + fileTransferRequestModel.getTransferId()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/9fd6be25/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/gridftp/CertFileReadTest.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/gridftp/CertFileReadTest.java b/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/gridftp/CertFileReadTest.java new file mode 100644 index 0000000..68578db --- /dev/null +++ b/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/gridftp/CertFileReadTest.java @@ -0,0 +1,126 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.remote.client.gridftp; + +import junit.framework.Assert; +import junit.framework.TestCase; +import org.globus.gsi.SigningPolicy; +import org.globus.gsi.SigningPolicyParser; +import org.globus.gsi.util.CertificateIOUtil; +import org.globus.util.GlobusResource; +import org.junit.Ignore; +import org.junit.Test; + +import javax.security.auth.x500.X500Principal; +import java.io.FileInputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.security.MessageDigest; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.util.Map; + + +@Ignore("This test case used to debug JGlobus-102. No need to run this test with other gridftp tests.") +public class CertFileReadTest extends TestCase { + + private static MessageDigest md5; + + private static String CERT_FILE_LOCATION = "/Users/supun/Work/airavata-sandbox/grid-tools/certificates/"; + + @Test + public void testCertFileRead() throws Exception { + + String path1 = CERT_FILE_LOCATION + "ffc3d59b"; + String path2 = CERT_FILE_LOCATION + "e5cc84c2"; + + + GlobusResource globusResource1 = new GlobusResource(path1 + ".signing_policy"); + GlobusResource globusResource2 = new GlobusResource(path2 + ".signing_policy"); + + // ===== Testing globusResource1 - This should pass (cos no DC components) ================ // + X509Certificate crt1 = readCertificate(path1 + ".0"); + X500Principal policySubjectCert1 = getPrincipal(globusResource1); + + String certHash1 = CertificateIOUtil.nameHash(crt1.getSubjectX500Principal()); + String principalHash1 = CertificateIOUtil.nameHash(policySubjectCert1); + + System.out.println("======== Printing hashes for 1 ================"); + System.out.println(certHash1); + System.out.println(principalHash1); + + Assert.assertEquals("Certificate hash value does not match with the hash value generated using principal name.", + certHash1, principalHash1); + + // ===== Testing globusResource1 - This should fail (cos we have DC components) ================ // + X509Certificate crt2 = readCertificate(path2 + ".0"); + X500Principal policySubjectCert2 = getPrincipal(globusResource2); + + String certHash2 = CertificateIOUtil.nameHash(crt2.getSubjectX500Principal()); + String principalHash2 = CertificateIOUtil.nameHash(policySubjectCert2); + + System.out.println("======== Printing hashes for 2 ================"); + System.out.println(certHash2); + System.out.println(principalHash2); + + Assert.assertEquals("Certificate hash value does not match with the hash value generated using principal name.", + certHash2, principalHash2); + } + + private X500Principal getPrincipal(GlobusResource globusResource) throws Exception{ + + SigningPolicyParser parser = new SigningPolicyParser(); + + Reader reader = new InputStreamReader(globusResource.getInputStream()); + + Map<X500Principal, SigningPolicy> policies = parser.parse(reader); + + return policies.keySet().iterator().next(); + + } + + private X509Certificate readCertificate(String certPath) { + try { + FileInputStream fr = new FileInputStream(certPath); + CertificateFactory cf = + CertificateFactory.getInstance("X509"); + X509Certificate crt = (X509Certificate) + cf.generateCertificate(fr); + System.out.println("Read certificate:"); + System.out.println("\tCertificate for: " + + crt.getSubjectDN()); + System.out.println("\tCertificate issued by: " + + crt.getIssuerDN()); + System.out.println("\tCertificate is valid from " + + crt.getNotBefore() + " to " + crt.getNotAfter()); + System.out.println("\tCertificate SN# " + + crt.getSerialNumber()); + System.out.println("\tGenerated with " + + crt.getSigAlgName()); + + return crt; + } catch (Exception e) { + e.printStackTrace(); + } + + return null; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/9fd6be25/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/gridftp/FileTransferTest.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/gridftp/FileTransferTest.java b/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/gridftp/FileTransferTest.java new file mode 100644 index 0000000..13d427d --- /dev/null +++ b/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/gridftp/FileTransferTest.java @@ -0,0 +1,168 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.remote.client.gridftp; + + +import junit.framework.Assert; +import junit.framework.TestCase; +import org.apache.airavata.file.manager.core.remote.client.gridftp.myproxy.SecurityContext; +import org.apache.log4j.BasicConfigurator; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.ietf.jgss.GSSCredential; +import org.junit.Ignore; + +import java.io.*; +import java.net.URI; + + +@Ignore +public class FileTransferTest extends TestCase { + + private GSSCredential gssCredential; + + private ExecutionContext executionContext; + + private static final Logger log = Logger.getLogger(FileTransferTest.class); + + + public void setUp() throws Exception { + + String userName = System.getProperty("myproxy.user"); + String password = System.getProperty("myproxy.password"); + + SecurityContext context = null; + + if (userName == null || password == null || userName.trim().equals("") || password.trim().equals("")) { + log.error("myproxy.user and myproxy.password system properties are not set. Example :- " + + "> mvn clean install -Dmyproxy.user=u1 -Dmyproxy.password=xxx"); + + Assert.fail("Please set myproxy.user and myproxy.password system properties."); + + } else { + context = new SecurityContext(userName, password); + } + + log.info("Using my proxy user name - " + userName); + + BasicConfigurator.configure(); + Logger logger = Logger.getLogger("GridFTPClient"); + Level lev = Level.toLevel("DEBUG"); + logger.setLevel(lev); + + + context.login(); + executionContext = new ExecutionContext(); + + + String targeterp = executionContext.getGridFTPServerDestination(); + String remoteDestFile = executionContext.getDestinationDataLocation(); + + URI dirLocation = GridFTP.createGsiftpURI(targeterp, + remoteDestFile.substring(0, remoteDestFile.lastIndexOf("/"))); + gssCredential = context.getGssCredential(); + System.out.println(dirLocation); + + } + + public void testMakeDir() throws Exception { + + String targetErp = executionContext.getGridFTPServerDestination(); + String remoteDestinationFile = executionContext.getDestinationDataLocation(); + + URI dirLocation = GridFTP.createGsiftpURI(targetErp, + remoteDestinationFile.substring(0, remoteDestinationFile.lastIndexOf("/"))); + + GridFTP ftp = new GridFTP(); + ftp.makeDir(dirLocation, gssCredential); + + Assert.assertTrue(ftp.exists(dirLocation, gssCredential)); + + } + + public void testTransferData() throws Exception { + + String sourceERP = executionContext.getGridFTPServerSource(); + String remoteSrcFile = executionContext.getSourceDataLocation(); + + String targetErp = executionContext.getGridFTPServerDestination(); + String remoteDestinationFile = executionContext.getDestinationDataLocation(); + + URI srcURI = GridFTP.createGsiftpURI(sourceERP, remoteSrcFile); + URI destURI = GridFTP.createGsiftpURI(targetErp, remoteDestinationFile); + + GridFTP ftp = new GridFTP(); + ftp.transfer(srcURI, destURI, gssCredential, true); + + Assert.assertTrue(ftp.exists(destURI, gssCredential)); + + } + + public void testDownloadFile() throws Exception { + + String fileName = "./downloaded"; + + File deleteFile = new File(fileName); + + if (deleteFile.exists()) { + if (!deleteFile.delete()) + throw new RuntimeException("Unable to delete file " + fileName); + } + + File f = new File(fileName); + + GridFTP ftp = new GridFTP(); + ftp.downloadFile(executionContext.getSourceDataFileUri(), + gssCredential, f); + + Assert.assertTrue(f.exists()); + + } + + public void testFileExists() throws Exception { + + GridFTP ftp = new GridFTP(); + Assert.assertTrue(ftp.exists(executionContext.getSourceDataFileUri(), gssCredential)); + } + + public void testUpdateFile() throws Exception { + + String currentDir = System.getProperty("projectDirectory"); + + if (currentDir == null) + currentDir = "src/test/resources"; + else + currentDir = currentDir + "/src/test/resources"; + + String file = currentDir + "/dummy"; + + System.out.println("File to upload is " + file); + + File fileToUpload = new File(file); + + Assert.assertTrue(fileToUpload.canRead()); + + GridFTP ftp = new GridFTP(); + ftp.updateFile(executionContext.getUploadingFilePathUri(), gssCredential, fileToUpload); + + Assert.assertTrue(ftp.exists(executionContext.getUploadingFilePathUri(), gssCredential)); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/9fd6be25/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/scp/SCPStorageClientTest.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/scp/SCPStorageClientTest.java b/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/scp/SCPStorageClientTest.java new file mode 100644 index 0000000..93fcba3 --- /dev/null +++ b/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/scp/SCPStorageClientTest.java @@ -0,0 +1,55 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.core.remote.client.scp; + +import org.apache.commons.io.IOUtils; +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; + +@Ignore +public class SCPStorageClientTest { + private final static Logger logger = LoggerFactory.getLogger(SCPStorageClientTest.class); + + @Test + public void testSCPStorageClient() throws Exception { + File privateKey = new File("/Users/supun/.ssh/id_rsa"); + byte[] privateKeyBytes = IOUtils.toByteArray(new FileInputStream(privateKey)); + + File publicKey = new File("/Users/supun/.ssh/id_rsa.pub"); + byte[] publicKeyBytes = IOUtils.toByteArray(new FileInputStream(publicKey)); + + String passPhrase = ""; + byte[] passPhraseBytes = passPhrase.getBytes(); + + SCPStorageClient scpStorageClient = new SCPStorageClient("gw75.iu.xsede.org", 22, "pga", privateKeyBytes, + publicKeyBytes, passPhraseBytes); + File file = scpStorageClient.readFile("/var/www/portals/gateway-user-data/testdrive/test.txt"); + System.out.println("File exists ? " + file.exists()); + scpStorageClient.writeFile(file, "/var/www/portals/gateway-user-data/testdrive/test2.txt"); + file.delete(); + System.out.println("File exists ? " + file.exists()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/9fd6be25/modules/file-manager/file-manager-core/src/test/resources/airavata-myproxy-client.properties ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/test/resources/airavata-myproxy-client.properties b/modules/file-manager/file-manager-core/src/test/resources/airavata-myproxy-client.properties new file mode 100644 index 0000000..407f09d --- /dev/null +++ b/modules/file-manager/file-manager-core/src/test/resources/airavata-myproxy-client.properties @@ -0,0 +1,28 @@ +#Download trusted certificates from - https://software.xsede.org/security/xsede-certs.tar.gz +trustedCertsFile=../certificates +myproxyServer=myproxy.teragrid.org +myproxy_lifetime=17280000 +myproxyPort=7512 + +testing.host=trestles +#testing.host=stampede +#testing.host=lonestar + +## TACC Lonestar +lonestar.gridftp.endpoint=gsiftp://gridftp1.ls4.tacc.utexas.edu:2811/ + +## TACC Stampede +stampede.gridftp.endpoint=gsiftp://gridftp.stampede.tacc.utexas.edu:2811/ + +## SDSC Trestles +trestles.gridftp.endpoint=gsiftp://trestles.sdsc.edu:2811/ + +gridftpserverSource=gsiftp://trestles-dm.sdsc.xsede.org:2811/ +gridftpSourcePath=/oasis/projects/nsf/sds128/ogce/file-transfer-tests/source/sample_wrfout.netcdf +gridftpUploadingFilePath = /oasis/projects/nsf/sds128/ogce/file-transfer-tests/source/dummy1 + +#gridftpserverDest=gsiftp://trestles-dm.sdsc.xsede.org:2811/ +#gridftpDestPath=/oasis/projects/nsf/sds128/ogce/file-transfer-tests/gf + +gridftpserverDest=gsiftp://gridftp.stampede.tacc.utexas.edu:2811/ +gridftpDestPath=/scratch/01437/ogce/file-transfer-tests/dest/xx \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/9fd6be25/modules/file-manager/file-manager-core/src/test/resources/dummy ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-core/src/test/resources/dummy b/modules/file-manager/file-manager-core/src/test/resources/dummy new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/airavata/blob/9fd6be25/thrift-interface-descriptions/data-models/file-manager-models/replica_models.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/data-models/file-manager-models/replica_models.thrift b/thrift-interface-descriptions/data-models/file-manager-models/replica_models.thrift index ff9b932..4e2ea00 100644 --- a/thrift-interface-descriptions/data-models/file-manager-models/replica_models.thrift +++ b/thrift-interface-descriptions/data-models/file-manager-models/replica_models.thrift @@ -64,7 +64,7 @@ struct FileModel { 8: optional string sha256Checksum, 9: optional FileModelType fileType, 10: optional i32 fileSize, - 11: optional string nativeFormat, + 11: optional string dataType, 12: optional i64 creationTime, 13: optional i64 lastModifiedTime, 14: optional list<FileReplicaModel> fileReplicas
