This is an automated email from the ASF dual-hosted git repository. dimuthuupe pushed a commit to branch jupyter-integration in repository https://gitbox.apache.org/repos/asf/airavata.git
commit 863bb5b10841189795ce2562898fb93795aca481 Author: dimuthu <[email protected]> AuthorDate: Fri Aug 31 05:47:09 2018 -0400 Optimizing FileStructure thrift model --- .../api/server/handler/AiravataServerHandler.java | 25 ++++-- .../model/appcatalog/datamodels/FileStructure.java | 96 +++++++++++++++++++++- .../registry-server/registry-api-service/pom.xml | 5 ++ .../api/service/handler/RegistryServerHandler.java | 27 +++--- 4 files changed, 130 insertions(+), 23 deletions(-) diff --git a/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java b/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java index c3cfae5..c4707bb 100644 --- a/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java +++ b/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java @@ -5809,6 +5809,7 @@ public class AiravataServerHandler implements Airavata.Iface { RegistryService.Client regClient = registryClientPool.getResource(); try { regClient.uploadFileToStorage(gatewayId, storageResourceId, userId, content, path, type); + registryClientPool.returnResource(regClient); } catch (Exception e) { String msg = "Error uploading file to storage resource: " + storageResourceId; AiravataSystemException exception = new AiravataSystemException(AiravataErrorType.INTERNAL_ERROR); @@ -5822,7 +5823,9 @@ public class AiravataServerHandler implements Airavata.Iface { public FileStructure downloadFileFromStorage(AuthzToken authzToken, String gatewayId, String storageResourceId, String userId, String path) throws InvalidRequestException, AiravataClientException, AiravataSystemException, AuthorizationException, TException { RegistryService.Client regClient = registryClientPool.getResource(); try { - return regClient.downloadFileFromStorage(gatewayId, storageResourceId, userId, path); + FileStructure structure = regClient.downloadFileFromStorage(gatewayId, storageResourceId, userId, path); + registryClientPool.returnResource(regClient); + return structure; } catch (Exception e) { String msg = "Error downloading file from storage resource: " + storageResourceId; AiravataSystemException exception = new AiravataSystemException(AiravataErrorType.INTERNAL_ERROR); @@ -5836,7 +5839,9 @@ public class AiravataServerHandler implements Airavata.Iface { public FileStructure getFileDetailsFromStorage(AuthzToken authzToken, String gatewayId, String storageResourceId, String userId, String filePath) throws InvalidRequestException, AiravataClientException, AiravataSystemException, AuthorizationException, TException { RegistryService.Client regClient = registryClientPool.getResource(); try { - return regClient.getFileDetailsFromStorage(gatewayId, storageResourceId, userId, filePath); + FileStructure structure = regClient.getFileDetailsFromStorage(gatewayId, storageResourceId, userId, filePath); + registryClientPool.returnResource(regClient); + return structure; } catch (Exception e) { String msg = "Error fetching file info from storage resource: " + storageResourceId + " for path : " + filePath; AiravataSystemException exception = new AiravataSystemException(AiravataErrorType.INTERNAL_ERROR); @@ -5850,7 +5855,9 @@ public class AiravataServerHandler implements Airavata.Iface { public List<FileStructure> listDirectoryFromStorage(AuthzToken authzToken, String gatewayId, String storageResourceId, String userId, String dirPath) throws InvalidRequestException, AiravataClientException, AiravataSystemException, AuthorizationException, TException { RegistryService.Client regClient = registryClientPool.getResource(); try { - return regClient.listDirectoryFromStorage(gatewayId, storageResourceId, userId, dirPath); + List<FileStructure> fileStructures = regClient.listDirectoryFromStorage(gatewayId, storageResourceId, userId, dirPath); + registryClientPool.returnResource(regClient); + return fileStructures; } catch (Exception e) { String msg = "Error listing directory " + dirPath + " of storage resource: " + storageResourceId; AiravataSystemException exception = new AiravataSystemException(AiravataErrorType.INTERNAL_ERROR); @@ -5865,6 +5872,7 @@ public class AiravataServerHandler implements Airavata.Iface { RegistryService.Client regClient = registryClientPool.getResource(); try { regClient.deleteFileFromStorage(gatewayId, storageResourceId, userId, path); + registryClientPool.returnResource(regClient); } catch (Exception e) { String msg = "Error deleting file " + path + " form storage resource: " + storageResourceId; AiravataSystemException exception = new AiravataSystemException(AiravataErrorType.INTERNAL_ERROR); @@ -5879,6 +5887,7 @@ public class AiravataServerHandler implements Airavata.Iface { RegistryService.Client regClient = registryClientPool.getResource(); try { regClient.deleteDirectoryFromStorage(gatewayId, storageResourceId, userId, path); + registryClientPool.returnResource(regClient); } catch (Exception e) { String msg = "Error deleting directory " + path + " form storage resource: " + storageResourceId; AiravataSystemException exception = new AiravataSystemException(AiravataErrorType.INTERNAL_ERROR); @@ -5892,7 +5901,9 @@ public class AiravataServerHandler implements Airavata.Iface { public boolean isExistInStorage(AuthzToken authzToken, String gatewayId, String storageResourceId, String userId, String path) throws InvalidRequestException, AiravataClientException, AiravataSystemException, AuthorizationException, TException { RegistryService.Client regClient = registryClientPool.getResource(); try { - return regClient.isExistInStorage(gatewayId, storageResourceId, userId, path); + boolean existInStorage = regClient.isExistInStorage(gatewayId, storageResourceId, userId, path); + registryClientPool.returnResource(regClient); + return existInStorage; } catch (Exception e) { String msg = "Error checking file " + path + " existance in storage resource: " + storageResourceId; AiravataSystemException exception = new AiravataSystemException(AiravataErrorType.INTERNAL_ERROR); @@ -5907,6 +5918,7 @@ public class AiravataServerHandler implements Airavata.Iface { RegistryService.Client regClient = registryClientPool.getResource(); try { regClient.createDirectoryInStorage(gatewayId, storageResourceId, userId, dirPath); + registryClientPool.returnResource(regClient); } catch (Exception e) { String msg = "Error creating directory " + dirPath + " in storage resource: " + storageResourceId; AiravataSystemException exception = new AiravataSystemException(AiravataErrorType.INTERNAL_ERROR); @@ -5920,7 +5932,9 @@ public class AiravataServerHandler implements Airavata.Iface { public boolean checkIsFileInStorage(AuthzToken authzToken, String gatewayId, String storageResourceId, String userId, String path) throws InvalidRequestException, AiravataClientException, AiravataSystemException, AuthorizationException, TException { RegistryService.Client regClient = registryClientPool.getResource(); try { - return regClient.checkIsFileInStorage(gatewayId, storageResourceId, userId, path); + boolean isFileInStorage = regClient.checkIsFileInStorage(gatewayId, storageResourceId, userId, path); + registryClientPool.returnResource(regClient); + return isFileInStorage; } catch (Exception e) { String msg = "Error checking is file " + path + " in storage resource: " + storageResourceId; AiravataSystemException exception = new AiravataSystemException(AiravataErrorType.INTERNAL_ERROR); @@ -5935,6 +5949,7 @@ public class AiravataServerHandler implements Airavata.Iface { RegistryService.Client regClient = registryClientPool.getResource(); try { regClient.renameFileInStorage(gatewayId, storageResourceId, userId, oldPath, newPath); + registryClientPool.returnResource(regClient); } catch (Exception e) { String msg = "Error renaming file from " + oldPath + " to " + newPath + " in storage resource: " + storageResourceId; AiravataSystemException exception = new AiravataSystemException(AiravataErrorType.INTERNAL_ERROR); diff --git a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/appcatalog/datamodels/FileStructure.java b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/appcatalog/datamodels/FileStructure.java index a94bceb..0105e4a 100644 --- a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/appcatalog/datamodels/FileStructure.java +++ b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/appcatalog/datamodels/FileStructure.java @@ -35,6 +35,7 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil private static final org.apache.thrift.protocol.TField MODIFIED_DATE_FIELD_DESC = new org.apache.thrift.protocol.TField("modifiedDate", org.apache.thrift.protocol.TType.I64, (short)5); private static final org.apache.thrift.protocol.TField CONTENT_FIELD_DESC = new org.apache.thrift.protocol.TField("content", org.apache.thrift.protocol.TType.STRING, (short)6); private static final org.apache.thrift.protocol.TField SIZE_FIELD_DESC = new org.apache.thrift.protocol.TField("size", org.apache.thrift.protocol.TType.I64, (short)7); + private static final org.apache.thrift.protocol.TField IS_EXIST_FIELD_DESC = new org.apache.thrift.protocol.TField("isExist", org.apache.thrift.protocol.TType.BOOL, (short)8); private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new FileStructureStandardSchemeFactory(); private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new FileStructureTupleSchemeFactory(); @@ -46,6 +47,7 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil private long modifiedDate; // required private java.nio.ByteBuffer content; // optional private long size; // required + private boolean isExist; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { @@ -55,7 +57,8 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil CREATED_DATE((short)4, "createdDate"), MODIFIED_DATE((short)5, "modifiedDate"), CONTENT((short)6, "content"), - SIZE((short)7, "size"); + SIZE((short)7, "size"), + IS_EXIST((short)8, "isExist"); private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>(); @@ -84,6 +87,8 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil return CONTENT; case 7: // SIZE return SIZE; + case 8: // IS_EXIST + return IS_EXIST; default: return null; } @@ -128,6 +133,7 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil private static final int __CREATEDDATE_ISSET_ID = 1; private static final int __MODIFIEDDATE_ISSET_ID = 2; private static final int __SIZE_ISSET_ID = 3; + private static final int __ISEXIST_ISSET_ID = 4; private byte __isset_bitfield = 0; private static final _Fields optionals[] = {_Fields.CONTENT}; public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; @@ -147,6 +153,8 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); tmpMap.put(_Fields.SIZE, new org.apache.thrift.meta_data.FieldMetaData("size", org.apache.thrift.TFieldRequirementType.REQUIRED, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.IS_EXIST, new org.apache.thrift.meta_data.FieldMetaData("isExist", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(FileStructure.class, metaDataMap); } @@ -160,7 +168,8 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil boolean isFile, long createdDate, long modifiedDate, - long size) + long size, + boolean isExist) { this(); this.name = name; @@ -173,6 +182,8 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil setModifiedDateIsSet(true); this.size = size; setSizeIsSet(true); + this.isExist = isExist; + setIsExistIsSet(true); } /** @@ -193,6 +204,7 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil this.content = org.apache.thrift.TBaseHelper.copyBinary(other.content); } this.size = other.size; + this.isExist = other.isExist; } public FileStructure deepCopy() { @@ -212,6 +224,8 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil this.content = null; setSizeIsSet(false); this.size = 0; + setIsExistIsSet(false); + this.isExist = false; } public java.lang.String getName() { @@ -380,6 +394,28 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __SIZE_ISSET_ID, value); } + public boolean isIsExist() { + return this.isExist; + } + + public void setIsExist(boolean isExist) { + this.isExist = isExist; + setIsExistIsSet(true); + } + + public void unsetIsExist() { + __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __ISEXIST_ISSET_ID); + } + + /** Returns true if field isExist is set (has been assigned a value) and false otherwise */ + public boolean isSetIsExist() { + return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __ISEXIST_ISSET_ID); + } + + public void setIsExistIsSet(boolean value) { + __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __ISEXIST_ISSET_ID, value); + } + public void setFieldValue(_Fields field, java.lang.Object value) { switch (field) { case NAME: @@ -442,6 +478,14 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil } break; + case IS_EXIST: + if (value == null) { + unsetIsExist(); + } else { + setIsExist((java.lang.Boolean)value); + } + break; + } } @@ -468,6 +512,9 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil case SIZE: return getSize(); + case IS_EXIST: + return isIsExist(); + } throw new java.lang.IllegalStateException(); } @@ -493,6 +540,8 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil return isSetContent(); case SIZE: return isSetSize(); + case IS_EXIST: + return isSetIsExist(); } throw new java.lang.IllegalStateException(); } @@ -575,6 +624,15 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil return false; } + boolean this_present_isExist = true; + boolean that_present_isExist = true; + if (this_present_isExist || that_present_isExist) { + if (!(this_present_isExist && that_present_isExist)) + return false; + if (this.isExist != that.isExist) + return false; + } + return true; } @@ -602,6 +660,8 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil hashCode = hashCode * 8191 + org.apache.thrift.TBaseHelper.hashCode(size); + hashCode = hashCode * 8191 + ((isExist) ? 131071 : 524287); + return hashCode; } @@ -683,6 +743,16 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil return lastComparison; } } + lastComparison = java.lang.Boolean.valueOf(isSetIsExist()).compareTo(other.isSetIsExist()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIsExist()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.isExist, other.isExist); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -744,6 +814,10 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil sb.append("size:"); sb.append(this.size); first = false; + if (!first) sb.append(", "); + sb.append("isExist:"); + sb.append(this.isExist); + first = false; sb.append(")"); return sb.toString(); } @@ -774,6 +848,10 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil throw new org.apache.thrift.protocol.TProtocolException("Required field 'size' is unset! Struct:" + toString()); } + if (!isSetIsExist()) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'isExist' is unset! Struct:" + toString()); + } + // check for sub-struct validity } @@ -869,6 +947,14 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 8: // IS_EXIST + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.isExist = iprot.readBool(); + struct.setIsExistIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -911,6 +997,9 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil oprot.writeFieldBegin(SIZE_FIELD_DESC); oprot.writeI64(struct.size); oprot.writeFieldEnd(); + oprot.writeFieldBegin(IS_EXIST_FIELD_DESC); + oprot.writeBool(struct.isExist); + oprot.writeFieldEnd(); oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -934,6 +1023,7 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil oprot.writeI64(struct.createdDate); oprot.writeI64(struct.modifiedDate); oprot.writeI64(struct.size); + oprot.writeBool(struct.isExist); java.util.BitSet optionals = new java.util.BitSet(); if (struct.isSetContent()) { optionals.set(0); @@ -959,6 +1049,8 @@ public class FileStructure implements org.apache.thrift.TBase<FileStructure, Fil struct.setModifiedDateIsSet(true); struct.size = iprot.readI64(); struct.setSizeIsSet(true); + struct.isExist = iprot.readBool(); + struct.setIsExistIsSet(true); java.util.BitSet incoming = iprot.readBitSet(1); if (incoming.get(0)) { struct.content = iprot.readBinary(); diff --git a/modules/registry/registry-server/registry-api-service/pom.xml b/modules/registry/registry-server/registry-api-service/pom.xml index 7f73224..46c1499 100644 --- a/modules/registry/registry-server/registry-api-service/pom.xml +++ b/modules/registry/registry-server/registry-api-service/pom.xml @@ -80,5 +80,10 @@ <artifactId>sshj-agent</artifactId> <version>0.17-SNAPSHOT</version> </dependency> + <dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>task-core</artifactId> + <version>0.17-SNAPSHOT</version> + </dependency> </dependencies> </project> \ No newline at end of file diff --git a/modules/registry/registry-server/registry-api-service/src/main/java/org/apache/airavata/registry/api/service/handler/RegistryServerHandler.java b/modules/registry/registry-server/registry-api-service/src/main/java/org/apache/airavata/registry/api/service/handler/RegistryServerHandler.java index 0dd344c..e73336e 100644 --- a/modules/registry/registry-server/registry-api-service/src/main/java/org/apache/airavata/registry/api/service/handler/RegistryServerHandler.java +++ b/modules/registry/registry-server/registry-api-service/src/main/java/org/apache/airavata/registry/api/service/handler/RegistryServerHandler.java @@ -24,7 +24,7 @@ import org.apache.airavata.agents.api.FileInfo; import org.apache.airavata.agents.api.StorageResourceAdaptor; import org.apache.airavata.common.utils.AiravataUtils; import org.apache.airavata.common.utils.ServerSettings; -import org.apache.airavata.helix.adaptor.SSHJStorageAdaptor; +import org.apache.airavata.helix.core.support.adaptor.AdaptorSupportImpl; import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription; import org.apache.airavata.model.appcatalog.appdeployment.ApplicationModule; import org.apache.airavata.model.appcatalog.appinterface.ApplicationInterfaceDescription; @@ -4968,21 +4968,14 @@ public class RegistryServerHandler implements RegistryService.Iface { String credentialStoreToken = resourceSpecificCredentialStoreToken != null ? resourceSpecificCredentialStoreToken : gatewayCredentialStoreToken; String storageLoginUserName = gatewayStoragePreference.getLoginUserName(); - switch (dataMovementProtocol) { - - case SCP: - SCPDataMovement scpDataMovement = getSCPDataMovement(storageResource.getDataMovementInterfaces().get(0).getDataMovementInterfaceId()); - SSHJStorageAdaptor storageAdaptor = new SSHJStorageAdaptor(); - try { - storageAdaptor.init(storageResourceId, gatewayId, storageLoginUserName, credentialStoreToken); - return storageAdaptor; - } catch (AgentException e) { - e.printStackTrace(); - } + try { + return AdaptorSupportImpl.getInstance().fetchStorageAdaptor(gatewayId, storageResourceId, dataMovementProtocol, credentialStoreToken, storageLoginUserName); + } catch (AgentException e) { + logger.error("Failed to fetch storage resource adaptor for storage resource " + storageResourceId, e); + throw new TException("Failed to fetch storage resource adaptor for storage resource " + storageResourceId); } - - return null; } + @Override public void uploadFileToStorage(String gatewayId, String storageResourceId, String userId, ByteBuffer content, String path, String type) throws TException { @@ -5009,6 +5002,8 @@ public class RegistryServerHandler implements RegistryService.Iface { @Override public FileStructure downloadFileFromStorage(String gatewayId, String storageResourceId, String userId, String path) throws TException { + FileStructure structure = getFileDetailsFromStorage(gatewayId, storageResourceId, userId, path); + File tmpFile; try { tmpFile = File.createTempFile("file", "download"); @@ -5025,8 +5020,6 @@ public class RegistryServerHandler implements RegistryService.Iface { logger.error("Failed to download remote file " + path + " to local path " + tmpFile.getAbsolutePath() + " from storage resource " + storageResourceId, e); } - FileStructure structure = new FileStructure(); - structure.setName(new File(path).getName()); try { structure.setContent(IOUtils.toByteArray(new FileInputStream(tmpFile))); } catch (IOException e) { @@ -5048,6 +5041,7 @@ public class RegistryServerHandler implements RegistryService.Iface { for (FileInfo fileInfo : fileInfos) { FileStructure fileStructure = new FileStructure(); + fileStructure.setIsExist(fileInfo.isExist()); fileStructure.setName(fileInfo.getName()); fileStructure.setPath(fileInfo.getPath()); fileStructure.setCreatedDate(fileInfo.getCreatedDate()); @@ -5070,6 +5064,7 @@ public class RegistryServerHandler implements RegistryService.Iface { try { FileInfo fileInfo = storageResourceAdaptor.getFileInfo(filePath); FileStructure fileStructure = new FileStructure(); + fileStructure.setIsExist(fileInfo.isExist()); fileStructure.setPath(filePath); fileStructure.setName(new File(filePath).getName()); fileStructure.setCreatedDate(fileInfo.getCreatedDate());
