http://git-wip-us.apache.org/repos/asf/airavata/blob/b61cfcd3/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileTransferService.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileTransferService.java b/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileTransferService.java index 0b971b7..02fbef1 100644 --- a/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileTransferService.java +++ b/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/FileTransferService.java @@ -20,9 +20,9 @@ */ package org.apache.airavata.file.manager.cpi; -import org.apache.airavata.model.file.FileNode; -import org.apache.airavata.model.file.FileTransferRequest; -import org.apache.airavata.model.file.StorageResourceProtocol; +import org.apache.airavata.model.file.transfer.FileTransferRequestModel; +import org.apache.airavata.model.file.transfer.LSEntryModel; +import org.apache.airavata.model.file.transfer.StorageResourceProtocol; import java.util.List; @@ -30,6 +30,8 @@ public interface FileTransferService { /** * Method to upload the give bytes to the destination storage system + * @param gatewayId + * @param username * @param fileData * @param destHostname * @param destLoginName @@ -40,12 +42,14 @@ public interface FileTransferService { * @return * @throws FileManagerException */ - String uploadFile(byte[] fileData, String destHostname, String destLoginName, int destPort, + String uploadFile(String gatewayId, String username, byte[] fileData, String destHostname, String destLoginName, int destPort, StorageResourceProtocol destProtocol, String destinationPath, String destHostCredToken) throws FileManagerException; /** * Transfer file between two storage resources synchronously. Returns the file transfer request id + * @param gatewayId + * @param username * @param srcHostname * @param srcPort * @param srcLoginName @@ -61,7 +65,7 @@ public interface FileTransferService { * @return * @throws FileManagerException */ - String transferFile(String srcHostname, String srcLoginName, int srcPort, + String transferFile(String gatewayId, String username, String srcHostname, String srcLoginName, int srcPort, StorageResourceProtocol srcProtocol, String srcPath, String srcHostCredToken, String destHostname, String destLoginName, int destPort, StorageResourceProtocol destProtocol, String destinationPath, String destHostCredToken) @@ -69,6 +73,8 @@ public interface FileTransferService { /** * Transfer file between two storage resources asynchronously. Returns the file transfer request id + * @param gatewayId + * @param username * @param srcHostname * @param srcLoginName * @param srcPort @@ -85,7 +91,7 @@ public interface FileTransferService { * @return * @throws FileManagerException */ - String transferFileAsync(String srcHostname, String srcLoginName, int srcPort, + String transferFileAsync(String gatewayId, String username, String srcHostname, String srcLoginName, int srcPort, StorageResourceProtocol srcProtocol, String srcPath, String srcHostCredToken, String destHostname, String destLoginName, int destPort, StorageResourceProtocol destProtocol, String destinationPath, String destHostCredToken, @@ -103,7 +109,7 @@ public interface FileTransferService { * @return * @throws FileManagerException */ - List<FileNode> getDirectoryListing(String hostname, String loginName, int port, + List<LSEntryModel> getDirectoryListing(String hostname, String loginName, int port, StorageResourceProtocol protocol, String path, String hostCredential) throws FileManagerException; @@ -201,6 +207,6 @@ public interface FileTransferService { * @return * @throws FileManagerException */ - FileTransferRequest getFileTransferRequestStatus(String transferId) + FileTransferRequestModel getFileTransferRequestStatus(String transferId) throws FileManagerException; } \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/b61cfcd3/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/MetadataCatalogService.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/MetadataCatalogService.java b/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/MetadataCatalogService.java new file mode 100644 index 0000000..89e060c --- /dev/null +++ b/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/MetadataCatalogService.java @@ -0,0 +1,56 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.cpi; + +import org.apache.airavata.model.file.metadata.MetadataModel; + +public interface MetadataCatalogService { + + /** + * Create new metadata model + * @param metadataModel + * @return + * @throws FileManagerException + */ + String createMetadata(MetadataModel metadataModel) throws FileManagerException; + + /** + * Update exisiting metadata model + * @param metadataModel + * @throws FileManagerException + */ + void updateMetadata(MetadataModel metadataModel) throws FileManagerException; + + /** + * Delete existing metadata model + * @param metadataId + * @throws FileManagerException + */ + void deleteMetadata(String metadataId) throws FileManagerException; + + /** + * Retrieve metadata model + * @param metadataId + * @return + * @throws FileManagerException + */ + MetadataModel getMetadata(String metadataId) throws FileManagerException; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/b61cfcd3/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/ReplicaCatalogService.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/ReplicaCatalogService.java b/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/ReplicaCatalogService.java new file mode 100644 index 0000000..cd7de53 --- /dev/null +++ b/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/ReplicaCatalogService.java @@ -0,0 +1,86 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.file.manager.cpi; + +import org.apache.airavata.model.file.replica.FileCollectionModel; +import org.apache.airavata.model.file.replica.FileModel; + +public interface ReplicaCatalogService { + + /** + * Creates a new file entry in the replica catalog + * @param fileModel + * @return + */ + String registerFileDetails(FileModel fileModel) throws FileManagerException; + + /** + * Updates an existing file information + * @param fileModel + */ + void updateFileDetails(FileModel fileModel) throws FileManagerException; + + + /** + * Deletes the specified file details entry + * @param fileId + */ + void deleteFileDetails(String fileId) throws FileManagerException; + + + /** + * Retrieves file details for the specified file id + * @param fileId + * @return + */ + FileModel getFileDetails(String fileId) throws FileManagerException; + + + /** + * Create new file collection entry + * @param fileCollectionModel + * @return + * @throws FileManagerException + */ + String registerFileCollection(FileCollectionModel fileCollectionModel) throws FileManagerException; + + /** + * Update existing file collection + * @param fileCollectionModel + * @throws FileManagerException + */ + void updateFileCollection(FileCollectionModel fileCollectionModel) throws FileManagerException; + + /** + * Delete exisiting file collection + * @param collectionId + * @throws FileManagerException + */ + void deleteFileCollection(String collectionId) throws FileManagerException; + + /** + * Retrieve file collection specifying the collection id + * @param collectionId + * @return + * @throws FileManagerException + */ + FileCollectionModel getFileCollection(String collectionId) throws FileManagerException; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/b61cfcd3/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/ReplicaManagementService.java ---------------------------------------------------------------------- diff --git a/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/ReplicaManagementService.java b/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/ReplicaManagementService.java deleted file mode 100644 index e0cce5b..0000000 --- a/modules/file-manager/file-manager-cpi/src/main/java/org/apache/airavata/file/manager/cpi/ReplicaManagementService.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.file.manager.cpi; - -public interface ReplicaManagementService { - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/b61cfcd3/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/AiravataClient.java ---------------------------------------------------------------------- diff --git a/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/AiravataClient.java b/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/AiravataClient.java index 8b4b48b..8db7d71 100644 --- a/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/AiravataClient.java +++ b/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/AiravataClient.java @@ -22,7 +22,7 @@ package org.apache.airavata.testsuite.multitenantedairavata; import org.apache.airavata.api.Airavata; -import org.apache.airavata.api.client.AiravataClientFactory; +import org.apache.airavata.client.AiravataClientFactory; import org.apache.airavata.model.error.AiravataClientException; import org.apache.airavata.testsuite.multitenantedairavata.utils.PropertyFileType; import org.apache.airavata.testsuite.multitenantedairavata.utils.PropertyReader; http://git-wip-us.apache.org/repos/asf/airavata/blob/b61cfcd3/thrift-interface-descriptions/airavata-apis/airavata_api.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/airavata-apis/airavata_api.thrift b/thrift-interface-descriptions/airavata-apis/airavata_api.thrift index 74d94b3..d5f4846 100644 --- a/thrift-interface-descriptions/airavata-apis/airavata_api.thrift +++ b/thrift-interface-descriptions/airavata-apis/airavata_api.thrift @@ -40,7 +40,6 @@ include "../data-models/resource-catalog-models/storage_resource_model.thrift" include "../data-models/resource-catalog-models/gateway_resource_profile_model.thrift" include "../data-models/resource-catalog-models/data_movement_models.thrift" include "../data-models/workflow-models/workflow_data_model.thrift" -include "../data-models/data-catalog-models/replica_models.thrift" namespace java org.apache.airavata.api namespace php Airavata.API http://git-wip-us.apache.org/repos/asf/airavata/blob/b61cfcd3/thrift-interface-descriptions/data-models/airavata_data_models.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/data-models/airavata_data_models.thrift b/thrift-interface-descriptions/data-models/airavata_data_models.thrift index 33783e5..940c89d 100644 --- a/thrift-interface-descriptions/data-models/airavata_data_models.thrift +++ b/thrift-interface-descriptions/data-models/airavata_data_models.thrift @@ -32,6 +32,7 @@ include "experiment-catalog-models/status_models.thrift" include "resource-catalog-models/data_movement_models.thrift" include "file-manager-models/replica_models.thrift" include "file-manager-models/file__transfer_models.thrift" +include "file-manager-models/metadata_models.thrift" namespace java org.apache.airavata.model namespace php Airavata.Model http://git-wip-us.apache.org/repos/asf/airavata/blob/b61cfcd3/thrift-interface-descriptions/data-models/file-manager-models/file__transfer_models.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/data-models/file-manager-models/file__transfer_models.thrift b/thrift-interface-descriptions/data-models/file-manager-models/file__transfer_models.thrift index 4600aa4..580959d 100644 --- a/thrift-interface-descriptions/data-models/file-manager-models/file__transfer_models.thrift +++ b/thrift-interface-descriptions/data-models/file-manager-models/file__transfer_models.thrift @@ -18,16 +18,16 @@ * */ - namespace java org.apache.airavata.model.file - namespace php Airavata.Model.File - namespace cpp apache.airavata.model.file - namespace py apache.airavata.model.file + namespace java org.apache.airavata.model.file.transfer + namespace php Airavata.Model.File.Transfer + namespace cpp apache.airavata.model.file.transfer + namespace py apache.airavata.model.file.transfer enum StorageResourceProtocol{ SCP,SFTP,HTTP,HTTPS,GridFTP,LOCAL } - enum FileNodeTypes{ + enum LSEntryType{ DIRECTORY, FILE } @@ -40,30 +40,33 @@ CREATED, QUEUED, RUNNING, COMPLETED, FAILED } - struct FileTransferRequest{ + struct FileTransferRequestModel{ 1: optional string transferId, - 2: optional string srcHostname, - 3: optional string srcLoginName, - 4: optional i64 srcPort, - 5: optional StorageResourceProtocol srcProtocol, - 6: optional string srcFilePath, - 7: optional string srcHostCredToken, - 8: optional string destHostname, - 9: optional string destLoginName, - 10: optional i64 destPort, - 11: optional StorageResourceProtocol destProtocol, - 12: optional string destFilePath, - 13: optional string destHostCredToken, - 14: optional FileTransferMode fileTransferMode, - 15: optional FileTransferStatus transferStatus, - 16: optional i64 fileSize, - 17: optional i64 transferTime, - 18: optional i64 createdTime, - 19: optional i64 lastModifiedType + 2: optional string gatewayId, + 3: optional string username, + 4: optional string srcHostname, + 5: optional string srcLoginName, + 6: optional i64 srcPort, + 7: optional StorageResourceProtocol srcProtocol, + 8: optional string srcFilePath, + 9: optional string srcHostCredToken, + 10: optional string destHostname, + 11: optional string destLoginName, + 12: optional i64 destPort, + 13: optional StorageResourceProtocol destProtocol, + 14: optional string destFilePath, + 15: optional string destHostCredToken, + 16: optional FileTransferMode fileTransferMode, + 17: optional FileTransferStatus transferStatus, + 18: optional i64 fileSize, + 19: optional i64 transferTime, + 20: optional i64 createdTime, + 21: optional i64 lastModifiedType, + 22: optional list<string> callbackEmails } - struct FileNode { - 1: optional FileNodeTypes type, + struct LSEntryModel { + 1: optional LSEntryType type, 2: optional i64 size, 3: optional string nativeType, 4: optional string name, http://git-wip-us.apache.org/repos/asf/airavata/blob/b61cfcd3/thrift-interface-descriptions/data-models/file-manager-models/metadata_models.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/data-models/file-manager-models/metadata_models.thrift b/thrift-interface-descriptions/data-models/file-manager-models/metadata_models.thrift index 927c92f..7ce6e88 100644 --- a/thrift-interface-descriptions/data-models/file-manager-models/metadata_models.thrift +++ b/thrift-interface-descriptions/data-models/file-manager-models/metadata_models.thrift @@ -17,3 +17,25 @@ * under the License. * */ + + namespace java org.apache.airavata.model.file.metadata + namespace php Airavata.Model.file.metadata + namespace cpp apache.airavata.model.file.metadata + namespace py apache.airavata.model.file.metadata + + enum MetadataType{ + FILE, COLLECTION + } + + struct MetadataModel{ + 1: optional string metadataId, + 2: optional string gatewayId, + 3: optional string username, + 4: optional list<string> sharedUsers, + 5: optional bool sharedPublic, + 6: optional string userFriendlyName, + 7: optional string userFriendlyDescription, + 8: optional MetadataType metadataType, + 9: optional string associatedEntityId, + 10: optional map<string,string> customInformation + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/b61cfcd3/thrift-interface-descriptions/data-models/file-manager-models/replica_models.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/data-models/file-manager-models/replica_models.thrift b/thrift-interface-descriptions/data-models/file-manager-models/replica_models.thrift index f010b6c..ff9b932 100644 --- a/thrift-interface-descriptions/data-models/file-manager-models/replica_models.thrift +++ b/thrift-interface-descriptions/data-models/file-manager-models/replica_models.thrift @@ -18,15 +18,20 @@ * */ - include "../resource-catalog-models/data_movement_models.thrift" +include "../resource-catalog-models/data_movement_models.thrift" - namespace java org.apache.airavata.model.replica - namespace php Airavata.Model.Replica - namespace cpp apache.airavata.model.replica - namespace py apache.airavata.model.replica +namespace java org.apache.airavata.model.file.replica +namespace php Airavata.Model.File.Replica +namespace cpp apache.airavata.model.file.replica +namespace py apache.airavata.model.file.replica -enum ReplicaLocationCategory { +enum FileModelType{ + FILE, DIRECTORY +} + +enum StorageResourceType { GATEWAY_DATA_STORE, + BACKUP_GATEWAY_DATA_STORE, COMPUTE_RESOURCE, LONG_TERM_STORAGE_RESOURCE, OTHER @@ -37,41 +42,42 @@ enum ReplicaPersistentType { PERSISTENT } -enum DataResourceType { - COLLECTION, - FILE +struct FileCollectionModel{ + 1: optional string collectionId, + 2: optional string gatewayId, + 3: optional string username, + 4: optional list<string> sharedUsers, + 5: optional bool sharedPublic, + 6: optional string collectionName, + 7: optional string collectionDescription, + 8: optional list<string> fileIdList } -struct DataResourceModel { - 1: optional string resourceId, +struct FileModel { + 1: optional string fileId, 2: optional string gatewayId, - 3: optional string parentResourceId, - 4: optional string resourceName, - 5: optional string resourceDescription, - 6: optional string ownerName, - 7: optional string sha256Checksum, - 8: optional DataResourceType dataResourceType, - 9: optional i32 resourceSize, - 10: optional string nativeFormat, - 11: optional i64 creationTime, - 12: optional i64 lastModifiedTime, - 13: optional map<string, string> resourceMetadata, - 14: optional list<DataReplicaLocationModel> replicaLocations, - 15: optional list<DataResourceModel> childResources + 3: optional string username, + 4: optional list<string> sharedUsers, + 5: optional bool sharedPublic, + 6: optional string fileName, + 7: optional string fileDescription, + 8: optional string sha256Checksum, + 9: optional FileModelType fileType, + 10: optional i32 fileSize, + 11: optional string nativeFormat, + 12: optional i64 creationTime, + 13: optional i64 lastModifiedTime, + 14: optional list<FileReplicaModel> fileReplicas } -struct DataReplicaLocationModel { - 1: optional string replicaId, - 2: optional string resourceId, - 3: optional string replicaName, - 4: optional string replicaDescription, - 5: optional string sourceReplicaId, +struct FileReplicaModel{ + 1: optional string replicaName, + 2: optional string replicaDescription, + 3: optional string storageHostname, + 4: optional string storageResourceId, + 5: optional string filePath, 6: optional i64 creationTime, - 7: optional i64 lastModifiedTime, - 8: optional i64 validUntilTime, - 9: optional ReplicaLocationCategory replicaLocationCategory, - 10: optional ReplicaPersistentType replicaPersistentType, - 11: optional string storageResourceId, - 12: optional string fileAbsolutePath, - 13: optional map<string, string> replicaMetadata -} + 7: optional i64 validUntilTime, + 8: optional StorageResourceType storageResourceType, + 9: optional ReplicaPersistentType replicaPersistentType +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/b61cfcd3/thrift-interface-descriptions/generate-thrift-stubs.sh ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/generate-thrift-stubs.sh b/thrift-interface-descriptions/generate-thrift-stubs.sh index 4a9550a..6b94ce6 100755 --- a/thrift-interface-descriptions/generate-thrift-stubs.sh +++ b/thrift-interface-descriptions/generate-thrift-stubs.sh @@ -139,9 +139,12 @@ copy_changed_files() { echo "Generated sources are in ${GENERATED_CODE_DIR}" echo "Destination workspace is in ${WORKSPACE_SRC_DIR}" - # Check if the newly generated files exist in the targeted workspace, if not copy. Only changed files will be synced. - # the extra slash to GENERATED_CODE_DIR is needed to ensure the parent directory itself is not copied. - rsync -auv ${GENERATED_CODE_DIR}/ ${WORKSPACE_SRC_DIR} + # Remove the existing directory (if exists) and then copies the files + if [ -d ${WORKSPACE_SRC_DIR} ]; then + rm -r ${WORKSPACE_SRC_DIR} + fi + cp -r ${GENERATED_CODE_DIR} ${WORKSPACE_SRC_DIR} + } #######################################
