This is an automated email from the ASF dual-hosted git repository. dimuthuupe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit 35c87225f13d30bce7e8f0edbf8b47a3d5ddacb0 Author: Dimuthu Wannipurage <[email protected]> AuthorDate: Mon Jan 9 12:56:59 2023 -0500 Azure metadata, streaming transport and cli support --- .gitignore | 6 +- agent/service/pom.xml | 26 +++- .../airavata/mft/core/ConnectorResolver.java | 6 + python-cli/mft_cli/mft_cli/main.py | 2 +- python-cli/mft_cli/mft_cli/storage/__init__.py | 3 + python-cli/mft_cli/mft_cli/storage/azure.py | 59 +++++++++ .../server/backend/sql/SQLResourceBackend.java | 28 ++++- .../backend/sql/entity/AzureStorageEntity.java | 68 +++++++++++ .../sql/repository/AzureStorageRepository.java | 31 +++++ .../server/backend/sql/SQLSecretBackend.java | 27 ++--- .../backend/sql/entity/AzureSecretEntity.java | 55 +++++++++ .../sql/repository/AzureSecretRepository.java | 30 +++++ transport/azure-transport/pom.xml | 4 +- .../azure/AzureIncomingStreamingConnector.java | 78 ++++++++++++ .../transport/azure/AzureMetadataCollector.java | 31 ++++- .../azure/AzureOutgoingStreamingConnector.java | 78 ++++++++++++ .../mft/transport/azure/AzureReceiver.java | 134 --------------------- .../airavata/mft/transport/azure/AzureSender.java | 102 ---------------- .../airavata/mft/transport/box/BoxReceiver.java | 2 - .../airavata/mft/transport/box/BoxSender.java | 2 - .../mft/transport/dropbox/DropboxReceiver.java | 2 - .../mft/transport/dropbox/DropboxSender.java | 2 - .../airavata/mft/transport/ftp/FTPReceiver.java | 2 - .../airavata/mft/transport/ftp/FTPSender.java | 2 - transport/pom.xml | 14 --- .../mft/transport/s3/S3OutgoingConnector.java | 4 - .../transport/swift/SwiftIncomingConnector.java | 4 - .../transport/swift/SwiftOutgoingConnector.java | 4 - 28 files changed, 503 insertions(+), 303 deletions(-) diff --git a/.gitignore b/.gitignore index 861afbf..d43a85d 100644 --- a/.gitignore +++ b/.gitignore @@ -1208,4 +1208,8 @@ airavata-mft/ # Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option) -python-sdk/env \ No newline at end of file +python-sdk/env +python-sdk/samples/env/* +python-sdk/venv/* +python-cli/venv/* +python-cli/mft_cli/poetry.lock \ No newline at end of file diff --git a/agent/service/pom.xml b/agent/service/pom.xml index 94660d9..0d1a5cb 100644 --- a/agent/service/pom.xml +++ b/agent/service/pom.xml @@ -37,6 +37,17 @@ <groupId>org.apache.airavata</groupId> <artifactId>mft-agent-stub</artifactId> <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-tcnative-boringssl-static</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-tcnative-boringssl-static</artifactId> + <version>2.0.54.Final</version> </dependency> <dependency> <groupId>org.apache.airavata</groupId> @@ -58,6 +69,17 @@ <groupId>org.apache.airavata</groupId> <artifactId>mft-s3-transport</artifactId> <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-common</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-common</artifactId> + <version>4.1.82.Final</version> </dependency> <dependency> <groupId>org.apache.airavata</groupId> @@ -141,7 +163,7 @@ </dependency> </dependencies> - <build> + <!--build> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -166,7 +188,7 @@ </executions> </plugin> </plugins> - </build> + </build--> <properties> <maven.compiler.source>11</maven.compiler.source> diff --git a/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java b/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java index ef5792c..ce5abf3 100644 --- a/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java +++ b/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java @@ -39,6 +39,9 @@ public final class ConnectorResolver { case "ODATA": className = "org.apache.airavata.mft.transport.odata.ODataIncomingConnector"; break; + case "AZURE": + className = "org.apache.airavata.mft.transport.azure.AzureIncomingStreamingConnector"; + break; } if (className != null) { @@ -59,6 +62,9 @@ public final class ConnectorResolver { case "S3": className = "org.apache.airavata.mft.transport.s3.S3OutgoingStreamingConnector"; break; + case "AZURE": + className = "org.apache.airavata.mft.transport.azure.AzureOutgoingStreamingConnector"; + break; } diff --git a/python-cli/mft_cli/mft_cli/main.py b/python-cli/mft_cli/mft_cli/main.py index 1accfd6..c89065f 100644 --- a/python-cli/mft_cli/mft_cli/main.py +++ b/python-cli/mft_cli/mft_cli/main.py @@ -102,7 +102,7 @@ def copy(source, destination): sourceSecretId = source_secret_id, destinationStorageId = dest_storage_id, destinationSecretId = dest_secret_id, - optimizeTransferPath = True) + optimizeTransferPath = False) if (source_metadata.WhichOneof('metadata') == 'directory') : if (destination[-1] != "/"): diff --git a/python-cli/mft_cli/mft_cli/storage/__init__.py b/python-cli/mft_cli/mft_cli/storage/__init__.py index 51fbdac..08799bc 100644 --- a/python-cli/mft_cli/mft_cli/storage/__init__.py +++ b/python-cli/mft_cli/mft_cli/storage/__init__.py @@ -1,6 +1,7 @@ import typer from pick import pick import mft_cli.storage.s3 as s3 +import mft_cli.storage.azure as azure from airavata_mft_sdk import mft_client from airavata_mft_sdk.common import StorageCommon_pb2 from rich.console import Console @@ -15,6 +16,8 @@ def add_storage(): option, index = pick(options, title, indicator="=>") if option == "S3": s3.handle_add_storage() + elif option == "Azure Storage": + azure.handle_add_storage() @app.command("list") diff --git a/python-cli/mft_cli/mft_cli/storage/azure.py b/python-cli/mft_cli/mft_cli/storage/azure.py new file mode 100644 index 0000000..c83b0bb --- /dev/null +++ b/python-cli/mft_cli/mft_cli/storage/azure.py @@ -0,0 +1,59 @@ +from rich import print +from pick import pick +import typer +from airavata_mft_sdk import mft_client +from airavata_mft_sdk.azure import AzureCredential_pb2 +from airavata_mft_sdk.azure import AzureStorage_pb2 +from airavata_mft_sdk import MFTTransferApi_pb2 +from airavata_mft_sdk import MFTAgentStubs_pb2 +from airavata_mft_sdk.common import StorageCommon_pb2 + +def handle_add_storage(): + + options = ["Through Azure Cli config file", "Enter manually" ] + option, index = pick(options, "How do you want to load credentials", indicator="=>") + + if index == 1: # Manual configuration + connection_string = typer.prompt("Connection String") + + client = mft_client.MFTClient() + + azure_secret = AzureCredential_pb2.AzureSecret(connectionString = connection_string) + secret_wrapper = MFTAgentStubs_pb2.SecretWrapper(azure=azure_secret) + + azure_storage = AzureStorage_pb2.AzureStorage() + storage_wrapper = MFTAgentStubs_pb2.StorageWrapper(azure=azure_storage) + + direct_req = MFTAgentStubs_pb2.GetResourceMetadataRequest(resourcePath="", secret=secret_wrapper, storage=storage_wrapper) + resource_medata_req = MFTTransferApi_pb2.FetchResourceMetadataRequest(directRequest = direct_req) + metadata_resp = client.transfer_api.resourceMetadata(resource_medata_req) + + container_options = ["Manually Enter"] + + container_list = metadata_resp.directory.directories + if len(container_list) > 0: + for c in container_list: + container_options.append(c.friendlyName) + + title = "Select the Container: " + selected_container, index = pick(container_options, title, indicator="=>") + + if index == 0: + selected_container = typer.prompt("Enter container name ") + storage_name = typer.prompt("Name of the storage ", selected_container) + + azure_storage_create_req = AzureStorage_pb2.AzureStorageCreateRequest(container= selected_container, name =storage_name) + + created_storage = client.azure_storage_api.createAzureStorage(azure_storage_create_req) + + secret_create_req= AzureCredential_pb2.AzureSecretCreateRequest(connectionString = connection_string) + created_secret = client.azure_secret_api.createAzureSecret(secret_create_req) + + secret_for_storage_req = StorageCommon_pb2.SecretForStorage(storageId = created_storage.storageId, + secretId = created_secret.secretId, + storageType = StorageCommon_pb2.StorageType.AZURE) + + client.common_api.registerSecretForStorage(secret_for_storage_req) + + print("Successfully added the Azure Bucket...") + diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java index ec9067c..df5d620 100644 --- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java @@ -54,6 +54,9 @@ public class SQLResourceBackend implements ResourceBackend { @Autowired private S3StorageRepository s3StorageRepository; + @Autowired + private AzureStorageRepository azureStorageRepository; + @Autowired private GCSStorageRepository gcsStorageRepository; @@ -318,27 +321,42 @@ public class SQLResourceBackend implements ResourceBackend { @Override public AzureStorageListResponse listAzureStorage(AzureStorageListRequest request) throws Exception { - throw new UnsupportedOperationException("Operation is not supported in backend"); + AzureStorageListResponse.Builder respBuilder = AzureStorageListResponse.newBuilder(); + List<AzureStorageEntity> all = azureStorageRepository.findAll(PageRequest.of(request.getOffset(), request.getLimit())); + all.forEach(ety -> respBuilder.addStorages(mapper.map(ety, AzureStorage.newBuilder().getClass()))); + return respBuilder.build(); } @Override public Optional<AzureStorage> getAzureStorage(AzureStorageGetRequest request) throws Exception { - throw new UnsupportedOperationException("Operation is not supported in backend"); + Optional<AzureStorageEntity> entity = azureStorageRepository.findById(request.getStorageId()); + return entity.map(e -> mapper.map(e, AzureStorage.newBuilder().getClass()).build()); } @Override public AzureStorage createAzureStorage(AzureStorageCreateRequest request) throws Exception { - throw new UnsupportedOperationException("Operation is not supported in backend"); + AzureStorageEntity savedEntity = azureStorageRepository.save(mapper.map(request, AzureStorageEntity.class)); + + ResolveStorageEntity storageTypeEty = new ResolveStorageEntity(); + storageTypeEty.setStorageId(savedEntity.getStorageId()); + storageTypeEty.setStorageType(ResolveStorageEntity.StorageType.AZURE); + storageTypeEty.setStorageName(savedEntity.getName()); + resolveStorageRepository.save(storageTypeEty); + + return mapper.map(savedEntity, AzureStorage.newBuilder().getClass()).build(); } @Override public boolean updateAzureStorage(AzureStorageUpdateRequest request) throws Exception { - throw new UnsupportedOperationException("Operation is not supported in backend"); + azureStorageRepository.save(mapper.map(request, AzureStorageEntity.class)); + return true; } @Override public boolean deleteAzureStorage(AzureStorageDeleteRequest request) throws Exception { - throw new UnsupportedOperationException("Operation is not supported in backend"); + azureStorageRepository.deleteById(request.getStorageId()); + resourceRepository.deleteByStorageIdAndStorageType(request.getStorageId(), GenericResourceEntity.StorageType.AZURE); + return true; } @Override diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/AzureStorageEntity.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/AzureStorageEntity.java new file mode 100644 index 0000000..8fadd5e --- /dev/null +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/AzureStorageEntity.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.resource.server.backend.sql.entity; + +import org.hibernate.annotations.GenericGenerator; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; + +@Entity +public class AzureStorageEntity { + + @Id + @Column(name = "AZURE_STORAGE_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator(name = "uuid", strategy = "uuid2") + private String storageId; + + @Column(name = "STORAGE_NAME") + private String name; + + @Column(name = "CONTAINER") + private String container; + + public String getStorageId() { + return storageId; + } + + public AzureStorageEntity setStorageId(String storageId) { + this.storageId = storageId; + return this; + } + + public String getName() { + return name; + } + + public AzureStorageEntity setName(String name) { + this.name = name; + return this; + } + + public String getContainer() { + return container; + } + + public AzureStorageEntity setContainer(String container) { + this.container = container; + return this; + } +} diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/AzureStorageRepository.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/AzureStorageRepository.java new file mode 100644 index 0000000..0a671b1 --- /dev/null +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/AzureStorageRepository.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.resource.server.backend.sql.repository; + +import org.apache.airavata.mft.resource.server.backend.sql.entity.AzureStorageEntity; +import org.apache.airavata.mft.resource.server.backend.sql.entity.S3StorageEntity; +import org.springframework.data.domain.Pageable; +import org.springframework.data.repository.CrudRepository; + +import java.util.List; + +public interface AzureStorageRepository extends CrudRepository<AzureStorageEntity, String> { + + List<AzureStorageEntity> findAll(Pageable pageable); + +} diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java index 2a6cd22..8c6c356 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java @@ -27,19 +27,11 @@ import org.apache.airavata.mft.credential.stubs.s3.*; import org.apache.airavata.mft.credential.stubs.scp.*; import org.apache.airavata.mft.credential.stubs.swift.*; import org.apache.airavata.mft.secret.server.backend.SecretBackend; -import org.apache.airavata.mft.secret.server.backend.sql.entity.FTPSecretEntity; -import org.apache.airavata.mft.secret.server.backend.sql.entity.GCSSecretEntity; -import org.apache.airavata.mft.secret.server.backend.sql.entity.ODataSecretEntity; -import org.apache.airavata.mft.secret.server.backend.sql.entity.S3SecretEntity; -import org.apache.airavata.mft.secret.server.backend.sql.entity.SCPSecretEntity; +import org.apache.airavata.mft.secret.server.backend.sql.entity.*; import org.apache.airavata.mft.secret.server.backend.sql.entity.swift.SwiftAuthCredentialSecretEntity; import org.apache.airavata.mft.secret.server.backend.sql.entity.swift.SwiftPasswordSecretEntity; import org.apache.airavata.mft.secret.server.backend.sql.entity.swift.SwiftSecretEntity; -import org.apache.airavata.mft.secret.server.backend.sql.repository.FTPSecretRepository; -import org.apache.airavata.mft.secret.server.backend.sql.repository.GCSSecretRepository; -import org.apache.airavata.mft.secret.server.backend.sql.repository.ODataSecretRepository; -import org.apache.airavata.mft.secret.server.backend.sql.repository.S3SecretRepository; -import org.apache.airavata.mft.secret.server.backend.sql.repository.SCPSecretRepository; +import org.apache.airavata.mft.secret.server.backend.sql.repository.*; import org.apache.airavata.mft.secret.server.backend.sql.repository.swift.SwiftAuthCredentialSecretRepository; import org.apache.airavata.mft.secret.server.backend.sql.repository.swift.SwiftPasswordSecretRepository; import org.apache.airavata.mft.secret.server.backend.sql.repository.swift.SwiftSecretRepository; @@ -63,6 +55,9 @@ public class SQLSecretBackend implements SecretBackend { @Autowired private S3SecretRepository s3SecretRepository; + @Autowired + private AzureSecretRepository azureSecretRepository; + @Autowired private SwiftSecretRepository swiftSecretRepository; @@ -160,22 +155,26 @@ public class SQLSecretBackend implements SecretBackend { @Override public Optional<AzureSecret> getAzureSecret(AzureSecretGetRequest request) throws Exception { - throw new UnsupportedOperationException("Operation is not supported in backend"); + Optional<AzureSecretEntity> secretEty = azureSecretRepository.findBySecretId(request.getSecretId()); + return secretEty.map(azSecretEntity -> mapper.map(azSecretEntity, AzureSecret.newBuilder().getClass()).build()); } @Override public AzureSecret createAzureSecret(AzureSecretCreateRequest request) throws Exception { - throw new UnsupportedOperationException("Operation is not supported in backend"); + AzureSecretEntity savedEntity = azureSecretRepository.save(mapper.map(request, AzureSecretEntity.class)); + return mapper.map(savedEntity, AzureSecret.newBuilder().getClass()).build(); } @Override public boolean updateAzureSecret(AzureSecretUpdateRequest request) throws Exception { - throw new UnsupportedOperationException("Operation is not supported in backend"); + azureSecretRepository.save(mapper.map(request, AzureSecretEntity.class)); + return true; } @Override public boolean deleteAzureSecret(AzureSecretDeleteRequest request) throws Exception { - throw new UnsupportedOperationException("Operation is not supported in backend"); + azureSecretRepository.deleteById(request.getSecretId()); + return true; } @Override diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/AzureSecretEntity.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/AzureSecretEntity.java new file mode 100644 index 0000000..c39ddd4 --- /dev/null +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/AzureSecretEntity.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.secret.server.backend.sql.entity; + +import org.hibernate.annotations.GenericGenerator; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; + +@Entity +public class AzureSecretEntity { + @Id + @Column(name = "SECRET_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator( name = "uuid", strategy = "uuid2") + private String secretId; + + @Column(name = "CONNECTION_STRING") + private String connectionString; + + public String getSecretId() { + return secretId; + } + + public AzureSecretEntity setSecretId(String secretId) { + this.secretId = secretId; + return this; + } + + public String getConnectionString() { + return connectionString; + } + + public AzureSecretEntity setConnectionString(String connectionString) { + this.connectionString = connectionString; + return this; + } +} diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/AzureSecretRepository.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/AzureSecretRepository.java new file mode 100644 index 0000000..3c240a8 --- /dev/null +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/AzureSecretRepository.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.secret.server.backend.sql.repository; + +import org.apache.airavata.mft.secret.server.backend.sql.entity.AzureSecretEntity; +import org.apache.airavata.mft.secret.server.backend.sql.entity.S3SecretEntity; +import org.springframework.data.repository.CrudRepository; + +import java.util.Optional; + +public interface AzureSecretRepository extends CrudRepository<AzureSecretEntity, String> { + + Optional<AzureSecretEntity> findBySecretId(String secretId); + +} diff --git a/transport/azure-transport/pom.xml b/transport/azure-transport/pom.xml index 6d05805..b6aff6e 100644 --- a/transport/azure-transport/pom.xml +++ b/transport/azure-transport/pom.xml @@ -36,12 +36,12 @@ <dependency> <groupId>com.azure</groupId> <artifactId>azure-storage-blob</artifactId> - <version>12.13.0</version> + <version>12.20.1</version> </dependency> <dependency> <groupId>org.apache.airavata</groupId> <artifactId>mft-core</artifactId> - <version>0.01-SNAPSHOT</version> + <version>${project.version}</version> </dependency> </dependencies> diff --git a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureIncomingStreamingConnector.java b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureIncomingStreamingConnector.java new file mode 100644 index 0000000..971f708 --- /dev/null +++ b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureIncomingStreamingConnector.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.transport.azure; + +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.BlobServiceClientBuilder; +import org.apache.airavata.mft.core.api.ConnectorConfig; +import org.apache.airavata.mft.core.api.IncomingStreamingConnector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.InputStream; + +public class AzureIncomingStreamingConnector implements IncomingStreamingConnector { + + private static final Logger logger = LoggerFactory.getLogger(AzureIncomingStreamingConnector.class); + + private ConnectorConfig connectorConfig; + private InputStream is; + + @Override + public void init(ConnectorConfig connectorConfig) throws Exception { + this.connectorConfig = connectorConfig; + } + + @Override + public void complete() throws Exception { + if (is != null) { + try { + is.close(); + } catch (Exception e) { + logger.warn("Failed to close the input stream", e); + } + } + } + + @Override + public void failed() throws Exception { + if (is != null) { + try { + is.close(); + } catch (Exception e) { + logger.warn("Failed to close the input stream", e); + } + } + } + + @Override + public InputStream fetchInputStream() throws Exception { + + BlobServiceClient blobServiceClient = new BlobServiceClientBuilder() + .connectionString(connectorConfig.getSecret().getAzure().getConnectionString()).buildClient(); + BlobContainerClient blobContainerClient = blobServiceClient + .getBlobContainerClient(connectorConfig.getStorage().getAzure().getContainer()); + + BlobClient blobClient = blobContainerClient.getBlobClient(connectorConfig.getResourcePath()); + + is = blobClient.openInputStream(); + return is; + } +} diff --git a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java index 5b08c80..0fb3f02 100644 --- a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java +++ b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java @@ -22,6 +22,7 @@ import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.blob.models.BlobContainerItem; import com.azure.storage.blob.models.BlobItem; import com.azure.storage.blob.models.BlobItemProperties; import com.azure.storage.blob.models.BlobProperties; @@ -55,17 +56,34 @@ public class AzureMetadataCollector implements MetadataCollector { // Azure does not have a concept called hierarchical containers. So we assume that there are no containers inside // the given container + ResourceMetadata.Builder metadataBuilder = ResourceMetadata.newBuilder(); - if (!isAvailable(resourcePath)) { - metadataBuilder.setError(MetadataFetchError.NOT_FOUND); + + BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(azureSecret.getConnectionString()).buildClient(); + + if (resourcePath.isEmpty() && azureStorage.getContainer().isEmpty()) { // List containers + PagedIterable<BlobContainerItem> blobContainerItems = blobServiceClient.listBlobContainers(); + DirectoryMetadata.Builder parentDir = DirectoryMetadata.newBuilder(); + parentDir.setResourcePath(""); + parentDir.setFriendlyName(""); + + blobContainerItems.forEach(containerItem -> { + DirectoryMetadata.Builder containerDir = DirectoryMetadata.newBuilder(); + containerDir.setFriendlyName(containerItem.getName()); + containerDir.setResourcePath(containerItem.getName()); + containerDir.setCreatedTime(containerItem.getProperties().getLastModified().toEpochSecond()); + containerDir.setUpdateTime(containerItem.getProperties().getLastModified().toEpochSecond()); + parentDir.addDirectories(containerDir); + }); + metadataBuilder.setDirectory(parentDir); + return metadataBuilder.build(); } - BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(azureSecret.getConnectionString()).buildClient(); BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(azureStorage.getContainer()); - if (resourcePath.isEmpty()) { // List the container + if (resourcePath.isEmpty()) { // List inside the container PagedIterable<BlobItem> blobItems = blobContainerClient.listBlobs(); DirectoryMetadata.Builder directoryBuilder = DirectoryMetadata.newBuilder(); blobItems.forEach(blobItem -> { @@ -89,6 +107,11 @@ public class AzureMetadataCollector implements MetadataCollector { } else { // If resource is a file + if (!isAvailable(resourcePath)) { + metadataBuilder.setError(MetadataFetchError.NOT_FOUND); + return metadataBuilder.build(); + } + BlobClient blobClient = blobContainerClient.getBlobClient(resourcePath); FileMetadata.Builder fileBuilder = FileMetadata.newBuilder(); BlobProperties properties = blobClient.getProperties(); diff --git a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureOutgoingStreamingConnector.java b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureOutgoingStreamingConnector.java new file mode 100644 index 0000000..45ca17d --- /dev/null +++ b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureOutgoingStreamingConnector.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.transport.azure; + +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.BlobServiceClientBuilder; +import org.apache.airavata.mft.core.api.ConnectorConfig; +import org.apache.airavata.mft.core.api.OutgoingStreamingConnector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.OutputStream; + +public class AzureOutgoingStreamingConnector implements OutgoingStreamingConnector { + + private static final Logger logger = LoggerFactory.getLogger(AzureOutgoingStreamingConnector.class); + + private ConnectorConfig connectorConfig; + private OutputStream os; + + @Override + public void init(ConnectorConfig connectorConfig) throws Exception { + this.connectorConfig = connectorConfig; + } + + @Override + public void complete() throws Exception { + if (os != null) { + try { + os.close(); + } catch (Exception e) { + logger.warn("Failed to close the output stream", e); + } + } + } + + @Override + public void failed() throws Exception { + if (os != null) { + try { + os.close(); + } catch (Exception e) { + logger.warn("Failed to close the output stream", e); + } + } + } + + @Override + public OutputStream fetchOutputStream() throws Exception { + + BlobServiceClient blobServiceClient = new BlobServiceClientBuilder() + .connectionString(connectorConfig.getSecret().getAzure().getConnectionString()).buildClient(); + BlobContainerClient blobContainerClient = blobServiceClient + .getBlobContainerClient(connectorConfig.getStorage().getAzure().getContainer()); + + BlobClient blobClient = blobContainerClient.getBlobClient(connectorConfig.getResourcePath()); + + os = blobClient.getBlockBlobClient().getBlobOutputStream(true); + return os; + } +} diff --git a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureReceiver.java b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureReceiver.java deleted file mode 100644 index 6d752ff..0000000 --- a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureReceiver.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.airavata.mft.transport.azure; - -import com.azure.storage.blob.BlobClient; -import com.azure.storage.blob.BlobContainerClient; -import com.azure.storage.blob.BlobServiceClient; -import com.azure.storage.blob.BlobServiceClientBuilder; -import com.azure.storage.blob.specialized.BlobInputStream; -import org.apache.airavata.mft.common.AuthToken; -import org.apache.airavata.mft.core.ConnectorContext; -import org.apache.airavata.mft.core.api.Connector; -import org.apache.airavata.mft.credential.stubs.azure.AzureSecret; -import org.apache.airavata.mft.credential.stubs.azure.AzureSecretGetRequest; -import org.apache.airavata.mft.resource.stubs.azure.storage.AzureStorage; -import org.apache.airavata.mft.secret.client.SecretServiceClient; -import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.OutputStream; - -public class AzureReceiver implements Connector { - - private static final Logger logger = LoggerFactory.getLogger(AzureReceiver.class); - - private boolean initialized = false; - private BlobContainerClient containerClient; - - private String resourceServiceHost; - private int resourceServicePort; - private String secretServiceHost; - private int secretServicePort; - - @Override - public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { - this.initialized = true; - - this.resourceServiceHost = resourceServiceHost; - this.resourceServicePort = resourceServicePort; - this.secretServiceHost = secretServiceHost; - this.secretServicePort = secretServicePort; - } - - @Override - public void destroy() { - - } - - private void checkInitialized() { - if (!initialized) { - throw new IllegalStateException("Azure Receiver is not initialized"); - } - } - - @Override - public void startStream(AuthToken authToken, String resourceId, String credentialToken, ConnectorContext context) throws Exception { - logger.info("Starting azure receive for remote server for transfer {}", context.getTransferId()); - /* - checkInitialized(); - - ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); - GenericResource resource = resourceClient.get().getGenericResource(GenericResourceGetRequest.newBuilder() - .setResourceId(resourceId).build()); - - if (resource.getStorageCase() != GenericResource.StorageCase.AZURESTORAGE) { - logger.error("Invalid storage type {} specified for resource {}", resource.getStorageCase(), resourceId); - throw new Exception("Invalid storage type specified for resource " + resourceId); - } - AzureStorage azureStorage = resource.getAzureStorage(); - - SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort); - AzureSecret azureSecret = secretClient.azure().getAzureSecret(AzureSecretGetRequest.newBuilder().setSecretId(credentialToken).build()); - - BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(azureSecret.getConnectionString()).buildClient(); - this.containerClient = blobServiceClient.getBlobContainerClient(azureStorage.getContainer()); - - BlobClient blobClient = containerClient.getBlobClient(resource.getFile().getResourcePath()); - BlobInputStream blobInputStream = blobClient.openInputStream(); - - OutputStream streamOs = context.getStreamBuffer().getOutputStream(); - - long fileSize = context.getMetadata().getResourceSize(); - - byte[] buf = new byte[1024]; - while (true) { - int bufSize = 0; - - if (buf.length < fileSize) { - bufSize = buf.length; - } else { - bufSize = (int) fileSize; - } - bufSize = blobInputStream.read(buf, 0, bufSize); - - if (bufSize < 0) { - break; - } - - streamOs.write(buf, 0, bufSize); - streamOs.flush(); - - fileSize -= bufSize; - if (fileSize == 0L) - break; - } - - streamOs.close(); - logger.info("Completed azure receive for remote server for transfer {}", context.getTransferId()); - - */ - } - - @Override - public void startStream(AuthToken authToken, String resourceId, String childResourcePath, String credentialToken, - ConnectorContext context) throws Exception { - throw new UnsupportedOperationException(); - } -} diff --git a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureSender.java b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureSender.java deleted file mode 100644 index 7b578b0..0000000 --- a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureSender.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.airavata.mft.transport.azure; - -import com.azure.storage.blob.BlobContainerClient; -import com.azure.storage.blob.BlobServiceClient; -import com.azure.storage.blob.BlobServiceClientBuilder; -import com.azure.storage.blob.specialized.BlockBlobClient; -import org.apache.airavata.mft.common.AuthToken; -import org.apache.airavata.mft.core.ConnectorContext; -import org.apache.airavata.mft.core.api.Connector; -import org.apache.airavata.mft.credential.stubs.azure.AzureSecret; -import org.apache.airavata.mft.credential.stubs.azure.AzureSecretGetRequest; -import org.apache.airavata.mft.resource.stubs.azure.storage.AzureStorage; -import org.apache.airavata.mft.secret.client.SecretServiceClient; -import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class AzureSender implements Connector { - - private static final Logger logger = LoggerFactory.getLogger(AzureSender.class); - - private boolean initialized = false; - BlobContainerClient containerClient; - - private String resourceServiceHost; - private int resourceServicePort; - private String secretServiceHost; - private int secretServicePort; - - @Override - public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { - this.initialized = true; - - this.resourceServiceHost = resourceServiceHost; - this.resourceServicePort = resourceServicePort; - this.secretServiceHost = secretServiceHost; - this.secretServicePort = secretServicePort; - } - - @Override - public void destroy() { - - } - - private void checkInitialized() { - if (!initialized) { - throw new IllegalStateException("Azure Sender is not initialized"); - } - } - - @Override - public void startStream(AuthToken authToken, String resourceId, String credentialToken, ConnectorContext context) throws Exception { - logger.info("Starting Azure send for remote server for transfer {}", context.getTransferId()); - /* - checkInitialized(); - - ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); - GenericResource resource = resourceClient.get().getGenericResource(GenericResourceGetRequest.newBuilder() - .setResourceId(resourceId).build()); - - if (resource.getStorageCase() != GenericResource.StorageCase.AZURESTORAGE) { - logger.error("Invalid storage type {} specified for resource {}", resource.getStorageCase(), resourceId); - throw new Exception("Invalid storage type specified for resource " + resourceId); - } - AzureStorage azureStorage = resource.getAzureStorage(); - - SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort); - AzureSecret azureSecret = secretClient.azure().getAzureSecret(AzureSecretGetRequest.newBuilder().setSecretId(credentialToken).build()); - - BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(azureSecret.getConnectionString()).buildClient(); - this.containerClient = blobServiceClient.getBlobContainerClient(azureStorage.getContainer()); - - BlockBlobClient blockBlobClient = containerClient.getBlobClient(resource.getFile().getResourcePath()).getBlockBlobClient(); - blockBlobClient.upload(context.getStreamBuffer().getInputStream(), context.getMetadata().getResourceSize(), true); - logger.info("Completed Azure send for remote server for transfer {}", context.getTransferId()); - - */ - } - - @Override - public void startStream(AuthToken authToken, String resourceId, String childResourcePath, String credentialToken, - ConnectorContext context) throws Exception { - throw new UnsupportedOperationException(); - } -} diff --git a/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxReceiver.java b/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxReceiver.java index 5103126..9dbd094 100644 --- a/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxReceiver.java +++ b/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxReceiver.java @@ -25,8 +25,6 @@ import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.api.Connector; import org.apache.airavata.mft.credential.stubs.box.BoxSecret; import org.apache.airavata.mft.credential.stubs.box.BoxSecretGetRequest; -import org.apache.airavata.mft.secret.client.SecretServiceClient; -import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxSender.java b/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxSender.java index 10b80cb..70724e8 100644 --- a/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxSender.java +++ b/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxSender.java @@ -25,8 +25,6 @@ import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.api.Connector; import org.apache.airavata.mft.credential.stubs.box.BoxSecret; import org.apache.airavata.mft.credential.stubs.box.BoxSecretGetRequest; -import org.apache.airavata.mft.secret.client.SecretServiceClient; -import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxReceiver.java b/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxReceiver.java index 9d96972..fab26fe 100644 --- a/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxReceiver.java +++ b/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxReceiver.java @@ -24,8 +24,6 @@ import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.api.Connector; import org.apache.airavata.mft.credential.stubs.dropbox.DropboxSecret; import org.apache.airavata.mft.credential.stubs.dropbox.DropboxSecretGetRequest; -import org.apache.airavata.mft.secret.client.SecretServiceClient; -import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxSender.java b/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxSender.java index 0000522..92cf62a 100644 --- a/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxSender.java +++ b/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxSender.java @@ -26,8 +26,6 @@ import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.api.Connector; import org.apache.airavata.mft.credential.stubs.dropbox.DropboxSecret; import org.apache.airavata.mft.credential.stubs.dropbox.DropboxSecretGetRequest; -import org.apache.airavata.mft.secret.client.SecretServiceClient; -import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java index 7aee312..68619ed 100644 --- a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java +++ b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java @@ -23,8 +23,6 @@ import org.apache.airavata.mft.core.api.Connector; import org.apache.airavata.mft.credential.stubs.ftp.FTPSecret; import org.apache.airavata.mft.credential.stubs.ftp.FTPSecretGetRequest; import org.apache.airavata.mft.resource.stubs.ftp.storage.FTPStorage; -import org.apache.airavata.mft.secret.client.SecretServiceClient; -import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder; import org.apache.commons.net.ftp.FTPClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java index 4960243..3b24c7a 100644 --- a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java +++ b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java @@ -23,8 +23,6 @@ import org.apache.airavata.mft.core.api.Connector; import org.apache.airavata.mft.credential.stubs.ftp.FTPSecret; import org.apache.airavata.mft.credential.stubs.ftp.FTPSecretGetRequest; import org.apache.airavata.mft.resource.stubs.ftp.storage.FTPStorage; -import org.apache.airavata.mft.secret.client.SecretServiceClient; -import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder; import org.apache.commons.net.ftp.FTPClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/transport/pom.xml b/transport/pom.xml index 2cc9729..4df6b25 100755 --- a/transport/pom.xml +++ b/transport/pom.xml @@ -44,18 +44,4 @@ <module>swift-transport</module> <module>odata-transport</module> </modules> - <dependencies> - <dependency> - <groupId>org.apache.airavata</groupId> - <artifactId>mft-resource-service-client</artifactId> - <version>0.01-SNAPSHOT</version> - </dependency> - <dependency> - <groupId>org.apache.airavata</groupId> - <artifactId>mft-secret-service-client</artifactId> - <version>0.01-SNAPSHOT</version> - </dependency> - </dependencies> - - </project> \ No newline at end of file diff --git a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java index f6eebaf..860a82e 100644 --- a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java +++ b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java @@ -12,12 +12,8 @@ import org.apache.airavata.mft.core.api.ConnectorConfig; import org.apache.airavata.mft.core.api.OutgoingChunkedConnector; import org.apache.airavata.mft.credential.stubs.s3.S3Secret; import org.apache.airavata.mft.credential.stubs.s3.S3SecretGetRequest; -import org.apache.airavata.mft.resource.client.StorageServiceClient; -import org.apache.airavata.mft.resource.client.StorageServiceClientBuilder; import org.apache.airavata.mft.resource.stubs.s3.storage.S3Storage; import org.apache.airavata.mft.resource.stubs.s3.storage.S3StorageGetRequest; -import org.apache.airavata.mft.secret.client.SecretServiceClient; -import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java index e4555c5..e578b5f 100644 --- a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java +++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java @@ -21,12 +21,8 @@ import org.apache.airavata.mft.core.api.ConnectorConfig; import org.apache.airavata.mft.core.api.IncomingChunkedConnector; import org.apache.airavata.mft.credential.stubs.swift.SwiftSecret; import org.apache.airavata.mft.credential.stubs.swift.SwiftSecretGetRequest; -import org.apache.airavata.mft.resource.client.StorageServiceClient; -import org.apache.airavata.mft.resource.client.StorageServiceClientBuilder; import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorage; import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorageGetRequest; -import org.apache.airavata.mft.secret.client.SecretServiceClient; -import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder; import org.jclouds.ContextBuilder; import org.jclouds.http.options.GetOptions; import org.jclouds.openstack.keystone.auth.config.CredentialTypes; diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java index 52f7a50..310143b 100644 --- a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java +++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java @@ -21,12 +21,8 @@ import org.apache.airavata.mft.core.api.ConnectorConfig; import org.apache.airavata.mft.core.api.OutgoingChunkedConnector; import org.apache.airavata.mft.credential.stubs.swift.SwiftSecret; import org.apache.airavata.mft.credential.stubs.swift.SwiftSecretGetRequest; -import org.apache.airavata.mft.resource.client.StorageServiceClient; -import org.apache.airavata.mft.resource.client.StorageServiceClientBuilder; import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorage; import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorageGetRequest; -import org.apache.airavata.mft.secret.client.SecretServiceClient; -import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder; import org.jclouds.ContextBuilder; import org.jclouds.io.payloads.InputStreamPayload; import org.jclouds.openstack.keystone.auth.config.CredentialTypes;
