This is an automated email from the ASF dual-hosted git repository. dimuthuupe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit 013ed494eb958990d0a6f90186a53103e1237bcd Author: Dimuthu Wannipurage <[email protected]> AuthorDate: Thu Apr 16 00:20:31 2020 -0400 Adding azure blob transfer support --- agent/pom.xml | 16 +++ .../airavata/mft/core/ConnectorResolver.java | 10 ++ .../mft/core/MetadataCollectorResolver.java | 3 + .../resource/server/backend/ResourceBackend.java | 5 + .../backend/airavata/AiravataResourceBackend.java | 20 ++++ .../backend/file/FileBasedResourceBackend.java | 44 +++++++- .../server/backend/sql/SQLResourceBackend.java | 20 ++++ .../server/handler/ResourceServiceHandler.java | 91 +++++++++++++---- .../resources/distribution/conf/resources.json | 6 ++ .../server/src/main/resources/resources.json | 6 ++ .../stub/src/main/proto/ResourceService.proto | 70 +++++++++++++ .../mft/secret/server/backend/SecretBackend.java | 5 + .../backend/airavata/AiravataSecretBackend.java | 20 ++++ .../backend/file/FileBasedSecretBackend.java | 40 ++++++++ .../server/backend/sql/SQLSecretBackend.java | 20 ++++ .../server/handler/SecretServiceHandler.java | 57 +++++++++++ .../main/resources/distribution/conf/secrets.json | 5 + .../server/src/main/resources/secrets.json | 5 + .../stub/src/main/proto/SecretService.proto | 70 +++++++++++++ transport/{ => azure-transport}/pom.xml | 21 ++-- .../transport/azure/AzureMetadataCollector.java | 112 +++++++++++++++++++++ .../mft/transport/azure/AzureReceiver.java | 111 ++++++++++++++++++++ .../airavata/mft/transport/azure/AzureSender.java | 77 ++++++++++++++ transport/pom.xml | 1 + 24 files changed, 801 insertions(+), 34 deletions(-) diff --git a/agent/pom.xml b/agent/pom.xml index cd00e66..80cab0c 100644 --- a/agent/pom.xml +++ b/agent/pom.xml @@ -50,6 +50,11 @@ </dependency> <dependency> <groupId>org.apache.airavata</groupId> + <artifactId>mft-azure-transport</artifactId> + <version>0.01-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.airavata</groupId> <artifactId>mft-admin</artifactId> <version>0.01-SNAPSHOT</version> </dependency> @@ -57,6 +62,17 @@ <groupId>com.orbitz.consul</groupId> <artifactId>consul-client</artifactId> <version>${consul.client}</version> + <exclusions> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + <version>2.10.0</version> </dependency> <dependency> <groupId>org.slf4j</groupId> diff --git a/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java b/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java index 3f76d07..5c49678 100644 --- a/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java +++ b/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java @@ -57,6 +57,16 @@ public final class ConnectorResolver { break; } break; + case "AZURE": + switch (direction) { + case "IN": + className = "org.apache.airavata.mft.transport.azure.AzureReceiver"; + break; + case "OUT": + className = "org.apache.airavata.mft.transport.azure.AzureSender"; + break; + } + break; } if (className != null) { diff --git a/core/src/main/java/org/apache/airavata/mft/core/MetadataCollectorResolver.java b/core/src/main/java/org/apache/airavata/mft/core/MetadataCollectorResolver.java index 653e56d..e8ca689 100644 --- a/core/src/main/java/org/apache/airavata/mft/core/MetadataCollectorResolver.java +++ b/core/src/main/java/org/apache/airavata/mft/core/MetadataCollectorResolver.java @@ -36,6 +36,9 @@ public final class MetadataCollectorResolver { case "S3": className = "org.apache.airavata.mft.transport.s3.S3MetadataCollector"; break; + case "AZURE": + className = "org.apache.airavata.mft.transport.azure.AzureMetadataCollector"; + break; } if (className != null) { diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java index 6857bde..231c738 100644 --- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java @@ -46,4 +46,9 @@ public interface ResourceBackend { public S3Resource createS3Resource(S3ResourceCreateRequest request) throws Exception; public boolean updateS3Resource(S3ResourceUpdateRequest request) throws Exception; public boolean deleteS3Resource(S3ResourceDeleteRequest request) throws Exception; + + public Optional<AzureResource> getAzureResource(AzureResourceGetRequest request) throws Exception; + public AzureResource createAzureResource(AzureResourceCreateRequest request) throws Exception; + public boolean updateAzureResource(AzureResourceUpdateRequest request) throws Exception; + public boolean deleteAzureResource(AzureResourceDeleteRequest request) throws Exception; } diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/airavata/AiravataResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/airavata/AiravataResourceBackend.java index f888134..09de364 100644 --- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/airavata/AiravataResourceBackend.java +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/airavata/AiravataResourceBackend.java @@ -191,4 +191,24 @@ public class AiravataResourceBackend implements ResourceBackend { throw new UnsupportedOperationException("Operation is not supported in backend"); } + + @Override + public Optional<AzureResource> getAzureResource(AzureResourceGetRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public AzureResource createAzureResource(AzureResourceCreateRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public boolean updateAzureResource(AzureResourceUpdateRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public boolean deleteAzureResource(AzureResourceDeleteRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } } diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java index cb504b3..c8d1d4c 100644 --- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java @@ -136,7 +136,6 @@ public class FileBasedResourceBackend implements ResourceBackend { JSONArray resourceList = (JSONArray) obj; - System.out.println("All resources "); List<LocalResource> localResources = (List<LocalResource>) resourceList.stream() .filter(resource -> "LOCAL".equals(((JSONObject) resource).get("type").toString())) .map(resource -> { @@ -180,7 +179,6 @@ public class FileBasedResourceBackend implements ResourceBackend { JSONArray resourceList = (JSONArray) obj; - System.out.println("All resources "); List<S3Resource> s3Resources = (List<S3Resource>) resourceList.stream() .filter(resource -> "S3".equals(((JSONObject) resource).get("type").toString())) .map(resource -> { @@ -217,4 +215,46 @@ public class FileBasedResourceBackend implements ResourceBackend { throw new UnsupportedOperationException("Operation is not supported in backend"); } + + @Override + public Optional<AzureResource> getAzureResource(AzureResourceGetRequest request) throws Exception { + JSONParser jsonParser = new JSONParser(); + InputStream inputStream = FileBasedResourceBackend.class.getClassLoader().getResourceAsStream(resourceFile); + + try (InputStreamReader reader = new InputStreamReader(inputStream)) { + Object obj = jsonParser.parse(reader); + + JSONArray resourceList = (JSONArray) obj; + + List<AzureResource> azureResources = (List<AzureResource>) resourceList.stream() + .filter(resource -> "AZURE".equals(((JSONObject) resource).get("type").toString())) + .map(resource -> { + JSONObject r = (JSONObject) resource; + + AzureResource azureResource = AzureResource.newBuilder() + .setBlobName(r.get("blobName").toString()) + .setContainer(r.get("container").toString()) + .setResourceId(r.get("resourceId").toString()) + .build(); + + return azureResource; + }).collect(Collectors.toList()); + return azureResources.stream().filter(r -> request.getResourceId().equals(r.getResourceId())).findFirst(); + } + } + + @Override + public AzureResource createAzureResource(AzureResourceCreateRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public boolean updateAzureResource(AzureResourceUpdateRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public boolean deleteAzureResource(AzureResourceDeleteRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } } diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java index a02d5a9..b02f9e2 100644 --- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java @@ -155,4 +155,24 @@ public class SQLResourceBackend implements ResourceBackend { throw new UnsupportedOperationException("Operation is not supported in backend"); } + + @Override + public Optional<AzureResource> getAzureResource(AzureResourceGetRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public AzureResource createAzureResource(AzureResourceCreateRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public boolean updateAzureResource(AzureResourceUpdateRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public boolean deleteAzureResource(AzureResourceDeleteRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } } diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/handler/ResourceServiceHandler.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/handler/ResourceServiceHandler.java index 16bd850..edcc7c9 100644 --- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/handler/ResourceServiceHandler.java +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/handler/ResourceServiceHandler.java @@ -14,22 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.apache.airavata.mft.resource.server.handler; @@ -294,7 +278,8 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI responseObserver.onError(Status.INTERNAL.withCause(e) .withDescription("Failed in creating the S3 resource") .asRuntimeException()); - } } + } + } @Override public void updateS3Resource(S3ResourceUpdateRequest request, StreamObserver<Empty> responseObserver) { @@ -307,7 +292,8 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI responseObserver.onError(Status.INTERNAL.withCause(e) .withDescription("Failed in updating the S3 resource with id " + request.getResourceId()) .asRuntimeException()); - } } + } + } @Override public void deleteS3Resource(S3ResourceDeleteRequest request, StreamObserver<Empty> responseObserver) { @@ -324,5 +310,72 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI responseObserver.onError(Status.INTERNAL.withCause(e) .withDescription("Failed in deleting the S3 resource with id " + request.getResourceId()) .asRuntimeException()); - } } + } + } + + @Override + public void getAzureResource(AzureResourceGetRequest request, StreamObserver<AzureResource> responseObserver) { + try { + this.backend.getAzureResource(request).ifPresentOrElse(resource -> { + responseObserver.onNext(resource); + responseObserver.onCompleted(); + }, () -> { + responseObserver.onError(Status.INTERNAL + .withDescription("No Azure Resource with id " + request.getResourceId()) + .asRuntimeException()); + }); + } catch (Exception e) { + logger.error("Failed in retrieving Azure resource with id {}", request.getResourceId(), e); + + responseObserver.onError(Status.INTERNAL.withCause(e) + .withDescription("Failed in retrieving Azure resource with id " + request.getResourceId()) + .asRuntimeException()); + } + } + + @Override + public void createAzureResource(AzureResourceCreateRequest request, StreamObserver<AzureResource> responseObserver) { + try { + responseObserver.onNext(this.backend.createAzureResource(request)); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.error("Failed in creating the Azure resource", e); + + responseObserver.onError(Status.INTERNAL.withCause(e) + .withDescription("Failed in creating the Azure resource") + .asRuntimeException()); + } + } + + @Override + public void updateAzureResource(AzureResourceUpdateRequest request, StreamObserver<Empty> responseObserver) { + try { + this.backend.updateAzureResource(request); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.error("Failed in updating the Azure resource {}", request.getResourceId(), e); + + responseObserver.onError(Status.INTERNAL.withCause(e) + .withDescription("Failed in updating the Azure resource with id " + request.getResourceId()) + .asRuntimeException()); + } + } + + @Override + public void deleteAzureResource(AzureResourceDeleteRequest request, StreamObserver<Empty> responseObserver) { + try { + boolean res = this.backend.deleteAzureResource(request); + if (res) { + responseObserver.onCompleted(); + } else { + responseObserver.onError(new Exception("Failed to delete Azure Resource with id " + request.getResourceId())); + } + } catch (Exception e) { + logger.error("Failed in deleting the Azure resource {}", request.getResourceId(), e); + + responseObserver.onError(Status.INTERNAL.withCause(e) + .withDescription("Failed in deleting the Azure resource with id " + request.getResourceId()) + .asRuntimeException()); + } + } } diff --git a/services/resource-service/server/src/main/resources/distribution/conf/resources.json b/services/resource-service/server/src/main/resources/distribution/conf/resources.json index 0791f2c..6b1f9c3 100644 --- a/services/resource-service/server/src/main/resources/distribution/conf/resources.json +++ b/services/resource-service/server/src/main/resources/distribution/conf/resources.json @@ -32,5 +32,11 @@ "resourcePath": "new-file.txt", "region": "us-east-2", "bucketName": "s3-bucket" + }, + { + "type": "AZURE", + "resourceId": "azure-blob", + "container": "sample-container", + "blobName": "sample.blob" } ] \ No newline at end of file diff --git a/services/resource-service/server/src/main/resources/resources.json b/services/resource-service/server/src/main/resources/resources.json index 0791f2c..6b1f9c3 100644 --- a/services/resource-service/server/src/main/resources/resources.json +++ b/services/resource-service/server/src/main/resources/resources.json @@ -32,5 +32,11 @@ "resourcePath": "new-file.txt", "region": "us-east-2", "bucketName": "s3-bucket" + }, + { + "type": "AZURE", + "resourceId": "azure-blob", + "container": "sample-container", + "blobName": "sample.blob" } ] \ No newline at end of file diff --git a/services/resource-service/stub/src/main/proto/ResourceService.proto b/services/resource-service/stub/src/main/proto/ResourceService.proto index 42af6b9..4b757b5 100644 --- a/services/resource-service/stub/src/main/proto/ResourceService.proto +++ b/services/resource-service/stub/src/main/proto/ResourceService.proto @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + syntax = "proto3"; option java_multiple_files = true; @@ -117,6 +134,33 @@ message S3ResourceDeleteRequest { string resourceId = 1; } +// Azure Resource + +message AzureResource { + string resourceId = 1; + string container = 2; + string blobName = 3; +} + +message AzureResourceGetRequest { + string resourceId = 1; +} + +message AzureResourceCreateRequest { + string container = 1; + string blobName = 2; +} + +message AzureResourceUpdateRequest { + string resourceId = 1; + string container = 2; + string blobName = 3; +} + +message AzureResourceDeleteRequest { + string resourceId = 1; +} + service ResourceService { // SCP Storage @@ -221,4 +265,30 @@ service ResourceService { delete: "/v1.0/resource/s3" }; } + + // Azure Resource + + rpc getAzureResource (AzureResourceGetRequest) returns (AzureResource) { + option (google.api.http) = { + get: "/v1.0/resource/azure" + }; + } + + rpc createAzureResource (AzureResourceCreateRequest) returns (AzureResource) { + option (google.api.http) = { + post: "/v1.0/resource/azure" + }; + } + + rpc updateAzureResource (AzureResourceUpdateRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + put: "/v1.0/resource/azure" + }; + } + + rpc deleteAzureResource (AzureResourceDeleteRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + delete: "/v1.0/resource/azure" + }; + } } \ No newline at end of file diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java index 3172387..e187beb 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java @@ -35,4 +35,9 @@ public interface SecretBackend { public S3Secret createS3Secret(S3SecretCreateRequest request) throws Exception; public boolean updateS3Secret(S3SecretUpdateRequest request) throws Exception; public boolean deleteS3Secret(S3SecretDeleteRequest request) throws Exception; + + public Optional<AzureSecret> getAzureSecret(AzureSecretGetRequest request) throws Exception; + public AzureSecret createAzureSecret(AzureSecretCreateRequest request) throws Exception; + public boolean updateAzureSecret(AzureSecretUpdateRequest request) throws Exception; + public boolean deleteAzureSecret(AzureSecretDeleteRequest request) throws Exception; } diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/airavata/AiravataSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/airavata/AiravataSecretBackend.java index a224c0a..9c52eb1 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/airavata/AiravataSecretBackend.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/airavata/AiravataSecretBackend.java @@ -99,4 +99,24 @@ public class AiravataSecretBackend implements SecretBackend { public boolean deleteS3Secret(S3SecretDeleteRequest request) throws Exception { throw new UnsupportedOperationException("Operation is not supported in backend"); } + + @Override + public Optional<AzureSecret> getAzureSecret(AzureSecretGetRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public AzureSecret createAzureSecret(AzureSecretCreateRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public boolean updateAzureSecret(AzureSecretUpdateRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public boolean deleteAzureSecret(AzureSecretDeleteRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } } diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/file/FileBasedSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/file/FileBasedSecretBackend.java index d2d97ad..4203929 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/file/FileBasedSecretBackend.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/file/FileBasedSecretBackend.java @@ -130,4 +130,44 @@ public class FileBasedSecretBackend implements SecretBackend { public boolean deleteS3Secret(S3SecretDeleteRequest request) throws Exception { throw new UnsupportedOperationException("Operation is not supported in backend"); } + + @Override + public Optional<AzureSecret> getAzureSecret(AzureSecretGetRequest request) throws Exception { + JSONParser jsonParser = new JSONParser(); + InputStream inputStream = FileBasedSecretBackend.class.getClassLoader().getResourceAsStream(secretFile); + + try (InputStreamReader reader = new InputStreamReader(inputStream)) { + Object obj = jsonParser.parse(reader); + + JSONArray resourceList = (JSONArray) obj; + + List<AzureSecret> azureSecrets = (List<AzureSecret>) resourceList.stream() + .filter(resource -> "AZURE".equals(((JSONObject) resource).get("type").toString())) + .map(resource -> { + JSONObject r = (JSONObject) resource; + + AzureSecret azureSecret = AzureSecret.newBuilder() + .setSecretId(r.get("secretId").toString()) + .setConnectionString(r.get("connectionString").toString()).build(); + + return azureSecret; + }).collect(Collectors.toList()); + return azureSecrets.stream().filter(r -> request.getSecretId().equals(r.getSecretId())).findFirst(); + } + } + + @Override + public AzureSecret createAzureSecret(AzureSecretCreateRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public boolean updateAzureSecret(AzureSecretUpdateRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public boolean deleteAzureSecret(AzureSecretDeleteRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } } diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java index 79da582..e9c1e40 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java @@ -90,4 +90,24 @@ public class SQLSecretBackend implements SecretBackend { public boolean deleteS3Secret(S3SecretDeleteRequest request) throws Exception { throw new UnsupportedOperationException("Operation is not supported in backend"); } + + @Override + public Optional<AzureSecret> getAzureSecret(AzureSecretGetRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public AzureSecret createAzureSecret(AzureSecretCreateRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public boolean updateAzureSecret(AzureSecretUpdateRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public boolean deleteAzureSecret(AzureSecretDeleteRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } } diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java index 72a9706..65ee4f8 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java @@ -137,4 +137,61 @@ public class SecretServiceHandler extends SecretServiceGrpc.SecretServiceImplBas .asRuntimeException()); } } + + @Override + public void getAzureSecret(AzureSecretGetRequest request, StreamObserver<AzureSecret> responseObserver) { + try { + this.backend.getAzureSecret(request).ifPresentOrElse(secret -> { + responseObserver.onNext(secret); + responseObserver.onCompleted(); + }, () -> { + responseObserver.onError(Status.INTERNAL + .withDescription("No Azure Secret with id " + request.getSecretId()) + .asRuntimeException()); + }); + + } catch (Exception e) { + logger.error("Error in retrieving Azure Secret with id " + request.getSecretId(), e); + responseObserver.onError(Status.INTERNAL.withCause(e) + .withDescription("Error in retrieving Azure Secret with id " + request.getSecretId()) + .asRuntimeException()); + } + super.getAzureSecret(request, responseObserver); + } + + @Override + public void createAzureSecret(AzureSecretCreateRequest request, StreamObserver<AzureSecret> responseObserver) { + try { + this.backend.createAzureSecret(request); + } catch (Exception e) { + logger.error("Error in creating Azure Secret", e); + responseObserver.onError(Status.INTERNAL.withCause(e) + .withDescription("Error in creating Azure Secret") + .asRuntimeException()); + } + } + + @Override + public void updateAzureSecret(AzureSecretUpdateRequest request, StreamObserver<Empty> responseObserver) { + try { + this.backend.updateAzureSecret(request); + } catch (Exception e) { + logger.error("Error in updating Azure Secret with id {}", request.getSecretId(), e); + responseObserver.onError(Status.INTERNAL.withCause(e) + .withDescription("Error in updating Azure Secret with id " + request.getSecretId()) + .asRuntimeException()); + } + } + + @Override + public void deleteAzureSecret(AzureSecretDeleteRequest request, StreamObserver<Empty> responseObserver) { + try { + this.backend.deleteAzureSecret(request); + } catch (Exception e) { + logger.error("Error in deleting Azure Secret with id {}", request.getSecretId(), e); + responseObserver.onError(Status.INTERNAL.withCause(e) + .withDescription("Error in deleting Azure Secret with id " + request.getSecretId()) + .asRuntimeException()); + } + } } diff --git a/services/secret-service/server/src/main/resources/distribution/conf/secrets.json b/services/secret-service/server/src/main/resources/distribution/conf/secrets.json index 042efaf..49cfc5d 100644 --- a/services/secret-service/server/src/main/resources/distribution/conf/secrets.json +++ b/services/secret-service/server/src/main/resources/distribution/conf/secrets.json @@ -11,5 +11,10 @@ "secretId": "s3-cred", "accessKey": "", "secretKey": "" + }, + { + "type": "AZURE", + "secretId": "azure-cred", + "connectionString": "azure-con" } ] \ No newline at end of file diff --git a/services/secret-service/server/src/main/resources/secrets.json b/services/secret-service/server/src/main/resources/secrets.json index fd5af0e..8262246 100644 --- a/services/secret-service/server/src/main/resources/secrets.json +++ b/services/secret-service/server/src/main/resources/secrets.json @@ -12,5 +12,10 @@ "secretId": "s3-cred", "accessKey": "", "secretKey": "" + }, + { + "type": "AZURE", + "secretId": "azure-cred", + "connectionString": "azure-con" } ] \ No newline at end of file diff --git a/services/secret-service/stub/src/main/proto/SecretService.proto b/services/secret-service/stub/src/main/proto/SecretService.proto index 93dfe3e..1fa8187 100644 --- a/services/secret-service/stub/src/main/proto/SecretService.proto +++ b/services/secret-service/stub/src/main/proto/SecretService.proto @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + syntax = "proto3"; option java_multiple_files = true; @@ -71,6 +88,33 @@ message S3SecretDeleteRequest { AuthToken authzToken = 2; } +// Azure + +message AzureSecret { + string secretId = 1; + string connectionString = 2; +} + +message AzureSecretGetRequest { + string secretId = 1; + AuthToken authzToken = 2; +} + +message AzureSecretCreateRequest { + string connectionString = 1; + AuthToken authzToken = 2; +} + +message AzureSecretUpdateRequest { + string secretId = 1; + string connectionString = 2; + AuthToken authzToken = 3; +} + +message AzureSecretDeleteRequest { + string secretId = 1; + AuthToken authzToken = 2; +} service SecretService { rpc getSCPSecret (SCPSecretGetRequest) returns (SCPSecret) { @@ -120,4 +164,30 @@ service SecretService { delete: "/v1.0/secret/s3" }; } + + // Azure + + rpc getAzureSecret (AzureSecretGetRequest) returns (AzureSecret) { + option (google.api.http) = { + get: "/v1.0/secret/azure" + }; + } + + rpc createAzureSecret (AzureSecretCreateRequest) returns (AzureSecret) { + option (google.api.http) = { + post: "/v1.0/secret/azure" + }; + } + + rpc updateAzureSecret (AzureSecretUpdateRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + put: "/v1.0/secret/azure" + }; + } + + rpc deleteAzureSecret (AzureSecretDeleteRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + delete: "/v1.0/secret/azure" + }; + } } \ No newline at end of file diff --git a/transport/pom.xml b/transport/azure-transport/pom.xml old mode 100755 new mode 100644 similarity index 74% copy from transport/pom.xml copy to transport/azure-transport/pom.xml index e1373da..d2d8d5f --- a/transport/pom.xml +++ b/transport/azure-transport/pom.xml @@ -24,31 +24,26 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <parent> - <artifactId>airavata-mft</artifactId> + <artifactId>mft-transport</artifactId> <groupId>org.apache.airavata</groupId> <version>0.01-SNAPSHOT</version> </parent> <modelVersion>4.0.0</modelVersion> - <artifactId>mft-transport</artifactId> - <packaging>pom</packaging> - <modules> - <module>scp-transport</module> - <module>local-transport</module> - <module>s3-transport</module> - </modules> + <artifactId>mft-azure-transport</artifactId> + <dependencies> <dependency> - <groupId>org.apache.airavata</groupId> - <artifactId>mft-resource-service-client</artifactId> - <version>0.01-SNAPSHOT</version> + <groupId>com.azure</groupId> + <artifactId>azure-storage-blob</artifactId> + <version>12.0.0</version> </dependency> <dependency> <groupId>org.apache.airavata</groupId> - <artifactId>mft-secret-service-client</artifactId> + <artifactId>mft-core</artifactId> <version>0.01-SNAPSHOT</version> </dependency> - </dependencies> + </dependencies> </project> \ No newline at end of file diff --git a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java new file mode 100644 index 0000000..cc6e669 --- /dev/null +++ b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.transport.azure; + +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.blob.models.BlobProperties; +import org.apache.airavata.mft.core.ResourceMetadata; +import org.apache.airavata.mft.core.api.MetadataCollector; +import org.apache.airavata.mft.resource.client.ResourceServiceClient; +import org.apache.airavata.mft.resource.service.AzureResource; +import org.apache.airavata.mft.resource.service.AzureResourceGetRequest; +import org.apache.airavata.mft.resource.service.ResourceServiceGrpc; +import org.apache.airavata.mft.secret.client.SecretServiceClient; +import org.apache.airavata.mft.secret.service.AzureSecret; +import org.apache.airavata.mft.secret.service.AzureSecretGetRequest; +import org.apache.airavata.mft.secret.service.SecretServiceGrpc; + +public class AzureMetadataCollector implements MetadataCollector { + + private String resourceServiceHost; + private int resourceServicePort; + private String secretServiceHost; + private int secretServicePort; + boolean initialized = false; + + @Override + public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) { + this.resourceServiceHost = resourceServiceHost; + this.resourceServicePort = resourceServicePort; + this.secretServiceHost = secretServiceHost; + this.secretServicePort = secretServicePort; + this.initialized = true; + } + + private void checkInitialized() { + if (!initialized) { + throw new IllegalStateException("Azure Metadata Collector is not initialized"); + } + } + + @Override + public ResourceMetadata getGetResourceMetadata(String resourceId, String credentialToken) throws Exception { + checkInitialized(); + + if (!isAvailable(resourceId, credentialToken)) { + throw new Exception("Azure blob can not find for resource id " + resourceId); + } + + ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient(resourceServiceHost, resourceServicePort); + AzureResource azureResource = resourceClient.getAzureResource(AzureResourceGetRequest.newBuilder().setResourceId(resourceId).build()); + + SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient(secretServiceHost, secretServicePort); + AzureSecret azureSecret = secretClient.getAzureSecret(AzureSecretGetRequest.newBuilder().setSecretId(credentialToken).build()); + + BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(azureSecret.getConnectionString()).buildClient(); + + BlobClient blobClient = blobServiceClient.getBlobContainerClient(azureResource.getContainer()).getBlobClient(azureResource.getBlobName()); + + BlobProperties properties = blobClient.getBlockBlobClient().getProperties(); + ResourceMetadata metadata = new ResourceMetadata(); + metadata.setResourceSize(properties.getBlobSize()); + metadata.setCreatedTime(properties.getCreationTime().toEpochSecond()); + metadata.setUpdateTime(properties.getCreationTime().toEpochSecond()); + + byte[] contentMd5 = properties.getContentMd5(); + StringBuilder md5sb = new StringBuilder(); + for (byte aByte : contentMd5) { + md5sb.append(Integer.toString((aByte & 0xff) + 0x100, 16).substring(1)); + } + + metadata.setMd5sum(md5sb.toString()); + + return metadata; + } + + @Override + public Boolean isAvailable(String resourceId, String credentialToken) throws Exception { + checkInitialized(); + + ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient(resourceServiceHost, resourceServicePort); + AzureResource azureResource = resourceClient.getAzureResource(AzureResourceGetRequest.newBuilder().setResourceId(resourceId).build()); + + SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient(secretServiceHost, secretServicePort); + AzureSecret azureSecret = secretClient.getAzureSecret(AzureSecretGetRequest.newBuilder().setSecretId(credentialToken).build()); + + BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(azureSecret.getConnectionString()).buildClient(); + BlobContainerClient containerClient = blobServiceClient.getBlobContainerClient(azureResource.getContainer()); + boolean containerExists = containerClient.exists(); + if (!containerExists) { + return false; + } + return containerClient.getBlobClient(azureResource.getBlobName()).exists(); + } +} diff --git a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureReceiver.java b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureReceiver.java new file mode 100644 index 0000000..6685bd5 --- /dev/null +++ b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureReceiver.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.transport.azure; + +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.blob.specialized.BlobInputStream; +import com.azure.storage.blob.specialized.BlockBlobClient; +import org.apache.airavata.mft.core.ConnectorContext; +import org.apache.airavata.mft.core.api.Connector; +import org.apache.airavata.mft.resource.client.ResourceServiceClient; +import org.apache.airavata.mft.resource.service.AzureResource; +import org.apache.airavata.mft.resource.service.AzureResourceGetRequest; +import org.apache.airavata.mft.resource.service.ResourceServiceGrpc; +import org.apache.airavata.mft.secret.client.SecretServiceClient; +import org.apache.airavata.mft.secret.service.AzureSecret; +import org.apache.airavata.mft.secret.service.AzureSecretGetRequest; +import org.apache.airavata.mft.secret.service.SecretServiceGrpc; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.OutputStream; + +public class AzureReceiver implements Connector { + + private static final Logger logger = LoggerFactory.getLogger(AzureReceiver.class); + + private boolean initialized = false; + private AzureResource azureResource; + BlobContainerClient containerClient; + + @Override + public void init(String resourceId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { + this.initialized = true; + + ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient(resourceServiceHost, resourceServicePort); + this.azureResource = resourceClient.getAzureResource(AzureResourceGetRequest.newBuilder().setResourceId(resourceId).build()); + + SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient(secretServiceHost, secretServicePort); + AzureSecret azureSecret = secretClient.getAzureSecret(AzureSecretGetRequest.newBuilder().setSecretId(credentialToken).build()); + + BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(azureSecret.getConnectionString()).buildClient(); + this.containerClient = blobServiceClient.getBlobContainerClient(azureResource.getContainer()); + } + + @Override + public void destroy() { + + } + + private void checkInitialized() { + if (!initialized) { + throw new IllegalStateException("Azure Receiver is not initialized"); + } + } + + @Override + public void startStream(ConnectorContext context) throws Exception { + logger.info("Starting azure receive for remote server for transfer {}", context.getTransferId()); + checkInitialized(); + BlobClient blobClient = containerClient.getBlobClient(azureResource.getBlobName()); + BlobInputStream blobInputStream = blobClient.openInputStream(); + + OutputStream streamOs = context.getStreamBuffer().getOutputStream(); + + long fileSize = context.getMetadata().getResourceSize(); + + byte[] buf = new byte[1024]; + while (true) { + int bufSize = 0; + + if (buf.length < fileSize) { + bufSize = buf.length; + } else { + bufSize = (int) fileSize; + } + bufSize = blobInputStream.read(buf, 0, bufSize); + + if (bufSize < 0) { + break; + } + + streamOs.write(buf, 0, bufSize); + streamOs.flush(); + + fileSize -= bufSize; + if (fileSize == 0L) + break; + } + + streamOs.close(); + logger.info("Completed azure receive for remote server for transfer {}", context.getTransferId()); + } +} diff --git a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureSender.java b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureSender.java new file mode 100644 index 0000000..cbca14e --- /dev/null +++ b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureSender.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.transport.azure; + +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.blob.specialized.BlockBlobClient; +import org.apache.airavata.mft.core.ConnectorContext; +import org.apache.airavata.mft.core.api.Connector; +import org.apache.airavata.mft.resource.client.ResourceServiceClient; +import org.apache.airavata.mft.resource.service.AzureResource; +import org.apache.airavata.mft.resource.service.AzureResourceGetRequest; +import org.apache.airavata.mft.resource.service.ResourceServiceGrpc; +import org.apache.airavata.mft.secret.client.SecretServiceClient; +import org.apache.airavata.mft.secret.service.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AzureSender implements Connector { + + private static final Logger logger = LoggerFactory.getLogger(AzureSender.class); + + private boolean initialized = false; + private AzureResource azureResource; + BlobContainerClient containerClient; + + @Override + public void init(String resourceId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { + this.initialized = true; + + ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient(resourceServiceHost, resourceServicePort); + this.azureResource = resourceClient.getAzureResource(AzureResourceGetRequest.newBuilder().setResourceId(resourceId).build()); + + SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient(secretServiceHost, secretServicePort); + AzureSecret azureSecret = secretClient.getAzureSecret(AzureSecretGetRequest.newBuilder().setSecretId(credentialToken).build()); + + BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(azureSecret.getConnectionString()).buildClient(); + this.containerClient = blobServiceClient.getBlobContainerClient(azureResource.getContainer()); + } + + @Override + public void destroy() { + + } + + private void checkInitialized() { + if (!initialized) { + throw new IllegalStateException("Azure Sender is not initialized"); + } + } + + @Override + public void startStream(ConnectorContext context) throws Exception { + logger.info("Starting scp send for remote server for transfer {}", context.getTransferId()); + checkInitialized(); + BlockBlobClient blockBlobClient = containerClient.getBlobClient(azureResource.getBlobName()).getBlockBlobClient(); + blockBlobClient.upload(context.getStreamBuffer().getInputStream(), context.getMetadata().getResourceSize(), true); + logger.info("Completed scp send for remote server for transfer {}", context.getTransferId()); + + } +} diff --git a/transport/pom.xml b/transport/pom.xml index e1373da..9a68166 100755 --- a/transport/pom.xml +++ b/transport/pom.xml @@ -36,6 +36,7 @@ <module>scp-transport</module> <module>local-transport</module> <module>s3-transport</module> + <module>azure-transport</module> </modules> <dependencies> <dependency>
