DImuthuUpe commented on a change in pull request #6:
URL: https://github.com/apache/airavata-mft/pull/6#discussion_r412717922
##########
File path:
services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/file/FileBasedSecretBackend.java
##########
@@ -211,4 +213,57 @@ public boolean updateAzureSecret(AzureSecretUpdateRequest
request) throws Except
public boolean deleteAzureSecret(AzureSecretDeleteRequest request) throws
Exception {
throw new UnsupportedOperationException("Operation is not supported in
backend");
}
+
+ @Override
+ public Optional<GCSSecret> getGCSSecret(GCSSecretGetRequest request)
throws Exception {
+ JSONParser jsonParser = new JSONParser();
+ InputStream inputStream =
FileBasedSecretBackend.class.getClassLoader().getResourceAsStream(secretFile);
+
+ try (InputStreamReader reader = new InputStreamReader(inputStream)) {
+ Object obj = jsonParser.parse(reader);
+ JSONArray resourceList = (JSONArray) obj;
+
+ List<GCSSecret> gcsSecrets = (List<GCSSecret>)
resourceList.stream()
+ .filter(resource -> "GCS".equals(((JSONObject)
resource).get("type").toString()))
+ .map(resource -> {
+ JSONObject r = (JSONObject) resource;
+ StringBuilder contentBuilder = new StringBuilder();
+ BufferedReader br = null;
+ String jsonContents = "";
+ try {
+ br = new BufferedReader(new
FileReader(r.get("jsonCredentialsFilePath").toString()));
Review comment:
Do not read from a path. Json should be stored in the secret service.
##########
File path: services/secret-service/stub/src/main/proto/SecretService.proto
##########
@@ -116,6 +116,34 @@ message AzureSecretDeleteRequest {
AuthToken authzToken = 2;
}
+// GCS
+
+message GCSSecret {
+ string secretId = 1;
+ string jsonCredentialsFilePath = 2;
Review comment:
I mentioned this in my previous reviews as well. This should have the
json content not the json path. You have to load the json file content not the
path
##########
File path:
services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/file/FileBasedSecretBackend.java
##########
@@ -211,4 +213,57 @@ public boolean updateAzureSecret(AzureSecretUpdateRequest
request) throws Except
public boolean deleteAzureSecret(AzureSecretDeleteRequest request) throws
Exception {
throw new UnsupportedOperationException("Operation is not supported in
backend");
}
+
+ @Override
+ public Optional<GCSSecret> getGCSSecret(GCSSecretGetRequest request)
throws Exception {
+ JSONParser jsonParser = new JSONParser();
+ InputStream inputStream =
FileBasedSecretBackend.class.getClassLoader().getResourceAsStream(secretFile);
+
+ try (InputStreamReader reader = new InputStreamReader(inputStream)) {
+ Object obj = jsonParser.parse(reader);
+ JSONArray resourceList = (JSONArray) obj;
+
+ List<GCSSecret> gcsSecrets = (List<GCSSecret>)
resourceList.stream()
+ .filter(resource -> "GCS".equals(((JSONObject)
resource).get("type").toString()))
+ .map(resource -> {
+ JSONObject r = (JSONObject) resource;
+ StringBuilder contentBuilder = new StringBuilder();
+ BufferedReader br = null;
+ String jsonContents = "";
+ try {
+ br = new BufferedReader(new
FileReader(r.get("jsonCredentialsFilePath").toString()));
+
+ while ((jsonContents = br.readLine()) != null)
+ {
+
contentBuilder.append(jsonContents).append("\n");
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
Review comment:
Do not print stack traces. These don't go to the log file. Always use
logs
##########
File path: transport/gcp-transport/pom.xml
##########
@@ -0,0 +1,39 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <parent>
+ <artifactId>mft-transport</artifactId>
+ <groupId>org.apache.airavata</groupId>
+ <version>0.01-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>mft-gcp-transport</artifactId>
+ <dependencyManagement>
Review comment:
Do not use dependency management in child modules. We should have a root
pom level dependency management implemented. For now bring this dependency to
<dependencies> section
##########
File path:
services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/file/FileBasedSecretBackend.java
##########
@@ -211,4 +213,57 @@ public boolean updateAzureSecret(AzureSecretUpdateRequest
request) throws Except
public boolean deleteAzureSecret(AzureSecretDeleteRequest request) throws
Exception {
throw new UnsupportedOperationException("Operation is not supported in
backend");
}
+
+ @Override
+ public Optional<GCSSecret> getGCSSecret(GCSSecretGetRequest request)
throws Exception {
+ JSONParser jsonParser = new JSONParser();
+ InputStream inputStream =
FileBasedSecretBackend.class.getClassLoader().getResourceAsStream(secretFile);
+
+ try (InputStreamReader reader = new InputStreamReader(inputStream)) {
+ Object obj = jsonParser.parse(reader);
+ JSONArray resourceList = (JSONArray) obj;
+
+ List<GCSSecret> gcsSecrets = (List<GCSSecret>)
resourceList.stream()
+ .filter(resource -> "GCS".equals(((JSONObject)
resource).get("type").toString()))
+ .map(resource -> {
+ JSONObject r = (JSONObject) resource;
+ StringBuilder contentBuilder = new StringBuilder();
+ BufferedReader br = null;
+ String jsonContents = "";
+ try {
+ br = new BufferedReader(new
FileReader(r.get("jsonCredentialsFilePath").toString()));
+
+ while ((jsonContents = br.readLine()) != null)
+ {
Review comment:
Bring { to upper line
while ((jsonContents = br.readLine()) != null) {
##########
File path:
transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSMetadataCollector.java
##########
@@ -0,0 +1,102 @@
+package org.apache.airavata.mft.transport.gcp;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.storage.Storage;
+import com.google.api.services.storage.StorageScopes;
+import com.google.api.services.storage.model.StorageObject;
+import org.apache.airavata.mft.core.ResourceMetadata;
+import org.apache.airavata.mft.core.api.MetadataCollector;
+import org.apache.airavata.mft.resource.client.ResourceServiceClient;
+import org.apache.airavata.mft.resource.service.GCSResource;
+import org.apache.airavata.mft.resource.service.GCSResourceGetRequest;
+import org.apache.airavata.mft.resource.service.ResourceServiceGrpc;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.service.GCSSecret;
+import org.apache.airavata.mft.secret.service.GCSSecretGetRequest;
+import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
+
+import java.io.ByteArrayInputStream;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Collection;
+
+public class GCSMetadataCollector implements MetadataCollector {
+
+ boolean initialized = false;
+ private String resourceServiceHost;
+ private int resourceServicePort;
+ private String secretServiceHost;
+ private int secretServicePort;
+
+ @Override
+ public void init(String resourceServiceHost, int resourceServicePort,
String secretServiceHost, int secretServicePort) {
+ this.resourceServiceHost = resourceServiceHost;
+ this.resourceServicePort = resourceServicePort;
+ this.secretServiceHost = secretServiceHost;
+ this.secretServicePort = secretServicePort;
+ this.initialized = true;
+ }
+
+ private void checkInitialized() {
+ if (!initialized) {
+ throw new IllegalStateException("GCS Metadata Collector is not
initialized");
+ }
+ }
+
+ @Override
+ public ResourceMetadata getGetResourceMetadata(String resourceId, String
credentialToken) throws Exception {
+ checkInitialized();
+ ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient =
ResourceServiceClient.buildClient(resourceServiceHost, resourceServicePort);
+ GCSResource gcsResource =
resourceClient.getGCSResource(GCSResourceGetRequest.newBuilder().setResourceId(resourceId).build());
+
+ SecretServiceGrpc.SecretServiceBlockingStub secretClient =
SecretServiceClient.buildClient(secretServiceHost, secretServicePort);
+ GCSSecret gcsSecret =
secretClient.getGCSSecret(GCSSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+
+ HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
+ JsonFactory jsonFactory = new JacksonFactory();
+ String jsonString = gcsSecret.getJsonCredentialsFilePath();
Review comment:
I don't understand what you are doing here. Do you get the file path and
feed it as the file content? Is this code working?
##########
File path:
transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSMetadataCollector.java
##########
@@ -0,0 +1,102 @@
+package org.apache.airavata.mft.transport.gcp;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.storage.Storage;
+import com.google.api.services.storage.StorageScopes;
+import com.google.api.services.storage.model.StorageObject;
+import org.apache.airavata.mft.core.ResourceMetadata;
+import org.apache.airavata.mft.core.api.MetadataCollector;
+import org.apache.airavata.mft.resource.client.ResourceServiceClient;
+import org.apache.airavata.mft.resource.service.GCSResource;
+import org.apache.airavata.mft.resource.service.GCSResourceGetRequest;
+import org.apache.airavata.mft.resource.service.ResourceServiceGrpc;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.service.GCSSecret;
+import org.apache.airavata.mft.secret.service.GCSSecretGetRequest;
+import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
+
+import java.io.ByteArrayInputStream;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Collection;
+
+public class GCSMetadataCollector implements MetadataCollector {
+
+ boolean initialized = false;
+ private String resourceServiceHost;
+ private int resourceServicePort;
+ private String secretServiceHost;
+ private int secretServicePort;
+
+ @Override
+ public void init(String resourceServiceHost, int resourceServicePort,
String secretServiceHost, int secretServicePort) {
+ this.resourceServiceHost = resourceServiceHost;
+ this.resourceServicePort = resourceServicePort;
+ this.secretServiceHost = secretServiceHost;
+ this.secretServicePort = secretServicePort;
+ this.initialized = true;
+ }
+
+ private void checkInitialized() {
+ if (!initialized) {
+ throw new IllegalStateException("GCS Metadata Collector is not
initialized");
+ }
+ }
+
+ @Override
+ public ResourceMetadata getGetResourceMetadata(String resourceId, String
credentialToken) throws Exception {
+ checkInitialized();
+ ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient =
ResourceServiceClient.buildClient(resourceServiceHost, resourceServicePort);
+ GCSResource gcsResource =
resourceClient.getGCSResource(GCSResourceGetRequest.newBuilder().setResourceId(resourceId).build());
+
+ SecretServiceGrpc.SecretServiceBlockingStub secretClient =
SecretServiceClient.buildClient(secretServiceHost, secretServicePort);
+ GCSSecret gcsSecret =
secretClient.getGCSSecret(GCSSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+
+ HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
+ JsonFactory jsonFactory = new JacksonFactory();
+ String jsonString = gcsSecret.getJsonCredentialsFilePath();
+ GoogleCredential credential = GoogleCredential.fromStream(new
ByteArrayInputStream(jsonString.getBytes(StandardCharsets.UTF_8)), transport,
jsonFactory);
+ if (credential.createScopedRequired()) {
+ Collection<String> scopes = StorageScopes.all();
+ credential = credential.createScoped(scopes);
+ }
+
+ Storage storage = new Storage.Builder(transport, jsonFactory,
credential).build();
+
+ ResourceMetadata metadata = new ResourceMetadata();
+ StorageObject gcsMetadata =
storage.objects().get(gcsResource.getBucketName(),
gcsResource.getResourcePath()).execute();
+ metadata.setResourceSize(gcsMetadata.getSize().longValue());
+ String md5Sum = String.format("%032x", new BigInteger(1,
Base64.getDecoder().decode(gcsMetadata.getMd5Hash())));
+ metadata.setMd5sum(md5Sum);
+
metadata.setUpdateTime(gcsMetadata.getTimeStorageClassUpdated().getValue());
+ metadata.setCreatedTime(gcsMetadata.getTimeCreated().getValue());
+ return metadata;
+ }
+
+ @Override
+ public Boolean isAvailable(String resourceId, String credentialToken)
throws Exception {
+ checkInitialized();
+ ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient =
ResourceServiceClient.buildClient(resourceServiceHost, resourceServicePort);
+ GCSResource gcsResource =
resourceClient.getGCSResource(GCSResourceGetRequest.newBuilder().setResourceId(resourceId).build());
+
+ SecretServiceGrpc.SecretServiceBlockingStub secretClient =
SecretServiceClient.buildClient(secretServiceHost, secretServicePort);
+ GCSSecret gcsSecret =
secretClient.getGCSSecret(GCSSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+
+ HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
+ JsonFactory jsonFactory = new JacksonFactory();
+ String jsonString = gcsSecret.getJsonCredentialsFilePath();
+ GoogleCredential credential = GoogleCredential.fromStream(new
ByteArrayInputStream(jsonString.getBytes(StandardCharsets.UTF_8)), transport,
jsonFactory);
Review comment:
Same as above comment
##########
File path:
transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSReceiver.java
##########
@@ -0,0 +1,77 @@
+package org.apache.airavata.mft.transport.gcp;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.storage.Storage;
+import com.google.api.services.storage.StorageScopes;
+import org.apache.airavata.mft.core.ConnectorContext;
+import org.apache.airavata.mft.core.api.Connector;
+import org.apache.airavata.mft.resource.client.ResourceServiceClient;
+import org.apache.airavata.mft.resource.service.GCSResource;
+import org.apache.airavata.mft.resource.service.GCSResourceGetRequest;
+import org.apache.airavata.mft.resource.service.ResourceServiceGrpc;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.service.GCSSecret;
+import org.apache.airavata.mft.secret.service.GCSSecretGetRequest;
+import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+
+
+public class GCSReceiver implements Connector {
+
+ private static final Logger logger =
LoggerFactory.getLogger(GCSReceiver.class);
+
+ private GCSResource gcsResource;
+ private Storage storage;
+
+ @Override
+ public void init(String resourceId, String credentialToken, String
resourceServiceHost, int resourceServicePort, String secretServiceHost, int
secretServicePort) throws Exception {
+ ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient =
ResourceServiceClient.buildClient(resourceServiceHost, resourceServicePort);
+ this.gcsResource =
resourceClient.getGCSResource(GCSResourceGetRequest.newBuilder().setResourceId(resourceId).build());
+
+ SecretServiceGrpc.SecretServiceBlockingStub secretClient =
SecretServiceClient.buildClient(secretServiceHost, secretServicePort);
+ GCSSecret gcsSecret =
secretClient.getGCSSecret(GCSSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+ HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
+ JsonFactory jsonFactory = new JacksonFactory();
+ String jsonString = gcsSecret.getJsonCredentialsFilePath();
+ GoogleCredential credential = GoogleCredential.fromStream(new
ByteArrayInputStream(jsonString.getBytes(StandardCharsets.UTF_8)));
Review comment:
Same as above comments
##########
File path:
transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSReceiver.java
##########
@@ -0,0 +1,77 @@
+package org.apache.airavata.mft.transport.gcp;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.storage.Storage;
+import com.google.api.services.storage.StorageScopes;
+import org.apache.airavata.mft.core.ConnectorContext;
+import org.apache.airavata.mft.core.api.Connector;
+import org.apache.airavata.mft.resource.client.ResourceServiceClient;
+import org.apache.airavata.mft.resource.service.GCSResource;
+import org.apache.airavata.mft.resource.service.GCSResourceGetRequest;
+import org.apache.airavata.mft.resource.service.ResourceServiceGrpc;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.service.GCSSecret;
+import org.apache.airavata.mft.secret.service.GCSSecretGetRequest;
+import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+
+
+public class GCSReceiver implements Connector {
+
+ private static final Logger logger =
LoggerFactory.getLogger(GCSReceiver.class);
+
+ private GCSResource gcsResource;
+ private Storage storage;
+
+ @Override
+ public void init(String resourceId, String credentialToken, String
resourceServiceHost, int resourceServicePort, String secretServiceHost, int
secretServicePort) throws Exception {
+ ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient =
ResourceServiceClient.buildClient(resourceServiceHost, resourceServicePort);
+ this.gcsResource =
resourceClient.getGCSResource(GCSResourceGetRequest.newBuilder().setResourceId(resourceId).build());
+
+ SecretServiceGrpc.SecretServiceBlockingStub secretClient =
SecretServiceClient.buildClient(secretServiceHost, secretServicePort);
+ GCSSecret gcsSecret =
secretClient.getGCSSecret(GCSSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+ HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
+ JsonFactory jsonFactory = new JacksonFactory();
+ String jsonString = gcsSecret.getJsonCredentialsFilePath();
+ GoogleCredential credential = GoogleCredential.fromStream(new
ByteArrayInputStream(jsonString.getBytes(StandardCharsets.UTF_8)));
+ if (credential.createScopedRequired()) {
+ Collection<String> scopes = StorageScopes.all();
+ credential = credential.createScoped(scopes);
+ }
+ storage = new Storage.Builder(transport, jsonFactory,
credential).build();
+ }
+
+ @Override
+ public void destroy() {
+
+ }
+
+ @Override
+ public void startStream(ConnectorContext context) throws Exception {
+ logger.info("Starting GCS Receiver stream for transfer {}",
context.getTransferId());
+
+ InputStream inputStream =
storage.objects().get(this.gcsResource.getBucketName(),
this.gcsResource.getResourcePath()).executeMediaAsInputStream();
+ OutputStream os = context.getStreamBuffer().getOutputStream();
+ int read;
+ long bytes = 0;
Review comment:
Instead of reading byte by byte, use read(byte[]) method of input
stream. This will improve the performance. Have a look at this example
https://github.com/apache/airavata-mft/blob/master/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureReceiver.java#L85
##########
File path:
transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSSender.java
##########
@@ -0,0 +1,86 @@
+package org.apache.airavata.mft.transport.gcp;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.InputStreamContent;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.storage.Storage;
+import com.google.api.services.storage.Storage.Objects.Insert;
+import com.google.api.services.storage.StorageScopes;
+import com.google.api.services.storage.model.ObjectAccessControl;
+import com.google.api.services.storage.model.StorageObject;
+import org.apache.airavata.mft.core.ConnectorContext;
+import org.apache.airavata.mft.core.api.Connector;
+import org.apache.airavata.mft.resource.client.ResourceServiceClient;
+import org.apache.airavata.mft.resource.service.GCSResource;
+import org.apache.airavata.mft.resource.service.GCSResourceGetRequest;
+import org.apache.airavata.mft.resource.service.ResourceServiceGrpc;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.service.GCSSecret;
+import org.apache.airavata.mft.secret.service.GCSSecretGetRequest;
+import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collection;
+
+
+public class GCSSender implements Connector {
+
+ private static final Logger logger =
LoggerFactory.getLogger(GCSSender.class);
+
+ private GCSResource gcsResource;
+ private Storage storage;
+
+ @Override
+ public void init(String resourceId, String credentialToken, String
resourceServiceHost, int resourceServicePort, String secretServiceHost, int
secretServicePort) throws Exception {
+
+ ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient =
ResourceServiceClient.buildClient(resourceServiceHost, resourceServicePort);
+ this.gcsResource =
resourceClient.getGCSResource(GCSResourceGetRequest.newBuilder().setResourceId(resourceId).build());
+
+ SecretServiceGrpc.SecretServiceBlockingStub secretClient =
SecretServiceClient.buildClient(secretServiceHost, secretServicePort);
+ GCSSecret gcsSecret =
secretClient.getGCSSecret(GCSSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+
+ HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
+ JsonFactory jsonFactory = new JacksonFactory();
+ String jsonString = gcsSecret.getJsonCredentialsFilePath();
+ GoogleCredential credential = GoogleCredential.fromStream(new
ByteArrayInputStream(jsonString.getBytes(StandardCharsets.UTF_8)));
+ if (credential.createScopedRequired()) {
+ Collection<String> scopes = StorageScopes.all();
+ credential = credential.createScoped(scopes);
+ }
+
+ storage = new Storage.Builder(transport, jsonFactory,
credential).build();
+ }
+
+ @Override
+ public void destroy() {
+
+ }
+
+ @Override
+ public void startStream(ConnectorContext context) throws Exception {
+ logger.info("Starting GCS Sender stream for transfer {}",
context.getTransferId());
+ logger.info("Content length for transfer {} {}",
context.getTransferId(), context.getMetadata().getResourceSize());
+
+ InputStreamContent contentStream = new InputStreamContent(
+ "text/plain", context.getStreamBuffer().getInputStream());
Review comment:
Is the content always "text/plain"?
##########
File path:
transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSSender.java
##########
@@ -0,0 +1,86 @@
+package org.apache.airavata.mft.transport.gcp;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.InputStreamContent;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.storage.Storage;
+import com.google.api.services.storage.Storage.Objects.Insert;
+import com.google.api.services.storage.StorageScopes;
+import com.google.api.services.storage.model.ObjectAccessControl;
+import com.google.api.services.storage.model.StorageObject;
+import org.apache.airavata.mft.core.ConnectorContext;
+import org.apache.airavata.mft.core.api.Connector;
+import org.apache.airavata.mft.resource.client.ResourceServiceClient;
+import org.apache.airavata.mft.resource.service.GCSResource;
+import org.apache.airavata.mft.resource.service.GCSResourceGetRequest;
+import org.apache.airavata.mft.resource.service.ResourceServiceGrpc;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.service.GCSSecret;
+import org.apache.airavata.mft.secret.service.GCSSecretGetRequest;
+import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collection;
+
+
+public class GCSSender implements Connector {
+
+ private static final Logger logger =
LoggerFactory.getLogger(GCSSender.class);
+
+ private GCSResource gcsResource;
+ private Storage storage;
+
+ @Override
+ public void init(String resourceId, String credentialToken, String
resourceServiceHost, int resourceServicePort, String secretServiceHost, int
secretServicePort) throws Exception {
+
+ ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient =
ResourceServiceClient.buildClient(resourceServiceHost, resourceServicePort);
+ this.gcsResource =
resourceClient.getGCSResource(GCSResourceGetRequest.newBuilder().setResourceId(resourceId).build());
+
+ SecretServiceGrpc.SecretServiceBlockingStub secretClient =
SecretServiceClient.buildClient(secretServiceHost, secretServicePort);
+ GCSSecret gcsSecret =
secretClient.getGCSSecret(GCSSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+
+ HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
+ JsonFactory jsonFactory = new JacksonFactory();
+ String jsonString = gcsSecret.getJsonCredentialsFilePath();
+ GoogleCredential credential = GoogleCredential.fromStream(new
ByteArrayInputStream(jsonString.getBytes(StandardCharsets.UTF_8)));
+ if (credential.createScopedRequired()) {
+ Collection<String> scopes = StorageScopes.all();
+ credential = credential.createScoped(scopes);
+ }
+
+ storage = new Storage.Builder(transport, jsonFactory,
credential).build();
+ }
+
+ @Override
+ public void destroy() {
+
+ }
+
+ @Override
+ public void startStream(ConnectorContext context) throws Exception {
+ logger.info("Starting GCS Sender stream for transfer {}",
context.getTransferId());
+ logger.info("Content length for transfer {} {}",
context.getTransferId(), context.getMetadata().getResourceSize());
+
+ InputStreamContent contentStream = new InputStreamContent(
+ "text/plain", context.getStreamBuffer().getInputStream());
+ StorageObject objectMetadata = new StorageObject()
+ // Set the destination object name
+ .setName(this.gcsResource.getResourcePath())
+ // Set the access control list to publicly read-only
+ .setAcl(Arrays.asList(new
ObjectAccessControl().setEntity("allUsers").setRole("READER")));
Review comment:
What is this allUsers and READER property? Is this making the resource
publicly available?
##########
File path:
transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSSender.java
##########
@@ -0,0 +1,86 @@
+package org.apache.airavata.mft.transport.gcp;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.InputStreamContent;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.storage.Storage;
+import com.google.api.services.storage.Storage.Objects.Insert;
+import com.google.api.services.storage.StorageScopes;
+import com.google.api.services.storage.model.ObjectAccessControl;
+import com.google.api.services.storage.model.StorageObject;
+import org.apache.airavata.mft.core.ConnectorContext;
+import org.apache.airavata.mft.core.api.Connector;
+import org.apache.airavata.mft.resource.client.ResourceServiceClient;
+import org.apache.airavata.mft.resource.service.GCSResource;
+import org.apache.airavata.mft.resource.service.GCSResourceGetRequest;
+import org.apache.airavata.mft.resource.service.ResourceServiceGrpc;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.service.GCSSecret;
+import org.apache.airavata.mft.secret.service.GCSSecretGetRequest;
+import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collection;
+
+
+public class GCSSender implements Connector {
+
+ private static final Logger logger =
LoggerFactory.getLogger(GCSSender.class);
+
+ private GCSResource gcsResource;
+ private Storage storage;
+
+ @Override
+ public void init(String resourceId, String credentialToken, String
resourceServiceHost, int resourceServicePort, String secretServiceHost, int
secretServicePort) throws Exception {
+
+ ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient =
ResourceServiceClient.buildClient(resourceServiceHost, resourceServicePort);
+ this.gcsResource =
resourceClient.getGCSResource(GCSResourceGetRequest.newBuilder().setResourceId(resourceId).build());
+
+ SecretServiceGrpc.SecretServiceBlockingStub secretClient =
SecretServiceClient.buildClient(secretServiceHost, secretServicePort);
+ GCSSecret gcsSecret =
secretClient.getGCSSecret(GCSSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+
+ HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
+ JsonFactory jsonFactory = new JacksonFactory();
+ String jsonString = gcsSecret.getJsonCredentialsFilePath();
Review comment:
Same as above comments
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
