This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
The following commit(s) were added to refs/heads/master by this push:
new f764874 Add transfermappings
new ec16722 Merge pull request #15 from isururanawaka/workflow_merge
f764874 is described below
commit f7648743014a75190d3327443c2175c1b8cb28c1
Author: Isuru Ranawaka <[email protected]>
AuthorDate: Tue Jul 6 17:36:23 2021 -0400
Add transfermappings
---
.../java/org/apache/airavata/drms/api/Client.java | 40 ++-
.../drms/api/handlers/ResourceServiceHandler.java | 25 +-
.../handlers/StoragePreferenceServiceHandler.java | 320 ++++++++++++++++-----
.../src/main/resources/application.properties | 3 +-
.../AnyStoragePreferenceDeserializer.java | 2 +-
.../deserializer/TransferMappingDeserializer.java | 57 ++++
.../preference/StoragePreferenceService.proto | 133 ++++++---
7 files changed, 458 insertions(+), 122 deletions(-)
diff --git
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Client.java
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Client.java
index 24f8c34..e0e9428 100644
---
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Client.java
+++
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Client.java
@@ -20,11 +20,10 @@ import com.google.protobuf.Struct;
import com.google.protobuf.util.JsonFormat;
import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
-import org.apache.airavata.datalake.drms.AuthCredentialType;
-import org.apache.airavata.datalake.drms.AuthenticatedUser;
import org.apache.airavata.datalake.drms.DRMSServiceAuthToken;
import org.apache.airavata.datalake.drms.resource.GenericResource;
import org.apache.airavata.datalake.drms.storage.*;
+import
org.apache.airavata.datalake.drms.storage.preference.ssh.SSHStoragePreference;
import org.apache.custos.clients.CustosClientProvider;
import org.apache.custos.identity.management.client.IdentityManagementClient;
import org.json.JSONObject;
@@ -114,7 +113,34 @@ public class Client {
//
//
storagePreferenceServiceBlockingStub.createStoragePreference(storagePreferenceCreateRequest);
+ TransferMapping transferMapping = TransferMapping.newBuilder()
+ .setUserId("[email protected]")
+ .setTransferScope(TransferScope.GLOBAL)
+
.setDestinationStoragePreference(AnyStoragePreference.newBuilder()
+
.setSshStoragePreference(SSHStoragePreference.newBuilder()
+
.setStoragePreferenceId("ssh_storage_preference").build()))
+ .setSourceStoragePreference(AnyStoragePreference.newBuilder()
+
.setSshStoragePreference(SSHStoragePreference.newBuilder()
+
.setStoragePreferenceId("ssh_storage_preference_2").build()))
+ .build();
+
+ CreateTransferMappingRequest request =
CreateTransferMappingRequest.newBuilder()
+ .setAuthToken(authToken)
+ .setTransferMapping(transferMapping)
+ .build();
+ FindTransferMappingsRequest findTransferMappingsRequest =
FindTransferMappingsRequest.newBuilder()
+ .setAuthToken(authToken)
+ .build();
+
+ DeleteTransferMappingRequest transferMappingRequest =
DeleteTransferMappingRequest.newBuilder()
+ .setAuthToken(authToken)
+ .setId("ssh_storage_preference_2_ssh_storage_preference")
+ .build();
+
+//
storagePreferenceServiceBlockingStub.deleteTransferMappings(transferMappingRequest);
+
+ storagePreferenceServiceBlockingStub.createTransferMapping(request);
ResourceServiceGrpc.ResourceServiceBlockingStub
resourceServiceBlockingStub = ResourceServiceGrpc.newBlockingStub(channel);
// ResourceSearchQuery query =
ResourceSearchQuery.newBuilder().setField("type").setValue("COLLECTION").build();
@@ -153,11 +179,11 @@ public class Client {
// .setTenantId("custos-whedmgamitu357p4wuke-10002708"))
//
.setAuthCredentialType(AuthCredentialType.AGENT_ACCOUNT_CREDENTIAL)
// .build();
- ResourceCreateRequest resourceCreateRequest =
ResourceCreateRequest.newBuilder()
- .setAuthToken(authToken)
- .setResource(genericResource)
- .build();
- resourceServiceBlockingStub.createResource(resourceCreateRequest);
+// ResourceCreateRequest resourceCreateRequest =
ResourceCreateRequest.newBuilder()
+// .setAuthToken(authToken)
+// .setResource(genericResource)
+// .build();
+// resourceServiceBlockingStub.createResource(resourceCreateRequest);
// System.out.println(authToken.getAccessToken());
diff --git
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
index ad345de..750b885 100644
---
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
+++
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
@@ -140,7 +140,7 @@ public class ResourceServiceHandler extends
ResourceServiceGrpc.ResourceServiceI
if (exEntity.isPresent()) {
serializedMap.put("description",
exEntity.get().getDescription());
- serializedMap.put("name", exEntity.get().getName());
+ serializedMap.put("resourceName", exEntity.get().getName());
serializedMap.put("createdTime",
String.valueOf(exEntity.get().getCreatedAt()));
serializedMap.put("tenantId", callUser.getTenantId());
serializedMap.put("entityId", exEntity.get().getId());
@@ -148,15 +148,12 @@ public class ResourceServiceHandler extends
ResourceServiceGrpc.ResourceServiceI
serializedMap.put("lastModifiedTime",
exEntity.get().getCreatedAt());
serializedMap.put("owner", exEntity.get().getOwnerId());
-
- HashMap<String, Object> hashMap = new HashMap<>();
-
if (!storagePreferenceId.isEmpty()) {
this.neo4JConnector.mergeNodesWithParentChildRelationShip(serializedMap, new
HashMap<>(),
request.getResource().getType(),
StoragePreferenceConstants.STORAGE_PREFERENCE_LABEL,
callUser.getUsername(), entityId,
storagePreferenceId, callUser.getTenantId());
} else {
- this.neo4JConnector.mergeNode(hashMap,
request.getResource().getType(),
+ this.neo4JConnector.mergeNode(serializedMap,
request.getResource().getType(),
callUser.getUsername(), entityId,
callUser.getTenantId());
}
} else {
@@ -181,9 +178,17 @@ public class ResourceServiceHandler extends
ResourceServiceGrpc.ResourceServiceI
List<Record> records = this.neo4JConnector.searchNodes(exProps,
query);
List<GenericResource> genericResourceList =
GenericResourceDeserializer.deserializeList(records);
+ GenericResource genericResource = genericResourceList.get(0);
+ if (genericResource.getPropertiesMap().containsKey("name")) {
+ genericResource = genericResource.toBuilder()
+
.setResourceName(genericResource.getPropertiesMap().get("name")).build();
+ } else if
(genericResource.getPropertiesMap().containsKey("resourceName")) {
+ genericResource = genericResource.toBuilder()
+
.setResourceName(genericResource.getPropertiesMap().get("resourceName")).build();
+ }
ResourceCreateResponse response = ResourceCreateResponse
.newBuilder()
- .setResource(genericResourceList.get(0))
+ .setResource(genericResource)
.build();
responseObserver.onNext(response);
responseObserver.onCompleted();
@@ -457,9 +462,9 @@ public class ResourceServiceHandler extends
ResourceServiceGrpc.ResourceServiceI
userProps.put("tenantId", callUser.getTenantId());
userProps.put("entityId", resource.getResourceId());
userProps.put("childEntityId", childResource.getResourceId());
- String query = "MATCH (r:" + resource.getType() + "), (cr:" +
childResource.getType() + ") where " +
+ String query = "MATCH (cr:" + childResource.getType() +
")-[crel:CHILD_OF]->(r:" + resource.getType() + ") where " +
" r.entityId = $entityId AND r.tenantId = $tenantId
AND cr.entityId = $childEntityId AND cr.tenantId = $tenantId " +
- " MATCH (cr)-[crel:CHILD_OF]->(r) delete crel";
+ "delete crel";
this.neo4JConnector.runTransactionalQuery(userProps, query);
});
@@ -514,6 +519,10 @@ public class ResourceServiceHandler extends
ResourceServiceGrpc.ResourceServiceI
builder.putAllProperties(genericResourceMap);
responseObserver.onNext(builder.build());
responseObserver.onCompleted();
+ } else {
+ ParentResourcesFetchResponse.Builder builder =
ParentResourcesFetchResponse.newBuilder();
+ responseObserver.onNext(builder.build());
+ responseObserver.onCompleted();
}
} else {
String msg = " Don't have access to change memberships";
diff --git
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java
index 4548745..ef07212 100644
---
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java
+++
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java
@@ -17,6 +17,7 @@
package org.apache.airavata.drms.api.handlers;
import com.google.protobuf.Empty;
+import io.grpc.Status;
import io.grpc.stub.StreamObserver;
import org.apache.airavata.datalake.drms.AuthenticatedUser;
import org.apache.airavata.datalake.drms.storage.*;
@@ -25,6 +26,7 @@ import org.apache.airavata.drms.core.Neo4JConnector;
import org.apache.airavata.drms.core.constants.StorageConstants;
import org.apache.airavata.drms.core.constants.StoragePreferenceConstants;
import
org.apache.airavata.drms.core.deserializer.AnyStoragePreferenceDeserializer;
+import org.apache.airavata.drms.core.deserializer.TransferMappingDeserializer;
import org.apache.airavata.drms.core.serializer.AnyStoragePreferenceSerializer;
import org.apache.airavata.drms.core.serializer.AnyStorageSerializer;
import org.apache.custos.clients.CustosClientProvider;
@@ -57,48 +59,54 @@ public class StoragePreferenceServiceHandler extends
StoragePreferenceServiceGrp
@Override
public void fetchStoragePreference(StoragePreferenceFetchRequest request,
StreamObserver<StoragePreferenceFetchResponse> responseObserver) {
- AuthenticatedUser callUser =
request.getAuthToken().getAuthenticatedUser();
+ try {
+ AuthenticatedUser callUser =
request.getAuthToken().getAuthenticatedUser();
- Map<String, Object> userProps = new HashMap<>();
- userProps.put("username", callUser.getUsername());
- userProps.put("tenantId", callUser.getTenantId());
- userProps.put("storagePreferenceId", request.getStoragePreferenceId());
+ Map<String, Object> userProps = new HashMap<>();
+ userProps.put("username", callUser.getUsername());
+ userProps.put("tenantId", callUser.getTenantId());
+ userProps.put("storagePreferenceId",
request.getStoragePreferenceId());
- List<Record> records = this.neo4JConnector.searchNodes(userProps,
- " MATCH (u:User) where u.username = $username AND u.tenantId =
$tenantId" +
- " OPTIONAL MATCH
(u)<-[:SHARED_WITH]-(s1:Storage)<-[:CHILD_OF]->(sp1:StoragePreference) where
sp1.storagePreferenceId = $storagePreferenceId" +
- " OPTIONAL MATCH
(cg:Group)-[:CHILD_OF*]->(g:Group)<-[:MEMBER_OF]-(u)" +
- " OPTIONAL MATCH
(sp2:StoragePreference)-[:CHILD_OF]->(s2:Storage)-[:SHARED_WITH]->(cg) where
sp2.storagePreferenceId = $storagePreferenceId" +
- " OPTIONAL MATCH
(sp3:StoragePreference)-[:CHILD_OF]->(s3:Storage)-[:SHARED_WITH]->(g) where
sp3.storagePreferenceId = $storagePreferenceId" +
- " OPTIONAL MATCH
(s4:Storage)<-[:CHILD_OF]->(sp4:StoragePreference)-[:SHARED_WITH]->(u) where
sp4.storagePreferenceId = $storagePreferenceId" +
- " OPTIONAL MATCH
(s5:Storage)<-[:CHILD_OF]->(sp5:StoragePreference)-[:SHARED_WITH]->(cg) where
sp5.storagePreferenceId = $storagePreferenceId" +
- " OPTIONAL MATCH
(s6:Storage)<-[:CHILD_OF]->(sp6:StoragePreference)-[:SHARED_WITH]->(g) where
sp6.storagePreferenceId = $storagePreferenceId" +
- " return distinct s1, sp1, s2, sp2, s3, sp3, s4,sp4,
s5,sp5, s6,sp6");
+ List<Record> records = this.neo4JConnector.searchNodes(userProps,
+ " MATCH (u:User) where u.username = $username AND
u.tenantId = $tenantId" +
+ " OPTIONAL MATCH
(u)<-[:SHARED_WITH]-(s1:Storage)<-[:CHILD_OF]->(sp1:StoragePreference) where
sp1.storagePreferenceId = $storagePreferenceId" +
+ " OPTIONAL MATCH
(cg:Group)-[:CHILD_OF*]->(g:Group)<-[:MEMBER_OF]-(u)" +
+ " OPTIONAL MATCH
(sp2:StoragePreference)-[:CHILD_OF]->(s2:Storage)-[:SHARED_WITH]->(cg) where
sp2.storagePreferenceId = $storagePreferenceId" +
+ " OPTIONAL MATCH
(sp3:StoragePreference)-[:CHILD_OF]->(s3:Storage)-[:SHARED_WITH]->(g) where
sp3.storagePreferenceId = $storagePreferenceId" +
+ " OPTIONAL MATCH
(s4:Storage)<-[:CHILD_OF]->(sp4:StoragePreference)-[:SHARED_WITH]->(u) where
sp4.storagePreferenceId = $storagePreferenceId" +
+ " OPTIONAL MATCH
(s5:Storage)<-[:CHILD_OF]->(sp5:StoragePreference)-[:SHARED_WITH]->(cg) where
sp5.storagePreferenceId = $storagePreferenceId" +
+ " OPTIONAL MATCH
(s6:Storage)<-[:CHILD_OF]->(sp6:StoragePreference)-[:SHARED_WITH]->(g) where
sp6.storagePreferenceId = $storagePreferenceId" +
+ " return distinct s1, sp1, s2, sp2, s3, sp3,
s4,sp4, s5,sp5, s6,sp6");
- if (!records.isEmpty()) {
- try {
- List keyList = new ArrayList();
- keyList.add("s1:sp1");
- keyList.add("s2:sp2");
- keyList.add("s3:sp3");
- keyList.add("s4:sp4");
- keyList.add("s5:sp5");
- keyList.add("s6:sp6");
- List<AnyStoragePreference> storagePrefList =
AnyStoragePreferenceDeserializer.deserializeList(records, keyList);
- StoragePreferenceFetchResponse.Builder builder =
StoragePreferenceFetchResponse.newBuilder();
- if (!storagePrefList.isEmpty()) {
- builder.setStoragePreference(storagePrefList.get(0));
+ if (!records.isEmpty()) {
+ try {
+ List keyList = new ArrayList();
+ keyList.add("s1:sp1");
+ keyList.add("s2:sp2");
+ keyList.add("s3:sp3");
+ keyList.add("s4:sp4");
+ keyList.add("s5:sp5");
+ keyList.add("s6:sp6");
+ List<AnyStoragePreference> storagePrefList =
AnyStoragePreferenceDeserializer.deserializeList(records, keyList);
+ StoragePreferenceFetchResponse.Builder builder =
StoragePreferenceFetchResponse.newBuilder();
+ if (!storagePrefList.isEmpty()) {
+ builder.setStoragePreference(storagePrefList.get(0));
+ }
+ responseObserver.onNext(builder.build());
+ responseObserver.onCompleted();
+ } catch (Exception e) {
+ String msg = "Errored while searching storage preferences;
Message:" + e.getMessage();
+ logger.error("Errored while searching storage preferences;
Message: {}", e.getMessage(), e);
+
responseObserver.onError(io.grpc.Status.INTERNAL.withDescription(msg).asRuntimeException());
}
- responseObserver.onNext(builder.build());
- responseObserver.onCompleted();
- } catch (Exception e) {
- String msg = "Errored while searching storage preferences;
Message:" + e.getMessage();
- logger.error("Errored while searching storage preferences;
Message: {}", e.getMessage(), e);
+ } else {
+ String msg = "Could not find a storage preference with id" +
request.getStoragePreferenceId();
+ logger.error("Could not find a storage preference with id {}",
request.getStoragePreferenceId());
responseObserver.onError(io.grpc.Status.INTERNAL.withDescription(msg).asRuntimeException());
}
- } else {
- String msg = "Could not find a storage preference with id" +
request.getStoragePreferenceId();
- logger.error("Could not find a storage preference with id {}",
request.getStoragePreferenceId());
+ } catch (Exception e) {
+ String msg = "Errored while searching storage preferences;
Message:" + e.getMessage();
+ logger.error("Errored while searching storage preferences;
Message: {}", e.getMessage(), e);
responseObserver.onError(io.grpc.Status.INTERNAL.withDescription(msg).asRuntimeException());
}
}
@@ -206,46 +214,224 @@ public class StoragePreferenceServiceHandler extends
StoragePreferenceServiceGrp
@Override
public void searchStoragePreference(StoragePreferenceSearchRequest
request, StreamObserver<StoragePreferenceSearchResponse> responseObserver) {
- AuthenticatedUser callUser =
request.getAuthToken().getAuthenticatedUser();
+ try {
+ AuthenticatedUser callUser =
request.getAuthToken().getAuthenticatedUser();
+
+ Map<String, Object> userProps = new HashMap<>();
+ userProps.put("username", callUser.getUsername());
+ userProps.put("tenantId", callUser.getTenantId());
+
+ List<Record> records = this.neo4JConnector.searchNodes(userProps,
+ " MATCH (u:User) where u.username = $username AND
u.tenantId = $tenantId" +
+ " OPTIONAL MATCH
(u)<-[:SHARED_WITH]-(s1:Storage)<-[:CHILD_OF]->(sp1:StoragePreference)" +
+ " OPTIONAL MATCH
(cg:Group)-[:CHILD_OF*]->(g:Group)<-[:MEMBER_OF]-(u)" +
+ " OPTIONAL MATCH
(sp2:StoragePreference)-[:CHILD_OF]->(s2:Storage)-[:SHARED_WITH]->(cg) " +
+ " OPTIONAL MATCH
(sp3:StoragePreference)-[:CHILD_OF]->(s3:Storage)-[:SHARED_WITH]->(g)" +
+ " OPTIONAL MATCH
(s4:Storage)<-[:CHILD_OF]->(sp4:StoragePreference)-[:SHARED_WITH]->(u)" +
+ " OPTIONAL MATCH
(s5:Storage)<-[:CHILD_OF]->(sp5:StoragePreference)-[:SHARED_WITH]->(cg)" +
+ " OPTIONAL MATCH
(s6:Storage)<-[:CHILD_OF]->(sp6:StoragePreference)-[:SHARED_WITH]->(g)" +
+ " return distinct s1, sp1, s2, sp2, s3, sp3,
s4,sp4, s5,sp5, s6,sp6");
+ if (!records.isEmpty()) {
+ try {
+ List keyList = new ArrayList();
+ keyList.add("s1:sp1");
+ keyList.add("s2:sp2");
+ keyList.add("s3:sp3");
+ keyList.add("s4:sp4");
+ keyList.add("s5:sp5");
+ keyList.add("s6:sp6");
+ List<AnyStoragePreference> storagePrefList =
AnyStoragePreferenceDeserializer.deserializeList(records, keyList);
+ StoragePreferenceSearchResponse.Builder builder =
StoragePreferenceSearchResponse.newBuilder();
+ builder.addAllStoragesPreference(storagePrefList);
+ responseObserver.onNext(builder.build());
+ responseObserver.onCompleted();
+ } catch (Exception e) {
+ String msg = "Errored while searching storage preferences;
Message:" + e.getMessage();
+ logger.error("Errored while searching storage preferences;
Message: {}", e.getMessage(), e);
+
responseObserver.onError(io.grpc.Status.INTERNAL.withDescription(msg).asRuntimeException());
+ }
+ } else {
+
responseObserver.onNext(StoragePreferenceSearchResponse.newBuilder().build());
+ responseObserver.onCompleted();
+ }
+ } catch (Exception e) {
+ String msg = "Errored while searching storage preferences;
Message:" + e.getMessage();
+ logger.error("Errored while searching storage preferences;
Message: {}", e.getMessage(), e);
+
responseObserver.onError(io.grpc.Status.INTERNAL.withDescription(msg).asRuntimeException());
+ }
+ }
+
+ @Override
+ public void createTransferMapping(CreateTransferMappingRequest request,
StreamObserver<CreateTransferMappingResponse> responseObserver) {
+ try {
+ AuthenticatedUser authenticatedUser =
request.getAuthToken().getAuthenticatedUser();
+ AnyStoragePreference sourceStoragePreference =
request.getTransferMapping().getSourceStoragePreference();
+ AnyStoragePreference destinationStoragePreference =
request.getTransferMapping().getDestinationStoragePreference();
+ String sourceId = getStorageId(sourceStoragePreference);
+ String destinationId = getStorageId(destinationStoragePreference);
+
+ TransferScope scope =
request.getTransferMapping().getTransferScope();
+ Map<String, Object> properties = new HashMap<>();
+ Map<String, Object> props = new HashMap<>();
+ props.put("tenantId", authenticatedUser.getTenantId());
+ props.put("owner", authenticatedUser.getUsername());
+ props.put("srcStoragePreferenceId", sourceId);
+ props.put("dstStoragePreferenceId", destinationId);
+ String entityId = sourceId + "_" + destinationId;
+ if (scope.equals(TransferScope.GLOBAL)) {
+ props.put("scope", TransferScope.GLOBAL.name());
+ } else {
+ props.put("scope", TransferScope.USER.name());
+ }
+ properties.put("props", props);
+ properties.put("tenantId", authenticatedUser.getTenantId());
+ properties.put("entityId", entityId);
+ properties.put("username", authenticatedUser.getUsername());
+ properties.put("srcStoragePreferenceId", sourceId);
+ properties.put("dstStoragePreferenceId", destinationId);
+ properties.put("owner", authenticatedUser.getUsername());
+
+
+ if (hasAccess(authenticatedUser.getUsername(),
authenticatedUser.getTenantId(), sourceId) &&
+ hasAccess(authenticatedUser.getUsername(),
authenticatedUser.getTenantId(), destinationId)) {
+ String query = " Match (u:User), (srcSp:StoragePreference),
(dstSp:StoragePreference) where " +
+ " u.username=$username AND u.tenantId=$tenantId AND " +
+ "srcSp.storagePreferenceId=$srcStoragePreferenceId AND
" +
+ "srcSp.tenantId = $tenantId AND
dstSp.storagePreferenceId=$dstStoragePreferenceId " +
+ "AND dstSp.tenantId =$tenantId " +
+ " Merge
(u)-[:HAS_TRANSFER_MAPPING]->(tm:TransferMapping{entityId:$entityId,
tenantId:$tenantId, " +
+ "srcStoragePreferenceId:$srcStoragePreferenceId," +
+
"dstStoragePreferenceId:$dstStoragePreferenceId,owner:$owner}) set tm +=
$props" +
+ " Merge (tm)<-[:TRANSFER_OUT]-(srcSp)" +
+ " Merge (tm)-[:TRANSFER_IN]->(dstSp) return (tm)";
+ this.neo4JConnector.runTransactionalQuery(properties, query);
+
+ String searchQuery = " Match (srcStr:Storage)<-[:CHILD_OF]-" +
+
"(srcSp:StoragePreference)-[:TRANSFER_OUT]->(tm:TransferMapping)" +
+
"-[:TRANSFER_IN]->(dstSp:StoragePreference)-[:CHILD_OF]->(dstStr:Storage)
where " +
+ " tm.entityId=$entityId AND tm.tenantId=$tenantId
return srcStr, srcSp, dstStr, dstSp, tm";
+ List<Record> records =
this.neo4JConnector.searchNodes(properties, searchQuery);
+ if (!records.isEmpty()) {
+ List<TransferMapping> transferMappings =
TransferMappingDeserializer.deserializeList(records);
+ if (!transferMappings.isEmpty()) {
+ CreateTransferMappingResponse response =
CreateTransferMappingResponse
+ .newBuilder()
+ .setTransferMapping(transferMappings.get(0))
+ .build();
+ responseObserver.onNext(response);
+ responseObserver.onCompleted();
+ } else {
+ String msg = "Errored while creating transfer mapping;
Message:";
+ logger.error("Errored while creating transfer mapping;
Message:");
+
responseObserver.onError(io.grpc.Status.INTERNAL.withDescription(msg).asRuntimeException());
+ }
+ } else {
+ String msg = "Errored while creating transfer mapping;
Message:";
+ logger.error("Errored while creating transfer mapping;
Message:");
+
responseObserver.onError(io.grpc.Status.INTERNAL.withDescription(msg).asRuntimeException());
+ }
+ } else {
+ String msg = "User does not have permission to create mapping
";
+ logger.error("User does not have permission to create mapping
");
+
responseObserver.onError(Status.PERMISSION_DENIED.withDescription(msg).asRuntimeException());
+
+ }
+ } catch (Exception e) {
+ String msg = "Errored while creating transfer mapping; Message:" +
e.getMessage();
+ logger.error("Errored while creating transfer mapping; Message:
{}", e.getMessage(), e);
+
responseObserver.onError(io.grpc.Status.INTERNAL.withDescription(msg).asRuntimeException());
+ }
+ }
+
+ @Override
+ public void getTransferMappings(FindTransferMappingsRequest request,
StreamObserver<FindTransferMappingsResponse> responseObserver) {
+ try {
+ List<TransferMapping> transferMappings = new ArrayList<>();
+ AuthenticatedUser authenticatedUser =
request.getAuthToken().getAuthenticatedUser();
+ Map<String, Object> properties = new HashMap<>();
+ properties.put("username", authenticatedUser.getUsername());
+ properties.put("tenantId", authenticatedUser.getTenantId());
+ properties.put("scope", TransferScope.USER.name());
+ String query = " MATCH
(u:User)-[:HAS_TRANSFER_MAPPING]->(t:TransferMapping{scope:$scope}) where
u.username = $username AND u.tenantId = $tenantId" +
+ " Match
(srcStr:Storage)<-[:CHILD_OF]-(srcSp:StoragePreference)-[:TRANSFER_OUT]->(t)-[:TRANSFER_IN]->(dstSp:StoragePreference)-[:CHILD_OF]->(dstStr:Storage)"
+
+ " return srcStr, srcSp, dstStr, dstSp, t";
+ List<Record> records = this.neo4JConnector.searchNodes(properties,
query);
+ properties.put("scope", TransferScope.GLOBAL.name());
+ String queryFetchGlobal = "Match (srcStr:Storage)<-[:CHILD_OF]-" +
+
"(srcSp:StoragePreference)-[:TRANSFER_OUT]->(t:TransferMapping{scope:$scope,
tenantId:$tenantId})-[:TRANSFER_IN]->(dstSp:StoragePreference)-[:CHILD_OF]->(dstStr:Storage)"
+
+ " return srcStr, srcSp, dstStr, dstSp, t";
+ List<Record> globalRecords =
this.neo4JConnector.searchNodes(properties, queryFetchGlobal);
+ if (!records.isEmpty()) {
+ transferMappings =
TransferMappingDeserializer.deserializeList(records);
+ }
+ if (!globalRecords.isEmpty()) {
+
transferMappings.addAll(TransferMappingDeserializer.deserializeList(globalRecords));
+ }
+ FindTransferMappingsResponse findTransferMappingsResponse =
FindTransferMappingsResponse
+ .newBuilder()
+ .addAllMappings(transferMappings)
+ .build();
+ responseObserver.onNext(findTransferMappingsResponse);
+ responseObserver.onCompleted();
+
+ } catch (Exception ex) {
+ String msg = "Errored while fetching transfer mappings, Message:"
+ ex.getMessage();
+ logger.error("Errored while fetching transfer mappings, Message:
{}", ex.getMessage(), ex);
+
responseObserver.onError(io.grpc.Status.INTERNAL.withDescription(msg).asRuntimeException());
+ }
+ }
+
+ @Override
+ public void deleteTransferMappings(DeleteTransferMappingRequest request,
StreamObserver<Empty> responseObserver) {
+ try {
+ AuthenticatedUser authenticatedUser =
request.getAuthToken().getAuthenticatedUser();
+ String transferMappingId = request.getId();
+ Map<String, Object> properties = new HashMap<>();
+ properties.put("username", authenticatedUser.getUsername());
+ properties.put("tenantId", authenticatedUser.getTenantId());
+ properties.put("entityId", transferMappingId);
+ String query = " MATCH (u:User)-[:HAS_TRANSFER_MAPPING]->" +
+ "(t:TransferMapping{entityId:$entityId})" +
+ " where u.username = $username AND u.tenantId = $tenantId
detach delete t";
+ this.neo4JConnector.runTransactionalQuery(properties, query);
+ responseObserver.onNext(Empty.newBuilder().build());
+ responseObserver.onCompleted();
+
+ } catch (Exception ex) {
+ String msg = "Errored while delete transfer mappings, Message:" +
ex.getMessage();
+ logger.error("Errored while delete transfer mappings, Message:
{}", ex.getMessage(), ex);
+
responseObserver.onError(io.grpc.Status.INTERNAL.withDescription(msg).asRuntimeException());
+ }
+ }
+ private boolean hasAccess(String username, String tenantId, String
storagePrefId) throws Exception {
Map<String, Object> userProps = new HashMap<>();
- userProps.put("username", callUser.getUsername());
- userProps.put("tenantId", callUser.getTenantId());
+ userProps.put("username", username);
+ userProps.put("tenantId", tenantId);
+ userProps.put("entityId", storagePrefId);
List<Record> records = this.neo4JConnector.searchNodes(userProps,
" MATCH (u:User) where u.username = $username AND u.tenantId =
$tenantId" +
- " OPTIONAL MATCH
(u)<-[:SHARED_WITH]-(s1:Storage)<-[:CHILD_OF]->(sp1:StoragePreference)" +
+ " OPTIONAL MATCH
(u)<-[:SHARED_WITH]-(s1:Storage)<-[:CHILD_OF]->(sp1:StoragePreference{entityId:$entityId})"
+
" OPTIONAL MATCH
(cg:Group)-[:CHILD_OF*]->(g:Group)<-[:MEMBER_OF]-(u)" +
- " OPTIONAL MATCH
(sp2:StoragePreference)-[:CHILD_OF]->(s2:Storage)-[:SHARED_WITH]->(cg) " +
- " OPTIONAL MATCH
(sp3:StoragePreference)-[:CHILD_OF]->(s3:Storage)-[:SHARED_WITH]->(g)" +
- " OPTIONAL MATCH
(s4:Storage)<-[:CHILD_OF]->(sp4:StoragePreference)-[:SHARED_WITH]->(u)" +
- " OPTIONAL MATCH
(s5:Storage)<-[:CHILD_OF]->(sp5:StoragePreference)-[:SHARED_WITH]->(cg)" +
- " OPTIONAL MATCH
(s6:Storage)<-[:CHILD_OF]->(sp6:StoragePreference)-[:SHARED_WITH]->(g)" +
+ " OPTIONAL MATCH
(sp2:StoragePreference{entityId:$entityId})-[:CHILD_OF]->(s2:Storage)-[:SHARED_WITH]->(cg)
" +
+ " OPTIONAL MATCH (sp3:StoragePreference
{entityId:$entityId})-[:CHILD_OF]->(s3:Storage)-[:SHARED_WITH]->(g)" +
+ " OPTIONAL MATCH
(s4:Storage)<-[:CHILD_OF]->(sp4:StoragePreference{entityId:$entityId})-[:SHARED_WITH]->(u)"
+
+ " OPTIONAL MATCH
(s5:Storage)<-[:CHILD_OF]->(sp5:StoragePreference{entityId:$entityId})-[:SHARED_WITH]->(cg)"
+
+ " OPTIONAL MATCH
(s6:Storage)<-[:CHILD_OF]->(sp6:StoragePreference{entityId:$entityId})-[:SHARED_WITH]->(g)"
+
" return distinct s1, sp1, s2, sp2, s3, sp3, s4,sp4,
s5,sp5, s6,sp6");
if (!records.isEmpty()) {
- try {
- List keyList = new ArrayList();
- keyList.add("s1:sp1");
- keyList.add("s2:sp2");
- keyList.add("s3:sp3");
- keyList.add("s4:sp4");
- keyList.add("s5:sp5");
- keyList.add("s6:sp6");
- List<AnyStoragePreference> storagePrefList =
AnyStoragePreferenceDeserializer.deserializeList(records, keyList);
- StoragePreferenceSearchResponse.Builder builder =
StoragePreferenceSearchResponse.newBuilder();
- builder.addAllStoragesPreference(storagePrefList);
- responseObserver.onNext(builder.build());
- responseObserver.onCompleted();
- } catch (Exception e) {
- String msg = "Errored while searching storage preferences;
Message:" + e.getMessage();
- logger.error("Errored while searching storage preferences;
Message: {}", e.getMessage(), e);
-
responseObserver.onError(io.grpc.Status.INTERNAL.withDescription(msg).asRuntimeException());
- }
- } else {
-
responseObserver.onNext(StoragePreferenceSearchResponse.newBuilder().build());
- responseObserver.onCompleted();
+ return true;
}
+ return false;
}
-
+ private String getStorageId(AnyStoragePreference storage) {
+ if (storage.getStorageCase()
+
.equals(AnyStoragePreference.StorageCase.S3_STORAGE_PREFERENCE)) {
+ return storage.getS3StoragePreference().getStoragePreferenceId();
+ } else {
+ return storage.getSshStoragePreference().getStoragePreferenceId();
+ }
+ }
}
diff --git
a/data-resource-management-service/drms-api/src/main/resources/application.properties
b/data-resource-management-service/drms-api/src/main/resources/application.properties
index 2829a56..0d9237a 100644
---
a/data-resource-management-service/drms-api/src/main/resources/application.properties
+++
b/data-resource-management-service/drms-api/src/main/resources/application.properties
@@ -25,4 +25,5 @@ custos.id=custos-whedmgamitu357p4wuke-10002708
custos.secret=mrMdl86Ia1H94cikW7CvHoh7L0ASNXQVt2aRzSIj
custos.host=custos.scigap.org
custos.port=31499
-grpc.port=7070
\ No newline at end of file
+grpc.port=7070
+local.grpc.port=7070
\ No newline at end of file
diff --git
a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/AnyStoragePreferenceDeserializer.java
b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/AnyStoragePreferenceDeserializer.java
index 314455f..8db4031 100644
---
a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/AnyStoragePreferenceDeserializer.java
+++
b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/AnyStoragePreferenceDeserializer.java
@@ -56,7 +56,7 @@ public class AnyStoragePreferenceDeserializer {
try {
storage =
AnyStorageDeserializer.deriveStorageFromMap(stVal.asMap());
AnyStoragePreference preference =
deriveStoragePrefFromMap(spVal.asMap(), storage);
- storagePreferenceMap.computeIfAbsent(stVal.id(), v
-> {
+ storagePreferenceMap.computeIfAbsent(spVal.id(), v
-> {
return new ArrayList<AnyStoragePreference>();
}).add(preference);
diff --git
a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/TransferMappingDeserializer.java
b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/TransferMappingDeserializer.java
new file mode 100644
index 0000000..461bbe8
--- /dev/null
+++
b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/TransferMappingDeserializer.java
@@ -0,0 +1,57 @@
+package org.apache.airavata.drms.core.deserializer;
+
+import org.apache.airavata.datalake.drms.storage.AnyStorage;
+import org.apache.airavata.datalake.drms.storage.AnyStoragePreference;
+import org.apache.airavata.datalake.drms.storage.TransferMapping;
+import org.apache.airavata.datalake.drms.storage.TransferScope;
+import org.neo4j.driver.Record;
+import org.neo4j.driver.Value;
+import org.neo4j.driver.internal.InternalRecord;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class TransferMappingDeserializer {
+
+ public static List<TransferMapping> deserializeList(List<Record>
neo4jRecords) throws Exception {
+ List<TransferMapping> transferMappings = new ArrayList<>();
+ for (Record record : neo4jRecords) {
+ InternalRecord internalRecord = (InternalRecord) record;
+ List<Value> values = internalRecord.values();
+
+ if (values.size() == 5) {
+ Value srcStr = values.get(0);
+ Value srcSpr = values.get(1);
+ Value dstStr = values.get(2);
+ Value dstSp = values.get(3);
+ Value tm = values.get(4);
+
+ if (!srcStr.isNull() && !srcSpr.isNull() && !tm.isNull()
+ && !dstStr.isNull() && !dstSp.isNull()) {
+ AnyStorage storage =
AnyStorageDeserializer.deriveStorageFromMap(srcStr.asMap());
+ AnyStoragePreference srcPreference =
AnyStoragePreferenceDeserializer
+ .deriveStoragePrefFromMap(srcSpr.asMap(), storage);
+
+ AnyStorage dstStorage =
AnyStorageDeserializer.deriveStorageFromMap(dstStr.asMap());
+ AnyStoragePreference dstPreference =
AnyStoragePreferenceDeserializer
+ .deriveStoragePrefFromMap(dstSp.asMap(),
dstStorage);
+
+ Map<String, Object> map = tm.asMap();
+ TransferMapping transferMapping =
TransferMapping.newBuilder()
+
.setTransferScope(TransferScope.valueOf(map.get("scope").toString()))
+ .setId(map.get("entityId").toString())
+ .setSourceStoragePreference(srcPreference)
+ .setDestinationStoragePreference(dstPreference)
+ .setUserId(map.get("owner").toString())
+ .build();
+
+ transferMappings.add(transferMapping);
+
+ }
+ }
+ }
+ return transferMappings;
+ }
+
+}
diff --git
a/data-resource-management-service/drms-stubs/src/main/proto/preference/StoragePreferenceService.proto
b/data-resource-management-service/drms-stubs/src/main/proto/preference/StoragePreferenceService.proto
index 117408b..6c2c6ef 100644
---
a/data-resource-management-service/drms-stubs/src/main/proto/preference/StoragePreferenceService.proto
+++
b/data-resource-management-service/drms-stubs/src/main/proto/preference/StoragePreferenceService.proto
@@ -27,89 +27,146 @@ import "preference/S3Preference.proto";
import "google/protobuf/empty.proto";
message AnyStoragePreference {
- oneof storage {
-
org.apache.airavata.datalake.drms.storage.preference.ssh.SSHStoragePreference
ssh_storage_preference = 1;
-
org.apache.airavata.datalake.drms.storage.preference.s3.S3StoragePreference
s3_storage_preference = 2;
- }
+ oneof storage {
+
org.apache.airavata.datalake.drms.storage.preference.ssh.SSHStoragePreference
ssh_storage_preference = 1;
+
org.apache.airavata.datalake.drms.storage.preference.s3.S3StoragePreference
s3_storage_preference = 2;
+ }
}
message StoragePreferenceFetchRequest {
- org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
- string storage_preference_id = 2;
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ string storage_preference_id = 2;
}
message StoragePreferenceFetchResponse {
- AnyStoragePreference storage_preference = 1;
+ AnyStoragePreference storage_preference = 1;
}
message StoragePreferenceCreateRequest {
- org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
- AnyStoragePreference storage_preference = 2;
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ AnyStoragePreference storage_preference = 2;
}
message StoragePreferenceCreateResponse {
- AnyStoragePreference storage_preference = 1;
+ AnyStoragePreference storage_preference = 1;
}
message StoragePreferenceUpdateRequest {
- org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
- string storage_preference_id = 2;
- AnyStoragePreference storage_preference = 3;
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ string storage_preference_id = 2;
+ AnyStoragePreference storage_preference = 3;
}
message StoragePreferenceUpdateResponse {
- AnyStoragePreference storage_preference = 1;
+ AnyStoragePreference storage_preference = 1;
}
message StoragePreferenceDeleteRequest {
- org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
- string storage_preference_id = 2;
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ string storage_preference_id = 2;
}
message StoragePreferenceSearchQuery {
- string field = 1;
- string value = 2;
- string options = 3; // LIKE, NOT, EQUALS, GREATER, LESS
+ string field = 1;
+ string value = 2;
+ string options = 3; // LIKE, NOT, EQUALS, GREATER, LESS
}
message StoragePreferenceSearchRequest {
- org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
- repeated StoragePreferenceSearchQuery queries = 2;
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ repeated StoragePreferenceSearchQuery queries = 2;
}
message StoragePreferenceSearchResponse {
- org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
- repeated AnyStoragePreference storages_preference = 2;
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ repeated AnyStoragePreference storages_preference = 2;
}
+enum TransferScope {
+ UNKNOWN = 0;
+ GLOBAL = 1;
+ USER = 2;
+}
+
+message TransferMapping {
+ string id = 1;
+ string user_id = 2;
+ AnyStoragePreference source_storage_preference = 3;
+ AnyStoragePreference destination_storage_preference = 4;
+ TransferScope transfer_scope = 5;
+}
+
+message CreateTransferMappingRequest {
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ TransferMapping transfer_mapping = 2;
+}
+
+message CreateTransferMappingResponse {
+ TransferMapping transfer_mapping = 1;
+}
+
+message FindTransferMappingsRequest {
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ string id = 2;
+ TransferScope transfer_scope = 3;
+}
+
+message FindTransferMappingsResponse {
+ repeated TransferMapping mappings = 1;
+}
+
+message DeleteTransferMappingRequest {
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken auth_token = 1;
+ string id = 2;
+}
+
+
service StoragePreferenceService {
- rpc fetchStoragePreference (StoragePreferenceFetchRequest) returns
(StoragePreferenceFetchResponse) {
- option (google.api.http) = {
+ rpc fetchStoragePreference (StoragePreferenceFetchRequest) returns
(StoragePreferenceFetchResponse) {
+ option (google.api.http) = {
get: "/v1.0/api/drms/storagePreference"
};
- }
+ }
- rpc createStoragePreference (StoragePreferenceCreateRequest) returns
(StoragePreferenceCreateResponse) {
- option (google.api.http) = {
+ rpc createStoragePreference (StoragePreferenceCreateRequest) returns
(StoragePreferenceCreateResponse) {
+ option (google.api.http) = {
post: "/v1.0/api/drms/storagePreference"
};
- }
+ }
- rpc updateStoragePreference (StoragePreferenceUpdateRequest) returns
(StoragePreferenceUpdateResponse) {
- option (google.api.http) = {
+ rpc updateStoragePreference (StoragePreferenceUpdateRequest) returns
(StoragePreferenceUpdateResponse) {
+ option (google.api.http) = {
put: "/v1.0/api/drms/storagePreference"
};
- }
+ }
- rpc deletePreferenceStorage (StoragePreferenceDeleteRequest) returns
(google.protobuf.Empty) {
- option (google.api.http) = {
+ rpc deletePreferenceStorage (StoragePreferenceDeleteRequest) returns
(google.protobuf.Empty) {
+ option (google.api.http) = {
delete: "/v1.0/api/drms/storagePreference"
};
- }
+ }
- rpc searchStoragePreference (StoragePreferenceSearchRequest) returns
(StoragePreferenceSearchResponse) {
- option (google.api.http) = {
+ rpc searchStoragePreference (StoragePreferenceSearchRequest) returns
(StoragePreferenceSearchResponse) {
+ option (google.api.http) = {
post: "/v1.0/api/drms/storagePreference/searchPreference"
};
- }
+ }
+
+ rpc createTransferMapping (CreateTransferMappingRequest) returns
(CreateTransferMappingResponse) {
+ option (google.api.http) = {
+ post: "/v1.0/api/drms/storagePreference/transferMapping"
+ };
+ }
+
+ rpc getTransferMappings (FindTransferMappingsRequest) returns
(FindTransferMappingsResponse) {
+ option (google.api.http) = {
+ get: "/v1.0/api/drms/storagePreference/transferMapping"
+ };
+ }
+
+ rpc deleteTransferMappings (DeleteTransferMappingRequest) returns
(google.protobuf.Empty) {
+ option (google.api.http) = {
+ delete: "/v1.0/api/drms/storagePreference/transferMapping"
+ };
+ }
}
\ No newline at end of file
