This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
The following commit(s) were added to refs/heads/master by this push:
new 1139218 drms storage and storage-preference
new ccbb746 Merge pull request #21 from isururanawaka/workflow_invocation
1139218 is described below
commit 1139218e74413b30ddcb9a9bb06e28b09b3d9903
Author: Isuru Ranawaka <[email protected]>
AuthorDate: Fri Jul 9 11:16:02 2021 -0400
drms storage and storage-preference
---
.../src/main/resources/config.yml | 2 +-
.../drms/api/handlers/ResourceServiceHandler.java | 45 ++++++++++++---------
.../handlers/StoragePreferenceServiceHandler.java | 44 +++++++++++++++-----
.../drms-rest-proxy/src/main/resources/drms.pb | Bin 109380 -> 109388 bytes
.../src/main/proto/resource/DRMSResource.proto | 10 ++---
5 files changed, 65 insertions(+), 36 deletions(-)
diff --git
a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/resources/config.yml
b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/resources/config.yml
index 2d11996..43fb605 100644
---
a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/resources/config.yml
+++
b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/resources/config.yml
@@ -12,6 +12,6 @@ outboundEventProcessor:
drmsPort: 7070
consumer:
brokerURL: "149.165.157.235:9092"
- consumerGroup: "DataOrchestratorGroup"
+ consumerGroup: "DataOrchestratorGroup2"
topic: "data-orchestrator-file-events"
maxPollRecordsConfig: 10
\ No newline at end of file
diff --git
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
index 214a6d7..d97f29d 100644
---
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
+++
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
@@ -29,8 +29,8 @@ import org.apache.airavata.drms.api.utils.CustosUtils;
import org.apache.airavata.drms.api.utils.Utils;
import org.apache.airavata.drms.core.Neo4JConnector;
import org.apache.airavata.drms.core.constants.StoragePreferenceConstants;
+import org.apache.airavata.drms.core.deserializer.AnyStorageDeserializer;
import org.apache.airavata.drms.core.deserializer.GenericResourceDeserializer;
-import org.apache.airavata.drms.core.deserializer.TransferMappingDeserializer;
import org.apache.airavata.drms.core.serializer.GenericResourceSerializer;
import org.apache.custos.clients.CustosClientProvider;
import org.apache.custos.sharing.service.Entity;
@@ -90,28 +90,16 @@ public class ResourceServiceHandler extends
ResourceServiceGrpc.ResourceServiceI
List<GenericResource> genericResourceList =
GenericResourceDeserializer.deserializeList(records);
ResourceFetchResponse.Builder builder =
ResourceFetchResponse.newBuilder();
if (!genericResourceList.isEmpty()) {
- // TODO: Move to Storage
- String searchQuery = "Match
(srcStr:Storage)<-[:CHILD_OF]-" +
-
"(srcSp:StoragePreference)-[:TRANSFER_OUT]->(t:TransferMapping" +
- "{scope:'GLOBAL',
tenantId:$tenantId})-[:TRANSFER_IN]->(dstSp:StoragePreference)-[:CHILD_OF]->(dstStr:Storage)"
+
- " return srcStr, srcSp, dstStr, dstSp, t";
-
- List<Record> globalRecords =
this.neo4JConnector.searchNodes(userProps, searchQuery);
-
- if (!globalRecords.isEmpty()) {
- List<TransferMapping> transferMappings =
TransferMappingDeserializer.deserializeList(globalRecords);
- if (!transferMappings.isEmpty()) {
- AnyStoragePreference anyStoragePreference =
transferMappings.get(0)
- .getSourceStoragePreference();
- GenericResource resource =
genericResourceList.get(0)
- .toBuilder()
-
.setSshPreference(anyStoragePreference.getSshStoragePreference())
- .build();
- builder.setResource(resource);
+ Optional<AnyStorage> anyStorage = findStorage(resourceId,
type, callUser.getTenantId());
+ GenericResource resource = genericResourceList.get(0);
+ if (anyStorage.isPresent()) {
+ if
(anyStorage.get().getStorageCase().equals(AnyStorage.StorageCase.SSH_STORAGE)) {
+ resource =
resource.toBuilder().setSshStorage(anyStorage.get().getSshStorage()).build();
} else {
- builder.setResource(genericResourceList.get(0));
+ resource =
resource.toBuilder().setS3Storage(anyStorage.get().getS3Storage()).build();
}
}
+ builder.setResource(resource);
}
responseObserver.onNext(builder.build());
responseObserver.onCompleted();
@@ -839,4 +827,21 @@ public class ResourceServiceHandler extends
ResourceServiceGrpc.ResourceServiceI
return Optional.empty();
}
+
+ private Optional<AnyStorage> findStorage(String entityId, String type,
String tenantId) throws Exception {
+ Map<String, Object> parameters = new HashMap<>();
+ parameters.put("entityId", entityId);
+ parameters.put("tenantId", tenantId);
+ String query = " Match (r" + type + ") where r.entityId=$entityId and
r.tenantId=$tenantId " +
+ " Match (s:Storage)<-[:CHILD_OF*]-(r) return (s)";
+
+ List<Record> records = this.neo4JConnector.searchNodes(parameters,
query);
+ if (!records.isEmpty()) {
+ List<AnyStorage> storageList =
AnyStorageDeserializer.deserializeList(records);
+ return Optional.ofNullable(storageList.get(0));
+
+ }
+ return Optional.empty();
+ }
+
}
diff --git
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java
index ef07212..7221e6d 100644
---
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java
+++
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java
@@ -221,16 +221,40 @@ public class StoragePreferenceServiceHandler extends
StoragePreferenceServiceGrp
userProps.put("username", callUser.getUsername());
userProps.put("tenantId", callUser.getTenantId());
- List<Record> records = this.neo4JConnector.searchNodes(userProps,
- " MATCH (u:User) where u.username = $username AND
u.tenantId = $tenantId" +
- " OPTIONAL MATCH
(u)<-[:SHARED_WITH]-(s1:Storage)<-[:CHILD_OF]->(sp1:StoragePreference)" +
- " OPTIONAL MATCH
(cg:Group)-[:CHILD_OF*]->(g:Group)<-[:MEMBER_OF]-(u)" +
- " OPTIONAL MATCH
(sp2:StoragePreference)-[:CHILD_OF]->(s2:Storage)-[:SHARED_WITH]->(cg) " +
- " OPTIONAL MATCH
(sp3:StoragePreference)-[:CHILD_OF]->(s3:Storage)-[:SHARED_WITH]->(g)" +
- " OPTIONAL MATCH
(s4:Storage)<-[:CHILD_OF]->(sp4:StoragePreference)-[:SHARED_WITH]->(u)" +
- " OPTIONAL MATCH
(s5:Storage)<-[:CHILD_OF]->(sp5:StoragePreference)-[:SHARED_WITH]->(cg)" +
- " OPTIONAL MATCH
(s6:Storage)<-[:CHILD_OF]->(sp6:StoragePreference)-[:SHARED_WITH]->(g)" +
- " return distinct s1, sp1, s2, sp2, s3, sp3,
s4,sp4, s5,sp5, s6,sp6");
+ String query = " MATCH (u:User) where u.username = $username AND
u.tenantId = $tenantId" +
+ " OPTIONAL MATCH
(u)<-[:SHARED_WITH]-(s1:Storage)<-[:CHILD_OF]->(sp1:StoragePreference)" +
+ " OPTIONAL MATCH
(cg:Group)-[:CHILD_OF*]->(g:Group)<-[:MEMBER_OF]-(u)" +
+ " OPTIONAL MATCH
(sp2:StoragePreference)-[:CHILD_OF]->(s2:Storage)-[:SHARED_WITH]->(cg) " +
+ " OPTIONAL MATCH
(sp3:StoragePreference)-[:CHILD_OF]->(s3:Storage)-[:SHARED_WITH]->(g)" +
+ " OPTIONAL MATCH
(s4:Storage)<-[:CHILD_OF]->(sp4:StoragePreference)-[:SHARED_WITH]->(u)" +
+ " OPTIONAL MATCH
(s5:Storage)<-[:CHILD_OF]->(sp5:StoragePreference)-[:SHARED_WITH]->(cg)" +
+ " OPTIONAL MATCH
(s6:Storage)<-[:CHILD_OF]->(sp6:StoragePreference)-[:SHARED_WITH]->(g)" +
+ " return distinct s1, sp1, s2, sp2, s3, sp3, s4,sp4,
s5,sp5, s6,sp6";
+
+
+ String storageId = null;
+ List<StoragePreferenceSearchQuery> storagePreferenceSearchQueries
= request.getQueriesList();
+
+ for (StoragePreferenceSearchQuery searchQuery :
storagePreferenceSearchQueries) {
+ if (searchQuery.getField().equals("storageId")) {
+ storageId = searchQuery.getValue();
+ }
+ }
+
+ if (storageId != null) {
+ query = " MATCH (u:User) where u.username = $username AND
u.tenantId = $tenantId" +
+ " OPTIONAL MATCH
(u)<-[:SHARED_WITH]-(s1:Storage{storageId:'" + storageId +
"'})<-[:CHILD_OF]->(sp1:StoragePreference)" +
+ " OPTIONAL MATCH
(cg:Group)-[:CHILD_OF*]->(g:Group)<-[:MEMBER_OF]-(u)" +
+ " OPTIONAL MATCH
(sp2:StoragePreference)-[:CHILD_OF]->(s2:Storage{storageId:'" + storageId +
"'})-[:SHARED_WITH]->(cg) " +
+ " OPTIONAL MATCH
(sp3:StoragePreference)-[:CHILD_OF]->(s3:Storage{storageId:'" + storageId +
"'})-[:SHARED_WITH]->(g)" +
+ " OPTIONAL MATCH (s4:Storage{storageId:'" + storageId
+ "'})<-[:CHILD_OF]->(sp4:StoragePreference)-[:SHARED_WITH]->(u)" +
+ " OPTIONAL MATCH (s5:Storage{storageId:'" + storageId
+ "'})<-[:CHILD_OF]->(sp5:StoragePreference)-[:SHARED_WITH]->(cg)" +
+ " OPTIONAL MATCH (s6:Storage{storageId:'" + storageId
+ "'})<-[:CHILD_OF]->(sp6:StoragePreference)-[:SHARED_WITH]->(g)" +
+ " return distinct s1, sp1, s2, sp2, s3, sp3, s4,sp4,
s5,sp5, s6,sp6";
+ }
+
+ List<Record> records = this.neo4JConnector.searchNodes(userProps,
query);
+
if (!records.isEmpty()) {
try {
List keyList = new ArrayList();
diff --git
a/data-resource-management-service/drms-rest-proxy/src/main/resources/drms.pb
b/data-resource-management-service/drms-rest-proxy/src/main/resources/drms.pb
index fd2c03a..21a39a1 100644
Binary files
a/data-resource-management-service/drms-rest-proxy/src/main/resources/drms.pb
and
b/data-resource-management-service/drms-rest-proxy/src/main/resources/drms.pb
differ
diff --git
a/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResource.proto
b/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResource.proto
index db7741a..c696cb0 100644
---
a/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResource.proto
+++
b/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResource.proto
@@ -21,15 +21,15 @@ option java_multiple_files = true;
package org.apache.airavata.datalake.drms.resource;
import "google/api/annotations.proto";
-import "preference/S3Preference.proto";
-import "preference/SSHPreference.proto";
+import "storage/S3Storage.proto";
+import "storage/SSHStorage.proto";
message GenericResource {
string resource_id = 1;
string resource_path = 2;
- oneof storagePreference {
-
org.apache.airavata.datalake.drms.storage.preference.s3.S3StoragePreference
s3_preference = 3;
-
org.apache.airavata.datalake.drms.storage.preference.ssh.SSHStoragePreference
ssh_preference = 4;
+ oneof storage {
+ org.apache.airavata.datalake.drms.storage.ssh.SSHStorage ssh_storage = 3;
+ org.apache.airavata.datalake.drms.storage.s3.S3Storage s3_storage = 4;
}
map<string, string> properties = 5;
string type=6;
