This is an automated email from the ASF dual-hosted git repository. dimuthuupe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
commit 4e9880d1d259859b05e881df39d3f73b594ca7cd Author: Isuru Ranawaka <[email protected]> AuthorDate: Mon Mar 22 16:03:15 2021 -0400 Add parser interface, remove duplicate object creation at Neo4J --- .../airavata/datalake/metadata/clients/Test.java | 39 +++++- .../airavata/datalake/metadata/AppConfig.java | 52 ++++--- .../metadata/backend/neo4j/model/nodes/Entity.java | 7 + .../metadata/backend/neo4j/model/nodes/Group.java | 5 + .../backend/neo4j/model/nodes/Resource.java | 5 + .../metadata/backend/neo4j/model/nodes/Tenant.java | 5 + .../metadata/backend/neo4j/model/nodes/User.java | 5 + .../metadata/handlers/TenantServiceHandler.java | 3 +- .../metadata/parsers/ExecutionContext.java | 17 +++ .../datalake/metadata/parsers/GroupParser.java | 106 ++++++++------ .../airavata/datalake/metadata/parsers/Parser.java | 13 ++ .../datalake/metadata/parsers/ResourceParser.java | 152 ++++++++++++++------- .../datalake/metadata/parsers/TenantParser.java | 114 +++++++++------- .../datalake/metadata/parsers/UserParser.java | 33 ++++- .../stub/src/main/proto/resource/Resource.proto | 13 +- 15 files changed, 389 insertions(+), 180 deletions(-) diff --git a/metadata-service/db-service/client/src/main/java/org/apache/airavata/datalake/metadata/clients/Test.java b/metadata-service/db-service/client/src/main/java/org/apache/airavata/datalake/metadata/clients/Test.java index dd8b392..a506d75 100644 --- a/metadata-service/db-service/client/src/main/java/org/apache/airavata/datalake/metadata/clients/Test.java +++ b/metadata-service/db-service/client/src/main/java/org/apache/airavata/datalake/metadata/clients/Test.java @@ -9,25 +9,27 @@ public class Test { TenantMetadataServiceGrpc.TenantMetadataServiceBlockingStub stub = serviceClient.tenant(); + Tenant tenant = Tenant.newBuilder() - .setTenantId("asdcfvf") + .setTenantId("100010402") .setName("TenantA") .build(); + Group group = Group.newBuilder() .setName("g1") + .setTenantId("100010402") .build(); + Group group2 = Group.newBuilder() .setName("g3") - .build(); - Group group1 = Group.newBuilder() - .setName("g2") - .addChildGroups(group2) + .setTenantId("100010402") .build(); User user = User.newBuilder() .setUsername("TestingUserA") .setFirstName("Isuru") .setLastName("Ranawaka") + .setTenantId("100010402") .build(); GroupMembership groupMemberships = GroupMembership @@ -36,26 +38,52 @@ public class Test { .setMembershipType("ADMIN") .build(); + Group group1 = Group.newBuilder() + .setName("g2") + .setTenantId("100010402") + .addChildGroups(group2) + .build(); + group1 = group1.toBuilder() .addGroupMembership(groupMemberships) + .setTenantId("100010402") .build(); + group = group.toBuilder() .addChildGroups(group1) + .setTenantId("100010402") .build(); Resource resource = Resource.newBuilder() .setName("R1") + .setTenantId("100010402") .build(); + Resource resource1 = Resource.newBuilder() .setName("R2") + .setTenantId("100010402") .build(); + Resource resource2 = Resource.newBuilder() .setName("R3") + .setTenantId("100010402") .build(); resource1 = resource1.toBuilder() .addChildResources(resource2) + .setTenantId("100010402") + .build(); + + ResourceSharings resourceSharings = ResourceSharings + .newBuilder() + .setPermissionType("READ") + .addGroups(group2) .build(); + + resource1 = resource1.toBuilder() + .addSharings(resourceSharings) + .build(); + resource = resource.toBuilder() .addChildResources(resource1) .build(); @@ -63,6 +91,7 @@ public class Test { tenant = tenant.toBuilder() .addGroups(group) .build(); + tenant = tenant.toBuilder() .addResources(resource) .build(); diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/AppConfig.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/AppConfig.java index 2c0f692..0e03c32 100644 --- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/AppConfig.java +++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/AppConfig.java @@ -1,18 +1,26 @@ package org.apache.airavata.datalake.metadata; import org.apache.airavata.datalake.metadata.backend.Connector; +import org.apache.airavata.datalake.metadata.backend.neo4j.curd.operators.ResourceServiceImpl; +import org.apache.airavata.datalake.metadata.backend.neo4j.curd.operators.SearchOperator; +import org.apache.airavata.datalake.metadata.backend.neo4j.curd.operators.TenantServiceImpl; import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group; import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Resource; import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Tenant; import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User; import org.dozer.DozerBeanMapper; import org.dozer.loader.api.BeanMappingBuilder; +import org.neo4j.ogm.cypher.ComparisonOperator; +import org.neo4j.ogm.cypher.Filter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import java.util.ArrayList; +import java.util.List; + @Configuration public class AppConfig { @@ -25,7 +33,9 @@ public class AppConfig { @Bean public DozerBeanMapper dozerBeanMapper() { + DozerBeanMapper mapper = new DozerBeanMapper(); + BeanMappingBuilder tenantMapping = new BeanMappingBuilder() { @Override protected void configure() { @@ -39,6 +49,7 @@ public class AppConfig { mapping(org.apache.airavata.datalake.metadata.service.Group.class, Group.class); } }; + BeanMappingBuilder resourceMapping = new BeanMappingBuilder() { @Override protected void configure() { @@ -52,10 +63,12 @@ public class AppConfig { mapping(org.apache.airavata.datalake.metadata.service.User.class, User.class); } }; + mapper.addMapping(tenantMapping); mapper.addMapping(groupMapping); mapper.addMapping(resourceMapping); mapper.addMapping(userMapping); + return mapper; } @@ -63,6 +76,7 @@ public class AppConfig { @Bean Tenant getTenant() { LOGGER.info("Calling get tenant############"); + Tenant tenant = new Tenant(); tenant.setTenantId("123456789"); tenant.setName("Tenant"); @@ -91,7 +105,6 @@ public class AppConfig { group.addChildGroup(group2, 0, 0, null); group2.addChildGroup(group3, 0, 0, null); - Resource resource = new Resource(); resource.setName("R1"); @@ -104,12 +117,10 @@ public class AppConfig { Resource resource3 = new Resource(); resource3.setName("R4"); - group.addChildUser(user, "ADMIN", 0, 0, null); resource.addChildResource(resource1, 0, 0, null); resource.shareWithAUser(user, "READ", 0, 0, null); - group2.addChildUser(user1, "ADMIN", 0, 0, null); group3.addChildUser(user2, "ADMIN", 0, 0, null); @@ -120,29 +131,26 @@ public class AppConfig { tenant.add(group, 0, 0, null); tenant.add(resource, 0, 0, null); + TenantServiceImpl tenantService = new TenantServiceImpl(connector); + tenantService.createOrUpdate(tenant); -// TenantServiceImpl tenantService = new TenantServiceImpl(connector); -// tenantService.createOrUpdate(tenant); -// -// Filter filter = new Filter("name", ComparisonOperator.EQUALS, "R3"); -// -// -// ResourceServiceImpl resourceService = new ResourceServiceImpl(connector); -// SearchOperator searchOperator = new SearchOperator(); -// searchOperator.setKey("name"); -// searchOperator.setValue("R2"); -// searchOperator.setComparisonOperator(ComparisonOperator.EQUALS); -// List searchList = new ArrayList<>(); -// searchList.add(searchOperator); -// List<Resource> collections = (List<Resource>) resourceService.search(searchList); -// LOGGER.info("Size", collections.size()); -// for (Resource collection : collections) { -// LOGGER.info("#############" + collection.getName() + "Created At" + collection.getCreatedAt()); -// } + Filter filter = new Filter("name", ComparisonOperator.EQUALS, "R3"); + ResourceServiceImpl resourceService = new ResourceServiceImpl(connector); + SearchOperator searchOperator = new SearchOperator(); + searchOperator.setKey("name"); + searchOperator.setValue("R2"); + searchOperator.setComparisonOperator(ComparisonOperator.EQUALS); + List searchList = new ArrayList<>(); + searchList.add(searchOperator); + List<Resource> collections = (List<Resource>) resourceService.search(searchList); + LOGGER.info("Size", collections.size()); + for (Resource collection : collections) { + LOGGER.info("#############" + collection.getName() + "Created At" + collection.getCreatedAt()); + } - return tenant; + return tenant; } diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Entity.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Entity.java index 69fbdca..aba5035 100644 --- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Entity.java +++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Entity.java @@ -35,6 +35,8 @@ public abstract class Entity { @Property(name = "tenant_id") private String tenantId; + + public Long getId() { return id; } @@ -106,4 +108,9 @@ public abstract class Entity { public void setId(Long id) { this.id = id; } + + + public String getSearchableId() { + return primaryExternalKey+"@"+tenantId; + } } diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Group.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Group.java index c3362c5..9b735ec 100644 --- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Group.java +++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Group.java @@ -142,6 +142,11 @@ public class Group extends Entity { this.memberUsersInPointers = memberUsersInPointers; } + @Override + public String getSearchableId() { + return this.getName()+"@"+this.getTenantId(); + } + public void addChildUser(User user, String membership, long relationshipCreatedAt, long relationshipModifiedAt, Map<String, String> relationshipProperties) { MemberOf memberOf = new MemberOf(); diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Resource.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Resource.java index b9808bf..a61848b 100644 --- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Resource.java +++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Resource.java @@ -108,6 +108,11 @@ public class Resource extends Entity { } @Override + public String getSearchableId() { + return this.getName()+"@"+this.getTenantId(); + } + + @Override public Long getId() { return id; } diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Tenant.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Tenant.java index 7967386..5dbe1ff 100644 --- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Tenant.java +++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/Tenant.java @@ -101,6 +101,11 @@ public class Tenant extends Entity { this.entities.add(entity); } + @Override + public String getSearchableId() { + return this.getName()+"@"+this.getTenantId(); + } + public void add(Entity entity, long relationShipCreatedAt, long relationShipModifiedAt, Map<String, String> relationshipProperties) { diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/User.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/User.java index d20f560..10a00ff 100644 --- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/User.java +++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/backend/neo4j/model/nodes/User.java @@ -129,4 +129,9 @@ public class User extends Entity { public void addAccessibleResources(HasAccess accessibleResources) { this.accessibleResources.add(accessibleResources); } + + @Override + public String getSearchableId() { + return this.getUsername() + "@" + this.getTenantId(); + } } diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/handlers/TenantServiceHandler.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/handlers/TenantServiceHandler.java index d4fce42..66a2f42 100644 --- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/handlers/TenantServiceHandler.java +++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/handlers/TenantServiceHandler.java @@ -33,7 +33,8 @@ public class TenantServiceHandler extends TenantMetadataServiceGrpc.TenantMetada try { Tenant tenant = request.getTenant(); org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Tenant parsedTenant = - tenantParser.parseTenant(tenant); + (org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Tenant) + tenantParser.parse(tenant); TenantServiceImpl tenantService = new TenantServiceImpl(connector); tenantService.createOrUpdate(parsedTenant); diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/ExecutionContext.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/ExecutionContext.java new file mode 100644 index 0000000..0a8365a --- /dev/null +++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/ExecutionContext.java @@ -0,0 +1,17 @@ +package org.apache.airavata.datalake.metadata.parsers; + +import java.util.HashMap; +import java.util.Map; + +public class ExecutionContext { + + private Map<String, Object> neo4JConvertedModels = new HashMap<>(); + + public Object getNeo4JConvertedModels(String key) { + return neo4JConvertedModels.get(key); + } + + public void addNeo4JConvertedModels(String key, Object obj) { + this.neo4JConvertedModels.put(key, obj); + } +} diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/GroupParser.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/GroupParser.java index eb31649..f486841 100644 --- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/GroupParser.java +++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/GroupParser.java @@ -1,5 +1,7 @@ package org.apache.airavata.datalake.metadata.parsers; +import com.google.protobuf.GeneratedMessageV3; +import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Entity; import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group; import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User; import org.apache.airavata.datalake.metadata.service.GroupMembership; @@ -10,59 +12,79 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.util.List; -import java.util.Optional; @Component -public class GroupParser { +public class GroupParser implements Parser { private static final Logger LOGGER = LoggerFactory.getLogger(GroupParser.class); @Autowired private DozerBeanMapper dozerBeanMapper; - public Group parseGroup(org.apache.airavata.datalake.metadata.service.Group group, Optional<Group> parentGroup) { - Group newParentGroup; - if (parentGroup.isEmpty()) { - newParentGroup = dozerBeanMapper.map(group, - org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group.class); + @Override + public Entity parse(GeneratedMessageV3 entity, Entity parentEntity, ExecutionContext executionContext) { + if (entity instanceof org.apache.airavata.datalake.metadata.service.Group) { + org.apache.airavata.datalake.metadata.service.Group group = (org.apache.airavata.datalake.metadata.service.Group) entity; + Group newParentGroup = null; + if (parentEntity == null) { + newParentGroup = dozerBeanMapper.map(group, + org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group.class); + LOGGER.info("Creating group "+ newParentGroup.getName() + " class"+ newParentGroup.toString()); + executionContext.addNeo4JConvertedModels(newParentGroup.getSearchableId(),newParentGroup); + } else if (parentEntity != null){ + newParentGroup = (Group) parentEntity; + Group childGroup = dozerBeanMapper.map(group, + org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group.class); + executionContext.addNeo4JConvertedModels(childGroup.getSearchableId(),childGroup); + LOGGER.info("Creating group "+ newParentGroup.getName() + " class"+ newParentGroup.toString()); + newParentGroup.addChildGroup(childGroup, + childGroup.getCreatedAt() != 0 ? childGroup.getCreatedAt() : System.currentTimeMillis(), + childGroup.getLastModifiedAt() != 0 ? childGroup.getLastModifiedAt() : System.currentTimeMillis(), + null); // Improve this with relatioship propertie + + newParentGroup = childGroup; + } + + List<GroupMembership> groupMemberships = group.getGroupMembershipList(); + + if (!groupMemberships.isEmpty()) { + Group finalNewParentGroup1 = newParentGroup; + groupMemberships.forEach(mebership -> { + User usr = dozerBeanMapper.map(mebership.getUser(), org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User.class); + executionContext.addNeo4JConvertedModels(usr.getSearchableId(),usr); + finalNewParentGroup1.addChildUser(usr, mebership.getMembershipType(), + mebership.getCreatedAt() != 0 ? mebership.getCreatedAt() : System.currentTimeMillis(), + mebership.getLastModifiedAt() != 0 ? mebership.getLastModifiedAt() : System.currentTimeMillis(), + null); + }); + + } + + List<org.apache.airavata.datalake.metadata.service.Group> groups = group.getChildGroupsList(); + + if (!groups.isEmpty()) { + + Group finalNewParentGroup = newParentGroup; + groups.forEach(gr -> { + this.parse(gr, finalNewParentGroup, executionContext); + }); + } + + return newParentGroup; } else { - newParentGroup = parentGroup.get(); - Group childGroup = dozerBeanMapper.map(group, - org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group.class); - newParentGroup.addChildGroup(childGroup, - childGroup.getCreatedAt() != 0 ? childGroup.getCreatedAt() : System.currentTimeMillis(), - childGroup.getLastModifiedAt() != 0 ? childGroup.getLastModifiedAt() : System.currentTimeMillis(), - null); // Improve this with relatioship propertie - - newParentGroup = childGroup; + String msg = "Wrong entity type detected for parser Group Parser, Expected Group"; + LOGGER.error(msg); + throw new RuntimeException(msg); } - - List<GroupMembership> groupMemberships = group.getGroupMembershipList(); - - if (!groupMemberships.isEmpty()) { - Group finalNewParentGroup1 = newParentGroup; - groupMemberships.forEach(mebership -> { - User usr = dozerBeanMapper.map(mebership.getUser(), org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User.class); - finalNewParentGroup1.addChildUser(usr, mebership.getMembershipType(), - mebership.getCreatedAt() != 0 ? mebership.getCreatedAt() : System.currentTimeMillis(), - mebership.getLastModifiedAt() != 0 ? mebership.getLastModifiedAt() : System.currentTimeMillis(), - null); - }); - - } - - List<org.apache.airavata.datalake.metadata.service.Group> groups = group.getChildGroupsList(); - - if (!groups.isEmpty()) { - - Group finalNewParentGroup = newParentGroup; - groups.forEach(gr -> { - this.parseGroup(gr, Optional.of(finalNewParentGroup)); - }); - } - return newParentGroup; - } + @Override + public Entity parse(GeneratedMessageV3 entity, ExecutionContext executionContext) { + return this.parse(entity,null, executionContext); + } + @Override + public Entity parse(GeneratedMessageV3 entity) { + return this.parse(entity,null,new ExecutionContext()); + } } diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/Parser.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/Parser.java new file mode 100644 index 0000000..9ed0683 --- /dev/null +++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/Parser.java @@ -0,0 +1,13 @@ +package org.apache.airavata.datalake.metadata.parsers; + +import com.google.protobuf.GeneratedMessageV3; +import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Entity; + +public interface Parser { + + public Entity parse(GeneratedMessageV3 entity, Entity parentEntity, ExecutionContext executionContext); + + public Entity parse(GeneratedMessageV3 entity, ExecutionContext executionContext); + + public Entity parse(GeneratedMessageV3 entity); +} diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/ResourceParser.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/ResourceParser.java index 6a14446..29a7a95 100644 --- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/ResourceParser.java +++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/ResourceParser.java @@ -1,6 +1,10 @@ package org.apache.airavata.datalake.metadata.parsers; +import com.google.protobuf.GeneratedMessageV3; +import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Entity; +import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group; import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Resource; +import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User; import org.apache.airavata.datalake.metadata.service.ResourceSharings; import org.dozer.DozerBeanMapper; import org.slf4j.Logger; @@ -9,69 +13,113 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.util.List; -import java.util.Optional; @Component -public class ResourceParser { +public class ResourceParser implements Parser { private static final Logger LOGGER = LoggerFactory.getLogger(ResourceParser.class); @Autowired private DozerBeanMapper dozerBeanMapper; + @Autowired + private UserParser userParser; - public Resource parseResource(org.apache.airavata.datalake.metadata.service.Resource resource, - Optional<Resource> parentResource) { - Resource newParentResource; - if (parentResource.isEmpty()) { - newParentResource = dozerBeanMapper.map(resource, - org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Resource.class); + @Autowired + private GroupParser groupParser; + + + @Override + public Entity parse(GeneratedMessageV3 entity, Entity parentEntity, ExecutionContext executionContext) { + if (entity instanceof org.apache.airavata.datalake.metadata.service.Resource) { + org.apache.airavata.datalake.metadata.service.Resource resource = + (org.apache.airavata.datalake.metadata.service.Resource) entity; + + Resource newParentResource = null; + if (parentEntity == null) { + newParentResource = dozerBeanMapper.map(resource, + org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Resource.class); + executionContext.addNeo4JConvertedModels(newParentResource.getSearchableId(), newParentResource); + } else { + newParentResource = (Resource) parentEntity; + Resource childResource = dozerBeanMapper.map(resource, + org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Resource.class); + executionContext.addNeo4JConvertedModels(newParentResource.getSearchableId(), newParentResource); + newParentResource.addChildResource(childResource, + childResource.getCreatedAt() != 0 ? childResource.getCreatedAt() : System.currentTimeMillis(), + childResource.getLastModifiedAt() != 0 ? childResource.getLastModifiedAt() : System.currentTimeMillis(), + null); // Improve this with relatioship properties + newParentResource = childResource; + } + + + List<ResourceSharings> resourceSharings = (resource).getSharingsList(); + + if (!resourceSharings.isEmpty()) { + Resource finalNewParentResource1 = newParentResource; + resourceSharings.forEach(reshr -> { + if (!reshr.getUsersList().isEmpty()) { + reshr.getUsersList().forEach(shr -> { + User user = null; + Object obj = executionContext.getNeo4JConvertedModels(shr.getUsername() + "@" + shr.getTenantId()); + if (obj != null) { + user = (User) obj; + } else { + user = (User) userParser.parse(shr, executionContext); + executionContext.addNeo4JConvertedModels(user.getSearchableId(), user); + } + finalNewParentResource1.shareWithAUser(user, + reshr.getPermissionType(), + reshr.getCreatedAt() != 0 ? reshr.getCreatedAt() : System.currentTimeMillis(), + reshr.getLastModifiedAt() != 0 ? reshr.getLastModifiedAt() : System.currentTimeMillis(), + null); + + }); + + } else if (!reshr.getGroupsList().isEmpty()) { + reshr.getGroupsList().forEach(gr -> { + Group group = null; + Object obj = executionContext.getNeo4JConvertedModels(gr.getName() + "@" + gr.getTenantId()); + if (obj != null) { + group = (Group) obj; + } else { + group = (Group) groupParser.parse(gr, executionContext); + executionContext.addNeo4JConvertedModels(group.getSearchableId(), group); + } + + LOGGER.info("Group resource " + group.toString()); + finalNewParentResource1.shareWithAGroup(group, reshr.getPermissionType(), + reshr.getCreatedAt() != 0 ? reshr.getCreatedAt() : System.currentTimeMillis(), + reshr.getLastModifiedAt() != 0 ? reshr.getLastModifiedAt() : System.currentTimeMillis(), + null); + }); + } + }); + } + + + List<org.apache.airavata.datalake.metadata.service.Resource> resources = resource.getChildResourcesList(); + + if (!resources.isEmpty()) { + Resource finalNewParentResource = newParentResource; + resources.forEach(gr -> { + this.parse(gr, finalNewParentResource, executionContext); + }); + } + return newParentResource; } else { - newParentResource = parentResource.get(); - Resource childResource = dozerBeanMapper.map(resource, - org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Resource.class); - newParentResource.addChildResource(childResource, - childResource.getCreatedAt() != 0 ? childResource.getCreatedAt() : System.currentTimeMillis(), - childResource.getLastModifiedAt() != 0 ? childResource.getLastModifiedAt() : System.currentTimeMillis(), - null); // Improve this with relatioship properties - newParentResource = childResource; - } - - - List<ResourceSharings> resourceSharings = resource.getSharingsList(); - - if (! resourceSharings.isEmpty()) { - resourceSharings.forEach(reshr-> { - if (!reshr.getUsersList().isEmpty()) { - reshr.getUsersList().forEach(shr-> { - // newParentResource.shareWithAUser(shr,reshr.get); - - }); - - } else if (!reshr.getGroupsList().isEmpty()) { - - } - - - }); - - + String msg = "Wrong entity type detected for parser Resource Parser, Expected Resource"; + LOGGER.error(msg); + throw new RuntimeException(msg); } - - - - - - List<org.apache.airavata.datalake.metadata.service.Resource> resources = resource.getChildResourcesList(); - - if (!resources.isEmpty()) { - Resource finalNewParentResource = newParentResource; - resources.forEach(gr -> { - this.parseResource(gr, Optional.of(finalNewParentResource)); - }); - } - return newParentResource; - } + @Override + public Entity parse(GeneratedMessageV3 entity, ExecutionContext executionContext) { + return this.parse(entity, null, executionContext); + } + @Override + public Entity parse(GeneratedMessageV3 entity) { + return this.parse(entity, null, new ExecutionContext()); + } } diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/TenantParser.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/TenantParser.java index c31379d..efba859 100644 --- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/TenantParser.java +++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/TenantParser.java @@ -1,6 +1,7 @@ package org.apache.airavata.datalake.metadata.parsers; -import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Tenant; +import com.google.protobuf.GeneratedMessageV3; +import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Entity; import org.apache.airavata.datalake.metadata.service.Group; import org.apache.airavata.datalake.metadata.service.Resource; import org.apache.airavata.datalake.metadata.service.User; @@ -11,10 +12,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.util.List; -import java.util.Optional; @Component -public class TenantParser { +public class TenantParser implements Parser { private static Logger LOGGER = LoggerFactory.getLogger(TenantParser.class); @Autowired @@ -29,51 +29,73 @@ public class TenantParser { @Autowired private ResourceParser resourceParser; - - public Tenant parseTenant(org.apache.airavata.datalake.metadata.service.Tenant tenant) { - List<Group> groups = tenant.getGroupsList(); - List<User> users = tenant.getUsersList(); - List<Resource> resources = tenant.getResourcesList(); - - - org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Tenant neo4JTenant = - dozerBeanMapper.map(tenant, org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Tenant.class); - - if (!groups.isEmpty()) { - groups.stream().forEach(group -> { - org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group neo4JGr = - groupParser.parseGroup(group, Optional.empty()); - neo4JTenant.add(neo4JGr, tenant.getCreatedAt() != 0 ? tenant.getCreatedAt() : System.currentTimeMillis(), - tenant.getLastModifiedAt() != 0 ? tenant.getLastModifiedAt() : System.currentTimeMillis(), - null); - - }); - } - - if (!users.isEmpty()) { - users.stream().forEach(user -> { - org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User usr = - userParser.parseUser(user); - neo4JTenant.add(usr, tenant.getCreatedAt() != 0 ? tenant.getCreatedAt() : System.currentTimeMillis(), - tenant.getLastModifiedAt() != 0 ? tenant.getLastModifiedAt() : System.currentTimeMillis(), - null); - - }); - } - - if (!resources.isEmpty()) { - resources.stream().forEach(resource -> { - org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Resource neo4JResource = - resourceParser.parseResource(resource, Optional.empty()); - neo4JTenant.add(neo4JResource, tenant.getCreatedAt() != 0 ? tenant.getCreatedAt() : System.currentTimeMillis(), - tenant.getLastModifiedAt() != 0 ? tenant.getLastModifiedAt() : System.currentTimeMillis(), - null); - - }); + @Override + public Entity parse(GeneratedMessageV3 entity, Entity parentEntity, ExecutionContext executionContext) { + if (entity instanceof org.apache.airavata.datalake.metadata.service.Tenant) { + + org.apache.airavata.datalake.metadata.service.Tenant tenant = + (org.apache.airavata.datalake.metadata.service.Tenant) entity; + + List<Group> groups = tenant.getGroupsList(); + List<User> users = tenant.getUsersList(); + List<Resource> resources = tenant.getResourcesList(); + + org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Tenant neo4JTenant = + dozerBeanMapper.map(tenant, + org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Tenant.class); + neo4JTenant.setPrimaryExternalKey(neo4JTenant.getTenantId()); + executionContext.addNeo4JConvertedModels(neo4JTenant.getSearchableId(),neo4JTenant); + + if (!groups.isEmpty()) { + groups.stream().forEach(group -> { + org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group neo4JGr = + (org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Group) + groupParser.parse(group, null, executionContext); + executionContext.addNeo4JConvertedModels(neo4JGr.getSearchableId(),neo4JGr); + neo4JTenant.add(neo4JGr, tenant.getCreatedAt() != 0 ? tenant.getCreatedAt() : System.currentTimeMillis(), + tenant.getLastModifiedAt() != 0 ? tenant.getLastModifiedAt() : System.currentTimeMillis(), + null); + }); + } + + if (!users.isEmpty()) { + users.stream().forEach(user -> { + org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User usr = + (org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User) + userParser.parse(user, null, executionContext); + executionContext.addNeo4JConvertedModels(usr.getSearchableId(),usr); + neo4JTenant.add(usr, tenant.getCreatedAt() != 0 ? tenant.getCreatedAt() : System.currentTimeMillis(), + tenant.getLastModifiedAt() != 0 ? tenant.getLastModifiedAt() : System.currentTimeMillis(), + null); + }); + } + + if (!resources.isEmpty()) { + resources.stream().forEach(resource -> { + org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Resource neo4JResource = + (org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Resource) + resourceParser.parse(resource, null, executionContext); + executionContext.addNeo4JConvertedModels(neo4JResource.getSearchableId(),neo4JResource); + neo4JTenant.add(neo4JResource, tenant.getCreatedAt() != 0 ? tenant.getCreatedAt() : System.currentTimeMillis(), + tenant.getLastModifiedAt() != 0 ? tenant.getLastModifiedAt() : System.currentTimeMillis(), + null); + }); + } + return neo4JTenant; + } else { + String msg = "Wrong entity type detected for parser Tenant Parser, Expected Tenant"; + LOGGER.error(msg); + throw new RuntimeException(msg); } - return neo4JTenant; - } + @Override + public Entity parse(GeneratedMessageV3 entity, ExecutionContext executionContext) { + return this.parse(entity, null, executionContext); + } + @Override + public Entity parse(GeneratedMessageV3 entity) { + return this.parse(entity, null, new ExecutionContext()); + } } diff --git a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/UserParser.java b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/UserParser.java index 2903381..371c316 100644 --- a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/UserParser.java +++ b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/UserParser.java @@ -1,22 +1,43 @@ package org.apache.airavata.datalake.metadata.parsers; -import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User; +import com.google.protobuf.GeneratedMessageV3; +import org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.Entity; import org.dozer.DozerBeanMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component -public class UserParser { +public class UserParser implements Parser { + + private static final Logger LOGGER = LoggerFactory.getLogger(UserParser.class); @Autowired private DozerBeanMapper dozerBeanMapper; - public User parseUser(org.apache.airavata.datalake.metadata.service.User user) { - - return dozerBeanMapper.map(user, - org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User.class); + @Override + public Entity parse(GeneratedMessageV3 entity, Entity parentEntity, ExecutionContext executionContext) { + if (entity instanceof org.apache.airavata.datalake.metadata.service.User) { + org.apache.airavata.datalake.metadata.service.User user = + (org.apache.airavata.datalake.metadata.service.User) entity; + return dozerBeanMapper.map(user, + org.apache.airavata.datalake.metadata.backend.neo4j.model.nodes.User.class); + } else { + String msg = "Wrong entity type detected for parser User Parser, Expected User"; + LOGGER.error(msg); + throw new RuntimeException(msg); + } } + @Override + public Entity parse(GeneratedMessageV3 entity, ExecutionContext executionContext) { + return this.parse(entity, null, executionContext); + } + @Override + public Entity parse(GeneratedMessageV3 entity) { + return this.parse(entity, null, new ExecutionContext()); + } } diff --git a/metadata-service/db-service/stub/src/main/proto/resource/Resource.proto b/metadata-service/db-service/stub/src/main/proto/resource/Resource.proto index 81f2e3a..ce52a04 100644 --- a/metadata-service/db-service/stub/src/main/proto/resource/Resource.proto +++ b/metadata-service/db-service/stub/src/main/proto/resource/Resource.proto @@ -47,12 +47,13 @@ message ResourceSharings { string shared_entity_id = 1; string shared_entity_type = 2; string sharing_type = 3; - map<string, string> properties = 4; - int64 created_at = 5; - int64 last_modified_at = 6; - string resource_id = 7; - repeated User users = 8; - repeated Group groups = 9; + string permission_type = 4; + map<string, string> properties = 5; + int64 created_at = 6; + int64 last_modified_at = 7; + string resource_id = 8; + repeated User users = 9; + repeated Group groups = 10; }
