This is an automated email from the ASF dual-hosted git repository. isjarana pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
commit 2056a2079209548cb09bcdf2bf72c5c163183286 Author: Dimuthu Wannipurage <[email protected]> AuthorDate: Wed Mar 24 09:37:30 2021 -0400 Implementing the storage layer with metadata and sharing support --- .gitignore | 3 + data-resource-management-service/drms-api/pom.xml | 29 ++++-- .../java/org/apache/airavata/drms/api/Client.java | 70 +++++++++++++ .../java/org/apache/airavata/drms/api/Config.java | 32 ++++++ .../apache/airavata/drms/api/DRMSApiRunner.java | 16 +++ .../drms/api/handlers/GroupServiceHandler.java | 95 ++++++++++++++++++ .../drms/api/handlers/ResourceServiceHandler.java | 16 +++ .../handlers/StoragePreferenceServiceHandler.java | 16 +++ .../drms/api/handlers/StorageServiceHandler.java | 109 ++++++++++++++++++++- .../src/main/resources/application.properties | 23 +++++ .../drms-api/src/main/resources/logback.xml | 45 +++++++++ data-resource-management-service/drms-core/pom.xml | 40 +++++++- .../apache/airavata/drms/core/Neo4JConnector.java | 66 +++++++++++++ .../drms/core/constants/StorageConstants.java | 24 +++++ .../storage/AnyStorageDeserializer.java | 81 +++++++++++++++ .../serializer/storage/AnyStorageSerializer.java | 58 +++++++++++ .../drms-stubs/pom.xml | 35 +++---- .../drms-stubs/src/main/proto/Common.proto | 17 ++++ .../drms-stubs/src/main/proto/group/Group.proto | 46 +++++++++ .../src/main/proto/group/GroupService.proto | 86 ++++++++++++++++ .../src/main/proto/preference/S3Preference.proto | 17 ++++ .../src/main/proto/preference/SSHPreference.proto | 17 ++++ .../preference/StoragePreferenceService.proto | 17 ++++ .../src/main/proto/resource/DRMSResource.proto | 17 ++++ .../main/proto/resource/DRMSResourceService.proto | 17 ++++ .../src/main/proto/sharing/Sharing.proto | 28 ++++++ .../src/main/proto/storage/S3Storage.proto | 17 ++++ .../src/main/proto/storage/SSHStorage.proto | 17 ++++ .../src/main/proto/storage/StorageService.proto | 30 ++++++ data-resource-management-service/pom.xml | 28 +++++- pom.xml | 4 + 31 files changed, 1083 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index cb794d5..f07bdb5 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,6 @@ data-orchestrator/data-orchestrator-api/target /metadata-service/metadata-service.iml /metadata-service/data-builders/data-builders.iml /metadata-service/db-service/db-service.iml +/metadata-service/db-service/client/client.iml +/metadata-service/db-service/server/server.iml +/metadata-service/db-service/stub/stub.iml diff --git a/data-resource-management-service/drms-api/pom.xml b/data-resource-management-service/drms-api/pom.xml index a6d5805..92e265b 100644 --- a/data-resource-management-service/drms-api/pom.xml +++ b/data-resource-management-service/drms-api/pom.xml @@ -1,4 +1,20 @@ <?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> @@ -15,11 +31,6 @@ <dependency> <groupId>org.apache.airavata.data.lake</groupId> <artifactId>drms-stubs</artifactId> - <version>0.01-SNAPSHOT</version> - </dependency> - <dependency> - <groupId>org.apache.airavata.data.lake</groupId> - <artifactId>stub</artifactId> <version>${project.version}</version> </dependency> <dependency> @@ -41,7 +52,7 @@ <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-web</artifactId> - <version>2.4.2</version> + <version>${spring.boot.version}</version> </dependency> <dependency> <groupId>net.sf.dozer</groupId> @@ -49,9 +60,9 @@ <version>${dozer}</version> </dependency> <dependency> - <groupId>io.grpc</groupId> - <artifactId>grpc-services</artifactId> - <version>1.25.0</version> + <groupId>org.apache.airavata.data.lake</groupId> + <artifactId>drms-core</artifactId> + <version>${project.version}</version> </dependency> </dependencies> </project> \ No newline at end of file diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Client.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Client.java new file mode 100644 index 0000000..8e589ef --- /dev/null +++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Client.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.airavata.drms.api; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import org.apache.airavata.datalake.drms.DRMSServiceAuthToken; +import org.apache.airavata.datalake.drms.groups.*; +import org.apache.airavata.datalake.drms.storage.*; + +public class Client { + public static void main(String ar[]) { + + DRMSServiceAuthToken token1 = DRMSServiceAuthToken.newBuilder().setAccessToken("Token-1").build(); + DRMSServiceAuthToken token2 = DRMSServiceAuthToken.newBuilder().setAccessToken("Token-2").build(); + + ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 6565).usePlaintext().build(); + StorageServiceGrpc.StorageServiceBlockingStub storageClient = StorageServiceGrpc.newBlockingStub(channel); + + System.out.println("List for user 1"); + StorageSearchResponse storages = storageClient.searchStorage( + StorageSearchRequest.newBuilder().setAuthToken(token1).build()); + System.out.println(storages); + + System.out.println("List for user 2"); + storages = storageClient.searchStorage( + StorageSearchRequest.newBuilder().setAuthToken(token2).build()); + System.out.println(storages); + + System.out.println("Fetch"); + StorageFetchResponse fetchResponse = storageClient.fetchStorage( + StorageFetchRequest.newBuilder().setAuthToken(token1).setStorageId("staging_pga_storage").buildPartial()); + + System.out.println(fetchResponse); + + GroupServiceGrpc.GroupServiceBlockingStub groupClient = GroupServiceGrpc.newBlockingStub(channel); + + System.out.println("User"); + FetchCurrentUserResponse currentUser = groupClient.fetchCurrentUser( + FetchCurrentUserRequest.newBuilder().setAuthToken(token1).build()); + System.out.println(currentUser); + + System.out.println("Groups"); + FetchCurrentUserGroupsResponse currentGroups = groupClient.fetchCurrentUserGroups( + FetchCurrentUserGroupsRequest.newBuilder().setAuthToken(token2).build()); + System.out.println(currentGroups); + + System.out.println("Adding metadata"); + storageClient.addStorageMetadata(AddStorageMetadataRequest.newBuilder() + .setAuthToken(token1) + .setStorageId("prod_pga") + .setKey("createdOn") + .setValue("02/15/2021") + .build()); + } +} diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Config.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Config.java index 494a21e..3ec0ab5 100644 --- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Config.java +++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Config.java @@ -1,7 +1,39 @@ package org.apache.airavata.drms.api; +import org.apache.airavata.drms.core.Neo4JConnector; +import org.springframework.context.annotation.Bean; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ import org.springframework.context.annotation.Configuration; @Configuration public class Config { + + @org.springframework.beans.factory.annotation.Value("${neo4j.server.uri}") + public String neo4jServerUri; + + @org.springframework.beans.factory.annotation.Value("${neo4j.server.user}") + public String neo4jServerUser; + + @org.springframework.beans.factory.annotation.Value("${neo4j.server.password}") + public String neo4jServerPassword; + + @Bean + public Neo4JConnector neo4JConnector() { + return new Neo4JConnector(neo4jServerUri, neo4jServerUser, neo4jServerPassword); + } } diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/DRMSApiRunner.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/DRMSApiRunner.java index 69a2203..21df7ec 100644 --- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/DRMSApiRunner.java +++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/DRMSApiRunner.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.airavata.drms.api; import org.springframework.boot.SpringApplication; diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/GroupServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/GroupServiceHandler.java new file mode 100644 index 0000000..5ad81d1 --- /dev/null +++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/GroupServiceHandler.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.airavata.drms.api.handlers; + +import io.grpc.stub.StreamObserver; +import org.apache.airavata.datalake.drms.groups.*; +import org.lognet.springboot.grpc.GRpcService; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@GRpcService +public class GroupServiceHandler extends GroupServiceGrpc.GroupServiceImplBase { + + @Override + public void fetchCurrentUser(FetchCurrentUserRequest request, StreamObserver<FetchCurrentUserResponse> responseObserver) { + try { + User user = getUserByAccessToken(request.getAuthToken().getAccessToken()); + responseObserver.onNext(FetchCurrentUserResponse.newBuilder().setUser(user).build()); + responseObserver.onCompleted(); + } catch (Exception e) { + responseObserver.onError(e); + } + } + + @Override + public void fetchUser(FetchUserRequest request, StreamObserver<FetchUserResponse> responseObserver) { + super.fetchUser(request, responseObserver); + } + + @Override + public void fetchUserGroups(FetchUserGroupsRequest request, StreamObserver<FetchUserGroupsResponse> responseObserver) { + super.fetchUserGroups(request, responseObserver); + } + + @Override + public void fetchCurrentUserGroups(FetchCurrentUserGroupsRequest request, + StreamObserver<FetchCurrentUserGroupsResponse> responseObserver) { + try { + User user = getUserByAccessToken(request.getAuthToken().getAccessToken()); + List<Group> userGroups = getUserGroups(user.getUserId()); + responseObserver.onNext(FetchCurrentUserGroupsResponse.newBuilder().addAllGroups(userGroups).build()); + responseObserver.onCompleted(); + + } catch (Exception e) { + responseObserver.onError(e); + } + } + + private User getUserByAccessToken(String accessToken) throws Exception { + switch (accessToken) { + case "Token-1": + return User.newBuilder().setUserId("user-1").setUsername("bob").setFirstName("Bob").setLastName("Leech") + .setEmailAddress("[email protected]").build(); + case "Token-2": + return User.newBuilder().setUserId("user-2").setUsername("alice").setFirstName("Alice").setLastName("Ann") + .setEmailAddress("[email protected]").build(); + } + throw new Exception("No user for given access token"); + } + + private List<Group> getUserGroups(String userId) { + Map<String, Group> groups = new HashMap<>(); + List<Group> groupList = new ArrayList<>(); + groups.put("group-1", Group.newBuilder().setGroupId("group-1").setName("Teachers").setDescription("Teachers").build()); + groups.put("group-2", Group.newBuilder().setGroupId("group-2").setName("Students").setDescription("Students").build()); + + switch (userId) { + case "user-1": + groupList.add(groups.get("group-1")); + groupList.add(groups.get("group-2")); + break; + case "user-2": + groupList.add(groups.get("group-2")); + break; + } + return groupList; + } +} diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java index ecc26a4..4e34e7a 100644 --- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java +++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.airavata.drms.api.handlers; import com.google.protobuf.Empty; diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java index 739921d..a010253 100644 --- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java +++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.airavata.drms.api.handlers; import com.google.protobuf.Empty; diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java index 47c6511..29e34e7 100644 --- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java +++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java @@ -1,21 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.airavata.drms.api.handlers; import com.google.protobuf.Empty; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; import io.grpc.stub.StreamObserver; +import org.apache.airavata.datalake.drms.DRMSServiceAuthToken; +import org.apache.airavata.datalake.drms.groups.FetchCurrentUserRequest; +import org.apache.airavata.datalake.drms.groups.FetchCurrentUserResponse; +import org.apache.airavata.datalake.drms.groups.GroupServiceGrpc; +import org.apache.airavata.datalake.drms.groups.User; import org.apache.airavata.datalake.drms.storage.*; +import org.apache.airavata.drms.core.Neo4JConnector; +import org.apache.airavata.drms.core.deserializer.storage.AnyStorageDeserializer; +import org.apache.airavata.drms.core.constants.StorageConstants; +import org.apache.airavata.drms.core.serializer.storage.AnyStorageSerializer; import org.lognet.springboot.grpc.GRpcService; +import org.neo4j.driver.Record; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Collections; +import java.util.List; +import java.util.Map; @GRpcService public class StorageServiceHandler extends StorageServiceGrpc.StorageServiceImplBase { + private static final Logger logger = LoggerFactory.getLogger(StorageServiceHandler.class); + + @Autowired + private Neo4JConnector neo4JConnector; + + @org.springframework.beans.factory.annotation.Value("${group.service.host}") + private String groupServiceHost; + + @org.springframework.beans.factory.annotation.Value("${group.service.port}") + private int groupServicePort; + + + private User getUser(DRMSServiceAuthToken authToken) { + ManagedChannel channel = ManagedChannelBuilder.forAddress(groupServiceHost, groupServicePort).usePlaintext().build(); + GroupServiceGrpc.GroupServiceBlockingStub groupClient = GroupServiceGrpc.newBlockingStub(channel); + FetchCurrentUserResponse userResponse = groupClient.fetchCurrentUser( + FetchCurrentUserRequest.newBuilder().setAuthToken(authToken).build()); + return userResponse.getUser(); + } + @Override public void fetchStorage(StorageFetchRequest request, StreamObserver<StorageFetchResponse> responseObserver) { - super.fetchStorage(request, responseObserver); + + User callUser = getUser(request.getAuthToken()); + + List<Record> records = this.neo4JConnector.searchNodes( + "MATCH (u:User)-[r1:MEMBER_OF]->(g:Group)<-[r2:SHARED_WITH]-(s:Storage) where " + + "s.storageId = '" + request.getStorageId() + "' and u.userId = '" + callUser.getUserId() + + "' return distinct s"); + + if (!records.isEmpty()) { + try { + List<AnyStorage> storageList = AnyStorageDeserializer.deserializeList(records); + responseObserver.onNext(StorageFetchResponse.newBuilder().setStorage(storageList.get(0)).build()); + responseObserver.onCompleted(); + } catch (Exception e) { + + logger.error("Errored while fetching storage with id {}", request.getStorageId(), e); + responseObserver.onError(new Exception("Errored while fetching storage with id " + request.getStorageId() + + ". Msg " + e.getMessage())); + } + } else { + logger.error("Could not find a storage with id {}", request.getStorageId()); + responseObserver.onError(new Exception("Could not find a storage with id " + request.getStorageId())); + } } @Override public void createStorage(StorageCreateRequest request, StreamObserver<StorageCreateResponse> responseObserver) { - super.createStorage(request, responseObserver); + User user = getUser(request.getAuthToken()); + AnyStorage storage = request.getStorage(); + Map<String, Object> serializedMap = AnyStorageSerializer.serializeToMap(storage); + this.neo4JConnector.createNode(serializedMap, StorageConstants.STORAGE_LABEL, user.getUserId()); } @Override @@ -29,7 +109,30 @@ public class StorageServiceHandler extends StorageServiceGrpc.StorageServiceImpl } @Override + public void addStorageMetadata(AddStorageMetadataRequest request, StreamObserver<Empty> responseObserver) { + User callUser = getUser(request.getAuthToken()); + this.neo4JConnector.createMetadataNode(StorageConstants.STORAGE_LABEL, "storageId", + request.getStorageId(), callUser.getUserId(), + request.getKey(), request.getValue()); + } + + @Override public void searchStorage(StorageSearchRequest request, StreamObserver<StorageSearchResponse> responseObserver) { - super.searchStorage(request, responseObserver); + User callUser = getUser(request.getAuthToken()); + + List<Record> records = this.neo4JConnector.searchNodes( + "MATCH (u:User)-[r1:MEMBER_OF]->(g:Group)<-[r2:SHARED_WITH]-(s:Storage) where u.userId ='" + + callUser.getUserId() + "' return distinct s"); + try { + List<AnyStorage> storageList = AnyStorageDeserializer.deserializeList(records); + StorageSearchResponse.Builder builder = StorageSearchResponse.newBuilder(); + builder.addAllStorages(storageList); + responseObserver.onNext(builder.build()); + responseObserver.onCompleted(); + + } catch (Exception e) { + logger.error("Errored while searching storages; Message: {}", e.getMessage(), e); + responseObserver.onError(e); + } } } diff --git a/data-resource-management-service/drms-api/src/main/resources/application.properties b/data-resource-management-service/drms-api/src/main/resources/application.properties new file mode 100644 index 0000000..d674f86 --- /dev/null +++ b/data-resource-management-service/drms-api/src/main/resources/application.properties @@ -0,0 +1,23 @@ + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +neo4j.server.uri=bolt://192.168.0.14:7687 +neo4j.server.user=neo4j +neo4j.server.password=123456 +group.service.host=localhost +group.service.port=6565 \ No newline at end of file diff --git a/data-resource-management-service/drms-api/src/main/resources/logback.xml b/data-resource-management-service/drms-api/src/main/resources/logback.xml new file mode 100644 index 0000000..dcf6d23 --- /dev/null +++ b/data-resource-management-service/drms-api/src/main/resources/logback.xml @@ -0,0 +1,45 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> +<configuration> + + <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <pattern>%d [%t] %-5p %c{30} %m [%X]%n</pattern> + </encoder> + </appender> + + <appender name="LOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <File>../logs/airavata.log</File> + <Append>true</Append> + <encoder> + <pattern>%d [%t] %-5p %c{30} %m [%X]%n</pattern> + </encoder> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>../logs/airavata.log.%d{yyyy-MM-dd}</fileNamePattern> + <maxHistory>30</maxHistory> + <totalSizeCap>1GB</totalSizeCap> + </rollingPolicy> + </appender> + + <logger name="ch.qos.logback" level="WARN"/> + <logger name="org.apache.airavata" level="INFO"/> + <root level="INFO"> + <appender-ref ref="CONSOLE"/> + <appender-ref ref="LOGFILE"/> + </root> +</configuration> \ No newline at end of file diff --git a/data-resource-management-service/drms-core/pom.xml b/data-resource-management-service/drms-core/pom.xml index 497ac63..a32e81f 100644 --- a/data-resource-management-service/drms-core/pom.xml +++ b/data-resource-management-service/drms-core/pom.xml @@ -1,4 +1,20 @@ <?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> @@ -15,12 +31,32 @@ <dependency> <groupId>org.neo4j</groupId> <artifactId>neo4j-ogm-core</artifactId> - <version>3.2.20</version> + <version>${neo4j.ogm.version}</version> </dependency> <dependency> <groupId>org.neo4j</groupId> <artifactId>neo4j-ogm-bolt-driver</artifactId> - <version>3.2.20</version> + <version>${neo4j.ogm.version}</version> + </dependency> + <dependency> + <groupId>org.neo4j</groupId> + <artifactId>neo4j</artifactId> + <version>${neo4j.version}</version> + </dependency> + <dependency> + <groupId>org.apache.airavata.data.lake</groupId> + <artifactId>drms-stubs</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>net.sf.dozer</groupId> + <artifactId>dozer</artifactId> + <version>${dozer}</version> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-web</artifactId> + <version>${spring.boot.version}</version> </dependency> </dependencies> diff --git a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/Neo4JConnector.java b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/Neo4JConnector.java new file mode 100644 index 0000000..1d39ce4 --- /dev/null +++ b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/Neo4JConnector.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.airavata.drms.core; + +import org.neo4j.driver.*; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Neo4JConnector { + + private String uri; + private String userName; + private String password; + + public Neo4JConnector(String uri, String userName, String password) { + this.uri = uri; + this.userName = userName; + this.password = password; + } + + public List<Record> searchNodes(String query) { + Driver driver = GraphDatabase.driver(uri, AuthTokens.basic(userName, password)); + Session session = driver.session(); + Result result = session.run(query); + return result.list(); + } + + public void createNode(Map<String, Object> properties, String label, String userId) { + Driver driver = GraphDatabase.driver(uri, AuthTokens.basic(userName, password)); + Session session = driver.session(); + Map<String, Object> parameters = new HashMap<>(); + parameters.put("props", properties); + Transaction tx = session.beginTransaction(); + Result result = tx.run("match (u:User)-[r1:MEMBER_OF {membershipType:'USER_GROUP'}]->(g:Group) where u.userId = '" + userId + "' CREATE (n:" + label + ")-[r2:SHARED_WITH {permission:'WRITE'}]->(g) SET n = $props return n", parameters); + tx.commit(); + tx.close(); + } + + public void createMetadataNode(String parentLabel, String parentIdName, String parentIdValue, + String userId, String key, String value) { + Driver driver = GraphDatabase.driver(uri, AuthTokens.basic(userName, password)); + Session session = driver.session(); + Transaction tx = session.beginTransaction(); + tx.run("match (u:User)-[r1:MEMBER_OF]->(g:Group)<-[r2:SHARED_WITH]-(s:" + parentLabel + ") where u.userId='" + userId + + "' and s." + parentIdName + "='" + parentIdValue + + "' merge (m:Metadata)<-[r3:HAS_METADATA]-(s) set m." + key + "='" + value + "' return m"); + tx.commit(); + tx.close(); + } +} diff --git a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/constants/StorageConstants.java b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/constants/StorageConstants.java new file mode 100644 index 0000000..f7d6a9d --- /dev/null +++ b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/constants/StorageConstants.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.airavata.drms.core.constants; + +public final class StorageConstants { + public static final String STORAGE_LABEL = "Storage"; + public static final String STORAGE_TYPE_LABEL = "type"; + public static final String SSH_STORAGE_TYPE_LABEL = "SSH"; + public static final String S3_STORAGE_TYPE_LABEL = "S3"; +} diff --git a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/storage/AnyStorageDeserializer.java b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/storage/AnyStorageDeserializer.java new file mode 100644 index 0000000..8d0fb95 --- /dev/null +++ b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/storage/AnyStorageDeserializer.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.airavata.drms.core.deserializer.storage; + +import org.apache.airavata.datalake.drms.storage.AnyStorage; +import org.apache.airavata.datalake.drms.storage.s3.S3Storage; +import org.apache.airavata.datalake.drms.storage.ssh.SSHStorage; +import org.apache.airavata.drms.core.constants.StorageConstants; +import org.neo4j.driver.Record; +import org.neo4j.driver.Value; +import org.neo4j.driver.internal.InternalRecord; +import org.neo4j.driver.types.Node; +import org.springframework.beans.BeanWrapper; +import org.springframework.beans.PropertyAccessorFactory; + +import java.util.*; + +public class AnyStorageDeserializer { + + public static List<AnyStorage> deserializeList(List<Record> neo4jRecords) throws Exception { + List<AnyStorage> storageList = new ArrayList<>(); + for (Record record : neo4jRecords) { + InternalRecord internalRecord = (InternalRecord) record; + List<Value> values = internalRecord.values(); + for (Value value : values) { + Node node = value.asNode(); + if (node.hasLabel(StorageConstants.STORAGE_LABEL)) { + storageList.add(deriveStorageFromMap(node.asMap())); + } + } + } + return storageList; + } + + private static AnyStorage deriveStorageFromMap(Map<String, Object> fixedMap) throws Exception { + + Map<String, Object> asMap = new HashMap<>(fixedMap); + AnyStorage.Builder anyStorageBuilder = AnyStorage.newBuilder(); + String type = (String)asMap.get(StorageConstants.STORAGE_TYPE_LABEL); + asMap.remove(StorageConstants.STORAGE_TYPE_LABEL); + + switch (type) { + case StorageConstants.SSH_STORAGE_TYPE_LABEL: + SSHStorage.Builder builder = SSHStorage.newBuilder(); + setObjectFieldsUsingMap(builder, asMap); + SSHStorage sshStorage = builder.build(); + anyStorageBuilder.setSshStorage(sshStorage); + break; + case StorageConstants.S3_STORAGE_TYPE_LABEL: + S3Storage.Builder s3Builder = S3Storage.newBuilder(); + setObjectFieldsUsingMap(s3Builder, asMap); + anyStorageBuilder.setS3Storage(s3Builder.build()); + break; + default: + throw new Exception("Unsupported storage type for deserializing : " + type); + } + + return anyStorageBuilder.build(); + } + + private static void setObjectFieldsUsingMap(Object target, Map<String, Object> values) { + for (String field :values.keySet()) { + BeanWrapper beanWrapper = PropertyAccessorFactory.forBeanPropertyAccess(target); + beanWrapper.setPropertyValue(field, values.get(field)); + } + } +} diff --git a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/serializer/storage/AnyStorageSerializer.java b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/serializer/storage/AnyStorageSerializer.java new file mode 100644 index 0000000..d286798 --- /dev/null +++ b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/serializer/storage/AnyStorageSerializer.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.airavata.drms.core.serializer.storage; + +import com.google.protobuf.Descriptors; +import org.apache.airavata.datalake.drms.storage.AnyStorage; +import org.apache.airavata.datalake.drms.storage.s3.S3Storage; +import org.apache.airavata.datalake.drms.storage.ssh.SSHStorage; +import org.apache.airavata.drms.core.constants.StorageConstants; + +import java.util.HashMap; +import java.util.Map; + +public class AnyStorageSerializer { + + public static Map<String, Object> serializeToMap(AnyStorage anyStorage) { + + Map<String, Object> fields = new HashMap<>(); + Map<Descriptors.FieldDescriptor, Object> allFields = null; + switch (anyStorage.getStorageCase()) { + case SSHSTORAGE: + SSHStorage sshStorage = anyStorage.getSshStorage(); + allFields = sshStorage.getAllFields(); + fields.put(StorageConstants.STORAGE_TYPE_LABEL, StorageConstants.SSH_STORAGE_TYPE_LABEL); + break; + case S3STORAGE: + S3Storage s3Storage = anyStorage.getS3Storage(); + allFields = s3Storage.getAllFields(); + fields.put(StorageConstants.STORAGE_TYPE_LABEL, StorageConstants.S3_STORAGE_TYPE_LABEL); + break; + case STORAGE_NOT_SET: + break; + } + + if (allFields != null) { + allFields.forEach((descriptor, value) -> { + String fieldName = descriptor.getJsonName(); + fields.put(fieldName, value); + }); + } + + return fields; + } +} diff --git a/data-resource-management-service/drms-stubs/pom.xml b/data-resource-management-service/drms-stubs/pom.xml index 915db71..abb8a3b 100644 --- a/data-resource-management-service/drms-stubs/pom.xml +++ b/data-resource-management-service/drms-stubs/pom.xml @@ -1,4 +1,20 @@ <?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> @@ -21,7 +37,7 @@ <dependency> <groupId>io.grpc</groupId> <artifactId>grpc-services</artifactId> - <version>1.35.0</version> + <version>${grpc.services}</version> <exclusions> <exclusion> <groupId>com.google.code.gson</groupId> @@ -32,7 +48,7 @@ <dependency> <groupId>javax.annotation</groupId> <artifactId>javax.annotation-api</artifactId> - <version>${javax.annotation}</version> + <version>${javax.annotation.version}</version> </dependency> </dependencies> <build> @@ -64,19 +80,4 @@ </plugin> </plugins> </build> - - <properties> - <maven.compiler.source>11</maven.compiler.source> - <maven.compiler.target>11</maven.compiler.target> - <protobuf.maven.plugin>0.5.1</protobuf.maven.plugin> - <os.maven.plugin>1.5.0.Final</os.maven.plugin> - <javax.annotation>1.3.2</javax.annotation> - <h2>1.4.191</h2> - <protobuf.java>3.10.0</protobuf.java> - <grpc.spring.boot>4.4.3</grpc.spring.boot> - <spring.boot.data.jpa>2.4.2</spring.boot.data.jpa> - <log4j.over.slf4j>1.7.26</log4j.over.slf4j> - <dozer>5.5.1</dozer> - <mariadb.jdbc>2.5.1</mariadb.jdbc> - </properties> </project> \ No newline at end of file diff --git a/data-resource-management-service/drms-stubs/src/main/proto/Common.proto b/data-resource-management-service/drms-stubs/src/main/proto/Common.proto index 3334881..8511c74 100644 --- a/data-resource-management-service/drms-stubs/src/main/proto/Common.proto +++ b/data-resource-management-service/drms-stubs/src/main/proto/Common.proto @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + syntax = "proto3"; option java_multiple_files = true; diff --git a/data-resource-management-service/drms-stubs/src/main/proto/group/Group.proto b/data-resource-management-service/drms-stubs/src/main/proto/group/Group.proto new file mode 100644 index 0000000..4515c21 --- /dev/null +++ b/data-resource-management-service/drms-stubs/src/main/proto/group/Group.proto @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.datalake.drms.groups; + +import "google/api/annotations.proto"; + +message User { + string userId = 1; + string username = 2; + string firstName = 3; + string lastName = 4; + string emailAddress = 5; +} + +message Group { + string groupId = 1; + string name = 2; + string description = 3; + repeated Group childGroups = 4; + string parentGroupId = 5; + repeated GroupMembership groupMembership = 6; +} + +message GroupMembership { + User user = 1; + string groupId = 2; + string membershipType = 3; +} \ No newline at end of file diff --git a/data-resource-management-service/drms-stubs/src/main/proto/group/GroupService.proto b/data-resource-management-service/drms-stubs/src/main/proto/group/GroupService.proto new file mode 100644 index 0000000..389d4cb --- /dev/null +++ b/data-resource-management-service/drms-stubs/src/main/proto/group/GroupService.proto @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.datalake.drms.groups; + +import "google/api/annotations.proto"; +import "group/Group.proto"; +import "Common.proto"; + +message FetchCurrentUserRequest { + org.apache.airavata.datalake.drms.DRMSServiceAuthToken authToken = 1; +} + +message FetchCurrentUserResponse { + User user = 1; +} + +message FetchUserRequest { + org.apache.airavata.datalake.drms.DRMSServiceAuthToken authToken = 1; + string userId = 2; +} + +message FetchUserResponse { + User user = 1; +} + +message FetchCurrentUserGroupsRequest { + org.apache.airavata.datalake.drms.DRMSServiceAuthToken authToken = 1; +} + +message FetchCurrentUserGroupsResponse { + repeated Group groups = 1; +} + +message FetchUserGroupsRequest { + org.apache.airavata.datalake.drms.DRMSServiceAuthToken authToken = 1; + string userId = 2; +} + +message FetchUserGroupsResponse { + repeated Group groups = 1; +} + +service GroupService { + + rpc fetchCurrentUser(FetchCurrentUserRequest) returns (FetchCurrentUserResponse) { + option (google.api.http) = { + get: "/v1.0/api/currentuser" + }; + } + + rpc fetchUser (FetchUserRequest) returns (FetchUserResponse) { + option (google.api.http) = { + get: "/v1.0/api/user" + }; + } + + rpc fetchUserGroups (FetchUserGroupsRequest) returns (FetchUserGroupsResponse) { + option (google.api.http) = { + get: "/v1.0/api/usergroups" + }; + } + + rpc fetchCurrentUserGroups (FetchCurrentUserGroupsRequest) returns (FetchCurrentUserGroupsResponse) { + option (google.api.http) = { + get: "/v1.0/api/currentusergroups" + }; + } +} \ No newline at end of file diff --git a/data-resource-management-service/drms-stubs/src/main/proto/preference/S3Preference.proto b/data-resource-management-service/drms-stubs/src/main/proto/preference/S3Preference.proto index b7eac3e..5717be6 100644 --- a/data-resource-management-service/drms-stubs/src/main/proto/preference/S3Preference.proto +++ b/data-resource-management-service/drms-stubs/src/main/proto/preference/S3Preference.proto @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + syntax = "proto3"; option java_multiple_files = true; diff --git a/data-resource-management-service/drms-stubs/src/main/proto/preference/SSHPreference.proto b/data-resource-management-service/drms-stubs/src/main/proto/preference/SSHPreference.proto index 8cf20c6..08f2b3f 100644 --- a/data-resource-management-service/drms-stubs/src/main/proto/preference/SSHPreference.proto +++ b/data-resource-management-service/drms-stubs/src/main/proto/preference/SSHPreference.proto @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + syntax = "proto3"; option java_multiple_files = true; diff --git a/data-resource-management-service/drms-stubs/src/main/proto/preference/StoragePreferenceService.proto b/data-resource-management-service/drms-stubs/src/main/proto/preference/StoragePreferenceService.proto index f8e296a..29e07c6 100644 --- a/data-resource-management-service/drms-stubs/src/main/proto/preference/StoragePreferenceService.proto +++ b/data-resource-management-service/drms-stubs/src/main/proto/preference/StoragePreferenceService.proto @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + syntax = "proto3"; option java_multiple_files = true; diff --git a/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResource.proto b/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResource.proto index 914e2ec..adf1231 100644 --- a/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResource.proto +++ b/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResource.proto @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + syntax = "proto3"; option java_multiple_files = true; diff --git a/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResourceService.proto b/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResourceService.proto index 6bd1fc7..600974a 100644 --- a/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResourceService.proto +++ b/data-resource-management-service/drms-stubs/src/main/proto/resource/DRMSResourceService.proto @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + syntax = "proto3"; option java_multiple_files = true; diff --git a/data-resource-management-service/drms-stubs/src/main/proto/sharing/Sharing.proto b/data-resource-management-service/drms-stubs/src/main/proto/sharing/Sharing.proto new file mode 100644 index 0000000..17517f3 --- /dev/null +++ b/data-resource-management-service/drms-stubs/src/main/proto/sharing/Sharing.proto @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.datalake.drms.sharing; + +import "google/api/annotations.proto"; + +message ShareStorageRequest { + string storageId = 1; + string groupId = 2; +} \ No newline at end of file diff --git a/data-resource-management-service/drms-stubs/src/main/proto/storage/S3Storage.proto b/data-resource-management-service/drms-stubs/src/main/proto/storage/S3Storage.proto index 58d4768..93b2a00 100644 --- a/data-resource-management-service/drms-stubs/src/main/proto/storage/S3Storage.proto +++ b/data-resource-management-service/drms-stubs/src/main/proto/storage/S3Storage.proto @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + syntax = "proto3"; option java_multiple_files = true; diff --git a/data-resource-management-service/drms-stubs/src/main/proto/storage/SSHStorage.proto b/data-resource-management-service/drms-stubs/src/main/proto/storage/SSHStorage.proto index d77e026..d7d8bb4 100644 --- a/data-resource-management-service/drms-stubs/src/main/proto/storage/SSHStorage.proto +++ b/data-resource-management-service/drms-stubs/src/main/proto/storage/SSHStorage.proto @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + syntax = "proto3"; option java_multiple_files = true; diff --git a/data-resource-management-service/drms-stubs/src/main/proto/storage/StorageService.proto b/data-resource-management-service/drms-stubs/src/main/proto/storage/StorageService.proto index 891962a..fca430a 100644 --- a/data-resource-management-service/drms-stubs/src/main/proto/storage/StorageService.proto +++ b/data-resource-management-service/drms-stubs/src/main/proto/storage/StorageService.proto @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + syntax = "proto3"; option java_multiple_files = true; @@ -64,6 +81,13 @@ message StorageSearchResponse { repeated AnyStorage storages = 2; } +message AddStorageMetadataRequest { + org.apache.airavata.datalake.drms.DRMSServiceAuthToken authToken = 1; + string storageId = 2; + string key = 3; + string value = 4; +} + service StorageService { rpc fetchStorage (StorageFetchRequest) returns (StorageFetchResponse) { @@ -95,4 +119,10 @@ service StorageService { post: "/v1.0/api/drms/storage/search" }; } + + rpc addStorageMetadata(AddStorageMetadataRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + post: "/v1.0/api/drms/storage/metadata" + }; + } } \ No newline at end of file diff --git a/data-resource-management-service/pom.xml b/data-resource-management-service/pom.xml index d05865a..d1c7685 100644 --- a/data-resource-management-service/pom.xml +++ b/data-resource-management-service/pom.xml @@ -1,4 +1,20 @@ <?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> @@ -12,10 +28,18 @@ <artifactId>data-resource-management-service</artifactId> <packaging>pom</packaging> <modules> - <module>drms-api</module> - <module>drms-core</module> <module>drms-stubs</module> + <module>drms-core</module> + <module>drms-api</module> </modules> + <dependencies> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>log4j-over-slf4j</artifactId> + <version>${log4j.over.slf4j}</version> + </dependency> + </dependencies> + </project> \ No newline at end of file diff --git a/pom.xml b/pom.xml index bb2d693..96bf525 100644 --- a/pom.xml +++ b/pom.xml @@ -94,6 +94,7 @@ <grpc.spring.boot>3.5.1</grpc.spring.boot> <spring.boot.data.jpa>2.2.1.RELEASE</spring.boot.data.jpa> <dozer>5.5.1</dozer> + <grpc.services>1.25.0</grpc.services> <!-- Spring dependencies --> <spring.boot.version>2.4.2</spring.boot.version> @@ -146,7 +147,10 @@ <com.codahale.version>0.7.0</com.codahale.version> + <neo4j.ogm.version>3.2.20</neo4j.ogm.version> + <neo4j.version>3.4.6</neo4j.version> + <log4j.over.slf4j>1.7.26</log4j.over.slf4j> </properties> </project>
