This is an automated email from the ASF dual-hosted git repository. isjarana pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
commit d1942b4710245ff5c691cef5f5a00c0512adf697 Author: Dimuthu Wannipurage <[email protected]> AuthorDate: Wed Mar 24 10:58:17 2021 -0400 Implementing storage preference fetch and search functionalities --- .../handlers/StoragePreferenceServiceHandler.java | 78 +++++++++++++++++- .../drms/api/handlers/StorageServiceHandler.java | 5 +- .../constants/StoragePreferenceConstants.java} | 24 ++---- .../{storage => }/AnyStorageDeserializer.java | 4 +- .../AnyStoragePreferenceDeserializer.java | 93 ++++++++++++++++++++++ .../{storage => }/AnyStorageSerializer.java | 2 +- .../src/main/proto/preference/S3Preference.proto | 6 +- .../src/main/proto/preference/SSHPreference.proto | 6 +- 8 files changed, 187 insertions(+), 31 deletions(-) diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java index a010253..3d7b44a 100644 --- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java +++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java @@ -17,16 +17,75 @@ package org.apache.airavata.drms.api.handlers; import com.google.protobuf.Empty; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; import io.grpc.stub.StreamObserver; +import org.apache.airavata.datalake.drms.DRMSServiceAuthToken; +import org.apache.airavata.datalake.drms.groups.FetchCurrentUserRequest; +import org.apache.airavata.datalake.drms.groups.FetchCurrentUserResponse; +import org.apache.airavata.datalake.drms.groups.GroupServiceGrpc; +import org.apache.airavata.datalake.drms.groups.User; import org.apache.airavata.datalake.drms.storage.*; +import org.apache.airavata.drms.core.Neo4JConnector; +import org.apache.airavata.drms.core.deserializer.AnyStorageDeserializer; +import org.apache.airavata.drms.core.deserializer.AnyStoragePreferenceDeserializer; import org.lognet.springboot.grpc.GRpcService; +import org.neo4j.driver.Record; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.List; @GRpcService public class StoragePreferenceServiceHandler extends StoragePreferenceServiceGrpc.StoragePreferenceServiceImplBase { + private static final Logger logger = LoggerFactory.getLogger(StoragePreferenceServiceHandler.class); + + @Autowired + private Neo4JConnector neo4JConnector; + + @org.springframework.beans.factory.annotation.Value("${group.service.host}") + private String groupServiceHost; + + @org.springframework.beans.factory.annotation.Value("${group.service.port}") + private int groupServicePort; + + + private User getUser(DRMSServiceAuthToken authToken) { + ManagedChannel channel = ManagedChannelBuilder.forAddress(groupServiceHost, groupServicePort).usePlaintext().build(); + GroupServiceGrpc.GroupServiceBlockingStub groupClient = GroupServiceGrpc.newBlockingStub(channel); + FetchCurrentUserResponse userResponse = groupClient.fetchCurrentUser( + FetchCurrentUserRequest.newBuilder().setAuthToken(authToken).build()); + return userResponse.getUser(); + } + @Override public void fetchStoragePreference(StoragePreferenceFetchRequest request, StreamObserver<StoragePreferenceFetchResponse> responseObserver) { - super.fetchStoragePreference(request, responseObserver); + User callUser = getUser(request.getAuthToken()); + + List<Record> records = this.neo4JConnector.searchNodes( + "MATCH (u:User)-[r1:MEMBER_OF]->(g:Group)<-[r2:SHARED_WITH]-(s:Storage)-[r3:HAS_PREFERENCE]->(sp:StoragePreference) " + + "where sp.storagePreferenceId = '" + request.getStoragePreferenceId() + "' and u.userId = '" + + callUser.getUserId() + "' return distinct sp, s"); + + if (!records.isEmpty()) { + try { + List<AnyStoragePreference> storagePrefList = AnyStoragePreferenceDeserializer.deserializeList(records); + responseObserver.onNext(StoragePreferenceFetchResponse.newBuilder().setStoragePreference( + storagePrefList.get(0)).build()); + responseObserver.onCompleted(); + } catch (Exception e) { + + logger.error("Errored while fetching storage preference with id {}", request.getStoragePreferenceId(), e); + responseObserver.onError(new Exception("Errored while fetching storage preference with id " + + request.getStoragePreferenceId() + ". Msg " + e.getMessage())); + } + } else { + logger.error("Could not find a storage preference with id {}", request.getStoragePreferenceId()); + responseObserver.onError(new Exception("Could not find a storage preference with id " + + request.getStoragePreferenceId())); + } } @Override @@ -46,6 +105,21 @@ public class StoragePreferenceServiceHandler extends StoragePreferenceServiceGrp @Override public void searchStoragePreference(StoragePreferenceSearchRequest request, StreamObserver<StoragePreferenceSearchResponse> responseObserver) { - super.searchStoragePreference(request, responseObserver); + User callUser = getUser(request.getAuthToken()); + + List<Record> records = this.neo4JConnector.searchNodes( + "MATCH (u:User)-[r1:MEMBER_OF]->(g:Group)<-[r2:SHARED_WITH]-(s:Storage)-[r3:HAS_PREFERENCE]->(sp:StoragePreference)" + + " where u.userId ='" + callUser.getUserId() + "' return distinct sp, s"); + try { + List<AnyStoragePreference> storagePrefList = AnyStoragePreferenceDeserializer.deserializeList(records); + StoragePreferenceSearchResponse.Builder builder = StoragePreferenceSearchResponse.newBuilder(); + builder.addAllStoragesPreference(storagePrefList); + responseObserver.onNext(builder.build()); + responseObserver.onCompleted(); + + } catch (Exception e) { + logger.error("Errored while searching storage preferences; Message: {}", e.getMessage(), e); + responseObserver.onError(e); + } } } diff --git a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java index 29e34e7..a806e70 100644 --- a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java +++ b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StorageServiceHandler.java @@ -27,16 +27,15 @@ import org.apache.airavata.datalake.drms.groups.GroupServiceGrpc; import org.apache.airavata.datalake.drms.groups.User; import org.apache.airavata.datalake.drms.storage.*; import org.apache.airavata.drms.core.Neo4JConnector; -import org.apache.airavata.drms.core.deserializer.storage.AnyStorageDeserializer; +import org.apache.airavata.drms.core.deserializer.AnyStorageDeserializer; import org.apache.airavata.drms.core.constants.StorageConstants; -import org.apache.airavata.drms.core.serializer.storage.AnyStorageSerializer; +import org.apache.airavata.drms.core.serializer.AnyStorageSerializer; import org.lognet.springboot.grpc.GRpcService; import org.neo4j.driver.Record; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import java.util.Collections; import java.util.List; import java.util.Map; diff --git a/data-resource-management-service/drms-stubs/src/main/proto/preference/S3Preference.proto b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/constants/StoragePreferenceConstants.java similarity index 63% copy from data-resource-management-service/drms-stubs/src/main/proto/preference/S3Preference.proto copy to data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/constants/StoragePreferenceConstants.java index 5717be6..e819c2b 100644 --- a/data-resource-management-service/drms-stubs/src/main/proto/preference/S3Preference.proto +++ b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/constants/StoragePreferenceConstants.java @@ -15,21 +15,11 @@ * limitations under the License. */ -syntax = "proto3"; +package org.apache.airavata.drms.core.constants; -option java_multiple_files = true; -package org.apache.airavata.datalake.drms.storage.preference.s3; - -import "google/api/annotations.proto"; -import "storage/S3Storage.proto"; - -message S3StoragePreference { - string storagePreferenceId = 1; - string credentialToken = 2; - enum AuthType { - CLIENT_ID_SECRET_ID = 0; - OPEN = 2; - } - AuthType authType = 3; - org.apache.airavata.datalake.drms.storage.s3.S3Storage storage = 4; -} \ No newline at end of file +public final class StoragePreferenceConstants { + public static final String STORAGE_PREFERENCE_LABEL = "StoragePreference"; + public static final String STORAGE_PREFERENCE_TYPE_LABEL = "type"; + public static final String SSH_STORAGE_PREFERENCE_TYPE_LABEL = "SSH"; + public static final String S3_STORAGE_PREFERENCE_TYPE_LABEL = "S3"; +} diff --git a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/storage/AnyStorageDeserializer.java b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/AnyStorageDeserializer.java similarity index 95% rename from data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/storage/AnyStorageDeserializer.java rename to data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/AnyStorageDeserializer.java index 8d0fb95..4edf2da 100644 --- a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/storage/AnyStorageDeserializer.java +++ b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/AnyStorageDeserializer.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.airavata.drms.core.deserializer.storage; +package org.apache.airavata.drms.core.deserializer; import org.apache.airavata.datalake.drms.storage.AnyStorage; import org.apache.airavata.datalake.drms.storage.s3.S3Storage; @@ -46,7 +46,7 @@ public class AnyStorageDeserializer { return storageList; } - private static AnyStorage deriveStorageFromMap(Map<String, Object> fixedMap) throws Exception { + public static AnyStorage deriveStorageFromMap(Map<String, Object> fixedMap) throws Exception { Map<String, Object> asMap = new HashMap<>(fixedMap); AnyStorage.Builder anyStorageBuilder = AnyStorage.newBuilder(); diff --git a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/AnyStoragePreferenceDeserializer.java b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/AnyStoragePreferenceDeserializer.java new file mode 100644 index 0000000..92159e0 --- /dev/null +++ b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/AnyStoragePreferenceDeserializer.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.drms.core.deserializer; + +import org.apache.airavata.datalake.drms.storage.AnyStorage; +import org.apache.airavata.datalake.drms.storage.AnyStoragePreference; +import org.apache.airavata.datalake.drms.storage.preference.s3.S3StoragePreference; +import org.apache.airavata.datalake.drms.storage.preference.ssh.SSHStoragePreference; +import org.apache.airavata.drms.core.constants.StorageConstants; +import org.apache.airavata.drms.core.constants.StoragePreferenceConstants; +import org.neo4j.driver.Record; +import org.neo4j.driver.Value; +import org.neo4j.driver.internal.InternalRecord; +import org.neo4j.driver.types.Node; +import org.springframework.beans.BeanWrapper; +import org.springframework.beans.PropertyAccessorFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class AnyStoragePreferenceDeserializer { + public static List<AnyStoragePreference> deserializeList(List<Record> neo4jRecords) throws Exception { + List<AnyStoragePreference> storagePrefList = new ArrayList<>(); + for (Record record : neo4jRecords) { + InternalRecord internalRecord = (InternalRecord) record; + List<Value> values = internalRecord.values(); + if (values.size() == 2) { + Value prfValue = values.get(0); + Value stoValue = values.get(1); + Node prefNode = prfValue.asNode(); + Node stoNode = stoValue.asNode(); + if (prefNode.hasLabel(StoragePreferenceConstants.STORAGE_PREFERENCE_LABEL) && stoNode.hasLabel(StorageConstants.STORAGE_LABEL)) { + AnyStorage storage = AnyStorageDeserializer.deriveStorageFromMap(stoNode.asMap()); + AnyStoragePreference preference = deriveStoragePrefFromMap(prefNode.asMap(), storage); + storagePrefList.add(preference); + } + } + } + return storagePrefList; + } + + public static AnyStoragePreference deriveStoragePrefFromMap(Map<String, Object> fixedMap, AnyStorage anyStorage) throws Exception { + + Map<String, Object> asMap = new HashMap<>(fixedMap); + AnyStoragePreference.Builder anyStoragePrefBuilder = AnyStoragePreference.newBuilder(); + String type = (String)asMap.get(StoragePreferenceConstants.STORAGE_PREFERENCE_TYPE_LABEL); + asMap.remove(StoragePreferenceConstants.STORAGE_PREFERENCE_TYPE_LABEL); + + switch (type) { + case StoragePreferenceConstants.SSH_STORAGE_PREFERENCE_TYPE_LABEL: + SSHStoragePreference.Builder builder = SSHStoragePreference.newBuilder(); + setObjectFieldsUsingMap(builder, asMap); + builder.setStorage(anyStorage.getSshStorage()); + SSHStoragePreference sshStoragePreference = builder.build(); + anyStoragePrefBuilder.setSshStoragePreference(sshStoragePreference); + break; + case StoragePreferenceConstants.S3_STORAGE_PREFERENCE_TYPE_LABEL: + S3StoragePreference.Builder s3Builder = S3StoragePreference.newBuilder(); + setObjectFieldsUsingMap(s3Builder, asMap); + s3Builder.setStorage(anyStorage.getS3Storage()); + anyStoragePrefBuilder.setS3StoragePreference(s3Builder.build()); + break; + default: + throw new Exception("Unsupported storage type for deserializing : " + type); + } + + return anyStoragePrefBuilder.build(); + } + + private static void setObjectFieldsUsingMap(Object target, Map<String, Object> values) { + for (String field :values.keySet()) { + BeanWrapper beanWrapper = PropertyAccessorFactory.forBeanPropertyAccess(target); + beanWrapper.setPropertyValue(field, values.get(field)); + } + } +} diff --git a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/serializer/storage/AnyStorageSerializer.java b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/serializer/AnyStorageSerializer.java similarity index 97% rename from data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/serializer/storage/AnyStorageSerializer.java rename to data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/serializer/AnyStorageSerializer.java index d286798..5f6d583 100644 --- a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/serializer/storage/AnyStorageSerializer.java +++ b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/serializer/AnyStorageSerializer.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.airavata.drms.core.serializer.storage; +package org.apache.airavata.drms.core.serializer; import com.google.protobuf.Descriptors; import org.apache.airavata.datalake.drms.storage.AnyStorage; diff --git a/data-resource-management-service/drms-stubs/src/main/proto/preference/S3Preference.proto b/data-resource-management-service/drms-stubs/src/main/proto/preference/S3Preference.proto index 5717be6..850b295 100644 --- a/data-resource-management-service/drms-stubs/src/main/proto/preference/S3Preference.proto +++ b/data-resource-management-service/drms-stubs/src/main/proto/preference/S3Preference.proto @@ -26,10 +26,10 @@ import "storage/S3Storage.proto"; message S3StoragePreference { string storagePreferenceId = 1; string credentialToken = 2; - enum AuthType { + /*enum AuthType { CLIENT_ID_SECRET_ID = 0; OPEN = 2; - } - AuthType authType = 3; + }*/ + string authType = 3; org.apache.airavata.datalake.drms.storage.s3.S3Storage storage = 4; } \ No newline at end of file diff --git a/data-resource-management-service/drms-stubs/src/main/proto/preference/SSHPreference.proto b/data-resource-management-service/drms-stubs/src/main/proto/preference/SSHPreference.proto index 08f2b3f..f071151 100644 --- a/data-resource-management-service/drms-stubs/src/main/proto/preference/SSHPreference.proto +++ b/data-resource-management-service/drms-stubs/src/main/proto/preference/SSHPreference.proto @@ -27,11 +27,11 @@ message SSHStoragePreference { string storagePreferenceId = 1; string credentialToken = 2; string userName = 3; - enum AuthType { + /*enum AuthType { SSH_KEY = 0; PASSWORD = 1; OPEN = 2; - } - AuthType authType = 4; + }*/ + string authType = 4; org.apache.airavata.datalake.drms.storage.ssh.SSHStorage storage = 5; } \ No newline at end of file
