[airavata-custos] branch develop updated: Improve get user by token api
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata-custos.git
The following commit(s) were added to refs/heads/develop by this push:
new ddb5487 Improve get user by token api
new 9b6434e Merge pull request #144 from isururanawaka/mft-related-changes
ddb5487 is described below
commit ddb54871d188f74d870e3bcb2e41049fc6a73d0e
Author: Isuru Ranawaka
AuthorDate: Thu Mar 25 21:39:33 2021 -0400
Improve get user by token api
---
.../client/IdentityManagementClient.java | 8 +
.../interceptors/AuthInterceptorImpl.java | 36 +-
.../service/TenantManagementService.java | 1 -
3 files changed, 9 insertions(+), 36 deletions(-)
diff --git
a/custos-client-sdks/custos-java-clients/identity-management-client/src/main/java/org/apache/custos/identity/management/client/IdentityManagementClient.java
b/custos-client-sdks/custos-java-clients/identity-management-client/src/main/java/org/apache/custos/identity/management/client/IdentityManagementClient.java
index aaf377a..65f74b4 100644
---
a/custos-client-sdks/custos-java-clients/identity-management-client/src/main/java/org/apache/custos/identity/management/client/IdentityManagementClient.java
+++
b/custos-client-sdks/custos-java-clients/identity-management-client/src/main/java/org/apache/custos/identity/management/client/IdentityManagementClient.java
@@ -190,6 +190,14 @@ public class IdentityManagementClient implements Closeable
{
}
+public User getUser(String accessToken) {
+AuthToken authToken = AuthToken.newBuilder()
+.setAccessToken(accessToken)
+.build();
+return blockingStub.getUser(authToken);
+
+}
+
public boolean isAuthenticated(String accessToken) {
try {
AuthToken authToken = AuthToken
diff --git
a/custos-integration-services/identity-management-service-parent/identity-management-service/src/main/java/org/apache/custos/identity/management/interceptors/AuthInterceptorImpl.java
b/custos-integration-services/identity-management-service-parent/identity-management-service/src/main/java/org/apache/custos/identity/management/interceptors/AuthInterceptorImpl.java
index a6145d6..2164eb7 100644
---
a/custos-integration-services/identity-management-service-parent/identity-management-service/src/main/java/org/apache/custos/identity/management/interceptors/AuthInterceptorImpl.java
+++
b/custos-integration-services/identity-management-service-parent/identity-management-service/src/main/java/org/apache/custos/identity/management/interceptors/AuthInterceptorImpl.java
@@ -25,7 +25,6 @@ import org.apache.custos.credential.store.service.Credentials;
import org.apache.custos.identity.client.IdentityClient;
import org.apache.custos.identity.management.service.EndSessionRequest;
import org.apache.custos.identity.management.service.GetCredentialsRequest;
-import org.apache.custos.identity.management.utils.Constants;
import org.apache.custos.identity.service.AuthToken;
import org.apache.custos.identity.service.AuthenticationRequest;
import org.apache.custos.identity.service.Claim;
@@ -74,7 +73,7 @@ public class AuthInterceptorImpl extends
MultiTenantAuthInterceptor {
.build();
return (ReqT) reqCore;
-} else if (method.equals("isAuthenticated")) {
+} else if (method.equals("isAuthenticated") ||
method.equals("getUser")) {
AuthClaim claim = authorize(headers);
if (claim == null) {
throw new UnAuthorizedException("Request is not authorized",
null);
@@ -97,39 +96,6 @@ public class AuthInterceptorImpl extends
MultiTenantAuthInterceptor {
return (ReqT) authzBuilder.build();
-} else if (method.equals("getUser")) {
-
-AuthToken authToken = ((AuthToken) reqT);
-String clientId = null;
-for (Claim claim : authToken.getClaimsList()) {
-if (claim.getKey().equals(Constants.CLIENT_ID)) {
-clientId = claim.getValue();
-}
-}
-
-String username = null;
-for (Claim claims : ((AuthToken) reqT).getClaimsList()) {
-if (claims.getKey().equals("username")) {
-username = claims.getValue();
-}
-}
-
-String accessToken = ((AuthToken) reqT).getAccessToken();
-AuthToken.Builder authzBuilder = AuthToken.newBuilder()
-.setAccessToken(accessToken);
-AuthClaim claim = authorize(headers, clientId);
-if (claim == null) {
-throw new UnAuthorizedException("Request is not authorized",
null);
-}
-
-long tenantId = claim.getTenantId();
-
-Claim userClaim =
[airavata-data-lake] 02/19: Initial project structure for data orchestrator
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
commit 1eb8a81b7e8fe7de1d4589833ad5358ea85195b2
Author: Dimuthu Wannipurage
AuthorDate: Fri Dec 11 04:19:07 2020 -0500
Initial project structure for data orchestrator
---
.gitignore | 5 ++
data-orchestrator/data-orchestrator-api/pom.xml| 15 ++
.../datalake/orchestrator/api/Application.java | 26 ++
data-orchestrator/data-orchestrator-core/pom.xml | 15 ++
.../orchestrator/core/task/AbstractTask.java | 26 ++
data-orchestrator/pom.xml | 20
pom.xml| 55 ++
7 files changed, 162 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 000..5559ec0
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,5 @@
+.idea/*
+airavata-data-lake.iml
+data-orchestrator/data-orchestrator.iml
+data-orchestrator/data-orchestrator-api/data-orchestrator-api.iml
+data-orchestrator/data-orchestrator-core/data-orchestrator-core.iml
\ No newline at end of file
diff --git a/data-orchestrator/data-orchestrator-api/pom.xml
b/data-orchestrator/data-orchestrator-api/pom.xml
new file mode 100644
index 000..720bd6c
--- /dev/null
+++ b/data-orchestrator/data-orchestrator-api/pom.xml
@@ -0,0 +1,15 @@
+
+http://maven.apache.org/POM/4.0.0;
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+
+data-orchestrator
+org.apache.airavata.data.lake
+0.01-SNAPSHOT
+
+4.0.0
+
+data-orchestrator-api
+
+
+
\ No newline at end of file
diff --git
a/data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/Application.java
b/data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/Application.java
new file mode 100644
index 000..dc91369
--- /dev/null
+++
b/data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/Application.java
@@ -0,0 +1,26 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.airavata.datalake.orchestrator.api;
+
+/**
+ * TODO: Spring Boot API
+ */
+public class Application {
+}
diff --git a/data-orchestrator/data-orchestrator-core/pom.xml
b/data-orchestrator/data-orchestrator-core/pom.xml
new file mode 100644
index 000..583e58a
--- /dev/null
+++ b/data-orchestrator/data-orchestrator-core/pom.xml
@@ -0,0 +1,15 @@
+
+http://maven.apache.org/POM/4.0.0;
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+
+data-orchestrator
+org.apache.airavata.data.lake
+0.01-SNAPSHOT
+
+4.0.0
+
+data-orchestrator-core
+
+
+
\ No newline at end of file
diff --git
a/data-orchestrator/data-orchestrator-core/src/main/java/org/apache/airavata/datalake/orchestrator/core/task/AbstractTask.java
b/data-orchestrator/data-orchestrator-core/src/main/java/org/apache/airavata/datalake/orchestrator/core/task/AbstractTask.java
new file mode 100644
index 000..1072614
--- /dev/null
+++
b/data-orchestrator/data-orchestrator-core/src/main/java/org/apache/airavata/datalake/orchestrator/core/task/AbstractTask.java
@@ -0,0 +1,26 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
[airavata-data-lake] 04/19: Showing the latest build status
This is an automated email from the ASF dual-hosted git repository. isjarana pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git commit d760234496163a8064b77373cfe762a4367b Author: Dimuthu Wannipurage AuthorDate: Fri Dec 11 04:26:20 2020 -0500 Showing the latest build status --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fd06c5a..2c0476f 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ # Apache Airavata Data Lake [](https://apache.org/licenses/LICENSE-2.0) +[](https://travis-ci.org/apache/airavata-data-lake) Apache Airavata use cases enable capture of data from observational and experimental instruments and computations resulting from Airavata's orchestration capabilities. As the data deluges into vast amounts, harvesting the data, capturing metadata, presenting it for easy and controlled access becomes unmanageable.
[airavata-data-lake] 03/19: Enabling the travis build
This is an automated email from the ASF dual-hosted git repository. isjarana pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git commit 10266978c902a3db2b7ae540f9563e373175eb97 Author: Dimuthu Wannipurage AuthorDate: Fri Dec 11 04:22:36 2020 -0500 Enabling the travis build --- .gitignore | 6 +- .travis.yml | 36 pom.xml | 15 ++- 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 5559ec0..753237f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,8 @@ airavata-data-lake.iml data-orchestrator/data-orchestrator.iml data-orchestrator/data-orchestrator-api/data-orchestrator-api.iml -data-orchestrator/data-orchestrator-core/data-orchestrator-core.iml \ No newline at end of file +data-orchestrator/data-orchestrator-core/data-orchestrator-core.iml +target/ +data-orchestrator/target +data-orchestrator/data-orchestrator-core/target +data-orchestrator/data-orchestrator-api/target \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000..9f37705 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +language: java + +sudo: false + +notifications: + email: +on_success: change +on_failure: always + +jdk: + - oraclejdk11 + +script: mvn clean install + +# Git-describe Maven plugin needs the full history +git: + depth: false diff --git a/pom.xml b/pom.xml index b3e53b6..48bcf01 100644 --- a/pom.xml +++ b/pom.xml @@ -50,6 +50,19 @@ https://issues.apache.org/jira/browse/AIRAVATA - + + + +org.apache.maven.plugins +maven-compiler-plugin +3.6.1 + +11 +11 +true + + + +
[airavata-data-lake] 13/19: Creating the grpc service handlers for DRMS API
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
commit e0d87c1bd01dc147a7534c51bb144d59f6e86f82
Author: DImuthuUpe
AuthorDate: Mon Mar 22 23:26:36 2021 -0400
Creating the grpc service handlers for DRMS API
---
.gitignore | 2 +
data-resource-management-service/drms-api/pom.xml | 44 +-
.../java/org/apache/airavata/drms/api/Config.java | 7
.../apache/airavata/drms/api/DRMSApiRunner.java| 13 +++
.../drms/api/handlers/ResourceServiceHandler.java | 34 +
.../handlers/StoragePreferenceServiceHandler.java | 35 +
.../drms/api/handlers/StorageServiceHandler.java | 35 +
data-resource-management-service/drms-core/pom.xml | 12 ++
metadata-service/pom.xml | 14 ---
pom.xml| 8
10 files changed, 189 insertions(+), 15 deletions(-)
diff --git a/.gitignore b/.gitignore
index 66ff5f6..cb794d5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,5 @@ data-orchestrator/data-orchestrator-api/target
/data-resource-management-service/drms-api/drms-api.iml
/data-resource-management-service/drms-core/drms-core.iml
/metadata-service/metadata-service.iml
+/metadata-service/data-builders/data-builders.iml
+/metadata-service/db-service/db-service.iml
diff --git a/data-resource-management-service/drms-api/pom.xml
b/data-resource-management-service/drms-api/pom.xml
index 2839b3b..a6d5805 100644
--- a/data-resource-management-service/drms-api/pom.xml
+++ b/data-resource-management-service/drms-api/pom.xml
@@ -11,5 +11,47 @@
drms-api
-
+
+
+org.apache.airavata.data.lake
+drms-stubs
+0.01-SNAPSHOT
+
+
+org.apache.airavata.data.lake
+stub
+${project.version}
+
+
+com.google.protobuf
+protobuf-java
+${protobuf.java}
+
+
+io.github.lognet
+grpc-spring-boot-starter
+${grpc.spring.boot}
+
+
+org.springframework.boot
+spring-boot-starter
+
+
+
+
+org.springframework.boot
+spring-boot-starter-web
+2.4.2
+
+
+net.sf.dozer
+dozer
+${dozer}
+
+
+io.grpc
+grpc-services
+1.25.0
+
+
\ No newline at end of file
diff --git
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Config.java
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Config.java
new file mode 100644
index 000..494a21e
--- /dev/null
+++
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Config.java
@@ -0,0 +1,7 @@
+package org.apache.airavata.drms.api;
+
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class Config {
+}
diff --git
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/DRMSApiRunner.java
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/DRMSApiRunner.java
new file mode 100644
index 000..69a2203
--- /dev/null
+++
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/DRMSApiRunner.java
@@ -0,0 +1,13 @@
+package org.apache.airavata.drms.api;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.ComponentScan;
+
+@ComponentScan(basePackages = {"org.apache.airavata.drms"})
+@SpringBootApplication
+public class DRMSApiRunner {
+public static void main(String[] args) {
+SpringApplication.run(DRMSApiRunner.class, args);
+}
+}
diff --git
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
new file mode 100644
index 000..ecc26a4
--- /dev/null
+++
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
@@ -0,0 +1,34 @@
+package org.apache.airavata.drms.api.handlers;
+
+import com.google.protobuf.Empty;
+import io.grpc.stub.StreamObserver;
+import org.apache.airavata.datalake.drms.storage.*;
+import org.lognet.springboot.grpc.GRpcService;
+
+@GRpcService
+public class ResourceServiceHandler extends
ResourceServiceGrpc.ResourceServiceImplBase {
+@Override
+public void
[airavata-data-lake] 19/19: Merge pull request #2 from isururanawaka/metadata_service
This is an automated email from the ASF dual-hosted git repository. isjarana pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git commit 1909dca870c31aaf33c29fda10def3c80e23af29 Merge: f368372 dc26af3 Author: Isuru Ranawaka AuthorDate: Thu Mar 25 12:46:22 2021 -0400 Merge pull request #2 from isururanawaka/metadata_service Add parser interface, remove duplicate object creation at Neo4J .../airavata/datalake/metadata/clients/Test.java | 70 - .../airavata/datalake/metadata/AppConfig.java | 41 -- .../datalake/metadata/Neo4JServiceInitializer.java | 2 + .../neo4j/curd/operators/GenericService.java | 14 +- .../backend/neo4j/curd/operators/GroupService.java | 1 + .../neo4j/curd/operators/GroupServiceImpl.java | 28 .../neo4j/curd/operators/ResourceService.java | 4 + .../neo4j/curd/operators/ResourceServiceImpl.java | 52 ++- .../backend/neo4j/curd/operators/Service.java | 6 +- .../neo4j/curd/operators/TenantService.java| 2 + .../neo4j/curd/operators/TenantServiceImpl.java| 34 + .../backend/neo4j/curd/operators/UserService.java | 1 + .../neo4j/curd/operators/UserServiceImpl.java | 28 .../metadata/backend/neo4j/model/nodes/Entity.java | 19 +++ .../metadata/backend/neo4j/model/nodes/Group.java | 5 + .../backend/neo4j/model/nodes/Resource.java| 5 + .../metadata/backend/neo4j/model/nodes/Tenant.java | 5 + .../metadata/backend/neo4j/model/nodes/User.java | 5 + .../metadata/handlers/GroupServiceHandler.java | 79 -- .../metadata/handlers/ResourceServiceHandler.java | 111 -- .../metadata/handlers/TenantServiceHandler.java| 77 -- .../metadata/handlers/UserServiceHandler.java | 48 ++- .../metadata/interceptors/Authenticator.java | 19 +++ .../interceptors/InterceptorPipelineExecutor.java | 69 + .../metadata/interceptors/ServiceInterceptor.java | 7 + .../datalake/metadata/mergers/GenericMerger.java | 46 ++ .../airavata/datalake/metadata/mergers/Merger.java | 9 ++ .../metadata/parsers/ExecutionContext.java | 21 +++ .../datalake/metadata/parsers/GroupParser.java | 111 -- .../airavata/datalake/metadata/parsers/Parser.java | 16 +++ .../datalake/metadata/parsers/ResourceParser.java | 160 ++--- .../datalake/metadata/parsers/TenantParser.java| 126 ++-- .../datalake/metadata/parsers/UserParser.java | 39 - .../stub/src/main/proto/resource/Resource.proto| 33 - .../stub/src/main/proto/tenant/Tenant.proto| 6 +- 35 files changed, 1095 insertions(+), 204 deletions(-)
[airavata-data-lake] 12/19: Add parser interface, remove duplicate object creation at Neo4J
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
commit 4e9880d1d259859b05e881df39d3f73b594ca7cd
Author: Isuru Ranawaka
AuthorDate: Mon Mar 22 16:03:15 2021 -0400
Add parser interface, remove duplicate object creation at Neo4J
---
.../airavata/datalake/metadata/clients/Test.java | 39 +-
.../airavata/datalake/metadata/AppConfig.java | 52 ---
.../metadata/backend/neo4j/model/nodes/Entity.java | 7 +
.../metadata/backend/neo4j/model/nodes/Group.java | 5 +
.../backend/neo4j/model/nodes/Resource.java| 5 +
.../metadata/backend/neo4j/model/nodes/Tenant.java | 5 +
.../metadata/backend/neo4j/model/nodes/User.java | 5 +
.../metadata/handlers/TenantServiceHandler.java| 3 +-
.../metadata/parsers/ExecutionContext.java | 17 +++
.../datalake/metadata/parsers/GroupParser.java | 106 --
.../airavata/datalake/metadata/parsers/Parser.java | 13 ++
.../datalake/metadata/parsers/ResourceParser.java | 152 ++---
.../datalake/metadata/parsers/TenantParser.java| 114 +---
.../datalake/metadata/parsers/UserParser.java | 33 -
.../stub/src/main/proto/resource/Resource.proto| 13 +-
15 files changed, 389 insertions(+), 180 deletions(-)
diff --git
a/metadata-service/db-service/client/src/main/java/org/apache/airavata/datalake/metadata/clients/Test.java
b/metadata-service/db-service/client/src/main/java/org/apache/airavata/datalake/metadata/clients/Test.java
index dd8b392..a506d75 100644
---
a/metadata-service/db-service/client/src/main/java/org/apache/airavata/datalake/metadata/clients/Test.java
+++
b/metadata-service/db-service/client/src/main/java/org/apache/airavata/datalake/metadata/clients/Test.java
@@ -9,25 +9,27 @@ public class Test {
TenantMetadataServiceGrpc.TenantMetadataServiceBlockingStub stub =
serviceClient.tenant();
+
Tenant tenant = Tenant.newBuilder()
-.setTenantId("asdcfvf")
+.setTenantId("100010402")
.setName("TenantA")
.build();
+
Group group = Group.newBuilder()
.setName("g1")
+.setTenantId("100010402")
.build();
+
Group group2 = Group.newBuilder()
.setName("g3")
-.build();
-Group group1 = Group.newBuilder()
-.setName("g2")
-.addChildGroups(group2)
+.setTenantId("100010402")
.build();
User user = User.newBuilder()
.setUsername("TestingUserA")
.setFirstName("Isuru")
.setLastName("Ranawaka")
+.setTenantId("100010402")
.build();
GroupMembership groupMemberships = GroupMembership
@@ -36,26 +38,52 @@ public class Test {
.setMembershipType("ADMIN")
.build();
+Group group1 = Group.newBuilder()
+.setName("g2")
+.setTenantId("100010402")
+.addChildGroups(group2)
+.build();
+
group1 = group1.toBuilder()
.addGroupMembership(groupMemberships)
+.setTenantId("100010402")
.build();
+
group = group.toBuilder()
.addChildGroups(group1)
+.setTenantId("100010402")
.build();
Resource resource = Resource.newBuilder()
.setName("R1")
+.setTenantId("100010402")
.build();
+
Resource resource1 = Resource.newBuilder()
.setName("R2")
+.setTenantId("100010402")
.build();
+
Resource resource2 = Resource.newBuilder()
.setName("R3")
+.setTenantId("100010402")
.build();
resource1 = resource1.toBuilder()
.addChildResources(resource2)
+.setTenantId("100010402")
+.build();
+
+ResourceSharings resourceSharings = ResourceSharings
+.newBuilder()
+.setPermissionType("READ")
+.addGroups(group2)
.build();
+
+resource1 = resource1.toBuilder()
+.addSharings(resourceSharings)
+.build();
+
resource = resource.toBuilder()
.addChildResources(resource1)
.build();
@@ -63,6 +91,7 @@ public class Test {
tenant = tenant.toBuilder()
.addGroups(group)
.build();
+
tenant = tenant.toBuilder()
.addResources(resource)
.build();
diff --git
a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/AppConfig.java
[airavata-data-lake] 16/19: Implementing generic resource handler and metadata ingestion
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
commit b6b5cc43be4931b427958e2863d6865b45b5c6ca
Author: Dimuthu Wannipurage
AuthorDate: Wed Mar 24 13:03:53 2021 -0400
Implementing generic resource handler and metadata ingestion
---
.../drms/api/handlers/ResourceServiceHandler.java | 99 +-
.../handlers/StoragePreferenceServiceHandler.java | 6 +-
.../drms/api/handlers/StorageServiceHandler.java | 2 +
.../drms/core/constants/ResourceConstants.java | 21 +
.../deserializer/GenericResourceDeserializer.java | 89 +++
.../main/proto/resource/DRMSResourceService.proto | 35 +++-
6 files changed, 247 insertions(+), 5 deletions(-)
diff --git
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
index 4e34e7a..34efc05 100644
---
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
+++
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
@@ -17,15 +17,78 @@
package org.apache.airavata.drms.api.handlers;
import com.google.protobuf.Empty;
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
import io.grpc.stub.StreamObserver;
+import org.apache.airavata.datalake.drms.DRMSServiceAuthToken;
+import org.apache.airavata.datalake.drms.groups.FetchCurrentUserRequest;
+import org.apache.airavata.datalake.drms.groups.FetchCurrentUserResponse;
+import org.apache.airavata.datalake.drms.groups.GroupServiceGrpc;
+import org.apache.airavata.datalake.drms.groups.User;
+import org.apache.airavata.datalake.drms.resource.GenericResource;
import org.apache.airavata.datalake.drms.storage.*;
+import org.apache.airavata.drms.core.Neo4JConnector;
+import org.apache.airavata.drms.core.constants.ResourceConstants;
+import org.apache.airavata.drms.core.constants.StorageConstants;
+import
org.apache.airavata.drms.core.deserializer.AnyStoragePreferenceDeserializer;
+import org.apache.airavata.drms.core.deserializer.GenericResourceDeserializer;
import org.lognet.springboot.grpc.GRpcService;
+import org.neo4j.driver.Record;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.List;
@GRpcService
public class ResourceServiceHandler extends
ResourceServiceGrpc.ResourceServiceImplBase {
+
+private static final Logger logger =
LoggerFactory.getLogger(ResourceServiceHandler.class);
+
+@Autowired
+private Neo4JConnector neo4JConnector;
+
+
@org.springframework.beans.factory.annotation.Value("${group.service.host}")
+private String groupServiceHost;
+
+
@org.springframework.beans.factory.annotation.Value("${group.service.port}")
+private int groupServicePort;
+
+private User getUser(DRMSServiceAuthToken authToken) {
+ManagedChannel channel =
ManagedChannelBuilder.forAddress(groupServiceHost,
groupServicePort).usePlaintext().build();
+GroupServiceGrpc.GroupServiceBlockingStub groupClient =
GroupServiceGrpc.newBlockingStub(channel);
+FetchCurrentUserResponse userResponse = groupClient.fetchCurrentUser(
+
FetchCurrentUserRequest.newBuilder().setAuthToken(authToken).build());
+return userResponse.getUser();
+}
+
@Override
public void fetchResource(ResourceFetchRequest request,
StreamObserver responseObserver) {
-super.fetchResource(request, responseObserver);
+User callUser = getUser(request.getAuthToken());
+
+// TODO review (u)-[r4:MEMBER_OF]->(g2:Group)<-[r5:SHARED_WITH]-(sp),
+List records = this.neo4JConnector.searchNodes(
+"MATCH
(u:User)-[r1:MEMBER_OF]->(g:Group)<-[r2:SHARED_WITH]-(s:Storage)-[r3:HAS_PREFERENCE]->(sp:StoragePreference)-[r6:HAS_RESOURCE]->(res:Resource),
" +
+
"(u)-[r7:MEMBER_OF]->(g3:Group)<-[r8:SHARED_WITH]-(res) " +
+"where res.resourceId = '" + request.getResourceId() +
"' and u.userId = '"
++ callUser.getUserId() + "' return distinct res, sp,
s");
+
+if (!records.isEmpty()) {
+try {
+List genericResourceList =
GenericResourceDeserializer.deserializeList(records);
+
responseObserver.onNext(ResourceFetchResponse.newBuilder().setResource(genericResourceList.get(0)).build());
+responseObserver.onCompleted();
+} catch (Exception e) {
+
+logger.error("Errored while fetching resource with id {}",
request.getResourceId(), e);
+responseObserver.onError(new
[airavata-data-lake] 17/19: Fetching resource level metadata
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
commit f3683725276e410fb0c55ff5096a71c773375b3d
Author: Dimuthu Wannipurage
AuthorDate: Wed Mar 24 13:52:43 2021 -0400
Fetching resource level metadata
---
.../drms/api/handlers/ResourceServiceHandler.java | 19 +++-
.../drms/core/constants/MetadataConstants.java | 22 +
.../core/deserializer/MetadataDeserializer.java| 54 ++
.../main/proto/resource/DRMSResourceService.proto | 6 ++-
4 files changed, 99 insertions(+), 2 deletions(-)
diff --git
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
index 34efc05..9917ee8 100644
---
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
+++
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/ResourceServiceHandler.java
@@ -32,6 +32,7 @@ import
org.apache.airavata.drms.core.constants.ResourceConstants;
import org.apache.airavata.drms.core.constants.StorageConstants;
import
org.apache.airavata.drms.core.deserializer.AnyStoragePreferenceDeserializer;
import org.apache.airavata.drms.core.deserializer.GenericResourceDeserializer;
+import org.apache.airavata.drms.core.deserializer.MetadataDeserializer;
import org.lognet.springboot.grpc.GRpcService;
import org.neo4j.driver.Record;
import org.slf4j.Logger;
@@ -140,6 +141,22 @@ public class ResourceServiceHandler extends
ResourceServiceGrpc.ResourceServiceI
@Override
public void fetchResourceMetadata(FetchResourceMetadataRequest request,
StreamObserver responseObserver) {
-super.fetchResourceMetadata(request, responseObserver);
+User callUser = getUser(request.getAuthToken());
+List records = neo4JConnector.searchNodes("match
(u:User)-[MEMBER_OF]->(g:Group)<-[SHARED_WITH]-(res:Resource)-[r:HAS_METADATA]->(m:Metadata)
" +
+"where u.userId ='" + callUser.getUserId()+ "' and
res.resourceId = '" + request.getResourceId() + "' return distinct m");
+try {
+List metadataNodes =
MetadataDeserializer.deserializeList(records);
+if (metadataNodes.size() == 1) {
+
responseObserver.onNext(FetchResourceMetadataResponse.newBuilder().setMetadataNode(metadataNodes.get(0)).build());
+responseObserver.onCompleted();
+} else {
+logger.error("No metadata entry for resource {}",
request.getResourceId());
+responseObserver.onError(new Exception("No metadata entry for
resource " + request.getResourceId()));
+}
+} catch (Exception e) {
+logger.error("Errored while fetching metadata for resource with id
{}", request.getResourceId(), e);
+responseObserver.onError(new Exception("Errored while fetching
metadata for resource with id "
++ request.getResourceId() + ". Msg " + e.getMessage()));
+}
}
}
diff --git
a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/constants/MetadataConstants.java
b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/constants/MetadataConstants.java
new file mode 100644
index 000..da573e8
--- /dev/null
+++
b/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/constants/MetadataConstants.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.drms.core.constants;
+
+public final class MetadataConstants {
+public static final String METADATA_LABEL = "Metadata";
+}
diff --git
a/data-resource-management-service/drms-core/src/main/java/org/apache/airavata/drms/core/deserializer/MetadataDeserializer.java
[airavata-data-lake] 11/19: Protobuf definitions for DRMS
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
commit a62e770395f4f6e2ef6657cfe7061a227ded23b6
Author: DImuthuUpe
AuthorDate: Sun Mar 21 21:56:19 2021 -0400
Protobuf definitions for DRMS
---
.../drms-stubs/src/main/proto/Common.proto | 10 +++
.../src/main/proto/preference/S3Preference.proto | 18
.../src/main/proto/preference/SSHPreference.proto | 20 +
.../preference/StoragePreferenceService.proto | 98 ++
.../src/main/proto/resource/DRMSResource.proto | 17
.../main/proto/resource/DRMSResourceService.proto | 92
.../src/main/proto/storage/S3Storage.proto | 12 +++
.../src/main/proto/storage/SSHStorage.proto| 12 +++
.../src/main/proto/storage/StorageService.proto| 98 ++
9 files changed, 377 insertions(+)
diff --git
a/data-resource-management-service/drms-stubs/src/main/proto/Common.proto
b/data-resource-management-service/drms-stubs/src/main/proto/Common.proto
new file mode 100644
index 000..3334881
--- /dev/null
+++ b/data-resource-management-service/drms-stubs/src/main/proto/Common.proto
@@ -0,0 +1,10 @@
+syntax = "proto3";
+
+option java_multiple_files = true;
+package org.apache.airavata.datalake.drms;
+
+import "google/api/annotations.proto";
+
+message DRMSServiceAuthToken {
+string access_token = 1;
+}
\ No newline at end of file
diff --git
a/data-resource-management-service/drms-stubs/src/main/proto/preference/S3Preference.proto
b/data-resource-management-service/drms-stubs/src/main/proto/preference/S3Preference.proto
new file mode 100644
index 000..b7eac3e
--- /dev/null
+++
b/data-resource-management-service/drms-stubs/src/main/proto/preference/S3Preference.proto
@@ -0,0 +1,18 @@
+syntax = "proto3";
+
+option java_multiple_files = true;
+package org.apache.airavata.datalake.drms.storage.preference.s3;
+
+import "google/api/annotations.proto";
+import "storage/S3Storage.proto";
+
+message S3StoragePreference {
+string storagePreferenceId = 1;
+string credentialToken = 2;
+enum AuthType {
+CLIENT_ID_SECRET_ID = 0;
+OPEN = 2;
+}
+AuthType authType = 3;
+org.apache.airavata.datalake.drms.storage.s3.S3Storage storage = 4;
+}
\ No newline at end of file
diff --git
a/data-resource-management-service/drms-stubs/src/main/proto/preference/SSHPreference.proto
b/data-resource-management-service/drms-stubs/src/main/proto/preference/SSHPreference.proto
new file mode 100644
index 000..8cf20c6
--- /dev/null
+++
b/data-resource-management-service/drms-stubs/src/main/proto/preference/SSHPreference.proto
@@ -0,0 +1,20 @@
+syntax = "proto3";
+
+option java_multiple_files = true;
+package org.apache.airavata.datalake.drms.storage.preference.ssh;
+
+import "google/api/annotations.proto";
+import "storage/SSHStorage.proto";
+
+message SSHStoragePreference {
+ string storagePreferenceId = 1;
+ string credentialToken = 2;
+ string userName = 3;
+ enum AuthType {
+SSH_KEY = 0;
+PASSWORD = 1;
+OPEN = 2;
+ }
+ AuthType authType = 4;
+ org.apache.airavata.datalake.drms.storage.ssh.SSHStorage storage = 5;
+}
\ No newline at end of file
diff --git
a/data-resource-management-service/drms-stubs/src/main/proto/preference/StoragePreferenceService.proto
b/data-resource-management-service/drms-stubs/src/main/proto/preference/StoragePreferenceService.proto
new file mode 100644
index 000..f8e296a
--- /dev/null
+++
b/data-resource-management-service/drms-stubs/src/main/proto/preference/StoragePreferenceService.proto
@@ -0,0 +1,98 @@
+syntax = "proto3";
+
+option java_multiple_files = true;
+package org.apache.airavata.datalake.drms.storage;
+
+import "google/api/annotations.proto";
+import "Common.proto";
+import "preference/SSHPreference.proto";
+import "preference/S3Preference.proto";
+import "google/protobuf/empty.proto";
+
+message AnyStoragePreference {
+ oneof storage {
+
org.apache.airavata.datalake.drms.storage.preference.ssh.SSHStoragePreference
sshStoragePreference = 1;
+
org.apache.airavata.datalake.drms.storage.preference.s3.S3StoragePreference
s3StoragePreference = 2;
+ }
+}
+message StoragePreferenceFetchRequest {
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken authToken = 1;
+ string storagePreferenceId = 2;
+}
+
+message StoragePreferenceFetchResponse {
+ AnyStoragePreference storagePreference = 1;
+}
+
+message StoragePreferenceCreateRequest {
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken authToken = 1;
+ AnyStoragePreference storagePreference = 2;
+}
+
+message StoragePreferenceCreateResponse {
+ AnyStoragePreference storagePreference = 1;
+}
+
+message StoragePreferenceUpdateRequest {
+ org.apache.airavata.datalake.drms.DRMSServiceAuthToken authToken = 1;
+ string storagePreferenceId = 2;
+ AnyStoragePreference
[airavata-data-lake] 06/19: Adding repository support for SFTP Remotes
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
commit ace1dc1bdfd9b3058574686cb1fc719901700bd8
Author: Dimuthu Wannipurage
AuthorDate: Tue Jan 5 19:00:19 2021 -0500
Adding repository support for SFTP Remotes
---
data-orchestrator/data-orchestrator-api/pom.xml| 19 +++
.../api/controller/remote/SFTPController.java | 46 +--
.../api/db/entity/SFTPCredentialEntity.java| 140 +
.../api/db/entity/SFTPRemoteEntity.java| 92 ++
.../api/db/repo/SFTPCredentialRepository.java | 26
.../api/db/repo/SFTPRemoteRepository.java | 26
6 files changed, 337 insertions(+), 12 deletions(-)
diff --git a/data-orchestrator/data-orchestrator-api/pom.xml
b/data-orchestrator/data-orchestrator-api/pom.xml
index c840c04..e6433cb 100644
--- a/data-orchestrator/data-orchestrator-api/pom.xml
+++ b/data-orchestrator/data-orchestrator-api/pom.xml
@@ -41,6 +41,25 @@
org.springframework.boot
+spring-boot-starter-data-jpa
+2.4.1
+
+
+
+net.sf.dozer
+dozer
+5.5.1
+
+
+
+com.h2database
+h2
+1.4.197
+runtime
+
+
+
+org.springframework.boot
spring-boot-starter-test
test
diff --git
a/data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/controller/remote/SFTPController.java
b/data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/controller/remote/SFTPController.java
index 5492889..e7bced6 100644
---
a/data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/controller/remote/SFTPController.java
+++
b/data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/controller/remote/SFTPController.java
@@ -19,39 +19,61 @@
*/
package org.apache.airavata.datalake.orchestrator.api.controller.remote;
-import
org.apache.airavata.datalake.orchestrator.api.model.remote.SFTPCredential;
+import
org.apache.airavata.datalake.orchestrator.api.db.entity.SFTPRemoteEntity;
+import
org.apache.airavata.datalake.orchestrator.api.db.repo.SFTPRemoteRepository;
import org.apache.airavata.datalake.orchestrator.api.model.remote.SFTPRemote;
+import org.dozer.DozerBeanMapper;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
+import org.springframework.web.server.ResponseStatusException;
+
+import java.util.Optional;
@RestController
@RequestMapping(path = "/remotes/sftp")
public class SFTPController {
+@Autowired
+private SFTPRemoteRepository sftpRemoteRepository;
+
@GetMapping(value = "/{remoteId}", produces = "application/json")
public SFTPRemote fetchSFTPRemote(@PathVariable(name = "remoteId") String
remoteId) {
-return new SFTPRemote()
-.setHost("localhost")
-.setPort(22)
-.setCredential(
-new SFTPCredential()
-.setAuthMethod(SFTPCredential.AuthMethod.SSH_KEY)
-.setPrivateKey("")
-.setPublicKey(""));
+
+Optional entityOp =
sftpRemoteRepository.findById(remoteId);
+SFTPRemoteEntity sftpRemoteEntity = entityOp.orElseThrow(
+() -> new
ResponseStatusException(HttpStatus.NOT_FOUND, remoteId + " not found"));
+DozerBeanMapper mapper = new DozerBeanMapper();
+return mapper.map(sftpRemoteEntity, SFTPRemote.class);
}
@PostMapping(value = "")
public String createSFTPRemote(@RequestBody SFTPRemote sftpRemote) {
-return "Remote id";
+
+DozerBeanMapper mapper = new DozerBeanMapper();
+SFTPRemoteEntity sftpRemoteEntity = mapper.map(sftpRemote,
SFTPRemoteEntity.class);
+SFTPRemoteEntity saved = sftpRemoteRepository.save(sftpRemoteEntity);
+return saved.getId();
}
@PutMapping(value = "/{remoteId}")
public String updateSFTPRemote(@RequestBody SFTPRemote sftpRemote,
@PathVariable(name = "remoteId") String
remoteId) {
-return "Remote id";
+
+DozerBeanMapper mapper = new DozerBeanMapper();
+SFTPRemoteEntity sftpRemoteEntity = mapper.map(sftpRemote,
SFTPRemoteEntity.class);
+sftpRemoteEntity.setId(remoteId);
+SFTPRemoteEntity saved = sftpRemoteRepository.save(sftpRemoteEntity);
+return saved.getId();
}
@DeleteMapping(value = "/{remoteId}")
public String removeSFTPRemote(@PathVariable(name = "remoteId") String
remoteId) {
-
[airavata-data-lake] 10/19: Merging metadata catalog branch and fixing build issue
This is an automated email from the ASF dual-hosted git repository. isjarana pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git commit 2277d7114c0bebee88251ad7102f528df57fd4ff Author: DImuthuUpe AuthorDate: Sat Mar 20 02:10:16 2021 -0400 Merging metadata catalog branch and fixing build issue --- .gitignore | 7 +- .../drms-api}/pom.xml | 33 +- .../drms-core}/pom.xml | 33 +- .../drms-stubs}/pom.xml| 137 ++-- .../pom.xml| 39 +- metadata-service/data-builders/pom.xml | 4 +- metadata-service/db-service/client/client.iml | 109 --- .../db-service/client/db-service-client.iml| 16 - metadata-service/db-service/client/pom.xml | 2 +- metadata-service/db-service/pom.xml| 6 +- .../db-service/server/db-service-server.iml| 462 metadata-service/db-service/server/server.iml | 109 --- .../db-service/stub/db-service-stub.iml| 84 --- metadata-service/db-service/stub/pom.xml | 2 +- metadata-service/db-service/stub/stub.iml | 111 --- metadata-service/metadata-service.iml | 804 - metadata-service/pom.xml | 5 +- pom.xml| 4 +- 18 files changed, 147 insertions(+), 1820 deletions(-) diff --git a/.gitignore b/.gitignore index 753237f..66ff5f6 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,9 @@ data-orchestrator/data-orchestrator-core/data-orchestrator-core.iml target/ data-orchestrator/target data-orchestrator/data-orchestrator-core/target -data-orchestrator/data-orchestrator-api/target \ No newline at end of file +data-orchestrator/data-orchestrator-api/target +/data-resource-management-service/data-resource-management-service.iml +/data-resource-management-service/drms-stubs/drms-stubs.iml +/data-resource-management-service/drms-api/drms-api.iml +/data-resource-management-service/drms-core/drms-core.iml +/metadata-service/metadata-service.iml diff --git a/metadata-service/data-builders/pom.xml b/data-resource-management-service/drms-api/pom.xml similarity index 58% copy from metadata-service/data-builders/pom.xml copy to data-resource-management-service/drms-api/pom.xml index 1af2131..2839b3b 100644 --- a/metadata-service/data-builders/pom.xml +++ b/data-resource-management-service/drms-api/pom.xml @@ -1,20 +1,15 @@ - -http://maven.apache.org/POM/4.0.0; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd;> - -airavata-data-lake -org.apache.airavata.data.lake -0.01-SNAPSHOT -../../pom.xml - -4.0.0 - -data-builders - - -11 -11 - - + +http://maven.apache.org/POM/4.0.0; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd;> + +data-resource-management-service +org.apache.airavata.data.lake +0.01-SNAPSHOT + +4.0.0 + +drms-api + + \ No newline at end of file diff --git a/metadata-service/data-builders/pom.xml b/data-resource-management-service/drms-core/pom.xml similarity index 58% copy from metadata-service/data-builders/pom.xml copy to data-resource-management-service/drms-core/pom.xml index 1af2131..385d937 100644 --- a/metadata-service/data-builders/pom.xml +++ b/data-resource-management-service/drms-core/pom.xml @@ -1,20 +1,15 @@ - -http://maven.apache.org/POM/4.0.0; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd;> - -airavata-data-lake -org.apache.airavata.data.lake -0.01-SNAPSHOT -../../pom.xml - -4.0.0 - -data-builders - - -11 -11 - - + +http://maven.apache.org/POM/4.0.0; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd;> + +data-resource-management-service +org.apache.airavata.data.lake +0.01-SNAPSHOT + +4.0.0 + +drms-core + + \ No newline at end of file diff --git a/metadata-service/db-service/stub/pom.xml b/data-resource-management-service/drms-stubs/pom.xml similarity index 61% copy from metadata-service/db-service/stub/pom.xml copy to data-resource-management-service/drms-stubs/pom.xml index e032813..915db71 100644 --- a/metadata-service/db-service/stub/pom.xml +++ b/data-resource-management-service/drms-stubs/pom.xml @@ -1,57 +1,82 @@ -
[airavata-data-lake] 18/19: Add merger for updates
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
commit dc26af311ae5e406a4e8f18afaf48db91d5777a2
Author: Isuru Ranawaka
AuthorDate: Wed Mar 24 14:46:37 2021 -0400
Add merger for updates
---
.../airavata/datalake/metadata/clients/Test.java | 31 +-
.../airavata/datalake/metadata/AppConfig.java | 47 +
.../datalake/metadata/Neo4JServiceInitializer.java | 2 +
.../neo4j/curd/operators/GenericService.java | 14 ++-
.../backend/neo4j/curd/operators/GroupService.java | 1 +
.../neo4j/curd/operators/GroupServiceImpl.java | 28 ++
.../neo4j/curd/operators/ResourceService.java | 4 +
.../neo4j/curd/operators/ResourceServiceImpl.java | 52 +-
.../backend/neo4j/curd/operators/Service.java | 6 +-
.../neo4j/curd/operators/TenantService.java| 2 +
.../neo4j/curd/operators/TenantServiceImpl.java| 34 +++
.../backend/neo4j/curd/operators/UserService.java | 1 +
.../neo4j/curd/operators/UserServiceImpl.java | 28 ++
.../metadata/backend/neo4j/model/nodes/Entity.java | 12 +++
.../metadata/handlers/GroupServiceHandler.java | 79 ---
.../metadata/handlers/ResourceServiceHandler.java | 111 ++---
.../metadata/handlers/TenantServiceHandler.java| 78 +--
.../metadata/handlers/UserServiceHandler.java | 48 -
.../metadata/interceptors/Authenticator.java | 19
.../interceptors/InterceptorPipelineExecutor.java | 69 +
.../metadata/interceptors/ServiceInterceptor.java | 7 ++
.../datalake/metadata/mergers/GenericMerger.java | 46 +
.../airavata/datalake/metadata/mergers/Merger.java | 9 ++
.../metadata/parsers/ExecutionContext.java | 4 +
.../datalake/metadata/parsers/GroupParser.java | 17 +++-
.../airavata/datalake/metadata/parsers/Parser.java | 5 +-
.../datalake/metadata/parsers/ResourceParser.java | 20 +++-
.../datalake/metadata/parsers/TenantParser.java| 52 +++---
.../datalake/metadata/parsers/UserParser.java | 18 +++-
.../stub/src/main/proto/resource/Resource.proto| 20
.../stub/src/main/proto/tenant/Tenant.proto| 6 +-
31 files changed, 776 insertions(+), 94 deletions(-)
diff --git
a/metadata-service/db-service/client/src/main/java/org/apache/airavata/datalake/metadata/clients/Test.java
b/metadata-service/db-service/client/src/main/java/org/apache/airavata/datalake/metadata/clients/Test.java
index a506d75..b70c246 100644
---
a/metadata-service/db-service/client/src/main/java/org/apache/airavata/datalake/metadata/clients/Test.java
+++
b/metadata-service/db-service/client/src/main/java/org/apache/airavata/datalake/metadata/clients/Test.java
@@ -101,7 +101,36 @@ public class Test {
.setTenant(tenant)
.build();
-stub.createTenant(request);
+//stub.createTenant(request);
+//
+
+//ResourceMetadataServiceGrpc.ResourceMetadataServiceBlockingStub
resourceMetadataServiceBlockingStub = serviceClient.resource();
+//
+//ResourcePermissionRequest permissionRequest =
ResourcePermissionRequest
+//.newBuilder()
+//.setPermissionType("READ")
+//.setUsername("TestingUserA")
+//.setResourceName("R5")
+//.setTenantId("100010402")
+//.build();
+// ResourcePermissionResponse response =
resourceMetadataServiceBlockingStub.hasAccess(permissionRequest);
+
+
+//TenantMetadataAPIRequest tenantMetadataAPIRequest =
TenantMetadataAPIRequest
+//.newBuilder()
+//.setTenant(tenant)
+//.build();
+//
+//stub.deleteTenant(tenantMetadataAPIRequest);
+
+tenant = tenant.toBuilder().setDomain("testing.com").build();
+
+
+TenantMetadataAPIRequest tenantMetadataAPIRequest =
TenantMetadataAPIRequest
+.newBuilder()
+.setTenant(tenant)
+.build();
+stub.updateTenant(tenantMetadataAPIRequest);
}
}
diff --git
a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/AppConfig.java
b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/AppConfig.java
index 0e03c32..05ee7a4 100644
---
a/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/AppConfig.java
+++
b/metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/AppConfig.java
@@ -1,15 +1,18 @@
package org.apache.airavata.datalake.metadata;
+import io.grpc.ServerInterceptor;
import org.apache.airavata.datalake.metadata.backend.Connector;
-import
org.apache.airavata.datalake.metadata.backend.neo4j.curd.operators.ResourceServiceImpl;
-import
[airavata-data-lake] 14/19: Implementing the storage layer with metadata and sharing support
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
commit 2056a2079209548cb09bcdf2bf72c5c163183286
Author: Dimuthu Wannipurage
AuthorDate: Wed Mar 24 09:37:30 2021 -0400
Implementing the storage layer with metadata and sharing support
---
.gitignore | 3 +
data-resource-management-service/drms-api/pom.xml | 29 --
.../java/org/apache/airavata/drms/api/Client.java | 70 +
.../java/org/apache/airavata/drms/api/Config.java | 32 ++
.../apache/airavata/drms/api/DRMSApiRunner.java| 16 +++
.../drms/api/handlers/GroupServiceHandler.java | 95 ++
.../drms/api/handlers/ResourceServiceHandler.java | 16 +++
.../handlers/StoragePreferenceServiceHandler.java | 16 +++
.../drms/api/handlers/StorageServiceHandler.java | 109 -
.../src/main/resources/application.properties | 23 +
.../drms-api/src/main/resources/logback.xml| 45 +
data-resource-management-service/drms-core/pom.xml | 40 +++-
.../apache/airavata/drms/core/Neo4JConnector.java | 66 +
.../drms/core/constants/StorageConstants.java | 24 +
.../storage/AnyStorageDeserializer.java| 81 +++
.../serializer/storage/AnyStorageSerializer.java | 58 +++
.../drms-stubs/pom.xml | 35 +++
.../drms-stubs/src/main/proto/Common.proto | 17
.../drms-stubs/src/main/proto/group/Group.proto| 46 +
.../src/main/proto/group/GroupService.proto| 86
.../src/main/proto/preference/S3Preference.proto | 17
.../src/main/proto/preference/SSHPreference.proto | 17
.../preference/StoragePreferenceService.proto | 17
.../src/main/proto/resource/DRMSResource.proto | 17
.../main/proto/resource/DRMSResourceService.proto | 17
.../src/main/proto/sharing/Sharing.proto | 28 ++
.../src/main/proto/storage/S3Storage.proto | 17
.../src/main/proto/storage/SSHStorage.proto| 17
.../src/main/proto/storage/StorageService.proto| 30 ++
data-resource-management-service/pom.xml | 28 +-
pom.xml| 4 +
31 files changed, 1083 insertions(+), 33 deletions(-)
diff --git a/.gitignore b/.gitignore
index cb794d5..f07bdb5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,6 @@ data-orchestrator/data-orchestrator-api/target
/metadata-service/metadata-service.iml
/metadata-service/data-builders/data-builders.iml
/metadata-service/db-service/db-service.iml
+/metadata-service/db-service/client/client.iml
+/metadata-service/db-service/server/server.iml
+/metadata-service/db-service/stub/stub.iml
diff --git a/data-resource-management-service/drms-api/pom.xml
b/data-resource-management-service/drms-api/pom.xml
index a6d5805..92e265b 100644
--- a/data-resource-management-service/drms-api/pom.xml
+++ b/data-resource-management-service/drms-api/pom.xml
@@ -1,4 +1,20 @@
+
http://maven.apache.org/POM/4.0.0;
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
@@ -15,11 +31,6 @@
org.apache.airavata.data.lake
drms-stubs
-0.01-SNAPSHOT
-
-
-org.apache.airavata.data.lake
-stub
${project.version}
@@ -41,7 +52,7 @@
org.springframework.boot
spring-boot-starter-web
-2.4.2
+${spring.boot.version}
net.sf.dozer
@@ -49,9 +60,9 @@
${dozer}
-io.grpc
-grpc-services
-1.25.0
+org.apache.airavata.data.lake
+drms-core
+${project.version}
\ No newline at end of file
diff --git
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Client.java
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Client.java
new file mode 100644
index 000..8e589ef
--- /dev/null
+++
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/Client.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *
[airavata-data-lake] 07/19: metadata service neo4j implementation
This is an automated email from the ASF dual-hosted git repository. isjarana pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git commit 2724b9a8e90f7a66150b69b359fe319d9b9dba0e Author: Isuru Ranawaka AuthorDate: Wed Mar 10 21:44:02 2021 -0500 metadata service neo4j implementation --- airavata-data-lake.ipr | 112 ++ airavata-data-lake.iws | 418 metadata-service/data-builders/data-builders.iml | 16 + metadata-service/data-builders/pom.xml | 20 + metadata-service/db-service/db-service.iml | 420 + metadata-service/db-service/pom.xml| 49 +++ .../datalake/metadata/db/service/AppConfig.java| 108 ++ .../db/service/Neo4JServiceInitializer.java| 14 + .../metadata/db/service/XmlConfiguration.java | 9 + .../metadata/db/service/backend/Connector.java | 10 + .../metadata/db/service/backend/Person.java| 74 .../db/service/backend/neo4j/Neo4JConnector.java | 53 +++ .../neo4j/curd/operators/GenericService.java | 79 .../backend/neo4j/curd/operators/GroupService.java | 4 + .../neo4j/curd/operators/ResourceService.java | 4 + .../neo4j/curd/operators/SearchOperator.java | 33 ++ .../backend/neo4j/curd/operators/Service.java | 24 ++ .../backend/neo4j/curd/operators/UserService.java | 4 + .../service/backend/neo4j/model/nodes/Entity.java | 109 ++ .../service/backend/neo4j/model/nodes/Group.java | 208 ++ .../backend/neo4j/model/nodes/Resource.java| 215 +++ .../db/service/backend/neo4j/model/nodes/Role.java | 37 ++ .../backend/neo4j/model/nodes/ServiceAccount.java | 29 ++ .../service/backend/neo4j/model/nodes/Tenant.java | 125 ++ .../db/service/backend/neo4j/model/nodes/User.java | 132 +++ .../backend/neo4j/model/relationships/Belongs.java | 9 + .../neo4j/model/relationships/ConfiguredWith.java | 7 + .../backend/neo4j/model/relationships/Has.java | 9 + .../neo4j/model/relationships/HasAccess.java | 23 ++ .../neo4j/model/relationships/HasChildGroup.java | 9 + .../model/relationships/HasChildResource.java | 8 + .../neo4j/model/relationships/HasChildUser.java| 19 + .../neo4j/model/relationships/HasParentGroup.java | 19 + .../model/relationships/HasParentResource.java | 7 + .../backend/neo4j/model/relationships/HasRole.java | 8 + .../neo4j/model/relationships/MemberOf.java| 19 + .../neo4j/model/relationships/Relationship.java| 81 .../neo4j/model/relationships/SharedWith.java | 25 ++ .../service/exceptions/DBConnectorException.java | 10 + .../src/main/resources/application.properties | 23 ++ .../src/main/resources/applicationContext.xml | 12 + metadata-service/metadata-service.iml | 14 + metadata-service/pom.xml | 130 +++ pom.xml| 77 +++- 44 files changed, 2814 insertions(+), 1 deletion(-) diff --git a/airavata-data-lake.ipr b/airavata-data-lake.ipr new file mode 100644 index 000..3bec1ac --- /dev/null +++ b/airavata-data-lake.ipr @@ -0,0 +1,112 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airavata-data-lake.iws b/airavata-data-lake.iws new file mode 100644 index 000..57de9a0 --- /dev/null +++ b/airavata-data-lake.iws @@ -0,0 +1,418 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
[airavata-data-lake] 09/19: Merge pull request #1 from isururanawaka/metadata_service
This is an automated email from the ASF dual-hosted git repository. isjarana pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git commit d44747ec51d5618683451328703b0866246db988 Merge: ace1dc1 0bac93c Author: Suresh Marru AuthorDate: Wed Mar 17 16:51:36 2021 -0400 Merge pull request #1 from isururanawaka/metadata_service Metadata service airavata-data-lake.ipr | 114 +++ airavata-data-lake.iws | 418 +++ metadata-service/data-builders/data-builders.iml | 16 + metadata-service/data-builders/pom.xml | 20 + metadata-service/db-service/client/client.iml | 109 +++ .../db-service/client/db-service-client.iml| 16 + metadata-service/db-service/client/pom.xml | 28 + .../metadata/clients/MetadataServiceClient.java| 35 + .../clients/MetadataServiceClientBuilder.java | 12 + .../airavata/datalake/metadata/clients/Test.java | 78 ++ metadata-service/db-service/db-service.iml | 420 +++ metadata-service/db-service/pom.xml| 28 + .../db-service/server/db-service-server.iml| 462 metadata-service/db-service/server/pom.xml | 69 ++ metadata-service/db-service/server/server.iml | 109 +++ .../airavata/datalake/metadata/AppConfig.java | 149 .../datalake/metadata/Neo4JServiceInitializer.java | 14 + .../datalake/metadata/XmlConfiguration.java| 9 + .../datalake/metadata/backend/Connector.java | 10 + .../airavata/datalake/metadata/backend/Person.java | 72 ++ .../metadata/backend/neo4j/Neo4JConnector.java | 53 ++ .../neo4j/curd/operators/GenericService.java | 90 +++ .../backend/neo4j/curd/operators/GroupService.java | 4 + .../neo4j/curd/operators/GroupServiceImpl.java | 16 + .../neo4j/curd/operators/ResourceService.java | 4 + .../neo4j/curd/operators/ResourceServiceImpl.java | 16 + .../neo4j/curd/operators/SearchOperator.java | 33 + .../backend/neo4j/curd/operators/Service.java | 27 + .../neo4j/curd/operators/TenantService.java| 4 + .../neo4j/curd/operators/TenantServiceImpl.java| 15 + .../backend/neo4j/curd/operators/UserService.java | 4 + .../neo4j/curd/operators/UserServiceImpl.java | 16 + .../metadata/backend/neo4j/model/nodes/Entity.java | 109 +++ .../metadata/backend/neo4j/model/nodes/Group.java | 208 ++ .../backend/neo4j/model/nodes/Resource.java| 215 ++ .../metadata/backend/neo4j/model/nodes/Role.java | 37 + .../backend/neo4j/model/nodes/ServiceAccount.java | 29 + .../metadata/backend/neo4j/model/nodes/Tenant.java | 125 .../metadata/backend/neo4j/model/nodes/User.java | 132 .../backend/neo4j/model/relationships/Belongs.java | 9 + .../neo4j/model/relationships/ConfiguredWith.java | 7 + .../backend/neo4j/model/relationships/Has.java | 9 + .../neo4j/model/relationships/HasAccess.java | 23 + .../neo4j/model/relationships/HasChildGroup.java | 9 + .../model/relationships/HasChildResource.java | 8 + .../neo4j/model/relationships/HasChildUser.java| 19 + .../neo4j/model/relationships/HasParentGroup.java | 19 + .../model/relationships/HasParentResource.java | 7 + .../backend/neo4j/model/relationships/HasRole.java | 8 + .../neo4j/model/relationships/MemberOf.java| 19 + .../neo4j/model/relationships/Relationship.java| 81 +++ .../neo4j/model/relationships/SharedWith.java | 25 + .../metadata/exceptions/DBConnectorException.java | 10 + .../metadata/handlers/GroupServiceHandler.java | 38 + .../metadata/handlers/ResourceServiceHandler.java | 38 + .../metadata/handlers/TenantServiceHandler.java| 83 +++ .../metadata/handlers/UserServiceHandler.java | 29 + .../datalake/metadata/parsers/GroupParser.java | 68 ++ .../datalake/metadata/parsers/ResourceParser.java | 77 ++ .../datalake/metadata/parsers/TenantParser.java| 79 ++ .../datalake/metadata/parsers/UserParser.java | 22 + .../src/main/resources/application.properties | 23 + .../src/main/resources/applicationContext.xml | 12 + .../db-service/stub/db-service-stub.iml| 84 +++ metadata-service/db-service/stub/pom.xml | 57 ++ .../stub/src/main/proto/common/Common.proto| 12 + .../stub/src/main/proto/group/Group.proto | 88 +++ .../stub/src/main/proto/resource/Resource.proto| 97 +++ .../stub/src/main/proto/tenant/Tenant.proto| 67 ++ .../db-service/stub/src/main/proto/user/User.proto | 66 ++ metadata-service/db-service/stub/stub.iml | 111 +++ metadata-service/metadata-service.iml | 804 + metadata-service/pom.xml | 69 ++ pom.xml| 80 +- 74 files changed, 5581 insertions(+), 1
[airavata-data-lake] 05/19: Initial framework for orchestrator API
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
commit 5c072e3b648a1b56f7aa19910764f83df6ca3f14
Author: Dimuthu Wannipurage
AuthorDate: Tue Jan 5 04:51:42 2021 -0500
Initial framework for orchestrator API
---
data-orchestrator/data-orchestrator-api/pom.xml| 51 ++
.../datalake/orchestrator/api/Application.java | 7 ++
.../HookController.java} | 7 +-
.../api/controller/TransferController.java | 48 +
.../api/controller/remote/SFTPController.java | 57 +++
.../orchestrator/api/model/TransferEntry.java | 75 ++
.../api/model/remote/SFTPCredential.java | 110 +
.../orchestrator/api/model/remote/SFTPRemote.java | 73 ++
data-orchestrator/pom.xml | 21
9 files changed, 444 insertions(+), 5 deletions(-)
diff --git a/data-orchestrator/data-orchestrator-api/pom.xml
b/data-orchestrator/data-orchestrator-api/pom.xml
index 720bd6c..c840c04 100644
--- a/data-orchestrator/data-orchestrator-api/pom.xml
+++ b/data-orchestrator/data-orchestrator-api/pom.xml
@@ -1,4 +1,25 @@
+
http://maven.apache.org/POM/4.0.0;
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
@@ -11,5 +32,35 @@
data-orchestrator-api
+
+
+org.springframework.boot
+spring-boot-starter-web
+2.4.1
+
+
+
+org.springframework.boot
+spring-boot-starter-test
+test
+
+
+org.junit.vintage
+junit-vintage-engine
+
+
+2.4.1
+
+
+
+
+
+
+org.springframework.boot
+spring-boot-maven-plugin
+2.4.1
+
+
+
\ No newline at end of file
diff --git
a/data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/Application.java
b/data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/Application.java
index dc91369..df78fc2 100644
---
a/data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/Application.java
+++
b/data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/Application.java
@@ -19,8 +19,15 @@
*/
package org.apache.airavata.datalake.orchestrator.api;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
/**
* TODO: Spring Boot API
*/
+@SpringBootApplication
public class Application {
+public static void main(String[] args) {
+SpringApplication.run(Application.class, args);
+}
}
diff --git
a/data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/Application.java
b/data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/controller/HookController.java
similarity index 87%
copy from
data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/Application.java
copy to
data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/controller/HookController.java
index dc91369..c171681 100644
---
a/data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/Application.java
+++
b/data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/controller/HookController.java
@@ -17,10 +17,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.airavata.datalake.orchestrator.api;
+package org.apache.airavata.datalake.orchestrator.api.controller;
-/**
- * TODO: Spring Boot API
- */
-public class Application {
+public class HookController {
}
diff --git
a/data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/controller/TransferController.java
b/data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/controller/TransferController.java
new file mode 100644
index 000..5e5b04c
--- /dev/null
+++
b/data-orchestrator/data-orchestrator-api/src/main/java/org/apache/airavata/datalake/orchestrator/api/controller/TransferController.java
@@ -0,0 +1,48 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright
[airavata-data-lake] 01/19: bootstrapping git repo
This is an automated email from the ASF dual-hosted git repository. isjarana pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git commit 3ed323a81d93a2da0cad1e748bc6e825102cb06d Author: Suresh Marru AuthorDate: Tue Nov 10 13:02:25 2020 -0500 bootstrapping git repo --- README.md | 35 +++ 1 file changed, 35 insertions(+) diff --git a/README.md b/README.md new file mode 100644 index 000..fd06c5a --- /dev/null +++ b/README.md @@ -0,0 +1,35 @@ + +# Apache Airavata Data Lake + +[](https://apache.org/licenses/LICENSE-2.0) + +Apache Airavata use cases enable capture of data from observational and experimental instruments and computations resulting from Airavata's orchestration capabilities. As the data deluges into vast amounts, harvesting the data, capturing metadata, presenting it for easy and controlled access becomes unmanageable. + +Airavata data lake will bundle stand alone services to catalog data in various data stores, extract and capture semantics and metadata descriptions of the data and preserve associated data provenance. The data lake will provide API's, query and search capabilities to programmatically search and retrieve data and power building user interactivity and data analysis applications on top of it. + + + +Airavata Data Lake will provide file watcher and other trigger capabilities to ingest data from scientific instruments as they become available. The framwork will enable pluggable data parsers to read structured and unstructured data files and extract meaningful descriptions. + +A bundled Data replica catalogs will associate pointers to data at multiple storgae locations. The replica catalog maps logical file names to the physical locations. Data Lake client SDK's will provide API functions to query replica location and resolve into multiple physical file locations. The client will be bundled with access protocols to retrive the data or to embedd into computational pipelines. + +Interfacing with Airavata [Managed File Transfer Service](https://github.com/apache/airavata-mft) Data can moved and archiving into longer term persistant storages like tapped archives. The Data archives will be indexed and have search capabilities + +Data Lake's provenance will provide information to capture parameters influenced the production or modification of the data. An abstraction API will enable plugging fine granted provenance based on Airavata tentant context. Interfacing with Airavata Orchestration Services, pointers to experiment catalog will enable restructuring of the underting computations. \ No newline at end of file
[airavata-data-lake] 15/19: Implementing storage preference fetch and search functionalities
This is an automated email from the ASF dual-hosted git repository.
isjarana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
commit d1942b4710245ff5c691cef5f5a00c0512adf697
Author: Dimuthu Wannipurage
AuthorDate: Wed Mar 24 10:58:17 2021 -0400
Implementing storage preference fetch and search functionalities
---
.../handlers/StoragePreferenceServiceHandler.java | 78 +-
.../drms/api/handlers/StorageServiceHandler.java | 5 +-
.../constants/StoragePreferenceConstants.java} | 24 ++
.../{storage => }/AnyStorageDeserializer.java | 4 +-
.../AnyStoragePreferenceDeserializer.java | 93 ++
.../{storage => }/AnyStorageSerializer.java| 2 +-
.../src/main/proto/preference/S3Preference.proto | 6 +-
.../src/main/proto/preference/SSHPreference.proto | 6 +-
8 files changed, 187 insertions(+), 31 deletions(-)
diff --git
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java
index a010253..3d7b44a 100644
---
a/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java
+++
b/data-resource-management-service/drms-api/src/main/java/org/apache/airavata/drms/api/handlers/StoragePreferenceServiceHandler.java
@@ -17,16 +17,75 @@
package org.apache.airavata.drms.api.handlers;
import com.google.protobuf.Empty;
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
import io.grpc.stub.StreamObserver;
+import org.apache.airavata.datalake.drms.DRMSServiceAuthToken;
+import org.apache.airavata.datalake.drms.groups.FetchCurrentUserRequest;
+import org.apache.airavata.datalake.drms.groups.FetchCurrentUserResponse;
+import org.apache.airavata.datalake.drms.groups.GroupServiceGrpc;
+import org.apache.airavata.datalake.drms.groups.User;
import org.apache.airavata.datalake.drms.storage.*;
+import org.apache.airavata.drms.core.Neo4JConnector;
+import org.apache.airavata.drms.core.deserializer.AnyStorageDeserializer;
+import
org.apache.airavata.drms.core.deserializer.AnyStoragePreferenceDeserializer;
import org.lognet.springboot.grpc.GRpcService;
+import org.neo4j.driver.Record;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.List;
@GRpcService
public class StoragePreferenceServiceHandler extends
StoragePreferenceServiceGrpc.StoragePreferenceServiceImplBase {
+private static final Logger logger =
LoggerFactory.getLogger(StoragePreferenceServiceHandler.class);
+
+@Autowired
+private Neo4JConnector neo4JConnector;
+
+
@org.springframework.beans.factory.annotation.Value("${group.service.host}")
+private String groupServiceHost;
+
+
@org.springframework.beans.factory.annotation.Value("${group.service.port}")
+private int groupServicePort;
+
+
+private User getUser(DRMSServiceAuthToken authToken) {
+ManagedChannel channel =
ManagedChannelBuilder.forAddress(groupServiceHost,
groupServicePort).usePlaintext().build();
+GroupServiceGrpc.GroupServiceBlockingStub groupClient =
GroupServiceGrpc.newBlockingStub(channel);
+FetchCurrentUserResponse userResponse = groupClient.fetchCurrentUser(
+
FetchCurrentUserRequest.newBuilder().setAuthToken(authToken).build());
+return userResponse.getUser();
+}
+
@Override
public void fetchStoragePreference(StoragePreferenceFetchRequest request,
StreamObserver responseObserver) {
-super.fetchStoragePreference(request, responseObserver);
+User callUser = getUser(request.getAuthToken());
+
+List records = this.neo4JConnector.searchNodes(
+"MATCH
(u:User)-[r1:MEMBER_OF]->(g:Group)<-[r2:SHARED_WITH]-(s:Storage)-[r3:HAS_PREFERENCE]->(sp:StoragePreference)
" +
+"where sp.storagePreferenceId = '" +
request.getStoragePreferenceId() + "' and u.userId = '"
++ callUser.getUserId() + "' return distinct sp, s");
+
+if (!records.isEmpty()) {
+try {
+List storagePrefList =
AnyStoragePreferenceDeserializer.deserializeList(records);
+
responseObserver.onNext(StoragePreferenceFetchResponse.newBuilder().setStoragePreference(
+storagePrefList.get(0)).build());
+responseObserver.onCompleted();
+} catch (Exception e) {
+
+logger.error("Errored while fetching storage preference with
id {}", request.getStoragePreferenceId(), e);
+responseObserver.onError(new Exception("Errored while fetching
storage preference with id "
++
[airavata-data-lake] branch master updated (f368372 -> 1909dca)
This is an automated email from the ASF dual-hosted git repository. isjarana pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git. from f368372 Fetching resource level metadata new 4e9880d Add parser interface, remove duplicate object creation at Neo4J new dc26af3 Add merger for updates new 1909dca Merge pull request #2 from isururanawaka/metadata_service The 19 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../airavata/datalake/metadata/clients/Test.java | 70 - .../airavata/datalake/metadata/AppConfig.java | 41 -- .../datalake/metadata/Neo4JServiceInitializer.java | 2 + .../neo4j/curd/operators/GenericService.java | 14 +- .../backend/neo4j/curd/operators/GroupService.java | 1 + .../neo4j/curd/operators/GroupServiceImpl.java | 28 .../neo4j/curd/operators/ResourceService.java | 4 + .../neo4j/curd/operators/ResourceServiceImpl.java | 52 ++- .../backend/neo4j/curd/operators/Service.java | 6 +- .../neo4j/curd/operators/TenantService.java| 2 + .../neo4j/curd/operators/TenantServiceImpl.java| 34 + .../backend/neo4j/curd/operators/UserService.java | 1 + .../neo4j/curd/operators/UserServiceImpl.java | 28 .../metadata/backend/neo4j/model/nodes/Entity.java | 19 +++ .../metadata/backend/neo4j/model/nodes/Group.java | 5 + .../backend/neo4j/model/nodes/Resource.java| 5 + .../metadata/backend/neo4j/model/nodes/Tenant.java | 5 + .../metadata/backend/neo4j/model/nodes/User.java | 5 + .../metadata/handlers/GroupServiceHandler.java | 79 -- .../metadata/handlers/ResourceServiceHandler.java | 111 -- .../metadata/handlers/TenantServiceHandler.java| 77 -- .../metadata/handlers/UserServiceHandler.java | 48 ++- .../metadata/interceptors/Authenticator.java | 19 +++ .../interceptors/InterceptorPipelineExecutor.java | 69 + .../metadata/interceptors/ServiceInterceptor.java | 7 + .../datalake/metadata/mergers/GenericMerger.java | 46 ++ .../airavata/datalake/metadata/mergers/Merger.java | 9 ++ .../metadata/parsers/ExecutionContext.java | 21 +++ .../datalake/metadata/parsers/GroupParser.java | 111 -- .../airavata/datalake/metadata/parsers/Parser.java | 16 +++ .../datalake/metadata/parsers/ResourceParser.java | 160 ++--- .../datalake/metadata/parsers/TenantParser.java| 126 ++-- .../datalake/metadata/parsers/UserParser.java | 39 - .../stub/src/main/proto/resource/Resource.proto| 33 - .../stub/src/main/proto/tenant/Tenant.proto| 6 +- 35 files changed, 1095 insertions(+), 204 deletions(-) create mode 100644 metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/interceptors/Authenticator.java create mode 100644 metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/interceptors/InterceptorPipelineExecutor.java create mode 100644 metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/interceptors/ServiceInterceptor.java create mode 100644 metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/mergers/GenericMerger.java create mode 100644 metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/mergers/Merger.java create mode 100644 metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/ExecutionContext.java create mode 100644 metadata-service/db-service/server/src/main/java/org/apache/airavata/datalake/metadata/parsers/Parser.java
[airavata] branch master updated: Ansible: set api server keystore file to letsencrypt
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/master by this push:
new d0cac32 Ansible: set api server keystore file to letsencrypt
d0cac32 is described below
commit d0cac328eda705d7361a4dd6d20bfbd21071f85e
Author: Marcus Christie
AuthorDate: Thu Mar 25 09:34:36 2021 -0400
Ansible: set api server keystore file to letsencrypt
---
dev-tools/ansible/inventories/scigap/production/group_vars/all/vars.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/dev-tools/ansible/inventories/scigap/production/group_vars/all/vars.yml
b/dev-tools/ansible/inventories/scigap/production/group_vars/all/vars.yml
index c2617ad..c684832 100644
--- a/dev-tools/ansible/inventories/scigap/production/group_vars/all/vars.yml
+++ b/dev-tools/ansible/inventories/scigap/production/group_vars/all/vars.yml
@@ -90,7 +90,7 @@ registry_port: 8970
default_gateway: "default"
# Credential and keystore related variables
-keystore_src_path: "{{inventory_dir}}/files/airavata.jks"
+keystore_src_path: "{{inventory_dir}}/files/letsencrypt_apiprod.jks"
keystore_passwd: "{{ vault_keystore_passwd }}"
# client_truststore_src_path: "{{inventory_dir}}/files/client_truststore.jks"
# client_truststore_passwd: "{{ vault_client_truststore_passwd }}"
[airavata] branch master updated: adding apiprod letsencrypt keystore
This is an automated email from the ASF dual-hosted git repository. smarru pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata.git The following commit(s) were added to refs/heads/master by this push: new d6ad14d adding apiprod letsencrypt keystore d6ad14d is described below commit d6ad14d1c5ee34ea2cfb4a25379c1209d351a4da Author: Suresh Marru AuthorDate: Thu Mar 25 09:10:44 2021 -0400 adding apiprod letsencrypt keystore --- .../scigap/production/files/letsencrypt_apiprod.jks | Bin 0 -> 4275 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/dev-tools/ansible/inventories/scigap/production/files/letsencrypt_apiprod.jks b/dev-tools/ansible/inventories/scigap/production/files/letsencrypt_apiprod.jks new file mode 100644 index 000..a6b5cff Binary files /dev/null and b/dev-tools/ansible/inventories/scigap/production/files/letsencrypt_apiprod.jks differ
[airavata-django-portal] 01/01: Merge branch 'master' into delta-topology-workshop
This is an automated email from the ASF dual-hosted git repository. machristie pushed a commit to branch delta-topology-workshop in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git commit a1f14fc91718efcbcf326f4b8ae80d7bcdb22b55 Merge: 65c62de 0870a6b Author: Marcus Christie AuthorDate: Thu Mar 25 08:27:45 2021 -0400 Merge branch 'master' into delta-topology-workshop .nvmrc | 1 + .travis.yml| 4 +- README.md | 7 +- .../GatewayResourceProfileEditorContainer.vue | 3 +- django_airavata/apps/api/output_views.py | 65 ++- django_airavata/apps/api/serializers.py| 38 +- .../api/static/django_airavata_api/js/index.js | 2 +- .../django_airavata_api/js/service_config.js | 14 +- django_airavata/apps/api/urls.py | 5 +- django_airavata/apps/api/view_utils.py | 10 +- django_airavata/apps/api/views.py | 77 ++-- django_airavata/apps/auth/backends.py | 36 +- django_airavata/apps/auth/signals.py | 28 ++ django_airavata/apps/auth/utils.py | 14 +- django_airavata/apps/workspace/apps.py | 3 - django_airavata/apps/workspace/package.json| 6 +- django_airavata/apps/workspace/signals.py | 26 -- .../js/components/experiment/ExperimentSummary.vue | 10 +- .../experiment/input-editors/FileInputEditor.vue | 41 +- .../input-editors/InputEditorContainer.vue | 4 + .../input-editors/RangeSliderInputEditor.vue | 95 .../experiment/input-editors/SliderInputEditor.vue | 83 .../output-displays/DefaultOutputDisplay.vue | 10 +- .../output-displays/OutputViewDataLoader.js| 5 + .../components/storage/UserStoragePathViewer.vue | 29 +- .../storage-edit/UserStorageAudioEditViewer.vue| 15 +- .../storage-edit/UserStorageDefaultEditViewer.vue | 13 +- .../storage-edit/UserStorageDownloadButton.vue | 12 +- .../storage/storage-edit/UserStorageEditViewer.vue | 61 ++- .../storage-edit/UserStorageImageEditViewer.vue| 15 +- .../storage/storage-edit/UserStorageLink.vue | 54 +++ .../storage-edit/UserStoragePdfEditViewer.vue | 15 +- .../storage-edit/UserStorageTextEditViewer.vue | 30 +- .../storage-edit/UserStorageVideoEditViewer.vue| 15 +- .../js/containers/ExperimentListContainer.vue | 195 +++- .../js/containers/UserStorageContainer.vue | 117 +++-- .../js/entry-create-experiment.js | 1 + .../js/entry-edit-experiment.js| 1 + .../js/entry-experiment-list.js| 3 + .../js/entry-user-storage.js | 2 +- .../apps/workspace/static/scss/styles.scss | 5 + django_airavata/apps/workspace/yarn.lock | 492 - docs/admin/app_inputs.md | 79 +++- docs/dev/custom_output_view_provider.md| 60 +++ docs/dev/developing_frontend.md| 6 + docs/tutorial/gateways_tutorial.md | 6 +- mkdocs.yml | 2 + requirements-mysql.txt | 2 +- 48 files changed, 1515 insertions(+), 302 deletions(-)
[airavata-django-portal] branch delta-topology-workshop updated (65c62de -> a1f14fc)
This is an automated email from the ASF dual-hosted git repository. machristie pushed a change to branch delta-topology-workshop in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git. from 65c62de Merge branch 'master' into delta-topology-workshop add 33878fe AIRAVATA-3310: Adding an enpoint to update a data product content by uri add ee9a9ee AIRAVATA-3310: Enabling the file edit at the experiment create add a574a2d AIRAVATA-3310: Refactoring add 26d15d5 AIRAVATA-3310: Refactoring add 7254130 AIRAVATA-3310: Adding a generic component for user storage links add 41852e0 AIRAVATA-3310: Refactoring add 4a229d4 AIRAVATA-3310: Enabling the file edit page to allow fetch by dataProductUri add 4200a51 AIRAVATA-3310: Revert changes in DataProductViewer.vue add 117a41d AIRAVATA-3310: Refactoring add 3c04928 AIRAVATA-3310: Fix the `Open in a new window` link of storage link component add 7a86fa4 AIRAVATA-3310: Refactoring add 2421097 AIRAVATA-3310: Fix unused imports add c990d4c Merge branch 'develop' of https://github.com/apache/airavata-django-portal into AIRAVATA-3310-enabling-file-edit-at-the-experiments add e6b035a flake8 fixes add c32774e AIRAVATA-3310: Refactoring add d5cdfd1 AIRAVATA-3399 Only send new user email when user profile created add d2a24e6 Handle adding authz_token to request in login handler add 5667aa9 Adding .nvmrc for specifying Node.js version to use add f8742b7 AIRAVATA-3398 Remove REST methods for any gateway resource profile add b21e754 Merge branch 'AIRAVATA-3398' into develop add 7962747 Adding recommended tools for frontend dev add 7e6a729 Merge branch 'develop' of https://github.com/apache/airavata-django-portal into AIRAVATA-3310-enabling-file-edit-at-the-experiments add 2671370 Refactoring add cce7e32 Refactoring add 37fb25b AIRAVATA-3310: Taking the breadcrumb off from the file preview add 07ba5f6 AIRAVATA-3310: Remove unstable `path` variable from the dataProduct model add e5f170c Adding support for URI_COLLECTION to output view providers add 8e4bc9d Upgrade mysqlclient dependency add babf1db Merge branch 'airavata-3403' into develop add 2affc2f AIRAVATA-3406 Fix ca cert bundle verification add 86d043d AIRAVATA-3406 provide better error message when login fails due to misconfiguration add 876de73 AIRAVATA-3362 moved get_rel_experiment_dir to user_storage in SDK add 611864f Preserve query params in pagination next/previous links add 2b4e51f AIRAVATA-3400 Slider and Range Slider components add 097697f Merge branch 'airavata-3400' into develop add ed52490 Merge branch 'develop' of https://github.com/apache/airavata-django-portal into AIRAVATA-3310-enabling-file-edit-at-the-experiments add 7638356 AIRAVATA-3310: Remove unused imports add b7f0ddc Merge pull request #52 from dinukadesilva/AIRAVATA-3310-enabling-file-edit-at-the-experiments add ff9996b Adding search options to the experiment list add 9df1b54 Merge pull request #53 from akbranam/develop add 64f3ce1 Merge branch 'airavata-3412' into develop add 789cf4e Use data_product_uri overload add 65635f1 Add back the test_output_file feature of output view providers add b20d047 fix typo add 0870a6b Merge branch 'staging' new a1f14fc Merge branch 'master' into delta-topology-workshop The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .nvmrc | 1 + .travis.yml| 4 +- README.md | 7 +- .../GatewayResourceProfileEditorContainer.vue | 3 +- django_airavata/apps/api/output_views.py | 65 ++- django_airavata/apps/api/serializers.py| 38 +- .../api/static/django_airavata_api/js/index.js | 2 +- .../django_airavata_api/js/service_config.js | 14 +- django_airavata/apps/api/urls.py | 5 +- django_airavata/apps/api/view_utils.py | 10 +- django_airavata/apps/api/views.py | 77 ++-- django_airavata/apps/auth/backends.py | 36 +- django_airavata/apps/auth/signals.py | 28 ++ django_airavata/apps/auth/utils.py | 14 +- django_airavata/apps/workspace/apps.py | 3 - django_airavata/apps/workspace/package.json| 6 +- django_airavata/apps/workspace/signals.py | 26 -- .../js/components/experiment/ExperimentSummary.vue | 10 +- .../experiment/input-editors/FileInputEditor.vue | 41 +- .../input-editors/InputEditorContainer.vue
[airavata-django-portal] branch staging updated (65635f1 -> b20d047)
This is an automated email from the ASF dual-hosted git repository. machristie pushed a change to branch staging in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git. from 65635f1 Add back the test_output_file feature of output view providers add b20d047 fix typo No new revisions were added by this update. Summary of changes: .../django_airavata_workspace/js/containers/ExperimentListContainer.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[airavata] branch master updated: AIRAVATA-3369 db migration added to ide-integration scripts
This is an automated email from the ASF dual-hosted git repository. machristie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata.git The following commit(s) were added to refs/heads/master by this push: new 19c817d AIRAVATA-3369 db migration added to ide-integration scripts 19c817d is described below commit 19c817d335d48de1820284c0b907ef5eb7c5d105 Author: Marcus Christie AuthorDate: Thu Mar 25 08:18:48 2021 -0400 AIRAVATA-3369 db migration added to ide-integration scripts --- .../resources/database_scripts/init/03-appcatalog-migrations.sql| 3 +++ .../resources/database_scripts/init/04-expcatalog-migrations.sql| 6 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/ide-integration/src/main/resources/database_scripts/init/03-appcatalog-migrations.sql b/modules/ide-integration/src/main/resources/database_scripts/init/03-appcatalog-migrations.sql index 0119ceb..df81298 100644 --- a/modules/ide-integration/src/main/resources/database_scripts/init/03-appcatalog-migrations.sql +++ b/modules/ide-integration/src/main/resources/database_scripts/init/03-appcatalog-migrations.sql @@ -14,3 +14,6 @@ ALTER TABLE COMPUTE_RESOURCE_RESERVATION ADD CONSTRAINT FK_COMPUTE_RESOURCE_RESE alter table GROUP_COMPUTE_RESOURCE_PREFERENCE drop column IF EXISTS RESERVATION; alter table GROUP_COMPUTE_RESOURCE_PREFERENCE drop column IF EXISTS RESERVATION_START_TIME; alter table GROUP_COMPUTE_RESOURCE_PREFERENCE drop column IF EXISTS RESERVATION_END_TIME; + +-- AIRAVATA-3369: Convert USER_FRIENDLY_DESC from VARCHAR to TEXT (CLOB) +alter table APPLICATION_INPUT modify column USER_FRIENDLY_DESC TEXT; diff --git a/modules/ide-integration/src/main/resources/database_scripts/init/04-expcatalog-migrations.sql b/modules/ide-integration/src/main/resources/database_scripts/init/04-expcatalog-migrations.sql index 1be66c6..4367d25 100644 --- a/modules/ide-integration/src/main/resources/database_scripts/init/04-expcatalog-migrations.sql +++ b/modules/ide-integration/src/main/resources/database_scripts/init/04-expcatalog-migrations.sql @@ -8,4 +8,8 @@ CREATE TABLE GATEWAY_USAGE_REPORTING_COMMAND ( COMPUTE_RESOURCE_ID VARCHAR(255) NOT NULL, COMMAND LONGTEXT NOT NULL, PRIMARY KEY (GATEWAY_ID, COMPUTE_RESOURCE_ID) -)ENGINE=InnoDB DEFAULT CHARSET=latin1; \ No newline at end of file +)ENGINE=InnoDB DEFAULT CHARSET=latin1; + +-- AIRAVATA-3369: Convert USER_FRIENDLY_DESCRIPTION from VARCHAR to TEXT (CLOB) +alter table EXPERIMENT_INPUT modify column USER_FRIENDLY_DESCRIPTION TEXT; +alter table PROCESS_INPUT modify column USER_FRIENDLY_DESCRIPTION TEXT;
[airavata-django-portal] 01/01: Merge branch 'staging'
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit 0870a6b568616620747da5e4d4c002065855f000
Merge: d2fca49 b20d047
Author: Marcus Christie
AuthorDate: Thu Mar 25 08:15:06 2021 -0400
Merge branch 'staging'
.nvmrc | 1 +
.travis.yml| 4 +-
README.md | 7 +-
.../GatewayResourceProfileEditorContainer.vue | 3 +-
django_airavata/apps/api/output_views.py | 65 ++-
django_airavata/apps/api/serializers.py| 38 +-
.../api/static/django_airavata_api/js/index.js | 2 +-
.../django_airavata_api/js/service_config.js | 14 +-
django_airavata/apps/api/urls.py | 5 +-
django_airavata/apps/api/view_utils.py | 10 +-
django_airavata/apps/api/views.py | 77 ++--
django_airavata/apps/auth/backends.py | 36 +-
django_airavata/apps/auth/signals.py | 28 ++
django_airavata/apps/auth/utils.py | 14 +-
django_airavata/apps/workspace/apps.py | 3 -
django_airavata/apps/workspace/package.json| 6 +-
django_airavata/apps/workspace/signals.py | 26 --
.../js/components/experiment/ExperimentSummary.vue | 10 +-
.../experiment/input-editors/FileInputEditor.vue | 41 +-
.../input-editors/InputEditorContainer.vue | 4 +
.../input-editors/RangeSliderInputEditor.vue | 95
.../experiment/input-editors/SliderInputEditor.vue | 83
.../output-displays/DefaultOutputDisplay.vue | 10 +-
.../output-displays/OutputViewDataLoader.js| 5 +
.../components/storage/UserStoragePathViewer.vue | 29 +-
.../storage-edit/UserStorageAudioEditViewer.vue| 15 +-
.../storage-edit/UserStorageDefaultEditViewer.vue | 13 +-
.../storage-edit/UserStorageDownloadButton.vue | 12 +-
.../storage/storage-edit/UserStorageEditViewer.vue | 61 ++-
.../storage-edit/UserStorageImageEditViewer.vue| 15 +-
.../storage/storage-edit/UserStorageLink.vue | 54 +++
.../storage-edit/UserStoragePdfEditViewer.vue | 15 +-
.../storage-edit/UserStorageTextEditViewer.vue | 30 +-
.../storage-edit/UserStorageVideoEditViewer.vue| 15 +-
.../js/containers/ExperimentListContainer.vue | 195 +++-
.../js/containers/UserStorageContainer.vue | 117 +++--
.../js/entry-create-experiment.js | 1 +
.../js/entry-edit-experiment.js| 1 +
.../js/entry-experiment-list.js| 3 +
.../js/entry-user-storage.js | 2 +-
.../apps/workspace/static/scss/styles.scss | 5 +
django_airavata/apps/workspace/yarn.lock | 492 -
docs/admin/app_inputs.md | 79 +++-
docs/dev/custom_output_view_provider.md| 60 +++
docs/dev/developing_frontend.md| 6 +
docs/tutorial/gateways_tutorial.md | 6 +-
mkdocs.yml | 2 +
requirements-mysql.txt | 2 +-
48 files changed, 1515 insertions(+), 302 deletions(-)
diff --cc docs/dev/custom_output_view_provider.md
index 324470d,84cdcb8..db4bc5e
--- a/docs/dev/custom_output_view_provider.md
+++ b/docs/dev/custom_output_view_provider.md
@@@ -89,21 -98,20 +98,35 @@@ def generate_data(self, request, experi
}
```
+ For the output view provider to work with experiment outputs of type
+ URI_COLLECTION, add `output_files=None` to the function signature and get the
+ output as a list of file objects.
+
+ ```python
+ # For URI_COLLECTION, add output_files=None to signature
+ def generate_data(self, request, experiment_output, experiment,
output_files=None, **kwargs):
+
+ # Return a dictionary
+ return {
+ #...
+ }
+ ```
+
+The arguments to the `generate_data` function are described below:
+
+- `request` -
+[Django
Request](https://docs.djangoproject.com/en/dev/ref/request-response/#django.http.HttpRequest)
+object.
+- `experiment_output` - Airavata metadata about output file(s), see
+[OutputDataObjectType
doc](http://airavata.apache.org/api-docs/master/application_io_models.html#Struct_OutputDataObjectType).
+- `experiment` - Airavata metadata about the experiment, see
+[ExperimentModel
doc](http://airavata.apache.org/api-docs/master/experiment_model.html#Struct_ExperimentModel).
+- `output_file` - Python
+[file-like
object](https://docs.python.org/3/glossary.html#term-file-object).
+Read from this file to process the contents of the output file.
+- `output_files` - If the output type is URI_COLLECTION, then the collection
+of files is given as a list of file-like objects.
+
