This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
The following commit(s) were added to refs/heads/master by this push:
new 063c10b Validating users before processing in orchestrator
063c10b is described below
commit 063c10bb74820a9f91832c29a7093d7a6c586359
Author: Dimuthu Wannipurage <[email protected]>
AuthorDate: Mon Nov 8 20:29:27 2021 -0500
Validating users before processing in orchestrator
---
.../templates/data-orchestrator/config.yml.j2 | 7 +-
.../src/main/dist/conf/config.yml | 7 +-
.../datalake/orchestrator/Configuration.java | 49 ++++++++++++++
.../orchestrator/connectors/CustosConnector.java | 77 ++++++++++++++++++++++
.../orchestrator/connectors/DRMSConnector.java | 17 +++++
.../connectors/WorkflowServiceConnector.java | 17 +++++
.../handlers/async/OrchestratorEventProcessor.java | 22 ++++++-
pom.xml | 4 +-
8 files changed, 194 insertions(+), 6 deletions(-)
diff --git a/ansible/roles/data_lake/templates/data-orchestrator/config.yml.j2
b/ansible/roles/data_lake/templates/data-orchestrator/config.yml.j2
index 7816641..aaefbf7 100644
--- a/ansible/roles/data_lake/templates/data-orchestrator/config.yml.j2
+++ b/ansible/roles/data_lake/templates/data-orchestrator/config.yml.j2
@@ -22,4 +22,9 @@ consumer:
tenantConfigs:
tenantId: "{{ datalake_synch_tenant_id }}"
userGroup: "{{ datalake_data_orch_user_group }}"
- adminGroup: "{{ datalake_data_orch_admin_group }}"
\ No newline at end of file
+ adminGroup: "{{ datalake_data_orch_admin_group }}"
+custosConfigs:
+ serverHost: "{{ datalake_drms_custos_host }}"
+ serverPort: {{ datalake_drms_custos_port }}
+ clientId: "{{ datalake_drms_custos_client_id }}"
+ clientSec: "{{ datalake_drms_custos_client_secret }}"
\ No newline at end of file
diff --git
a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/dist/conf/config.yml
b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/dist/conf/config.yml
index 8887fb4..1fea939 100644
---
a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/dist/conf/config.yml
+++
b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/dist/conf/config.yml
@@ -20,4 +20,9 @@ consumer:
tenantConfigs:
tenantId: "custos-ii8g0cfwsz6ruwezykn9-10002640"
userGroup: "emc-users_e8a37f6d-e3b3-4e4a-9081-265bb42b1b99"
- adminGroup: "admin-group_c1aac070-1512-4e98-b1d1-e06a94f03665"
\ No newline at end of file
+ adminGroup: "admin-group_c1aac070-1512-4e98-b1d1-e06a94f03665"
+custosConfigs:
+ serverHost: "custos.scigap.org"
+ serverPort: 31499
+ clientId: "custos-ii8g0cfwsz6ruwezykn9-10002640"
+ clientSec: "secret"
\ No newline at end of file
diff --git
a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/Configuration.java
b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/Configuration.java
index 0f33438..232e47f 100644
---
a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/Configuration.java
+++
b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/Configuration.java
@@ -22,6 +22,8 @@ public class Configuration {
public TenantConfigs tenantConfigs;
+ private CustosConfigs custosConfigs;
+
public Configuration() {
}
@@ -70,6 +72,14 @@ public class Configuration {
this.tenantConfigs = tenantConfigs;
}
+ public CustosConfigs getCustosConfigs() {
+ return custosConfigs;
+ }
+
+ public void setCustosConfigs(CustosConfigs custosConfigs) {
+ this.custosConfigs = custosConfigs;
+ }
+
public static class Consumer {
private String brokerURL;
@@ -275,5 +285,44 @@ public class Configuration {
}
}
+ public static class CustosConfigs {
+
+ private String serverHost;
+ private int serverPort;
+ private String clientId;
+ private String clientSec;
+
+ public String getServerHost() {
+ return serverHost;
+ }
+
+ public void setServerHost(String serverHost) {
+ this.serverHost = serverHost;
+ }
+
+ public int getServerPort() {
+ return serverPort;
+ }
+
+ public void setServerPort(int serverPort) {
+ this.serverPort = serverPort;
+ }
+
+ public String getClientId() {
+ return clientId;
+ }
+
+ public void setClientId(String clientId) {
+ this.clientId = clientId;
+ }
+
+ public String getClientSec() {
+ return clientSec;
+ }
+
+ public void setClientSec(String clientSec) {
+ this.clientSec = clientSec;
+ }
+ }
}
diff --git
a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/CustosConnector.java
b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/CustosConnector.java
new file mode 100644
index 0000000..937e714
--- /dev/null
+++
b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/CustosConnector.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.datalake.orchestrator.connectors;
+
+import org.apache.airavata.datalake.orchestrator.Configuration;
+import
org.apache.airavata.datalake.orchestrator.core.connector.AbstractConnector;
+import org.apache.custos.clients.CustosClientProvider;
+import org.apache.custos.iam.service.FindUsersResponse;
+import org.apache.custos.iam.service.UserRepresentation;
+import org.apache.custos.user.management.client.UserManagementClient;
+
+import java.util.Optional;
+
+public class CustosConnector implements AbstractConnector<Configuration> {
+
+ private UserManagementClient umClient = null;
+
+ public CustosConnector(Configuration configuration) throws Exception {
+ this.init(configuration);
+ }
+
+ @Override
+ public void init(Configuration configuration) throws Exception {
+ CustosClientProvider clientProvider = new
CustosClientProvider.Builder()
+
.setServerHost(configuration.getCustosConfigs().getServerHost())
+
.setServerPort(configuration.getCustosConfigs().getServerPort())
+ .setClientId(configuration.getCustosConfigs().getClientId())
+
.setClientSec(configuration.getCustosConfigs().getClientSec()).build();
+
+ this.umClient = clientProvider.getUserManagementClient();
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (isOpen()) {
+ umClient.close();
+ }
+ }
+
+ public Optional<UserRepresentation> findUserByUserName(String userName) {
+ FindUsersResponse userResp = umClient.findUser(userName, "", "", "",
0, 1);
+ if (userResp.getUsersCount() == 0) {
+ return Optional.empty();
+ } else {
+ return Optional.of(userResp.getUsers(0));
+ }
+ }
+
+ public Optional<UserRepresentation> findUserByEmail(String email) {
+ FindUsersResponse userResp = umClient.findUser("", "", "", email, 0,
1);
+ if (userResp.getUsersCount() == 0) {
+ return Optional.empty();
+ } else {
+ return Optional.of(userResp.getUsers(0));
+ }
+ }
+
+ @Override
+ public boolean isOpen() {
+ return umClient != null;
+ }
+}
diff --git
a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/DRMSConnector.java
b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/DRMSConnector.java
index d653482..f943449 100644
---
a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/DRMSConnector.java
+++
b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/DRMSConnector.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.airavata.datalake.orchestrator.connectors;
import com.google.protobuf.Struct;
diff --git
a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/WorkflowServiceConnector.java
b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/WorkflowServiceConnector.java
index d1898cb..11f90a1 100644
---
a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/WorkflowServiceConnector.java
+++
b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/connectors/WorkflowServiceConnector.java
@@ -1,5 +1,22 @@
package org.apache.airavata.datalake.orchestrator.connectors;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
import org.apache.airavata.datalake.orchestrator.Configuration;
diff --git
a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/handlers/async/OrchestratorEventProcessor.java
b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/handlers/async/OrchestratorEventProcessor.java
index 9bd855d..308a661 100644
---
a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/handlers/async/OrchestratorEventProcessor.java
+++
b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/handlers/async/OrchestratorEventProcessor.java
@@ -25,6 +25,7 @@ import
org.apache.airavata.datalake.drms.storage.AnyStoragePreference;
import org.apache.airavata.datalake.drms.storage.TransferMapping;
import org.apache.airavata.datalake.orchestrator.Configuration;
import org.apache.airavata.datalake.orchestrator.Utils;
+import org.apache.airavata.datalake.orchestrator.connectors.CustosConnector;
import org.apache.airavata.datalake.orchestrator.connectors.DRMSConnector;
import
org.apache.airavata.datalake.orchestrator.connectors.WorkflowServiceConnector;
import org.apache.airavata.dataorchestrator.clients.core.NotificationClient;
@@ -35,6 +36,7 @@ import
org.apache.airavata.mft.api.service.FileMetadataResponse;
import org.apache.airavata.mft.api.service.MFTApiServiceGrpc;
import org.apache.airavata.mft.common.AuthToken;
import org.apache.airavata.mft.common.DelegateAuth;
+import org.apache.custos.iam.service.UserRepresentation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,6 +53,7 @@ public class OrchestratorEventProcessor implements Runnable {
private final WorkflowServiceConnector workflowServiceConnector;
private final Set<String> eventCache;
private final NotificationClient notificationClient;
+ private final CustosConnector custosConnector;
public OrchestratorEventProcessor(Configuration configuration,
Notification notificationEvent,
Set<String> eventCache,
NotificationClient notificationClient) throws Exception {
@@ -60,6 +63,7 @@ public class OrchestratorEventProcessor implements Runnable {
this.workflowServiceConnector = new
WorkflowServiceConnector(configuration);
this.configuration = configuration;
this.notificationClient = notificationClient;
+ this.custosConnector = new CustosConnector(configuration);
}
private List<GenericResource> createResourceWithParentDirectories(String
hostName, String storageId, String basePath,
@@ -156,6 +160,20 @@ public class OrchestratorEventProcessor implements
Runnable {
}
}
+ private String verifyUser(String userName) throws Exception {
+ if (custosConnector.findUserByUserName(userName).isEmpty()) {
+ Optional<UserRepresentation> userByEmail =
custosConnector.findUserByEmail(userName);
+ if (userByEmail.isPresent()) {
+ return userByEmail.get().getUsername();
+ } else {
+ logger.error("No user {} by email or user name", userName);
+ throw new Exception("Could not find the user " + userName);
+ }
+ } else {
+ return userName;
+ }
+ }
+
@Override
public void run() {
logger.info("Processing resource path {} on storage {}",
notification.getResourcePath(),
@@ -188,8 +206,8 @@ public class OrchestratorEventProcessor implements Runnable
{
throw new Exception("Invalid path. Need at least two folder
levels from base");
}
- String adminUser = splitted[0];
- String owner = splitted[1].split("_")[0];
+ String adminUser = verifyUser(splitted[0]);
+ String owner = verifyUser(splitted[1].split("_")[0]);
Map<String, String> ownerRules = new HashMap<>();
ownerRules.put(adminUser, "VIEWER");
diff --git a/pom.xml b/pom.xml
index cba4a14..24676ba 100644
--- a/pom.xml
+++ b/pom.xml
@@ -59,8 +59,8 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.6.1</version>
<configuration>
- <source>1.8</source>
- <target>1.8</target>
+ <source>11</source>
+ <target>11</target>
<fork>true</fork>
</configuration>
</plugin>
