[airavata-mft] branch master updated: Decoupling transfer request accept and process threads in Agent for optimal performance
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
The following commit(s) were added to refs/heads/master by this push:
new 2b2eab6 Decoupling transfer request accept and process threads in
Agent for optimal performance
2b2eab6 is described below
commit 2b2eab69ce4888b7fe3ab2ded844eab9f8384671
Author: Dimuthu Wannipurage
AuthorDate: Tue Apr 5 19:22:43 2022 -0400
Decoupling transfer request accept and process threads in Agent for optimal
performance
---
.../org/apache/airavata/mft/agent/MFTAgent.java| 218 +-
.../airavata/mft/agent/TransportMediator.java | 247 ++---
.../mft/core/api/IncomingChunkedConnector.java | 2 +
.../mft/core/api/OutgoingChunkedConnector.java | 2 +
.../mft/transport/s3/S3IncomingConnector.java | 4 +-
.../mft/transport/s3/S3OutgoingConnector.java | 3 +-
6 files changed, 241 insertions(+), 235 deletions(-)
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
index 15f316e..5a61d31 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
+++ b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
@@ -55,10 +55,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
@SpringBootApplication(exclude = DataSourceAutoConfiguration.class)
public class MFTAgent implements CommandLineRunner {
@@ -122,6 +119,8 @@ public class MFTAgent implements CommandLineRunner {
private long sessionRenewSeconds = 4;
private long sessionTTLSeconds = 10;
private String session;
+private ExecutorService transferRequestExecutor;
+
private TransportMediator mediator;
@@ -141,6 +140,7 @@ public class MFTAgent implements CommandLineRunner {
transferMessageCache = KVCache.newCache(mftConsulClient.getKvClient(),
MFTConsulClient.AGENTS_TRANSFER_REQUEST_MESSAGE_PATH + agentId);
rpcMessageCache = KVCache.newCache(mftConsulClient.getKvClient(),
MFTConsulClient.AGENTS_RPC_REQUEST_MESSAGE_PATH + agentId);
mediator = new TransportMediator(tempDataDir, concurrentTransfers,
concurrentChunkedThreads, chunkedSize);
+transferRequestExecutor =
Executors.newFixedThreadPool(concurrentTransfers);
}
private void acceptRPCRequests() {
@@ -164,6 +164,110 @@ public class MFTAgent implements CommandLineRunner {
rpcMessageCache.start();
}
+private void processTransfer(String transferId, String
transferRequestJson) {
+logger.info("Received raw message: {}", transferRequestJson);
+TransferApiRequest request = null;
+try {
+TransferApiRequest.Builder builder =
TransferApiRequest.newBuilder();
+JsonFormat.parser().merge(transferRequestJson, builder);
+request = builder.build();
+
+logger.info("Received request " + transferId);
+mftConsulClient.submitTransferStateToProcess(transferId, agentId,
new TransferState()
+.setState("STARTING")
+.setPercentage(0)
+.setUpdateTimeMils(System.currentTimeMillis())
+.setPublisher(agentId)
+.setDescription("Starting the transfer"));
+
+Optional srcMetadataCollectorOp =
MetadataCollectorResolver.resolveMetadataCollector(request.getSourceType());
+MetadataCollector srcMetadataCollector =
srcMetadataCollectorOp.orElseThrow(() -> new Exception("Could not find a
metadata collector for source"));
+srcMetadataCollector.init(resourceServiceHost,
resourceServicePort, secretServiceHost, secretServicePort);
+
+Optional dstMetadataCollectorOp =
MetadataCollectorResolver.resolveMetadataCollector(request.getDestinationType());
+MetadataCollector dstMetadataCollector =
dstMetadataCollectorOp.orElseThrow(() -> new Exception("Could not find a
metadata collector for destination"));
+dstMetadataCollector.init(resourceServiceHost,
resourceServicePort, secretServiceHost, secretServicePort);
+
+FileResourceMetadata srcMetadata =
srcMetadataCollector.getFileResourceMetadata(
+request.getMftAuthorizationToken(),
+request.getSourceResourceId(),
+request.getSourceToken());
+
+
+ConnectorConfig srcCC =
ConnectorConfig.ConnectorConfigBuilder.newBuilder()
+.withAuthToken(request.getMftAuthorizationToken())
+.withResourceServiceHost(resourceServiceHost)
+
[airavata-mft] branch master updated: Proper error handling for transfer API
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
The following commit(s) were added to refs/heads/master by this push:
new e14d60b Proper error handling for transfer API
e14d60b is described below
commit e14d60b91bd760ac9d8e0134b0e3f7dd6ff4dc0a
Author: Dimuthu Wannipurage
AuthorDate: Tue Apr 5 19:19:40 2022 -0400
Proper error handling for transfer API
---
.../airavata/mft/api/handler/MFTApiHandler.java| 55 --
1 file changed, 40 insertions(+), 15 deletions(-)
diff --git
a/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java
b/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java
index cfca425..7e5d525 100644
---
a/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java
+++
b/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java
@@ -19,6 +19,7 @@ package org.apache.airavata.mft.api.handler;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.protobuf.util.JsonFormat;
+import io.grpc.Status;
import io.grpc.stub.StreamObserver;
import org.apache.airavata.mft.admin.MFTConsulClient;
import org.apache.airavata.mft.admin.SyncRPCClient;
@@ -85,7 +86,9 @@ public class MFTApiHandler extends
MFTTransferServiceGrpc.MFTTransferServiceImpl
responseObserver.onCompleted();
} catch (Exception e) {
logger.error("Error in submitting transfer request", e);
-responseObserver.onError(new Exception("Failed to submit
transfer", e));
+responseObserver.onError(Status.INTERNAL
+.withDescription("Failed to submit http download request.
" + e.getMessage())
+.asException());
}
}
@@ -127,14 +130,19 @@ public class MFTApiHandler extends
MFTTransferServiceGrpc.MFTTransferServiceImpl
case FAIL:
logger.error("Errored while processing the download
request to resource {}. Error msg : {}",
request.getSourceResourceId(),
rpcResponse.getErrorAsStr());
-responseObserver.onError(new Exception("Errored while
processing the the fetch file metadata response. Error msg : " +
-rpcResponse.getErrorAsStr()));
+
+responseObserver.onError(Status.INTERNAL
+.withDescription("Errored while processing the the
fetch file metadata response. Error msg : " +
+rpcResponse.getErrorAsStr())
+.asException());
}
} catch (Exception e) {
logger.error("Error while submitting http download request to
resource {}",
request.getSourceResourceId()
, e);
-responseObserver.onError(new Exception("Failed to submit http
download request", e));
+responseObserver.onError(Status.INTERNAL
+.withDescription("Failed to submit http download request.
" + e.getMessage())
+.asException());
}
}
@@ -149,7 +157,9 @@ public class MFTApiHandler extends
MFTTransferServiceGrpc.MFTTransferServiceImpl
responseObserver.onCompleted();
} catch (Exception e) {
logger.error("Error in fetching transfer states", e);
-responseObserver.onError(new Exception("Failed to retrieve
transfer states", e));
+responseObserver.onError(Status.INTERNAL
+.withDescription("Failed to retrieve transfer states. " +
e.getMessage())
+.asException());
}
}
@@ -162,13 +172,18 @@ public class MFTApiHandler extends
MFTTransferServiceGrpc.MFTTransferServiceImpl
TransferStateApiResponse s = dozerBeanMapper.map(stateOp.get(),
TransferStateApiResponse.newBuilder().getClass()).build();
responseObserver.onNext(s);
+responseObserver.onCompleted();
} else {
-
responseObserver.onNext(TransferStateApiResponse.getDefaultInstance());
+logger.error("There is no state for transfer " +
request.getTransferId());
+responseObserver.onError(Status.NOT_FOUND
+.withDescription("There is no state for transfer " +
request.getTransferId())
+.asRuntimeException());
}
-responseObserver.onCompleted();
} catch (Exception e) {
logger.error("Error in fetching transfer state", e);
-responseObserver.onError(new Exception("Failed to retrieve
transfer state", e));
+responseObserver.onError(Status.INTERNAL
+.withDescription("Failed to retrieve transfer state. " +
[airavata-django-portal] branch develop updated: Fixing typo in input dependencies schema
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
The following commit(s) were added to refs/heads/develop by this push:
new aac5a81b Fixing typo in input dependencies schema
aac5a81b is described below
commit aac5a81b7f4bb73b943db66d919288fbdc92495c
Author: Marcus Christie
AuthorDate: Tue Apr 5 16:17:48 2022 -0400
Fixing typo in input dependencies schema
---
.../static/django_airavata_api/js/models/InputDataObjectType.js | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git
a/django_airavata/apps/api/static/django_airavata_api/js/models/InputDataObjectType.js
b/django_airavata/apps/api/static/django_airavata_api/js/models/InputDataObjectType.js
index 320b8512..e3f1cb00 100644
---
a/django_airavata/apps/api/static/django_airavata_api/js/models/InputDataObjectType.js
+++
b/django_airavata/apps/api/static/django_airavata_api/js/models/InputDataObjectType.js
@@ -153,10 +153,10 @@ export default class InputDataObjectType extends
BaseModel {
* "editor": {
* "dependencies": {
* "show": {
- * "AND": [ // Boolean operator ("AND", "OR")
- * "INPUT_1": {// Name of other application input
- * "type": "equals", // Name of comparison type
- * "value": "1" // Value to compare with
+ * "AND": [ // Boolean operator ("AND", "OR")
+ * "INPUT_1": { // Name of other application input
+ * "comparison": "equals", // Name of comparison type
+ * "value": "1" // Value to compare with
* },
* "NOT": {// "NOT" is given a single input comparison
or "AND" or "OR" expression
* "INPUT_2": {
[airavata] branch master updated (f412aeb290 -> e12c2497d5)
This is an automated email from the ASF dual-hosted git repository. smarru pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/airavata.git from f412aeb290 AIRAVATA-3590 upgrade log4j and commons-io add e12c2497d5 AIRAVATA-3590 remove xmlbeans No new revisions were added by this update. Summary of changes: modules/distribution/pom.xml | 11 --- pom.xml | 2 -- 2 files changed, 13 deletions(-)
[airavata] branch dependabot/maven/org.apache.logging.log4j-log4j-core-2.17.1 created (now a90f183802)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch dependabot/maven/org.apache.logging.log4j-log4j-core-2.17.1 in repository https://gitbox.apache.org/repos/asf/airavata.git at a90f183802 Bump log4j-core from 2.16.0 to 2.17.1 No new revisions were added by this update.
[airavata] branch dependabot/maven/modules/ide-integration/src/main/containers/pga/airavata-php-gateway/app/tests/selenium/commons-io-commons-io-2.7 updated (a0357a67c9 -> 533fa8ddb5)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch dependabot/maven/modules/ide-integration/src/main/containers/pga/airavata-php-gateway/app/tests/selenium/commons-io-commons-io-2.7 in repository https://gitbox.apache.org/repos/asf/airavata.git discard a0357a67c9 Bump commons-io add f412aeb290 AIRAVATA-3590 upgrade log4j and commons-io add 533fa8ddb5 Bump commons-io This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (a0357a67c9) \ N -- N -- N refs/heads/dependabot/maven/modules/ide-integration/src/main/containers/pga/airavata-php-gateway/app/tests/selenium/commons-io-commons-io-2.7 (533fa8ddb5) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
[airavata] branch dependabot/maven/modules/ide-integration/src/main/containers/pga/airavata-php-gateway/app/tests/selenium/commons-io-commons-io-2.7 created (now a0357a67c9)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch dependabot/maven/modules/ide-integration/src/main/containers/pga/airavata-php-gateway/app/tests/selenium/commons-io-commons-io-2.7 in repository https://gitbox.apache.org/repos/asf/airavata.git at a0357a67c9 Bump commons-io No new revisions were added by this update.
[airavata] branch master updated: AIRAVATA-3590 upgrade log4j and commons-io
This is an automated email from the ASF dual-hosted git repository. smarru pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata.git The following commit(s) were added to refs/heads/master by this push: new f412aeb290 AIRAVATA-3590 upgrade log4j and commons-io f412aeb290 is described below commit f412aeb290dda5be31e950dd3a0d8561280dd357 Author: PJ Fanning AuthorDate: Thu Mar 17 13:42:39 2022 +0100 AIRAVATA-3590 upgrade log4j and commons-io --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 64507cd0ed..d7c9562e1c 100644 --- a/pom.xml +++ b/pom.xml @@ -79,7 +79,7 @@ UTF-8 10.13.1.1 1.7.25 -2.16.0 +2.17.2 3.0.0-M4 4.12 2.8.0 @@ -111,7 +111,7 @@ 3.3.0 1.0.0 20160212 -2.4 +2.11.0 20.0 0.8.1 3.1.0
[airavata] branch dependabot/maven/modules/ide-integration/src/main/containers/pga/airavata-php-gateway/app/tests/selenium/org.apache.httpcomponents-httpclient-4.5.13 created (now 14332ce082)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch dependabot/maven/modules/ide-integration/src/main/containers/pga/airavata-php-gateway/app/tests/selenium/org.apache.httpcomponents-httpclient-4.5.13 in repository https://gitbox.apache.org/repos/asf/airavata.git at 14332ce082 Bump httpclient No new revisions were added by this update.
[airavata] branch dependabot/maven/modules/ide-integration/src/main/containers/pga/airavata-php-gateway/app/tests/selenium/junit-junit-4.13.1 created (now 19218a0e69)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch dependabot/maven/modules/ide-integration/src/main/containers/pga/airavata-php-gateway/app/tests/selenium/junit-junit-4.13.1 in repository https://gitbox.apache.org/repos/asf/airavata.git at 19218a0e69 Bump junit No new revisions were added by this update.
[airavata] branch master updated (d3e29bcf03 -> 97a205cbc2)
This is an automated email from the ASF dual-hosted git repository. smarru pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/airavata.git from d3e29bcf03 Update email-config.yaml.j2 new 08a15d1123 jackson 2.9.10.8 new 97a205cbc2 update versions The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../profile-service/iam-admin-services-core/pom.xml | 2 +- modules/distribution/pom.xml | 16 pom.xml | 2 ++ 3 files changed, 11 insertions(+), 9 deletions(-)
[airavata] 01/02: jackson 2.9.10.8
This is an automated email from the ASF dual-hosted git repository.
smarru pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata.git
commit 08a15d11238a8a0444f12d7247eba54732f259a9
Author: PJ Fanning
AuthorDate: Thu Mar 17 14:19:25 2022 +0100
jackson 2.9.10.8
---
airavata-services/profile-service/iam-admin-services-core/pom.xml | 2 +-
modules/distribution/pom.xml | 2 +-
pom.xml | 1 +
3 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/airavata-services/profile-service/iam-admin-services-core/pom.xml
b/airavata-services/profile-service/iam-admin-services-core/pom.xml
index fd3b0acab1..7028f7b2d0 100644
--- a/airavata-services/profile-service/iam-admin-services-core/pom.xml
+++ b/airavata-services/profile-service/iam-admin-services-core/pom.xml
@@ -68,7 +68,7 @@
com.fasterxml.jackson.core
jackson-databind
-2.5.4
+${jackson.databind.version}
diff --git a/modules/distribution/pom.xml b/modules/distribution/pom.xml
index 2e2c35db04..37a8087aa6 100644
--- a/modules/distribution/pom.xml
+++ b/modules/distribution/pom.xml
@@ -324,7 +324,7 @@
com.fasterxml.jackson.core
jackson-databind
-2.9.4
+${jackson.databind.version}
com.fasterxml.jackson.core
diff --git a/pom.xml b/pom.xml
index ffe1ae8d69..ea291e72c9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -91,6 +91,7 @@
2.10.7
1.2.7
0.9.4
+2.9.10.8
1.13
1.3.1
1.1.3
