This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch staging
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/staging by this push:
new 8faef3b Adding a configuration to enable streaming transfer
8faef3b is described below
commit 8faef3bb9ff8c53907b14ea17b2dbcc867c83a32
Author: Dimuthu Wannipurage <dimuthu.upeks...@gmail.com>
AuthorDate: Mon Sep 16 00:19:03 2019 -0400
Adding a configuration to enable streaming transfer
---
.../participant/airavata-server.properties.j2 | 5 +++++
.../helix/impl/task/staging/DataStagingTask.java | 22 +++++++++++++++-------
.../apache/airavata/common/utils/Constants.java | 2 ++
.../airavata/common/utils/ServerSettings.java | 4 ++++
4 files changed, 26 insertions(+), 7 deletions(-)
diff --git
a/dev-tools/ansible/roles/helix_setup/templates/participant/airavata-server.properties.j2
b/dev-tools/ansible/roles/helix_setup/templates/participant/airavata-server.properties.j2
index 6a307a0..2e5e292 100644
---
a/dev-tools/ansible/roles/helix_setup/templates/participant/airavata-server.properties.j2
+++
b/dev-tools/ansible/roles/helix_setup/templates/participant/airavata-server.properties.j2
@@ -88,3 +88,8 @@ local.data.location={{ local_data_location }}
# Data Parsing Task Level Configurations
###########################################################################
data.parser.delete.container=True
+
+###########################################################################
+# Data Staging Task Level Configurations
+###########################################################################
+enable.streaming.transfer=True
\ No newline at end of file
diff --git
a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java
b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java
index 1fe08cb..9258dd0 100644
---
a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java
+++
b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java
@@ -302,10 +302,14 @@ public abstract class DataStagingTask extends
AiravataTask {
throw new TaskOnFailException("Failed to fetch metadata for file "
+ sourcePath, false, e);
}
- String sourceFileName =
sourcePath.substring(sourcePath.lastIndexOf(File.separator) + 1,
sourcePath.length());
- //String tempPath = getLocalDataPath(sourceFileName);
- //naiveTransfer(storageAdaptor, sourcePath, computeAdaptor, destPath,
tempPath);
- passThroughTransfer(storageAdaptor, sourcePath, computeAdaptor,
destPath);
+ if (ServerSettings.isSteamingEnabled()) {
+ passThroughTransfer(storageAdaptor, sourcePath, computeAdaptor,
destPath);
+ } else {
+ String sourceFileName =
sourcePath.substring(sourcePath.lastIndexOf(File.separator) + 1,
sourcePath.length());
+ String tempPath = getLocalDataPath(sourceFileName);
+ naiveTransfer(storageAdaptor, sourcePath, computeAdaptor,
destPath, tempPath);
+ }
+
}
protected boolean transferFileToStorage(String sourcePath, String
destPath, String fileName, AgentAdaptor adaptor,
@@ -338,9 +342,13 @@ public abstract class DataStagingTask extends AiravataTask
{
throw new TaskOnFailException("Error while checking the file " +
sourcePath + " existence", false, e);
}
- //String tempPath = getLocalDataPath(fileName);
- //naiveTransfer(adaptor, sourcePath, storageResourceAdaptor, destPath,
tempPath);
- passThroughTransfer(adaptor, sourcePath, storageResourceAdaptor,
destPath);
+
+ if (ServerSettings.isSteamingEnabled()) {
+ passThroughTransfer(adaptor, sourcePath, storageResourceAdaptor,
destPath);
+ } else {
+ String tempPath = getLocalDataPath(fileName);
+ naiveTransfer(adaptor, sourcePath, storageResourceAdaptor,
destPath, tempPath);
+ }
return true;
}
diff --git
a/modules/commons/src/main/java/org/apache/airavata/common/utils/Constants.java
b/modules/commons/src/main/java/org/apache/airavata/common/utils/Constants.java
index 8820d45..63c99ba 100644
---
a/modules/commons/src/main/java/org/apache/airavata/common/utils/Constants.java
+++
b/modules/commons/src/main/java/org/apache/airavata/common/utils/Constants.java
@@ -69,4 +69,6 @@ public final class Constants {
public static final String TRUSTED_CERTIFICATE_SYSTEM_PROPERTY =
"X509_CERT_DIR";
public static final String NEWLINE = System.getProperty("line.separator");
+
+ public static final String ENABLE_STREAMING_TRANSFER =
"enable.streaming.transfer";
}
diff --git
a/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
b/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
index e8ff43f..e740cf5 100644
---
a/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
+++
b/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
@@ -502,4 +502,8 @@ public class ServerSettings extends ApplicationSettings {
public static String getSharingRegistryHost() {
return getSetting(SHARING_REGISTRY_HOST, "localhost");
}
+
+ public static Boolean isSteamingEnabled() {
+ return Boolean.valueOf(getSetting(Constants.ENABLE_STREAMING_TRANSFER,
"True"));
+ }
}
