This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch jupyter-integration
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/jupyter-integration by this
push:
new 96a1529 Optimizing directory listing in Adaptors
96a1529 is described below
commit 96a1529f73c86f171ba9f79dd38272cf0c205ab5
Author: dimuthu <[email protected]>
AuthorDate: Wed Aug 29 15:45:06 2018 -0400
Optimizing directory listing in Adaptors
---
.../apache/airavata/agents/api/AgentAdaptor.java | 2 ++
.../org/apache/airavata/agents/api/FileInfo.java | 18 +++++++++++++++
.../agents/api/StorageResourceAdaptor.java | 1 +
.../airavata/helix/agent/ssh/SshAgentAdaptor.java | 5 ++++
.../airavata/helix/adaptor/SSHJAgentAdaptor.java | 27 ++++++++++++++++++----
.../api/service/handler/RegistryServerHandler.java | 14 +++++++----
6 files changed, 59 insertions(+), 8 deletions(-)
diff --git
a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java
b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java
index 5355d5c..5e119f2 100644
---
a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java
+++
b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java
@@ -46,6 +46,8 @@ public interface AgentAdaptor {
public List<String> listDirectory(String path) throws AgentException;
+ public List<FileInfo> listDirectoryWithInfo(String path) throws
AgentException;
+
public Boolean doesFileExist(String filePath) throws AgentException;
public List<String> getFileNameFromExtension(String fileName, String
parentPath) throws AgentException;
diff --git
a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/FileInfo.java
b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/FileInfo.java
index b435452..281a6cc 100644
---
a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/FileInfo.java
+++
b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/FileInfo.java
@@ -20,6 +20,8 @@
package org.apache.airavata.agents.api;
public class FileInfo {
+ private String name;
+ private String path;
private boolean isExist;
private boolean isFile;
private long size;
@@ -27,6 +29,22 @@ public class FileInfo {
private long modifiedDate;
private String owner;
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
public boolean isExist() {
return isExist;
}
diff --git
a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/StorageResourceAdaptor.java
b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/StorageResourceAdaptor.java
index d4c32f8..3bc9704 100644
---
a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/StorageResourceAdaptor.java
+++
b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/StorageResourceAdaptor.java
@@ -26,6 +26,7 @@ public interface StorageResourceAdaptor {
public void uploadFile(String sourceFile, String destFile) throws
AgentException;
public void downloadFile(String sourceFile, String destFile) throws
AgentException;
public List<String> listDirectory(String path) throws AgentException;
+ public List<FileInfo> listDirectoryWithInfo(String path) throws
AgentException;
public void deleteFile(String path) throws AgentException;
public void deleteDirectory(String path) throws AgentException;
public Boolean doesFileExist(String filePath) throws AgentException;
diff --git
a/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java
b/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java
index e4cffe9..9018f01 100644
---
a/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java
+++
b/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java
@@ -455,6 +455,11 @@ public class SshAgentAdaptor implements AgentAdaptor {
}
@Override
+ public List<FileInfo> listDirectoryWithInfo(String path) throws
AgentException {
+ throw new AgentException("Operation not implemented");
+ }
+
+ @Override
public Boolean doesFileExist(String filePath) throws AgentException {
String command = "ls " + filePath;
ChannelExec channelExec = null;
diff --git
a/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java
b/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java
index 182942b..54dc928 100644
---
a/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java
+++
b/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java
@@ -31,10 +31,7 @@ import
net.schmizz.sshj.userauth.method.ChallengeResponseProvider;
import net.schmizz.sshj.userauth.password.PasswordFinder;
import net.schmizz.sshj.userauth.password.PasswordUtils;
import net.schmizz.sshj.userauth.password.Resource;
-import org.apache.airavata.agents.api.AgentAdaptor;
-import org.apache.airavata.agents.api.AgentException;
-import org.apache.airavata.agents.api.AgentUtils;
-import org.apache.airavata.agents.api.CommandOutput;
+import org.apache.airavata.agents.api.*;
import org.apache.airavata.helix.adaptor.wrapper.SCPFileTransferWrapper;
import org.apache.airavata.helix.agent.ssh.StandardOutReader;
import
org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
@@ -208,6 +205,28 @@ public class SSHJAgentAdaptor implements AgentAdaptor {
}
@Override
+ public List<FileInfo> listDirectoryWithInfo(String path) throws
AgentException {
+ try (SFTPClient sftpClient = sshjClient.newSFTPClientWrapper()) {
+ List<RemoteResourceInfo> ls = sftpClient.ls(path);
+
+ return ls.stream().map(rInfo -> {
+ FileInfo fileInfo = new FileInfo();
+ fileInfo.setName(rInfo.getName());
+ fileInfo.setPath(rInfo.getPath());
+ fileInfo.setExist(true);
+ fileInfo.setCreatedDate(rInfo.getAttributes().getAtime());
+ fileInfo.setModifiedDate(rInfo.getAttributes().getMtime());
+ fileInfo.setFile(rInfo.getAttributes().getType() !=
FileMode.Type.DIRECTORY);
+ fileInfo.setSize(rInfo.getAttributes().getSize());
+ return fileInfo;
+ }).collect(Collectors.toList());
+
+ } catch (Exception e) {
+ throw new AgentException(e);
+ }
+ }
+
+ @Override
public Boolean doesFileExist(String filePath) throws AgentException {
try (SFTPClient sftpClient = sshjClient.newSFTPClientWrapper()) {
return sftpClient.statExistence(filePath) != null;
diff --git
a/modules/registry/registry-server/registry-api-service/src/main/java/org/apache/airavata/registry/api/service/handler/RegistryServerHandler.java
b/modules/registry/registry-server/registry-api-service/src/main/java/org/apache/airavata/registry/api/service/handler/RegistryServerHandler.java
index 65ccb6f..359de71 100644
---
a/modules/registry/registry-server/registry-api-service/src/main/java/org/apache/airavata/registry/api/service/handler/RegistryServerHandler.java
+++
b/modules/registry/registry-server/registry-api-service/src/main/java/org/apache/airavata/registry/api/service/handler/RegistryServerHandler.java
@@ -5044,10 +5044,16 @@ public class RegistryServerHandler implements
RegistryService.Iface {
try {
List<FileStructure> fileStructures = new ArrayList<>();
- List<String> fileNames =
storageResourceAdaptor.listDirectory(dirPath);
- for (String fileName : fileNames) {
- String filePath = dirPath.endsWith(File.separator) ? dirPath +
fileName : dirPath + File.separator + fileName;
- fileStructures.add(getFileDetailsFromStorage(gatewayId,
storageResourceId, userId, filePath));
+ List<FileInfo> fileInfos =
storageResourceAdaptor.listDirectoryWithInfo(dirPath);
+ for (FileInfo fileInfo : fileInfos) {
+
+ FileStructure fileStructure = new FileStructure();
+ fileStructure.setName(fileInfo.getName());
+ fileStructure.setPath(fileInfo.getPath());
+ fileStructure.setCreatedDate(fileInfo.getCreatedDate());
+ fileStructure.setModifiedDate(fileInfo.getModifiedDate());
+ fileStructure.setIsFile(fileInfo.isFile());
+ fileStructures.add(fileStructure);
}
return fileStructures;
} catch (AgentException e) {
