This is an automated email from the ASF dual-hosted git repository.
lahirujayathilake pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/develop by this push:
new fa2a324538 Add getResourceStorageInfo API to retrieve disk usage from
compute/storage resources
fa2a324538 is described below
commit fa2a3245382da39e9bdc3ba93cbdb5159e38a7f5
Author: lahiruj <[email protected]>
AuthorDate: Fri Nov 7 20:17:57 2025 -0500
Add getResourceStorageInfo API to retrieve disk usage from compute/storage
resources
---
.../apache/airavata/agents/api/AgentAdaptor.java | 31 +--
.../api/server/handler/AiravataServerHandler.java | 214 +++++++++++++++++++++
.../airavata/helix/adaptor/SSHJAgentAdaptor.java | 112 +++++++++++
.../airavata/helix/adaptor/SSHJStorageAdaptor.java | 5 +
.../airavata/helix/agent/ssh/SshAgentAdaptor.java | 6 +
.../core/support/adaptor/AdaptorSupportImpl.java | 66 ++++++-
.../helix/core/support/adaptor/AgentStore.java | 33 ++++
.../helix/task/api/support/AdaptorSupport.java | 14 +-
.../airavata_sdk/clients/api_server_client.py | 1 +
.../airavata-apis/airavata_api.thrift | 23 ++-
.../data-models/storage_resource_model.thrift | 27 +++
11 files changed, 509 insertions(+), 23 deletions(-)
diff --git
a/airavata-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java
b/airavata-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java
index 38375c714f..4a9b338991 100644
---
a/airavata-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java
+++
b/airavata-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java
@@ -19,6 +19,8 @@
*/
package org.apache.airavata.agents.api;
+import org.apache.airavata.model.appcatalog.storageresource.StorageVolumeInfo;
+
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
@@ -31,30 +33,31 @@ import java.util.List;
*/
public interface AgentAdaptor {
- public void init(String computeResource, String gatewayId, String userId,
String token) throws AgentException;
+ void init(String computeResource, String gatewayId, String userId, String
token) throws AgentException;
+
+ void destroy();
- public void destroy();
+ CommandOutput executeCommand(String command, String workingDirectory)
throws AgentException;
- public CommandOutput executeCommand(String command, String
workingDirectory) throws AgentException;
+ void createDirectory(String path) throws AgentException;
- public void createDirectory(String path) throws AgentException;
+ void createDirectory(String path, boolean recursive) throws AgentException;
- public void createDirectory(String path, boolean recursive) throws
AgentException;
+ void uploadFile(String localFile, String remoteFile) throws AgentException;
- public void uploadFile(String localFile, String remoteFile) throws
AgentException;
+ void uploadFile(InputStream localInStream, FileMetadata metadata, String
remoteFile) throws AgentException;
- public void uploadFile(InputStream localInStream, FileMetadata metadata,
String remoteFile) throws AgentException;
+ void downloadFile(String remoteFile, String localFile) throws
AgentException;
- public void downloadFile(String remoteFile, String localFile) throws
AgentException;
+ void downloadFile(String remoteFile, OutputStream localOutStream,
FileMetadata metadata) throws AgentException;
- public void downloadFile(String remoteFile, OutputStream localOutStream,
FileMetadata metadata)
- throws AgentException;
+ List<String> listDirectory(String path) throws AgentException;
- public List<String> listDirectory(String path) throws AgentException;
+ Boolean doesFileExist(String filePath) throws AgentException;
- public Boolean doesFileExist(String filePath) throws AgentException;
+ List<String> getFileNameFromExtension(String fileName, String parentPath)
throws AgentException;
- public List<String> getFileNameFromExtension(String fileName, String
parentPath) throws AgentException;
+ FileMetadata getFileMetadata(String remoteFile) throws AgentException;
- public FileMetadata getFileMetadata(String remoteFile) throws
AgentException;
+ StorageVolumeInfo getStorageVolumeInfo(String location) throws
AgentException;
}
diff --git
a/airavata-api/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
b/airavata-api/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
index 4be740eebd..e7a29ab87a 100644
---
a/airavata-api/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
+++
b/airavata-api/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
@@ -57,6 +57,7 @@ import
org.apache.airavata.model.appcatalog.groupresourceprofile.GroupResourcePr
import org.apache.airavata.model.appcatalog.parser.Parser;
import org.apache.airavata.model.appcatalog.parser.ParsingTemplate;
import
org.apache.airavata.model.appcatalog.storageresource.StorageResourceDescription;
+import org.apache.airavata.model.appcatalog.storageresource.StorageVolumeInfo;
import
org.apache.airavata.model.appcatalog.userresourceprofile.UserComputeResourcePreference;
import
org.apache.airavata.model.appcatalog.userresourceprofile.UserResourceProfile;
import
org.apache.airavata.model.appcatalog.userresourceprofile.UserStoragePreference;
@@ -93,6 +94,9 @@ import org.apache.airavata.model.workspace.Notification;
import org.apache.airavata.model.workspace.Project;
import org.apache.airavata.registry.api.RegistryService;
import org.apache.airavata.registry.api.exception.RegistryServiceException;
+import org.apache.airavata.agents.api.AgentAdaptor;
+import org.apache.airavata.agents.api.AgentException;
+import org.apache.airavata.helix.core.support.adaptor.AdaptorSupportImpl;
import org.apache.airavata.service.security.GatewayGroupsInitializer;
import org.apache.airavata.service.security.interceptor.SecurityCheck;
import org.apache.airavata.sharing.registry.models.*;
@@ -3839,6 +3843,58 @@ public class AiravataServerHandler implements
Airavata.Iface {
}
}
+ @Override
+ @SecurityCheck
+ public StorageVolumeInfo getResourceStorageInfo(AuthzToken authzToken,
String resourceId, String location) throws TException {
+ String gatewayId = authzToken.getClaimsMap().get(Constants.GATEWAY_ID);
+ String userId = authzToken.getClaimsMap().get(Constants.USER_NAME);
+ RegistryService.Client regClient = registryClientPool.getResource();
+ StorageInfoContext context;
+
+ try {
+ Optional<ComputeResourceDescription> computeResourceOp =
Optional.ofNullable(regClient.getComputeResource(resourceId));
+ Optional<StorageResourceDescription> storageResourceOp =
Optional.empty();
+
+ if (computeResourceOp.isEmpty()) {
+ storageResourceOp =
Optional.ofNullable(regClient.getStorageResource(resourceId));
+ }
+
+ if (computeResourceOp.isEmpty() && storageResourceOp.isEmpty()) {
+ logger.error("Resource with ID {} not found as either compute
resource or storage resource", resourceId);
+ throw new InvalidRequestException("Resource with ID '" +
resourceId + "' not found as either compute resource or storage resource");
+ }
+
+ if (computeResourceOp.isPresent()) {
+ logger.debug("Found compute resource with ID {}. Resolving
login username and credentials", resourceId);
+ context = resolveComputeStorageInfoContext(authzToken,
gatewayId, userId, resourceId);
+ } else {
+ logger.debug("Found storage resource with ID {}. Resolving
login username and credentials", resourceId);
+ context = resolveStorageStorageInfoContext(authzToken,
gatewayId, userId, resourceId);
+ }
+
+ registryClientPool.returnResource(regClient);
+ regClient = null;
+
+ return context.adaptor.getStorageVolumeInfo(location);
+
+ } catch (InvalidRequestException | AiravataClientException e) {
+ if (regClient != null) {
+ registryClientPool.returnResource(regClient);
+ }
+ logger.error("Error while retrieving storage resource.", e);
+ throw e;
+
+ } catch (Exception e) {
+ logger.error("Error while retrieving storage volume info for
resource {}", resourceId, e);
+ registryClientPool.returnBrokenResource(regClient);
+
+ AiravataSystemException exception = new AiravataSystemException();
+ exception.setAiravataErrorType(AiravataErrorType.INTERNAL_ERROR);
+ exception.setMessage("Error while retrieving storage volume info.
More info: " + e.getMessage());
+ throw exception;
+ }
+ }
+
/**
* Add a Local Job Submission details to a compute resource
* App catalog will return a jobSubmissionInterfaceId which will be added
to the jobSubmissionInterfaces.
@@ -7232,4 +7288,162 @@ public class AiravataServerHandler implements
Airavata.Iface {
return GatewayGroupsInitializer.initializeGatewayGroups(gatewayId);
}
}
+
+ /**
+ * To hold storage info context (login username, credential token, and
adaptor)
+ */
+ private record StorageInfoContext(String loginUserName, String
credentialToken, AgentAdaptor adaptor) {
+ }
+
+ private AiravataClientException clientException(AiravataErrorType
errorType, String parameter) {
+ AiravataClientException exception = new AiravataClientException();
+ exception.setAiravataErrorType(errorType);
+ exception.setParameter(parameter);
+ return exception;
+ }
+
+ /**
+ * Resolves compute resource storage info context (login username,
credential token, and adaptor).
+ * Handles user preference → group preference fallback for both login and
credentials.
+ */
+ private StorageInfoContext resolveComputeStorageInfoContext(AuthzToken
authzToken, String gatewayId, String userId, String resourceId) throws
AgentException, TException {
+ String loginUserName = null;
+ boolean loginFromUserPref = false;
+ GroupComputeResourcePreference groupComputePref = null;
+ GroupResourceProfile groupResourceProfile = null;
+
+ UserComputeResourcePreference userComputePref =
getUserComputeResourcePreference(authzToken, userId, gatewayId, resourceId);
+
+ if (userComputePref != null && userComputePref.getLoginUserName() !=
null && !userComputePref.getLoginUserName().trim().isEmpty()) {
+ loginUserName = userComputePref.getLoginUserName();
+ loginFromUserPref = true;
+ logger.debug("Using user preference login username: {}",
loginUserName);
+
+ } else {
+ // Fallback to GroupComputeResourcePreference
+ List<GroupResourceProfile> groupResourceProfiles =
getGroupResourceList(authzToken, gatewayId);
+ for (GroupResourceProfile groupProfile : groupResourceProfiles) {
+ List<GroupComputeResourcePreference> groupComputePrefs =
groupProfile.getComputePreferences();
+
+ if (groupComputePrefs != null && !groupComputePrefs.isEmpty())
{
+ for (GroupComputeResourcePreference groupPref :
groupComputePrefs) {
+ if
(resourceId.equals(groupPref.getComputeResourceId()) &&
groupPref.getLoginUserName() != null &&
!groupPref.getLoginUserName().trim().isEmpty()) {
+ loginUserName = groupPref.getLoginUserName();
+ groupComputePref = groupPref;
+ groupResourceProfile = groupProfile;
+ logger.debug("Using login username from group
compute resource preference for resource {}", resourceId);
+ break;
+ }
+ }
+ }
+ if (loginUserName != null) {
+ break;
+ }
+ }
+ if (loginUserName == null) {
+ logger.debug("No login username found for compute resource
{}", resourceId);
+ throw new InvalidRequestException("No login username found for
compute resource " + resourceId);
+ }
+ }
+
+ // Resolve credential token based on where login came from
+ String credentialToken;
+ if (loginFromUserPref) {
+ // Login username came from user preference. Use user preference
token → user profile token
+ if (userComputePref != null &&
userComputePref.getResourceSpecificCredentialStoreToken() != null &&
!userComputePref.getResourceSpecificCredentialStoreToken().trim().isEmpty()) {
+ credentialToken =
userComputePref.getResourceSpecificCredentialStoreToken();
+ } else {
+ UserResourceProfile userResourceProfile =
getUserResourceProfile(authzToken, userId, gatewayId);
+ if (userResourceProfile == null ||
userResourceProfile.getCredentialStoreToken() == null ||
userResourceProfile.getCredentialStoreToken().trim().isEmpty()) {
+ logger.error("No credential store token found for user {}
in gateway {}", userId, gatewayId);
+ throw
clientException(AiravataErrorType.AUTHENTICATION_FAILURE, "No credential store
token found for user " + userId + " in gateway " + gatewayId);
+ }
+ credentialToken =
userResourceProfile.getCredentialStoreToken();
+ }
+ } else {
+ // Login username came from group preference. Use group preference
token → group profile default token → user profile token (fallback)
+ if (groupComputePref != null &&
groupComputePref.getResourceSpecificCredentialStoreToken() != null &&
!groupComputePref.getResourceSpecificCredentialStoreToken().trim().isEmpty()) {
+ credentialToken =
groupComputePref.getResourceSpecificCredentialStoreToken();
+
+ } else if (groupResourceProfile != null &&
groupResourceProfile.getDefaultCredentialStoreToken() != null &&
!groupResourceProfile.getDefaultCredentialStoreToken().trim().isEmpty()) {
+ credentialToken =
groupResourceProfile.getDefaultCredentialStoreToken();
+
+ } else {
+ UserResourceProfile userResourceProfile =
getUserResourceProfile(authzToken, userId, gatewayId);
+ if (userResourceProfile == null ||
userResourceProfile.getCredentialStoreToken() == null ||
userResourceProfile.getCredentialStoreToken().trim().isEmpty()) {
+ logger.error("No credential store token found for user {}
in gateway {}", userId, gatewayId);
+ throw
clientException(AiravataErrorType.AUTHENTICATION_FAILURE, "No credential store
token found for compute resource " + resourceId);
+ }
+ credentialToken =
userResourceProfile.getCredentialStoreToken();
+ }
+ }
+
+ AgentAdaptor adaptor =
AdaptorSupportImpl.getInstance().fetchComputeSSHAdaptor(gatewayId, resourceId,
credentialToken, userId, loginUserName);
+ logger.info("Resolved resource {} as compute resource to fetch storage
details", resourceId);
+
+ return new StorageInfoContext(loginUserName, credentialToken, adaptor);
+ }
+
+ /**
+ * Resolves storage resource storage info context (login username,
credential token, and adaptor).
+ * Handles user preference → gateway preference fallback for both login
and credentials.
+ */
+ private StorageInfoContext resolveStorageStorageInfoContext(AuthzToken
authzToken, String gatewayId, String userId, String resourceId) throws
AgentException, TException {
+ UserStoragePreference userStoragePref =
getUserStoragePreference(authzToken, userId, gatewayId, resourceId);
+ StoragePreference storagePref =
getGatewayStoragePreference(authzToken, gatewayId, resourceId);
+
+ String loginUserName;
+ boolean loginFromUserPref;
+
+ if (userStoragePref != null && userStoragePref.getLoginUserName() !=
null && !userStoragePref.getLoginUserName().trim().isEmpty()) {
+ loginUserName = userStoragePref.getLoginUserName();
+ loginFromUserPref = true;
+ logger.debug("Using login username from user storage preference
for resource {}", resourceId);
+
+ } else if (storagePref != null && storagePref.getLoginUserName() !=
null && !storagePref.getLoginUserName().trim().isEmpty()) {
+ loginUserName = storagePref.getLoginUserName();
+ loginFromUserPref = false;
+ logger.debug("Using login username from gateway storage preference
for resource {}", resourceId);
+
+ } else {
+ logger.error("No login username found for storage resource {}",
resourceId);
+ throw new InvalidRequestException("No login username found for
storage resource " + resourceId);
+ }
+
+ // Resolve credential token based on where login came from
+ String credentialToken;
+ if (loginFromUserPref) {
+ // Login came from user preference. Use user preference token or
user profile token
+ if (userStoragePref != null &&
userStoragePref.getResourceSpecificCredentialStoreToken() != null &&
!userStoragePref.getResourceSpecificCredentialStoreToken().trim().isEmpty()) {
+ credentialToken =
userStoragePref.getResourceSpecificCredentialStoreToken();
+ logger.debug("Using login username from user preference for
resource {}", resourceId);
+
+ } else {
+ UserResourceProfile userResourceProfile =
getUserResourceProfile(authzToken, userId, gatewayId);
+ if (userResourceProfile == null ||
userResourceProfile.getCredentialStoreToken() == null ||
userResourceProfile.getCredentialStoreToken().trim().isEmpty()) {
+ logger.error("No credential store token found for user {}
in gateway {}", userId, gatewayId);
+ throw
clientException(AiravataErrorType.AUTHENTICATION_FAILURE, "No credential store
token found for user " + userId + " in gateway " + gatewayId);
+ }
+ credentialToken =
userResourceProfile.getCredentialStoreToken();
+ }
+ } else {
+ // Login came from gateway preference. Use gateway preference
token or gateway profile token
+ if (storagePref != null &&
storagePref.getResourceSpecificCredentialStoreToken() != null &&
!storagePref.getResourceSpecificCredentialStoreToken().trim().isEmpty()) {
+ credentialToken =
storagePref.getResourceSpecificCredentialStoreToken();
+
+ } else {
+ GatewayResourceProfile gatewayResourceProfile =
getGatewayResourceProfile(authzToken, gatewayId);
+ if (gatewayResourceProfile == null ||
gatewayResourceProfile.getCredentialStoreToken() == null ||
gatewayResourceProfile.getCredentialStoreToken().trim().isEmpty()) {
+ logger.error("No credential store token found for gateway
{}", gatewayId);
+ throw
clientException(AiravataErrorType.AUTHENTICATION_FAILURE, "No credential store
token found for gateway " + gatewayId);
+ }
+ credentialToken =
gatewayResourceProfile.getCredentialStoreToken();
+ }
+ }
+
+ AgentAdaptor adaptor =
AdaptorSupportImpl.getInstance().fetchStorageSSHAdaptor(gatewayId, resourceId,
credentialToken, userId, loginUserName);
+ logger.info("Resolved resource {} as storage resource to fetch storage
details", resourceId);
+
+ return new StorageInfoContext(loginUserName, credentialToken, adaptor);
+ }
}
diff --git
a/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java
b/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java
index 7b715f4cd3..410b57dc60 100644
---
a/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java
+++
b/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java
@@ -51,6 +51,7 @@ import
org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescr
import
org.apache.airavata.model.appcatalog.computeresource.JobSubmissionInterface;
import
org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
import org.apache.airavata.model.appcatalog.computeresource.SSHJobSubmission;
+import org.apache.airavata.model.appcatalog.storageresource.StorageVolumeInfo;
import org.apache.airavata.model.credential.store.SSHCredential;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -593,4 +594,115 @@ public class SSHJAgentAdaptor implements AgentAdaptor {
}
return j == p.length();
}
+
+ @Override
+ public StorageVolumeInfo getStorageVolumeInfo(String location) throws
AgentException {
+ try {
+ String targetLocation = location;
+ if (targetLocation == null || targetLocation.trim().isEmpty()) {
+ CommandOutput homeOutput = executeCommand("echo $HOME", null);
+
+ if (homeOutput.getExitCode() != 0 || homeOutput.getStdOut() ==
null || homeOutput.getStdOut().trim().isEmpty()) {
+ logger.error("Failed to determine user's home directory:
{}", homeOutput.getStdError());
+ throw new AgentException("Failed to determine user's home
directory: " + homeOutput.getStdError());
+ }
+ targetLocation = homeOutput.getStdOut().trim();
+ }
+
+ // Escape location to prevent command injection and handle spaces
+ String escapedLocation = targetLocation.replace("'", "'\"'\"'");
+ String dfCommand = "df -P -T -h '" + escapedLocation + "'";
+ String dfBytesCommand = "df -P -T '" + escapedLocation + "'";
+
+ CommandOutput dfHumanOutput = executeCommand(dfCommand, null);
+ CommandOutput dfBytesOutput = executeCommand(dfBytesCommand, null);
+
+ if (dfHumanOutput.getExitCode() != 0) {
+ logger.error("Failed to execute df command for location {}:
{}", targetLocation, dfHumanOutput.getStdError());
+ throw new AgentException("Failed to execute df command for
location " + targetLocation + ": " + dfHumanOutput.getStdError());
+ }
+
+ if (dfBytesOutput.getExitCode() != 0) {
+ logger.error("Failed to execute df command for location {}:
{}", targetLocation, dfBytesOutput.getStdError());
+ throw new AgentException("Failed to execute df command for
location " + targetLocation + ": " + dfBytesOutput.getStdError());
+ }
+
+ return parseDfOutput(dfHumanOutput.getStdOut(),
dfBytesOutput.getStdOut(), targetLocation);
+
+ } catch (Exception e) {
+ logger.error("Error while retrieving storage volume info for
location " + location, e);
+ throw new AgentException("Error while retrieving storage volume
info for location " + location, e);
+ }
+ }
+
+ private StorageVolumeInfo parseDfOutput(String dfHumanOutput, String
dfBytesOutput, String targetLocation) throws AgentException {
+ try {
+ // Parse df -P -T -h output (POSIX format with filesystem type)
+ String[] humanLines = dfHumanOutput.split("\n");
+ String[] bytesLines = dfBytesOutput.split("\n");
+
+ if (humanLines.length < 2 || bytesLines.length < 2) {
+ logger.error("Unexpected df output format while parsing
storage volume info for location {}", targetLocation);
+ throw new AgentException("Unexpected df output format while
parsing storage volume info for location " + targetLocation);
+ }
+
+ // Skip the header line and get the data line
+ String humanDataLine = humanLines[1].trim();
+ String bytesDataLine = bytesLines[1].trim();
+
+ // Split by whitespace. POSIX format uses fixed width columns
separated by spaces
+ String[] humanFields = humanDataLine.split("\\s+");
+ String[] bytesFields = bytesDataLine.split("\\s+");
+
+ if (humanFields.length < 7 || bytesFields.length < 7) {
+ logger.error("Unexpected df output format - insufficient
fields while parsing storage volume info for location {}", targetLocation);
+ throw new AgentException("Unexpected df output format -
insufficient fields while parsing storage volume info for location " +
targetLocation);
+ }
+
+ String filesystemType = humanFields[1]; // ext4, xfs, etc.
+ String totalSizeHuman = humanFields[2];
+ String usedSizeHuman = humanFields[3];
+ String availableSizeHuman = humanFields[4];
+ String capacityStr = humanFields[5].replace("%", "");
+
+ // If Mount point contains spaces
+ StringBuilder mountPointBuilder = new StringBuilder();
+ for (int i = 6; i < humanFields.length; i++) {
+ if (i > 6) {
+ mountPointBuilder.append(" ");
+ }
+ mountPointBuilder.append(humanFields[i]);
+ }
+ String mountPoint = mountPointBuilder.toString();
+
+ // Parse bytes output. Same format but in 1024-byte blocks
+ long totalSizeBlocks = Long.parseLong(bytesFields[2]);
+ long usedSizeBlocks = Long.parseLong(bytesFields[3]);
+ long availableSizeBlocks = Long.parseLong(bytesFields[4]);
+
+ // Convert 1024-byte blocks to bytes
+ long totalSizeBytes = totalSizeBlocks * 1024L;
+ long usedSizeBytes = usedSizeBlocks * 1024L;
+ long availableSizeBytes = availableSizeBlocks * 1024L;
+
+ double percentageUsed = Double.parseDouble(capacityStr);
+
+ StorageVolumeInfo volumeInfo = new StorageVolumeInfo();
+ volumeInfo.setTotalSize(totalSizeHuman);
+ volumeInfo.setUsedSize(usedSizeHuman);
+ volumeInfo.setAvailableSize(availableSizeHuman);
+ volumeInfo.setTotalSizeBytes(totalSizeBytes);
+ volumeInfo.setUsedSizeBytes(usedSizeBytes);
+ volumeInfo.setAvailableSizeBytes(availableSizeBytes);
+ volumeInfo.setPercentageUsed(percentageUsed);
+ volumeInfo.setMountPoint(mountPoint);
+ volumeInfo.setFilesystemType(filesystemType);
+
+ return volumeInfo;
+
+ } catch (Exception e) {
+ logger.error("Error parsing df output: {} for location {}",
e.getMessage(), targetLocation, e);
+ throw new AgentException("Error parsing df output: " +
e.getMessage() + " for location " + targetLocation, e);
+ }
+ }
}
diff --git
a/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJStorageAdaptor.java
b/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJStorageAdaptor.java
index f9a0751b79..d309cdb632 100644
---
a/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJStorageAdaptor.java
+++
b/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJStorageAdaptor.java
@@ -108,4 +108,9 @@ public class SSHJStorageAdaptor extends SSHJAgentAdaptor
implements StorageResou
public CommandOutput executeCommand(String command, String
workingDirectory) throws AgentException {
return super.executeCommand(command, workingDirectory);
}
+
+ @Override
+ public
org.apache.airavata.model.appcatalog.storageresource.StorageVolumeInfo
getStorageVolumeInfo(String location) throws AgentException {
+ return super.getStorageVolumeInfo(location);
+ }
}
diff --git
a/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java
b/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java
index a59be3c5fe..96a0eeed74 100644
---
a/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java
+++
b/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.UUID;
import org.apache.airavata.agents.api.*;
import org.apache.airavata.model.appcatalog.computeresource.*;
+import org.apache.airavata.model.appcatalog.storageresource.StorageVolumeInfo;
import org.apache.airavata.model.credential.store.SSHCredential;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -540,6 +541,11 @@ public class SshAgentAdaptor implements AgentAdaptor {
throw new AgentException("Operation not implemented");
}
+ @Override
+ public StorageVolumeInfo getStorageVolumeInfo(String location) {
+ throw new UnsupportedOperationException("Operation not supported by
SshAgentAdaptor. Use SSHJAgentAdaptor instead.");
+ }
+
private static class DefaultUserInfo implements UserInfo,
UIKeyboardInteractive {
private String userName;
diff --git
a/airavata-api/src/main/java/org/apache/airavata/helix/core/support/adaptor/AdaptorSupportImpl.java
b/airavata-api/src/main/java/org/apache/airavata/helix/core/support/adaptor/AdaptorSupportImpl.java
index b5c47b1b05..4fbbcc58a2 100644
---
a/airavata-api/src/main/java/org/apache/airavata/helix/core/support/adaptor/AdaptorSupportImpl.java
+++
b/airavata-api/src/main/java/org/apache/airavata/helix/core/support/adaptor/AdaptorSupportImpl.java
@@ -19,8 +19,9 @@
*/
package org.apache.airavata.helix.core.support.adaptor;
-import java.util.Optional;
-import org.apache.airavata.agents.api.*;
+import org.apache.airavata.agents.api.AgentAdaptor;
+import org.apache.airavata.agents.api.AgentException;
+import org.apache.airavata.agents.api.StorageResourceAdaptor;
import org.apache.airavata.helix.adaptor.SSHJAgentAdaptor;
import org.apache.airavata.helix.adaptor.SSHJStorageAdaptor;
import org.apache.airavata.helix.task.api.support.AdaptorSupport;
@@ -29,6 +30,8 @@ import
org.apache.airavata.model.data.movement.DataMovementProtocol;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Optional;
+
/**
* TODO: Class level comments please
*
@@ -131,4 +134,63 @@ public class AdaptorSupportImpl implements AdaptorSupport {
}
}
}
+
+ @Override
+ public AgentAdaptor fetchComputeSSHAdaptor(String gatewayId, String
resourceId, String authToken, String gatewayUserId, String loginUserName)
throws AgentException {
+ String cacheKey = "compute-" + resourceId;
+
+ logger.debug("Fetching SSH adaptor for compute resource {} with token
{} for gateway user {} with login username {}", resourceId, authToken,
gatewayUserId, loginUserName);
+
+ Optional<AgentAdaptor> adaptorOp = agentStore.getSSHAdaptor(cacheKey,
authToken, gatewayUserId, loginUserName);
+ if (adaptorOp.isPresent()) {
+ logger.debug("Reusing SSH adaptor for gateway {}, compute resource
{}, gateway user {}, login username {}", gatewayId, resourceId, gatewayUserId,
loginUserName);
+ return adaptorOp.get();
+
+ } else {
+ synchronized (this) {
+ adaptorOp = agentStore.getSSHAdaptor(cacheKey, authToken,
gatewayUserId, loginUserName);
+ if (adaptorOp.isPresent()) {
+ return adaptorOp.get();
+
+ } else {
+ logger.debug("Could not find SSH adaptor for gateway {},
compute resource {}, gateway user {}, login username {}. Creating new one",
gatewayId, resourceId, gatewayUserId, loginUserName);
+
+ SSHJAgentAdaptor agentAdaptor = new SSHJAgentAdaptor();
+ agentAdaptor.init(resourceId, gatewayId, loginUserName,
authToken);
+
+ agentStore.putSSHAdaptor(cacheKey, authToken,
gatewayUserId, loginUserName, agentAdaptor);
+ return agentAdaptor;
+ }
+ }
+ }
+ }
+
+ @Override
+ public StorageResourceAdaptor fetchStorageSSHAdaptor(String gatewayId,
String resourceId, String authToken, String gatewayUserId, String
loginUserName) throws AgentException {
+ String cacheKey = "storage-" + resourceId;
+
+ logger.debug("Fetching SSH adaptor for storage resource {} with token
{} for gateway user {} with login username {}", resourceId, authToken,
gatewayUserId, loginUserName);
+
+ Optional<AgentAdaptor> adaptorOp = agentStore.getSSHAdaptor(cacheKey,
authToken, gatewayUserId, loginUserName);
+ if (adaptorOp.isPresent()) {
+ logger.debug("Reusing SSH adaptor for gateway {}, storage resource
{}, gateway user {}, login username {}", gatewayId, resourceId, gatewayUserId,
loginUserName);
+ return (StorageResourceAdaptor) adaptorOp.get();
+
+ } else {
+ synchronized (this) {
+ adaptorOp = agentStore.getSSHAdaptor(cacheKey, authToken,
gatewayUserId, loginUserName);
+ if (adaptorOp.isPresent()) {
+ return (StorageResourceAdaptor) adaptorOp.get();
+ } else {
+ logger.debug("Could not find SSH adaptor for gateway {},
storage resource {}, gateway user {}, login username {}. Creating new one",
gatewayId, resourceId, gatewayUserId, loginUserName);
+
+ SSHJStorageAdaptor storageAdaptor = new
SSHJStorageAdaptor();
+ storageAdaptor.init(resourceId, gatewayId, loginUserName,
authToken);
+
+ agentStore.putSSHAdaptor(cacheKey, authToken,
gatewayUserId, loginUserName, storageAdaptor);
+ return storageAdaptor;
+ }
+ }
+ }
+ }
}
diff --git
a/airavata-api/src/main/java/org/apache/airavata/helix/core/support/adaptor/AgentStore.java
b/airavata-api/src/main/java/org/apache/airavata/helix/core/support/adaptor/AgentStore.java
index 48d1c04eee..c9a0943686 100644
---
a/airavata-api/src/main/java/org/apache/airavata/helix/core/support/adaptor/AgentStore.java
+++
b/airavata-api/src/main/java/org/apache/airavata/helix/core/support/adaptor/AgentStore.java
@@ -40,6 +40,8 @@ public class AgentStore {
new HashMap<>();
private final Map<String, Map<DataMovementProtocol, Map<String,
Map<String, StorageResourceAdaptor>>>>
storageAdaptorCache = new HashMap<>();
+ // SSH adaptor cache: resourceId (with compute/storage prefix): auth
token: gatewayUserId: loginUserName: adaptor
+ private final Map<String, Map<String, Map<String, Map<String,
AgentAdaptor>>>> sshAdaptorCache = new HashMap<>();
public Optional<AgentAdaptor> getAgentAdaptor(
String computeResource, JobSubmissionProtocol submissionProtocol,
String authToken, String userId) {
@@ -111,4 +113,35 @@ public class AgentStore {
tokenToUserMap.computeIfAbsent(authToken, k -> new
HashMap<>());
userToAdaptorMap.put(userId, storageResourceAdaptor);
}
+
+ public Optional<AgentAdaptor> getSSHAdaptor(String resourceId, String
authToken, String gatewayUserId, String loginUserName) {
+ Map<String, Map<String, Map<String, AgentAdaptor>>>
tokenToGatewayUserMap = sshAdaptorCache.get(resourceId);
+
+ if (tokenToGatewayUserMap != null) {
+ Map<String, Map<String, AgentAdaptor>> gatewayUserToLoginUserMap =
tokenToGatewayUserMap.get(authToken);
+
+ if (gatewayUserToLoginUserMap != null) {
+ Map<String, AgentAdaptor> loginUserToAdaptorMap =
gatewayUserToLoginUserMap.get(gatewayUserId);
+
+ if (loginUserToAdaptorMap != null) {
+ return
Optional.ofNullable(loginUserToAdaptorMap.get(loginUserName));
+ } else {
+ return Optional.empty();
+ }
+ } else {
+ return Optional.empty();
+ }
+ } else {
+ return Optional.empty();
+ }
+ }
+
+ public void putSSHAdaptor(String resourceId, String authToken, String
gatewayUserId, String loginUserName, AgentAdaptor adaptor) {
+
+ Map<String, Map<String, Map<String, AgentAdaptor>>>
tokenToGatewayUserMap = sshAdaptorCache.computeIfAbsent(resourceId, k -> new
HashMap<>());
+ Map<String, Map<String, AgentAdaptor>> gatewayUserToLoginUserMap =
tokenToGatewayUserMap.computeIfAbsent(authToken, k -> new HashMap<>());
+ Map<String, AgentAdaptor> loginUserToAdaptorMap =
gatewayUserToLoginUserMap.computeIfAbsent(gatewayUserId, k -> new HashMap<>());
+
+ loginUserToAdaptorMap.put(loginUserName, adaptor);
+ }
}
diff --git
a/airavata-api/src/main/java/org/apache/airavata/helix/task/api/support/AdaptorSupport.java
b/airavata-api/src/main/java/org/apache/airavata/helix/task/api/support/AdaptorSupport.java
index d87e9dec9f..7e27b2a62b 100644
---
a/airavata-api/src/main/java/org/apache/airavata/helix/task/api/support/AdaptorSupport.java
+++
b/airavata-api/src/main/java/org/apache/airavata/helix/task/api/support/AdaptorSupport.java
@@ -19,7 +19,9 @@
*/
package org.apache.airavata.helix.task.api.support;
-import org.apache.airavata.agents.api.*;
+import org.apache.airavata.agents.api.AgentAdaptor;
+import org.apache.airavata.agents.api.AgentException;
+import org.apache.airavata.agents.api.StorageResourceAdaptor;
import
org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
import org.apache.airavata.model.data.movement.DataMovementProtocol;
@@ -30,13 +32,17 @@ import
org.apache.airavata.model.data.movement.DataMovementProtocol;
* @since 1.0.0-SNAPSHOT
*/
public interface AdaptorSupport {
- public void initializeAdaptor();
+ void initializeAdaptor();
- public AgentAdaptor fetchAdaptor(
+ AgentAdaptor fetchAdaptor(
String gatewayId, String computeResource, JobSubmissionProtocol
protocol, String authToken, String userId)
throws Exception;
- public StorageResourceAdaptor fetchStorageAdaptor(
+ StorageResourceAdaptor fetchStorageAdaptor(
String gatewayId, String storageResourceId, DataMovementProtocol
protocol, String authToken, String userId)
throws AgentException;
+
+ AgentAdaptor fetchComputeSSHAdaptor(String gatewayId, String resourceId,
String authToken, String gatewayUserId, String loginUserName) throws
AgentException;
+
+ StorageResourceAdaptor fetchStorageSSHAdaptor(String gatewayId, String
resourceId, String authToken, String gatewayUserId, String loginUserName)
throws AgentException;
}
diff --git
a/dev-tools/airavata-python-sdk/airavata_sdk/clients/api_server_client.py
b/dev-tools/airavata-python-sdk/airavata_sdk/clients/api_server_client.py
index cf6fd2edaa..08c9e15ee0 100644
--- a/dev-tools/airavata-python-sdk/airavata_sdk/clients/api_server_client.py
+++ b/dev-tools/airavata-python-sdk/airavata_sdk/clients/api_server_client.py
@@ -113,6 +113,7 @@ class APIServerClient:
self.get_all_storage_resource_names =
self.client.getAllStorageResourceNames
self.update_storage_resource = self.client.updateStorageResource
self.delete_storage_resource = self.client.deleteStorageResource
+ self.get_resource_storage_info = self.client.getResourceStorageInfo
self.add_local_submission_details =
self.client.addLocalSubmissionDetails
self.update_local_submission_details =
self.client.updateLocalSubmissionDetails
self.get_local_job_submission = self.client.getLocalJobSubmission
diff --git a/thrift-interface-descriptions/airavata-apis/airavata_api.thrift
b/thrift-interface-descriptions/airavata-apis/airavata_api.thrift
index 122a2e6935..c5a95d325d 100644
--- a/thrift-interface-descriptions/airavata-apis/airavata_api.thrift
+++ b/thrift-interface-descriptions/airavata-apis/airavata_api.thrift
@@ -3610,7 +3610,24 @@ service Airavata extends base_api.BaseAPI {
2: airavata_errors.AiravataClientException ace,
3: airavata_errors.AiravataSystemException ase,
4: airavata_errors.AuthorizationException ae);
- //
- //End of API
- }
+
+ /**
+ * Get storage volume information for a compute or storage resource.
+ *
+ * @param authzToken
+ * @param resourceId Can be either a compute resource ID or storage resource
ID
+ * @param location Optional path/mount point. If null/empty, defaults to
user's home directory ($HOME)
+ * @return StorageVolumeInfo containing disk usage information
+ */
+ storage_resource_model.StorageVolumeInfo getResourceStorageInfo(1: required
security_model.AuthzToken authzToken,
+ 2: required
string resourceId,
+ 3: optional
string location)
+ throws (1:
airavata_errors.InvalidRequestException ire,
+ 2:
airavata_errors.AiravataClientException ace,
+ 3:
airavata_errors.AiravataSystemException ase,
+ 4:
airavata_errors.AuthorizationException ae)
+
+//
+//End of API
+}
diff --git
a/thrift-interface-descriptions/data-models/storage_resource_model.thrift
b/thrift-interface-descriptions/data-models/storage_resource_model.thrift
index f6b65d80c4..8b41079379 100644
--- a/thrift-interface-descriptions/data-models/storage_resource_model.thrift
+++ b/thrift-interface-descriptions/data-models/storage_resource_model.thrift
@@ -52,3 +52,30 @@ struct StorageResourceDescription {
6: optional i64 creationTime,
7: optional i64 updateTime,
}
+
+/**
+ * Storage Volume Information
+ *
+ * Contains disk usage information for a filesystem/mount point.
+ *
+ * totalSize: Total size in human-readable format (e.g., "100G", "500M")
+ * usedSize: Used size in human-readable format
+ * availableSize: Available size in human-readable format
+ * totalSizeBytes: Total size in bytes
+ * usedSizeBytes: Used size in bytes
+ * availableSizeBytes: Available size in bytes
+ * percentageUsed: Percentage used
+ * mountPoint: Mount point/filesystem path
+ * filesystemType: Filesystem type if available
+ */
+struct StorageVolumeInfo {
+ 1: required string totalSize,
+ 2: required string usedSize,
+ 3: required string availableSize,
+ 4: required i64 totalSizeBytes,
+ 5: required i64 usedSizeBytes,
+ 6: required i64 availableSizeBytes,
+ 7: required double percentageUsed,
+ 8: required string mountPoint,
+ 9: optional string filesystemType,
+}
