This is an automated email from the ASF dual-hosted git repository. dimuthuupe pushed a commit to branch storage-directory-size-api in repository https://gitbox.apache.org/repos/asf/airavata.git
commit 8a0469eb961611482d61ee1831c201877a58ebbe Author: DImuthuUpe <[email protected]> AuthorDate: Tue Nov 11 16:16:21 2025 -0500 Enabling storage directory size listing api --- .../apache/airavata/agents/api/AgentAdaptor.java | 4 ++ .../api/server/handler/AiravataServerHandler.java | 76 ++++++++++++++++++++++ .../airavata/helix/adaptor/SSHJAgentAdaptor.java | 49 ++++++++++++++ .../airavata/helix/agent/ssh/SshAgentAdaptor.java | 7 ++ .../airavata-apis/airavata_api.thrift | 52 ++++++++++----- .../data-models/storage_resource_model.thrift | 11 ++++ 6 files changed, 181 insertions(+), 18 deletions(-) diff --git a/airavata-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java b/airavata-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java index 245cad8c69..fd2e5d79b4 100644 --- a/airavata-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java +++ b/airavata-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java @@ -22,6 +22,8 @@ package org.apache.airavata.agents.api; import java.io.InputStream; import java.io.OutputStream; import java.util.List; + +import org.apache.airavata.model.appcatalog.storageresource.StorageDirectoryInfo; import org.apache.airavata.model.appcatalog.storageresource.StorageVolumeInfo; /** @@ -59,4 +61,6 @@ public interface AgentAdaptor { FileMetadata getFileMetadata(String remoteFile) throws AgentException; StorageVolumeInfo getStorageVolumeInfo(String location) throws AgentException; + + StorageDirectoryInfo getStorageDirectoryInfo(String location) throws AgentException; } diff --git a/airavata-api/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java b/airavata-api/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java index c8a89566b3..4da1eb7494 100644 --- a/airavata-api/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java +++ b/airavata-api/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java @@ -59,6 +59,7 @@ import org.apache.airavata.model.appcatalog.groupresourceprofile.GroupComputeRes import org.apache.airavata.model.appcatalog.groupresourceprofile.GroupResourceProfile; import org.apache.airavata.model.appcatalog.parser.Parser; import org.apache.airavata.model.appcatalog.parser.ParsingTemplate; +import org.apache.airavata.model.appcatalog.storageresource.StorageDirectoryInfo; import org.apache.airavata.model.appcatalog.storageresource.StorageResourceDescription; import org.apache.airavata.model.appcatalog.storageresource.StorageVolumeInfo; import org.apache.airavata.model.appcatalog.userresourceprofile.UserComputeResourcePreference; @@ -3919,6 +3920,81 @@ public class AiravataServerHandler implements Airavata.Iface { } } + @Override + @SecurityCheck + public StorageDirectoryInfo getStorageDirectoryInfo(AuthzToken authzToken, String resourceId, String location) + throws TException { + String gatewayId = authzToken.getClaimsMap().get(Constants.GATEWAY_ID); + String userId = authzToken.getClaimsMap().get(Constants.USER_NAME); + RegistryService.Client regClient = registryClientPool.getResource(); + StorageInfoContext context; + + try { + Optional<ComputeResourceDescription> computeResourceOp = Optional.empty(); + try { + ComputeResourceDescription computeResource = regClient.getComputeResource(resourceId); + if (computeResource != null) { + computeResourceOp = Optional.of(computeResource); + } + } catch (TApplicationException e) { + // TApplicationException with "unknown result" means resource not found (null return for non-nullable + // type) + logger.debug("Compute resource {} not found (TApplicationException): {}", resourceId, e.getMessage()); + } + + Optional<StorageResourceDescription> storageResourceOp = Optional.empty(); + if (computeResourceOp.isEmpty()) { + try { + StorageResourceDescription storageResource = regClient.getStorageResource(resourceId); + if (storageResource != null) { + storageResourceOp = Optional.of(storageResource); + } + } catch (TApplicationException e) { + // TApplicationException with "unknown result" means resource not found (null return for + // non-nullable type) + logger.debug( + "Storage resource {} not found (TApplicationException): {}", resourceId, e.getMessage()); + } + } + + if (computeResourceOp.isEmpty() && storageResourceOp.isEmpty()) { + logger.error( + "Resource with ID {} not found as either compute resource or storage resource", resourceId); + throw new InvalidRequestException("Resource with ID '" + resourceId + + "' not found as either compute resource or storage resource"); + } + + if (computeResourceOp.isPresent()) { + logger.debug("Found compute resource with ID {}. Resolving login username and credentials", resourceId); + context = resolveComputeStorageInfoContext(authzToken, gatewayId, userId, resourceId); + } else { + logger.debug("Found storage resource with ID {}. Resolving login username and credentials", resourceId); + context = resolveStorageStorageInfoContext(authzToken, gatewayId, userId, resourceId); + } + + registryClientPool.returnResource(regClient); + regClient = null; + + return context.adaptor.getStorageDirectoryInfo(location); + + } catch (InvalidRequestException | AiravataClientException e) { + if (regClient != null) { + registryClientPool.returnResource(regClient); + } + logger.error("Error while retrieving storage resource.", e); + throw e; + + } catch (Exception e) { + logger.error("Error while retrieving storage volume info for resource {}", resourceId, e); + registryClientPool.returnBrokenResource(regClient); + + AiravataSystemException exception = new AiravataSystemException(); + exception.setAiravataErrorType(AiravataErrorType.INTERNAL_ERROR); + exception.setMessage("Error while retrieving storage volume info. More info: " + e.getMessage()); + throw exception; + } + } + /** * Add a Local Job Submission details to a compute resource * App catalog will return a jobSubmissionInterfaceId which will be added to the jobSubmissionInterfaces. diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java b/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java index 9e9caaefc1..2845afb1ef 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java @@ -51,6 +51,7 @@ import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescr import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionInterface; import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol; import org.apache.airavata.model.appcatalog.computeresource.SSHJobSubmission; +import org.apache.airavata.model.appcatalog.storageresource.StorageDirectoryInfo; import org.apache.airavata.model.appcatalog.storageresource.StorageVolumeInfo; import org.apache.airavata.model.credential.store.SSHCredential; import org.slf4j.Logger; @@ -645,6 +646,54 @@ public class SSHJAgentAdaptor implements AgentAdaptor { } } + @Override + public StorageDirectoryInfo getStorageDirectoryInfo(String location) throws AgentException { + try { + String targetLocation = location; + if (targetLocation == null || targetLocation.trim().isEmpty()) { + CommandOutput homeOutput = executeCommand("echo $HOME", null); + + if (homeOutput.getExitCode() != 0 + || homeOutput.getStdOut() == null + || homeOutput.getStdOut().trim().isEmpty()) { + logger.error("Failed to determine user's home directory: {}", homeOutput.getStdError()); + throw new AgentException("Failed to determine user's home directory: " + homeOutput.getStdError()); + } + targetLocation = homeOutput.getStdOut().trim(); + } + + // Escape location to prevent command injection and handle spaces + String escapedLocation = targetLocation.replace("'", "'\"'\"'"); + String duKBytesCommand = "du -sk '" + escapedLocation + "'"; + + CommandOutput duKBytesOutput = executeCommand(duKBytesCommand, null); + + if (duKBytesOutput.getExitCode() != 0) { + logger.error( + "Failed to execute du -sk command for location {}: {}", + targetLocation, + duKBytesOutput.getStdError()); + throw new AgentException("Failed to execute du -sk command for location " + targetLocation + ": " + + duKBytesOutput.getStdError()); + } + + String outputKbStr = duKBytesOutput.getStdOut().trim(); + logger.info("OutputKbStr: for du -ku {} is {}", location, outputKbStr); + String numberOfKBytesStr = outputKbStr.split(" ")[0]; + + long numberOfKBytes = Long.parseLong(numberOfKBytesStr); + + StorageDirectoryInfo storageDirectoryInfo = new StorageDirectoryInfo(); + storageDirectoryInfo.setTotalSizeBytes(numberOfKBytes * 1024); + storageDirectoryInfo.setTotalSize(numberOfKBytes + "kb"); + return storageDirectoryInfo; + + } catch (Exception e) { + logger.error("Error while retrieving storage directory info for location " + location, e); + throw new AgentException("Error while retrieving storage directory info for location " + location, e); + } + } + private StorageVolumeInfo parseDfOutput(String dfHumanOutput, String dfBytesOutput, String targetLocation) throws AgentException { try { diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java b/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java index 40096595b2..8ee4237fd7 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.UUID; import org.apache.airavata.agents.api.*; import org.apache.airavata.model.appcatalog.computeresource.*; +import org.apache.airavata.model.appcatalog.storageresource.StorageDirectoryInfo; import org.apache.airavata.model.appcatalog.storageresource.StorageVolumeInfo; import org.apache.airavata.model.credential.store.SSHCredential; import org.slf4j.Logger; @@ -547,6 +548,12 @@ public class SshAgentAdaptor implements AgentAdaptor { "Operation not supported by SshAgentAdaptor. Use SSHJAgentAdaptor instead."); } + @Override + public StorageDirectoryInfo getStorageDirectoryInfo(String location) throws AgentException { + throw new UnsupportedOperationException( + "Operation not supported by SshAgentAdaptor. Use SSHJAgentAdaptor instead."); + } + private static class DefaultUserInfo implements UserInfo, UIKeyboardInteractive { private String userName; diff --git a/thrift-interface-descriptions/airavata-apis/airavata_api.thrift b/thrift-interface-descriptions/airavata-apis/airavata_api.thrift index c5a95d325d..964a55ceb5 100644 --- a/thrift-interface-descriptions/airavata-apis/airavata_api.thrift +++ b/thrift-interface-descriptions/airavata-apis/airavata_api.thrift @@ -20,7 +20,7 @@ /** * Application Programming Interface definition for Apache Airavata Services. - * this parent thrift file is contains all service interfaces. The data models are + * this parent thrift file is contains all service interfaces. The data models are * described in respective thrift files. */ @@ -102,7 +102,7 @@ service Airavata extends base_api.BaseAPI { * * @param gateway * The gateway data model. - * + * * @return gatewayId * Th unique identifier of the newly registered gateway. * @@ -673,13 +673,13 @@ service Airavata extends base_api.BaseAPI { * * @throws org.apache.airavata.model.error.InvalidRequestException * For any incorrect forming of the request itself. - * + * * @throws org.apache.airavata.model.error.ExperimentNotFoundException * If the specified experiment is not previously created, then an Experiment Not Found Exception is thrown. - * + * * @throws org.apache.airavata.model.error.AiravataClientException * The following list of exceptions are thrown which Airavata Client can take corrective actions to resolve: - * + * * UNKNOWN_GATEWAY_ID - If a Gateway is not registered with Airavata as a one time administrative * step, then Airavata Registry will not have a provenance area setup. The client has to follow * gateway registration steps and retry this request. @@ -811,13 +811,13 @@ service Airavata extends base_api.BaseAPI { * * @throws org.apache.airavata.model.error.InvalidRequestException * For any incorrect forming of the request itself. - * + * * @throws org.apache.airavata.model.error.ExperimentNotFoundException * If the specified experiment is not previously created, then an Experiment Not Found Exception is thrown. - * + * * @throws org.apache.airavata.model.error.AiravataClientException * The following list of exceptions are thrown which Airavata Client can take corrective actions to resolve: - * + * * UNKNOWN_GATEWAY_ID - If a Gateway is not registered with Airavata as a one time administrative * step, then Airavata Registry will not have a provenance area setup. The client has to follow * gateway registration steps and retry this request. @@ -888,13 +888,13 @@ service Airavata extends base_api.BaseAPI { * * @throws org.apache.airavata.model.error.InvalidRequestException * For any incorrect forming of the request itself. - * + * * @throws org.apache.airavata.model.error.ExperimentNotFoundException * If the specified experiment is not previously created, then an Experiment Not Found Exception is thrown. - * + * * @throws org.apache.airavata.model.error.AiravataClientException * The following list of exceptions are thrown which Airavata Client can take corrective actions to resolve: - * + * * UNKNOWN_GATEWAY_ID - If a Gateway is not registered with Airavata as a one time administrative * step, then Airavata Registry will not have a provenance area setup. The client has to follow * gateway registration steps and retry this request. @@ -1076,7 +1076,7 @@ service Airavata extends base_api.BaseAPI { * * Clone an Existing Experiment * Existing specified experiment is cloned and a new name is provided. A copy of the experiment configuration is made and is persisted with new metadata. - * The client has to subsequently update this configuration if needed and launch the cloned experiment. + * The client has to subsequently update this configuration if needed and launch the cloned experiment. * * @param newExperimentName * experiment name that should be used in the cloned experiment @@ -1094,13 +1094,13 @@ service Airavata extends base_api.BaseAPI { * * @throws org.apache.airavata.model.error.InvalidRequestException * For any incorrect forming of the request itself. - * + * * @throws org.apache.airavata.model.error.ExperimentNotFoundException * If the specified experiment is not previously created, then an Experiment Not Found Exception is thrown. - * + * * @throws org.apache.airavata.model.error.AiravataClientException * The following list of exceptions are thrown which Airavata Client can take corrective actions to resolve: - * + * * UNKNOWN_GATEWAY_ID - If a Gateway is not registered with Airavata as a one time administrative * step, then Airavata Registry will not have a provenance area setup. The client has to follow * gateway registration steps and retry this request. @@ -1196,13 +1196,13 @@ service Airavata extends base_api.BaseAPI { * * @throws org.apache.airavata.model.error.InvalidRequestException * For any incorrect forming of the request itself. - * + * * @throws org.apache.airavata.model.error.ExperimentNotFoundException * If the specified experiment is not previously created, then an Experiment Not Found Exception is thrown. - * + * * @throws org.apache.airavata.model.error.AiravataClientException * The following list of exceptions are thrown which Airavata Client can take corrective actions to resolve: - * + * * UNKNOWN_GATEWAY_ID - If a Gateway is not registered with Airavata as a one time administrative * step, then Airavata Registry will not have a provenance area setup. The client has to follow * gateway registration steps and retry this request. @@ -3627,6 +3627,22 @@ service Airavata extends base_api.BaseAPI { 3: airavata_errors.AiravataSystemException ase, 4: airavata_errors.AuthorizationException ae) +/** + * Get storage directory information for a compute or storage resource. + * + * @param authzToken + * @param resourceId Can be either a compute resource ID or storage resource ID + * @param location Optional path/mount point. If null/empty, defaults to user's home directory ($HOME) + * @return StorageDirectoryInfo containing directory size information + */ + storage_resource_model.StorageDirectoryInfo getStorageDirectoryInfo(1: required security_model.AuthzToken authzToken, + 2: required string resourceId, + 3: optional string location) + throws (1: airavata_errors.InvalidRequestException ire, + 2: airavata_errors.AiravataClientException ace, + 3: airavata_errors.AiravataSystemException ase, + 4: airavata_errors.AuthorizationException ae) + // //End of API } diff --git a/thrift-interface-descriptions/data-models/storage_resource_model.thrift b/thrift-interface-descriptions/data-models/storage_resource_model.thrift index 8b41079379..9d6fa4d70e 100644 --- a/thrift-interface-descriptions/data-models/storage_resource_model.thrift +++ b/thrift-interface-descriptions/data-models/storage_resource_model.thrift @@ -79,3 +79,14 @@ struct StorageVolumeInfo { 8: required string mountPoint, 9: optional string filesystemType, } + +/** + * Provides Directory Size Information of a given storage + * + * totalSize: Total size in human-readable format (e.g., "100G", "500M") + * totalSizeBytes: Total size in bytes + */ +struct StorageDirectoryInfo { + 1: required string totalSize, + 2: required i64 totalSizeBytes, +}
