(asterixdb) branch master updated: [NO ISSUE][MISC] Add method to compute remote storage size used

[peeyush]

This is an automated email from the ASF dual-hosted git repository.

peeyush pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git


The following commit(s) were added to refs/heads/master by this push:
     new a931a5759a [NO ISSUE][MISC] Add method to compute remote storage size 
used
a931a5759a is described below

commit a931a5759ac534f5bd5478f1874568e9856b81ee
Author: Peeyush Gupta <peeyush.gu...@couchbase.com>
AuthorDate: Tue Jul 16 15:33:44 2024 -0700

    [NO ISSUE][MISC] Add method to compute remote storage size used
    
    - user model changes: no
    - storage format changes: no
    - interface changes: no
    
    Ext-ref: MB-62718
    Change-Id: I3496da75ecf2fcd3e6306b8fd2e2d722f0462516
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18491
    Reviewed-by: Peeyush Gupta <peeyush.gu...@couchbase.com>
    Reviewed-by: Murtadha Hubail <mhub...@apache.org>
    Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
    Tested-by: Peeyush Gupta <peeyush.gu...@couchbase.com>
---
 .../asterix/cloud/AbstractCloudIOManager.java      | 69 ++++++++++++----------
 1 file changed, 39 insertions(+), 30 deletions(-)

diff --git 
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
 
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
index 4913f83037..c005253293 100644
--- 
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
+++ 
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
@@ -375,43 +375,14 @@ public abstract class AbstractCloudIOManager extends 
IOManager implements IParti
      */
     public final JsonNode listAsJson(ObjectMapper objectMapper) {
         ArrayNode objectsInfo = objectMapper.createArrayNode();
-        List<CloudFile> allFiles = new ArrayList<>();
         try {
-            // get cached files (read from disk)
-            for (IODeviceHandle deviceHandle : getIODevices()) {
-                FileReference storageRoot = 
deviceHandle.createFileRef(STORAGE_ROOT_DIR_NAME);
-
-                Set<FileReference> deviceFiles;
-                try {
-                    deviceFiles = localIoManager.list(storageRoot, 
IoUtil.NO_OP_FILTER);
-                } catch (Throwable th) {
-                    LOGGER.warn("Failed to get local storage files for root 
{}", storageRoot.getRelativePath(), th);
-                    continue;
-                }
-
-                for (FileReference fileReference : deviceFiles) {
-                    try {
-                        
allFiles.add(CloudFile.of(fileReference.getRelativePath(), 
fileReference.getFile().length()));
-                    } catch (Throwable th) {
-                        LOGGER.warn("Encountered issue for local storage file 
{}", fileReference.getRelativePath(), th);
-                    }
-                }
-            }
-
-            // get uncached files from uncached files tracker
-            for (UncachedFileReference uncachedFile : getUncachedFiles()) {
-                allFiles.add(CloudFile.of(uncachedFile.getRelativePath(), 
uncachedFile.getSize()));
-            }
-
-            // combine all and sort
+            List<CloudFile> allFiles = list();
             allFiles.sort((x, y) -> 
String.CASE_INSENSITIVE_ORDER.compare(x.getPath(), y.getPath()));
-
             for (CloudFile file : allFiles) {
                 ObjectNode objectInfo = objectsInfo.addObject();
                 objectInfo.put("path", file.getPath());
                 objectInfo.put("size", file.getSize());
             }
-
             return objectsInfo;
         } catch (Throwable th) {
             LOGGER.warn("Failed to retrieve list of all cloud files", th);
@@ -422,6 +393,36 @@ public abstract class AbstractCloudIOManager extends 
IOManager implements IParti
         }
     }
 
+    private List<CloudFile> list() {
+        List<CloudFile> allFiles = new ArrayList<>();
+        // get cached files (read from disk)
+        for (IODeviceHandle deviceHandle : getIODevices()) {
+            FileReference storageRoot = 
deviceHandle.createFileRef(STORAGE_ROOT_DIR_NAME);
+
+            Set<FileReference> deviceFiles;
+            try {
+                deviceFiles = localIoManager.list(storageRoot, 
IoUtil.NO_OP_FILTER);
+            } catch (Throwable th) {
+                LOGGER.warn("Failed to get local storage files for root {}", 
storageRoot.getRelativePath(), th);
+                continue;
+            }
+
+            for (FileReference fileReference : deviceFiles) {
+                try {
+                    allFiles.add(CloudFile.of(fileReference.getRelativePath(), 
fileReference.getFile().length()));
+                } catch (Throwable th) {
+                    LOGGER.warn("Encountered issue for local storage file {}", 
fileReference.getRelativePath(), th);
+                }
+            }
+        }
+
+        // get uncached files from uncached files tracker
+        for (UncachedFileReference uncachedFile : getUncachedFiles()) {
+            allFiles.add(CloudFile.of(uncachedFile.getRelativePath(), 
uncachedFile.getSize()));
+        }
+        return allFiles;
+    }
+
     /**
      * Writes the bytes to the specified key in the bucket
      *
@@ -465,4 +466,12 @@ public abstract class AbstractCloudIOManager extends 
IOManager implements IParti
                     + ", cloudOffset: " + cloudOffset + " != 
requestedWriteOffset: " + requestedWriteOffset);
         }
     }
+
+    public long getTotalRemoteStorageSizeForNodeBytes() {
+        long size = 0;
+        for (CloudFile file : list()) {
+            size += file.getSize();
+        }
+        return size;
+    }
 }