(asterixdb) branch master updated: [ASTERIXDB-3420][STO] Account for file changes on directory traversal

[mhubail]

This is an automated email from the ASF dual-hosted git repository.

mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 8b1003bd93 [ASTERIXDB-3420][STO] Account for file changes on directory 
traversal
8b1003bd93 is described below

commit 8b1003bd931136520f15f6eac103d0616026d229
Author: Savyasach Reddy <esavyared...@gmail.com>
AuthorDate: Wed Jun 5 23:59:42 2024 +0530

    [ASTERIXDB-3420][STO] Account for file changes on directory traversal
    
    - user model changes: no
    - storage format changes: no
    - interface changes: no
    
    Details:
    
    - Replace FileUtils.sizeOfDirectory with custom directory size function
      that accounts for file changes while traversing a directory.
    
    Change-Id: I660233e6b74429c75b9dd63103db1248846054a0
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18343
    Reviewed-by: Murtadha Hubail <mhub...@apache.org>
    Tested-by: Murtadha Hubail <mhub...@apache.org>
    Integration-Tests: Murtadha Hubail <mhub...@apache.org>
---
 .../java/org/apache/hyracks/api/util/IoUtil.java   | 31 ++++++++++++++++++++++
 .../apache/hyracks/control/nc/io/IOManager.java    |  2 +-
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git 
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java
 
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java
index fcf8c7a645..ddb8ef6cfc 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java
@@ -24,15 +24,19 @@ import java.io.FileNotFoundException;
 import java.io.FilenameFilter;
 import java.io.IOException;
 import java.nio.channels.FileChannel;
+import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
 import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 
 import org.apache.commons.io.FileUtils;
@@ -210,4 +214,31 @@ public class IoUtil {
         }
         return files;
     }
+
+    public static long sizeOfDirectory(final Path path) {
+        final AtomicLong size = new AtomicLong(0);
+        try {
+            Files.walkFileTree(path, new SimpleFileVisitor<>() {
+                @Override
+                public FileVisitResult visitFile(Path file, 
BasicFileAttributes attrs) {
+                    size.addAndGet(attrs.size());
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFileFailed(Path file, IOException 
exc) {
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult postVisitDirectory(Path dir, 
IOException exc) {
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+        } catch (IOException e) {
+            // This should never happen
+            throw new IllegalStateException("Cannot get the size of directory 
" + path);
+        }
+        return size.get();
+    }
 }
diff --git 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
index 9909e97a5d..b35111e386 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
@@ -473,7 +473,7 @@ public class IOManager implements IIOManager {
     public long getTotalDiskUsage() {
         long totalSize = 0;
         for (IODeviceHandle handle : ioDevices) {
-            totalSize += FileUtils.sizeOfDirectory(handle.getMount());
+            totalSize += IoUtil.sizeOfDirectory(handle.getMount().toPath());
         }
         return totalSize;
     }