(orc) branch branch-2.0 updated: ORC-1631: Support `summary` output in `sizes` command

[dongjoon]

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/orc.git


The following commit(s) were added to refs/heads/branch-2.0 by this push:
     new 95cb283f3 ORC-1631: Support `summary` output in `sizes` command
95cb283f3 is described below

commit 95cb283f3fc9a56007d8983378609c77bf0e6132
Author: sychen <syc...@ctrip.com>
AuthorDate: Sun Feb 25 23:20:51 2024 -0800

    ORC-1631: Support `summary` output in `sizes` command
    
    ### What changes were proposed in this pull request?
    Add support for summarizing the number of files, file sizes and file lines 
in the sizes command.
    
    ### Why are the changes needed?
    When we count the size of each field, we only know the percentage and the 
average size of each row, but we do not know the overall value.
    
    ### How was this patch tested?
    local test
    
    ```bash
    java -jar orc-tools-2.1.0-SNAPSHOT-uber.jar sizes -h
    usage: sizes
     -h,--help              Print help message
     -i,--ignoreExtension   Ignore ORC file extension
     -s,--summary           Summarize the number of files, file sizes, and
                            file rows
    ```
    
    ```
    java -jar orc-tools-2.1.0-SNAPSHOT-uber.jar sizes -s
    ```
    
    ```
    Total Files: 5
    Total Sizes: 4803687270
    Total Rows: 39820045
    Percent  Bytes/Row  Name
      26.41  31.86
    ```
    
    ### Was this patch authored or co-authored using generative AI tooling?
    No
    
    Closes #1816 from cxzl25/ORC-1631.
    
    Authored-by: sychen <syc...@ctrip.com>
    Signed-off-by: Dongjoon Hyun <dongj...@apache.org>
    (cherry picked from commit f46e55af8dd11f978f6ced35f8b59789101cd780)
    Signed-off-by: Dongjoon Hyun <dongj...@apache.org>
---
 .../src/java/org/apache/orc/tools/ColumnSizes.java    | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/java/tools/src/java/org/apache/orc/tools/ColumnSizes.java 
b/java/tools/src/java/org/apache/orc/tools/ColumnSizes.java
index b9cfb081b..2347ac744 100644
--- a/java/tools/src/java/org/apache/orc/tools/ColumnSizes.java
+++ b/java/tools/src/java/org/apache/orc/tools/ColumnSizes.java
@@ -126,7 +126,7 @@ public class ColumnSizes {
     }
   }
 
-  private void printResults(PrintStream out) {
+  private void printResults(PrintStream out, boolean summary) {
     List<StringLongPair> sizes = new ArrayList<>(columnSizes.length + 5);
     for(int column = 0; column < columnSizes.length; ++column) {
       if (columnSizes[column] > 0) {
@@ -153,6 +153,10 @@ public class ColumnSizes {
     // sort by descending size, ascending name
     sizes.sort((x, y) -> x.size != y.size ?
         Long.compare(y.size, x.size) : x.name.compareTo(y.name));
+    if (summary) {
+      out.printf("Total Sizes: %d%n", totalSize);
+      out.printf("Total Rows: %d%n", rows);
+    }
     out.println("Percent  Bytes/Row  Name");
     for (StringLongPair item: sizes) {
       out.println(String.format("  %-5.2f  %-9.2f  %s",
@@ -169,9 +173,11 @@ public class ColumnSizes {
       return;
     }
     boolean ignoreExtension = cli.hasOption("ignoreExtension");
+    boolean summary = cli.hasOption("summary");
     String[] files = cli.getArgs();
 
     ColumnSizes result = null;
+    int totalFiles = 0;
     int badFiles = 0;
     for(String root: files) {
       Path rootPath = new Path(root);
@@ -179,6 +185,7 @@ public class ColumnSizes {
       for(RemoteIterator<LocatedFileStatus> itr = fs.listFiles(rootPath, 
true); itr.hasNext(); ) {
         LocatedFileStatus status = itr.next();
         if (status.isFile() && (ignoreExtension || 
status.getPath().getName().endsWith(".orc"))) {
+          totalFiles += 1;
           try {
             if (result == null) {
               result = new ColumnSizes(conf, status);
@@ -197,7 +204,10 @@ public class ColumnSizes {
     if (result == null) {
       System.err.println("No files found");
     } else {
-      result.printResults(System.out);
+      if (summary) {
+        System.out.printf("Total Files: %d%n", totalFiles);
+      }
+      result.printResults(System.out, summary);
     }
     if (badFiles > 0) {
       System.err.println(badFiles + " bad ORC files found.");
@@ -217,6 +227,11 @@ public class ColumnSizes {
         .desc("Ignore ORC file extension")
         .build());
 
+    result.addOption(Option.builder("s")
+        .longOpt("summary")
+        .desc("Summarize the number of files, file sizes, and file rows")
+        .build());
+
     result.addOption(Option.builder("h")
         .longOpt("help")
         .desc("Print help message")