Repository: carbondata
Updated Branches:
  refs/heads/master 63a28a951 -> 789055e19


[CARBONDATA-3060]Improve the command for cli and fixed other issues

improve the syntax for CLI DDL: CarbonCli for table <table_name> options('-cmd 
summary/benchmark -a -s -v -c <column_name> -m')

This closes #2882


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/789055e1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/789055e1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/789055e1

Branch: refs/heads/master
Commit: 789055e1972c9456998c3a1a7516720fb7d506c2
Parents: 63a28a9
Author: akashrn5 <akashnilu...@gmail.com>
Authored: Mon Oct 29 18:14:24 2018 +0530
Committer: Jacky Li <jacky.li...@qq.com>
Committed: Mon Nov 5 15:20:56 2018 +0800

----------------------------------------------------------------------
 .../schema/table/column/ColumnSchema.java       |  9 ++
 .../examples/sdk/CarbonReaderExample.java       |  4 +-
 .../TestNonTransactionalCarbonTable.scala       |  2 +-
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala | 10 +-
 .../command/management/CarbonCliCommand.scala   | 62 +++++++++++++
 .../management/CarbonShowSummaryCommand.scala   | 62 -------------
 .../sql/parser/CarbonSpark2SqlParser.scala      | 12 +--
 .../org/apache/carbondata/tool/CarbonCli.java   | 21 ++++-
 .../org/apache/carbondata/tool/DataFile.java    | 15 ++-
 .../org/apache/carbondata/tool/DataSummary.java | 98 +++++++++++---------
 10 files changed, 172 insertions(+), 123 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/789055e1/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git 
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
 
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
index 8eba101..cf7ecab 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
@@ -571,4 +571,13 @@ public class ColumnSchema implements Serializable, 
Writable {
     }
     this.isLocalDictColumn = in.readBoolean();
   }
+
+  /**
+   * returns whether column is complex column based on column name for child 
column
+   * @return
+   */
+  public boolean isComplexColumn() {
+    return this.getColumnName()
+        .contains(".val") || this.getColumnName().contains(".");
+  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/789055e1/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
----------------------------------------------------------------------
diff --git 
a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
 
b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
index ef4ae7a..a2f2eee 100644
--- 
a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
+++ 
b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
@@ -61,7 +61,9 @@ public class CarbonReaderExample {
             CarbonWriter writer = CarbonWriter.builder()
                 .outputPath(path)
                 .withLoadOptions(map)
-                .withCsvInput(new Schema(fields)).build();
+                .withCsvInput(new Schema(fields))
+                .writtenBy("CarbonReaderExample")
+                .build();
 
             for (int i = 0; i < 10; i++) {
                 String[] row2 = new String[]{

http://git-wip-us.apache.org/repos/asf/carbondata/blob/789055e1/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
----------------------------------------------------------------------
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
index a3a3fc3..5b93553 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -389,7 +389,7 @@ class TestNonTransactionalCarbonTable extends QueryTest 
with BeforeAndAfterAll {
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
 
-    val output = sql("show summary for table sdkOutputTable 
options('command'='-cmd,summary,-p,-a,-v,-c,age')").collect()
+    val output = sql("Carboncli for table sdkOutputTable options('-cmd summary 
-a -v -c age')").collect()
 
     assert(output.toList.contains(Row("written_by                       
Version         ")))
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/789055e1/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git 
a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
 
b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 107a303..4a37a20 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -188,7 +188,7 @@ abstract class CarbonDDLSqlParser extends 
AbstractCarbonSparkSQLParser {
   protected val STREAM = carbonKeyWord("STREAM")
   protected val STREAMS = carbonKeyWord("STREAMS")
   protected val STMPROPERTIES = carbonKeyWord("STMPROPERTIES")
-  protected val SUMMARY = carbonKeyWord("SUMMARY")
+  protected val CARBONCLI = carbonKeyWord("CARBONCLI")
 
   protected val doubleQuotedString = "\"([^\"]+)\"".r
   protected val singleQuotedString = "'([^']+)'".r
@@ -1145,10 +1145,10 @@ abstract class CarbonDDLSqlParser extends 
AbstractCarbonSparkSQLParser {
       case _ => ("", "")
     }
 
-  protected lazy val summaryOptions: Parser[(String, String)] =
-    (stringLit <~ "=") ~ stringLit ^^ {
-      case opt ~ optvalue => (opt.trim.toLowerCase(), optvalue)
-      case _ => ("", "")
+  protected lazy val commandOptions: Parser[String] =
+    stringLit ^^ {
+      case optValue => optValue
+      case _ => ""
     }
 
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/789055e1/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCliCommand.scala
----------------------------------------------------------------------
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCliCommand.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCliCommand.scala
new file mode 100644
index 0000000..bf5adc3
--- /dev/null
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCliCommand.scala
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.management
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, 
AttributeReference}
+import org.apache.spark.sql.execution.command.{Checker, DataCommand}
+import org.apache.spark.sql.types.StringType
+
+import org.apache.carbondata.tool.CarbonCli
+
+/**
+ * CarbonCLi command class which is integrated to cli and sql support is 
provided via this class
+ * @param databaseNameOp
+ * @param tableName
+ * @param commandOptions
+ */
+case class CarbonCliCommand(
+    databaseNameOp: Option[String],
+    tableName: String,
+    commandOptions: String)
+  extends DataCommand {
+
+  override def output: Seq[Attribute] = {
+      Seq(AttributeReference("CarbonCli", StringType, nullable = false)())
+  }
+
+  override def processData(sparkSession: SparkSession): Seq[Row] = {
+    Checker.validateTableExists(databaseNameOp, tableName, sparkSession)
+    val carbonTable = CarbonEnv.getCarbonTable(databaseNameOp, 
tableName)(sparkSession)
+    val commandArgs: Seq[String] = commandOptions.split("\\s+")
+    val finalCommands = commandArgs.collect {
+      case a if a.trim.equalsIgnoreCase("summary") || 
a.trim.equalsIgnoreCase("benchmark") =>
+        Seq(a, "-p", carbonTable.getTablePath)
+      case x => Seq(x.trim)
+    }.flatten
+    val summaryOutput = new util.ArrayList[String]()
+    CarbonCli.run(finalCommands.toArray, summaryOutput)
+    summaryOutput.asScala.map(x =>
+      Row(x)
+    )
+  }
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/789055e1/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowSummaryCommand.scala
----------------------------------------------------------------------
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowSummaryCommand.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowSummaryCommand.scala
deleted file mode 100644
index 461f31f..0000000
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowSummaryCommand.scala
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.command.management
-
-import java.util
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
-import org.apache.spark.sql.catalyst.expressions.{Attribute, 
AttributeReference}
-import org.apache.spark.sql.execution.command.{Checker, DataCommand}
-import org.apache.spark.sql.types.StringType
-
-import org.apache.carbondata.tool.CarbonCli
-
-/**
- * Show summary command class which is integrated to cli and sql support is 
provided via this class
- * @param databaseNameOp
- * @param tableName
- * @param commandOptions
- */
-case class CarbonShowSummaryCommand(
-    databaseNameOp: Option[String],
-    tableName: String,
-    commandOptions: Map[String, String])
-  extends DataCommand {
-
-  override def output: Seq[Attribute] = {
-      Seq(AttributeReference("Table Summary", StringType, nullable = false)())
-  }
-
-  override def processData(sparkSession: SparkSession): Seq[Row] = {
-    Checker.validateTableExists(databaseNameOp, tableName, sparkSession)
-    val carbonTable = CarbonEnv.getCarbonTable(databaseNameOp, 
tableName)(sparkSession)
-    val commandArgs: Seq[String] = commandOptions("command").split(",")
-    val finalCommands = commandArgs.collect {
-      case a if a.trim.equalsIgnoreCase("-p") =>
-        Seq(a, carbonTable.getTablePath)
-      case x => Seq(x.trim)
-    }.flatten
-    val summaryOutput = new util.ArrayList[String]()
-    CarbonCli.run(finalCommands.toArray, summaryOutput)
-    summaryOutput.asScala.map(x =>
-      Row(x)
-    )
-  }
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/789055e1/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
----------------------------------------------------------------------
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
index 5427168..39e2f30 100644
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
@@ -497,18 +497,18 @@ class CarbonSpark2SqlParser extends CarbonDDLSqlParser {
 
 
   protected lazy val cli: Parser[LogicalPlan] =
-    (SHOW ~> SUMMARY ~> FOR ~> TABLE) ~> (ident <~ ".").? ~ ident ~
-    (OPTIONS ~> "(" ~> repsep(summaryOptions, ",") <~ ")").? <~
+    (CARBONCLI ~> FOR ~> TABLE) ~> (ident <~ ".").? ~ ident ~
+    (OPTIONS ~> "(" ~> commandOptions <~ ")").? <~
     opt(";") ^^ {
       case databaseName ~ tableName ~ commandList =>
-        var commandOptions: Map[String, String] = null
+        var commandOptions: String = null
         if (commandList.isDefined) {
-          commandOptions = commandList.getOrElse(List.empty[(String, 
String)]).toMap
+          commandOptions = commandList.get
         }
-        CarbonShowSummaryCommand(
+        CarbonCliCommand(
           convertDbNameToLowerCase(databaseName),
           tableName.toLowerCase(),
-          commandOptions.map { case (key, value) => key.toLowerCase -> value })
+          commandOptions)
     }
 
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/789055e1/tools/cli/src/main/java/org/apache/carbondata/tool/CarbonCli.java
----------------------------------------------------------------------
diff --git a/tools/cli/src/main/java/org/apache/carbondata/tool/CarbonCli.java 
b/tools/cli/src/main/java/org/apache/carbondata/tool/CarbonCli.java
index bb2260b..6fc3128 100644
--- a/tools/cli/src/main/java/org/apache/carbondata/tool/CarbonCli.java
+++ b/tools/cli/src/main/java/org/apache/carbondata/tool/CarbonCli.java
@@ -19,6 +19,8 @@ package org.apache.carbondata.tool;
 
 import java.io.IOException;
 import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -148,7 +150,10 @@ public class CarbonCli {
       outPuts = new ArrayList<>();
     }
     if (line.hasOption("h")) {
-      printHelp(options);
+      collectHelpInfo(options);
+      for (String output : outPuts) {
+        out.println(output);
+      }
       return;
     }
 
@@ -167,7 +172,10 @@ public class CarbonCli {
     } else {
       out.println("command " + cmd + " is not supported");
       outPuts.add("command " + cmd + " is not supported");
-      printHelp(options);
+      collectHelpInfo(options);
+      for (String output : outPuts) {
+        out.println(output);
+      }
       return;
     }
 
@@ -186,9 +194,14 @@ public class CarbonCli {
     }
   }
 
-  private static void printHelp(Options options) {
+  private static void collectHelpInfo(Options options) {
     HelpFormatter formatter = new HelpFormatter();
-    formatter.printHelp("CarbonCli", options);
+    StringWriter stringWriter = new StringWriter();
+    PrintWriter printWriter = new PrintWriter(stringWriter);
+    formatter.printHelp(printWriter, formatter.getWidth(), "CarbonCli", null, 
options,
+        formatter.getLeftPadding(), formatter.getDescPadding(), null, false);
+    printWriter.flush();
+    outPuts.add(stringWriter.toString());
   }
 
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/789055e1/tools/cli/src/main/java/org/apache/carbondata/tool/DataFile.java
----------------------------------------------------------------------
diff --git a/tools/cli/src/main/java/org/apache/carbondata/tool/DataFile.java 
b/tools/cli/src/main/java/org/apache/carbondata/tool/DataFile.java
index 457ef0c..e553a78 100644
--- a/tools/cli/src/main/java/org/apache/carbondata/tool/DataFile.java
+++ b/tools/cli/src/main/java/org/apache/carbondata/tool/DataFile.java
@@ -33,6 +33,7 @@ import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.memory.MemoryException;
 import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
 import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
 import org.apache.carbondata.core.reader.CarbonFooterReaderV3;
 import org.apache.carbondata.core.reader.CarbonHeaderReader;
@@ -446,7 +447,8 @@ class DataFile {
      * @return result
      */
     private double computePercentage(byte[] data, byte[] min, byte[] max, 
ColumnSchema column) {
-      if (column.getDataType() == DataTypes.STRING) {
+      if (column.getDataType() == DataTypes.STRING || column.getDataType() == 
DataTypes.BOOLEAN
+          || column.hasEncoding(Encoding.DICTIONARY) || 
column.getDataType().isComplexType()) {
         // for string, we do not calculate
         return 0;
       } else if (DataTypes.isDecimal(column.getDataType())) {
@@ -456,7 +458,16 @@ class DataFile {
         return dataValue.divide(factorValue).doubleValue();
       }
       double dataValue, minValue, factorValue;
-      if (column.getDataType() == DataTypes.SHORT) {
+      if (columnChunk.column.isDimensionColumn() &&
+          DataTypeUtil.isPrimitiveColumn(columnChunk.column.getDataType())) {
+        minValue = Double.valueOf(String.valueOf(
+            DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(min, 
column.getDataType())));
+        dataValue = Double.valueOf(String.valueOf(
+            DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(data, 
column.getDataType())));
+        factorValue = Double.valueOf(String.valueOf(
+            DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(max, 
column.getDataType())))
+            - minValue;
+      } else if (column.getDataType() == DataTypes.SHORT) {
         minValue = ByteUtil.toShort(min, 0);
         dataValue = ByteUtil.toShort(data, 0) - minValue;
         factorValue = ByteUtil.toShort(max, 0) - ByteUtil.toShort(min, 0);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/789055e1/tools/cli/src/main/java/org/apache/carbondata/tool/DataSummary.java
----------------------------------------------------------------------
diff --git 
a/tools/cli/src/main/java/org/apache/carbondata/tool/DataSummary.java 
b/tools/cli/src/main/java/org/apache/carbondata/tool/DataSummary.java
index cd4c040..1930bf5 100644
--- a/tools/cli/src/main/java/org/apache/carbondata/tool/DataSummary.java
+++ b/tools/cli/src/main/java/org/apache/carbondata/tool/DataSummary.java
@@ -20,12 +20,7 @@ package org.apache.carbondata.tool;
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.Charset;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 import org.apache.carbondata.common.Strings;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -33,11 +28,11 @@ import 
org.apache.carbondata.core.datastore.filesystem.CarbonFile;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.memory.MemoryException;
 import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
 import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
 import org.apache.carbondata.core.reader.CarbonHeaderReader;
 import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
 import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
-import org.apache.carbondata.core.util.ByteUtil;
 import org.apache.carbondata.core.util.CarbonUtil;
 import org.apache.carbondata.core.util.DataTypeUtil;
 import org.apache.carbondata.format.BlockletInfo3;
@@ -80,7 +75,9 @@ class DataSummary implements Command {
     }
     if (line.hasOption("s") || printAll) {
       if (dataFiles.size() > 0) {
-        
collectSchemaDetails(dataFiles.entrySet().iterator().next().getValue());
+        List<String> dataFilesSet = new ArrayList<>(dataFiles.keySet());
+        Collections.reverse(dataFilesSet);
+        collectSchemaDetails(dataFiles.get(dataFilesSet.get(0)));
       }
     }
     if (line.hasOption("m") || printAll) {
@@ -175,8 +172,8 @@ class DataSummary implements Command {
         tableFormatter.addRow(new String[]{
             segment.getLoadName(),
             segment.getSegmentStatus().toString(),
-            new java.sql.Date(segment.getLoadStartTime()).toString(),
-            new java.sql.Date(segment.getLoadEndTime()).toString(),
+            new java.sql.Timestamp(segment.getLoadStartTime()).toString(),
+            new java.sql.Timestamp(segment.getLoadEndTime()).toString(),
             segment.getMergedLoadName() == null ? "NA" : 
segment.getMergedLoadName(),
             segment.getFileFormat().toString(),
             dataSize,
@@ -306,9 +303,9 @@ class DataSummary implements Command {
           maxPercent = "NA";
           // for complex types min max can be given as NA and for varchar 
where min max is not
           // written, can give NA
-          if 
(blocklet.getColumnChunk().column.getColumnName().contains(".val") || blocklet
-              .getColumnChunk().column.getColumnName().contains(".") || 
!blocklet
-              .getColumnChunk().isMinMaxPresent) {
+          if 
(blocklet.getColumnChunk().column.hasEncoding(Encoding.DICTIONARY) ||
+              blocklet.getColumnChunk().column.isComplexColumn() ||
+              !blocklet.getColumnChunk().isMinMaxPresent) {
             min = "NA";
             max = "NA";
           } else {
@@ -316,26 +313,41 @@ class DataSummary implements Command {
             max = new String(blockletMax, Charset.forName(DEFAULT_CHARSET));
           }
         } else {
-          minPercent = String.format("%.1f", 
blocklet.getColumnChunk().getMinPercentage() * 100);
-          maxPercent = String.format("%.1f", 
blocklet.getColumnChunk().getMaxPercentage() * 100);
+          // for column has global dictionary and for complex columns,min and 
max percentage can be
+          // NA
+          if 
(blocklet.getColumnChunk().column.hasEncoding(Encoding.DICTIONARY) ||
+              blocklet.getColumnChunk().column.isComplexColumn() ||
+              blocklet.getColumnChunk().column.getDataType().isComplexType()) {
+            minPercent = "NA";
+            maxPercent = "NA";
+          } else {
+            minPercent =
+                String.format("%.1f", 
Math.abs(blocklet.getColumnChunk().getMinPercentage() * 100));
+            maxPercent =
+                String.format("%.1f", 
Math.abs(blocklet.getColumnChunk().getMaxPercentage() * 100));
+          }
           DataFile.ColumnChunk columnChunk = blocklet.columnChunk;
-          if (columnChunk.column.isDimensionColumn() && DataTypeUtil
+          // need to consider dictionary and complex columns
+          if (columnChunk.column.hasEncoding(Encoding.DICTIONARY) ||
+              blocklet.getColumnChunk().column.isComplexColumn() ||
+              blocklet.getColumnChunk().column.getDataType().isComplexType()) {
+            min = "NA";
+            max = "NA";
+          } else if (columnChunk.column.isDimensionColumn() && DataTypeUtil
               .isPrimitiveColumn(columnChunk.column.getDataType())) {
             min = 
DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(blockletMin,
                 columnChunk.column.getDataType()).toString();
             max = 
DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(blockletMax,
                 columnChunk.column.getDataType()).toString();
-          } else {
-            if (blockletMin.length > 4) {
-              min = String.valueOf(ByteUtil.toLong(blockletMin, 0, 
blockletMin.length));
-            } else {
-              min = String.valueOf(ByteUtil.toInt(blockletMin, 0, 
blockletMin.length));
-            }
-            if (blockletMax.length > 4) {
-              max = String.valueOf(ByteUtil.toLong(blockletMax, 0, 
blockletMax.length));
-            } else {
-              max = String.valueOf(ByteUtil.toInt(blockletMax, 0, 
blockletMax.length));
+            if (columnChunk.column.getDataType().equals(DataTypes.TIMESTAMP)) {
+              min = new java.sql.Timestamp(Long.parseLong(min) / 
1000).toString();
+              max = new java.sql.Timestamp(Long.parseLong(max) / 
1000).toString();
             }
+          } else {
+            min = String.valueOf(DataTypeUtil
+                .getMeasureObjectFromDataType(blockletMin, 
columnChunk.column.getDataType()));
+            max = String.valueOf(DataTypeUtil
+                .getMeasureObjectFromDataType(blockletMax, 
columnChunk.column.getDataType()));
           }
         }
         printer.addRow(
@@ -370,24 +382,26 @@ class DataSummary implements Command {
   }
 
   private void collectColumnChunkMeta(String columnName) throws IOException, 
MemoryException {
-    DataFile file = dataFiles.entrySet().iterator().next().getValue();
-    outPuts.add("");
-    outPuts.add("## Page Meta for column '" + columnName + "' in file " + 
file.getFilePath());
-    collectStats(columnName);
-    for (int i = 0; i < file.getAllBlocklets().size(); i++) {
-      DataFile.Blocklet blocklet = file.getAllBlocklets().get(i);
-      DataChunk3 dataChunk3 = blocklet.getColumnChunk().getDataChunk3();
-      List<DataChunk2> dataChunk2List = dataChunk3.getData_chunk_list();
-      outPuts.add(String.format("Blocklet %d:", i));
+    for (Map.Entry<String, DataFile> entry : dataFiles.entrySet()) {
+      DataFile file = entry.getValue();
+      outPuts.add("");
+      outPuts.add("## Page Meta for column '" + columnName + "' in file " + 
file.getFilePath());
+      collectStats(columnName);
+      for (int i = 0; i < file.getAllBlocklets().size(); i++) {
+        DataFile.Blocklet blocklet = file.getAllBlocklets().get(i);
+        DataChunk3 dataChunk3 = blocklet.getColumnChunk().getDataChunk3();
+        List<DataChunk2> dataChunk2List = dataChunk3.getData_chunk_list();
+        outPuts.add(String.format("Blocklet %d:", i));
 
-      // There will be many pages, for debugging purpose,
-      // just print 3 page for each blocklet is enough
-      for (int j = 0; j < dataChunk2List.size() && j < 3; j++) {
-        outPuts.add(String.format("Page %d (offset %d, length %d): %s",
-            j, dataChunk3.page_offset.get(j), dataChunk3.page_length.get(j),
-            dataChunk2List.get(j).toString()));
+        // There will be many pages, for debugging purpose,
+        // just print 3 page for each blocklet is enough
+        for (int j = 0; j < dataChunk2List.size() && j < 3; j++) {
+          outPuts.add(String
+              .format("Page %d (offset %d, length %d): %s", j, 
dataChunk3.page_offset.get(j),
+                  dataChunk3.page_length.get(j), 
dataChunk2List.get(j).toString()));
+        }
+        outPuts.add("");
       }
-      outPuts.add("");
     }
   }
 

Reply via email to