Repository: incubator-carbondata Updated Branches: refs/heads/master 826485c9d -> 41c5ad491
add block size info in descFormatted and executor log Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/cd61beb5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/cd61beb5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/cd61beb5 Branch: refs/heads/master Commit: cd61beb59dd27006906d5e6e088acaeb0379d75a Parents: 826485c Author: Jay357089 <liujunj...@huawei.com> Authored: Wed Oct 12 11:21:44 2016 +0800 Committer: jackylk <jacky.li...@huawei.com> Committed: Sat Oct 22 00:47:13 2016 +0800 ---------------------------------------------------------------------- .../metadata/schema/table/CarbonTable.java | 16 +++++++------- .../core/constants/CarbonCommonConstants.java | 13 +++++++++--- .../apache/carbondata/core/util/ByteUtil.java | 22 ++++++++++++++++++++ .../execution/command/carbonTableSchema.scala | 3 ++- .../describeTable/TestDescribeTable.scala | 7 +++++++ .../processing/mdkeygen/MDKeyGenStep.java | 8 +++---- .../store/CarbonFactDataHandlerColumnar.java | 4 ++-- .../store/CarbonFactDataHandlerModel.java | 12 +++++------ .../store/writer/AbstractFactDataWriter.java | 7 +++++++ 9 files changed, 68 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd61beb5/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java index c4d5594..c0a7032 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java @@ -87,9 +87,9 @@ public class CarbonTable implements Serializable { private long tableLastUpdatedTime; /** - * table block size + * table block size in MB */ - private int blocksize; + private int blockSize; public CarbonTable() { this.tableDimensionsMap = new HashMap<String, List<CarbonDimension>>(); @@ -101,7 +101,7 @@ public class CarbonTable implements Serializable { * @param tableInfo */ public void loadCarbonTable(TableInfo tableInfo) { - this.blocksize = getTableBlockSize(tableInfo); + this.blockSize = getTableBlockSizeInMB(tableInfo); this.tableLastUpdatedTime = tableInfo.getLastUpdatedTime(); this.tableUniqueName = tableInfo.getTableUniqueName(); this.metaDataFilepath = tableInfo.getMetaDataFilepath(); @@ -127,7 +127,7 @@ public class CarbonTable implements Serializable { * @param tableInfo * @return */ - private int getTableBlockSize(TableInfo tableInfo) { + private int getTableBlockSizeInMB(TableInfo tableInfo) { String tableBlockSize = null; // In case of old store there will not be any map for table properties so table properties // will be null @@ -430,12 +430,12 @@ public class CarbonTable implements Serializable { return 1; } - public int getBlocksize() { - return blocksize; + public int getBlockSizeInMB() { + return blockSize; } - public void setBlocksize(int blocksize) { - this.blocksize = blocksize; + public void setBlockSizeInMB(int blockSize) { + this.blockSize = blockSize; } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd61beb5/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java index 310b1c3..997f412 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java @@ -88,11 +88,17 @@ public final class CarbonCommonConstants { * max blocklet size */ public static final int BLOCKLET_SIZE_MAX_VAL = 12000000; - + /** + * default block size in MB + */ public static final String BLOCK_SIZE_DEFAULT_VAL = "1024"; - + /** + * min block size in MB + */ public static final int BLOCK_SIZE_MIN_VAL = 1; - + /** + * max block size in MB + */ public static final int BLOCK_SIZE_MAX_VAL = 2048; /** * default value of number of cores to be used @@ -711,6 +717,7 @@ public final class CarbonCommonConstants { public static final String PARTITIONCLASS = "partitionclass"; public static final String PARTITIONCOUNT = "partitioncount"; public static final String COLUMN_PROPERTIES = "columnproperties"; + // table block size in MB public static final String TABLE_BLOCKSIZE = "table_blocksize"; /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd61beb5/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java index 2f91d1e..85685ff 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java @@ -25,6 +25,8 @@ import java.nio.ByteOrder; import java.security.AccessController; import java.security.PrivilegedAction; +import org.apache.carbondata.core.constants.CarbonCommonConstants; + /** * Util class for byte comparision */ @@ -77,6 +79,26 @@ public final class ByteUtil { } /** + * convert number in byte to more readable format + * @param sizeInbyte + * @return + */ + public static String convertByteToReadable(long sizeInbyte) { + + String readableSize; + if (sizeInbyte < CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR) { + readableSize = sizeInbyte + " Byte"; + } else if (sizeInbyte < CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR * + CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR) { + readableSize = sizeInbyte / CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR + " KB"; + } else { + readableSize = sizeInbyte / CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR / + CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR + " MB"; + } + return readableSize; + } + + /** * Unsafe comparator */ public enum UnsafeComparer { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd61beb5/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala index c265f7f..919f310 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala @@ -1384,6 +1384,7 @@ private[sql] case class DescribeCommandFormatted( results ++= Seq(("Table Name : ", relation.tableMeta.carbonTableIdentifier.getTableName, "")) results ++= Seq(("CARBON Store Path : ", relation.tableMeta.storePath, "")) val carbonTable = relation.tableMeta.carbonTable + results ++= Seq(("Table Block Size : ", carbonTable.getBlockSizeInMB + " MB", "")) results ++= Seq(("", "", ""), ("##Detailed Column property", "", "")) if (colPropStr.length() > 0) { results ++= Seq((colPropStr, "", "")) @@ -1391,7 +1392,7 @@ private[sql] case class DescribeCommandFormatted( results ++= Seq(("NONE", "", "")) } val dimension = carbonTable - .getDimensionByTableName(relation.tableMeta.carbonTableIdentifier.getTableName); + .getDimensionByTableName(relation.tableMeta.carbonTableIdentifier.getTableName) results ++= getColumnGroups(dimension.asScala.toList) results.map { case (name, dataType, comment) => Row(f"$name%-36s $dataType%-80s $comment%-72s") http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd61beb5/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala index 1b27842..1f94646 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala @@ -20,6 +20,8 @@ package org.apache.carbondata.spark.testsuite.describeTable import org.apache.spark.sql.common.util.CarbonHiveContext._ import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.Row + import org.scalatest.BeforeAndAfterAll /** @@ -41,6 +43,11 @@ class TestDescribeTable extends QueryTest with BeforeAndAfterAll { checkAnswer(sql("DESC Desc1"), sql("DESC Desc2")) } + test("test describe formatted table") { + checkExistence(sql("DESC FORMATTED Desc1"), true, + "Table Block Size : 1024 MB") + } + override def afterAll: Unit = { sql("DROP TABLE Desc1") sql("DROP TABLE Desc2") http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd61beb5/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStep.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStep.java b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStep.java index fef79ea..57d74c3 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStep.java +++ b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStep.java @@ -90,9 +90,9 @@ public class MDKeyGenStep extends BaseStep { private String tableName; /** - * table blocksize + * table blocksize in MB */ - private int blocksize; + private int blockSize; /** * File manager @@ -314,7 +314,7 @@ public class MDKeyGenStep extends BaseStep { wrapperColumnSchema = CarbonUtil .getColumnSchemaList(carbonTable.getDimensionByTableName(tableName), carbonTable.getMeasureByTableName(tableName)); - blocksize = carbonTable.getBlocksize(); + blockSize = carbonTable.getBlockSizeInMB(); colCardinality = CarbonUtil.getFormattedCardinality(dimLensWithComplex, wrapperColumnSchema); segmentProperties = new SegmentProperties(wrapperColumnSchema, colCardinality); @@ -399,7 +399,7 @@ public class MDKeyGenStep extends BaseStep { carbonFactDataHandlerModel.setAggType(aggType); carbonFactDataHandlerModel.setFactDimLens(dimLens); carbonFactDataHandlerModel.setWrapperColumnSchema(wrapperColumnSchema); - carbonFactDataHandlerModel.setBlocksize(blocksize); + carbonFactDataHandlerModel.setBlockSizeInMB(blockSize); return carbonFactDataHandlerModel; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd61beb5/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java index b963329..8dd5ce2 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java @@ -134,7 +134,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { */ private String tableName; /** - * table block size + * table block size in MB */ private int tableBlockSize; /** @@ -343,7 +343,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { private void initParameters(CarbonFactDataHandlerModel carbonFactDataHandlerModel) { this.databaseName = carbonFactDataHandlerModel.getDatabaseName(); - this.tableBlockSize = carbonFactDataHandlerModel.getBlocksize(); + this.tableBlockSize = carbonFactDataHandlerModel.getBlockSizeInMB(); this.tableName = carbonFactDataHandlerModel.getTableName(); this.type = carbonFactDataHandlerModel.getAggType(); this.segmentProperties = carbonFactDataHandlerModel.getSegmentProperties(); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd61beb5/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java index a835d61..49c9410 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java @@ -40,18 +40,18 @@ public class CarbonFactDataHandlerModel { */ private String tableName; - public int getBlocksize() { - return blocksize; + public int getBlockSizeInMB() { + return blockSize; } - public void setBlocksize(int blocksize) { - this.blocksize = blocksize; + public void setBlockSizeInMB(int blockSize) { + this.blockSize = blockSize; } /** - * table blocksize + * table blocksize in MB */ - private int blocksize; + private int blockSize; /** * flag to check whether to group the similar data */ http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd61beb5/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java index 56134d5..f56a5d2 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java @@ -253,6 +253,13 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter< if (remainder > 0) { maxSize = maxSize + HDFS_CHECKSUM_LENGTH - remainder; } + // convert to make block size more readable. + String readableBlockSize = ByteUtil.convertByteToReadable(blockSize); + String readableFileSize = ByteUtil.convertByteToReadable(fileSize); + String readableMaxSize = ByteUtil.convertByteToReadable(maxSize); + LOGGER.info("The configured block size is " + readableBlockSize + + ", the actual carbon file size is " + readableFileSize + + ", choose the max value " + readableMaxSize + " as the block size on HDFS"); return maxSize; }
