Repository: incubator-carbondata Updated Branches: refs/heads/master 617b69aac -> 8235f82fe
[CARBONDATA-319] Bad Records logging for column data type is not proper in case of long, carbon system while creating the table metadata uses BIGINT instead of LONG , internally it converts the bigint to long type and processes , while processing the data if any long type data is having issue it will be logged into the bad record with data type Long which is not proper since as per the metadata the datatype of column is BIGINT. Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/274b8db0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/274b8db0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/274b8db0 Branch: refs/heads/master Commit: 274b8db060391b271248ede5bd77e3518759b57a Parents: 617b69a Author: sujith71955 <sujithchacko.2...@gmail.com> Authored: Sat Oct 15 21:09:27 2016 +0530 Committer: Venkata Ramana G <ramana.gollam...@huawei.com> Committed: Tue Oct 18 00:17:07 2016 +0530 ---------------------------------------------------------------------- .../core/carbon/metadata/datatype/DataType.java | 36 ++++++++++++-------- .../carbondata/core/util/DataTypeUtil.java | 27 +++++++++++++++ .../csvbased/CarbonCSVBasedSeqGenStep.java | 1 + 3 files changed, 49 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/274b8db0/core/src/main/java/org/apache/carbondata/core/carbon/metadata/datatype/DataType.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/datatype/DataType.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/datatype/DataType.java index ac7ef32..28bce74 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/datatype/DataType.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/datatype/DataType.java @@ -21,28 +21,34 @@ package org.apache.carbondata.core.carbon.metadata.datatype; public enum DataType { - STRING(0), - DATE(1), - TIMESTAMP(2), - BOOLEAN(1), - SHORT(2), - INT(3), - FLOAT(4), - LONG(5), - DOUBLE(6), - NULL(7), - DECIMAL(8), - ARRAY(9), - STRUCT(10), - MAP(11); + STRING(0, "STRING"), + DATE(1, "DATE"), + TIMESTAMP(2, "TIMESTAMP"), + BOOLEAN(1, "BOOLEAN"), + SHORT(2, "SHORT"), + INT(3, "INT"), + FLOAT(4, "FLOAT"), + LONG(5, "BIGINT"), + DOUBLE(6, "DOUBLE"), + NULL(7, "NULL"), + DECIMAL(8, "DECIMAL"), + ARRAY(9, "ARRAY"), + STRUCT(10, "STRUCT"), + MAP(11, "MAP"); private int presedenceOrder; + private String name ; - DataType(int value) { + DataType(int value ,String name) { this.presedenceOrder = value; + this.name = name; } public int getPresedenceOrder() { return presedenceOrder; } + + public String getName() { + return name; + } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/274b8db0/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java index 1af87ca..d00fc39 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java @@ -25,6 +25,8 @@ import java.math.RoundingMode; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.HashMap; +import java.util.Map; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; @@ -43,6 +45,23 @@ public final class DataTypeUtil { */ private static final LogService LOGGER = LogServiceFactory.getLogService(DataTypeUtil.class.getName()); + private static final Map<String, String> dataTypeDisplayNames; + + static { + dataTypeDisplayNames = new HashMap<String, String>(16); + dataTypeDisplayNames.put(DataType.DATE.toString(), DataType.DATE.getName()); + dataTypeDisplayNames.put(DataType.LONG.toString(), DataType.LONG.getName()); + dataTypeDisplayNames.put(DataType.INT.toString(), DataType.INT.getName()); + dataTypeDisplayNames.put(DataType.FLOAT.toString(), DataType.FLOAT.getName()); + dataTypeDisplayNames.put(DataType.BOOLEAN.toString(), DataType.BOOLEAN.getName()); + dataTypeDisplayNames.put(DataType.NULL.toString(), DataType.NULL.getName()); + dataTypeDisplayNames.put(DataType.DECIMAL.toString(), DataType.DECIMAL.getName()); + dataTypeDisplayNames.put(DataType.ARRAY.toString(), DataType.ARRAY.getName()); + dataTypeDisplayNames.put(DataType.STRUCT.toString(), DataType.STRUCT.getName()); + dataTypeDisplayNames.put(DataType.TIMESTAMP.toString(), DataType.TIMESTAMP.getName()); + dataTypeDisplayNames.put(DataType.SHORT.toString(), DataType.SHORT.getName()); + dataTypeDisplayNames.put(DataType.STRING.toString(), DataType.STRING.getName()); + } /** * This method will convert a given value to its specific type @@ -73,6 +92,14 @@ public final class DataTypeUtil { } /** + * @param dataType + * @return + */ + public static String getColumnDataTypeDisplayName(String dataType) { + return dataTypeDisplayNames.get(dataType); + } + + /** * This method will check the digits before dot with the max precision allowed * * @param bigDecimal http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/274b8db0/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java index 94b2df8..ca2c7ce 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java +++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java @@ -1237,6 +1237,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep { private void addEntryToBadRecords(Object[] r, int j, String columnName, String dataType) { + dataType= DataTypeUtil.getColumnDataTypeDisplayName(dataType); badRecordslogger.addBadRecordsToBuilder(r, "The value " + " \"" + r[j] + "\"" + " with column name " + columnName + " and column data type " + dataType + " is not a valid " + dataType + " type.",
