http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java index 7a80c88..87e7dea 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java @@ -113,9 +113,13 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { DataType msrType = getMeasureDataType(msrColumnEvaluatorInfo); for (int i = 0; i < ColumnPages.length; i++) { BitSet bitSet = - getFilteredIndexesForMeasure(measureRawColumnChunk.convertToColumnPage(i), - measureRawColumnChunk.getRowCount()[i], useBitsetPipeLine, - blockChunkHolder.getBitSetGroup(), i, msrType);; + getFilteredIndexesForMeasure( + measureRawColumnChunk.convertToColumnPage(i), + measureRawColumnChunk.getRowCount()[i], + useBitsetPipeLine, + blockChunkHolder.getBitSetGroup(), + i, + msrType); bitSetGroup.setBitSet(bitSet, i); } return bitSetGroup; @@ -132,15 +136,14 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { return DataTypes.INT; } else if (msrColumnEvaluatorInfo.getType() == DataTypes.LONG) { return DataTypes.LONG; - } else if (msrColumnEvaluatorInfo.getType() == DataTypes.DECIMAL) { - return DataTypes.DECIMAL; + } else if (DataTypes.isDecimal(msrColumnEvaluatorInfo.getType())) { + return DataTypes.createDefaultDecimalType(); } else { return DataTypes.DOUBLE; } } - protected BitSet getFilteredIndexes(ColumnPage columnPage, - int numerOfRows, DataType msrType) { + private BitSet getFilteredIndexes(ColumnPage columnPage, int numerOfRows, DataType msrType) { // Here the algorithm is // Get the measure values from the chunk. compare sequentially with the // the filter values. The one that matches sets it Bitset. @@ -311,6 +314,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { } return bitSet; } + private BitSet setFilterdIndexToBitSetWithColumnIndex( DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) { BitSet bitSet = new BitSet(numerOfRows); @@ -383,14 +387,14 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { } @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException { - if (isDimensionPresentInCurrentBlock == true) { + if (isDimensionPresentInCurrentBlock) { int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping() .get(dimColEvaluatorInfo.getColumnIndex()); if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) { blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex); } - } else if (isMeasurePresentInCurrentBlock == true) { + } else if (isMeasurePresentInCurrentBlock) { int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping() .get(msrColumnEvaluatorInfo.getColumnIndex()); if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java index 843da71..0022a72 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java @@ -156,8 +156,8 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { return DataTypes.INT; } else if (msrColumnEvaluatorInfo.getType() == DataTypes.LONG) { return DataTypes.LONG; - } else if (msrColumnEvaluatorInfo.getType() == DataTypes.DECIMAL) { - return DataTypes.DECIMAL; + } else if (DataTypes.isDecimal(msrColumnEvaluatorInfo.getType())) { + return DataTypes.createDefaultDecimalType(); } else { return DataTypes.DOUBLE; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java index dbf8d4c..10664c8 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java @@ -359,8 +359,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { msrType = DataTypes.INT; } else if (dataType == DataTypes.LONG) { msrType = DataTypes.LONG; - } else if (dataType == DataTypes.DECIMAL) { - msrType = DataTypes.DECIMAL; + } else if (DataTypes.isDecimal(dataType)) { + msrType = DataTypes.createDefaultDecimalType(); } else { msrType = DataTypes.DOUBLE; } @@ -386,7 +386,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { msrValue = (int) columnPage.getLong(index); } else if (msrType == DataTypes.LONG) { msrValue = columnPage.getLong(index); - } else if (msrType == DataTypes.DECIMAL) { + } else if (DataTypes.isDecimal(msrType)) { BigDecimal bigDecimalValue = columnPage.getDecimal(index); if (null != bigDecimalValue && msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema().getScale() http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/scan/partition/PartitionUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/partition/PartitionUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/partition/PartitionUtil.java index 85e6cef..676cf48 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/partition/PartitionUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/partition/PartitionUtil.java @@ -66,7 +66,7 @@ public class PartitionUtil { return dateFormatter.parse(data).getTime(); } else if (actualDataType == DataTypes.TIMESTAMP) { return timestampFormatter.parse(data).getTime(); - } else if (actualDataType == DataTypes.DECIMAL) { + } else if (DataTypes.isDecimal(actualDataType)) { return new BigDecimal(data); } else { return data; @@ -102,7 +102,7 @@ public class PartitionUtil { return Long.parseLong(data); } else if (actualDataType == DataTypes.DATE || actualDataType == DataTypes.TIMESTAMP) { return Long.parseLong(data) / 1000; - } else if (actualDataType == DataTypes.DECIMAL) { + } else if (DataTypes.isDecimal(actualDataType)) { return new BigDecimal(data); } else { return data; http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java index cf6c88b..5916fd3 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java @@ -372,7 +372,7 @@ public class MeasureDataVectorProcessor { return new IntegralMeasureVectorFiller(); } else if (dataType == DataTypes.LONG) { return new LongMeasureVectorFiller(); - } else if (dataType == DataTypes.DECIMAL) { + } else if (DataTypes.isDecimal(dataType)) { return new DecimalMeasureVectorFiller(); } else { return new DefaultMeasureVectorFiller(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java index cba9931..aca2fd6 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java +++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java @@ -37,6 +37,7 @@ import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex; import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.datatype.DecimalType; import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; import org.apache.carbondata.core.reader.CarbonIndexFileReader; @@ -262,8 +263,13 @@ public abstract class AbstractDataFileFooterConverter { wrapperColumnSchema.setColumnUniqueId(externalColumnSchema.getColumn_id()); wrapperColumnSchema.setColumnName(externalColumnSchema.getColumn_name()); wrapperColumnSchema.setColumnar(externalColumnSchema.isColumnar()); - wrapperColumnSchema - .setDataType(thriftDataTyopeToWrapperDataType(externalColumnSchema.data_type)); + DataType dataType = thriftDataTyopeToWrapperDataType(externalColumnSchema.data_type); + if (DataTypes.isDecimal(dataType)) { + DecimalType decimalType = (DecimalType) dataType; + decimalType.setPrecision(externalColumnSchema.getPrecision()); + decimalType.setScale(externalColumnSchema.getScale()); + } + wrapperColumnSchema.setDataType(dataType); wrapperColumnSchema.setDimensionColumn(externalColumnSchema.isDimension()); List<Encoding> encoders = new ArrayList<Encoding>(); for (org.apache.carbondata.format.Encoding encoder : externalColumnSchema.getEncoders()) { @@ -369,7 +375,7 @@ public abstract class AbstractDataFileFooterConverter { case DOUBLE: return DataTypes.DOUBLE; case DECIMAL: - return DataTypes.DECIMAL; + return DataTypes.createDefaultDecimalType(); case DATE: return DataTypes.DATE; case TIMESTAMP: http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java index d232d61..6579a6f 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java @@ -363,7 +363,7 @@ public class CarbonMetadataUtil { firstBuffer.flip(); secondBuffer.flip(); return (int) (firstBuffer.getLong() - secondBuffer.getLong()); - } else if (dataType == DataTypes.DECIMAL) { + } else if (DataTypes.isDecimal(dataType)) { return DataTypeUtil.byteToBigDecimal(first).compareTo(DataTypeUtil.byteToBigDecimal(second)); } else { throw new IllegalArgumentException("Invalid data type"); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 77789c5..17a4b5f 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -1848,6 +1848,7 @@ public final class CarbonUtil { return map; } + // TODO: move this to carbon store API as it is related to TableInfo creation public static TableInfo convertGsonToTableInfo(Map<String, String> properties) { Gson gson = new Gson(); String partsNo = properties.get("carbonSchemaPartsNo"); @@ -1864,9 +1865,35 @@ public final class CarbonUtil { builder.append(part); } TableInfo tableInfo = gson.fromJson(builder.toString(), TableInfo.class); + + // The tableInfo is deserialized from GSON string, need to update the scale and + // precision if there are any decimal field, because DecimalType is added in Carbon 1.3, + // If it is not updated, read compactibility will be break for table generated before Carbon 1.3 + updateDecimalType(tableInfo); return tableInfo; } + // Update decimal type inside `tableInfo` to set scale and precision, if there are any decimal + private static void updateDecimalType(TableInfo tableInfo) { + List<ColumnSchema> deserializedColumns = tableInfo.getFactTable().getListOfColumns(); + for (ColumnSchema column : deserializedColumns) { + DataType dataType = column.getDataType(); + if (DataTypes.isDecimal(dataType)) { + column.setDataType(DataTypes.createDecimalType(column.getPrecision(), column.getScale())); + } + } + if (tableInfo.getFactTable().getPartitionInfo() != null) { + List<ColumnSchema> partitionColumns = + tableInfo.getFactTable().getPartitionInfo().getColumnSchemaList(); + for (ColumnSchema column : partitionColumns) { + DataType dataType = column.getDataType(); + if (DataTypes.isDecimal(dataType)) { + column.setDataType(DataTypes.createDecimalType(column.getPrecision(), column.getScale())); + } + } + } + } + /** * Removes schema from properties * @param properties @@ -1964,7 +1991,7 @@ public final class CarbonUtil { b.putDouble((double) value); b.flip(); return b.array(); - } else if (dataType == DataTypes.DECIMAL) { + } else if (DataTypes.isDecimal(dataType)) { return DataTypeUtil.bigDecimalToByte((BigDecimal) value); } else if (dataType == DataTypes.BYTE_ARRAY) { return (byte[]) value; http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java index 01e34a7..0961a63 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java @@ -27,8 +27,6 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Date; -import java.util.HashMap; -import java.util.Map; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; @@ -51,7 +49,6 @@ public final class DataTypeUtil { */ private static final LogService LOGGER = LogServiceFactory.getLogService(DataTypeUtil.class.getName()); - private static final Map<String, String> dataTypeDisplayNames; private static final ThreadLocal<DateFormat> timeStampformatter = new ThreadLocal<DateFormat>() { @Override protected DateFormat initialValue() { @@ -71,23 +68,6 @@ public final class DataTypeUtil { } }; - static { - dataTypeDisplayNames = new HashMap<String, String>(16); - dataTypeDisplayNames.put(DataTypes.DATE.toString(), DataTypes.DATE.getName()); - dataTypeDisplayNames.put(DataTypes.LONG.toString(), DataTypes.LONG.getName()); - dataTypeDisplayNames.put(DataTypes.INT.toString(), DataTypes.INT.getName()); - dataTypeDisplayNames.put(DataTypes.FLOAT.toString(), DataTypes.FLOAT.getName()); - dataTypeDisplayNames.put(DataTypes.BOOLEAN.toString(), DataTypes.BOOLEAN.getName()); - dataTypeDisplayNames.put(DataTypes.NULL.toString(), DataTypes.NULL.getName()); - dataTypeDisplayNames.put(DataTypes.DECIMAL.toString(), DataTypes.DECIMAL.getName()); - dataTypeDisplayNames.put(DataTypes.ARRAY.toString(), DataTypes.ARRAY.getName()); - dataTypeDisplayNames.put(DataTypes.STRUCT.toString(), DataTypes.STRUCT.getName()); - dataTypeDisplayNames.put(DataTypes.TIMESTAMP.toString(), DataTypes.TIMESTAMP.getName()); - dataTypeDisplayNames.put(DataTypes.DATE.toString(), DataTypes.DATE.getName()); - dataTypeDisplayNames.put(DataTypes.SHORT.toString(), DataTypes.SHORT.getName()); - dataTypeDisplayNames.put(DataTypes.STRING.toString(), DataTypes.STRING.getName()); - } - /** * DataType converter for different computing engines */ @@ -105,7 +85,7 @@ public final class DataTypeUtil { CarbonMeasure carbonMeasure) { if (dataType == DataTypes.BOOLEAN) { return BooleanConvert.parseBoolean(msrValue); - } else if (dataType == DataTypes.DECIMAL) { + } else if (DataTypes.isDecimal(dataType)) { BigDecimal bigDecimal = new BigDecimal(msrValue).setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP); return normalizeDecimalValue(bigDecimal, carbonMeasure.getPrecision()); @@ -137,7 +117,7 @@ public final class DataTypeUtil { return (int) bb.getLong(); } else if (dataType == DataTypes.LONG) { return bb.getLong(); - } else if (dataType == DataTypes.DECIMAL) { + } else if (DataTypes.isDecimal(dataType)) { return byteToBigDecimal(data); } else { return bb.getDouble(); @@ -154,7 +134,7 @@ public final class DataTypeUtil { return (int) measurePage.getLong(index); } else if (dataType == DataTypes.LONG) { return measurePage.getLong(index); - } else if (dataType == DataTypes.DECIMAL) { + } else if (DataTypes.isDecimal(dataType)) { BigDecimal bigDecimalMsrValue = measurePage.getDecimal(index); if (null != bigDecimalMsrValue && carbonMeasure.getScale() > bigDecimalMsrValue.scale()) { bigDecimalMsrValue = @@ -163,22 +143,13 @@ public final class DataTypeUtil { if (null != bigDecimalMsrValue) { return normalizeDecimalValue(bigDecimalMsrValue, carbonMeasure.getPrecision()); } else { - return bigDecimalMsrValue; + return null; } } else { return measurePage.getDouble(index); } } - - /** - * @param dataType - * @return - */ - public static String getColumnDataTypeDisplayName(String dataType) { - return dataTypeDisplayNames.get(dataType); - } - /** * This method will check the digits before dot with the max precision allowed * @@ -239,55 +210,6 @@ public final class DataTypeUtil { } /** - * returns the SqlStatement.Type of corresponding string value - * - * @param dataTypeStr - * @return return the SqlStatement.Type - */ - public static DataType getDataType(String dataTypeStr) { - DataType dataType = null; - switch (dataTypeStr) { - case "BOOLEAN": - dataType = DataTypes.BOOLEAN; - break; - case "DATE": - dataType = DataTypes.DATE; - break; - case "TIMESTAMP": - dataType = DataTypes.TIMESTAMP; - break; - case "STRING": - dataType = DataTypes.STRING; - break; - case "INT": - dataType = DataTypes.INT; - break; - case "SMALLINT": - dataType = DataTypes.SHORT; - break; - case "LONG": - dataType = DataTypes.LONG; - break; - case "DOUBLE": - dataType = DataTypes.DOUBLE; - break; - case "DECIMAL": - dataType = DataTypes.DECIMAL; - break; - case "ARRAY": - dataType = DataTypes.ARRAY; - break; - case "STRUCT": - dataType = DataTypes.STRUCT; - break; - case "MAP": - default: - dataType = DataTypes.STRING; - } - return dataType; - } - - /** * Below method will be used to convert the data passed to its actual data * type * @@ -365,7 +287,7 @@ public final class DataTypeUtil { LOGGER.error("Cannot convert value to Time/Long type value" + e.getMessage()); return null; } - } else if (actualDataType == DataTypes.DECIMAL) { + } else if (DataTypes.isDecimal(actualDataType)) { if (data.isEmpty()) { return null; } @@ -518,7 +440,7 @@ public final class DataTypeUtil { LOGGER.error("Cannot convert value to Time/Long type value" + e.getMessage()); return null; } - } else if (dataType == DataTypes.DECIMAL) { + } else if (DataTypes.isDecimal(dataType)) { String data7 = new String(dataInBytes, CarbonCommonConstants.DEFAULT_CHARSET_CLASS); if (data7.isEmpty()) { return null; @@ -585,7 +507,7 @@ public final class DataTypeUtil { // validation will not be done for timestamp datatype as for timestamp direct dictionary // is generated. No dictionary file is created for timestamp datatype column DataType dataType = dimension.getDataType(); - if (dataType == DataTypes.DECIMAL) { + if (DataTypes.isDecimal(dataType)) { return parseStringToBigDecimal(value, dimension); } else if (dataType == DataTypes.SHORT || dataType == DataTypes.INT || dataType == DataTypes.LONG) { @@ -618,7 +540,7 @@ public final class DataTypeUtil { } try { DataType dataType = dimension.getDataType(); - if (dataType == DataTypes.DECIMAL) { + if (DataTypes.isDecimal(dataType)) { return parseStringToBigDecimal(value, dimension); } else if (dataType == DataTypes.INT) { Integer.parseInt(value); @@ -721,7 +643,7 @@ public final class DataTypeUtil { return null; } } - } else if (dataType == DataTypes.DECIMAL) { + } else if (DataTypes.isDecimal(dataType)) { String parsedValue = parseStringToBigDecimal(data, columnSchema); if (null == parsedValue) { return null; @@ -749,7 +671,7 @@ public final class DataTypeUtil { try { Object parsedValue = null; DataType dataType = columnSchema.getDataType(); - if (dataType == DataTypes.DECIMAL) { + if (DataTypes.isDecimal(dataType)) { return parseStringToBigDecimal(value, columnSchema); } else if (dataType == DataTypes.SHORT || dataType == DataTypes.INT || dataType == DataTypes.LONG) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java b/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java index d1beb16..c162396 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java +++ b/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java @@ -37,7 +37,7 @@ public final class Comparator { } else if (dataType == DataTypes.LONG || dataType == DataTypes.DATE || dataType == DataTypes.TIMESTAMP) { return new LongSerializableComparator(); - } else if (dataType == DataTypes.DECIMAL) { + } else if (DataTypes.isDecimal(dataType)) { return new BigDecimalSerializableComparator(); } else { return new ByteArraySerializableComparator(); @@ -61,7 +61,7 @@ public final class Comparator { return new LongSerializableComparator(); } else if (dataType == DataTypes.DOUBLE) { return new DoubleSerializableComparator(); - } else if (dataType == DataTypes.DECIMAL) { + } else if (DataTypes.isDecimal(dataType)) { return new BigDecimalSerializableComparator(); } else { throw new IllegalArgumentException("Unsupported data type"); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java index 4396d6a..5756c0d 100644 --- a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java +++ b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java @@ -82,7 +82,7 @@ public class CarbonDictionarySortModel implements Comparable<CarbonDictionarySor return -1; } return d1.compareTo(d2); - } else if (dataType == DataTypes.DECIMAL) { + } else if (DataTypes.isDecimal(dataType)) { java.math.BigDecimal val1 = null; java.math.BigDecimal val2 = null; try { http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfoTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfoTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfoTest.java index 33121e2..cb0892f 100644 --- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfoTest.java +++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfoTest.java @@ -25,6 +25,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.datatype.DecimalType; import org.apache.carbondata.core.util.CarbonUtil; import mockit.Mock; @@ -244,7 +245,7 @@ public class ColumnDictionaryInfoTest { } @Test public void testGetIncrementalSurrogateKeyFromDictionaryWithDecimalType() { - columnDictionaryInfo = new ColumnDictionaryInfo(DataTypes.DECIMAL); + columnDictionaryInfo = new ColumnDictionaryInfo(DataTypes.createDefaultDecimalType()); List<String> evaluateResultList = Arrays.asList("150011.550"); List<byte[]> byteValuesOfFilterMembers = convertListElementsIntoByteArray(evaluateResultList); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/RLECodecSuite.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/RLECodecSuite.java b/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/RLECodecSuite.java index be898c2..8be5f9d 100644 --- a/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/RLECodecSuite.java +++ b/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/RLECodecSuite.java @@ -46,9 +46,9 @@ public class RLECodecSuite { TestData(byte[] inputByteData, byte[] expectedEncodedByteData) throws IOException, MemoryException { this.inputByteData = inputByteData; inputBytePage = ColumnPage.newPage( - new TableSpec.ColumnSpec("test", DataTypes.BYTE, ColumnType.MEASURE), + TableSpec.ColumnSpec.newInstance("test", DataTypes.BYTE, ColumnType.MEASURE), DataTypes.BYTE, inputByteData.length); - inputBytePage.setStatsCollector(PrimitivePageStatsCollector.newInstance(DataTypes.BYTE, 0, 0)); + inputBytePage.setStatsCollector(PrimitivePageStatsCollector.newInstance(DataTypes.BYTE)); for (int i = 0; i < inputByteData.length; i++) { inputBytePage.putData(i, inputByteData[i]); } @@ -131,7 +131,7 @@ public class RLECodecSuite { private void testBytePageDecode(byte[] inputBytes, byte[] expectedDecodedBytes) throws IOException, MemoryException { RLECodec codec = new RLECodec(); RLEEncoderMeta meta = new RLEEncoderMeta( - new TableSpec.ColumnSpec("test", DataTypes.BYTE, ColumnType.MEASURE), + TableSpec.ColumnSpec.newInstance("test", DataTypes.BYTE, ColumnType.MEASURE), DataTypes.BYTE, expectedDecodedBytes.length, null); ColumnPageDecoder decoder = codec.createDecoder(meta); ColumnPage page = decoder.decode(inputBytes, 0, inputBytes.length); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java index 4a3ef32..42c0ad6 100644 --- a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java +++ b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java @@ -23,6 +23,8 @@ import java.util.Map; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.datatype.DecimalType; +import org.apache.carbondata.core.metadata.datatype.IntType; import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.schema.SchemaEvolution; import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry; @@ -874,7 +876,7 @@ public class ThriftWrapperSchemaConverterImplTest { } @Mock public DataType getDataType() { - return DataTypes.DECIMAL; + return DataTypes.createDefaultDecimalType(); } @Mock public String getColumnName() { @@ -1158,7 +1160,12 @@ public class ThriftWrapperSchemaConverterImplTest { @Test public void testFromWrapperToExternalColumnSchemaForDatatypeNullCase() { org.apache.carbondata.format.ColumnSchema thriftColumnSchema = - new org.apache.carbondata.format.ColumnSchema(null, "columnName", "1", true, encoders, + new org.apache.carbondata.format.ColumnSchema( + org.apache.carbondata.format.DataType.INT, + "columnName", + "1", + true, + encoders, true); thriftColumnSchema.setSchemaOrdinal(1); @@ -1172,7 +1179,7 @@ public class ThriftWrapperSchemaConverterImplTest { } @Mock public DataType getDataType() { - return null; + return DataTypes.INT; } @Mock public String getColumnName() { @@ -1548,9 +1555,7 @@ long time =1112745600000L; wrapperColumnSchema.setDimensionColumn(true); wrapperColumnSchema.setEncodingList(encodings); wrapperColumnSchema.setNumberOfChild(1); - wrapperColumnSchema.setPrecision(1); wrapperColumnSchema.setColumnGroup(1); - wrapperColumnSchema.setScale(1); wrapperColumnSchema.setDefaultValue(new byte[] { 1, 2 }); wrapperColumnSchema.setColumnProperties(columnPropertyMap); wrapperColumnSchema.setInvisible(true); @@ -1686,8 +1691,9 @@ long time =1112745600000L; @Test public void testFromExternalToWrapperColumnSchemaForDatatypeNullCase() { org.apache.carbondata.format.ColumnSchema thriftColumnSchema = - new org.apache.carbondata.format.ColumnSchema(null, "columnName", "1", true, encoders, - true); + new org.apache.carbondata.format.ColumnSchema( + org.apache.carbondata.format.DataType.STRING, + "columnName", "1", true, encoders, true); ColumnSchema actualResult = thriftWrapperSchemaConverter.fromExternalToWrapperColumnSchema(thriftColumnSchema); Boolean expectedResult = true; http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java index 048b194..172a53e 100644 --- a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java @@ -45,22 +45,27 @@ public class RestructureUtilTest { encodingList.add(Encoding.DICTIONARY); ColumnSchema columnSchema1 = new ColumnSchema(); columnSchema1.setColumnName("Id"); + columnSchema1.setDataType(DataTypes.STRING); columnSchema1.setColumnUniqueId(UUID.randomUUID().toString()); columnSchema1.setEncodingList(encodingList); ColumnSchema columnSchema2 = new ColumnSchema(); columnSchema2.setColumnName("Name"); + columnSchema2.setDataType(DataTypes.STRING); columnSchema2.setColumnUniqueId(UUID.randomUUID().toString()); columnSchema2.setEncodingList(encodingList); ColumnSchema columnSchema3 = new ColumnSchema(); columnSchema3.setColumnName("Age"); + columnSchema3.setDataType(DataTypes.INT); columnSchema3.setColumnUniqueId(UUID.randomUUID().toString()); columnSchema3.setEncodingList(encodingList); ColumnSchema columnSchema4 = new ColumnSchema(); columnSchema4.setColumnName("Salary"); + columnSchema4.setDataType(DataTypes.INT); columnSchema4.setColumnUniqueId(UUID.randomUUID().toString()); columnSchema4.setEncodingList(encodingList); ColumnSchema columnSchema5 = new ColumnSchema(); columnSchema5.setColumnName("Address"); + columnSchema5.setDataType(DataTypes.STRING); columnSchema5.setColumnUniqueId(UUID.randomUUID().toString()); columnSchema5.setEncodingList(encodingList); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/expression/ExpressionResultTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/ExpressionResultTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/ExpressionResultTest.java index cdc7fb9..1be4c2e 100644 --- a/core/src/test/java/org/apache/carbondata/core/scan/expression/ExpressionResultTest.java +++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/ExpressionResultTest.java @@ -26,6 +26,7 @@ import java.util.List; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.datatype.DecimalType; import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException; import org.junit.BeforeClass; @@ -255,7 +256,7 @@ public class ExpressionResultTest { } @Test public void testGetDecimalForDecimal() throws FilterIllegalMemberException { - ExpressionResult expressionResultForDecimal = new ExpressionResult(DataTypes.DECIMAL, 5); + ExpressionResult expressionResultForDecimal = new ExpressionResult(DataTypes.createDefaultDecimalType(), 5); BigDecimal actualValue = expressionResultForDecimal.getDecimal(); BigDecimal expectedValue = new BigDecimal(5); assertEquals(expectedValue, actualValue); @@ -403,8 +404,8 @@ public class ExpressionResultTest { } @Test public void testEqualsForDecimal() { - ExpressionResult expressionResult = new ExpressionResult(DataTypes.DECIMAL, 5); - ExpressionResult objToCompare = new ExpressionResult(DataTypes.DECIMAL, 6); + ExpressionResult expressionResult = new ExpressionResult(DataTypes.createDefaultDecimalType(), 5); + ExpressionResult objToCompare = new ExpressionResult(DataTypes.createDefaultDecimalType(), 6); boolean result = expressionResult.equals(objToCompare); assertFalse(result); } @@ -443,8 +444,8 @@ public class ExpressionResultTest { } @Test public void compareToForDecimal() { - ExpressionResult obj = new ExpressionResult(DataTypes.DECIMAL, 5); - ExpressionResult expressionResult = new ExpressionResult(DataTypes.DECIMAL, 6); + ExpressionResult obj = new ExpressionResult(DataTypes.createDefaultDecimalType(), 5); + ExpressionResult expressionResult = new ExpressionResult(DataTypes.createDefaultDecimalType(), 6); int actualValue = expressionResult.compareTo(obj); int expectedValue = 1; assertEquals(expectedValue, actualValue); @@ -452,7 +453,7 @@ public class ExpressionResultTest { @Test public void compareToForException() { ExpressionResult obj = new ExpressionResult(DataTypes.INT, 5); - ExpressionResult expressionResult = new ExpressionResult(DataTypes.DECIMAL, 6); + ExpressionResult expressionResult = new ExpressionResult(DataTypes.createDefaultDecimalType(), 6); int actualValue = expressionResult.compareTo(obj); int expectedValue = -1; assertEquals(expectedValue, actualValue); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java index 51bb0fe..cef1953 100644 --- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java +++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java @@ -25,6 +25,7 @@ import java.util.Date; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.datatype.DecimalType; import org.apache.carbondata.core.scan.expression.ColumnExpression; import org.apache.carbondata.core.scan.expression.ExpressionResult; import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException; @@ -294,7 +295,7 @@ public class EqualToExpressionUnitTest { @Test public void testEvaluateForEqualToExpressionWithDecimalDataType() throws FilterUnsupportedException, FilterIllegalMemberException { - ColumnExpression right = new ColumnExpression("contact", DataTypes.DECIMAL); + ColumnExpression right = new ColumnExpression("contact", DataTypes.createDefaultDecimalType()); right.setColIndex(0); equalToExpression = new EqualToExpression(right, right); RowImpl value = new RowImpl(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java index 0375a6d..de45721 100644 --- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java +++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java @@ -25,6 +25,7 @@ import java.util.Date; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.datatype.DecimalType; import org.apache.carbondata.core.scan.expression.ColumnExpression; import org.apache.carbondata.core.scan.expression.ExpressionResult; import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException; @@ -205,9 +206,9 @@ public class GreaterThanEqualToExpressionUnitTest { @Test public void testEvaluateForGreaterThanEqualToExpressionWithDecimalDataType() throws FilterUnsupportedException, FilterIllegalMemberException { - ColumnExpression right = new ColumnExpression("contact", DataTypes.DECIMAL); + ColumnExpression right = new ColumnExpression("contact", DataTypes.createDefaultDecimalType()); right.setColIndex(0); - ColumnExpression left = new ColumnExpression("contact", DataTypes.DECIMAL); + ColumnExpression left = new ColumnExpression("contact", DataTypes.createDefaultDecimalType()); left.setColIndex(1); greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right); RowImpl value = new RowImpl(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java index 1940069..ef32f69 100644 --- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java +++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java @@ -25,6 +25,7 @@ import java.util.Date; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.datatype.DecimalType; import org.apache.carbondata.core.scan.expression.ColumnExpression; import org.apache.carbondata.core.scan.expression.ExpressionResult; import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException; @@ -251,9 +252,9 @@ public class GreaterThanExpressionUnitTest { @Test public void testEvaluateForGreaterThanExpressionWithDecimalDataType() throws FilterUnsupportedException, FilterIllegalMemberException { - ColumnExpression right = new ColumnExpression("contact", DataTypes.DECIMAL); + ColumnExpression right = new ColumnExpression("contact", DataTypes.createDefaultDecimalType()); right.setColIndex(0); - ColumnExpression left = new ColumnExpression("contact", DataTypes.DECIMAL); + ColumnExpression left = new ColumnExpression("contact", DataTypes.createDefaultDecimalType()); left.setColIndex(1); greaterThanExpression = new GreaterThanExpression(left, right); RowImpl value = new RowImpl(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java index d31ec31..aa3c208 100644 --- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java +++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java @@ -25,6 +25,7 @@ import java.util.Date; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.datatype.DecimalType; import org.apache.carbondata.core.scan.expression.ColumnExpression; import org.apache.carbondata.core.scan.expression.ExpressionResult; import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException; @@ -203,9 +204,9 @@ public class InExpressionUnitTest { @Test public void testEvaluateForInExpressionWithDecimalDataType() throws FilterUnsupportedException, FilterIllegalMemberException { - ColumnExpression left = new ColumnExpression("left_contact", DataTypes.DECIMAL); + ColumnExpression left = new ColumnExpression("left_contact", DataTypes.createDefaultDecimalType()); left.setColIndex(0); - ColumnExpression right = new ColumnExpression("right_contact", DataTypes.DECIMAL); + ColumnExpression right = new ColumnExpression("right_contact", DataTypes.createDefaultDecimalType()); right.setColIndex(1); inExpression = new InExpression(left, right); RowImpl value = new RowImpl(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java index a2a5d2a..9b0aa66 100644 --- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java +++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java @@ -25,6 +25,7 @@ import java.util.Date; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.datatype.DecimalType; import org.apache.carbondata.core.scan.expression.ColumnExpression; import org.apache.carbondata.core.scan.expression.ExpressionResult; import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException; @@ -254,9 +255,9 @@ public class LessThanEqualToExpressionUnitTest { @Test public void testEvaluateForLessThanEqualToExpressionWithDecimalDataType() throws FilterUnsupportedException, FilterIllegalMemberException { - ColumnExpression right = new ColumnExpression("right_contact", DataTypes.DECIMAL); + ColumnExpression right = new ColumnExpression("right_contact", DataTypes.createDefaultDecimalType()); right.setColIndex(0); - ColumnExpression left = new ColumnExpression("left_contact", DataTypes.DECIMAL); + ColumnExpression left = new ColumnExpression("left_contact", DataTypes.createDefaultDecimalType()); left.setColIndex(1); lessThanEqualToExpression = new LessThanEqualToExpression(left, right); RowImpl value = new RowImpl(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java index d385dc1..4bd02af 100644 --- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java +++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java @@ -25,6 +25,7 @@ import java.util.Date; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.datatype.DecimalType; import org.apache.carbondata.core.scan.expression.ColumnExpression; import org.apache.carbondata.core.scan.expression.ExpressionResult; import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException; @@ -251,9 +252,9 @@ public class LessThanExpressionUnitTest { @Test public void testEvaluateForLessThanExpressionWithDecimalDataType() throws FilterUnsupportedException, FilterIllegalMemberException { - ColumnExpression right = new ColumnExpression("contact", DataTypes.DECIMAL); + ColumnExpression right = new ColumnExpression("contact", DataTypes.createDefaultDecimalType()); right.setColIndex(0); - ColumnExpression left = new ColumnExpression("contact", DataTypes.DECIMAL); + ColumnExpression left = new ColumnExpression("contact", DataTypes.createDefaultDecimalType()); left.setColIndex(1); lessThanExpression = new LessThanExpression(left, right); RowImpl value = new RowImpl(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java index e7fa544..5ee6a15 100644 --- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java +++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java @@ -25,6 +25,7 @@ import java.util.Date; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.datatype.DecimalType; import org.apache.carbondata.core.scan.expression.ColumnExpression; import org.apache.carbondata.core.scan.expression.ExpressionResult; import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException; @@ -261,9 +262,9 @@ public class NotEqualsExpressionUnitTest { @Test public void testEvaluateWithForNotEqualsExpressionDecimalDataType() throws FilterUnsupportedException, FilterIllegalMemberException { - ColumnExpression right = new ColumnExpression("contact", DataTypes.DECIMAL); + ColumnExpression right = new ColumnExpression("contact", DataTypes.createDefaultDecimalType()); right.setColIndex(1); - ColumnExpression left = new ColumnExpression("contact", DataTypes.DECIMAL); + ColumnExpression left = new ColumnExpression("contact", DataTypes.createDefaultDecimalType()); left.setColIndex(0); notEqualsExpression = new NotEqualsExpression(left, right); RowImpl value = new RowImpl(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java index 5758625..c797ce9 100644 --- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java +++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java @@ -25,6 +25,7 @@ import java.util.Date; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.datatype.DecimalType; import org.apache.carbondata.core.scan.expression.ColumnExpression; import org.apache.carbondata.core.scan.expression.ExpressionResult; import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException; @@ -202,9 +203,9 @@ public class NotInExpressionUnitTest { @Test public void testEvaluateForNotInExpressionWithDecimalDataType() throws FilterUnsupportedException, FilterIllegalMemberException { - ColumnExpression left = new ColumnExpression("left_contact", DataTypes.DECIMAL); + ColumnExpression left = new ColumnExpression("left_contact", DataTypes.createDefaultDecimalType()); left.setColIndex(0); - ColumnExpression right = new ColumnExpression("right_contact", DataTypes.DECIMAL); + ColumnExpression right = new ColumnExpression("right_contact", DataTypes.createDefaultDecimalType()); right.setColIndex(1); notInExpression = new NotInExpression(left, right); RowImpl value = new RowImpl(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java index a6f483b..f1f05f0 100644 --- a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java @@ -27,6 +27,7 @@ import org.apache.carbondata.core.datastore.IndexKey; import org.apache.carbondata.core.datastore.block.SegmentProperties; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.datatype.DecimalType; import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; @@ -149,7 +150,7 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest { String dictionaryVal = "1111111"; String memberVal = "1111"; int actualResult = - FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataTypes.DECIMAL); + FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataTypes.createDefaultDecimalType()); int expectedResult = 1; assertEquals(expectedResult, actualResult); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java index ca78e66..4cf3d11 100644 --- a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java @@ -189,7 +189,7 @@ public class CarbonMetadataUtilTest { final EncodedColumnPage measure = new EncodedColumnPage(new DataChunk2(), new byte[]{0,1}, PrimitivePageStatsCollector.newInstance( - org.apache.carbondata.core.metadata.datatype.DataTypes.BYTE, 0, 0)); + org.apache.carbondata.core.metadata.datatype.DataTypes.BYTE)); new MockUp<EncodedTablePage>() { @SuppressWarnings("unused") @Mock public EncodedColumnPage getMeasure(int measureIndex) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java index 658dda9..56c59e1 100644 --- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java @@ -40,6 +40,7 @@ import org.apache.carbondata.core.metadata.blocklet.DataFileFooter; import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.datatype.DecimalType; import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; @@ -508,7 +509,7 @@ public class CarbonUtilTest { } @Test public void testForHasDataTypes() { - DataType[] dataTypes = { DataTypes.DECIMAL, DataTypes.BOOLEAN, DataTypes.INT }; + DataType[] dataTypes = {DataTypes.createDefaultDecimalType(), DataTypes.BOOLEAN, DataTypes.INT }; assertTrue(CarbonUtil.hasDataType(DataTypes.BOOLEAN, dataTypes)); assertTrue(!CarbonUtil.hasDataType(DataTypes.DATE, dataTypes)); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java index b6ea765..c944d18 100644 --- a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java @@ -21,6 +21,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.datatype.DecimalType; import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; @@ -30,9 +31,7 @@ import static junit.framework.TestCase.assertEquals; import static junit.framework.TestCase.assertTrue; import static org.apache.carbondata.core.util.DataTypeUtil.bigDecimalToByte; import static org.apache.carbondata.core.util.DataTypeUtil.byteToBigDecimal; -import static org.apache.carbondata.core.util.DataTypeUtil.getColumnDataTypeDisplayName; import static org.apache.carbondata.core.util.DataTypeUtil.getDataBasedOnDataType; -import static org.apache.carbondata.core.util.DataTypeUtil.getDataType; import static org.apache.carbondata.core.util.DataTypeUtil.getMeasureValueBasedOnDataType; import static org.apache.carbondata.core.util.DataTypeUtil.normalizeIntAndLongValues; @@ -40,7 +39,7 @@ public class DataTypeUtilTest { @Test public void testGetColumnDataTypeDisplayName() { String expected = DataTypes.INT.getName(); - String result = getColumnDataTypeDisplayName("INT"); + String result = "INT"; assertEquals(expected, result); } @@ -60,22 +59,6 @@ public class DataTypeUtilTest { assertTrue(result == result); } - @Test public void testGetDataType() { - assertEquals(DataTypes.TIMESTAMP, getDataType("TIMESTAMP")); - assertEquals(DataTypes.DATE, getDataType("DATE")); - assertEquals(DataTypes.STRING, getDataType("STRING")); - assertEquals(DataTypes.INT, getDataType("INT")); - assertEquals(DataTypes.SHORT, getDataType("SMALLINT")); - assertEquals(DataTypes.LONG, getDataType("LONG")); - assertEquals(DataTypes.DOUBLE, getDataType("DOUBLE")); - assertEquals(DataTypes.DECIMAL, getDataType("DECIMAL")); - assertEquals(DataTypes.ARRAY, getDataType("ARRAY")); - assertEquals(DataTypes.STRUCT, getDataType("STRUCT")); - assertEquals(DataTypes.STRING, getDataType("MAP")); - assertEquals(DataTypes.STRING, getDataType("default")); - - } - @Test public void testGetDataBasedOnDataType() throws NumberFormatException { String data = " "; if (data.isEmpty()) { @@ -87,7 +70,7 @@ public class DataTypeUtilTest { assertEquals(getDataBasedOnDataType("0", DataTypes.LONG), 0L); java.math.BigDecimal javaDecVal = new java.math.BigDecimal(1); scala.math.BigDecimal scalaDecVal = new scala.math.BigDecimal(javaDecVal); - assertEquals(getDataBasedOnDataType("1", DataTypes.DECIMAL), + assertEquals(getDataBasedOnDataType("1", DataTypes.createDefaultDecimalType()), DataTypeUtil.getDataTypeConverter().convertToDecimal(scalaDecVal)); assertEquals(getDataBasedOnDataType("default", DataTypes.NULL), DataTypeUtil.getDataTypeConverter().convertFromStringToUTF8String("default")); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModelTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModelTest.java b/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModelTest.java index a42b214..f89128d 100644 --- a/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModelTest.java +++ b/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModelTest.java @@ -18,6 +18,7 @@ package org.apache.carbondata.core.writer.sortindex; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.datatype.DecimalType; import org.junit.Test; @@ -69,27 +70,27 @@ public class CarbonDictionarySortModelTest { } @Test public void testCompareToForDataTypeDecimalCase() { - carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "72.34"); + carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "72.34"); CarbonDictionarySortModel testCarbonDictionarySortModel = - new CarbonDictionarySortModel(2, DataTypes.DECIMAL, "56.78"); + new CarbonDictionarySortModel(2, DataTypes.createDefaultDecimalType(), "56.78"); int expectedResult = 1; int actualResult = carbonDictionarySortModel.compareTo(testCarbonDictionarySortModel); assertEquals(actualResult, expectedResult); } @Test public void testCompareToForDataTypeDecimalExceptionCase() { - carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "decimal"); + carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "decimal"); CarbonDictionarySortModel testCarbonDictionarySortModel = - new CarbonDictionarySortModel(2, DataTypes.DECIMAL, "@NU#LL$!"); + new CarbonDictionarySortModel(2, DataTypes.createDefaultDecimalType(), "@NU#LL$!"); int expectedResult = -1; int actualResult = carbonDictionarySortModel.compareTo(testCarbonDictionarySortModel); assertEquals(actualResult, expectedResult); } @Test public void testCompareToForDataTypeDecimalExceptionCaseForOtherObject() { - carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "15.24"); + carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "15.24"); CarbonDictionarySortModel testCarbonDictionarySortModel = - new CarbonDictionarySortModel(2, DataTypes.DECIMAL, "@NU#LL$!"); + new CarbonDictionarySortModel(2, DataTypes.createDefaultDecimalType(), "@NU#LL$!"); int expectedResult = -1; int actualResult = carbonDictionarySortModel.compareTo(testCarbonDictionarySortModel); assertEquals(actualResult, expectedResult); @@ -126,28 +127,28 @@ public class CarbonDictionarySortModelTest { } @Test public void testHashCode() { - carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "15.24"); + carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "15.24"); int actualResult = carbonDictionarySortModel.hashCode(); int expectedResult = 46877260; assertTrue(actualResult == expectedResult); } @Test public void testHashCodeNullCaseForMemberValue() { - carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, null); + carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), null); int actualResult = carbonDictionarySortModel.hashCode(); int expectedResult = 0; assertEquals(actualResult, expectedResult); } @Test public void testEquals() { - carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "15.24"); + carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "15.24"); CarbonDictionarySortModel testCarbonDictionarySortModel = carbonDictionarySortModel; boolean result = carbonDictionarySortModel.equals(testCarbonDictionarySortModel); assertTrue(result); } @Test public void testEqualsMemberValueNullCase() { - carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, null); + carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), null); CarbonDictionarySortModel testCarbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.BOOLEAN, "false"); boolean result = carbonDictionarySortModel.equals(testCarbonDictionarySortModel); @@ -155,7 +156,7 @@ public class CarbonDictionarySortModelTest { } @Test public void testEqualsWhenMemberValueDiffers() { - carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "12.45"); + carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "12.45"); CarbonDictionarySortModel testCarbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.BOOLEAN, "false"); boolean result = carbonDictionarySortModel.equals(testCarbonDictionarySortModel); @@ -163,15 +164,15 @@ public class CarbonDictionarySortModelTest { } @Test public void testEqualsWhenMemberValueIsSame() { - carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "12.45"); + carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "12.45"); CarbonDictionarySortModel testCarbonDictionarySortModel = - new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "12.45"); + new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "12.45"); boolean result = carbonDictionarySortModel.equals(testCarbonDictionarySortModel); assertTrue(result); } @Test public void testEqualsForDifferentObjects() { - carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "12.45"); + carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "12.45"); Object testCarbonDictionarySortModel = new Object(); boolean result = carbonDictionarySortModel.equals(testCarbonDictionarySortModel); assertFalse(result); @@ -187,9 +188,9 @@ public class CarbonDictionarySortModelTest { } @Test public void testCompareToForDataTypeDecimalExceptionCaseForDifferentObject() { - carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "12.il"); + carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "12.il"); CarbonDictionarySortModel testCarbonDictionarySortModel = - new CarbonDictionarySortModel(2, DataTypes.DECIMAL, "12.89"); + new CarbonDictionarySortModel(2, DataTypes.createDefaultDecimalType(), "12.89"); int expectedResult = 1; int actualResult = carbonDictionarySortModel.compareTo(testCarbonDictionarySortModel); assertEquals(actualResult, expectedResult); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java ---------------------------------------------------------------------- diff --git a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java index 36ae65c..05ad93a 100644 --- a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java +++ b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java @@ -239,7 +239,7 @@ public class CarbonDictionaryDecodeReadSupport<T> implements CarbonReadSupport<T return new TimestampWritable(new Timestamp((long) obj / 1000)); } else if (dataType == DataTypes.STRING) { return new Text(obj.toString()); - } else if (dataType == DataTypes.DECIMAL) { + } else if (DataTypes.isDecimal(dataType)) { return new HiveDecimalWritable(HiveDecimal.create(new java.math.BigDecimal(obj.toString()))); } else { throw new IOException("unsupported data type:" + dataType); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonTypeUtil.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonTypeUtil.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonTypeUtil.java index e2c9c68..ddc8d9e 100644 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonTypeUtil.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonTypeUtil.java @@ -20,7 +20,7 @@ public class CarbonTypeUtil { return DataTypes.DoubleType; } else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.BOOLEAN) { return DataTypes.BooleanType; - } else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.DECIMAL) { + } else if (org.apache.carbondata.core.metadata.datatype.DataTypes.isDecimal(carbonDataType)) { return DataTypes.createDecimalType(); } else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.TIMESTAMP) { return DataTypes.TimestampType; http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java index 2db2d23..f129474 100644 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java @@ -205,7 +205,7 @@ class CarbonVectorizedRecordReader extends AbstractRecordReader<Object> { fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(), CarbonTypeUtil.convertCarbonToSparkDataType(msr.getMeasure().getDataType()), true, null); - } else if (dataType == DataTypes.DECIMAL) { + } else if (DataTypes.isDecimal(dataType)) { fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(), new DecimalType(msr.getMeasure().getPrecision(), msr.getMeasure().getScale()), true, null); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java index 4a9b7ed..fb9a0ba 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java @@ -76,8 +76,10 @@ public class CarbondataColumnHandle implements ColumnHandle { return columnUniqueId; } - @JsonCreator public CarbondataColumnHandle(@JsonProperty("connectorId") String connectorId, - @JsonProperty("columnName") String columnName, @JsonProperty("columnType") Type columnType, + @JsonCreator public CarbondataColumnHandle( + @JsonProperty("connectorId") String connectorId, + @JsonProperty("columnName") String columnName, + @JsonProperty("columnType") Type columnType, @JsonProperty("ordinalPosition") int ordinalPosition, @JsonProperty("keyOrdinal") int keyOrdinal, @JsonProperty("columnGroupOrdinal") int columnGroupOrdinal, http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java index 99109d1..f106a08 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java @@ -232,7 +232,7 @@ public class CarbondataMetadata implements ConnectorMetadata { return BigintType.BIGINT; } else if (colType == DataTypes.FLOAT || colType == DataTypes.DOUBLE) { return DoubleType.DOUBLE; - } else if (colType == DataTypes.DECIMAL) { + } else if (DataTypes.isDecimal(colType)) { if (columnSchema.getPrecision() > 0) { return DecimalType.createDecimalType(columnSchema.getPrecision(), columnSchema.getScale()); } else { http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java index c9fb177..0982254 100644 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java @@ -79,7 +79,10 @@ public class PrestoFilterUtil { else if (colType == DateType.DATE) return DataTypes.DATE; else if (colType == TimestampType.TIMESTAMP) return DataTypes.TIMESTAMP; else if (colType.equals(DecimalType.createDecimalType(carbondataColumnHandle.getPrecision(), - carbondataColumnHandle.getScale()))) return DataTypes.DECIMAL; + carbondataColumnHandle.getScale()))) + return org.apache.carbondata.core.metadata.datatype.DataTypes.createDecimalType( + carbondataColumnHandle.getPrecision(), + carbondataColumnHandle.getScale()); else return DataTypes.STRING; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala ---------------------------------------------------------------------- diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala index b8031a2..acee71b 100644 --- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala +++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala @@ -40,7 +40,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.datastore.impl.FileFactory import org.apache.carbondata.core.fileoperations.{AtomicFileOperations, AtomicFileOperationsImpl, FileWriteOperation} import org.apache.carbondata.core.metadata.converter.{SchemaConverter, ThriftWrapperSchemaConverterImpl} -import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes} +import org.apache.carbondata.core.metadata.datatype.DataTypes import org.apache.carbondata.core.metadata.encoder.Encoding import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension, CarbonMeasure, ColumnSchema} import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo, TableSchema} @@ -236,7 +236,7 @@ object CarbonDataStoreCreator { val bonus: ColumnSchema = new ColumnSchema() bonus.setColumnName("bonus") bonus.setColumnar(true) - bonus.setDataType(DataTypes.DECIMAL) + bonus.setDataType(DataTypes.createDecimalType(10, 4)) bonus.setPrecision(10) bonus.setScale(4) bonus.setEncodingList(encodings) http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala index cf37a18..2043ecf 100644 --- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala +++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala @@ -25,12 +25,11 @@ import org.apache.spark.sql.execution.command.DataTypeInfo import org.apache.spark.sql.types._ import org.apache.carbondata.core.constants.CarbonCommonConstants -import org.apache.carbondata.core.metadata.datatype.{DataType => CarbonDataType, DataTypes => CarbonDataTypes} +import org.apache.carbondata.core.metadata.datatype.{DataType => CarbonDataType, DataTypes => CarbonDataTypes, DecimalType => CarbonDecimalType} import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn object CarbonScalaUtil { - def convertSparkToCarbonDataType( - dataType: org.apache.spark.sql.types.DataType): CarbonDataType = { + def convertSparkToCarbonDataType(dataType: DataType): CarbonDataType = { dataType match { case StringType => CarbonDataTypes.STRING case ShortType => CarbonDataTypes.SHORT @@ -44,7 +43,9 @@ object CarbonScalaUtil { case ArrayType(_, _) => CarbonDataTypes.ARRAY case StructType(_) => CarbonDataTypes.STRUCT case NullType => CarbonDataTypes.NULL - case _ => CarbonDataTypes.DECIMAL + case decimal: DecimalType => + CarbonDataTypes.createDecimalType(decimal.precision, decimal.scale) + case _ => throw new UnsupportedOperationException("getting " + dataType + " from spark") } } @@ -66,27 +67,19 @@ object CarbonScalaUtil { } def convertCarbonToSparkDataType(dataType: CarbonDataType): types.DataType = { - dataType match { - case CarbonDataTypes.STRING => StringType - case CarbonDataTypes.SHORT => ShortType - case CarbonDataTypes.INT => IntegerType - case CarbonDataTypes.LONG => LongType - case CarbonDataTypes.DOUBLE => DoubleType - case CarbonDataTypes.BOOLEAN => BooleanType - case CarbonDataTypes.DECIMAL => DecimalType.SYSTEM_DEFAULT - case CarbonDataTypes.TIMESTAMP => TimestampType - case CarbonDataTypes.DATE => DateType - } - } - - def updateDataType( - currentDataType: org.apache.spark.sql.types.DataType): org.apache.spark.sql.types.DataType = { - currentDataType match { - case decimal: DecimalType => - val scale = currentDataType.asInstanceOf[DecimalType].scale - DecimalType(DecimalType.MAX_PRECISION, scale) - case _ => - currentDataType + if (CarbonDataTypes.isDecimal(dataType)) { + DecimalType.SYSTEM_DEFAULT + } else { + dataType match { + case CarbonDataTypes.STRING => StringType + case CarbonDataTypes.SHORT => ShortType + case CarbonDataTypes.INT => IntegerType + case CarbonDataTypes.LONG => LongType + case CarbonDataTypes.DOUBLE => DoubleType + case CarbonDataTypes.BOOLEAN => BooleanType + case CarbonDataTypes.TIMESTAMP => TimestampType + case CarbonDataTypes.DATE => DateType + } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala index 89a3ac3..0460e30 100644 --- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala +++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala @@ -17,7 +17,7 @@ package org.apache.carbondata.spark.util -import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes} +import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes, DecimalType} import org.apache.carbondata.format.{DataType => ThriftDataType} object DataTypeConverterUtil { @@ -37,8 +37,8 @@ object DataTypeConverterUtil { case "numeric" => DataTypes.DOUBLE case "double" => DataTypes.DOUBLE case "float" => DataTypes.DOUBLE - case "decimal" => DataTypes.DECIMAL - case FIXED_DECIMAL(_, _) => DataTypes.DECIMAL + case "decimal" => DataTypes.createDefaultDecimalType + case FIXED_DECIMAL(_, _) => DataTypes.createDefaultDecimalType case "timestamp" => DataTypes.TIMESTAMP case "date" => DataTypes.DATE case "array" => DataTypes.ARRAY @@ -60,8 +60,8 @@ object DataTypeConverterUtil { case "numerictype" => DataTypes.DOUBLE case "doubletype" => DataTypes.DOUBLE case "floattype" => DataTypes.DOUBLE - case "decimaltype" => DataTypes.DECIMAL - case FIXED_DECIMALTYPE(_, _) => DataTypes.DECIMAL + case "decimaltype" => DataTypes.createDefaultDecimalType + case FIXED_DECIMALTYPE(_, _) => DataTypes.createDefaultDecimalType case "timestamptype" => DataTypes.TIMESTAMP case "datetype" => DataTypes.DATE case others => @@ -80,19 +80,22 @@ object DataTypeConverterUtil { } def convertToString(dataType: DataType): String = { - dataType match { - case DataTypes.BOOLEAN => "boolean" + if (DataTypes.isDecimal(dataType)) { + "decimal" + } else { + dataType match { + case DataTypes.BOOLEAN => "boolean" case DataTypes.STRING => "string" - case DataTypes.SHORT => "smallint" - case DataTypes.INT => "int" - case DataTypes.LONG => "bigint" - case DataTypes.DOUBLE => "double" - case DataTypes.FLOAT => "double" - case DataTypes.DECIMAL => "decimal" - case DataTypes.TIMESTAMP => "timestamp" - case DataTypes.DATE => "date" - case DataTypes.ARRAY => "array" - case DataTypes.STRUCT => "struct" + case DataTypes.SHORT => "smallint" + case DataTypes.INT => "int" + case DataTypes.LONG => "bigint" + case DataTypes.DOUBLE => "double" + case DataTypes.FLOAT => "double" + case DataTypes.TIMESTAMP => "timestamp" + case DataTypes.DATE => "date" + case DataTypes.ARRAY => "array" + case DataTypes.STRUCT => "struct" + } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala index 1163b3f..edb471e 100644 --- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala +++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala @@ -622,7 +622,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser { dimFields += field } else if (dictIncludeCols.exists(x => x.equalsIgnoreCase(field.column))) { dimFields += field - } else if (DataTypeUtil.getDataType(field.dataType.get.toUpperCase) == DataTypes.TIMESTAMP && + } else if (field.dataType.get.toUpperCase.equals("TIMESTAMP") && !dictIncludeCols.exists(x => x.equalsIgnoreCase(field.column))) { noDictionaryDims :+= field.column dimFields += field
