[CARBONDATA-1253] Sort_columns should not support float,double,decimal This closes #1122
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5f9741eb Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5f9741eb Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5f9741eb Branch: refs/heads/datamap Commit: 5f9741ebcb5b2b606f4d710785ce97c4d6b49229 Parents: 0d46976 Author: QiangCai <qiang...@qq.com> Authored: Fri Jun 30 19:51:19 2017 +0800 Committer: jackylk <jacky.li...@huawei.com> Committed: Mon Jul 3 20:53:20 2017 +0800 ---------------------------------------------------------------------- ...feVariableLengthDimensionDataChunkStore.java | 12 --- ...afeVariableLengthDimesionDataChunkStore.java | 12 --- .../apache/carbondata/core/util/ByteUtil.java | 81 -------------------- .../carbondata/core/util/DataTypeUtil.java | 12 --- .../spark/sql/catalyst/CarbonDDLSqlParser.scala | 12 ++- 5 files changed, 10 insertions(+), 119 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java index a65d745..2079811 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java @@ -25,10 +25,6 @@ import org.apache.carbondata.core.util.ByteUtil; import org.apache.spark.sql.types.BooleanType; import org.apache.spark.sql.types.DataType; -import org.apache.spark.sql.types.Decimal; -import org.apache.spark.sql.types.DecimalType; -import org.apache.spark.sql.types.DoubleType; -import org.apache.spark.sql.types.FloatType; import org.apache.spark.sql.types.IntegerType; import org.apache.spark.sql.types.LongType; import org.apache.spark.sql.types.ShortType; @@ -157,16 +153,8 @@ public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimens vector.putShort(vectorRow, ByteUtil.toShort(data, currentDataOffset, length)); } else if (dt instanceof IntegerType) { vector.putInt(vectorRow, ByteUtil.toInt(data, currentDataOffset, length)); - } else if (dt instanceof FloatType) { - vector.putFloat(vectorRow, ByteUtil.toFloat(data, currentDataOffset)); - } else if (dt instanceof DoubleType) { - vector.putDouble(vectorRow, ByteUtil.toDouble(data, currentDataOffset)); } else if (dt instanceof LongType) { vector.putLong(vectorRow, ByteUtil.toLong(data, currentDataOffset, length)); - } else if (dt instanceof DecimalType) { - vector.putDecimal(vectorRow, - Decimal.apply(ByteUtil.toBigDecimal(data, currentDataOffset, length)), - DecimalType.MAX_PRECISION()); } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java index 03ba34e..c6c98f9 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java @@ -26,10 +26,6 @@ import org.apache.carbondata.core.util.ByteUtil; import org.apache.spark.sql.types.BooleanType; import org.apache.spark.sql.types.DataType; -import org.apache.spark.sql.types.Decimal; -import org.apache.spark.sql.types.DecimalType; -import org.apache.spark.sql.types.DoubleType; -import org.apache.spark.sql.types.FloatType; import org.apache.spark.sql.types.IntegerType; import org.apache.spark.sql.types.LongType; import org.apache.spark.sql.types.ShortType; @@ -184,16 +180,8 @@ public class UnsafeVariableLengthDimesionDataChunkStore vector.putShort(vectorRow, ByteUtil.toShort(value, 0, value.length)); } else if (dt instanceof IntegerType) { vector.putInt(vectorRow, ByteUtil.toInt(value, 0, value.length)); - } else if (dt instanceof FloatType) { - vector.putFloat(vectorRow, ByteUtil.toFloat(value, 0)); - } else if (dt instanceof DoubleType) { - vector.putDouble(vectorRow, ByteUtil.toDouble(value, 0)); } else if (dt instanceof LongType) { vector.putLong(vectorRow, ByteUtil.toLong(value, 0, value.length)); - } else if (dt instanceof DecimalType) { - vector.putDecimal(vectorRow, - Decimal.apply(ByteUtil.toBigDecimal(value, 0, value.length)), - DecimalType.MAX_PRECISION()); } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java index 2761d71..b74f206 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java @@ -18,8 +18,6 @@ package org.apache.carbondata.core.util; import java.io.UnsupportedEncodingException; -import java.math.BigDecimal; -import java.math.BigInteger; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -39,8 +37,6 @@ public final class ByteUtil { public static final String UTF8_CSN = StandardCharsets.UTF_8.name(); - public static final byte[] ZERO_IN_BYTES = toBytes(0); - private ByteUtil() { } @@ -531,28 +527,6 @@ public final class ByteUtil { } /** - * float => byte[] - * - * @param f - * @return - */ - public static byte[] toBytes(final float f) { - // Encode it as int - return toBytes(Float.floatToRawIntBits(f)); - } - - /** - * byte[] => float - * - * @param bytes - * @param offset - * @return - */ - public static float toFloat(byte[] bytes, int offset) { - return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT)); - } - - /** * long => byte[] * * @param val @@ -593,61 +567,6 @@ public final class ByteUtil { return l ^ Long.MIN_VALUE; } - /** - * doube => byte[] - * - * @param d - * @return - */ - public static byte[] toBytes(final double d) { - // Encode it as a long - return toBytes(Double.doubleToRawLongBits(d)); - } - - /** - * byte[] => double - * - * @param bytes - * @param offset - * @return - */ - public static double toDouble(final byte[] bytes, final int offset) { - return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG)); - } - - /** - * BigDecimal => byte[] - * - * @param val - * @return - */ - public static byte[] toBytes(BigDecimal val) { - byte[] valueBytes = val.unscaledValue().toByteArray(); - byte[] result = new byte[valueBytes.length + SIZEOF_INT]; - int offset = putInt(result, 0, val.scale()); - putBytes(result, offset, valueBytes, 0, valueBytes.length); - return result; - } - - /** - * byte[] => BigDecimal - * - * @param bytes - * @param offset - * @param length - * @return - */ - public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) { - if (bytes == null || length < SIZEOF_INT + 1 || (offset + length > bytes.length)) { - return null; - } - - int scale = toInt(bytes, offset, bytes.length); - byte[] tcBytes = new byte[length - SIZEOF_INT]; - System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT); - return new BigDecimal(new BigInteger(tcBytes), scale); - } - private static IllegalArgumentException explainWrongLengthOrOffset(final byte[] bytes, final int offset, final int length, final int expectedLength) { String reason; http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java index 1b1884e..37ae5bb 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java @@ -334,14 +334,8 @@ public final class DataTypeUtil { return ByteUtil.toBytes(Short.parseShort(dimensionValue)); case INT: return ByteUtil.toBytes(Integer.parseInt(dimensionValue)); - case FLOAT: - return ByteUtil.toBytes(Float.parseFloat(dimensionValue)); case LONG: return ByteUtil.toBytes(Long.parseLong(dimensionValue)); - case DOUBLE: - return ByteUtil.toBytes(Double.parseDouble(dimensionValue)); - case DECIMAL: - return ByteUtil.toBytes(new BigDecimal(dimensionValue)); default: return ByteUtil.toBytes(dimensionValue); } @@ -372,14 +366,8 @@ public final class DataTypeUtil { return ByteUtil.toShort(dataInBytes, 0, dataInBytes.length); case INT: return ByteUtil.toInt(dataInBytes, 0, dataInBytes.length); - case FLOAT: - return ByteUtil.toFloat(dataInBytes, 0); case LONG: return ByteUtil.toLong(dataInBytes, 0, dataInBytes.length); - case DOUBLE: - return ByteUtil.toDouble(dataInBytes, 0); - case DECIMAL: - return ByteUtil.toBigDecimal(dataInBytes, 0, dataInBytes.length); default: return ByteUtil.toString(dataInBytes, 0, dataInBytes.length); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala index c565c31..8207a9d 100644 --- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala +++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala @@ -551,8 +551,8 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser { } else { val dataType = fields.find(x => x.column.equalsIgnoreCase(column)).get.dataType.get - if (isComplexDimDictionaryExclude(dataType)) { - val errormsg = "sort_columns is unsupported for complex datatype column: " + column + if (isDataTypeSupportedForSortColumn(dataType)) { + val errormsg = s"sort_columns is unsupported for ${dataType} datatype column: " + column throw new MalformedCarbonCommandException(errormsg) } } @@ -691,6 +691,14 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser { } /** + * detects whether datatype is part of sort_column + */ + private def isDataTypeSupportedForSortColumn(columnDataType: String): Boolean = { + val dataTypes = Array("array", "struct", "double", "float", "decimal") + dataTypes.exists(x => x.equalsIgnoreCase(columnDataType)) + } + + /** * detects whether datatype is part of dictionary_exclude */ def isDataTypeSupportedForDictionary_Exclude(columnDataType: String): Boolean = {