Repository: incubator-carbondata Updated Branches: refs/heads/master 7213ac057 -> c95e565e4
Implement BigInt value compression fix for review comment Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/20af74ba Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/20af74ba Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/20af74ba Branch: refs/heads/master Commit: 20af74ba719141002c68bc5e82bc05131e47a2d9 Parents: 7213ac0 Author: Ashok Kumar <ashok.bl...@gmail.com> Authored: Sun Jul 24 00:43:58 2016 +0530 Committer: jackylk <jacky.li...@huawei.com> Committed: Thu Dec 1 19:38:55 2016 +0800 ---------------------------------------------------------------------- .../core/compression/BigIntCompressor.java | 110 +++++++++ .../core/compression/DoubleCompressor.java | 242 +++++++++++++++++++ .../core/compression/ValueCompressor.java | 103 ++++++++ .../compression/type/UnCompressDefaultLong.java | 11 +- .../compression/type/UnCompressMaxMinByte.java | 39 ++- .../type/UnCompressMaxMinByteForLong.java | 8 +- .../type/UnCompressMaxMinDefault.java | 13 +- .../type/UnCompressMaxMinDefaultLong.java | 9 +- .../compression/type/UnCompressMaxMinFloat.java | 10 +- .../compression/type/UnCompressMaxMinInt.java | 36 ++- .../compression/type/UnCompressMaxMinLong.java | 37 ++- .../compression/type/UnCompressMaxMinShort.java | 36 ++- .../compression/type/UnCompressNoneByte.java | 39 ++- .../compression/type/UnCompressNoneDefault.java | 10 +- .../compression/type/UnCompressNoneFloat.java | 11 +- .../compression/type/UnCompressNoneInt.java | 29 ++- .../compression/type/UnCompressNoneLong.java | 39 ++- .../compression/type/UnCompressNoneShort.java | 38 ++- ...ractHeavyCompressedDoubleArrayDataStore.java | 17 +- .../core/util/ValueCompressionUtil.java | 80 ++++-- .../core/util/ValueCompressionUtilTest.java | 8 +- .../store/CarbonFactDataHandlerColumnar.java | 2 - 22 files changed, 844 insertions(+), 83 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java new file mode 100644 index 0000000..7b9e52f --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.compression; + +import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder; +import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; + +/** + * It compresses big int data + */ +public class BigIntCompressor extends ValueCompressor { + + @Override protected Object compressNonDecimalMaxMin(DataType changedDataType, + CarbonWriteDataHolder dataHolder, int decimal, Object max) { + // in case if bigint, decimal will be 0 + return compressMaxMin(changedDataType, dataHolder, max); + } + + @Override + protected Object compressNonDecimal(DataType changedDataType, CarbonWriteDataHolder dataHolder, + int decimal) { + // in case if bigint, decimal will be 0 + return compressNone(changedDataType, dataHolder); + } + + @Override + protected Object compressMaxMin(DataType changedDataType, CarbonWriteDataHolder dataHolder, + Object max) { + long maxValue = (long) max; + long[] value = dataHolder.getWritableLongValues(); + int i = 0; + switch (changedDataType) { + case DATA_BYTE: + byte[] result = new byte[value.length]; + for (int j = 0; j < value.length; j++) { + result[i] = (byte) (maxValue - value[j]); + i++; + } + return result; + case DATA_SHORT: + short[] shortResult = new short[value.length]; + for (int j = 0; j < value.length; j++) { + shortResult[i] = (short) (maxValue - value[j]); + i++; + } + return shortResult; + case DATA_INT: + int[] intResult = new int[value.length]; + for (int j = 0; j < value.length; j++) { + intResult[i] = (int) (maxValue - value[j]); + i++; + } + return intResult; + default: + long[] defaultResult = new long[value.length]; + for (int j = 0; j < value.length; j++) { + defaultResult[i] = (long) (maxValue - value[j]); + i++; + } + return defaultResult; + } + } + + @Override + protected Object compressNone(DataType changedDataType, CarbonWriteDataHolder dataHolder) { + long[] value = dataHolder.getWritableLongValues(); + int i = 0; + switch (changedDataType) { + case DATA_BYTE: + byte[] result = new byte[value.length]; + for (int j = 0; j < value.length; j++) { + result[i] = (byte) value[j]; + i++; + } + return result; + case DATA_SHORT: + short[] shortResult = new short[value.length]; + for (int j = 0; j < value.length; j++) { + shortResult[i] = (short) value[j]; + i++; + } + return shortResult; + case DATA_INT: + int[] intResult = new int[value.length]; + for (int j = 0; j < value.length; j++) { + intResult[i] = (int) value[j]; + i++; + } + return intResult; + default: + return value; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java new file mode 100644 index 0000000..8d20f18 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.compression; + +import java.math.BigDecimal; + +import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder; +import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; + +/** + * Double compressor + */ +public class DoubleCompressor extends ValueCompressor { + + @Override protected Object compressNonDecimalMaxMin(DataType changedDataType, + CarbonWriteDataHolder dataHolder, int decimal, Object maxValue) { + int i = 0; + BigDecimal max = BigDecimal.valueOf((double)maxValue); + double[] value = dataHolder.getWritableDoubleValues(); + switch (changedDataType) { + case DATA_BYTE: + byte[] result = new byte[value.length]; + for (int j = 0; j < value.length; j++) { + BigDecimal val = BigDecimal.valueOf(value[j]); + double diff = max.subtract(val).doubleValue(); + result[i] = (byte) (Math.round(diff * Math.pow(10, decimal))); + i++; + } + return result; + case DATA_SHORT: + short[] shortResult = new short[value.length]; + for (int j = 0; j < value.length; j++) { + BigDecimal val = BigDecimal.valueOf(value[j]); + double diff = max.subtract(val).doubleValue(); + shortResult[i] = (short) (Math.round(diff * Math.pow(10, decimal))); + i++; + } + return shortResult; + case DATA_INT: + int[] intResult = new int[value.length]; + for (int j = 0; j < value.length; j++) { + BigDecimal val = BigDecimal.valueOf(value[j]); + double diff = max.subtract(val).doubleValue(); + intResult[i] = (int) (Math.round(diff * Math.pow(10, decimal))); + i++; + } + return intResult; + case DATA_LONG: + long[] longResult = new long[value.length]; + for (int j = 0; j < value.length; j++) { + BigDecimal val = BigDecimal.valueOf(value[j]); + double diff = max.subtract(val).doubleValue(); + longResult[i] = (long) (Math.round(diff * Math.pow(10, decimal))); + i++; + } + return longResult; + case DATA_FLOAT: + float[] floatResult = new float[value.length]; + for (int j = 0; j < value.length; j++) { + BigDecimal val = BigDecimal.valueOf(value[j]); + double diff = max.subtract(val).doubleValue(); + floatResult[i] = (float) (Math.round(diff * Math.pow(10, decimal))); + i++; + } + return floatResult; + default: + double[] defaultResult = new double[value.length]; + for (int j = 0; j < value.length; j++) { + BigDecimal val = BigDecimal.valueOf(value[j]); + double diff = max.subtract(val).doubleValue(); + defaultResult[i] = (Math.round(diff * Math.pow(10, decimal))); + i++; + } + return defaultResult; + } + } + + @Override + protected Object compressNonDecimal(DataType changedDataType, CarbonWriteDataHolder dataHolder, + int decimal) { + int i = 0; + double[] value = dataHolder.getWritableDoubleValues(); + switch (changedDataType) { + case DATA_BYTE: + byte[] result = new byte[value.length]; + for (int j = 0; j < value.length; j++) { + result[i] = (byte) (Math.round(Math.pow(10, decimal) * value[j])); + i++; + } + return result; + case DATA_SHORT: + short[] shortResult = new short[value.length]; + for (int j = 0; j < value.length; j++) { + shortResult[i] = (short) (Math.round(Math.pow(10, decimal) * value[j])); + i++; + } + return shortResult; + case DATA_INT: + int[] intResult = new int[value.length]; + for (int j = 0; j < value.length; j++) { + intResult[i] = (int) (Math.round(Math.pow(10, decimal) * value[j])); + i++; + } + return intResult; + case DATA_LONG: + long[] longResult = new long[value.length]; + for (int j = 0; j < value.length; j++) { + longResult[i] = (long) (Math.round(Math.pow(10, decimal) * value[j])); + i++; + } + return longResult; + case DATA_FLOAT: + float[] floatResult = new float[value.length]; + for (int j = 0; j < value.length; j++) { + floatResult[i] = (float) (Math.round(Math.pow(10, decimal) * value[j])); + i++; + } + return floatResult; + default: + double[] defaultResult = new double[value.length]; + for (int j = 0; j < value.length; j++) { + defaultResult[i] = (double) (Math.round(Math.pow(10, decimal) * value[j])); + i++; + } + return defaultResult; + } + } + + @Override + protected Object compressMaxMin(DataType changedDataType, CarbonWriteDataHolder dataHolder, + Object max) { + double maxValue = (double) max; + double[] value = dataHolder.getWritableDoubleValues(); + int i = 0; + switch (changedDataType) { + case DATA_BYTE: + byte[] result = new byte[value.length]; + for (int j = 0; j < value.length; j++) { + result[i] = (byte) (maxValue - value[j]); + i++; + } + return result; + case DATA_SHORT: + short[] shortResult = new short[value.length]; + for (int j = 0; j < value.length; j++) { + shortResult[i] = (short) (maxValue - value[j]); + i++; + } + return shortResult; + case DATA_INT: + int[] intResult = new int[value.length]; + for (int j = 0; j < value.length; j++) { + intResult[i] = (int) (maxValue - value[j]); + i++; + } + return intResult; + case DATA_LONG: + long[] longResult = new long[value.length]; + for (int j = 0; j < value.length; j++) { + longResult[i] = (long) (maxValue - value[j]); + i++; + } + return longResult; + case DATA_FLOAT: + float[] floatResult = new float[value.length]; + for (int j = 0; j < value.length; j++) { + floatResult[i] = (float) (maxValue - value[j]); + i++; + } + return floatResult; + default: + double[] defaultResult = new double[value.length]; + for (int j = 0; j < value.length; j++) { + defaultResult[i] = (double) (maxValue - value[j]); + i++; + } + return defaultResult; + } + } + + @Override + protected Object compressNone(DataType changedDataType, CarbonWriteDataHolder dataHolder) { + double[] value = dataHolder.getWritableDoubleValues(); + int i = 0; + switch (changedDataType) { + case DATA_BYTE: + byte[] result = new byte[value.length]; + for (int j = 0; j < value.length; j++) { + result[i] = (byte) value[j]; + i++; + } + return result; + case DATA_SHORT: + short[] shortResult = new short[value.length]; + for (int j = 0; j < value.length; j++) { + shortResult[i] = (short) value[j]; + i++; + } + return shortResult; + case DATA_INT: + int[] intResult = new int[value.length]; + for (int j = 0; j < value.length; j++) { + intResult[i] = (int) value[j]; + i++; + } + return intResult; + case DATA_LONG: + case DATA_BIGINT: + long[] longResult = new long[value.length]; + for (int j = 0; j < value.length; j++) { + longResult[i] = (long) value[j]; + i++; + } + return longResult; + case DATA_FLOAT: + float[] floatResult = new float[value.length]; + for (int j = 0; j < value.length; j++) { + floatResult[i] = (float) value[j]; + i++; + } + return floatResult; + default: + return value; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java new file mode 100644 index 0000000..5ecb90e --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.compression; + +import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder; +import org.apache.carbondata.core.util.ValueCompressionUtil.COMPRESSION_TYPE; +import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; + +/** + * Measure compressor + */ +public abstract class ValueCompressor { + + /** + * + * @param compType + * @param dataHolder + * @param changedDataType + * @param maxValue + * @param decimal + * @return compressed data + */ + public Object getCompressedValues(COMPRESSION_TYPE compType, CarbonWriteDataHolder dataHolder, + DataType changedDataType, Object maxValue, int decimal) { + Object o; + switch (compType) { + case NONE: + + o = compressNone(changedDataType, dataHolder); + return o; + + case MAX_MIN: + + o = compressMaxMin(changedDataType, dataHolder, maxValue); + return o; + + case NON_DECIMAL_CONVERT: + + o = compressNonDecimal(changedDataType, dataHolder, decimal); + return o; + + default: + o = compressNonDecimalMaxMin(changedDataType, dataHolder, decimal, maxValue); + return o; + } + } + + /** + * + * @param changedDataType + * @param dataHolder + * @param decimal + * @param maxValue + * @return compressed data + */ + protected abstract Object compressNonDecimalMaxMin(DataType changedDataType, + CarbonWriteDataHolder dataHolder, int decimal, Object maxValue); + + /** + * + * @param changedDataType + * @param dataHolder + * @param decimal + * @return compressed data + */ + protected abstract Object compressNonDecimal(DataType changedDataType, + CarbonWriteDataHolder dataHolder, int decimal); + + /** + * + * @param changedDataType + * @param dataHolder + * @param maxValue + * @return compressed data + */ + protected abstract Object compressMaxMin(DataType changedDataType, + CarbonWriteDataHolder dataHolder, Object maxValue); + + /** + * + * @param changedDataType + * @param dataHolder + * @return compressed data + */ + protected abstract Object compressNone(DataType changedDataType, + CarbonWriteDataHolder dataHolder); +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressDefaultLong.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressDefaultLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressDefaultLong.java index a4d3b96..444fb5c 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressDefaultLong.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressDefaultLong.java @@ -22,18 +22,21 @@ import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder; import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder; +import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; public class UnCompressDefaultLong extends UnCompressNoneLong { + private static final LogService LOGGER = LogServiceFactory + .getLogService(UnCompressDefaultLong.class.getName()); - private static final LogService LOGGER = - LogServiceFactory.getLogService(UnCompressDefaultLong.class.getName()); + public UnCompressDefaultLong(DataType actualDataType) { + super(actualDataType); + } public ValueCompressonHolder.UnCompressValue getNew() { try { return (ValueCompressonHolder.UnCompressValue) clone(); } catch (CloneNotSupportedException clnNotSupportedExc) { - LOGGER.error(clnNotSupportedExc, - clnNotSupportedExc.getMessage()); + LOGGER.error(clnNotSupportedExc, clnNotSupportedExc.getMessage()); } return null; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByte.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByte.java index e6486c2..db7f085 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByte.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByte.java @@ -46,8 +46,17 @@ public class UnCompressMaxMinByte implements UnCompressValue<byte[]> { */ protected byte[] value; + /** + * actual data type + */ + protected DataType actualDataType; + //TODO SIMIAN + public UnCompressMaxMinByte(DataType actualDataType) { + this.actualDataType = actualDataType; + } + @Override public void setValue(byte[] value) { this.value = value; @@ -64,13 +73,13 @@ public class UnCompressMaxMinByte implements UnCompressValue<byte[]> { @Override public UnCompressValue compress() { - UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte(); + UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte(actualDataType); byte1.setValue(byteCompressor.compress(value)); return byte1; } @Override public UnCompressValue uncompress(DataType dataType) { - UnCompressValue byte1 = ValueCompressionUtil.unCompressMaxMin(dataType, dataType); + UnCompressValue byte1 = ValueCompressionUtil.unCompressMaxMin(dataType, actualDataType); ValueCompressonHolder.unCompress(dataType, byte1, value); return byte1; } @@ -87,10 +96,34 @@ public class UnCompressMaxMinByte implements UnCompressValue<byte[]> { * @see ValueCompressonHolder.UnCompressValue#getCompressorObject() */ @Override public UnCompressValue getCompressorObject() { - return new UnCompressMaxMinByte(); + return new UnCompressMaxMinByte(actualDataType); } @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) { + switch (actualDataType) { + case DATA_BIGINT: + return unCompressLong(decimal, maxValueObject); + default: + return unCompressDouble(decimal, maxValueObject); + } + } + + private CarbonReadDataHolder unCompressLong(int decimal, Object maxValueObject) { + long maxValue = (long) maxValueObject; + long[] vals = new long[value.length]; + CarbonReadDataHolder dataHolder = new CarbonReadDataHolder(); + for (int i = 0; i < vals.length; i++) { + if (value[i] == 0) { + vals[i] = maxValue; + } else { + vals[i] = maxValue - value[i]; + } + } + dataHolder.setReadableLongValues(vals); + return dataHolder; + } + + private CarbonReadDataHolder unCompressDouble(int decimal, Object maxValueObject) { double maxValue = (double) maxValueObject; double[] vals = new double[value.length]; CarbonReadDataHolder dataHolder = new CarbonReadDataHolder(); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByteForLong.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByteForLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByteForLong.java index c265a44..b5d42b3 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByteForLong.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByteForLong.java @@ -25,6 +25,7 @@ import org.apache.carbondata.core.datastorage.store.compression.SnappyCompressio import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder; import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder; import org.apache.carbondata.core.util.ValueCompressionUtil; +import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; public class UnCompressMaxMinByteForLong extends UnCompressMaxMinByte { @@ -33,6 +34,9 @@ public class UnCompressMaxMinByteForLong extends UnCompressMaxMinByte { private static Compressor<byte[]> byteCompressor = SnappyCompression.SnappyByteCompression.INSTANCE; + public UnCompressMaxMinByteForLong(DataType actualDataType) { + super(actualDataType); + } @Override public ValueCompressonHolder.UnCompressValue getNew() { try { return (ValueCompressonHolder.UnCompressValue) clone(); @@ -44,7 +48,7 @@ public class UnCompressMaxMinByteForLong extends UnCompressMaxMinByte { @Override public ValueCompressonHolder.UnCompressValue compress() { - UnCompressMaxMinByteForLong byte1 = new UnCompressMaxMinByteForLong(); + UnCompressMaxMinByteForLong byte1 = new UnCompressMaxMinByteForLong(actualDataType); byte1.setValue(byteCompressor.compress(value)); return byte1; } @@ -58,7 +62,7 @@ public class UnCompressMaxMinByteForLong extends UnCompressMaxMinByte { } @Override public ValueCompressonHolder.UnCompressValue getCompressorObject() { - return new UnCompressMaxMinByteForLong(); + return new UnCompressMaxMinByteForLong(actualDataType); } @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefault.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefault.java index df72c61..dea9858 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefault.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefault.java @@ -48,6 +48,15 @@ public class UnCompressMaxMinDefault implements ValueCompressonHolder.UnCompress */ private double[] value; + /** + * actual data type + */ + private DataType actualDataType; + + public UnCompressMaxMinDefault(DataType actualDataType) { + this.actualDataType = actualDataType; + } + @Override public void setValue(double[] value) { this.value = (double[]) value; @@ -63,7 +72,7 @@ public class UnCompressMaxMinDefault implements ValueCompressonHolder.UnCompress } @Override public ValueCompressonHolder.UnCompressValue compress() { - UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte(); + UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte(actualDataType); byte1.setValue(doubleCompressor.compress(value)); return byte1; } @@ -85,7 +94,7 @@ public class UnCompressMaxMinDefault implements ValueCompressonHolder.UnCompress * @see ValueCompressonHolder.UnCompressValue#getCompressorObject() */ @Override public ValueCompressonHolder.UnCompressValue getCompressorObject() { - return new UnCompressMaxMinByte(); + return new UnCompressMaxMinByte(actualDataType); } //TODO SIMIAN http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefaultLong.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefaultLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefaultLong.java index 57a25eb..376b487 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefaultLong.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefaultLong.java @@ -25,6 +25,7 @@ import org.apache.carbondata.core.datastorage.store.compression.SnappyCompressio import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder; import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder; import org.apache.carbondata.core.util.ValueCompressionUtil; +import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; public class UnCompressMaxMinDefaultLong extends UnCompressMaxMinLong { @@ -33,6 +34,10 @@ public class UnCompressMaxMinDefaultLong extends UnCompressMaxMinLong { private static Compressor<long[]> longCompressor = SnappyCompression.SnappyLongCompression.INSTANCE; + public UnCompressMaxMinDefaultLong(DataType actualDataType) { + super(actualDataType); + // TODO Auto-generated constructor stub + } @Override public ValueCompressonHolder.UnCompressValue getNew() { try { return (ValueCompressonHolder.UnCompressValue) clone(); @@ -43,7 +48,7 @@ public class UnCompressMaxMinDefaultLong extends UnCompressMaxMinLong { } @Override public ValueCompressonHolder.UnCompressValue compress() { - UnCompressMaxMinByteForLong byte1 = new UnCompressMaxMinByteForLong(); + UnCompressMaxMinByteForLong byte1 = new UnCompressMaxMinByteForLong(actualDataType); byte1.setValue(longCompressor.compress(value)); return byte1; } @@ -53,7 +58,7 @@ public class UnCompressMaxMinDefaultLong extends UnCompressMaxMinLong { } @Override public ValueCompressonHolder.UnCompressValue getCompressorObject() { - return new UnCompressMaxMinByteForLong(); + return new UnCompressMaxMinByteForLong(actualDataType); } @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinFloat.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinFloat.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinFloat.java index 396c5c0..0958069 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinFloat.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinFloat.java @@ -47,6 +47,12 @@ public class UnCompressMaxMinFloat implements UnCompressValue<float[]> { */ private float[] value; + private DataType actualDataType; + + public UnCompressMaxMinFloat(DataType actualDataType) { + this.actualDataType = actualDataType; + } + @Override public void setValue(float[] value) { this.value = (float[]) value; @@ -63,7 +69,7 @@ public class UnCompressMaxMinFloat implements UnCompressValue<float[]> { @Override public UnCompressValue compress() { - UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte(); + UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte(actualDataType); byte1.setValue(floatCompressor.compress(value)); return byte1; } @@ -85,7 +91,7 @@ public class UnCompressMaxMinFloat implements UnCompressValue<float[]> { * @see ValueCompressonHolder.UnCompressValue#getCompressorObject() */ @Override public UnCompressValue getCompressorObject() { - return new UnCompressMaxMinByte(); + return new UnCompressMaxMinByte(actualDataType); } @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinInt.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinInt.java index a2af61d..c25f75b 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinInt.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinInt.java @@ -28,6 +28,7 @@ import org.apache.carbondata.core.datastorage.store.compression.SnappyCompressio import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder; import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder; import org.apache.carbondata.core.util.ValueCompressionUtil; +import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; public class UnCompressMaxMinInt implements ValueCompressonHolder.UnCompressValue<int[]> { /** @@ -45,6 +46,12 @@ public class UnCompressMaxMinInt implements ValueCompressonHolder.UnCompressValu */ private int[] value; + private DataType actualDataType; + + public UnCompressMaxMinInt(DataType actualDataType) { + this.actualDataType = actualDataType; + } + @Override public void setValue(int[] value) { this.value = value; @@ -60,7 +67,7 @@ public class UnCompressMaxMinInt implements ValueCompressonHolder.UnCompressValu } @Override public ValueCompressonHolder.UnCompressValue compress() { - UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte(); + UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte(actualDataType); byte1.setValue(intCompressor.compress(value)); return byte1; } @@ -83,10 +90,19 @@ public class UnCompressMaxMinInt implements ValueCompressonHolder.UnCompressValu * @see ValueCompressonHolder.UnCompressValue#getCompressorObject() */ @Override public ValueCompressonHolder.UnCompressValue getCompressorObject() { - return new UnCompressMaxMinByte(); + return new UnCompressMaxMinByte(actualDataType); } @Override public CarbonReadDataHolder getValues(int decVal, Object maxValueObject) { + switch (actualDataType) { + case DATA_BIGINT: + return unCompressLong(decVal, maxValueObject); + default: + return unCompressDouble(decVal, maxValueObject); + } + } + + private CarbonReadDataHolder unCompressDouble(int decVal, Object maxValueObject) { double maxValue = (double) maxValueObject; double[] vals = new double[value.length]; CarbonReadDataHolder dataHolder = new CarbonReadDataHolder(); @@ -102,4 +118,20 @@ public class UnCompressMaxMinInt implements ValueCompressonHolder.UnCompressValu return dataHolder; } + private CarbonReadDataHolder unCompressLong(int decVal, Object maxValueObject) { + long maxValue = (long) maxValueObject; + long[] vals = new long[value.length]; + CarbonReadDataHolder dataHolder = new CarbonReadDataHolder(); + for (int i = 0; i < vals.length; i++) { + if (value[i] == 0) { + vals[i] = maxValue; + } else { + vals[i] = maxValue - value[i]; + } + + } + dataHolder.setReadableLongValues(vals); + return dataHolder; + } + } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinLong.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinLong.java index cab3aa2..0dc1c5b 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinLong.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinLong.java @@ -28,6 +28,7 @@ import org.apache.carbondata.core.datastorage.store.compression.SnappyCompressio import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder; import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder; import org.apache.carbondata.core.util.ValueCompressionUtil; +import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; public class UnCompressMaxMinLong implements ValueCompressonHolder.UnCompressValue<long[]> { /** @@ -45,6 +46,12 @@ public class UnCompressMaxMinLong implements ValueCompressonHolder.UnCompressVal */ protected long[] value; + protected DataType actualDataType; + + public UnCompressMaxMinLong(DataType actualDataType) { + this.actualDataType = actualDataType; + } + @Override public ValueCompressonHolder.UnCompressValue getNew() { try { return (ValueCompressonHolder.UnCompressValue) clone(); @@ -55,7 +62,7 @@ public class UnCompressMaxMinLong implements ValueCompressonHolder.UnCompressVal } @Override public ValueCompressonHolder.UnCompressValue compress() { - UnCompressMaxMinByte unCompressByte = new UnCompressMaxMinByte(); + UnCompressMaxMinByte unCompressByte = new UnCompressMaxMinByte(actualDataType); unCompressByte.setValue(longCompressor.compress(value)); return unCompressByte; } @@ -83,10 +90,20 @@ public class UnCompressMaxMinLong implements ValueCompressonHolder.UnCompressVal * @see ValueCompressonHolder.UnCompressValue#getCompressorObject() */ @Override public ValueCompressonHolder.UnCompressValue getCompressorObject() { - return new UnCompressMaxMinByte(); + return new UnCompressMaxMinByte(actualDataType); } @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) { + switch (actualDataType) { + case DATA_BIGINT: + return unCompressLong(decimal, maxValueObject); + default: + return unCompressDouble(decimal, maxValueObject); + } + + } + + private CarbonReadDataHolder unCompressDouble(int decimal, Object maxValueObject) { double maxValue = (double) maxValueObject; double[] vals = new double[value.length]; CarbonReadDataHolder data = new CarbonReadDataHolder(); @@ -102,4 +119,20 @@ public class UnCompressMaxMinLong implements ValueCompressonHolder.UnCompressVal return data; } + private CarbonReadDataHolder unCompressLong(int decimal, Object maxValueObject) { + long maxValue = (long) maxValueObject; + long[] vals = new long[value.length]; + CarbonReadDataHolder data = new CarbonReadDataHolder(); + for (int i = 0; i < vals.length; i++) { + if (value[i] == 0) { + vals[i] = maxValue; + } else { + vals[i] = maxValue - value[i]; + } + + } + data.setReadableLongValues(vals); + return data; + } + } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinShort.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinShort.java index 884c430..faca753 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinShort.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinShort.java @@ -46,6 +46,12 @@ public class UnCompressMaxMinShort implements ValueCompressonHolder.UnCompressVa */ private short[] value; + private DataType actualDataType; + + public UnCompressMaxMinShort(DataType actualDataType) { + this.actualDataType = actualDataType; + } + @Override public void setValue(short[] value) { this.value = value; @@ -70,7 +76,7 @@ public class UnCompressMaxMinShort implements ValueCompressonHolder.UnCompressVa @Override public ValueCompressonHolder.UnCompressValue compress() { - UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte(); + UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte(actualDataType); byte1.setValue(shortCompressor.compress(value)); return byte1; } @@ -84,10 +90,20 @@ public class UnCompressMaxMinShort implements ValueCompressonHolder.UnCompressVa * @see ValueCompressonHolder.UnCompressValue#getCompressorObject() */ @Override public ValueCompressonHolder.UnCompressValue getCompressorObject() { - return new UnCompressMaxMinByte(); + return new UnCompressMaxMinByte(actualDataType); } @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) { + switch (actualDataType) { + case DATA_BIGINT: + return unCompressLong(decimal, maxValueObject); + default: + return unCompressDouble(decimal, maxValueObject); + } + + } + + private CarbonReadDataHolder unCompressDouble(int decimal, Object maxValueObject) { double maxValue = (double) maxValueObject; double[] vals = new double[value.length]; CarbonReadDataHolder carbonDataHolderObj = new CarbonReadDataHolder(); @@ -103,4 +119,20 @@ public class UnCompressMaxMinShort implements ValueCompressonHolder.UnCompressVa return carbonDataHolderObj; } + private CarbonReadDataHolder unCompressLong(int decimal, Object maxValueObject) { + long maxValue = (long) maxValueObject; + long[] vals = new long[value.length]; + CarbonReadDataHolder carbonDataHolderObj = new CarbonReadDataHolder(); + for (int i = 0; i < vals.length; i++) { + if (value[i] == 0) { + vals[i] = maxValue; + } else { + vals[i] = maxValue - value[i]; + } + + } + carbonDataHolderObj.setReadableLongValues(vals); + return carbonDataHolderObj; + } + } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneByte.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneByte.java index d689f93..91959a2 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneByte.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneByte.java @@ -47,6 +47,15 @@ public class UnCompressNoneByte implements UnCompressValue<byte[]> { */ private byte[] value; + /** + * actual data type + */ + private DataType actualDataType; + + public UnCompressNoneByte(DataType actualDataType) { + this.actualDataType = actualDataType; + } + @Override public UnCompressValue getNew() { try { return (UnCompressValue) clone(); @@ -61,13 +70,13 @@ public class UnCompressNoneByte implements UnCompressValue<byte[]> { } @Override public UnCompressValue uncompress(DataType dataType) { - UnCompressValue byte1 = ValueCompressionUtil.unCompressNone(dataType, dataType); + UnCompressValue byte1 = ValueCompressionUtil.unCompressNone(dataType, actualDataType); ValueCompressonHolder.unCompress(dataType, byte1, value); return byte1; } @Override public UnCompressValue compress() { - UnCompressNoneByte byte1 = new UnCompressNoneByte(); + UnCompressNoneByte byte1 = new UnCompressNoneByte(actualDataType); byte1.setValue(byteCompressor.compress(value)); return byte1; } @@ -84,12 +93,24 @@ public class UnCompressNoneByte implements UnCompressValue<byte[]> { * @see ValueCompressonHolder.UnCompressValue#getCompressorObject() */ @Override public UnCompressValue getCompressorObject() { - return new UnCompressNoneByte(); + return new UnCompressNoneByte(actualDataType); } @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) { + switch (actualDataType) { + case DATA_BIGINT: + return unCompressLong(); + default: + return unCompressDouble(); + } + + } + + private CarbonReadDataHolder unCompressDouble() { CarbonReadDataHolder dataHldr = new CarbonReadDataHolder(); + double[] vals = new double[value.length]; + for (int i = 0; i < vals.length; i++) { vals[i] = value[i]; } @@ -97,4 +118,16 @@ public class UnCompressNoneByte implements UnCompressValue<byte[]> { return dataHldr; } + private CarbonReadDataHolder unCompressLong() { + CarbonReadDataHolder dataHldr = new CarbonReadDataHolder(); + + long[] vals = new long[value.length]; + + for (int i = 0; i < vals.length; i++) { + vals[i] = value[i]; + } + dataHldr.setReadableLongValues(vals); + return dataHldr; + } + } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneDefault.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneDefault.java index c1395b4..abf6ae3 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneDefault.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneDefault.java @@ -46,6 +46,12 @@ public class UnCompressNoneDefault implements UnCompressValue<double[]> { */ private double[] value; + private DataType actualDataType; + + public UnCompressNoneDefault(DataType actualDataType) { + this.actualDataType = actualDataType; + } + @Override public void setValue(double[] value) { this.value = value; @@ -61,7 +67,7 @@ public class UnCompressNoneDefault implements UnCompressValue<double[]> { } @Override public UnCompressValue compress() { - UnCompressNoneByte byte1 = new UnCompressNoneByte(); + UnCompressNoneByte byte1 = new UnCompressNoneByte(actualDataType); byte1.setValue(doubleCompressor.compress(value)); return byte1; @@ -75,7 +81,7 @@ public class UnCompressNoneDefault implements UnCompressValue<double[]> { * @see ValueCompressonHolder.UnCompressValue#getCompressorObject() */ @Override public UnCompressValue getCompressorObject() { - return new UnCompressNoneByte(); + return new UnCompressNoneByte(actualDataType); } @Override public byte[] getBackArrayData() { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneFloat.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneFloat.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneFloat.java index 103b1c9..6167c54 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneFloat.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneFloat.java @@ -28,6 +28,7 @@ import org.apache.carbondata.core.datastorage.store.compression.SnappyCompressio import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder; import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder; import org.apache.carbondata.core.util.ValueCompressionUtil; +import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; public class UnCompressNoneFloat implements ValueCompressonHolder.UnCompressValue<float[]> { /** @@ -45,6 +46,12 @@ public class UnCompressNoneFloat implements ValueCompressonHolder.UnCompressValu */ private float[] value; + private DataType actualDataType; + + public UnCompressNoneFloat(DataType actualDataType) { + this.actualDataType = actualDataType; + } + @Override public void setValue(float[] value) { this.value = value; @@ -60,7 +67,7 @@ public class UnCompressNoneFloat implements ValueCompressonHolder.UnCompressValu } @Override public ValueCompressonHolder.UnCompressValue compress() { - UnCompressNoneByte byte1 = new UnCompressNoneByte(); + UnCompressNoneByte byte1 = new UnCompressNoneByte(this.actualDataType); byte1.setValue(floatCompressor.compress(value)); return byte1; @@ -76,7 +83,7 @@ public class UnCompressNoneFloat implements ValueCompressonHolder.UnCompressValu * @see ValueCompressonHolder.UnCompressValue#getCompressorObject() */ @Override public ValueCompressonHolder.UnCompressValue getCompressorObject() { - return new UnCompressNoneByte(); + return new UnCompressNoneByte(this.actualDataType); } @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneInt.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneInt.java index ca5593d..d12ebe1 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneInt.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneInt.java @@ -45,6 +45,12 @@ public class UnCompressNoneInt implements ValueCompressonHolder.UnCompressValue< */ private int[] value; + private DataType actualDataType; + + public UnCompressNoneInt(DataType actualDataType) { + this.actualDataType = actualDataType; + } + @Override public void setValue(int[] value) { this.value = value; @@ -64,7 +70,7 @@ public class UnCompressNoneInt implements ValueCompressonHolder.UnCompressValue< } @Override public ValueCompressonHolder.UnCompressValue compress() { - UnCompressNoneByte byte1 = new UnCompressNoneByte(); + UnCompressNoneByte byte1 = new UnCompressNoneByte(this.actualDataType); byte1.setValue(intCompressor.compress(value)); return byte1; @@ -84,10 +90,19 @@ public class UnCompressNoneInt implements ValueCompressonHolder.UnCompressValue< * @see ValueCompressonHolder.UnCompressValue#getCompressorObject() */ @Override public ValueCompressonHolder.UnCompressValue getCompressorObject() { - return new UnCompressNoneByte(); + return new UnCompressNoneByte(this.actualDataType); } @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) { + switch (actualDataType) { + case DATA_BIGINT: + return unCompressLong(); + default: + return unCompressDouble(); + } + } + + private CarbonReadDataHolder unCompressDouble() { CarbonReadDataHolder dataHolderInfoObj = new CarbonReadDataHolder(); double[] vals = new double[value.length]; for (int i = 0; i < vals.length; i++) { @@ -98,4 +113,14 @@ public class UnCompressNoneInt implements ValueCompressonHolder.UnCompressValue< return dataHolderInfoObj; } + private CarbonReadDataHolder unCompressLong() { + CarbonReadDataHolder dataHolderInfoObj = new CarbonReadDataHolder(); + long[] vals = new long[value.length]; + for (int i = 0; i < vals.length; i++) { + vals[i] = value[i]; + } + + dataHolderInfoObj.setReadableLongValues(vals); + return dataHolderInfoObj; + } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneLong.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneLong.java index 47220c5..4661b7a 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneLong.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneLong.java @@ -28,6 +28,7 @@ import org.apache.carbondata.core.datastorage.store.compression.SnappyCompressio import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder; import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder; import org.apache.carbondata.core.util.ValueCompressionUtil; +import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; public class UnCompressNoneLong implements ValueCompressonHolder.UnCompressValue<long[]> { /** @@ -45,6 +46,12 @@ public class UnCompressNoneLong implements ValueCompressonHolder.UnCompressValue */ protected long[] value; + private DataType actualDataType; + + public UnCompressNoneLong(DataType actualDataType) { + this.actualDataType = actualDataType; + } + @Override public void setValue(long[] value) { this.value = value; @@ -61,7 +68,7 @@ public class UnCompressNoneLong implements ValueCompressonHolder.UnCompressValue } @Override public ValueCompressonHolder.UnCompressValue compress() { - UnCompressNoneByte byte1 = new UnCompressNoneByte(); + UnCompressNoneByte byte1 = new UnCompressNoneByte(this.actualDataType); byte1.setValue(longCompressor.compress(value)); return byte1; @@ -85,17 +92,39 @@ public class UnCompressNoneLong implements ValueCompressonHolder.UnCompressValue * @see ValueCompressonHolder.UnCompressValue#getCompressorObject() */ @Override public ValueCompressonHolder.UnCompressValue getCompressorObject() { - return new UnCompressNoneByte(); + return new UnCompressNoneByte(this.actualDataType); } @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) { - CarbonReadDataHolder dataHolder = new CarbonReadDataHolder(); + switch (actualDataType) { + case DATA_BIGINT: + return unCompressLong(); + default: + return unCompressDouble(); + } + } + + private CarbonReadDataHolder unCompressDouble() { + CarbonReadDataHolder dataHldr = new CarbonReadDataHolder(); + double[] vals = new double[value.length]; + for (int i = 0; i < vals.length; i++) { vals[i] = value[i]; } - dataHolder.setReadableDoubleValues(vals); - return dataHolder; + dataHldr.setReadableDoubleValues(vals); + return dataHldr; } + private CarbonReadDataHolder unCompressLong() { + CarbonReadDataHolder dataHldr = new CarbonReadDataHolder(); + + long[] vals = new long[value.length]; + + for (int i = 0; i < vals.length; i++) { + vals[i] = value[i]; + } + dataHldr.setReadableLongValues(vals); + return dataHldr; + } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneShort.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneShort.java index ed41ca5..2fdf514 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneShort.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneShort.java @@ -48,6 +48,12 @@ public class UnCompressNoneShort implements ValueCompressonHolder.UnCompressValu */ private short[] shortValue; + private DataType actualDataType; + + public UnCompressNoneShort(DataType actualDataType) { + this.actualDataType = actualDataType; + } + @Override public void setValue(short[] shortValue) { this.shortValue = shortValue; @@ -64,7 +70,7 @@ public class UnCompressNoneShort implements ValueCompressonHolder.UnCompressValu @Override public ValueCompressonHolder.UnCompressValue compress() { - UnCompressNoneByte byte1 = new UnCompressNoneByte(); + UnCompressNoneByte byte1 = new UnCompressNoneByte(this.actualDataType); byte1.setValue(shortCompressor.compress(shortValue)); return byte1; @@ -88,17 +94,39 @@ public class UnCompressNoneShort implements ValueCompressonHolder.UnCompressValu * @see ValueCompressonHolder.UnCompressValue#getCompressorObject() */ @Override public ValueCompressonHolder.UnCompressValue getCompressorObject() { - return new UnCompressNoneByte(); + return new UnCompressNoneByte(this.actualDataType); } @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) { - CarbonReadDataHolder dataHolder = new CarbonReadDataHolder(); + switch (actualDataType) { + case DATA_BIGINT: + return unCompressLong(); + default: + return unCompressDouble(); + } + } + + private CarbonReadDataHolder unCompressDouble() { + CarbonReadDataHolder dataHldr = new CarbonReadDataHolder(); + double[] vals = new double[shortValue.length]; + for (int i = 0; i < vals.length; i++) { vals[i] = shortValue[i]; } - dataHolder.setReadableDoubleValues(vals); - return dataHolder; + dataHldr.setReadableDoubleValues(vals); + return dataHldr; } + private CarbonReadDataHolder unCompressLong() { + CarbonReadDataHolder dataHldr = new CarbonReadDataHolder(); + + long[] vals = new long[shortValue.length]; + + for (int i = 0; i < vals.length; i++) { + vals[i] = shortValue[i]; + } + dataHldr.setReadableLongValues(vals); + return dataHldr; + } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java index 21c2a60..6051736 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java @@ -64,17 +64,12 @@ public abstract class AbstractHeavyCompressedDoubleArrayDataStore values[i] = compressionModel.getUnCompressValues()[i].getNew(); if (type[i] != CarbonCommonConstants.BYTE_VALUE_MEASURE && type[i] != CarbonCommonConstants.BIG_DECIMAL_MEASURE) { - if (type[i] == CarbonCommonConstants.BIG_INT_MEASURE) { - values[i].setValue(ValueCompressionUtil - .getCompressedValues(compressionModel.getCompType()[i], - dataHolder[i].getWritableLongValues(), compressionModel.getChangedDataType()[i], - (long) compressionModel.getMaxValue()[i], compressionModel.getDecimal()[i])); - } else { - values[i].setValue(ValueCompressionUtil - .getCompressedValues(compressionModel.getCompType()[i], - dataHolder[i].getWritableDoubleValues(), compressionModel.getChangedDataType()[i], - (double) compressionModel.getMaxValue()[i], compressionModel.getDecimal()[i])); - } + + values[i].setValue( + ValueCompressionUtil.getValueCompressor(compressionModel.getActualDataType()[i]) + .getCompressedValues(compressionModel.getCompType()[i], dataHolder[i], + compressionModel.getChangedDataType()[i], compressionModel.getMaxValue()[i], + compressionModel.getDecimal()[i])); } else { values[i].setValue(dataHolder[i].getWritableByteArrayValues()); } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java index d3448f1..5e68411 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java @@ -25,11 +25,13 @@ import java.util.Arrays; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; +import org.apache.carbondata.core.compression.BigIntCompressor; +import org.apache.carbondata.core.compression.DoubleCompressor; +import org.apache.carbondata.core.compression.ValueCompressor; import org.apache.carbondata.core.datastorage.store.compression.MeasureMetaDataModel; import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel; import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder; import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressByteArray; -import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressDefaultLong; import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressMaxMinByte; import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressMaxMinDefault; import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressMaxMinFloat; @@ -149,8 +151,8 @@ public final class ValueCompressionUtil { return new CompressionFinder(COMPRESSION_TYPE.CUSTOM_BIGDECIMAL, DataType.DATA_BYTE, DataType.DATA_BYTE); case 'l': - return new CompressionFinder(COMPRESSION_TYPE.NONE, - DataType.DATA_BIGINT, DataType.DATA_BIGINT); + return getLongCompressorFinder(maxValue, minValue, decimal, aggregatorType, + dataTypeSelected); default: break; } @@ -199,6 +201,22 @@ public final class ValueCompressionUtil { } } + private static CompressionFinder getLongCompressorFinder(Object maxValue, Object minValue, + int decimal, char aggregatorType, byte dataTypeSelected) { + if (getSize(getDataType((long) maxValue, decimal, dataTypeSelected)) > getSize( + getDataType((long) maxValue - (long) minValue, decimal, dataTypeSelected))) { + return new CompressionFinder(COMPRESSION_TYPE.MAX_MIN, DataType.DATA_BIGINT, + getDataType((long) maxValue - (long) minValue, decimal, dataTypeSelected)); + } else if (getSize(getDataType((long) maxValue, decimal, dataTypeSelected)) < getSize( + getDataType((long) maxValue - (long) minValue, decimal, dataTypeSelected))) { + return new CompressionFinder(COMPRESSION_TYPE.NONE, DataType.DATA_BIGINT, + getDataType((long) maxValue - (long) minValue, decimal, dataTypeSelected)); + } else { + return new CompressionFinder(COMPRESSION_TYPE.NONE, DataType.DATA_BIGINT, + getDataType((long) maxValue, decimal, dataTypeSelected)); + } + } + /** * @param compType : compression type * @param values : the data of one measure @@ -243,6 +261,20 @@ public final class ValueCompressionUtil { } } + /** + * It returns Compressor for given datatype + * @param actualDataType + * @return compressor based on actualdatatype + */ + public static ValueCompressor getValueCompressor(DataType actualDataType) { + switch (actualDataType) { + case DATA_BIGINT: + return new BigIntCompressor(); + default: + return new DoubleCompressor(); + } + } + private static ValueCompressonHolder.UnCompressValue[] getUncompressedValues( COMPRESSION_TYPE[] compType, DataType[] actualDataType, DataType[] changedDataType) { @@ -569,34 +601,30 @@ public final class ValueCompressionUtil { */ public static ValueCompressonHolder.UnCompressValue unCompressNone(DataType compDataType, DataType actualDataType) { - if (actualDataType == DataType.DATA_BIGINT) { - return new UnCompressDefaultLong(); - } else { - switch (compDataType) { - case DATA_BYTE: + switch (compDataType) { + case DATA_BYTE: - return new UnCompressNoneByte(); + return new UnCompressNoneByte(actualDataType); - case DATA_SHORT: + case DATA_SHORT: - return new UnCompressNoneShort(); + return new UnCompressNoneShort(actualDataType); - case DATA_INT: + case DATA_INT: - return new UnCompressNoneInt(); + return new UnCompressNoneInt(actualDataType); - case DATA_LONG: + case DATA_LONG: - return new UnCompressNoneLong(); + return new UnCompressNoneLong(actualDataType); - case DATA_FLOAT: + case DATA_FLOAT: - return new UnCompressNoneFloat(); + return new UnCompressNoneFloat(actualDataType); - default: + default: - return new UnCompressNoneDefault(); - } + return new UnCompressNoneDefault(actualDataType); } } @@ -608,27 +636,27 @@ public final class ValueCompressionUtil { switch (compDataType) { case DATA_BYTE: - return new UnCompressMaxMinByte(); + return new UnCompressMaxMinByte(actualDataType); case DATA_SHORT: - return new UnCompressMaxMinShort(); + return new UnCompressMaxMinShort(actualDataType); case DATA_INT: - return new UnCompressMaxMinInt(); + return new UnCompressMaxMinInt(actualDataType); case DATA_LONG: - return new UnCompressMaxMinLong(); + return new UnCompressMaxMinLong(actualDataType); case DATA_FLOAT: - return new UnCompressMaxMinFloat(); + return new UnCompressMaxMinFloat(actualDataType); default: - return new UnCompressMaxMinDefault(); + return new UnCompressMaxMinDefault(actualDataType); } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java index db64011..25d1426 100644 --- a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java @@ -320,7 +320,7 @@ public class ValueCompressionUtilTest { @Test public void testToUnCompressNone() { ValueCompressonHolder.UnCompressValue result = ValueCompressionUtil.unCompressNone(DataType.DATA_BIGINT, DataType.DATA_BIGINT); - assertEquals(result.getClass(), UnCompressDefaultLong.class); + assertEquals(result.getClass(), UnCompressNoneDefault.class); } @Test public void testToUnCompressNoneForByte() { @@ -563,10 +563,10 @@ public class ValueCompressionUtilTest { } @Test public void testToGetValueCompressionModel() { - Object[] maxValues = { 10, 20, 30 }; - Object[] minValues = { 1, 2, 3 }; + Object[] maxValues = { 10, 20, 30l }; + Object[] minValues = { 1, 2, 3l }; int[] decimalLength = { 0, 0, 0 }; - Object[] uniqueValues = { 5, 3, 2 }; + Object[] uniqueValues = { 5, 3, 2l }; char[] types = { 'c', 'b', 'l' }; byte[] dataTypeSelected = { 1, 2, 4 }; MeasureMetaDataModel measureMetaDataModel = http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java index 1fa14f0..e560784 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java @@ -1195,8 +1195,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { long minVal = (long) min[count]; max[count] = (maxVal > value ? max[count] : value); min[count] = (minVal < value ? min[count] : value); - int num = getDecimalCount(value); - decimal[count] = (decimal[count] > num ? decimal[count] : num); } else if (type[count] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) { byte[] buff = null; // in compaction flow the measure with decimal type will come as spark decimal.
