This is an automated email from the ASF dual-hosted git repository. arina pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git
commit bdbd93fc726224ec45d06f09434c7406e444242e Author: Volodymyr Vysotskyi <[email protected]> AuthorDate: Wed May 16 21:34:56 2018 +0300 DRILL-6421: Refactor DecimalUtility and CoreDecimalUtility classes closes #1267 --- .../java/org/apache/drill/common/types/Types.java | 106 ++-- .../drill/common/util/CoreDecimalUtility.java | 55 -- .../drill/exec/store/hive/HiveUtilities.java | 8 +- .../templates/Decimal/CastVarDecimalDecimal.java | 6 +- .../exec/expr/ExpressionTreeMaterializer.java | 43 +- .../drill/exec/physical/impl/join/JoinUtils.java | 6 +- .../drill/exec/planner/sql/TypeInferenceUtils.java | 4 +- .../exec/store/parquet/ParquetRecordWriter.java | 3 +- .../columnreaders/ParquetToDrillTypeConverter.java | 4 +- .../main/codegen/templates/ColumnAccessors.java | 14 +- .../main/codegen/templates/FixedValueVectors.java | 2 +- .../org/apache/drill/exec/util/DecimalUtility.java | 672 ++++----------------- .../drill/exec/vector/ValueHolderHelper.java | 8 +- .../drill/common/expression/ValueExpressions.java | 21 +- 14 files changed, 185 insertions(+), 767 deletions(-) diff --git a/common/src/main/java/org/apache/drill/common/types/Types.java b/common/src/main/java/org/apache/drill/common/types/Types.java index 5c6e7f3..e66a340 100644 --- a/common/src/main/java/org/apache/drill/common/types/Types.java +++ b/common/src/main/java/org/apache/drill/common/types/Types.java @@ -27,12 +27,11 @@ import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; -import org.apache.drill.common.util.CoreDecimalUtility; import com.google.protobuf.TextFormat; +@SuppressWarnings("WeakerAccess") public class Types { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Types.class); public static final int MAX_VARCHAR_LENGTH = 65535; public static final int UNDEFINED = 0; @@ -47,10 +46,6 @@ public class Types { return toType.getMinorType() == MinorType.UNION; } - public enum Comparability { - UNKNOWN, NONE, EQUAL, ORDERED - } - public static boolean isComplex(final MajorType type) { switch(type.getMinorType()) { case LIST: @@ -90,6 +85,37 @@ public class Types { case UINT4: case UINT8: return true; + default: + return false; + } + } + + /** + * Returns true if specified type is decimal data type. + * + * @param type type to check + * @return true if specified type is decimal data type. + */ + public static boolean isDecimalType(MajorType type) { + return isDecimalType(type.getMinorType()); + } + + /** + * Returns true if specified type is decimal data type. + * + * @param minorType type to check + * @return true if specified type is decimal data type. + */ + public static boolean isDecimalType(MinorType minorType) { + switch(minorType) { + case VARDECIMAL: + case DECIMAL38SPARSE: + case DECIMAL38DENSE: + case DECIMAL28SPARSE: + case DECIMAL28DENSE: + case DECIMAL18: + case DECIMAL9: + return true; default: return false; } @@ -182,11 +208,9 @@ public class Types { * Extend decimal type with precision and scale. * * @param type major type - * @param typeName type converted to a string * @return type name augmented with precision and scale, * if type is a decimal */ - public static String getExtendedSqlTypeName(MajorType type) { String typeName = getSqlTypeName(type); @@ -266,7 +290,7 @@ public class Types { /** * Reports whether given RPC-level type is a signed type (per semantics of - * {@link ResultSetMetaData#isSigned(int)}). + * {@link java.sql.ResultSetMetaData#isSigned(int)}). */ public static boolean isJdbcSignedType( final MajorType type ) { final boolean isSigned; @@ -477,41 +501,6 @@ public class Types { } } - public static boolean isBytesScalarType(final MajorType type) { - if (type.getMode() == REPEATED) { - return false; - } - switch(type.getMinorType()) { - case FIXEDBINARY: - case VARBINARY: - return true; - default: - return false; - } - } - - public static Comparability getComparability(final MajorType type) { - if (type.getMode() == REPEATED) { - return Comparability.NONE; - } - if (type.getMinorType() == MinorType.LATE) { - return Comparability.UNKNOWN; - } - - switch(type.getMinorType()) { - case LATE: - return Comparability.UNKNOWN; - case MAP: - return Comparability.NONE; - case BIT: - return Comparability.EQUAL; - default: - return Comparability.ORDERED; - } - - } - - public static boolean softEquals(final MajorType a, final MajorType b, final boolean allowNullSwap) { if (a.getMinorType() != b.getMinorType()) { return false; @@ -532,10 +521,6 @@ public class Types { return a.getMode() == b.getMode(); } - public static boolean isLateBind(final MajorType type) { - return type.getMinorType() == MinorType.LATE; - } - public static boolean isUntypedNull(final MajorType type) { return type.getMinorType() == MinorType.NULL; } @@ -572,19 +557,6 @@ public class Types { return MajorType.newBuilder().setMode(DataMode.OPTIONAL).setMinorType(type).build(); } - public static MajorType overrideMinorType(final MajorType originalMajorType, final MinorType overrideMinorType) { - switch (originalMajorType.getMode()) { - case REPEATED: - return repeated(overrideMinorType); - case OPTIONAL: - return optional(overrideMinorType); - case REQUIRED: - return required(overrideMinorType); - default: - throw new UnsupportedOperationException(); - } - } - public static MajorType overrideMode(final MajorType originalMajorType, final DataMode overrideMode) { return withScaleAndPrecision(originalMajorType.getMinorType(), overrideMode, originalMajorType.getScale(), originalMajorType.getPrecision()); } @@ -765,7 +737,7 @@ public class Types { public static MajorType.Builder calculateTypePrecisionAndScale(MajorType leftType, MajorType rightType, MajorType.Builder typeBuilder) { if (leftType.getMinorType().equals(rightType.getMinorType())) { boolean isScalarString = Types.isScalarStringType(leftType) && Types.isScalarStringType(rightType); - boolean isDecimal = CoreDecimalUtility.isDecimalType(leftType); + boolean isDecimal = isDecimalType(leftType); if ((isScalarString || isDecimal) && leftType.hasPrecision() && rightType.hasPrecision()) { typeBuilder.setPrecision(Math.max(leftType.getPrecision(), rightType.getPrecision())); @@ -778,10 +750,6 @@ public class Types { return typeBuilder; } - public static boolean isLaterType(MajorType type) { - return type.getMinorType() == MinorType.LATE; - } - public static boolean isEquivalent(MajorType type1, MajorType type2) { // Requires full type equality, including fields such as precision and scale. @@ -815,10 +783,8 @@ public class Types { // Now it gets slow because subtype lists are not ordered. - List<MinorType> copy1 = new ArrayList<>(); - List<MinorType> copy2 = new ArrayList<>(); - copy1.addAll(subtypes1); - copy2.addAll(subtypes2); + List<MinorType> copy1 = new ArrayList<>(subtypes1); + List<MinorType> copy2 = new ArrayList<>(subtypes2); Collections.sort(copy1); Collections.sort(copy2); return copy1.equals(copy2); diff --git a/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java b/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java deleted file mode 100644 index 0e8a1c7..0000000 --- a/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.common.util; - -import java.math.BigDecimal; - -import org.apache.drill.common.types.TypeProtos; - -public class CoreDecimalUtility { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CoreDecimalUtility.class); - - public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) { - // Truncate or pad to set the input to the correct scale - input = input.setScale(scale, BigDecimal.ROUND_HALF_UP); - - return (input.unscaledValue().longValue()); - } - - public static int getDecimal9FromBigDecimal(BigDecimal input, int scale, int precision) { - // Truncate/ or pad to set the input to the correct scale - input = input.setScale(scale, BigDecimal.ROUND_HALF_UP); - - return (input.unscaledValue().intValue()); - } - - /* - * Helper function to detect if the given data type is Decimal - */ - public static boolean isDecimalType(TypeProtos.MajorType type) { - return isDecimalType(type.getMinorType()); - } - - public static boolean isDecimalType(TypeProtos.MinorType minorType) { - return minorType == TypeProtos.MinorType.VARDECIMAL || - minorType == TypeProtos.MinorType.DECIMAL9 || - minorType == TypeProtos.MinorType.DECIMAL18 || - minorType == TypeProtos.MinorType.DECIMAL28SPARSE || - minorType == TypeProtos.MinorType.DECIMAL38SPARSE; - } -} diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java index 8f34584..5279f28 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java @@ -233,7 +233,7 @@ public class HiveUtilities { final Decimal9Holder holder = new Decimal9Holder(); holder.scale = v.getField().getScale(); holder.precision = v.getField().getPrecision(); - holder.value = DecimalUtility.getDecimal9FromBigDecimal(value, holder.scale, holder.precision); + holder.value = DecimalUtility.getDecimal9FromBigDecimal(value, holder.scale); for (int i = start; i < end; i++) { v.getMutator().setSafe(i, holder); } @@ -246,7 +246,7 @@ public class HiveUtilities { final Decimal18Holder holder = new Decimal18Holder(); holder.scale = v.getField().getScale(); holder.precision = v.getField().getPrecision(); - holder.value = DecimalUtility.getDecimal18FromBigDecimal(value, holder.scale, holder.precision); + holder.value = DecimalUtility.getDecimal18FromBigDecimal(value, holder.scale); for (int i = start; i < end; i++) { v.getMutator().setSafe(i, holder); } @@ -266,7 +266,7 @@ public class HiveUtilities { holder.precision = v.getField().getPrecision(); holder.buffer = managedBuffer; holder.start = 0; - DecimalUtility.getSparseFromBigDecimal(value, holder.buffer, 0, holder.scale, holder.precision, + DecimalUtility.getSparseFromBigDecimal(value, holder.buffer, 0, holder.scale, Decimal28SparseHolder.nDecimalDigits); for (int i = start; i < end; i++) { v.getMutator().setSafe(i, holder); @@ -286,7 +286,7 @@ public class HiveUtilities { holder.precision = v.getField().getPrecision(); holder.buffer = managedBuffer; holder.start = 0; - DecimalUtility.getSparseFromBigDecimal(value, holder.buffer, 0, holder.scale, holder.precision, + DecimalUtility.getSparseFromBigDecimal(value, holder.buffer, 0, holder.scale, Decimal38SparseHolder.nDecimalDigits); for (int i = start; i < end; i++) { v.getMutator().setSafe(i, holder); diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastVarDecimalDecimal.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastVarDecimalDecimal.java index 8ef2bf9..828d486 100644 --- a/exec/java-exec/src/main/codegen/templates/Decimal/CastVarDecimalDecimal.java +++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastVarDecimalDecimal.java @@ -77,14 +77,14 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc { .round(new java.math.MathContext(precision.value, java.math.RoundingMode.HALF_UP)); <#if type.to.endsWith("Decimal9")> - out.value = org.apache.drill.exec.util.DecimalUtility.getDecimal9FromBigDecimal(bd, out.scale, out.precision); + out.value = org.apache.drill.exec.util.DecimalUtility.getDecimal9FromBigDecimal(bd, out.scale); <#elseif type.to.endsWith("Decimal18")> - out.value = org.apache.drill.exec.util.DecimalUtility.getDecimal18FromBigDecimal(bd, out.scale, out.precision); + out.value = org.apache.drill.exec.util.DecimalUtility.getDecimal18FromBigDecimal(bd, out.scale); <#elseif type.to.endsWith("Sparse")> out.start = 0; out.buffer = buffer; org.apache.drill.exec.util.DecimalUtility - .getSparseFromBigDecimal(bd, out.buffer, out.start, out.scale, out.precision, out.nDecimalDigits); + .getSparseFromBigDecimal(bd, out.buffer, out.start, out.scale, out.nDecimalDigits); </#if> } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java index ff38e0c..283953a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java @@ -21,15 +21,12 @@ import java.util.ArrayDeque; import java.util.Arrays; import java.util.Collections; import java.util.Deque; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Queue; -import java.util.Set; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.expression.BooleanOperator; import org.apache.drill.common.expression.CastExpression; @@ -72,7 +69,6 @@ import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; -import org.apache.drill.common.util.CoreDecimalUtility; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.expr.annotations.FunctionTemplate; import org.apache.drill.exec.expr.fn.AbstractFuncHolder; @@ -121,8 +117,7 @@ public class ExpressionTreeMaterializer { public static LogicalExpression materializeFilterExpr(LogicalExpression expr, Map<SchemaPath, ColumnStatistics> fieldTypes, ErrorCollector errorCollector, FunctionLookupContext functionLookupContext) { final FilterMaterializeVisitor filterMaterializeVisitor = new FilterMaterializeVisitor(fieldTypes, errorCollector); - LogicalExpression out = expr.accept(filterMaterializeVisitor, functionLookupContext); - return out; + return expr.accept(filterMaterializeVisitor, functionLookupContext); } /** @@ -211,7 +206,7 @@ public class ExpressionTreeMaterializer { return fromExpr; } - if (CoreDecimalUtility.isDecimalType(toType)) { + if (Types.isDecimalType(toType)) { // Add the scale and precision to the arguments of the implicit cast castArgs.add(new ValueExpressions.IntExpression(toType.getPrecision(), null)); castArgs.add(new ValueExpressions.IntExpression(toType.getScale(), null)); @@ -338,10 +333,6 @@ public class ExpressionTreeMaterializer { * Once this code is more well tested, we will probably remove this flag */ private final boolean unionTypeEnabled; - /** - * Avoid revisiting portions of the tree that have already been materialized - */ - private Set<LogicalExpression> materializedExpressions = Sets.newIdentityHashSet(); public AbstractMaterializeVisitor(ErrorCollector errorCollector, boolean allowComplexWriter, boolean unionTypeEnabled) { this.errorCollector = errorCollector; @@ -430,7 +421,7 @@ public class ExpressionTreeMaterializer { argsWithCast.add(currentArg); } else { //Case 3: insert cast if param type is different from arg type. - if (CoreDecimalUtility.isDecimalType(parmType)) { + if (Types.isDecimalType(parmType)) { // We are implicitly promoting a decimal type, set the required scale and precision parmType = MajorType.newBuilder().setMinorType(parmType.getMinorType()).setMode(parmType.getMode()). setScale(currentArg.getMajorType().getScale()).setPrecision(computePrecision(currentArg)).build(); @@ -468,7 +459,7 @@ public class ExpressionTreeMaterializer { extArgsWithCast.add(currentArg); } else { // Insert cast if param type is different from arg type. - if (CoreDecimalUtility.isDecimalType(parmType)) { + if (Types.isDecimalType(parmType)) { // We are implicitly promoting a decimal type, set the required scale and precision parmType = MajorType.newBuilder().setMinorType(parmType.getMinorType()).setMode(parmType.getMode()). setScale(currentArg.getMajorType().getScale()).setPrecision(computePrecision(currentArg)).build(); @@ -488,17 +479,6 @@ public class ExpressionTreeMaterializer { return NullExpression.INSTANCE; } - private static final Set<String> UNION_FUNCTIONS; - - static { - UNION_FUNCTIONS = new HashSet<>(); - for (MinorType t : MinorType.values()) { - UNION_FUNCTIONS.add("assert_" + t.name().toLowerCase()); - UNION_FUNCTIONS.add("is_" + t.name().toLowerCase()); - } - UNION_FUNCTIONS.add("typeof"); - } - private boolean hasUnionInput(FunctionCall call) { for (LogicalExpression arg : call.args) { if (arg.getMajorType().getMinorType() == MinorType.UNION) { @@ -583,8 +563,7 @@ public class ExpressionTreeMaterializer { QuotedString msg = new QuotedString(message, message.length(), ExpressionPosition.UNKNOWN); List<LogicalExpression> args = Lists.newArrayList(); args.add(msg); - FunctionCall call = new FunctionCall(ExceptionFunction.EXCEPTION_FUNCTION_NAME, args, ExpressionPosition.UNKNOWN); - return call; + return new FunctionCall(ExceptionFunction.EXCEPTION_FUNCTION_NAME, args, ExpressionPosition.UNKNOWN); } /** @@ -602,7 +581,6 @@ public class ExpressionTreeMaterializer { return getExceptionFunction("Unable to cast union to " + type); } String castFuncName = String.format("assert_%s", type.toString()); - Collections.singletonList(arg); return new FunctionCall(castFuncName, Collections.singletonList(arg), ExpressionPosition.UNKNOWN); } @@ -634,7 +612,6 @@ public class ExpressionTreeMaterializer { if (unionTypeEnabled) { if (thenType != elseType && !(thenType == MinorType.NULL || elseType == MinorType.NULL)) { - MinorType leastRestrictive = MinorType.UNION; MajorType.Builder builder = MajorType.newBuilder().setMinorType(MinorType.UNION).setMode(DataMode.OPTIONAL); if (thenType == MinorType.UNION) { for (MinorType subType : conditions.expression.getMajorType().getSubTypeList()) { @@ -726,8 +703,12 @@ public class ExpressionTreeMaterializer { } } - LogicalExpression expr = validateNewExpr(IfExpression.newBuilder().setElse(newElseExpr).setIfCondition(conditions).setOutputType(outputType).build()); - return expr; + return validateNewExpr( + IfExpression.newBuilder() + .setElse(newElseExpr) + .setIfCondition(conditions) + .setOutputType(outputType) + .build()); } private LogicalExpression getConvertToNullableExpr(List<LogicalExpression> args, MinorType minorType, @@ -880,7 +861,7 @@ public class ExpressionTreeMaterializer { List<LogicalExpression> newArgs = Lists.newArrayList(); newArgs.add(input); //input_expr - if (CoreDecimalUtility.isDecimalType(type)) { + if (Types.isDecimalType(type)) { newArgs.add(new ValueExpressions.IntExpression(type.getPrecision(), null)); newArgs.add(new ValueExpressions.IntExpression(type.getScale(), null)); } else if (!Types.isFixedWidthType(type)) { //VarLen type diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinUtils.java index b6604d2..e4dab91 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinUtils.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinUtils.java @@ -25,6 +25,7 @@ import org.apache.drill.common.logical.data.JoinCondition; import org.apache.calcite.rel.RelNode; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.drill.common.types.Types; import org.apache.drill.exec.physical.impl.common.Comparator; import org.apache.drill.exec.planner.logical.DrillAggregateRel; import org.apache.drill.common.exceptions.DrillRuntimeException; @@ -36,7 +37,6 @@ import org.apache.drill.exec.expr.ExpressionTreeMaterializer; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.record.VectorAccessible; import org.apache.drill.exec.resolver.TypeCastRules; -import org.apache.drill.exec.util.DecimalUtility; import java.util.LinkedList; import java.util.List; @@ -123,8 +123,8 @@ public class JoinUtils { // allow implicit cast if both the input types are numeric and any of them is non-decimal // or both of them are decimal if (TypeCastRules.isNumericType(input1) && TypeCastRules.isNumericType(input2) - && ((!DecimalUtility.isDecimalType(input1) && !DecimalUtility.isDecimalType(input2)) - || DecimalUtility.isDecimalType(input1) && DecimalUtility.isDecimalType(input2))) { + && ((!Types.isDecimalType(input1) && !Types.isDecimalType(input2)) + || Types.isDecimalType(input1) && Types.isDecimalType(input2))) { return true; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/TypeInferenceUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/TypeInferenceUtils.java index 529546f..4b5ac7b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/TypeInferenceUtils.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/TypeInferenceUtils.java @@ -46,7 +46,6 @@ import org.apache.drill.common.expression.MajorTypeInLogicalExpression; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.Types; -import org.apache.drill.common.util.CoreDecimalUtility; import org.apache.drill.exec.expr.annotations.FunctionTemplate; import org.apache.drill.exec.expr.fn.DrillFuncHolder; import org.apache.drill.exec.planner.types.DrillRelDataTypeSystem; @@ -57,6 +56,7 @@ import org.apache.drill.exec.resolver.TypeCastRules; import java.util.List; import java.util.Set; +@SuppressWarnings("WeakerAccess") public class TypeInferenceUtils { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TypeInferenceUtils.class); @@ -906,7 +906,7 @@ public class TypeInferenceUtils { .setMode(dataMode) .setMinorType(minorType); - if (CoreDecimalUtility.isDecimalType(minorType)) { + if (Types.isDecimalType(minorType)) { builder .setScale(type.getScale()) .setPrecision(type.getPrecision()); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java index ec78619..114bae4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java @@ -30,6 +30,7 @@ import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.common.types.Types; import org.apache.drill.common.util.DrillVersionInfo; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.exception.OutOfMemoryException; @@ -251,7 +252,7 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter { String name = field.getName(); int length = ParquetTypeHelper.getLengthForMinorType(minorType); PrimitiveTypeName primitiveTypeName = ParquetTypeHelper.getPrimitiveTypeNameForMinorType(minorType); - if (DecimalUtility.isDecimalType(minorType)) { + if (Types.isDecimalType(minorType)) { primitiveTypeName = logicalTypeForDecimals; if (usePrimitiveTypesForDecimals) { if (field.getPrecision() <= ParquetTypeHelper.getMaxPrecisionForPrimitiveType(PrimitiveTypeName.INT32)) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java index 309ed89..cb688fd 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java @@ -20,7 +20,7 @@ package org.apache.drill.exec.store.parquet.columnreaders; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.TypeProtos.MinorType; -import org.apache.drill.common.util.CoreDecimalUtility; +import org.apache.drill.common.types.Types; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.store.parquet.ParquetReaderUtility; @@ -128,7 +128,7 @@ public class ParquetToDrillTypeConverter { MinorType minorType = getMinorType(primitiveTypeName, length, convertedType, options); TypeProtos.MajorType.Builder typeBuilder = TypeProtos.MajorType.newBuilder().setMinorType(minorType).setMode(mode); - if (CoreDecimalUtility.isDecimalType(minorType)) { + if (Types.isDecimalType(minorType)) { int precision = schemaElement.getPrecision(); int scale = schemaElement.getScale(); diff --git a/exec/vector/src/main/codegen/templates/ColumnAccessors.java b/exec/vector/src/main/codegen/templates/ColumnAccessors.java index accf1b2..15349a2 100644 --- a/exec/vector/src/main/codegen/templates/ColumnAccessors.java +++ b/exec/vector/src/main/codegen/templates/ColumnAccessors.java @@ -178,13 +178,11 @@ public class ColumnAccessors { <#elseif drillType == "Decimal9"> return DecimalUtility.getBigDecimalFromPrimitiveTypes( buf.getInt(${getOffset}), - type.getScale(), - type.getPrecision()); + type.getScale()); <#elseif drillType == "Decimal18"> return DecimalUtility.getBigDecimalFromPrimitiveTypes( buf.getLong(${getOffset}), - type.getScale(), - type.getPrecision()); + type.getScale()); <#elseif drillType == "IntervalYear"> return DateUtilities.fromIntervalYear( buf.getInt(${getOffset})); @@ -296,21 +294,21 @@ public class ColumnAccessors { <#elseif drillType == "Decimal9"> drillBuf.setInt(${putOffset}, DecimalUtility.getDecimal9FromBigDecimal(value, - type.getScale(), type.getPrecision())); + type.getScale())); <#elseif drillType == "Decimal18"> drillBuf.setLong(${putOffset}, DecimalUtility.getDecimal18FromBigDecimal(value, - type.getScale(), type.getPrecision())); + type.getScale())); <#elseif drillType == "Decimal38Sparse"> <#-- Hard to optimize this case. Just use the available tools. --> DecimalUtility.getSparseFromBigDecimal(value, drillBuf, ${putOffset}, - type.getScale(), type.getPrecision(), 6); + type.getScale(), 6); <#elseif drillType == "Decimal28Sparse"> <#-- Hard to optimize this case. Just use the available tools. --> DecimalUtility.getSparseFromBigDecimal(value, drillBuf, ${putOffset}, - type.getScale(), type.getPrecision(), 5); + type.getScale(), 5); <#elseif drillType == "IntervalYear"> drillBuf.setInt(${putOffset}, value.getYears() * 12 + value.getMonths()); diff --git a/exec/vector/src/main/codegen/templates/FixedValueVectors.java b/exec/vector/src/main/codegen/templates/FixedValueVectors.java index 4a98c26..eb2c93d 100644 --- a/exec/vector/src/main/codegen/templates/FixedValueVectors.java +++ b/exec/vector/src/main/codegen/templates/FixedValueVectors.java @@ -699,7 +699,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F public void set(int index, BigDecimal value) { DecimalUtility.getSparseFromBigDecimal(value, data, index * VALUE_WIDTH, - field.getScale(), field.getPrecision(), ${minor.nDecimalDigits}); + field.getScale(), ${minor.nDecimalDigits}); } public void setSafe(int index, BigDecimal value) { diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java b/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java index a451b97..f0a06d8 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java @@ -25,142 +25,40 @@ import io.netty.buffer.UnpooledByteBufAllocator; import java.math.BigDecimal; import java.math.BigInteger; import java.math.RoundingMode; -import java.nio.ByteBuffer; -import java.util.Arrays; import org.apache.drill.common.types.TypeProtos; -import org.apache.drill.common.util.CoreDecimalUtility; -import org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers; -import org.apache.drill.exec.expr.holders.Decimal38SparseHolder; -public class DecimalUtility extends CoreDecimalUtility { +@SuppressWarnings("WeakerAccess") +public class DecimalUtility { public final static int MAX_DIGITS = 9; public final static int MAX_DIGITS_INT = 10; public final static int MAX_DIGITS_BIGINT = 19; public final static int DIGITS_BASE = 1000000000; - public final static int DIGITS_MAX = 999999999; - public final static int INTEGER_SIZE = (Integer.SIZE/8); - - public final static String[] decimalToString = {"", - "0", - "00", - "000", - "0000", - "00000", - "000000", - "0000000", - "00000000", - "000000000"}; - - public final static long[] scale_long_constants = { - 1, - 10, - 100, - 1000, - 10000, - 100000, - 1000000, - 10000000, - 100000000, - 1000000000, - 10000000000l, - 100000000000l, - 1000000000000l, - 10000000000000l, - 100000000000000l, - 1000000000000000l, - 10000000000000000l, - 100000000000000000l, - 1000000000000000000l}; - - /* - * Simple function that returns the static precomputed - * power of ten, instead of using Math.pow - */ - public static long getPowerOfTen(int power) { - assert power >= 0 && power < scale_long_constants.length; - return scale_long_constants[(power)]; - } - - /* - * Math.pow returns a double and while multiplying with large digits - * in the decimal data type we encounter noise. So instead of multiplying - * with Math.pow we use the static constants to perform the multiplication - */ - public static long adjustScaleMultiply(long input, int factor) { - int index = Math.abs(factor); - assert index >= 0 && index < scale_long_constants.length; - if (factor >= 0) { - return input * scale_long_constants[index]; - } else { - return input / scale_long_constants[index]; - } - } - - public static long adjustScaleDivide(long input, int factor) { - int index = Math.abs(factor); - assert index >= 0 && index < scale_long_constants.length; - if (factor >= 0) { - return input / scale_long_constants[index]; - } else { - return input * scale_long_constants[index]; - } - } + public final static int INTEGER_SIZE = Integer.SIZE / 8; - /* Given the number of actual digits this function returns the + /** + * Given the number of actual digits this function returns the * number of indexes it will occupy in the array of integers * which are stored in base 1 billion */ public static int roundUp(int ndigits) { - return (ndigits + MAX_DIGITS - 1)/MAX_DIGITS; + return (ndigits + MAX_DIGITS - 1) / MAX_DIGITS; } - /** Returns a string representation of the given integer - * If the length of the given integer is less than the - * passed length, this function will prepend zeroes to the string + /** + * Create a BigDecimal object using the data in the DrillBuf. + * This function assumes that data is provided in a sparse format. */ - public static StringBuilder toStringWithZeroes(int number, int desiredLength) { - String value = ((Integer) number).toString(); - int length = value.length(); - - StringBuilder str = new StringBuilder(); - str.append(decimalToString[desiredLength - length]); - str.append(value); - - return str; - } - - public static StringBuilder toStringWithZeroes(long number, int desiredLength) { - String value = ((Long) number).toString(); - int length = value.length(); - - StringBuilder str = new StringBuilder(); - - // Desired length can be > MAX_DIGITS - int zeroesLength = desiredLength - length; - while (zeroesLength > MAX_DIGITS) { - str.append(decimalToString[MAX_DIGITS]); - zeroesLength -= MAX_DIGITS; - } - str.append(decimalToString[zeroesLength]); - str.append(value); - - return str; - } - - public static BigDecimal getBigDecimalFromIntermediate(ByteBuf data, int startIndex, int nDecimalDigits, int scale) { - - // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false - return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, false); - } - public static BigDecimal getBigDecimalFromSparse(DrillBuf data, int startIndex, int nDecimalDigits, int scale) { - // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, true); } + /** + * Create a BigDecimal object using the data in the DrillBuf. + * This function assumes that data is provided in format supported by {@link BigInteger}. + */ public static BigDecimal getBigDecimalFromDrillBuf(DrillBuf bytebuf, int start, int length, int scale) { byte[] value = new byte[length]; bytebuf.getBytes(start, value, 0, length); @@ -168,14 +66,8 @@ public class DecimalUtility extends CoreDecimalUtility { return new BigDecimal(unscaledValue, scale); } - public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int start, int length, int scale) { - byte[] value = new byte[length]; - bytebuf.get(value); - BigInteger unscaledValue = new BigInteger(value); - return new BigDecimal(unscaledValue, scale); - } - - /** Create a BigDecimal object using the data in the DrillBuf. + /** + * Create a BigDecimal object using the data in the DrillBuf. * This function assumes that data is provided in a non-dense format * It works on both sparse and intermediate representations. */ @@ -183,48 +75,47 @@ public class DecimalUtility extends CoreDecimalUtility { boolean truncateScale) { // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal. - int actualDigits; + int actualDigits = scale % MAX_DIGITS; // Initialize the BigDecimal, first digit in the DrillBuf has the sign so mask it out - BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF); + BigInteger decimalDigits = BigInteger.valueOf(data.getInt(startIndex) & 0x7FFFFFFF); BigInteger base = BigInteger.valueOf(DIGITS_BASE); for (int i = 1; i < nDecimalDigits; i++) { - - BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + (i * INTEGER_SIZE))); - decimalDigits = decimalDigits.multiply(base); - decimalDigits = decimalDigits.add(temp); + BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + i * INTEGER_SIZE)); + decimalDigits = decimalDigits.multiply(base); + decimalDigits = decimalDigits.add(temp); } // Truncate any additional padding we might have added - if (truncateScale == true && scale > 0 && (actualDigits = scale % MAX_DIGITS) != 0) { - BigInteger truncate = BigInteger.valueOf((int)Math.pow(10, (MAX_DIGITS - actualDigits))); - decimalDigits = decimalDigits.divide(truncate); + if (truncateScale && scale > 0 && actualDigits != 0) { + BigInteger truncate = BigInteger.valueOf((int) Math.pow(10, MAX_DIGITS - actualDigits)); + decimalDigits = decimalDigits.divide(truncate); } // set the sign if ((data.getInt(startIndex) & 0x80000000) != 0) { - decimalDigits = decimalDigits.negate(); + decimalDigits = decimalDigits.negate(); } - BigDecimal decimal = new BigDecimal(decimalDigits, scale); - - return decimal; + return new BigDecimal(decimalDigits, scale); } - /* This function returns a BigDecimal object from the dense decimal representation. + /** + * Returns a BigDecimal object from the dense decimal representation. * First step is to convert the dense representation into an intermediate representation * and then invoke getBigDecimalFromDrillBuf() to get the BigDecimal object */ - public static BigDecimal getBigDecimalFromDense(DrillBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) { + public static BigDecimal getBigDecimalFromDense(DrillBuf data, int startIndex, int nDecimalDigits, + int scale, int maxPrecision, int width) { /* This method converts the dense representation to * an intermediate representation. The intermediate * representation has one more integer than the dense * representation. */ - byte[] intermediateBytes = new byte[((nDecimalDigits + 1) * INTEGER_SIZE)]; + byte[] intermediateBytes = new byte[(nDecimalDigits + 1) * INTEGER_SIZE]; // Start storing from the least significant byte of the first integer int intermediateIndex = 3; @@ -236,80 +127,72 @@ public class DecimalUtility extends CoreDecimalUtility { int shiftOrder; byte shiftBits; - // TODO: Some of the logic here is common with casting from Dense to Sparse types, factor out common code if (maxPrecision == 38) { - maskIndex = 0; - shiftOrder = 6; - shiftBits = 0x00; - intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F); + maskIndex = 0; + shiftOrder = 6; + shiftBits = 0x00; + intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F); } else if (maxPrecision == 28) { - maskIndex = 1; - shiftOrder = 4; - shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder); - intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2); + maskIndex = 1; + shiftOrder = 4; + shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder); + intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2); } else { - throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported"); + throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported"); } int inputIndex = 1; boolean sign = false; if ((data.getByte(startIndex) & 0x80) != 0) { - sign = true; + sign = true; } while (inputIndex < width) { - intermediateBytes[intermediateIndex] = (byte) ((shiftBits) | (((data.getByte(startIndex + inputIndex) & reverseMask[maskIndex]) & 0xFF) >>> (8 - shiftOrder))); - shiftBits = (byte) ((data.getByte(startIndex + inputIndex) & mask[maskIndex]) << shiftOrder); inputIndex++; intermediateIndex++; if (((inputIndex - 1) % INTEGER_SIZE) == 0) { - shiftBits = (byte) ((shiftBits & 0xFF) >>> 2); - maskIndex++; - shiftOrder -= 2; + shiftBits = (byte) ((shiftBits & 0xFF) >>> 2); + maskIndex++; + shiftOrder -= 2; } - } /* copy the last byte */ intermediateBytes[intermediateIndex] = shiftBits; - if (sign == true) { - intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80); + if (sign) { + intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80); } final ByteBuf intermediate = UnpooledByteBufAllocator.DEFAULT.buffer(intermediateBytes.length); try { - intermediate.setBytes(0, intermediateBytes); + intermediate.setBytes(0, intermediateBytes); - BigDecimal ret = getBigDecimalFromIntermediate(intermediate, 0, nDecimalDigits + 1, scale); - return ret; + // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false + return getBigDecimalFromDrillBuf(intermediate, 0, nDecimalDigits + 1, scale, false); } finally { intermediate.release(); } - } /** * Function converts the BigDecimal and stores it in out internal sparse representation */ - public static void getSparseFromBigDecimal(BigDecimal input, ByteBuf data, int startIndex, int scale, int precision, - int nDecimalDigits) { + public static void getSparseFromBigDecimal(BigDecimal input, ByteBuf data, int startIndex, int scale, int nDecimalDigits) { // Initialize the buffer - for (int i = 0; i < nDecimalDigits; i++) { - data.setInt(startIndex + (i * INTEGER_SIZE), 0); - } + data.setZero(startIndex, nDecimalDigits * INTEGER_SIZE); boolean sign = false; if (input.signum() == -1) { - // negative input - sign = true; - input = input.abs(); + // negative input + sign = true; + input = input.abs(); } // Truncate the input as per the scale provided @@ -320,25 +203,25 @@ public class DecimalUtility extends CoreDecimalUtility { int destIndex = nDecimalDigits - roundUp(scale) - 1; - // we use base 1 billion integer digits for out integernal representation + // we use base 1 billion integer digits for out internal representation BigDecimal base = new BigDecimal(DIGITS_BASE); while (integerPart.compareTo(BigDecimal.ZERO) == 1) { - // store the modulo as the integer value - data.setInt(startIndex + (destIndex * INTEGER_SIZE), (integerPart.remainder(base)).intValue()); - destIndex--; - // Divide by base 1 billion - integerPart = (integerPart.divide(base)).setScale(0, BigDecimal.ROUND_DOWN); + // store the modulo as the integer value + data.setInt(startIndex + destIndex * INTEGER_SIZE, integerPart.remainder(base).intValue()); + destIndex--; + // Divide by base 1 billion + integerPart = integerPart.divide(base, BigDecimal.ROUND_DOWN).setScale(0, BigDecimal.ROUND_DOWN); } /* Sparse representation contains padding of additional zeroes * so each digit contains MAX_DIGITS for ease of arithmetic */ - int actualDigits; - if ((actualDigits = (scale % MAX_DIGITS)) != 0) { - // Pad additional zeroes - scale = scale + (MAX_DIGITS - actualDigits); - input = input.setScale(scale, BigDecimal.ROUND_DOWN); + int actualDigits = scale % MAX_DIGITS; + if (actualDigits != 0) { + // Pad additional zeroes + scale = scale + MAX_DIGITS - actualDigits; + input = input.setScale(scale, BigDecimal.ROUND_DOWN); } //separate out the fractional part @@ -347,15 +230,15 @@ public class DecimalUtility extends CoreDecimalUtility { destIndex = nDecimalDigits - 1; while (scale > 0) { - // Get next set of MAX_DIGITS (9) store it in the DrillBuf - fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS); - BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE); + // Get next set of MAX_DIGITS (9) store it in the DrillBuf + fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS); + BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE); - data.setInt(startIndex + (destIndex * INTEGER_SIZE), (temp.unscaledValue().intValue())); - destIndex--; + data.setInt(startIndex + destIndex * INTEGER_SIZE, temp.unscaledValue().intValue()); + destIndex--; - fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN); - scale -= MAX_DIGITS; + fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN); + scale -= MAX_DIGITS; } // Set the negative sign @@ -365,93 +248,55 @@ public class DecimalUtility extends CoreDecimalUtility { } /** - * Converts from an input BigDecimal into varying width "VarDecimal" representation. - * The object that manages the "data" is assumed to already have the proper scale set, - * matching that of input.scale(). - * @param input input decimal number to be stored - * @param data destination buffer to store the byte array representation of input - * @param startIndex starting index in data to hold the bytes - * @return startIndex + length of bytes stored (i.e., the next startIndex in the data buffer) + * Returns unsigned long value taken from specified {@link BigDecimal} input with specified scale + * + * @param input {@link BigDecimal} with desired value + * @param scale scale of the value + * @return long value taken from specified {@link BigDecimal} */ - public static int getVarDecimalFromBigDecimal(BigDecimal input, ByteBuf data, int startIndex) { - byte[] bytes = input.unscaledValue().toByteArray(); - int len = bytes.length; - data.setBytes(startIndex, bytes); - return startIndex + len; - } - - public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) { + public static long getDecimal18FromBigDecimal(BigDecimal input, int scale) { // Truncate or pad to set the input to the correct scale input = input.setScale(scale, BigDecimal.ROUND_HALF_UP); return input.unscaledValue().longValue(); } - public static int getDecimal9FromBigDecimal(BigDecimal input, int scale, int precision) { + /** + * Returns unsigned int value taken from specified {@link BigDecimal} input with specified scale. + * + * @param input {@link BigDecimal} with desired value + * @param scale scale of the value + * @return int value taken from specified {@link BigDecimal} + */ + public static int getDecimal9FromBigDecimal(BigDecimal input, int scale) { // Truncate or pad to set the input to the correct scale input = input.setScale(scale, BigDecimal.ROUND_HALF_UP); return input.unscaledValue().intValue(); } - public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale, int precision) { + /** + * Returns {@link BigDecimal} value created from specified integer value with specified scale. + * + * @param input integer value to use for creating of {@link BigDecimal} + * @param scale scale for resulting {@link BigDecimal} + * @return {@link BigDecimal} value + */ + public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale) { return BigDecimal.valueOf(input, scale); } - public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale, int precision) { + /** + * Returns {@link BigDecimal} value created from specified long value with specified scale. + * + * @param input long value to use for creating of {@link BigDecimal} + * @param scale scale for resulting {@link BigDecimal} + * @return {@link BigDecimal} value + */ + public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale) { return BigDecimal.valueOf(input, scale); } - public static int compareDenseBytes(DrillBuf left, int leftStart, boolean leftSign, DrillBuf right, int rightStart, boolean rightSign, int width) { - - int invert = 1; - - /* If signs are different then simply look at the - * sign of the two inputs and determine which is greater - */ - if (leftSign != rightSign) { - - return((leftSign == true) ? -1 : 1); - } else if(leftSign == true) { - /* Both inputs are negative, at the end we will - * have to invert the comparison - */ - invert = -1; - } - - int cmp = 0; - - for (int i = 0; i < width; i++) { - byte leftByte = left.getByte(leftStart + i); - byte rightByte = right.getByte(rightStart + i); - // Unsigned byte comparison - if ((leftByte & 0xFF) > (rightByte & 0xFF)) { - cmp = 1; - break; - } else if ((leftByte & 0xFF) < (rightByte & 0xFF)) { - cmp = -1; - break; - } - } - cmp *= invert; // invert the comparison if both were negative values - - return cmp; - } - - public static int getIntegerFromSparseBuffer(DrillBuf buffer, int start, int index) { - int value = buffer.getInt(start + (index * 4)); - - if (index == 0) { - /* the first byte contains sign bit, return value without it */ - value = (value & 0x7FFFFFFF); - } - return value; - } - - public static void setInteger(DrillBuf buffer, int start, int index, int value) { - buffer.setInt(start + (index * 4), value); - } - /** * Compares two VarDecimal values, still stored in their respective Drill buffers * @@ -497,314 +342,6 @@ public class DecimalUtility extends CoreDecimalUtility { return bdLeft.compareTo(bdRight); } - public static int compareSparseBytes(DrillBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, DrillBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits, boolean absCompare) { - - int invert = 1; - - if (absCompare == false) { - if (leftSign != rightSign) { - return (leftSign == true) ? -1 : 1; - } - - // Both values are negative invert the outcome of the comparison - if (leftSign == true) { - invert = -1; - } - } - - int cmp = compareSparseBytesInner(left, leftStart, leftSign, leftScale, leftPrecision, right, rightStart, rightSign, rightPrecision, rightScale, width, nDecimalDigits); - return cmp * invert; - } - - public static int compareSparseBytesInner(DrillBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, DrillBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits) { - /* compute the number of integer digits in each decimal */ - int leftInt = leftPrecision - leftScale; - int rightInt = rightPrecision - rightScale; - - /* compute the number of indexes required for storing integer digits */ - int leftIntRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(leftInt); - int rightIntRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(rightInt); - - /* compute number of indexes required for storing scale */ - int leftScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(leftScale); - int rightScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(rightScale); - - /* compute index of the most significant integer digits */ - int leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp; - int rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp; - - int leftStopIndex = nDecimalDigits - leftScaleRoundedUp; - int rightStopIndex = nDecimalDigits - rightScaleRoundedUp; - - /* Discard the zeroes in the integer part */ - while (leftIndex1 < leftStopIndex) { - if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) { - break; - } - - /* Digit in this location is zero, decrement the actual number - * of integer digits - */ - leftIntRoundedUp--; - leftIndex1++; - } - - /* If we reached the stop index then the number of integers is zero */ - if (leftIndex1 == leftStopIndex) { - leftIntRoundedUp = 0; - } - - while (rightIndex1 < rightStopIndex) { - if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) { - break; - } - - /* Digit in this location is zero, decrement the actual number - * of integer digits - */ - rightIntRoundedUp--; - rightIndex1++; - } - - if (rightIndex1 == rightStopIndex) { - rightIntRoundedUp = 0; - } - - /* We have the accurate number of non-zero integer digits, - * if the number of integer digits are different then we can determine - * which decimal is larger and needn't go down to comparing individual values - */ - if (leftIntRoundedUp > rightIntRoundedUp) { - return 1; - } - else if (rightIntRoundedUp > leftIntRoundedUp) { - return -1; - } - - /* The number of integer digits are the same, set the each index - * to the first non-zero integer and compare each digit - */ - leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp; - rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp; - - while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) { - if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) { - return 1; - } - else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) { - return -1; - } - - leftIndex1++; - rightIndex1++; - } - - /* The integer part of both the decimal's are equal, now compare - * each individual fractional part. Set the index to be at the - * beginning of the fractional part - */ - leftIndex1 = leftStopIndex; - rightIndex1 = rightStopIndex; - - /* Stop indexes will be the end of the array */ - leftStopIndex = nDecimalDigits; - rightStopIndex = nDecimalDigits; - - /* compare the two fractional parts of the decimal */ - while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) { - if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) { - return 1; - } - else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) { - return -1; - } - - leftIndex1++; - rightIndex1++; - } - - /* Till now the fractional part of the decimals are equal, check - * if one of the decimal has fractional part that is remaining - * and is non-zero - */ - while (leftIndex1 < leftStopIndex) { - if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) { - return 1; - } - leftIndex1++; - } - - while(rightIndex1 < rightStopIndex) { - if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) { - return -1; - } - rightIndex1++; - } - - /* Both decimal values are equal */ - return 0; - } - - public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) { - byte[] value = Arrays.copyOfRange(bytes, start, start + length); - BigInteger unscaledValue = new BigInteger(value); - return new BigDecimal(unscaledValue, scale); - } - - public static void roundDecimal(DrillBuf result, int start, int nDecimalDigits, int desiredScale, int currentScale) { - int newScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(desiredScale); - int origScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(currentScale); - - if (desiredScale < currentScale) { - - boolean roundUp = false; - - //Extract the first digit to be truncated to check if we need to round up - int truncatedScaleIndex = desiredScale + 1; - if (truncatedScaleIndex <= currentScale) { - int extractDigitIndex = nDecimalDigits - origScaleRoundedUp -1; - extractDigitIndex += org.apache.drill.exec.util.DecimalUtility.roundUp(truncatedScaleIndex); - int extractDigit = getIntegerFromSparseBuffer(result, start, extractDigitIndex); - int temp = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (truncatedScaleIndex % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS); - if (temp != 0) { - extractDigit = extractDigit / (int) (Math.pow(10, temp)); - } - if ((extractDigit % 10) > 4) { - roundUp = true; - } - } - - // Get the source index beyond which we will truncate - int srcIntIndex = nDecimalDigits - origScaleRoundedUp - 1; - int srcIndex = srcIntIndex + newScaleRoundedUp; - - // Truncate the remaining fractional part, move the integer part - int destIndex = nDecimalDigits - 1; - if (srcIndex != destIndex) { - while (srcIndex >= 0) { - setInteger(result, start, destIndex--, getIntegerFromSparseBuffer(result, start, srcIndex--)); - } - - // Set the remaining portion of the decimal to be zeroes - while (destIndex >= 0) { - setInteger(result, start, destIndex--, 0); - } - srcIndex = nDecimalDigits - 1; - } - - // We truncated the decimal digit. Now we need to truncate within the base 1 billion fractional digit - int truncateFactor = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (desiredScale % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS); - if (truncateFactor != org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) { - truncateFactor = (int) Math.pow(10, truncateFactor); - int fractionalDigits = getIntegerFromSparseBuffer(result, start, nDecimalDigits - 1); - fractionalDigits /= truncateFactor; - setInteger(result, start, nDecimalDigits - 1, fractionalDigits * truncateFactor); - } - - // Finally round up the digit if needed - if (roundUp == true) { - srcIndex = nDecimalDigits - 1; - int carry; - if (truncateFactor != org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) { - carry = truncateFactor; - } else { - carry = 1; - } - - while (srcIndex >= 0) { - int value = getIntegerFromSparseBuffer(result, start, srcIndex); - value += carry; - - if (value >= org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) { - setInteger(result, start, srcIndex--, value % org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE); - carry = value / org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE; - } else { - setInteger(result, start, srcIndex--, value); - carry = 0; - break; - } - } - } - } else if (desiredScale > currentScale) { - // Add fractional digits to the decimal - - // Check if we need to shift the decimal digits to the left - if (newScaleRoundedUp > origScaleRoundedUp) { - int srcIndex = 0; - int destIndex = newScaleRoundedUp - origScaleRoundedUp; - - // Check while extending scale, we are not overwriting integer part - while (srcIndex < destIndex) { - if (getIntegerFromSparseBuffer(result, start, srcIndex++) != 0) { - throw new org.apache.drill.common.exceptions.DrillRuntimeException("Truncate resulting in loss of integer part, reduce scale specified"); - } - } - - srcIndex = 0; - while (destIndex < nDecimalDigits) { - setInteger(result, start, srcIndex++, getIntegerFromSparseBuffer(result, start, destIndex++)); - } - - // Clear the remaining part - while (srcIndex < nDecimalDigits) { - setInteger(result, start, srcIndex++, 0); - } - } - } - } - - public static int getFirstFractionalDigit(int decimal, int scale) { - if (scale == 0) { - return 0; - } - int temp = (int) adjustScaleDivide(decimal, scale - 1); - return Math.abs(temp % 10); - } - - public static int getFirstFractionalDigit(long decimal, int scale) { - if (scale == 0) { - return 0; - } - long temp = adjustScaleDivide(decimal, scale - 1); - return (int) (Math.abs(temp % 10)); - } - - public static int getFirstFractionalDigit(DrillBuf data, int scale, int start, int nDecimalDigits) { - if (scale == 0) { - return 0; - } - - int index = nDecimalDigits - roundUp(scale); - return (int) (adjustScaleDivide(data.getInt(start + (index * INTEGER_SIZE)), MAX_DIGITS - 1)); - } - - public static int compareSparseSamePrecScale(DrillBuf left, int lStart, byte[] right, int length) { - // check the sign first - boolean lSign = (left.getInt(lStart) & 0x80000000) != 0; - boolean rSign = ByteFunctionHelpers.getSign(right); - int cmp = 0; - - if (lSign != rSign) { - return (lSign == false) ? 1 : -1; - } - - // invert the comparison if we are comparing negative numbers - int invert = (lSign == true) ? -1 : 1; - - // compare byte by byte - int n = 0; - while (n < length/4) { - int leftInt = Decimal38SparseHolder.getInteger(n, lStart, left); - int rightInt = ByteFunctionHelpers.getInteger(right, n); - if (leftInt != rightInt) { - cmp = (leftInt - rightInt ) > 0 ? 1 : -1; - break; - } - n++; - } - return cmp * invert; - } - /** * Returns max length of byte array, required to store value with specified precision. * @@ -817,6 +354,13 @@ public class DecimalUtility extends CoreDecimalUtility { return 0; } if (precision < 300) { // normal case, use exact heuristic formula + // Math.pow(10, precision) - 1 returns max integer value for the specified precision, example 999 for precision 3 + // Math.log(Math.pow(10, precision) - 1) / Math.log(2) is just log with base 2 to calculate size + // in bits for max integer value mentioned above + // Math.log(Math.pow(10, precision) - 1) / Math.log(2) + 1 in this expression was added 1 to the count of bits to + // take into account case with negative value + // size in bits was divided by size of byte to calculate bytes count required to store value + // with the specified precision return (int) Math.ceil((Math.log(Math.pow(10, precision) - 1) / Math.log(2) + 1) / Byte.SIZE); } else { // for values greater than 304 Math.pow(10, precision) returns Infinity, therefore 1 is neglected in Math.log() diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java index d1cc3b4..73027aa 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java @@ -127,7 +127,7 @@ public class ValueHolderHelper { byte[] b = s.getBytes(Charsets.UTF_8); vch.start = 0; vch.end = b.length; - vch.buffer = a.buffer(b.length); // + vch.buffer = a.buffer(b.length); vch.buffer.setBytes(0, b); return vch; } @@ -180,7 +180,7 @@ public class ValueHolderHelper { dch.start = 0; dch.buffer = buf.reallocIfNeeded(5 * DecimalUtility.INTEGER_SIZE); DecimalUtility - .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits); + .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, Decimal28SparseHolder.nDecimalDigits); return dch; } @@ -195,9 +195,9 @@ public class ValueHolderHelper { dch.precision = bigDecimal.precision(); Decimal38SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer); dch.start = 0; - dch.buffer = buf.reallocIfNeeded(dch.maxPrecision * DecimalUtility.INTEGER_SIZE); + dch.buffer = buf.reallocIfNeeded(Decimal38SparseHolder.maxPrecision * DecimalUtility.INTEGER_SIZE); DecimalUtility - .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits); + .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, Decimal38SparseHolder.nDecimalDigits); return dch; } diff --git a/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java b/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java index 2ef809a..538e6c8 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java +++ b/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java @@ -28,7 +28,6 @@ import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; -import org.apache.drill.common.util.CoreDecimalUtility; import com.google.common.collect.Iterators; @@ -94,22 +93,6 @@ public class ValueExpressions { return new IntervalDayExpression(intervalInMillis); } - public static LogicalExpression getDecimal9(BigDecimal i) { - return new Decimal9Expression(i, ExpressionPosition.UNKNOWN); - } - - public static LogicalExpression getDecimal18(BigDecimal i) { - return new Decimal18Expression(i, ExpressionPosition.UNKNOWN); - } - - public static LogicalExpression getDecimal28(BigDecimal i) { - return new Decimal28Expression(i, ExpressionPosition.UNKNOWN); - } - - public static LogicalExpression getDecimal38(BigDecimal i) { - return new Decimal38Expression(i, ExpressionPosition.UNKNOWN); - } - public static LogicalExpression getVarDecimal(BigDecimal input, int precision, int scale) { return new VarDecimalExpression(input, precision, scale, ExpressionPosition.UNKNOWN); } @@ -265,7 +248,7 @@ public class ValueExpressions { super(pos); this.scale = input.scale(); this.precision = input.precision(); - this.decimal = CoreDecimalUtility.getDecimal9FromBigDecimal(input, scale, precision); + this.decimal = input.setScale(scale, BigDecimal.ROUND_HALF_UP).intValue(); } @@ -307,7 +290,7 @@ public class ValueExpressions { super(pos); this.scale = input.scale(); this.precision = input.precision(); - this.decimal = CoreDecimalUtility.getDecimal18FromBigDecimal(input, scale, precision); + this.decimal = input.setScale(scale, BigDecimal.ROUND_HALF_UP).longValue(); } -- To stop receiving notification emails like this one, please contact [email protected].
