Author: jitendra
Date: Sat Mar 8 23:31:13 2014
New Revision: 1575633
URL: http://svn.apache.org/r1575633
Log:
HIVE-6508 : Mismatched aggregation results between vector and non-vector mode
with decimal field (Remus Rusanu via jitendra)
Modified:
hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java
hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt
hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFSumDecimal.java
hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java
Modified:
hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java?rev=1575633&r1=1575632&r2=1575633&view=diff
==============================================================================
---
hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java
(original)
+++
hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java
Sat Mar 8 23:31:13 2014
@@ -261,8 +261,19 @@ public final class Decimal128 extends Nu
* object to copy from
*/
public Decimal128 update(Decimal128 o) {
+ update(o, o.scale);
+ return this;
+ }
+
+ /**
+ * Copy the value of given object and assigns a custom scale.
+ *
+ * @param o
+ * object to copy from
+ */
+ public Decimal128 update(Decimal128 o, short scale) {
this.unscaledValue.update(o.unscaledValue);
- this.scale = o.scale;
+ this.scale = scale;
this.signum = o.signum;
return this;
}
Modified:
hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt?rev=1575633&r1=1575632&r2=1575633&view=diff
==============================================================================
---
hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt
(original)
+++
hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt
Sat Mar 8 23:31:13 2014
@@ -60,12 +60,12 @@ public class <ClassName> extends VectorA
value = new Decimal128();
}
- public void checkValue(Decimal128 value) {
+ public void checkValue(Decimal128 value, short scale) {
if (isNull) {
isNull = false;
this.value.update(value);
} else if (this.value.compareTo(value) <OperatorSymbol> 0) {
- this.value.update(value);
+ this.value.update(value, scale);
}
}
@@ -124,16 +124,16 @@ public class <ClassName> extends VectorA
if (inputVector.isRepeating) {
iterateNoNullsRepeatingWithAggregationSelection(
aggregationBufferSets, aggregrateIndex,
- vector[0], batchSize);
+ vector[0], inputVector.scale, batchSize);
} else {
if (batch.selectedInUse) {
iterateNoNullsSelectionWithAggregationSelection(
aggregationBufferSets, aggregrateIndex,
- vector, batch.selected, batchSize);
+ vector, inputVector.scale, batch.selected, batchSize);
} else {
iterateNoNullsWithAggregationSelection(
aggregationBufferSets, aggregrateIndex,
- vector, batchSize);
+ vector, inputVector.scale, batchSize);
}
}
} else {
@@ -141,21 +141,21 @@ public class <ClassName> extends VectorA
if (batch.selectedInUse) {
iterateHasNullsRepeatingSelectionWithAggregationSelection(
aggregationBufferSets, aggregrateIndex,
- vector[0], batchSize, batch.selected, inputVector.isNull);
+ vector[0], inputVector.scale, batchSize, batch.selected,
inputVector.isNull);
} else {
iterateHasNullsRepeatingWithAggregationSelection(
aggregationBufferSets, aggregrateIndex,
- vector[0], batchSize, inputVector.isNull);
+ vector[0], inputVector.scale, batchSize, inputVector.isNull);
}
} else {
if (batch.selectedInUse) {
iterateHasNullsSelectionWithAggregationSelection(
aggregationBufferSets, aggregrateIndex,
- vector, batchSize, batch.selected, inputVector.isNull);
+ vector, inputVector.scale, batchSize, batch.selected,
inputVector.isNull);
} else {
iterateHasNullsWithAggregationSelection(
aggregationBufferSets, aggregrateIndex,
- vector, batchSize, inputVector.isNull);
+ vector, inputVector.scale, batchSize, inputVector.isNull);
}
}
}
@@ -165,6 +165,7 @@ public class <ClassName> extends VectorA
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregrateIndex,
Decimal128 value,
+ short scale,
int batchSize) {
for (int i=0; i < batchSize; ++i) {
@@ -172,7 +173,7 @@ public class <ClassName> extends VectorA
aggregationBufferSets,
aggregrateIndex,
i);
- myagg.checkValue(value);
+ myagg.checkValue(value, scale);
}
}
@@ -180,6 +181,7 @@ public class <ClassName> extends VectorA
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregrateIndex,
Decimal128[] values,
+ short scale,
int[] selection,
int batchSize) {
@@ -188,7 +190,7 @@ public class <ClassName> extends VectorA
aggregationBufferSets,
aggregrateIndex,
i);
- myagg.checkValue(values[selection[i]]);
+ myagg.checkValue(values[selection[i]], scale);
}
}
@@ -196,13 +198,14 @@ public class <ClassName> extends VectorA
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregrateIndex,
Decimal128[] values,
+ short scale,
int batchSize) {
for (int i=0; i < batchSize; ++i) {
Aggregation myagg = getCurrentAggregationBuffer(
aggregationBufferSets,
aggregrateIndex,
i);
- myagg.checkValue(values[i]);
+ myagg.checkValue(values[i], scale);
}
}
@@ -210,6 +213,7 @@ public class <ClassName> extends VectorA
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregrateIndex,
Decimal128 value,
+ short scale,
int batchSize,
int[] selection,
boolean[] isNull) {
@@ -220,7 +224,7 @@ public class <ClassName> extends VectorA
aggregationBufferSets,
aggregrateIndex,
i);
- myagg.checkValue(value);
+ myagg.checkValue(value, scale);
}
}
@@ -230,6 +234,7 @@ public class <ClassName> extends VectorA
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregrateIndex,
Decimal128 value,
+ short scale,
int batchSize,
boolean[] isNull) {
@@ -239,7 +244,7 @@ public class <ClassName> extends VectorA
aggregationBufferSets,
aggregrateIndex,
i);
- myagg.checkValue(value);
+ myagg.checkValue(value, scale);
}
}
}
@@ -248,6 +253,7 @@ public class <ClassName> extends VectorA
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregrateIndex,
Decimal128[] values,
+ short scale,
int batchSize,
int[] selection,
boolean[] isNull) {
@@ -259,7 +265,7 @@ public class <ClassName> extends VectorA
aggregationBufferSets,
aggregrateIndex,
j);
- myagg.checkValue(values[i]);
+ myagg.checkValue(values[i], scale);
}
}
}
@@ -268,6 +274,7 @@ public class <ClassName> extends VectorA
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregrateIndex,
Decimal128[] values,
+ short scale,
int batchSize,
boolean[] isNull) {
@@ -277,7 +284,7 @@ public class <ClassName> extends VectorA
aggregationBufferSets,
aggregrateIndex,
i);
- myagg.checkValue(values[i]);
+ myagg.checkValue(values[i], scale);
}
}
}
@@ -305,28 +312,31 @@ public class <ClassName> extends VectorA
if (inputVector.noNulls &&
(myagg.isNull || (myagg.value.compareTo(vector[0])
<OperatorSymbol> 0))) {
myagg.isNull = false;
- myagg.value.update(vector[0]);
+ myagg.value.update(vector[0], inputVector.scale);
}
return;
}
if (!batch.selectedInUse && inputVector.noNulls) {
- iterateNoSelectionNoNulls(myagg, vector, batchSize);
+ iterateNoSelectionNoNulls(myagg, vector, inputVector.scale,
batchSize);
}
else if (!batch.selectedInUse) {
- iterateNoSelectionHasNulls(myagg, vector, batchSize,
inputVector.isNull);
+ iterateNoSelectionHasNulls(myagg, vector, inputVector.scale,
+ batchSize, inputVector.isNull);
}
else if (inputVector.noNulls){
- iterateSelectionNoNulls(myagg, vector, batchSize, batch.selected);
+ iterateSelectionNoNulls(myagg, vector, inputVector.scale, batchSize,
batch.selected);
}
else {
- iterateSelectionHasNulls(myagg, vector, batchSize,
inputVector.isNull, batch.selected);
+ iterateSelectionHasNulls(myagg, vector, inputVector.scale,
+ batchSize, inputVector.isNull, batch.selected);
}
}
private void iterateSelectionHasNulls(
Aggregation myagg,
Decimal128[] vector,
+ short scale,
int batchSize,
boolean[] isNull,
int[] selected) {
@@ -340,7 +350,7 @@ public class <ClassName> extends VectorA
myagg.value.update(value);
}
else if (myagg.value.compareTo(value) <OperatorSymbol> 0) {
- myagg.value.update(value);
+ myagg.value.update(value, scale);
}
}
}
@@ -349,6 +359,7 @@ public class <ClassName> extends VectorA
private void iterateSelectionNoNulls(
Aggregation myagg,
Decimal128[] vector,
+ short scale,
int batchSize,
int[] selected) {
@@ -360,7 +371,7 @@ public class <ClassName> extends VectorA
for (int i=0; i< batchSize; ++i) {
Decimal128 value = vector[selected[i]];
if (myagg.value.compareTo(value) <OperatorSymbol> 0) {
- myagg.value.update(value);
+ myagg.value.update(value, scale);
}
}
}
@@ -368,6 +379,7 @@ public class <ClassName> extends VectorA
private void iterateNoSelectionHasNulls(
Aggregation myagg,
Decimal128[] vector,
+ short scale,
int batchSize,
boolean[] isNull) {
@@ -375,11 +387,11 @@ public class <ClassName> extends VectorA
if (!isNull[i]) {
Decimal128 value = vector[i];
if (myagg.isNull) {
- myagg.value.update(value);
+ myagg.value.update(value, scale);
myagg.isNull = false;
}
else if (myagg.value.compareTo(value) <OperatorSymbol> 0) {
- myagg.value.update(value);
+ myagg.value.update(value, scale);
}
}
}
@@ -388,6 +400,7 @@ public class <ClassName> extends VectorA
private void iterateNoSelectionNoNulls(
Aggregation myagg,
Decimal128[] vector,
+ short scale,
int batchSize) {
if (myagg.isNull) {
myagg.value.update(vector[0]);
@@ -397,7 +410,7 @@ public class <ClassName> extends VectorA
for (int i=0;i<batchSize;++i) {
Decimal128 value = vector[i];
if (myagg.value.compareTo(value) <OperatorSymbol> 0) {
- myagg.value.update(value);
+ myagg.value.update(value, scale);
}
}
}
Modified:
hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt?rev=1575633&r1=1575632&r2=1575633&view=diff
==============================================================================
---
hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt
(original)
+++
hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt
Sat Mar 8 23:31:13 2014
@@ -79,27 +79,27 @@ public class <ClassName> extends VectorA
throw new UnsupportedOperationException();
}
- public void updateValueWithCheckAndInit(Decimal128 scratch, Decimal128
value) {
+ public void updateValueWithCheckAndInit(Decimal128 scratch, Decimal128
value, short scale) {
if (this.isNull) {
this.init();
}
- this.sum.addDestructive(value, value.getScale());
+ this.sum.addDestructive(value, scale);
this.count += 1;
if(this.count > 1) {
scratch.update(count);
- scratch.multiplyDestructive(value, value.getScale());
- scratch.subtractDestructive(sum, sum.getScale());
+ scratch.multiplyDestructive(value, scale);
+ scratch.subtractDestructive(sum, scale);
double t = scratch.doubleValue();
this.variance += (t*t) / ((double)this.count*(this.count-1));
}
}
- public void updateValueNoCheck(Decimal128 scratch, Decimal128 value) {
- this.sum.addDestructive(value, value.getScale());
+ public void updateValueNoCheck(Decimal128 scratch, Decimal128 value,
short scale) {
+ this.sum.addDestructive(value, scale);
this.count += 1;
scratch.update(count);
- scratch.multiplyDestructive(value, value.getScale());
- scratch.subtractDestructive(sum, sum.getScale());
+ scratch.multiplyDestructive(value, scale);
+ scratch.subtractDestructive(sum, scale);
double t = scratch.doubleValue();
this.variance += (t*t) / ((double)this.count*(this.count-1));
}
@@ -181,24 +181,26 @@ public class <ClassName> extends VectorA
if (inputVector.isRepeating) {
if (inputVector.noNulls || !inputVector.isNull[0]) {
iterateRepeatingNoNullsWithAggregationSelection(
- aggregationBufferSets, aggregateIndex, vector[0], batchSize);
+ aggregationBufferSets, aggregateIndex, vector[0],
inputVector.scale, batchSize);
}
}
else if (!batch.selectedInUse && inputVector.noNulls) {
iterateNoSelectionNoNullsWithAggregationSelection(
- aggregationBufferSets, aggregateIndex, vector, batchSize);
+ aggregationBufferSets, aggregateIndex, vector, inputVector.scale,
batchSize);
}
else if (!batch.selectedInUse) {
iterateNoSelectionHasNullsWithAggregationSelection(
- aggregationBufferSets, aggregateIndex, vector, batchSize,
inputVector.isNull);
+ aggregationBufferSets, aggregateIndex, vector, inputVector.scale,
+ batchSize, inputVector.isNull);
}
else if (inputVector.noNulls){
iterateSelectionNoNullsWithAggregationSelection(
- aggregationBufferSets, aggregateIndex, vector, batchSize,
batch.selected);
+ aggregationBufferSets, aggregateIndex, vector, inputVector.scale,
+ batchSize, batch.selected);
}
else {
iterateSelectionHasNullsWithAggregationSelection(
- aggregationBufferSets, aggregateIndex, vector, batchSize,
+ aggregationBufferSets, aggregateIndex, vector, inputVector.scale,
batchSize,
inputVector.isNull, batch.selected);
}
@@ -208,6 +210,7 @@ public class <ClassName> extends VectorA
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregateIndex,
Decimal128 value,
+ short scale,
int batchSize) {
for (int i=0; i<batchSize; ++i) {
@@ -215,7 +218,7 @@ public class <ClassName> extends VectorA
aggregationBufferSets,
aggregateIndex,
i);
- myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+ myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
}
}
@@ -223,6 +226,7 @@ public class <ClassName> extends VectorA
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregateIndex,
Decimal128[] vector,
+ short scale,
int batchSize,
boolean[] isNull,
int[] selected) {
@@ -235,7 +239,7 @@ public class <ClassName> extends VectorA
int i = selected[j];
if (!isNull[i]) {
Decimal128 value = vector[i];
- myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+ myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
}
}
}
@@ -244,6 +248,7 @@ public class <ClassName> extends VectorA
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregateIndex,
Decimal128[] vector,
+ short scale,
int batchSize,
int[] selected) {
@@ -253,7 +258,7 @@ public class <ClassName> extends VectorA
aggregateIndex,
i);
Decimal128 value = vector[selected[i]];
- myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+ myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
}
}
@@ -261,6 +266,7 @@ public class <ClassName> extends VectorA
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregateIndex,
Decimal128[] vector,
+ short scale,
int batchSize,
boolean[] isNull) {
@@ -271,7 +277,7 @@ public class <ClassName> extends VectorA
aggregateIndex,
i);
Decimal128 value = vector[i];
- myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+ myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
}
}
}
@@ -280,6 +286,7 @@ public class <ClassName> extends VectorA
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregateIndex,
Decimal128[] vector,
+ short scale,
int batchSize) {
for (int i=0; i<batchSize; ++i) {
@@ -288,7 +295,7 @@ public class <ClassName> extends VectorA
aggregateIndex,
i);
Decimal128 value = vector[i];
- myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+ myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
}
}
@@ -313,42 +320,45 @@ public class <ClassName> extends VectorA
if (inputVector.isRepeating) {
if (inputVector.noNulls) {
- iterateRepeatingNoNulls(myagg, vector[0], batchSize);
+ iterateRepeatingNoNulls(myagg, vector[0], inputVector.scale,
batchSize);
}
}
else if (!batch.selectedInUse && inputVector.noNulls) {
- iterateNoSelectionNoNulls(myagg, vector, batchSize);
+ iterateNoSelectionNoNulls(myagg, vector, inputVector.scale, batchSize);
}
else if (!batch.selectedInUse) {
- iterateNoSelectionHasNulls(myagg, vector, batchSize,
inputVector.isNull);
+ iterateNoSelectionHasNulls(myagg, vector, inputVector.scale,
batchSize, inputVector.isNull);
}
else if (inputVector.noNulls){
- iterateSelectionNoNulls(myagg, vector, batchSize, batch.selected);
+ iterateSelectionNoNulls(myagg, vector, inputVector.scale, batchSize,
batch.selected);
}
else {
- iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull,
batch.selected);
+ iterateSelectionHasNulls(myagg, vector, inputVector.scale,
+ batchSize, inputVector.isNull, batch.selected);
}
}
private void iterateRepeatingNoNulls(
Aggregation myagg,
Decimal128 value,
+ short scale,
int batchSize) {
// TODO: conjure a formula w/o iterating
//
- myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+ myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
// We pulled out i=0 so we can remove the count > 1 check in the loop
for (int i=1; i<batchSize; ++i) {
- myagg.updateValueNoCheck(scratchDecimal, value);
+ myagg.updateValueNoCheck(scratchDecimal, value, scale);
}
}
private void iterateSelectionHasNulls(
Aggregation myagg,
Decimal128[] vector,
+ short scale,
int batchSize,
boolean[] isNull,
int[] selected) {
@@ -357,7 +367,7 @@ public class <ClassName> extends VectorA
int i = selected[j];
if (!isNull[i]) {
Decimal128 value = vector[i];
- myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+ myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
}
}
}
@@ -365,6 +375,7 @@ public class <ClassName> extends VectorA
private void iterateSelectionNoNulls(
Aggregation myagg,
Decimal128[] vector,
+ short scale,
int batchSize,
int[] selected) {
@@ -373,26 +384,27 @@ public class <ClassName> extends VectorA
}
Decimal128 value = vector[selected[0]];
- myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+ myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
// i=0 was pulled out to remove the count > 1 check in the loop
//
for (int i=1; i< batchSize; ++i) {
value = vector[selected[i]];
- myagg.updateValueNoCheck(scratchDecimal, value);
+ myagg.updateValueNoCheck(scratchDecimal, value, scale);
}
}
private void iterateNoSelectionHasNulls(
Aggregation myagg,
Decimal128[] vector,
+ short scale,
int batchSize,
boolean[] isNull) {
for(int i=0;i<batchSize;++i) {
if (!isNull[i]) {
Decimal128 value = vector[i];
- myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+ myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
}
}
}
@@ -400,6 +412,7 @@ public class <ClassName> extends VectorA
private void iterateNoSelectionNoNulls(
Aggregation myagg,
Decimal128[] vector,
+ short scale,
int batchSize) {
if (myagg.isNull) {
@@ -407,12 +420,12 @@ public class <ClassName> extends VectorA
}
Decimal128 value = vector[0];
- myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+ myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
// i=0 was pulled out to remove count > 1 check
for (int i=1; i<batchSize; ++i) {
value = vector[i];
- myagg.updateValueNoCheck(scratchDecimal, value);
+ myagg.updateValueNoCheck(scratchDecimal, value, scale);
}
}
Modified:
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFSumDecimal.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFSumDecimal.java?rev=1575633&r1=1575632&r2=1575633&view=diff
==============================================================================
---
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFSumDecimal.java
(original)
+++
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFSumDecimal.java
Sat Mar 8 23:31:13 2014
@@ -51,12 +51,12 @@ public class VectorUDAFSumDecimal extend
transient private Decimal128 sum = new Decimal128();
transient private boolean isNull;
- public void sumValue(Decimal128 value) {
+ public void sumValue(Decimal128 value, short scale) {
if (isNull) {
- sum.update(value);
+ sum.update(value, scale);
isNull = false;
} else {
- sum.addDestructive(value, value.getScale());
+ sum.addDestructive(value, scale);
}
}
@@ -110,16 +110,19 @@ public class VectorUDAFSumDecimal extend
if (inputVector.isRepeating) {
iterateNoNullsRepeatingWithAggregationSelection(
aggregationBufferSets, aggregateIndex,
- vector[0], batchSize);
+ vector[0], inputVector.scale,
+ batchSize);
} else {
if (batch.selectedInUse) {
iterateNoNullsSelectionWithAggregationSelection(
aggregationBufferSets, aggregateIndex,
- vector, batch.selected, batchSize);
+ vector, inputVector.scale,
+ batch.selected, batchSize);
} else {
iterateNoNullsWithAggregationSelection(
aggregationBufferSets, aggregateIndex,
- vector, batchSize);
+ vector, inputVector.scale,
+ batchSize);
}
}
} else {
@@ -127,21 +130,25 @@ public class VectorUDAFSumDecimal extend
if (batch.selectedInUse) {
iterateHasNullsRepeatingSelectionWithAggregationSelection(
aggregationBufferSets, aggregateIndex,
- vector[0], batchSize, batch.selected, inputVector.isNull);
+ vector[0], inputVector.scale,
+ batchSize, batch.selected, inputVector.isNull);
} else {
iterateHasNullsRepeatingWithAggregationSelection(
aggregationBufferSets, aggregateIndex,
- vector[0], batchSize, inputVector.isNull);
+ vector[0], inputVector.scale,
+ batchSize, inputVector.isNull);
}
} else {
if (batch.selectedInUse) {
iterateHasNullsSelectionWithAggregationSelection(
aggregationBufferSets, aggregateIndex,
- vector, batchSize, batch.selected, inputVector.isNull);
+ vector, inputVector.scale,
+ batchSize, batch.selected, inputVector.isNull);
} else {
iterateHasNullsWithAggregationSelection(
aggregationBufferSets, aggregateIndex,
- vector, batchSize, inputVector.isNull);
+ vector,inputVector.scale,
+ batchSize, inputVector.isNull);
}
}
}
@@ -151,6 +158,7 @@ public class VectorUDAFSumDecimal extend
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregateIndex,
Decimal128 value,
+ short scale,
int batchSize) {
for (int i=0; i < batchSize; ++i) {
@@ -158,7 +166,7 @@ public class VectorUDAFSumDecimal extend
aggregationBufferSets,
aggregateIndex,
i);
- myagg.sumValue(value);
+ myagg.sumValue(value, scale);
}
}
@@ -166,6 +174,7 @@ public class VectorUDAFSumDecimal extend
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregateIndex,
Decimal128[] values,
+ short scale,
int[] selection,
int batchSize) {
@@ -174,7 +183,7 @@ public class VectorUDAFSumDecimal extend
aggregationBufferSets,
aggregateIndex,
i);
- myagg.sumValue(values[selection[i]]);
+ myagg.sumValue(values[selection[i]], scale);
}
}
@@ -182,13 +191,14 @@ public class VectorUDAFSumDecimal extend
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregateIndex,
Decimal128[] values,
+ short scale,
int batchSize) {
for (int i=0; i < batchSize; ++i) {
Aggregation myagg = getCurrentAggregationBuffer(
aggregationBufferSets,
aggregateIndex,
i);
- myagg.sumValue(values[i]);
+ myagg.sumValue(values[i], scale);
}
}
@@ -196,6 +206,7 @@ public class VectorUDAFSumDecimal extend
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregateIndex,
Decimal128 value,
+ short scale,
int batchSize,
int[] selection,
boolean[] isNull) {
@@ -206,7 +217,7 @@ public class VectorUDAFSumDecimal extend
aggregationBufferSets,
aggregateIndex,
i);
- myagg.sumValue(value);
+ myagg.sumValue(value, scale);
}
}
@@ -216,6 +227,7 @@ public class VectorUDAFSumDecimal extend
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregateIndex,
Decimal128 value,
+ short scale,
int batchSize,
boolean[] isNull) {
@@ -225,7 +237,7 @@ public class VectorUDAFSumDecimal extend
aggregationBufferSets,
aggregateIndex,
i);
- myagg.sumValue(value);
+ myagg.sumValue(value, scale);
}
}
}
@@ -234,6 +246,7 @@ public class VectorUDAFSumDecimal extend
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregateIndex,
Decimal128[] values,
+ short scale,
int batchSize,
int[] selection,
boolean[] isNull) {
@@ -245,7 +258,7 @@ public class VectorUDAFSumDecimal extend
aggregationBufferSets,
aggregateIndex,
j);
- myagg.sumValue(values[i]);
+ myagg.sumValue(values[i], scale);
}
}
}
@@ -254,6 +267,7 @@ public class VectorUDAFSumDecimal extend
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregateIndex,
Decimal128[] values,
+ short scale,
int batchSize,
boolean[] isNull) {
@@ -263,7 +277,7 @@ public class VectorUDAFSumDecimal extend
aggregationBufferSets,
aggregateIndex,
i);
- myagg.sumValue(values[i]);
+ myagg.sumValue(values[i], scale);
}
}
}
@@ -295,29 +309,30 @@ public class VectorUDAFSumDecimal extend
myagg.sum.zeroClear();
}
scratchDecimal.update(batchSize);
- scratchDecimal.multiplyDestructive(vector[0], vector[0].getScale());
+ scratchDecimal.multiplyDestructive(vector[0], inputVector.scale);
myagg.sum.update(scratchDecimal);
}
return;
}
if (!batch.selectedInUse && inputVector.noNulls) {
- iterateNoSelectionNoNulls(myagg, vector, batchSize);
+ iterateNoSelectionNoNulls(myagg, vector, inputVector.scale, batchSize);
}
else if (!batch.selectedInUse) {
- iterateNoSelectionHasNulls(myagg, vector, batchSize,
inputVector.isNull);
+ iterateNoSelectionHasNulls(myagg, vector, inputVector.scale,
batchSize, inputVector.isNull);
}
else if (inputVector.noNulls){
- iterateSelectionNoNulls(myagg, vector, batchSize, batch.selected);
+ iterateSelectionNoNulls(myagg, vector, inputVector.scale, batchSize,
batch.selected);
}
else {
- iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull,
batch.selected);
+ iterateSelectionHasNulls(myagg, vector, inputVector.scale, batchSize,
inputVector.isNull, batch.selected);
}
}
private void iterateSelectionHasNulls(
Aggregation myagg,
Decimal128[] vector,
+ short scale,
int batchSize,
boolean[] isNull,
int[] selected) {
@@ -330,7 +345,7 @@ public class VectorUDAFSumDecimal extend
myagg.isNull = false;
myagg.sum.zeroClear();
}
- myagg.sum.addDestructive(value, value.getScale());
+ myagg.sum.addDestructive(value, scale);
}
}
}
@@ -338,6 +353,7 @@ public class VectorUDAFSumDecimal extend
private void iterateSelectionNoNulls(
Aggregation myagg,
Decimal128[] vector,
+ short scale,
int batchSize,
int[] selected) {
@@ -348,13 +364,14 @@ public class VectorUDAFSumDecimal extend
for (int i=0; i< batchSize; ++i) {
Decimal128 value = vector[selected[i]];
- myagg.sum.addDestructive(value, value.getScale());
+ myagg.sum.addDestructive(value, scale);
}
}
private void iterateNoSelectionHasNulls(
Aggregation myagg,
Decimal128[] vector,
+ short scale,
int batchSize,
boolean[] isNull) {
@@ -365,7 +382,7 @@ public class VectorUDAFSumDecimal extend
myagg.sum.zeroClear();
myagg.isNull = false;
}
- myagg.sum.addDestructive(value, value.getScale());
+ myagg.sum.addDestructive(value, scale);
}
}
}
@@ -373,6 +390,7 @@ public class VectorUDAFSumDecimal extend
private void iterateNoSelectionNoNulls(
Aggregation myagg,
Decimal128[] vector,
+ short scale,
int batchSize) {
if (myagg.isNull) {
myagg.sum.zeroClear();
@@ -381,7 +399,7 @@ public class VectorUDAFSumDecimal extend
for (int i=0;i<batchSize;++i) {
Decimal128 value = vector[i];
- myagg.sum.addDestructive(value, value.getScale());
+ myagg.sum.addDestructive(value, scale);
}
}
Modified:
hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java?rev=1575633&r1=1575632&r2=1575633&view=diff
==============================================================================
---
hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
(original)
+++
hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
Sat Mar 8 23:31:13 2014
@@ -603,6 +603,7 @@ public class TestVectorGroupByOperator {
@Test
public void testCountDecimal() throws HiveException {
testAggregateDecimal(
+ "Decimal",
"count",
2,
Arrays.asList(new Object[]{
@@ -615,6 +616,7 @@ public class TestVectorGroupByOperator {
@Test
public void testMaxDecimal() throws HiveException {
testAggregateDecimal(
+ "Decimal",
"max",
2,
Arrays.asList(new Object[]{
@@ -623,26 +625,29 @@ public class TestVectorGroupByOperator {
new Decimal128(3)}),
new Decimal128(3));
testAggregateDecimal(
- "max",
- 2,
- Arrays.asList(new Object[]{
- new Decimal128(3),
- new Decimal128(2),
- new Decimal128(1)}),
- new Decimal128(3));
+ "Decimal",
+ "max",
+ 2,
+ Arrays.asList(new Object[]{
+ new Decimal128(3),
+ new Decimal128(2),
+ new Decimal128(1)}),
+ new Decimal128(3));
testAggregateDecimal(
- "max",
- 2,
- Arrays.asList(new Object[]{
- new Decimal128(2),
- new Decimal128(3),
- new Decimal128(1)}),
- new Decimal128(3));
+ "Decimal",
+ "max",
+ 2,
+ Arrays.asList(new Object[]{
+ new Decimal128(2),
+ new Decimal128(3),
+ new Decimal128(1)}),
+ new Decimal128(3));
}
@Test
public void testMinDecimal() throws HiveException {
testAggregateDecimal(
+ "Decimal",
"min",
2,
Arrays.asList(new Object[]{
@@ -651,28 +656,31 @@ public class TestVectorGroupByOperator {
new Decimal128(3)}),
new Decimal128(1));
testAggregateDecimal(
- "min",
- 2,
- Arrays.asList(new Object[]{
- new Decimal128(3),
- new Decimal128(2),
- new Decimal128(1)}),
- new Decimal128(1));
+ "Decimal",
+ "min",
+ 2,
+ Arrays.asList(new Object[]{
+ new Decimal128(3),
+ new Decimal128(2),
+ new Decimal128(1)}),
+ new Decimal128(1));
testAggregateDecimal(
- "min",
- 2,
- Arrays.asList(new Object[]{
- new Decimal128(2),
- new Decimal128(1),
- new Decimal128(3)}),
- new Decimal128(1));
+ "Decimal",
+ "min",
+ 2,
+ Arrays.asList(new Object[]{
+ new Decimal128(2),
+ new Decimal128(1),
+ new Decimal128(3)}),
+ new Decimal128(1));
}
@Test
public void testSumDecimal() throws HiveException {
testAggregateDecimal(
- "sum",
+ "Decimal",
+ "sum",
2,
Arrays.asList(new Object[]{
new Decimal128(1),
@@ -682,8 +690,25 @@ public class TestVectorGroupByOperator {
}
@Test
+ public void testSumDecimalHive6508() throws HiveException {
+ short scale = 4;
+ testAggregateDecimal(
+ "Decimal(10,4)",
+ "sum",
+ 4,
+ Arrays.asList(new Object[]{
+ new Decimal128("1234.2401", scale),
+ new Decimal128("1868.52", scale),
+ new Decimal128(0L, (short) 0),
+ new Decimal128("456.84", scale),
+ new Decimal128("121.89", scale)}),
+ new Decimal128("3681.4901", scale));
+ }
+
+ @Test
public void testAvgDecimal() throws HiveException {
testAggregateDecimal(
+ "Decimal",
"avg",
2,
Arrays.asList(new Object[]{
@@ -696,18 +721,20 @@ public class TestVectorGroupByOperator {
@Test
public void testAvgDecimalNegative() throws HiveException {
testAggregateDecimal(
- "avg",
- 2,
- Arrays.asList(new Object[]{
- new Decimal128(-1),
- new Decimal128(-2),
- new Decimal128(-3)}),
- HiveDecimal.create((-1-2-3)/3));
+ "Decimal",
+ "avg",
+ 2,
+ Arrays.asList(new Object[]{
+ new Decimal128(-1),
+ new Decimal128(-2),
+ new Decimal128(-3)}),
+ HiveDecimal.create((-1-2-3)/3));
}
@Test
public void testVarianceDecimal () throws HiveException {
testAggregateDecimal(
+ "Decimal",
"variance",
2,
Arrays.asList(new Object[]{
@@ -721,6 +748,7 @@ public class TestVectorGroupByOperator {
@Test
public void testVarSampDecimal () throws HiveException {
testAggregateDecimal(
+ "Decimal",
"var_samp",
2,
Arrays.asList(new Object[]{
@@ -734,6 +762,7 @@ public class TestVectorGroupByOperator {
@Test
public void testStdPopDecimal () throws HiveException {
testAggregateDecimal(
+ "Decimal",
"stddev_pop",
2,
Arrays.asList(new Object[]{
@@ -747,6 +776,7 @@ public class TestVectorGroupByOperator {
@Test
public void testStdSampDecimal () throws HiveException {
testAggregateDecimal(
+ "Decimal",
"stddev_samp",
2,
Arrays.asList(new Object[]{
@@ -1936,14 +1966,15 @@ public class TestVectorGroupByOperator {
}
public void testAggregateDecimal (
- String aggregateName,
- int batchSize,
- Iterable<Object> values,
- Object expected) throws HiveException {
+ String typeName,
+ String aggregateName,
+ int batchSize,
+ Iterable<Object> values,
+ Object expected) throws HiveException {
@SuppressWarnings("unchecked")
FakeVectorRowBatchFromObjectIterables fdr = new
FakeVectorRowBatchFromObjectIterables(
- batchSize, new String[] {"Decimal"}, values);
+ batchSize, new String[] {typeName}, values);
testAggregateDecimalIterable (aggregateName, fdr, expected);
}
Modified:
hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java?rev=1575633&r1=1575632&r2=1575633&view=diff
==============================================================================
---
hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java
(original)
+++
hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java
Sat Mar 8 23:31:13 2014
@@ -22,6 +22,9 @@ import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.regex.MatchResult;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.hadoop.hive.common.type.Decimal128;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
@@ -141,7 +144,23 @@ public class FakeVectorRowBatchFromObjec
}
};
} else if (types[i].toLowerCase().startsWith("decimal")) {
- batch.cols[i] = new DecimalColumnVector(batchSize, 38, 0);
+ Pattern decimalPattern = Pattern.compile(
+ "decimal(?:\\((\\d+)(?:\\,(\\d+))?\\))?",
Pattern.CASE_INSENSITIVE);
+ Matcher mr = decimalPattern.matcher(types[i]);
+ int precission = 38;
+ int scale = 0;
+ if (mr.matches()) {
+ String typePrecission = mr.group(1);
+ if (typePrecission != null) {
+ precission = Integer.parseInt(typePrecission);
+ }
+ String typeScale = mr.group(2);
+ if (typeScale != null) {
+ scale = Integer.parseInt(typeScale);
+ }
+ }
+
+ batch.cols[i] = new DecimalColumnVector(batchSize, precission,
scale);
columnAssign[i] = new ColumnVectorAssign() {
@Override
public void assign(
