Author: jitendra
Date: Mon Mar 17 20:00:53 2014
New Revision: 1578538
URL: http://svn.apache.org/r1578538
Log:
HIVE-6664. Vectorized variance computation differs from row mode computation.
(jitendra, reviewed by Eric Hanson)
Modified:
hive/trunk/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt
hive/trunk/ql/src/test/results/clientpositive/vector_decimal_aggregate.q.out
Modified:
hive/trunk/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt?rev=1578538&r1=1578537&r2=1578538&view=diff
==============================================================================
--- hive/trunk/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt
(original)
+++ hive/trunk/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt
Mon Mar 17 20:00:53 2014
@@ -54,7 +54,7 @@ public class <ClassName> extends VectorA
private static final long serialVersionUID = 1L;
- transient private final Decimal128 sum;
+ transient private double sum;
transient private long count;
transient private double variance;
@@ -64,12 +64,11 @@ public class <ClassName> extends VectorA
transient private boolean isNull = true;
public Aggregation() {
- sum = new Decimal128();
}
public void init() {
isNull = false;
- sum.zeroClear();
+ sum = 0f;
count = 0;
variance = 0f;
}
@@ -79,28 +78,25 @@ public class <ClassName> extends VectorA
throw new UnsupportedOperationException();
}
- public void updateValueWithCheckAndInit(Decimal128 scratch, Decimal128
value, short scale) {
+ public void updateValueWithCheckAndInit(Decimal128 value, short scale) {
if (this.isNull) {
this.init();
}
- this.sum.addDestructive(value, scale);
+
+ double dval = value.doubleValue();
+ this.sum += dval;
this.count += 1;
if(this.count > 1) {
- scratch.update(count);
- scratch.multiplyDestructive(value, scale);
- scratch.subtractDestructive(sum, scale);
- double t = scratch.doubleValue();
+ double t = this.count*dval - this.sum;
this.variance += (t*t) / ((double)this.count*(this.count-1));
}
}
- public void updateValueNoCheck(Decimal128 scratch, Decimal128 value,
short scale) {
- this.sum.addDestructive(value, scale);
+ public void updateValueNoCheck(Decimal128 value, short scale) {
+ double dval = value.doubleValue();
+ this.sum += dval;
this.count += 1;
- scratch.update(count);
- scratch.multiplyDestructive(value, scale);
- scratch.subtractDestructive(sum, scale);
- double t = scratch.doubleValue();
+ double t = this.count*dval - this.sum;
this.variance += (t*t) / ((double)this.count*(this.count-1));
}
@@ -114,9 +110,6 @@ public class <ClassName> extends VectorA
transient private ObjectInspector soi;
- transient private final Decimal128 scratchDecimal;
-
-
public <ClassName>(VectorExpression inputExpression) {
this();
this.inputExpression = inputExpression;
@@ -132,7 +125,6 @@ public class <ClassName> extends VectorA
partialResult[1] = resultSum;
partialResult[2] = resultVariance;
initPartialResultInspector();
- scratchDecimal = new Decimal128();
}
private void initPartialResultInspector() {
@@ -218,7 +210,7 @@ public class <ClassName> extends VectorA
aggregationBufferSets,
aggregateIndex,
i);
- myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
+ myagg.updateValueWithCheckAndInit(value, scale);
}
}
@@ -239,7 +231,7 @@ public class <ClassName> extends VectorA
int i = selected[j];
if (!isNull[i]) {
Decimal128 value = vector[i];
- myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
+ myagg.updateValueWithCheckAndInit(value, scale);
}
}
}
@@ -258,7 +250,7 @@ public class <ClassName> extends VectorA
aggregateIndex,
i);
Decimal128 value = vector[selected[i]];
- myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
+ myagg.updateValueWithCheckAndInit(value, scale);
}
}
@@ -277,7 +269,7 @@ public class <ClassName> extends VectorA
aggregateIndex,
i);
Decimal128 value = vector[i];
- myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
+ myagg.updateValueWithCheckAndInit(value, scale);
}
}
}
@@ -295,7 +287,7 @@ public class <ClassName> extends VectorA
aggregateIndex,
i);
Decimal128 value = vector[i];
- myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
+ myagg.updateValueWithCheckAndInit(value, scale);
}
}
@@ -347,11 +339,11 @@ public class <ClassName> extends VectorA
// TODO: conjure a formula w/o iterating
//
- myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
+ myagg.updateValueWithCheckAndInit(value, scale);
// We pulled out i=0 so we can remove the count > 1 check in the loop
for (int i=1; i<batchSize; ++i) {
- myagg.updateValueNoCheck(scratchDecimal, value, scale);
+ myagg.updateValueNoCheck(value, scale);
}
}
@@ -367,7 +359,7 @@ public class <ClassName> extends VectorA
int i = selected[j];
if (!isNull[i]) {
Decimal128 value = vector[i];
- myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
+ myagg.updateValueWithCheckAndInit(value, scale);
}
}
}
@@ -384,13 +376,13 @@ public class <ClassName> extends VectorA
}
Decimal128 value = vector[selected[0]];
- myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
+ myagg.updateValueWithCheckAndInit(value, scale);
// i=0 was pulled out to remove the count > 1 check in the loop
//
for (int i=1; i< batchSize; ++i) {
value = vector[selected[i]];
- myagg.updateValueNoCheck(scratchDecimal, value, scale);
+ myagg.updateValueNoCheck(value, scale);
}
}
@@ -404,7 +396,7 @@ public class <ClassName> extends VectorA
for(int i=0;i<batchSize;++i) {
if (!isNull[i]) {
Decimal128 value = vector[i];
- myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
+ myagg.updateValueWithCheckAndInit(value, scale);
}
}
}
@@ -420,12 +412,12 @@ public class <ClassName> extends VectorA
}
Decimal128 value = vector[0];
- myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
+ myagg.updateValueWithCheckAndInit(value, scale);
// i=0 was pulled out to remove count > 1 check
for (int i=1; i<batchSize; ++i) {
value = vector[i];
- myagg.updateValueNoCheck(scratchDecimal, value, scale);
+ myagg.updateValueNoCheck(value, scale);
}
}
@@ -449,9 +441,9 @@ public class <ClassName> extends VectorA
}
else {
assert(0 < myagg.count);
- resultCount.set (myagg.count);
- resultSum.set (myagg.sum.doubleValue());
- resultVariance.set (myagg.variance);
+ resultCount.set(myagg.count);
+ resultSum.set(myagg.sum);
+ resultVariance.set(myagg.variance);
return partialResult;
}
}
@@ -482,4 +474,4 @@ public class <ClassName> extends VectorA
public void setInputExpression(VectorExpression inputExpression) {
this.inputExpression = inputExpression;
}
-}
\ No newline at end of file
+}
Modified:
hive/trunk/ql/src/test/results/clientpositive/vector_decimal_aggregate.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/vector_decimal_aggregate.q.out?rev=1578538&r1=1578537&r2=1578538&view=diff
==============================================================================
---
hive/trunk/ql/src/test/results/clientpositive/vector_decimal_aggregate.q.out
(original)
+++
hive/trunk/ql/src/test/results/clientpositive/vector_decimal_aggregate.q.out
Mon Mar 17 20:00:53 2014
@@ -99,11 +99,11 @@ POSTHOOK: query: SELECT cint,
POSTHOOK: type: QUERY
POSTHOOK: Input: default@decimal_vgby
#### A masked pattern was here ####
-NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808
1633.60810810806667 5695.4830821353335 5696.410307714474 3072
11160.715384615385 -5147.907692307693 6010604.3076923073536
1956.576923076922966667 6821.495748565141 6822.606289190906
+NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808
1633.60810810806667 5695.483082135364 5696.4103077145055 3072
11160.715384615385 -5147.907692307693 6010604.3076923073536
1956.576923076922966667 6821.495748565159 6822.606289190924
-3728 6 5831542.269248378 -3367.6517567568
5817556.0411483778 969592.67352472963333 2174330.2092403853
2381859.406131774 6 6984454.211097692 -4033.445769230769
6967702.8672438458471 1161283.811207307641183333 2604201.2704476737
2852759.5602156054
--563 2 -515.621072973 -3367.6517567568 -3883.2728297298
-1941.6364148649 1426.0153418919 2016.6902366556312 2
-617.5607769230769 -4033.445769230769 -4651.0065461538459
-2325.50327307692295 1707.9424961538462 2415.395441814127
-762 2 5831542.269248378 1531.2194054054 5833073.4886537834
2916536.7443268917 2915005.524921486 4122440.347736469 2
6984454.211097692 1833.9456923076925 6986288.1567899996925
3493144.07839499984625 3491310.132702692 4937458.140118757
-6981 3 5831542.269248378 -515.621072973 5830511.027102432
1943503.67570081066667 2749258.4550124914 3367140.192906513 3
6984454.211097692 -617.5607769230769 6983219.0895438458462
2327739.696514615282066667 3292794.4113115156 4032833.0678006653
+-563 2 -515.621072973 -3367.6517567568 -3883.2728297298
-1941.6364148649 1426.0153418918999 2016.6902366556308 2
-617.5607769230769 -4033.445769230769 -4651.0065461538459
-2325.50327307692295 1707.9424961538462 2415.395441814127
+762 2 5831542.269248378 1531.2194054054 5833073.4886537834
2916536.7443268917 2915005.5249214866 4122440.3477364695 2
6984454.211097692 1833.9456923076925 6986288.1567899996925
3493144.07839499984625 3491310.1327026924 4937458.140118758
+6981 3 5831542.269248378 -515.621072973 5830511.027102432
1943503.67570081066667 2749258.455012492 3367140.1929065133 3
6984454.211097692 -617.5607769230769 6983219.0895438458462
2327739.696514615282066667 3292794.4113115156 4032833.0678006653
253665376 1024 9767.0054054054 -9779.5486486487
-347484.0818378374 -339.33992366976309 5708.9563478862
5711.745967572779 1024 11697.969230769231 -11712.99230769231
-416182.64030769233089 -406.428359675480791885 6837.632716002934
6840.973851172274
-528534767 1024 5831542.269248378 -9777.1594594595
11646372.8607481068 11373.41099682432305 257528.9298820665
257654.76860439766 1024 6984454.211097692 -11710.130769230771
13948892.79980307629003 13621.965624807691689482 308443.1074570801
308593.82484083984
-626923679 1024 9723.4027027027 -9778.9513513514
10541.0525297287 10.29399661106318 5742.09145323734
5744.897264034267 1024 11645.746153846154 -11712.276923076923
12625.04759999997746 12.329148046874977988 6877.318722794877
6880.679250101604
+528534767 1024 5831542.269248378 -9777.1594594595
11646372.8607481068 11373.41099682432305 257528.92988206653
257654.7686043977 1024 6984454.211097692 -11710.130769230771
13948892.79980307629003 13621.965624807691689482 308443.1074570801
308593.82484083984
+626923679 1024 9723.4027027027 -9778.9513513514
10541.0525297287 10.29399661106318 5742.09145323734
5744.897264034267 1024 11645.746153846154 -11712.276923076923
12625.04759999997746 12.329148046874977988 6877.318722794877
6880.679250101603
