Author: hashutosh
Date: Thu Sep 18 16:12:10 2014
New Revision: 1626018
URL: http://svn.apache.org/r1626018
Log:
HIVE-8092 : Vectorized Tez count(*) returns NULL instead of 0 when result is
empty (Matt McCline via Gopal V)
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountMerge.java
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountStar.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
hive/trunk/ql/src/test/queries/clientpositive/vectorization_short_regress.q
hive/trunk/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
hive/trunk/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java?rev=1626018&r1=1626017&r2=1626018&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
Thu Sep 18 16:12:10 2014
@@ -48,6 +48,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import
org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
import
org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFAvgDecimal;
import
org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFCount;
+import
org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFCountMerge;
import
org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFCountStar;
import
org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFSumDecimal;
import
org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFAvgDouble;
@@ -1898,7 +1899,7 @@ public class VectorizationContext {
add(new AggregateDefinition("max",
VectorExpressionDescriptor.ArgumentType.DECIMAL, null,
VectorUDAFMaxDecimal.class));
add(new AggregateDefinition("count",
VectorExpressionDescriptor.ArgumentType.NONE, GroupByDesc.Mode.HASH,
VectorUDAFCountStar.class));
add(new AggregateDefinition("count",
VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFCount.class));
- add(new AggregateDefinition("count",
VectorExpressionDescriptor.ArgumentType.INT_FAMILY,
GroupByDesc.Mode.MERGEPARTIAL, VectorUDAFSumLong.class));
+ add(new AggregateDefinition("count",
VectorExpressionDescriptor.ArgumentType.INT_FAMILY,
GroupByDesc.Mode.MERGEPARTIAL, VectorUDAFCountMerge.class));
add(new AggregateDefinition("count",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFCount.class));
add(new AggregateDefinition("count",
VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFCount.class));
add(new AggregateDefinition("count",
VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH,
VectorUDAFCount.class));
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java?rev=1626018&r1=1626017&r2=1626018&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java
Thu Sep 18 16:12:10 2014
@@ -46,15 +46,7 @@ public class VectorUDAFCount extends Vec
private static final long serialVersionUID = 1L;
- transient private long value;
- transient private boolean isNull;
-
- public void initIfNull() {
- if (isNull) {
- isNull = false;
- value = 0;
- }
- }
+ transient private long count;
@Override
public int getVariableSize() {
@@ -63,8 +55,7 @@ public class VectorUDAFCount extends Vec
@Override
public void reset() {
- isNull = true;
- value = 0L;
+ count = 0L;
}
}
@@ -131,8 +122,7 @@ public class VectorUDAFCount extends Vec
aggregationBufferSets,
aggregateIndex,
i);
- myagg.initIfNull();
- myagg.value++;
+ myagg.count++;
}
}
@@ -148,8 +138,7 @@ public class VectorUDAFCount extends Vec
aggregationBufferSets,
aggregateIndex,
i);
- myagg.initIfNull();
- myagg.value++;
+ myagg.count++;
}
}
}
@@ -168,8 +157,7 @@ public class VectorUDAFCount extends Vec
aggregationBufferSets,
aggregateIndex,
j);
- myagg.initIfNull();
- myagg.value++;
+ myagg.count++;
}
}
}
@@ -191,17 +179,15 @@ public class VectorUDAFCount extends Vec
Aggregation myagg = (Aggregation)agg;
- myagg.initIfNull();
-
if (inputVector.isRepeating) {
if (inputVector.noNulls || !inputVector.isNull[0]) {
- myagg.value += batchSize;
+ myagg.count += batchSize;
}
return;
}
if (inputVector.noNulls) {
- myagg.value += batchSize;
+ myagg.count += batchSize;
return;
}
else if (!batch.selectedInUse) {
@@ -221,7 +207,7 @@ public class VectorUDAFCount extends Vec
for (int j=0; j< batchSize; ++j) {
int i = selected[j];
if (!isNull[i]) {
- myagg.value += 1;
+ myagg.count += 1;
}
}
}
@@ -233,7 +219,7 @@ public class VectorUDAFCount extends Vec
for (int i=0; i< batchSize; ++i) {
if (!isNull[i]) {
- myagg.value += 1;
+ myagg.count += 1;
}
}
}
@@ -251,14 +237,9 @@ public class VectorUDAFCount extends Vec
@Override
public Object evaluateOutput(AggregationBuffer agg) throws HiveException {
- Aggregation myagg = (Aggregation) agg;
- if (myagg.isNull) {
- return null;
- }
- else {
- result.set (myagg.value);
+ Aggregation myagg = (Aggregation) agg;
+ result.set (myagg.count);
return result;
- }
}
@Override
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountMerge.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountMerge.java?rev=1626018&view=auto
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountMerge.java
(added)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountMerge.java
Thu Sep 18 16:12:10 2014
@@ -0,0 +1,400 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.AggregationDesc;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.io.LongWritable;
+
+
+/**
+ * VectorUDAFCountMerge. Vectorized implementation for COUNT aggregate on
reduce-side (merge).
+ */
+@Description(name = "count", value = "_FUNC_(expr) - Returns the merged
sum value of expr (vectorized, type: long)")
+
+public class VectorUDAFCountMerge extends VectorAggregateExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * class for storing the current aggregate value.
+ */
+ static class Aggregation implements AggregationBuffer {
+
+ private static final long serialVersionUID = 1L;
+
+ transient private long value;
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void reset() {
+ value = 0L;
+ }
+ }
+
+ private VectorExpression inputExpression = null;
+ transient private final LongWritable result;
+
+ public VectorUDAFCountMerge(VectorExpression inputExpression) {
+ this();
+ this.inputExpression = inputExpression;
+ }
+
+ public VectorUDAFCountMerge() {
+ super();
+ result = new LongWritable(0);
+ }
+
+ private Aggregation getCurrentAggregationBuffer(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int aggregateIndex,
+ int row) {
+ VectorAggregationBufferRow mySet = aggregationBufferSets[row];
+ Aggregation myagg = (Aggregation)
mySet.getAggregationBuffer(aggregateIndex);
+ return myagg;
+ }
+
+ @Override
+ public void aggregateInputSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int aggregateIndex,
+ VectorizedRowBatch batch) throws HiveException {
+
+ int batchSize = batch.size;
+
+ if (batchSize == 0) {
+ return;
+ }
+
+ inputExpression.evaluate(batch);
+
+ LongColumnVector inputVector = (LongColumnVector)batch.
+ cols[this.inputExpression.getOutputColumn()];
+ long[] vector = inputVector.vector;
+
+ if (inputVector.noNulls) {
+ if (inputVector.isRepeating) {
+ iterateNoNullsRepeatingWithAggregationSelection(
+ aggregationBufferSets, aggregateIndex,
+ vector[0], batchSize);
+ } else {
+ if (batch.selectedInUse) {
+ iterateNoNullsSelectionWithAggregationSelection(
+ aggregationBufferSets, aggregateIndex,
+ vector, batch.selected, batchSize);
+ } else {
+ iterateNoNullsWithAggregationSelection(
+ aggregationBufferSets, aggregateIndex,
+ vector, batchSize);
+ }
+ }
+ } else {
+ if (inputVector.isRepeating) {
+ if (batch.selectedInUse) {
+ iterateHasNullsRepeatingSelectionWithAggregationSelection(
+ aggregationBufferSets, aggregateIndex,
+ vector[0], batchSize, batch.selected, inputVector.isNull);
+ } else {
+ iterateHasNullsRepeatingWithAggregationSelection(
+ aggregationBufferSets, aggregateIndex,
+ vector[0], batchSize, inputVector.isNull);
+ }
+ } else {
+ if (batch.selectedInUse) {
+ iterateHasNullsSelectionWithAggregationSelection(
+ aggregationBufferSets, aggregateIndex,
+ vector, batchSize, batch.selected, inputVector.isNull);
+ } else {
+ iterateHasNullsWithAggregationSelection(
+ aggregationBufferSets, aggregateIndex,
+ vector, batchSize, inputVector.isNull);
+ }
+ }
+ }
+ }
+
+ private void iterateNoNullsRepeatingWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int aggregateIndex,
+ long value,
+ int batchSize) {
+
+ for (int i=0; i < batchSize; ++i) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ aggregateIndex,
+ i);
+ myagg.value += value;
+ }
+ }
+
+ private void iterateNoNullsSelectionWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int aggregateIndex,
+ long[] values,
+ int[] selection,
+ int batchSize) {
+
+ for (int i=0; i < batchSize; ++i) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ aggregateIndex,
+ i);
+ myagg.value += values[selection[i]];
+ }
+ }
+
+ private void iterateNoNullsWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int aggregateIndex,
+ long[] values,
+ int batchSize) {
+ for (int i=0; i < batchSize; ++i) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ aggregateIndex,
+ i);
+ myagg.value += values[i];
+ }
+ }
+
+ private void iterateHasNullsRepeatingSelectionWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int aggregateIndex,
+ long value,
+ int batchSize,
+ int[] selection,
+ boolean[] isNull) {
+
+ for (int i=0; i < batchSize; ++i) {
+ if (!isNull[selection[i]]) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ aggregateIndex,
+ i);
+ myagg.value += value;
+ }
+ }
+
+ }
+
+ private void iterateHasNullsRepeatingWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int aggregateIndex,
+ long value,
+ int batchSize,
+ boolean[] isNull) {
+
+ for (int i=0; i < batchSize; ++i) {
+ if (!isNull[i]) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ aggregateIndex,
+ i);
+ myagg.value += value;
+ }
+ }
+ }
+
+ private void iterateHasNullsSelectionWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int aggregateIndex,
+ long[] values,
+ int batchSize,
+ int[] selection,
+ boolean[] isNull) {
+
+ for (int j=0; j < batchSize; ++j) {
+ int i = selection[j];
+ if (!isNull[i]) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ aggregateIndex,
+ j);
+ myagg.value += values[i];
+ }
+ }
+ }
+
+ private void iterateHasNullsWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int aggregateIndex,
+ long[] values,
+ int batchSize,
+ boolean[] isNull) {
+
+ for (int i=0; i < batchSize; ++i) {
+ if (!isNull[i]) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ aggregateIndex,
+ i);
+ myagg.value += values[i];
+ }
+ }
+ }
+
+ @Override
+ public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch)
+ throws HiveException {
+
+ inputExpression.evaluate(batch);
+
+ LongColumnVector inputVector = (LongColumnVector)batch.
+ cols[this.inputExpression.getOutputColumn()];
+
+ int batchSize = batch.size;
+
+ if (batchSize == 0) {
+ return;
+ }
+
+ Aggregation myagg = (Aggregation)agg;
+
+ long[] vector = inputVector.vector;
+
+ if (inputVector.isRepeating) {
+ if (inputVector.noNulls) {
+ myagg.value += vector[0]*batchSize;
+ }
+ return;
+ }
+
+ if (!batch.selectedInUse && inputVector.noNulls) {
+ iterateNoSelectionNoNulls(myagg, vector, batchSize);
+ }
+ else if (!batch.selectedInUse) {
+ iterateNoSelectionHasNulls(myagg, vector, batchSize,
inputVector.isNull);
+ }
+ else if (inputVector.noNulls){
+ iterateSelectionNoNulls(myagg, vector, batchSize, batch.selected);
+ }
+ else {
+ iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull,
batch.selected);
+ }
+ }
+
+ private void iterateSelectionHasNulls(
+ Aggregation myagg,
+ long[] vector,
+ int batchSize,
+ boolean[] isNull,
+ int[] selected) {
+
+ for (int j=0; j< batchSize; ++j) {
+ int i = selected[j];
+ if (!isNull[i]) {
+ myagg.value += vector[i];
+ }
+ }
+ }
+
+ private void iterateSelectionNoNulls(
+ Aggregation myagg,
+ long[] vector,
+ int batchSize,
+ int[] selected) {
+
+ for (int i=0; i< batchSize; ++i) {
+ myagg.value += vector[selected[i]];
+ }
+ }
+
+ private void iterateNoSelectionHasNulls(
+ Aggregation myagg,
+ long[] vector,
+ int batchSize,
+ boolean[] isNull) {
+
+ for(int i=0;i<batchSize;++i) {
+ if (!isNull[i]) {
+ myagg.value += vector[i];
+ }
+ }
+ }
+
+ private void iterateNoSelectionNoNulls(
+ Aggregation myagg,
+ long[] vector,
+ int batchSize) {
+
+ for (int i=0;i<batchSize;++i) {
+ myagg.value += vector[i];
+ }
+ }
+
+ @Override
+ public AggregationBuffer getNewAggregationBuffer() throws HiveException {
+ return new Aggregation();
+ }
+
+ @Override
+ public void reset(AggregationBuffer agg) throws HiveException {
+ Aggregation myAgg = (Aggregation) agg;
+ myAgg.reset();
+ }
+
+ @Override
+ public Object evaluateOutput(AggregationBuffer agg) throws HiveException {
+ Aggregation myagg = (Aggregation) agg;
+ result.set (myagg.value);
+ return result;
+ }
+
+ @Override
+ public ObjectInspector getOutputObjectInspector() {
+ return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
+ }
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2() +
+ model.primitive1(),
+ model.memoryAlign());
+ }
+
+ @Override
+ public void init(AggregationDesc desc) throws HiveException {
+ // No-op
+ }
+
+ public VectorExpression getInputExpression() {
+ return inputExpression;
+ }
+
+ public void setInputExpression(VectorExpression inputExpression) {
+ this.inputExpression = inputExpression;
+ }
+}
+
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountStar.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountStar.java?rev=1626018&r1=1626017&r2=1626018&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountStar.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountStar.java
Thu Sep 18 16:12:10 2014
@@ -44,8 +44,7 @@ public class VectorUDAFCountStar extends
private static final long serialVersionUID = 1L;
- transient private long value;
- transient private boolean isNull;
+ transient private long count;
@Override
public int getVariableSize() {
@@ -54,8 +53,7 @@ public class VectorUDAFCountStar extends
@Override
public void reset() {
- isNull = true;
- value = 0L;
+ count = 0L;
}
}
@@ -95,8 +93,7 @@ public class VectorUDAFCountStar extends
for (int i=0; i < batchSize; ++i) {
Aggregation myAgg = getCurrentAggregationBuffer(
aggregationBufferSets, aggregateIndex, i);
- myAgg.isNull = false;
- ++myAgg.value;
+ ++myAgg.count;
}
}
@@ -111,8 +108,7 @@ public class VectorUDAFCountStar extends
}
Aggregation myagg = (Aggregation)agg;
- myagg.isNull = false;
- myagg.value += batchSize;
+ myagg.count += batchSize;
}
@Override
@@ -128,14 +124,9 @@ public class VectorUDAFCountStar extends
@Override
public Object evaluateOutput(AggregationBuffer agg) throws HiveException {
- Aggregation myagg = (Aggregation) agg;
- if (myagg.isNull) {
- return null;
- }
- else {
- result.set (myagg.value);
+ Aggregation myagg = (Aggregation) agg;
+ result.set (myagg.count);
return result;
- }
}
@Override
Modified:
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java?rev=1626018&r1=1626017&r2=1626018&view=diff
==============================================================================
---
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
(original)
+++
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
Thu Sep 18 16:12:10 2014
@@ -50,6 +50,7 @@ import org.apache.hadoop.hive.ql.plan.Ag
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
+import org.apache.hadoop.hive.ql.plan.VectorGroupByDesc;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
@@ -601,6 +602,30 @@ public class TestVectorGroupByOperator {
}
@Test
+ public void testCountReduce() throws HiveException {
+ testAggregateCountReduce(
+ 2,
+ Arrays.asList(new Long[]{}),
+ 0L);
+ testAggregateCountReduce(
+ 2,
+ Arrays.asList(new Long[]{0L}),
+ 0L);
+ testAggregateCountReduce(
+ 2,
+ Arrays.asList(new Long[]{0L,0L}),
+ 0L);
+ testAggregateCountReduce(
+ 2,
+ Arrays.asList(new Long[]{0L,1L,0L}),
+ 1L);
+ testAggregateCountReduce(
+ 2,
+ Arrays.asList(new Long[]{13L,0L,7L,19L}),
+ 39L);
+ }
+
+ @Test
public void testCountDecimal() throws HiveException {
testAggregateDecimal(
"Decimal",
@@ -1210,7 +1235,7 @@ public class TestVectorGroupByOperator {
"count",
2,
Arrays.asList(new Long[]{}),
- null);
+ 0L);
}
@Test
@@ -2027,6 +2052,17 @@ public class TestVectorGroupByOperator {
testAggregateCountStarIterable (fdr, expected);
}
+ public void testAggregateCountReduce (
+ int batchSize,
+ Iterable<Long> values,
+ Object expected) throws HiveException {
+
+ @SuppressWarnings("unchecked")
+ FakeVectorRowBatchFromLongIterables fdr = new
FakeVectorRowBatchFromLongIterables(batchSize,
+ values);
+ testAggregateCountReduceIterable (fdr, expected);
+ }
+
public static interface Validator {
void validate (String key, Object expected, Object result);
@@ -2223,6 +2259,37 @@ public class TestVectorGroupByOperator {
validator.validate("_total", expected, result);
}
+ public void testAggregateCountReduceIterable (
+ Iterable<VectorizedRowBatch> data,
+ Object expected) throws HiveException {
+ Map<String, Integer> mapColumnNames = new HashMap<String, Integer>();
+ mapColumnNames.put("A", 0);
+ VectorizationContext ctx = new VectorizationContext(mapColumnNames, 1);
+
+ GroupByDesc desc = buildGroupByDescType(ctx, "count", "A",
TypeInfoFactory.longTypeInfo);
+ VectorGroupByDesc vectorDesc = desc.getVectorDesc();
+ vectorDesc.setIsReduce(true);
+
+ VectorGroupByOperator vgo = new VectorGroupByOperator(ctx, desc);
+
+ FakeCaptureOutputOperator out =
FakeCaptureOutputOperator.addCaptureOutputChild(vgo);
+ vgo.initialize(null, null);
+
+ for (VectorizedRowBatch unit: data) {
+ vgo.processOp(unit, 0);
+ }
+ vgo.close(false);
+
+ List<Object> outBatchList = out.getCapturedRows();
+ assertNotNull(outBatchList);
+ assertEquals(1, outBatchList.size());
+
+ Object result = outBatchList.get(0);
+
+ Validator validator = getValidator("count");
+ validator.validate("_total", expected, result);
+ }
+
public void testAggregateStringIterable (
String aggregateName,
Iterable<VectorizedRowBatch> data,
Modified:
hive/trunk/ql/src/test/queries/clientpositive/vectorization_short_regress.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/vectorization_short_regress.q?rev=1626018&r1=1626017&r2=1626018&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/vectorization_short_regress.q
(original)
+++ hive/trunk/ql/src/test/queries/clientpositive/vectorization_short_regress.q
Thu Sep 18 16:12:10 2014
@@ -850,3 +850,52 @@ WHERE (((cboolean1 IS NOT NULL))
GROUP BY cboolean1
ORDER BY cboolean1;
+-- These tests verify COUNT on empty or null colulmns work correctly.
+create table test_count(i int) stored as orc;
+
+explain
+select count(*) from test_count;
+
+select count(*) from test_count;
+
+explain
+select count(i) from test_count;
+
+select count(i) from test_count;
+
+create table alltypesnull like alltypesorc;
+alter table alltypesnull set fileformat textfile;
+
+insert into table alltypesnull select null, null, null, null, null, null,
null, null, null, null, null, null from alltypesorc;
+
+create table alltypesnullorc stored as orc as select * from alltypesnull;
+
+explain
+select count(*) from alltypesnullorc;
+
+select count(*) from alltypesnullorc;
+
+explain
+select count(ctinyint) from alltypesnullorc;
+
+select count(ctinyint) from alltypesnullorc;
+
+explain
+select count(cint) from alltypesnullorc;
+
+select count(cint) from alltypesnullorc;
+
+explain
+select count(cfloat) from alltypesnullorc;
+
+select count(cfloat) from alltypesnullorc;
+
+explain
+select count(cstring1) from alltypesnullorc;
+
+select count(cstring1) from alltypesnullorc;
+
+explain
+select count(cboolean1) from alltypesnullorc;
+
+select count(cboolean1) from alltypesnullorc;
Modified:
hive/trunk/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out?rev=1626018&r1=1626017&r2=1626018&view=diff
==============================================================================
---
hive/trunk/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
(original)
+++
hive/trunk/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
Thu Sep 18 16:12:10 2014
@@ -6618,3 +6618,623 @@ POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
false 11.0 -11.0 -2.389090909090909 -17881597706
-1.7881597716175E10 3.8953387713327066E17 6.0 -0.8249999999999993
-2454.8879999999995 3.8953385925167296E17 -2145884705
1.66288903197104486E18 0.8249999999999993 4.7840233756130287E-17
4.098424268084119E-17 0.8249999999999993 -1051696618
28.692556844886422 2.980633855245E9 -4.032330473245E9
85.79562278396777 4.032330473245E9 -3983699.3106060605
3983699.3106060605 4.1896430920933255E15
true 79.553 -79.553 -0.33034580136836733 -401322621137
-4.01322621147175E11 7.9255373737244976E16 34.727455139160156
-69.3780014038086 4856.6352637899645 7.9254972414623824E16
-2130544867 2.30133924842409523E18 69.3780014038086
3.456813247089758E-17 2.0387240975807185E-18 69.3780014038086
2182477964777 34.654968050508266 2.959326820263E9
2.179518637956737E12 9461.197516216069 -2.179518637956737E12
4.592756659884259E8 -4.592756659884259E8 1.002359020778021E21
+PREHOOK: query: -- These tests verify COUNT on empty or null colulmns work
correctly.
+create table test_count(i int) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test_count
+POSTHOOK: query: -- These tests verify COUNT on empty or null colulmns work
correctly.
+create table test_count(i int) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test_count
+PREHOOK: query: explain
+select count(*) from test_count
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from test_count
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: test_count
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE
Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE
Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from test_count
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_count
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from test_count
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_count
+#### A masked pattern was here ####
+0
+PREHOOK: query: explain
+select count(i) from test_count
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(i) from test_count
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: test_count
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE
Column stats: NONE
+ Select Operator
+ expressions: i (type: int)
+ outputColumnNames: i
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE
Column stats: NONE
+ Group By Operator
+ aggregations: count(i)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(i) from test_count
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_count
+#### A masked pattern was here ####
+POSTHOOK: query: select count(i) from test_count
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_count
+#### A masked pattern was here ####
+0
+PREHOOK: query: create table alltypesnull like alltypesorc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@alltypesnull
+POSTHOOK: query: create table alltypesnull like alltypesorc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@alltypesnull
+PREHOOK: query: alter table alltypesnull set fileformat textfile
+PREHOOK: type: ALTERTABLE_FILEFORMAT
+PREHOOK: Input: default@alltypesnull
+PREHOOK: Output: default@alltypesnull
+POSTHOOK: query: alter table alltypesnull set fileformat textfile
+POSTHOOK: type: ALTERTABLE_FILEFORMAT
+POSTHOOK: Input: default@alltypesnull
+POSTHOOK: Output: default@alltypesnull
+PREHOOK: query: insert into table alltypesnull select null, null, null, null,
null, null, null, null, null, null, null, null from alltypesorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@alltypesnull
+POSTHOOK: query: insert into table alltypesnull select null, null, null, null,
null, null, null, null, null, null, null, null from alltypesorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@alltypesnull
+POSTHOOK: Lineage: alltypesnull.cbigint EXPRESSION []
+POSTHOOK: Lineage: alltypesnull.cboolean1 EXPRESSION []
+POSTHOOK: Lineage: alltypesnull.cboolean2 EXPRESSION []
+POSTHOOK: Lineage: alltypesnull.cdouble EXPRESSION []
+POSTHOOK: Lineage: alltypesnull.cfloat EXPRESSION []
+POSTHOOK: Lineage: alltypesnull.cint EXPRESSION []
+POSTHOOK: Lineage: alltypesnull.csmallint EXPRESSION []
+POSTHOOK: Lineage: alltypesnull.cstring1 SIMPLE []
+POSTHOOK: Lineage: alltypesnull.cstring2 SIMPLE []
+POSTHOOK: Lineage: alltypesnull.ctimestamp1 EXPRESSION []
+POSTHOOK: Lineage: alltypesnull.ctimestamp2 EXPRESSION []
+POSTHOOK: Lineage: alltypesnull.ctinyint EXPRESSION []
+PREHOOK: query: create table alltypesnullorc stored as orc as select * from
alltypesnull
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@alltypesnull
+PREHOOK: Output: database:default
+PREHOOK: Output: default@alltypesnullorc
+POSTHOOK: query: create table alltypesnullorc stored as orc as select * from
alltypesnull
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@alltypesnull
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@alltypesnullorc
+PREHOOK: query: explain
+select count(*) from alltypesnullorc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from alltypesnullorc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesnullorc
+ Statistics: Num rows: 12288 Data size: 168 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 12288 Data size: 168 Basic stats:
COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from alltypesnullorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from alltypesnullorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+12288
+PREHOOK: query: explain
+select count(ctinyint) from alltypesnullorc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(ctinyint) from alltypesnullorc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesnullorc
+ Statistics: Num rows: 12288 Data size: 168 Basic stats:
COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint)
+ outputColumnNames: ctinyint
+ Statistics: Num rows: 12288 Data size: 168 Basic stats:
COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(ctinyint)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(ctinyint) from alltypesnullorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+POSTHOOK: query: select count(ctinyint) from alltypesnullorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+0
+PREHOOK: query: explain
+select count(cint) from alltypesnullorc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(cint) from alltypesnullorc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesnullorc
+ Statistics: Num rows: 12288 Data size: 168 Basic stats:
COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cint (type: int)
+ outputColumnNames: cint
+ Statistics: Num rows: 12288 Data size: 168 Basic stats:
COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(cint)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(cint) from alltypesnullorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+POSTHOOK: query: select count(cint) from alltypesnullorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+0
+PREHOOK: query: explain
+select count(cfloat) from alltypesnullorc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(cfloat) from alltypesnullorc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesnullorc
+ Statistics: Num rows: 12288 Data size: 168 Basic stats:
COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cfloat (type: float)
+ outputColumnNames: cfloat
+ Statistics: Num rows: 12288 Data size: 168 Basic stats:
COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(cfloat)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(cfloat) from alltypesnullorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+POSTHOOK: query: select count(cfloat) from alltypesnullorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+0
+PREHOOK: query: explain
+select count(cstring1) from alltypesnullorc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(cstring1) from alltypesnullorc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesnullorc
+ Statistics: Num rows: 12288 Data size: 168 Basic stats:
COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cstring1 (type: string)
+ outputColumnNames: cstring1
+ Statistics: Num rows: 12288 Data size: 168 Basic stats:
COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(cstring1)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(cstring1) from alltypesnullorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+POSTHOOK: query: select count(cstring1) from alltypesnullorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+0
+PREHOOK: query: explain
+select count(cboolean1) from alltypesnullorc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(cboolean1) from alltypesnullorc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesnullorc
+ Statistics: Num rows: 12288 Data size: 168 Basic stats:
COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cboolean1 (type: boolean)
+ outputColumnNames: cboolean1
+ Statistics: Num rows: 12288 Data size: 168 Basic stats:
COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(cboolean1)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(cboolean1) from alltypesnullorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+POSTHOOK: query: select count(cboolean1) from alltypesnullorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+0
Modified:
hive/trunk/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/vectorization_short_regress.q.out?rev=1626018&r1=1626017&r2=1626018&view=diff
==============================================================================
---
hive/trunk/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
(original)
+++
hive/trunk/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
Thu Sep 18 16:12:10 2014
@@ -6615,3 +6615,567 @@ POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
false 11.0 -11.0 -2.389090909090909 -17881597706
-1.7881597716175E10 3.8953387713327066E17 6.0 -0.8249999999999993
-2454.8879999999995 3.8953385925167296E17 -2145884705
1.66288903197104486E18 0.8249999999999993 4.7840233756130287E-17
4.098424268084119E-17 0.8249999999999993 -1051696618
28.692556844886422 2.980633855245E9 -4.032330473245E9
85.79562278396777 4.032330473245E9 -3983699.3106060605
3983699.3106060605 4.1896430920933255E15
true 79.553 -79.553 -0.33034580136836733 -401322621137
-4.01322621147175E11 7.9255373737244976E16 34.727455139160156
-69.3780014038086 4856.6352637899645 7.9254972414623824E16
-2130544867 2.30133924842409523E18 69.3780014038086
3.456813247089758E-17 2.0387240975807185E-18 69.3780014038086
2182477964777 34.654968050508266 2.959326820263E9
2.179518637956737E12 9461.197516216069 -2.179518637956737E12
4.592756659884259E8 -4.592756659884259E8 1.002359020778021E21
+PREHOOK: query: -- These tests verify COUNT on empty or null colulmns work
correctly.
+create table test_count(i int) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test_count
+POSTHOOK: query: -- These tests verify COUNT on empty or null colulmns work
correctly.
+create table test_count(i int) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test_count
+PREHOOK: query: explain
+select count(*) from test_count
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from test_count
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: test_count
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column
stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column
stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from test_count
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_count
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from test_count
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_count
+#### A masked pattern was here ####
+0
+PREHOOK: query: explain
+select count(i) from test_count
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(i) from test_count
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: test_count
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column
stats: NONE
+ Select Operator
+ expressions: i (type: int)
+ outputColumnNames: i
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column
stats: NONE
+ Group By Operator
+ aggregations: count(i)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(i) from test_count
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_count
+#### A masked pattern was here ####
+POSTHOOK: query: select count(i) from test_count
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_count
+#### A masked pattern was here ####
+0
+PREHOOK: query: create table alltypesnull like alltypesorc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@alltypesnull
+POSTHOOK: query: create table alltypesnull like alltypesorc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@alltypesnull
+PREHOOK: query: alter table alltypesnull set fileformat textfile
+PREHOOK: type: ALTERTABLE_FILEFORMAT
+PREHOOK: Input: default@alltypesnull
+PREHOOK: Output: default@alltypesnull
+POSTHOOK: query: alter table alltypesnull set fileformat textfile
+POSTHOOK: type: ALTERTABLE_FILEFORMAT
+POSTHOOK: Input: default@alltypesnull
+POSTHOOK: Output: default@alltypesnull
+PREHOOK: query: insert into table alltypesnull select null, null, null, null,
null, null, null, null, null, null, null, null from alltypesorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@alltypesnull
+POSTHOOK: query: insert into table alltypesnull select null, null, null, null,
null, null, null, null, null, null, null, null from alltypesorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@alltypesnull
+POSTHOOK: Lineage: alltypesnull.cbigint EXPRESSION []
+POSTHOOK: Lineage: alltypesnull.cboolean1 EXPRESSION []
+POSTHOOK: Lineage: alltypesnull.cboolean2 EXPRESSION []
+POSTHOOK: Lineage: alltypesnull.cdouble EXPRESSION []
+POSTHOOK: Lineage: alltypesnull.cfloat EXPRESSION []
+POSTHOOK: Lineage: alltypesnull.cint EXPRESSION []
+POSTHOOK: Lineage: alltypesnull.csmallint EXPRESSION []
+POSTHOOK: Lineage: alltypesnull.cstring1 SIMPLE []
+POSTHOOK: Lineage: alltypesnull.cstring2 SIMPLE []
+POSTHOOK: Lineage: alltypesnull.ctimestamp1 EXPRESSION []
+POSTHOOK: Lineage: alltypesnull.ctimestamp2 EXPRESSION []
+POSTHOOK: Lineage: alltypesnull.ctinyint EXPRESSION []
+PREHOOK: query: create table alltypesnullorc stored as orc as select * from
alltypesnull
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@alltypesnull
+PREHOOK: Output: database:default
+PREHOOK: Output: default@alltypesnullorc
+POSTHOOK: query: create table alltypesnullorc stored as orc as select * from
alltypesnull
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@alltypesnull
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@alltypesnullorc
+PREHOOK: query: explain
+select count(*) from alltypesnullorc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from alltypesnullorc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypesnullorc
+ Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE
Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from alltypesnullorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from alltypesnullorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+12288
+PREHOOK: query: explain
+select count(ctinyint) from alltypesnullorc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(ctinyint) from alltypesnullorc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypesnullorc
+ Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE
Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint)
+ outputColumnNames: ctinyint
+ Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE
Column stats: NONE
+ Group By Operator
+ aggregations: count(ctinyint)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(ctinyint) from alltypesnullorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+POSTHOOK: query: select count(ctinyint) from alltypesnullorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+0
+PREHOOK: query: explain
+select count(cint) from alltypesnullorc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(cint) from alltypesnullorc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypesnullorc
+ Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE
Column stats: NONE
+ Select Operator
+ expressions: cint (type: int)
+ outputColumnNames: cint
+ Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE
Column stats: NONE
+ Group By Operator
+ aggregations: count(cint)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(cint) from alltypesnullorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+POSTHOOK: query: select count(cint) from alltypesnullorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+0
+PREHOOK: query: explain
+select count(cfloat) from alltypesnullorc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(cfloat) from alltypesnullorc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypesnullorc
+ Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE
Column stats: NONE
+ Select Operator
+ expressions: cfloat (type: float)
+ outputColumnNames: cfloat
+ Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE
Column stats: NONE
+ Group By Operator
+ aggregations: count(cfloat)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(cfloat) from alltypesnullorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+POSTHOOK: query: select count(cfloat) from alltypesnullorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+0
+PREHOOK: query: explain
+select count(cstring1) from alltypesnullorc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(cstring1) from alltypesnullorc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypesnullorc
+ Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE
Column stats: NONE
+ Select Operator
+ expressions: cstring1 (type: string)
+ outputColumnNames: cstring1
+ Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE
Column stats: NONE
+ Group By Operator
+ aggregations: count(cstring1)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(cstring1) from alltypesnullorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+POSTHOOK: query: select count(cstring1) from alltypesnullorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+0
+PREHOOK: query: explain
+select count(cboolean1) from alltypesnullorc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(cboolean1) from alltypesnullorc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypesnullorc
+ Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE
Column stats: NONE
+ Select Operator
+ expressions: cboolean1 (type: boolean)
+ outputColumnNames: cboolean1
+ Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE
Column stats: NONE
+ Group By Operator
+ aggregations: count(cboolean1)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(cboolean1) from alltypesnullorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+POSTHOOK: query: select count(cboolean1) from alltypesnullorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesnullorc
+#### A masked pattern was here ####
+0
