HIVE-18359: Extend grouping set limits from int to long (Prasanth Jayachandran reviewed by Jesus Camacho Rodriguez)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ddd4c9ae Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ddd4c9ae Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ddd4c9ae Branch: refs/heads/standalone-metastore Commit: ddd4c9aea6166129be289757e1721d0cfccfef66 Parents: 89e75c7 Author: Prasanth Jayachandran <prasan...@apache.org> Authored: Sat Feb 10 11:22:05 2018 -0600 Committer: Prasanth Jayachandran <prasan...@apache.org> Committed: Sat Feb 10 11:22:05 2018 -0600 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/ql/ErrorMsg.java | 2 + .../hadoop/hive/ql/exec/GroupByOperator.java | 20 +- .../ql/exec/vector/VectorGroupByOperator.java | 8 +- .../hadoop/hive/ql/metadata/VirtualColumn.java | 2 +- .../calcite/reloperators/HiveGroupingID.java | 2 +- .../rules/HiveExpandDistinctAggregatesRule.java | 2 +- .../calcite/translator/HiveGBOpConvUtil.java | 8 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 20 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 110 +- .../apache/hadoop/hive/ql/plan/GroupByDesc.java | 12 +- .../hive/ql/udf/generic/GenericUDFGrouping.java | 39 +- .../test/queries/clientnegative/groupby_cube3.q | 90 + .../clientnegative/groupby_grouping_sets8.q | 98 + .../queries/clientnegative/groupby_rollup3.q | 90 + ql/src/test/queries/clientpositive/cte_1.q | 2 +- .../clientpositive/groupingset_high_columns.q | 259 + .../results/clientnegative/groupby_cube3.q.out | 18 + .../clientnegative/groupby_grouping_sets8.q.out | 18 + .../clientnegative/groupby_rollup3.q.out | 18 + .../clientpositive/annotate_stats_groupby.q.out | 192 +- .../annotate_stats_groupby2.q.out | 40 +- .../cbo_rp_annotate_stats_groupby.q.out | 64 +- .../results/clientpositive/groupby_cube1.q.out | 74 +- .../clientpositive/groupby_cube_multi_gby.q.out | 16 +- .../clientpositive/groupby_grouping_id3.q.out | 20 +- .../clientpositive/groupby_grouping_sets1.q.out | 40 +- .../clientpositive/groupby_grouping_sets2.q.out | 32 +- .../clientpositive/groupby_grouping_sets3.q.out | 24 +- .../clientpositive/groupby_grouping_sets4.q.out | 48 +- .../clientpositive/groupby_grouping_sets5.q.out | 24 +- .../clientpositive/groupby_grouping_sets6.q.out | 16 +- .../groupby_grouping_sets_grouping.q.out | 128 +- .../groupby_grouping_sets_limit.q.out | 32 +- .../groupby_grouping_window.q.out | 8 +- .../clientpositive/groupby_rollup1.q.out | 56 +- .../clientpositive/groupby_rollup_empty.q.out | 10 +- .../groupingset_high_columns.q.out | 1169 + .../infer_bucket_sort_grouping_operators.q.out | 24 +- .../clientpositive/limit_pushdown2.q.out | 16 +- .../results/clientpositive/llap/cte_1.q.out | 36670 ++++++++--------- .../llap/groupby_rollup_empty.q.out | 10 +- .../results/clientpositive/llap/llap_acid.q.out | 60 +- .../clientpositive/llap/llap_acid_fast.q.out | 60 +- .../llap/multi_count_distinct_null.q.out | 58 +- .../llap/vector_groupby_cube1.q.out | 74 +- .../llap/vector_groupby_grouping_id1.q.out | 100 +- .../llap/vector_groupby_grouping_id2.q.out | 306 +- .../llap/vector_groupby_grouping_id3.q.out | 42 +- .../llap/vector_groupby_grouping_sets1.q.out | 70 +- .../llap/vector_groupby_grouping_sets2.q.out | 62 +- .../llap/vector_groupby_grouping_sets3.q.out | 38 +- .../vector_groupby_grouping_sets3_dec.q.out | 42 +- .../llap/vector_groupby_grouping_sets4.q.out | 72 +- .../llap/vector_groupby_grouping_sets5.q.out | 42 +- .../llap/vector_groupby_grouping_sets6.q.out | 28 +- .../vector_groupby_grouping_sets_grouping.q.out | 230 +- .../vector_groupby_grouping_sets_limit.q.out | 56 +- .../llap/vector_groupby_grouping_window.q.out | 28 +- .../llap/vector_groupby_rollup1.q.out | 96 +- .../llap/vector_grouping_sets.q.out | 44 +- .../clientpositive/perf/spark/query18.q.out | 8 +- .../clientpositive/perf/spark/query22.q.out | 8 +- .../clientpositive/perf/spark/query27.q.out | 14 +- .../clientpositive/perf/spark/query36.q.out | 26 +- .../clientpositive/perf/spark/query5.q.out | 20 +- .../clientpositive/perf/spark/query67.q.out | 8 +- .../clientpositive/perf/spark/query70.q.out | 26 +- .../clientpositive/perf/spark/query77.q.out | 20 +- .../clientpositive/perf/spark/query80.q.out | 20 +- .../clientpositive/perf/spark/query86.q.out | 26 +- .../clientpositive/spark/groupby_cube1.q.out | 74 +- .../clientpositive/spark/groupby_rollup1.q.out | 56 +- .../clientpositive/spark/limit_pushdown2.q.out | 16 +- .../tez/multi_count_distinct.q.out | 12 +- .../clientpositive/vector_grouping_sets.q.out | 32 +- .../test/results/clientpositive/view_cbo.q.out | 8 +- 76 files changed, 21594 insertions(+), 19819 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/ddd4c9ae/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 134faee..46d876d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -456,6 +456,8 @@ public enum ErrorMsg { HIVE_GROUPING_FUNCTION_EXPR_NOT_IN_GROUPBY(10409, "Expression in GROUPING function not present in GROUP BY"), ALTER_TABLE_NON_PARTITIONED_TABLE_CASCADE_NOT_SUPPORTED(10410, "Alter table with non-partitioned table does not support cascade"), + HIVE_GROUPING_SETS_SIZE_LIMIT(10411, + "Grouping sets size cannot be greater than 64"), //========================== 20000 range starts here ========================// http://git-wip-us.apache.org/repos/asf/hive/blob/ddd4c9ae/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java index 6de979e..4b76638 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java @@ -65,7 +65,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobConf; @@ -130,9 +130,9 @@ public class GroupByOperator extends Operator<GroupByDesc> implements IConfigure private transient boolean groupingSetsPresent; // generates grouping set private transient int groupingSetsPosition; // position of grouping set, generally the last of keys - private transient List<Integer> groupingSets; // declared grouping set values + private transient List<Long> groupingSets; // declared grouping set values private transient FastBitSet[] groupingSetsBitSet; // bitsets acquired from grouping set values - private transient IntWritable[] newKeysGroupingSets; + private transient LongWritable[] newKeysGroupingSets; // for these positions, some variable primitive type (String) is used, so size // cannot be estimated. sample it at runtime. @@ -180,7 +180,7 @@ public class GroupByOperator extends Operator<GroupByDesc> implements IConfigure * @param length * @return */ - public static FastBitSet groupingSet2BitSet(int value, int length) { + public static FastBitSet groupingSet2BitSet(long value, int length) { FastBitSet bits = new FastBitSet(); for (int index = length - 1; index >= 0; index--) { if (value % 2 != 0) { @@ -231,13 +231,13 @@ public class GroupByOperator extends Operator<GroupByDesc> implements IConfigure if (groupingSetsPresent) { groupingSets = conf.getListGroupingSets(); groupingSetsPosition = conf.getGroupingSetPosition(); - newKeysGroupingSets = new IntWritable[groupingSets.size()]; + newKeysGroupingSets = new LongWritable[groupingSets.size()]; groupingSetsBitSet = new FastBitSet[groupingSets.size()]; int pos = 0; - for (Integer groupingSet: groupingSets) { + for (Long groupingSet: groupingSets) { // Create the mapping corresponding to the grouping set - newKeysGroupingSets[pos] = new IntWritable(groupingSet); + newKeysGroupingSets[pos] = new LongWritable(groupingSet); groupingSetsBitSet[pos] = groupingSet2BitSet(groupingSet, groupingSetsPosition); pos++; } @@ -1102,7 +1102,7 @@ public class GroupByOperator extends Operator<GroupByDesc> implements IConfigure Object[] keys=new Object[outputKeyLength]; int pos = conf.getGroupingSetPosition(); if (pos >= 0 && pos < outputKeyLength) { - keys[pos] = new IntWritable((1 << pos) - 1); + keys[pos] = new LongWritable((1L << pos) - 1); } forward(keys, aggregations); } else { @@ -1175,11 +1175,11 @@ public class GroupByOperator extends Operator<GroupByDesc> implements IConfigure } int groupingSetPosition = desc.getGroupingSetPosition(); - List<Integer> listGroupingSets = desc.getListGroupingSets(); + List<Long> listGroupingSets = desc.getListGroupingSets(); // groupingSets are known at map/reducer side; but have to do real processing // hence grouppingSetsPresent is true only at map side if (groupingSetPosition >= 0 && listGroupingSets != null) { - Integer emptyGrouping = (1 << groupingSetPosition) - 1; + Long emptyGrouping = (1L << groupingSetPosition) - 1; if (listGroupingSets.contains(emptyGrouping)) { return true; } http://git-wip-us.apache.org/repos/asf/hive/blob/ddd4c9ae/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java index e670409..75efc29 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java @@ -126,7 +126,7 @@ public class VectorGroupByOperator extends Operator<GroupByDesc> private transient boolean groupingSetsPresent; // The field bits (i.e. which fields to include) or "id" for each grouping set. - private transient int[] groupingSets; + private transient long[] groupingSets; // The position in the column keys of the dummy grouping set id column. private transient int groupingSetsPosition; @@ -457,7 +457,7 @@ public class VectorGroupByOperator extends Operator<GroupByDesc> kw.setNull(); int pos = conf.getGroupingSetPosition(); if (pos >= 0) { - long val = (1 << pos) - 1; + long val = (1L << pos) - 1; keyWrappersBatch.setLongValue(kw, pos, val); } VectorAggregationBufferRow groupAggregators = allocateAggregationBuffer(); @@ -938,13 +938,13 @@ public class VectorGroupByOperator extends Operator<GroupByDesc> return; } - groupingSets = ArrayUtils.toPrimitive(conf.getListGroupingSets().toArray(new Integer[0])); + groupingSets = ArrayUtils.toPrimitive(conf.getListGroupingSets().toArray(new Long[0])); groupingSetsPosition = conf.getGroupingSetPosition(); allGroupingSetsOverrideIsNulls = new boolean[groupingSets.length][]; int pos = 0; - for (int groupingSet: groupingSets) { + for (long groupingSet: groupingSets) { // Create the mapping corresponding to the grouping set http://git-wip-us.apache.org/repos/asf/hive/blob/ddd4c9ae/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java index 2411d3a..48623e5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java @@ -61,7 +61,7 @@ public enum VirtualColumn { * set if that column has been aggregated in that row. Otherwise the * value is "0". Returns the decimal representation of the bit vector. */ - GROUPINGID("GROUPING__ID", TypeInfoFactory.intTypeInfo); + GROUPINGID("GROUPING__ID", TypeInfoFactory.longTypeInfo); public static final ImmutableSet<String> VIRTUAL_COLUMN_NAMES = ImmutableSet.of(FILENAME.getName(), BLOCKOFFSET.getName(), ROWOFFSET.getName(), http://git-wip-us.apache.org/repos/asf/hive/blob/ddd4c9ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java index 4ba27a2..dcbccf6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java @@ -33,7 +33,7 @@ public class HiveGroupingID extends SqlAbstractGroupFunction { private HiveGroupingID() { super(VirtualColumn.GROUPINGID.getName(), SqlKind.OTHER, - ReturnTypes.INTEGER, + ReturnTypes.BIGINT, InferTypes.BOOLEAN, OperandTypes.NILADIC, SqlFunctionCategory.SYSTEM); http://git-wip-us.apache.org/repos/asf/hive/blob/ddd4c9ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java index 864efa4..b33c4c5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java @@ -327,7 +327,7 @@ public final class HiveExpandDistinctAggregatesRule extends RelOptRule { // Create GroupingID column AggregateCall aggCall = AggregateCall.create(HiveGroupingID.INSTANCE, false, new ImmutableList.Builder<Integer>().build(), -1, this.cluster.getTypeFactory() - .createSqlType(SqlTypeName.INTEGER), HiveGroupingID.INSTANCE.getName()); + .createSqlType(SqlTypeName.BIGINT), HiveGroupingID.INSTANCE.getName()); aggregateCalls.add(aggCall); return new HiveAggregate(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), aggregate.getInput(), groupSet, origGroupSets, aggregateCalls); http://git-wip-us.apache.org/repos/asf/hive/blob/ddd4c9ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveGBOpConvUtil.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveGBOpConvUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveGBOpConvUtil.java index f22cd94..70f8343 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveGBOpConvUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveGBOpConvUtil.java @@ -101,7 +101,7 @@ public class HiveGBOpConvUtil { private final List<TypeInfo> gbKeyTypes = new ArrayList<TypeInfo>(); private final List<ExprNodeDesc> gbKeys = new ArrayList<ExprNodeDesc>(); - private final List<Integer> grpSets = new ArrayList<Integer>(); + private final List<Long> grpSets = new ArrayList<Long>(); private boolean grpSetRqrAdditionalMRJob; private boolean grpIdFunctionNeeded; @@ -177,7 +177,7 @@ public class HiveGBOpConvUtil { if (aggRel.getGroupType() != Group.SIMPLE) { // 2.1 Translate Grouping set col bitset ImmutableList<ImmutableBitSet> lstGrpSet = aggRel.getGroupSets(); - int bitmap = 0; + long bitmap = 0; for (ImmutableBitSet grpSet : lstGrpSet) { bitmap = 0; for (Integer bitIdx : grpSet.asList()) { @@ -863,7 +863,7 @@ public class HiveGBOpConvUtil { groupingSetsColPosition = gbInfo.gbKeys.size(); if (computeGrpSet) { // GrpSet Col needs to be constructed - gbKeys.add(new ExprNodeConstantDesc("0")); + gbKeys.add(new ExprNodeConstantDesc("0L")); } else { // GrpSet Col already part of input RS // TODO: Can't we just copy the ExprNodeDEsc from input (Do we need to @@ -1185,7 +1185,7 @@ public class HiveGBOpConvUtil { ExprNodeDesc grpSetColExpr = null; if (createConstantExpr) { - grpSetColExpr = new ExprNodeConstantDesc("0"); + grpSetColExpr = new ExprNodeConstantDesc("0L"); } else { grpSetColExpr = new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, grpSetIDExprName, null, false); http://git-wip-us.apache.org/repos/asf/hive/blob/ddd4c9ae/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index cf2bc13..8e0a454 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -3028,7 +3028,7 @@ public class CalcitePlanner extends SemanticAnalyzer { } private RelNode genGBRelNode(List<ExprNodeDesc> gbExprs, List<AggInfo> aggInfoLst, - List<Integer> groupSets, RelNode srcRel) throws SemanticException { + List<Long> groupSets, RelNode srcRel) throws SemanticException { ImmutableMap<String, Integer> posMap = this.relToHiveColNameCalcitePosMap.get(srcRel); RexNodeConverter converter = new RexNodeConverter(this.cluster, srcRel.getRowType(), posMap, 0, false); @@ -3054,7 +3054,7 @@ public class CalcitePlanner extends SemanticAnalyzer { if(hasGroupSets) { Set<ImmutableBitSet> setTransformedGroupSets = new HashSet<ImmutableBitSet>(groupSets.size()); - for(int val: groupSets) { + for(long val: groupSets) { setTransformedGroupSets.add(convert(val, groupSet.cardinality())); } // Calcite expects the grouping sets sorted and without duplicates @@ -3071,7 +3071,7 @@ public class CalcitePlanner extends SemanticAnalyzer { // Create GroupingID column AggregateCall aggCall = AggregateCall.create(HiveGroupingID.INSTANCE, false, new ImmutableList.Builder<Integer>().build(), -1, - this.cluster.getTypeFactory().createSqlType(SqlTypeName.INTEGER), + this.cluster.getTypeFactory().createSqlType(SqlTypeName.BIGINT), HiveGroupingID.INSTANCE.getName()); aggregateCalls.add(aggCall); } @@ -3090,7 +3090,7 @@ public class CalcitePlanner extends SemanticAnalyzer { } /* This method returns the flip big-endian representation of value */ - private ImmutableBitSet convert(int value, int length) { + private ImmutableBitSet convert(long value, int length) { BitSet bits = new BitSet(); for (int index = length - 1; index >= 0; index--) { if (value % 2 != 0) { @@ -3333,15 +3333,9 @@ public class CalcitePlanner extends SemanticAnalyzer { // 5. GroupingSets, Cube, Rollup int groupingColsSize = gbExprNDescLst.size(); - List<Integer> groupingSets = null; + List<Long> groupingSets = null; if (cubeRollupGrpSetPresent) { - if (qbp.getDestRollups().contains(detsClauseName)) { - groupingSets = getGroupingSetsForRollup(grpByAstExprs.size()); - } else if (qbp.getDestCubes().contains(detsClauseName)) { - groupingSets = getGroupingSetsForCube(grpByAstExprs.size()); - } else if (qbp.getDestGroupingSets().contains(detsClauseName)) { - groupingSets = getGroupingSets(grpByAstExprs, qbp, detsClauseName); - } + groupingSets = getGroupByGroupingSetsForClause(qbp, detsClauseName).getSecond(); } // 6. Construct aggregation function Info @@ -3386,7 +3380,7 @@ public class CalcitePlanner extends SemanticAnalyzer { groupByOutputRowResolver.put(null, VirtualColumn.GROUPINGID.getName(), new ColumnInfo( field, - TypeInfoFactory.intTypeInfo, + VirtualColumn.GROUPINGID.getTypeInfo(), null, true)); } http://git-wip-us.apache.org/repos/asf/hive/blob/ddd4c9ae/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 8e587f1..5c96653 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -255,6 +255,7 @@ import com.google.common.base.Strings; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import com.google.common.math.IntMath; +import com.google.common.math.LongMath; /** * Implementation of the semantic analyzer. It generates the query plan. @@ -2972,7 +2973,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // Query does not contain CUBE, ROLLUP, or GROUPING SETS, and thus, // grouping should return 0 childGroupingID = (ASTNode) ParseDriver.adaptor.create(HiveParser.IntegralLiteral, - String.valueOf(0)); + "0L"); } else { // We refer to grouping_id column childGroupingID = (ASTNode) ParseDriver.adaptor.create( @@ -2990,7 +2991,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // Create and add AST node with position of grouping function input // in group by clause ASTNode childN = (ASTNode) ParseDriver.adaptor.create(HiveParser.IntegralLiteral, - String.valueOf(IntMath.mod(-j-1, grpByAstExprs.size()))); + String.valueOf(IntMath.mod(-j-1, grpByAstExprs.size())) + "L"); newRoot.addChild(childN); break; } @@ -3830,18 +3831,18 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } } - protected List<Integer> getGroupingSetsForRollup(int size) { - List<Integer> groupingSetKeys = new ArrayList<Integer>(); + protected List<Long> getGroupingSetsForRollup(int size) { + List<Long> groupingSetKeys = new ArrayList<Long>(); for (int i = 0; i <= size; i++) { - groupingSetKeys.add((1 << i) - 1); + groupingSetKeys.add((1L << i) - 1); } return groupingSetKeys; } - protected List<Integer> getGroupingSetsForCube(int size) { - int count = 1 << size; - List<Integer> results = new ArrayList<Integer>(count); - for (int i = 0; i < count; ++i) { + protected List<Long> getGroupingSetsForCube(int size) { + long count = 1L << size; + List<Long> results = new ArrayList<Long>(); + for (long i = 0; i < count; ++i) { results.add(i); } return results; @@ -3850,10 +3851,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // This function returns the grouping sets along with the grouping expressions // Even if rollups and cubes are present in the query, they are converted to // grouping sets at this point - private ObjectPair<List<ASTNode>, List<Integer>> getGroupByGroupingSetsForClause( - QBParseInfo parseInfo, String dest) throws SemanticException { - List<Integer> groupingSets = new ArrayList<Integer>(); + ObjectPair<List<ASTNode>, List<Long>> getGroupByGroupingSetsForClause( + QBParseInfo parseInfo, String dest) throws SemanticException { + List<Long> groupingSets = new ArrayList<Long>(); List<ASTNode> groupByExprs = getGroupByForClause(parseInfo, dest); + if (groupByExprs.size() > Long.SIZE) { + throw new SemanticException(ErrorMsg.HIVE_GROUPING_SETS_SIZE_LIMIT.getMsg()); + } + if (parseInfo.getDestRollups().contains(dest)) { groupingSets = getGroupingSetsForRollup(groupByExprs.size()); } else if (parseInfo.getDestCubes().contains(dest)) { @@ -3862,11 +3867,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { groupingSets = getGroupingSets(groupByExprs, parseInfo, dest); } - return new ObjectPair<List<ASTNode>, List<Integer>>(groupByExprs, groupingSets); + return new ObjectPair<List<ASTNode>, List<Long>>(groupByExprs, groupingSets); } - protected List<Integer> getGroupingSets(List<ASTNode> groupByExpr, QBParseInfo parseInfo, - String dest) throws SemanticException { + protected List<Long> getGroupingSets(List<ASTNode> groupByExpr, QBParseInfo parseInfo, + String dest) throws SemanticException { Map<String, Integer> exprPos = new HashMap<String, Integer>(); for (int i = 0; i < groupByExpr.size(); ++i) { ASTNode node = groupByExpr.get(i); @@ -3874,14 +3879,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } ASTNode root = parseInfo.getGroupByForClause(dest); - List<Integer> result = new ArrayList<Integer>(root == null ? 0 : root.getChildCount()); + List<Long> result = new ArrayList<Long>(root == null ? 0 : root.getChildCount()); if (root != null) { for (int i = 0; i < root.getChildCount(); ++i) { ASTNode child = (ASTNode) root.getChild(i); if (child.getType() != HiveParser.TOK_GROUPING_SETS_EXPRESSION) { continue; } - int bitmap = IntMath.pow(2, groupByExpr.size()) - 1; + long bitmap = LongMath.pow(2, groupByExpr.size()) - 1; for (int j = 0; j < child.getChildCount(); ++j) { String treeAsString = child.getChild(j).toStringTree(); Integer pos = exprPos.get(treeAsString); @@ -3895,27 +3900,28 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { result.add(bitmap); } } - if (checkForEmptyGroupingSets(result, IntMath.pow(2, groupByExpr.size()) - 1)) { + + if (checkForEmptyGroupingSets(result, LongMath.pow(2, groupByExpr.size()) - 1)) { throw new SemanticException( - ErrorMsg.HIVE_GROUPING_SETS_EMPTY.getMsg()); + ErrorMsg.HIVE_GROUPING_SETS_EMPTY.getMsg()); } return result; } - private boolean checkForEmptyGroupingSets(List<Integer> bitmaps, int groupingIdAllSet) { + private boolean checkForEmptyGroupingSets(List<Long> bitmaps, long groupingIdAllSet) { boolean ret = true; - for (int mask : bitmaps) { + for (long mask : bitmaps) { ret &= mask == groupingIdAllSet; } return ret; } - public static int setBit(int bitmap, int bitIdx) { - return bitmap | (1 << bitIdx); + public static long setBit(long bitmap, int bitIdx) { + return bitmap | (1L << bitIdx); } - public static int unsetBit(int bitmap, int bitIdx) { - return bitmap & ~(1 << bitIdx); + public static long unsetBit(long bitmap, int bitIdx) { + return bitmap & ~(1L << bitIdx); } /** @@ -4774,7 +4780,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // For grouping sets, add a dummy grouping key String groupingSetColumnName = groupByInputRowResolver.get(null, VirtualColumn.GROUPINGID.getName()).getInternalName(); - ExprNodeDesc inputExpr = new ExprNodeColumnDesc(TypeInfoFactory.intTypeInfo, + ExprNodeDesc inputExpr = new ExprNodeColumnDesc(VirtualColumn.GROUPINGID.getTypeInfo(), groupingSetColumnName, null, false); groupByKeys.add(inputExpr); @@ -4783,7 +4789,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { groupByOutputRowResolver.put(null, VirtualColumn.GROUPINGID.getName(), new ColumnInfo( field, - TypeInfoFactory.intTypeInfo, + VirtualColumn.GROUPINGID.getTypeInfo(), null, true)); colExprMap.put(field, groupByKeys.get(groupByKeys.size() - 1)); @@ -4805,7 +4811,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // add a key for reduce sink String groupingSetColumnName = reduceSinkInputRowResolver.get(null, VirtualColumn.GROUPINGID.getName()).getInternalName(); - ExprNodeDesc inputExpr = new ExprNodeColumnDesc(TypeInfoFactory.intTypeInfo, + ExprNodeDesc inputExpr = new ExprNodeColumnDesc(VirtualColumn.GROUPINGID.getTypeInfo(), groupingSetColumnName, null, false); reduceKeys.add(inputExpr); @@ -4841,11 +4847,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { */ @SuppressWarnings("nls") private Operator genGroupByPlanGroupByOperator1(QBParseInfo parseInfo, - String dest, Operator reduceSinkOperatorInfo, GroupByDesc.Mode mode, - Map<String, GenericUDAFEvaluator> genericUDAFEvaluators, - List<Integer> groupingSets, - boolean groupingSetsPresent, - boolean groupingSetsNeedAdditionalMRJob) throws SemanticException { + String dest, Operator reduceSinkOperatorInfo, GroupByDesc.Mode mode, + Map<String, GenericUDAFEvaluator> genericUDAFEvaluators, + List<Long> groupingSets, + boolean groupingSetsPresent, + boolean groupingSetsNeedAdditionalMRJob) throws SemanticException { ArrayList<String> outputColumnNames = new ArrayList<String>(); RowResolver groupByInputRowResolver = opParseCtx .get(reduceSinkOperatorInfo).getRowResolver(); @@ -5037,14 +5043,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { Map<String, ExprNodeDesc> colExprMap) { // The value for the constant does not matter. It is replaced by the grouping set // value for the actual implementation - ExprNodeConstantDesc constant = new ExprNodeConstantDesc(0); + ExprNodeConstantDesc constant = new ExprNodeConstantDesc(VirtualColumn.GROUPINGID.getTypeInfo(), 0L); groupByKeys.add(constant); String field = getColumnInternalName(groupByKeys.size() - 1); outputColumnNames.add(field); groupByOutputRowResolver.put(null, VirtualColumn.GROUPINGID.getName(), new ColumnInfo( field, - TypeInfoFactory.intTypeInfo, + VirtualColumn.GROUPINGID.getTypeInfo(), null, true)); colExprMap.put(field, constant); @@ -5065,13 +5071,13 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { */ @SuppressWarnings("nls") private Operator genGroupByPlanMapGroupByOperator(QB qb, - String dest, - List<ASTNode> grpByExprs, - Operator inputOperatorInfo, - GroupByDesc.Mode mode, - Map<String, GenericUDAFEvaluator> genericUDAFEvaluators, - List<Integer> groupingSetKeys, - boolean groupingSetsPresent) throws SemanticException { + String dest, + List<ASTNode> grpByExprs, + Operator inputOperatorInfo, + GroupByDesc.Mode mode, + Map<String, GenericUDAFEvaluator> genericUDAFEvaluators, + List<Long> groupingSetKeys, + boolean groupingSetsPresent) throws SemanticException { RowResolver groupByInputRowResolver = opParseCtx.get(inputOperatorInfo) .getRowResolver(); @@ -5770,11 +5776,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { QBParseInfo parseInfo = qb.getParseInfo(); int numReducers = -1; - ObjectPair<List<ASTNode>, List<Integer>> grpByExprsGroupingSets = + ObjectPair<List<ASTNode>, List<Long>> grpByExprsGroupingSets = getGroupByGroupingSetsForClause(parseInfo, dest); List<ASTNode> grpByExprs = grpByExprsGroupingSets.getFirst(); - List<Integer> groupingSets = grpByExprsGroupingSets.getSecond(); + List<Long> groupingSets = grpByExprsGroupingSets.getSecond(); if (grpByExprs.isEmpty()) { numReducers = 1; @@ -5819,10 +5825,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { List<ExprNodeDesc.ExprNodeDescEqualityWrapper> whereExpressions = new ArrayList<ExprNodeDesc.ExprNodeDescEqualityWrapper>(); for (String dest : dests) { - ObjectPair<List<ASTNode>, List<Integer>> grpByExprsGroupingSets = + ObjectPair<List<ASTNode>, List<Long>> grpByExprsGroupingSets = getGroupByGroupingSetsForClause(parseInfo, dest); - List<Integer> groupingSets = grpByExprsGroupingSets.getSecond(); + List<Long> groupingSets = grpByExprsGroupingSets.getSecond(); if (!groupingSets.isEmpty()) { throw new SemanticException(ErrorMsg.HIVE_GROUPING_SETS_AGGR_NOMAPAGGR_MULTIGBY.getMsg()); } @@ -5965,11 +5971,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { QBParseInfo parseInfo = qb.getParseInfo(); - ObjectPair<List<ASTNode>, List<Integer>> grpByExprsGroupingSets = + ObjectPair<List<ASTNode>, List<Long>> grpByExprsGroupingSets = getGroupByGroupingSetsForClause(parseInfo, dest); List<ASTNode> grpByExprs = grpByExprsGroupingSets.getFirst(); - List<Integer> groupingSets = grpByExprsGroupingSets.getSecond(); + List<Long> groupingSets = grpByExprsGroupingSets.getSecond(); // Grouping sets are not allowed // This restriction can be lifted in future. @@ -6161,11 +6167,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { Operator inputOperatorInfo) throws SemanticException { QBParseInfo parseInfo = qb.getParseInfo(); - ObjectPair<List<ASTNode>, List<Integer>> grpByExprsGroupingSets = + ObjectPair<List<ASTNode>, List<Long>> grpByExprsGroupingSets = getGroupByGroupingSetsForClause(parseInfo, dest); List<ASTNode> grpByExprs = grpByExprsGroupingSets.getFirst(); - List<Integer> groupingSets = grpByExprsGroupingSets.getSecond(); + List<Long> groupingSets = grpByExprsGroupingSets.getSecond(); boolean groupingSetsPresent = !groupingSets.isEmpty(); int newMRJobGroupingSetsThreshold = @@ -6330,11 +6336,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { QBParseInfo parseInfo = qb.getParseInfo(); - ObjectPair<List<ASTNode>, List<Integer>> grpByExprsGroupingSets = + ObjectPair<List<ASTNode>, List<Long>> grpByExprsGroupingSets = getGroupByGroupingSetsForClause(parseInfo, dest); List<ASTNode> grpByExprs = grpByExprsGroupingSets.getFirst(); - List<Integer> groupingSets = grpByExprsGroupingSets.getSecond(); + List<Long> groupingSets = grpByExprsGroupingSets.getSecond(); boolean groupingSetsPresent = !groupingSets.isEmpty(); if (groupingSetsPresent) { http://git-wip-us.apache.org/repos/asf/hive/blob/ddd4c9ae/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java index e90a398..86cc77d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java @@ -70,7 +70,7 @@ public class GroupByDesc extends AbstractOperatorDesc { private boolean bucketGroup; private ArrayList<ExprNodeDesc> keys; - private List<Integer> listGroupingSets; + private List<Long> listGroupingSets; private boolean groupingSetsPresent; private int groupingSetPosition = -1; // /* in case of grouping sets; groupby1 will output values for every setgroup; this is the index of the column that information will be sent */ private ArrayList<org.apache.hadoop.hive.ql.plan.AggregationDesc> aggregators; @@ -90,7 +90,7 @@ public class GroupByDesc extends AbstractOperatorDesc { final ArrayList<org.apache.hadoop.hive.ql.plan.AggregationDesc> aggregators, final float groupByMemoryUsage, final float memoryThreshold, - final List<Integer> listGroupingSets, + final List<Long> listGroupingSets, final boolean groupingSetsPresent, final int groupingSetsPosition, final boolean isDistinct) { @@ -107,7 +107,7 @@ public class GroupByDesc extends AbstractOperatorDesc { final boolean bucketGroup, final float groupByMemoryUsage, final float memoryThreshold, - final List<Integer> listGroupingSets, + final List<Long> listGroupingSets, final boolean groupingSetsPresent, final int groupingSetsPosition, final boolean isDistinct) { @@ -267,11 +267,11 @@ public class GroupByDesc extends AbstractOperatorDesc { // in which case the group by would execute as a single map-reduce job. // For the group-by, the group by keys should be: a,b,groupingSet(for rollup), c // So, the starting position of grouping set need to be known - public List<Integer> getListGroupingSets() { + public List<Long> getListGroupingSets() { return listGroupingSets; } - public void setListGroupingSets(final List<Integer> listGroupingSets) { + public void setListGroupingSets(final List<Long> listGroupingSets) { this.listGroupingSets = listGroupingSets; } @@ -315,7 +315,7 @@ public class GroupByDesc extends AbstractOperatorDesc { keys.addAll(this.keys); ArrayList<org.apache.hadoop.hive.ql.plan.AggregationDesc> aggregators = new ArrayList<>(); aggregators.addAll(this.aggregators); - List<Integer> listGroupingSets = new ArrayList<>(); + List<Long> listGroupingSets = new ArrayList<>(); listGroupingSets.addAll(this.listGroupingSets); return new GroupByDesc(this.mode, outputColumnNames, keys, aggregators, this.groupByMemoryUsage, this.memoryThreshold, listGroupingSets, this.groupingSetsPresent, http://git-wip-us.apache.org/repos/asf/hive/blob/ddd4c9ae/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFGrouping.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFGrouping.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFGrouping.java index c0c3015..d11e51b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFGrouping.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFGrouping.java @@ -23,17 +23,18 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.udf.UDFType; +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableConstantIntObjectInspector; -import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableConstantLongObjectInspector; +import org.apache.hadoop.io.LongWritable; -import com.google.common.math.IntMath; +import com.google.common.math.LongMath; /** * UDF grouping @@ -45,9 +46,9 @@ extended = "a is the grouping id, p1...pn are the indices we want to extract") @UDFType(deterministic = true) public class GenericUDFGrouping extends GenericUDF { - private transient IntObjectInspector groupingIdOI; + private transient PrimitiveObjectInspector groupingIdOI; private int[] indices; - private IntWritable intWritable = new IntWritable(); + private LongWritable longWritable = new LongWritable(); @Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { @@ -60,37 +61,41 @@ public class GenericUDFGrouping extends GenericUDF { throw new UDFArgumentTypeException(0, "The first argument to grouping() must be primitive"); } PrimitiveObjectInspector arg1OI = (PrimitiveObjectInspector) arguments[0]; - if (arg1OI.getPrimitiveCategory() != PrimitiveCategory.INT) { - throw new UDFArgumentTypeException(0, "The first argument to grouping() must be an integer"); + // INT can happen in cases where grouping() is used without grouping sets, in all other cases it should be LONG. + if (!(arg1OI.getPrimitiveCategory() == PrimitiveCategory.INT || + arg1OI.getPrimitiveCategory() == PrimitiveCategory.LONG)) { + throw new UDFArgumentTypeException(0, + "The first argument to grouping() must be an int/long. Got: " + arg1OI.getPrimitiveCategory()); } - groupingIdOI = (IntObjectInspector) arguments[0]; + groupingIdOI = arg1OI; indices = new int[arguments.length - 1]; for (int i = 1; i < arguments.length; i++) { PrimitiveObjectInspector arg2OI = (PrimitiveObjectInspector) arguments[i]; - if (!(arg2OI instanceof WritableConstantIntObjectInspector)) { - throw new UDFArgumentTypeException(i, "Must be a constant"); + if (!(arg2OI instanceof ConstantObjectInspector)) { + throw new UDFArgumentTypeException(i, "Must be a constant. Got: " + arg2OI.getClass().getSimpleName()); } - indices[i - 1] = ((WritableConstantIntObjectInspector)arg2OI).getWritableConstantValue().get(); + indices[i - 1] = PrimitiveObjectInspectorUtils + .getInt(((ConstantObjectInspector) arguments[i]).getWritableConstantValue(), arg2OI); } - return PrimitiveObjectInspectorFactory.writableIntObjectInspector; + return PrimitiveObjectInspectorFactory.writableLongObjectInspector; } @Override public Object evaluate(DeferredObject[] arguments) throws HiveException { // groupingId = PrimitiveObjectInspectorUtils.getInt(arguments[0].get(), groupingIdOI); // Check that the bit at the given index is '1' or '0' - int result = 0; + long result = 0; // grouping(c1, c2, c3) // is equivalent to // 4 * grouping(c1) + 2 * grouping(c2) + grouping(c3) for (int a = 1; a < arguments.length; a++) { - result += IntMath.pow(2, indices.length - a) * - ((PrimitiveObjectInspectorUtils.getInt(arguments[0].get(), groupingIdOI) >> indices[a - 1]) & 1); + result += LongMath.pow(2, indices.length - a) * + ((PrimitiveObjectInspectorUtils.getLong(arguments[0].get(), groupingIdOI) >> indices[a - 1]) & 1); } - intWritable.set(result); - return intWritable; + longWritable.set(result); + return longWritable; } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/ddd4c9ae/ql/src/test/queries/clientnegative/groupby_cube3.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/groupby_cube3.q b/ql/src/test/queries/clientnegative/groupby_cube3.q new file mode 100644 index 0000000..2804dbe --- /dev/null +++ b/ql/src/test/queries/clientnegative/groupby_cube3.q @@ -0,0 +1,90 @@ +create table facts (val string); + +insert into facts values ('abcdefghijklmnopqrstuvwxyz0123456789'); + +create table groupingsets65 as +select +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32,c33,c34,c35,c36,c37,c38,c39, +c40,c41,c42,c43,c44,c45,c46,c47,c48,c49, +c50,c51,c52,c53,c54,c55,c56,c57,c58,c59, +c60,c61,c62,c63,c64 +,count(*) as n from ( +select +substring(val,01,1) as c00, +substring(val,02,1) as c01, +substring(val,03,1) as c02, +substring(val,04,1) as c03, +substring(val,05,1) as c04, +substring(val,06,1) as c05, +substring(val,07,1) as c06, +substring(val,08,1) as c07, +substring(val,09,1) as c08, +substring(val,10,1) as c09, +substring(val,11,1) as c10, +substring(val,12,1) as c11, +substring(val,13,1) as c12, +substring(val,14,1) as c13, +substring(val,15,1) as c14, +substring(val,16,1) as c15, +substring(val,17,1) as c16, +substring(val,18,1) as c17, +substring(val,19,1) as c18, +substring(val,20,1) as c19, +substring(val,21,1) as c20, +substring(val,22,1) as c21, +substring(val,23,1) as c22, +substring(val,24,1) as c23, +substring(val,25,1) as c24, +substring(val,26,1) as c25, +substring(val,27,1) as c26, +substring(val,28,1) as c27, +substring(val,29,1) as c28, +substring(val,30,1) as c29, +substring(val,31,1) as c30, +substring(val,32,1) as c31, +substring(val,01,1) as c32, +substring(val,02,1) as c33, +substring(val,03,1) as c34, +substring(val,04,1) as c35, +substring(val,05,1) as c36, +substring(val,06,1) as c37, +substring(val,07,1) as c38, +substring(val,08,1) as c39, +substring(val,09,1) as c40, +substring(val,10,1) as c41, +substring(val,11,1) as c42, +substring(val,12,1) as c43, +substring(val,13,1) as c44, +substring(val,14,1) as c45, +substring(val,15,1) as c46, +substring(val,16,1) as c47, +substring(val,17,1) as c48, +substring(val,18,1) as c49, +substring(val,19,1) as c50, +substring(val,20,1) as c51, +substring(val,21,1) as c52, +substring(val,22,1) as c53, +substring(val,23,1) as c54, +substring(val,24,1) as c55, +substring(val,25,1) as c56, +substring(val,26,1) as c57, +substring(val,27,1) as c58, +substring(val,28,1) as c59, +substring(val,29,1) as c60, +substring(val,30,1) as c61, +substring(val,31,1) as c62, +substring(val,32,1) as c63, +substring(val,32,1) as c64 +from facts ) x +group by +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32,c33,c34,c35,c36,c37,c38,c39, +c40,c41,c42,c43,c44,c45,c46,c47,c48,c49, +c50,c51,c52,c53,c54,c55,c56,c57,c58,c59, +c60,c61,c62,c63,c64 +with cube; http://git-wip-us.apache.org/repos/asf/hive/blob/ddd4c9ae/ql/src/test/queries/clientnegative/groupby_grouping_sets8.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/groupby_grouping_sets8.q b/ql/src/test/queries/clientnegative/groupby_grouping_sets8.q new file mode 100644 index 0000000..082da7a --- /dev/null +++ b/ql/src/test/queries/clientnegative/groupby_grouping_sets8.q @@ -0,0 +1,98 @@ +create table facts (val string); + +insert into facts values ('abcdefghijklmnopqrstuvwxyz0123456789'); + +create table groupingsets65 as +select +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32,c33,c34,c35,c36,c37,c38,c39, +c40,c41,c42,c43,c44,c45,c46,c47,c48,c49, +c50,c51,c52,c53,c54,c55,c56,c57,c58,c59, +c60,c61,c62,c63,c64 +,count(*) as n from ( +select +substring(val,01,1) as c00, +substring(val,02,1) as c01, +substring(val,03,1) as c02, +substring(val,04,1) as c03, +substring(val,05,1) as c04, +substring(val,06,1) as c05, +substring(val,07,1) as c06, +substring(val,08,1) as c07, +substring(val,09,1) as c08, +substring(val,10,1) as c09, +substring(val,11,1) as c10, +substring(val,12,1) as c11, +substring(val,13,1) as c12, +substring(val,14,1) as c13, +substring(val,15,1) as c14, +substring(val,16,1) as c15, +substring(val,17,1) as c16, +substring(val,18,1) as c17, +substring(val,19,1) as c18, +substring(val,20,1) as c19, +substring(val,21,1) as c20, +substring(val,22,1) as c21, +substring(val,23,1) as c22, +substring(val,24,1) as c23, +substring(val,25,1) as c24, +substring(val,26,1) as c25, +substring(val,27,1) as c26, +substring(val,28,1) as c27, +substring(val,29,1) as c28, +substring(val,30,1) as c29, +substring(val,31,1) as c30, +substring(val,32,1) as c31, +substring(val,01,1) as c32, +substring(val,02,1) as c33, +substring(val,03,1) as c34, +substring(val,04,1) as c35, +substring(val,05,1) as c36, +substring(val,06,1) as c37, +substring(val,07,1) as c38, +substring(val,08,1) as c39, +substring(val,09,1) as c40, +substring(val,10,1) as c41, +substring(val,11,1) as c42, +substring(val,12,1) as c43, +substring(val,13,1) as c44, +substring(val,14,1) as c45, +substring(val,15,1) as c46, +substring(val,16,1) as c47, +substring(val,17,1) as c48, +substring(val,18,1) as c49, +substring(val,19,1) as c50, +substring(val,20,1) as c51, +substring(val,21,1) as c52, +substring(val,22,1) as c53, +substring(val,23,1) as c54, +substring(val,24,1) as c55, +substring(val,25,1) as c56, +substring(val,26,1) as c57, +substring(val,27,1) as c58, +substring(val,28,1) as c59, +substring(val,29,1) as c60, +substring(val,30,1) as c61, +substring(val,31,1) as c62, +substring(val,32,1) as c63, +substring(val,32,1) as c64 +from facts ) x +group by +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32,c33,c34,c35,c36,c37,c38,c39, +c40,c41,c42,c43,c44,c45,c46,c47,c48,c49, +c50,c51,c52,c53,c54,c55,c56,c57,c58,c59, +c60,c61,c62,c63,c64 +grouping sets ( +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32,c33,c34,c35,c36,c37,c38,c39, +c40,c41,c42,c43,c44,c45,c46,c47,c48,c49, +c50,c51,c52,c53,c54,c55,c56,c57,c58,c59, +c60,c61,c62,c63,c64 +); http://git-wip-us.apache.org/repos/asf/hive/blob/ddd4c9ae/ql/src/test/queries/clientnegative/groupby_rollup3.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/groupby_rollup3.q b/ql/src/test/queries/clientnegative/groupby_rollup3.q new file mode 100644 index 0000000..c205ca0 --- /dev/null +++ b/ql/src/test/queries/clientnegative/groupby_rollup3.q @@ -0,0 +1,90 @@ +create table facts (val string); + +insert into facts values ('abcdefghijklmnopqrstuvwxyz0123456789'); + +create table groupingsets65 as +select +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32,c33,c34,c35,c36,c37,c38,c39, +c40,c41,c42,c43,c44,c45,c46,c47,c48,c49, +c50,c51,c52,c53,c54,c55,c56,c57,c58,c59, +c60,c61,c62,c63,c64 +,count(*) as n from ( +select +substring(val,01,1) as c00, +substring(val,02,1) as c01, +substring(val,03,1) as c02, +substring(val,04,1) as c03, +substring(val,05,1) as c04, +substring(val,06,1) as c05, +substring(val,07,1) as c06, +substring(val,08,1) as c07, +substring(val,09,1) as c08, +substring(val,10,1) as c09, +substring(val,11,1) as c10, +substring(val,12,1) as c11, +substring(val,13,1) as c12, +substring(val,14,1) as c13, +substring(val,15,1) as c14, +substring(val,16,1) as c15, +substring(val,17,1) as c16, +substring(val,18,1) as c17, +substring(val,19,1) as c18, +substring(val,20,1) as c19, +substring(val,21,1) as c20, +substring(val,22,1) as c21, +substring(val,23,1) as c22, +substring(val,24,1) as c23, +substring(val,25,1) as c24, +substring(val,26,1) as c25, +substring(val,27,1) as c26, +substring(val,28,1) as c27, +substring(val,29,1) as c28, +substring(val,30,1) as c29, +substring(val,31,1) as c30, +substring(val,32,1) as c31, +substring(val,01,1) as c32, +substring(val,02,1) as c33, +substring(val,03,1) as c34, +substring(val,04,1) as c35, +substring(val,05,1) as c36, +substring(val,06,1) as c37, +substring(val,07,1) as c38, +substring(val,08,1) as c39, +substring(val,09,1) as c40, +substring(val,10,1) as c41, +substring(val,11,1) as c42, +substring(val,12,1) as c43, +substring(val,13,1) as c44, +substring(val,14,1) as c45, +substring(val,15,1) as c46, +substring(val,16,1) as c47, +substring(val,17,1) as c48, +substring(val,18,1) as c49, +substring(val,19,1) as c50, +substring(val,20,1) as c51, +substring(val,21,1) as c52, +substring(val,22,1) as c53, +substring(val,23,1) as c54, +substring(val,24,1) as c55, +substring(val,25,1) as c56, +substring(val,26,1) as c57, +substring(val,27,1) as c58, +substring(val,28,1) as c59, +substring(val,29,1) as c60, +substring(val,30,1) as c61, +substring(val,31,1) as c62, +substring(val,32,1) as c63, +substring(val,32,1) as c64 +from facts ) x +group by +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32,c33,c34,c35,c36,c37,c38,c39, +c40,c41,c42,c43,c44,c45,c46,c47,c48,c49, +c50,c51,c52,c53,c54,c55,c56,c57,c58,c59, +c60,c61,c62,c63,c64 +with rollup; http://git-wip-us.apache.org/repos/asf/hive/blob/ddd4c9ae/ql/src/test/queries/clientpositive/cte_1.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/cte_1.q b/ql/src/test/queries/clientpositive/cte_1.q index 15d3f06..6d4e05d 100644 --- a/ql/src/test/queries/clientpositive/cte_1.q +++ b/ql/src/test/queries/clientpositive/cte_1.q @@ -60,4 +60,4 @@ create table cte10_t1 as with q1 as (select cint, cstring1 from alltypesorc where cint > 70) select * from q1; with q1 as (select cint , cstring1 from alltypesorc where age < 50) - select * from cte10_t1; \ No newline at end of file + select * from cte10_t1; http://git-wip-us.apache.org/repos/asf/hive/blob/ddd4c9ae/ql/src/test/queries/clientpositive/groupingset_high_columns.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/groupingset_high_columns.q b/ql/src/test/queries/clientpositive/groupingset_high_columns.q new file mode 100644 index 0000000..977ced6 --- /dev/null +++ b/ql/src/test/queries/clientpositive/groupingset_high_columns.q @@ -0,0 +1,259 @@ +create table facts (val string); + +insert into facts values ('abcdefghijklmnopqrstuvwxyz0123456789'); + +set hive.vectorized.execution.enabled=false; +drop table groupingsets32; +drop table groupingsets33; +drop table groupingsets32a; +drop table groupingsets33a; + +create table groupingsets32 as +select +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31 +,count(*) as n from ( +select +substring(val,01,1) as c00, substring(val,02,1) as c01, substring(val,03,1) as c02,substring(val,04,1) as c03,substring(val,05,1) as c04,substring(val,06,1) as c05,substring(val,07,1) as c06, substring(val,08,1) as c07,substring(val,09,1) as c08,substring(val,10,1) as c09, +substring(val,11,1) as c10, substring(val,12,1) as c11, substring(val,13,1) as c12,substring(val,14,1) as c13,substring(val,15,1) as c14,substring(val,16,1) as c15,substring(val,17,1) as c16, substring(val,18,1) as c17,substring(val,19,1) as c18,substring(val,20,1) as c19, +substring(val,21,1) as c20, substring(val,22,1) as c21, substring(val,23,1) as c22,substring(val,24,1) as c23,substring(val,25,1) as c24,substring(val,26,1) as c25,substring(val,27,1) as c26, substring(val,28,1) as c27,substring(val,29,1) as c28,substring(val,30,1) as c29, +substring(val,31,1) as c30,substring(val,32,1) as c31 +from facts ) x +group by +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31 +grouping sets ( +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31 +); + +select * from groupingsets32; + +create table groupingsets32a as +select +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31, +count(*) as n, +grouping__id, +grouping(c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31) +from ( +select +substring(val,01,1) as c00, substring(val,02,1) as c01, substring(val,03,1) as c02,substring(val,04,1) as c03,substring(val,05,1) as c04,substring(val,06,1) as c05,substring(val,07,1) as c06, substring(val,08,1) as c07,substring(val,09,1) as c08,substring(val,10,1) as c09, +substring(val,11,1) as c10, substring(val,12,1) as c11, substring(val,13,1) as c12,substring(val,14,1) as c13,substring(val,15,1) as c14,substring(val,16,1) as c15,substring(val,17,1) as c16, substring(val,18,1) as c17,substring(val,19,1) as c18,substring(val,20,1) as c19, +substring(val,21,1) as c20, substring(val,22,1) as c21, substring(val,23,1) as c22,substring(val,24,1) as c23,substring(val,25,1) as c24,substring(val,26,1) as c25,substring(val,27,1) as c26, substring(val,28,1) as c27,substring(val,29,1) as c28,substring(val,30,1) as c29, +substring(val,31,1) as c30,substring(val,32,1) as c31 +from facts ) x +group by +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31 +grouping sets ( +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31 +); + +select * from groupingsets32a; + +create table groupingsets33 as +select +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32 +,count(*) as n from ( +select +substring(val,01,1) as c00, substring(val,02,1) as c01, substring(val,03,1) as c02,substring(val,04,1) as c03,substring(val,05,1) as c04,substring(val,06,1) as c05,substring(val,07,1) as c06, substring(val,08,1) as c07,substring(val,09,1) as c08,substring(val,10,1) as c09, +substring(val,11,1) as c10, substring(val,12,1) as c11, substring(val,13,1) as c12,substring(val,14,1) as c13,substring(val,15,1) as c14,substring(val,16,1) as c15,substring(val,17,1) as c16, substring(val,18,1) as c17,substring(val,19,1) as c18,substring(val,20,1) as c19, +substring(val,21,1) as c20, substring(val,22,1) as c21, substring(val,23,1) as c22,substring(val,24,1) as c23,substring(val,25,1) as c24,substring(val,26,1) as c25,substring(val,27,1) as c26, substring(val,28,1) as c27,substring(val,29,1) as c28,substring(val,30,1) as c29, +substring(val,31,1) as c30,substring(val,32,1) as c31,substring(val,33,1) as c32 +from facts ) x +group by +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32 +grouping sets ( +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32 +) ; + +select * from groupingsets33; + +create table groupingsets33a as +select +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32 +,count(*) as n, +grouping__id, +grouping(c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32) +from ( +select +substring(val,01,1) as c00, substring(val,02,1) as c01, substring(val,03,1) as c02,substring(val,04,1) as c03,substring(val,05,1) as c04,substring(val,06,1) as c05,substring(val,07,1) as c06, substring(val,08,1) as c07,substring(val,09,1) as c08,substring(val,10,1) as c09, +substring(val,11,1) as c10, substring(val,12,1) as c11, substring(val,13,1) as c12,substring(val,14,1) as c13,substring(val,15,1) as c14,substring(val,16,1) as c15,substring(val,17,1) as c16, substring(val,18,1) as c17,substring(val,19,1) as c18,substring(val,20,1) as c19, +substring(val,21,1) as c20, substring(val,22,1) as c21, substring(val,23,1) as c22,substring(val,24,1) as c23,substring(val,25,1) as c24,substring(val,26,1) as c25,substring(val,27,1) as c26, substring(val,28,1) as c27,substring(val,29,1) as c28,substring(val,30,1) as c29, +substring(val,31,1) as c30,substring(val,32,1) as c31,substring(val,33,1) as c32 +from facts ) x +group by +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32 +grouping sets ( +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32 +) ; + +select * from groupingsets33a; + +set hive.vectorized.execution.enabled=true; + +drop table groupingsets32; +drop table groupingsets33; + +drop table groupingsets32; +drop table groupingsets33; +drop table groupingsets32a; +drop table groupingsets33a; + +create table groupingsets32 as +select +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31 +,count(*) as n from ( +select +substring(val,01,1) as c00, substring(val,02,1) as c01, substring(val,03,1) as c02,substring(val,04,1) as c03,substring(val,05,1) as c04,substring(val,06,1) as c05,substring(val,07,1) as c06, substring(val,08,1) as c07,substring(val,09,1) as c08,substring(val,10,1) as c09, +substring(val,11,1) as c10, substring(val,12,1) as c11, substring(val,13,1) as c12,substring(val,14,1) as c13,substring(val,15,1) as c14,substring(val,16,1) as c15,substring(val,17,1) as c16, substring(val,18,1) as c17,substring(val,19,1) as c18,substring(val,20,1) as c19, +substring(val,21,1) as c20, substring(val,22,1) as c21, substring(val,23,1) as c22,substring(val,24,1) as c23,substring(val,25,1) as c24,substring(val,26,1) as c25,substring(val,27,1) as c26, substring(val,28,1) as c27,substring(val,29,1) as c28,substring(val,30,1) as c29, +substring(val,31,1) as c30,substring(val,32,1) as c31 +from facts ) x +group by +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31 +grouping sets ( +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31 +); + +select * from groupingsets32; + +create table groupingsets32a as +select +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31, +count(*) as n, +grouping__id, +grouping(c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31) +from ( +select +substring(val,01,1) as c00, substring(val,02,1) as c01, substring(val,03,1) as c02,substring(val,04,1) as c03,substring(val,05,1) as c04,substring(val,06,1) as c05,substring(val,07,1) as c06, substring(val,08,1) as c07,substring(val,09,1) as c08,substring(val,10,1) as c09, +substring(val,11,1) as c10, substring(val,12,1) as c11, substring(val,13,1) as c12,substring(val,14,1) as c13,substring(val,15,1) as c14,substring(val,16,1) as c15,substring(val,17,1) as c16, substring(val,18,1) as c17,substring(val,19,1) as c18,substring(val,20,1) as c19, +substring(val,21,1) as c20, substring(val,22,1) as c21, substring(val,23,1) as c22,substring(val,24,1) as c23,substring(val,25,1) as c24,substring(val,26,1) as c25,substring(val,27,1) as c26, substring(val,28,1) as c27,substring(val,29,1) as c28,substring(val,30,1) as c29, +substring(val,31,1) as c30,substring(val,32,1) as c31 +from facts ) x +group by +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31 +grouping sets ( +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31 +); + +select * from groupingsets32a; + +create table groupingsets33 as +select +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32 +,count(*) as n from ( +select +substring(val,01,1) as c00, substring(val,02,1) as c01, substring(val,03,1) as c02,substring(val,04,1) as c03,substring(val,05,1) as c04,substring(val,06,1) as c05,substring(val,07,1) as c06, substring(val,08,1) as c07,substring(val,09,1) as c08,substring(val,10,1) as c09, +substring(val,11,1) as c10, substring(val,12,1) as c11, substring(val,13,1) as c12,substring(val,14,1) as c13,substring(val,15,1) as c14,substring(val,16,1) as c15,substring(val,17,1) as c16, substring(val,18,1) as c17,substring(val,19,1) as c18,substring(val,20,1) as c19, +substring(val,21,1) as c20, substring(val,22,1) as c21, substring(val,23,1) as c22,substring(val,24,1) as c23,substring(val,25,1) as c24,substring(val,26,1) as c25,substring(val,27,1) as c26, substring(val,28,1) as c27,substring(val,29,1) as c28,substring(val,30,1) as c29, +substring(val,31,1) as c30,substring(val,32,1) as c31,substring(val,33,1) as c32 +from facts ) x +group by +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32 +grouping sets ( +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32 +) ; + +select * from groupingsets33; + +create table groupingsets33a as +select +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32 +,count(*) as n, +grouping__id, +grouping(c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32) +from ( +select +substring(val,01,1) as c00, substring(val,02,1) as c01, substring(val,03,1) as c02,substring(val,04,1) as c03,substring(val,05,1) as c04,substring(val,06,1) as c05,substring(val,07,1) as c06, substring(val,08,1) as c07,substring(val,09,1) as c08,substring(val,10,1) as c09, +substring(val,11,1) as c10, substring(val,12,1) as c11, substring(val,13,1) as c12,substring(val,14,1) as c13,substring(val,15,1) as c14,substring(val,16,1) as c15,substring(val,17,1) as c16, substring(val,18,1) as c17,substring(val,19,1) as c18,substring(val,20,1) as c19, +substring(val,21,1) as c20, substring(val,22,1) as c21, substring(val,23,1) as c22,substring(val,24,1) as c23,substring(val,25,1) as c24,substring(val,26,1) as c25,substring(val,27,1) as c26, substring(val,28,1) as c27,substring(val,29,1) as c28,substring(val,30,1) as c29, +substring(val,31,1) as c30,substring(val,32,1) as c31,substring(val,33,1) as c32 +from facts ) x +group by +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32 +grouping sets ( +c00,c01,c02,c03,c04,c05,c06,c07,c08,c09, +c10,c11,c12,c13,c14,c15,c16,c17,c18,c19, +c20,c21,c22,c23,c24,c25,c26,c27,c28,c29, +c30,c31,c32 +) ; + +select * from groupingsets33a; http://git-wip-us.apache.org/repos/asf/hive/blob/ddd4c9ae/ql/src/test/results/clientnegative/groupby_cube3.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/groupby_cube3.q.out b/ql/src/test/results/clientnegative/groupby_cube3.q.out new file mode 100644 index 0000000..937cc01 --- /dev/null +++ b/ql/src/test/results/clientnegative/groupby_cube3.q.out @@ -0,0 +1,18 @@ +PREHOOK: query: create table facts (val string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@facts +POSTHOOK: query: create table facts (val string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@facts +PREHOOK: query: insert into facts values ('abcdefghijklmnopqrstuvwxyz0123456789') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@facts +POSTHOOK: query: insert into facts values ('abcdefghijklmnopqrstuvwxyz0123456789') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@facts +POSTHOOK: Lineage: facts.val SCRIPT [] +FAILED: SemanticException [Error 10411]: Grouping sets size cannot be greater than 64 http://git-wip-us.apache.org/repos/asf/hive/blob/ddd4c9ae/ql/src/test/results/clientnegative/groupby_grouping_sets8.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/groupby_grouping_sets8.q.out b/ql/src/test/results/clientnegative/groupby_grouping_sets8.q.out new file mode 100644 index 0000000..937cc01 --- /dev/null +++ b/ql/src/test/results/clientnegative/groupby_grouping_sets8.q.out @@ -0,0 +1,18 @@ +PREHOOK: query: create table facts (val string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@facts +POSTHOOK: query: create table facts (val string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@facts +PREHOOK: query: insert into facts values ('abcdefghijklmnopqrstuvwxyz0123456789') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@facts +POSTHOOK: query: insert into facts values ('abcdefghijklmnopqrstuvwxyz0123456789') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@facts +POSTHOOK: Lineage: facts.val SCRIPT [] +FAILED: SemanticException [Error 10411]: Grouping sets size cannot be greater than 64 http://git-wip-us.apache.org/repos/asf/hive/blob/ddd4c9ae/ql/src/test/results/clientnegative/groupby_rollup3.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/groupby_rollup3.q.out b/ql/src/test/results/clientnegative/groupby_rollup3.q.out new file mode 100644 index 0000000..937cc01 --- /dev/null +++ b/ql/src/test/results/clientnegative/groupby_rollup3.q.out @@ -0,0 +1,18 @@ +PREHOOK: query: create table facts (val string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@facts +POSTHOOK: query: create table facts (val string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@facts +PREHOOK: query: insert into facts values ('abcdefghijklmnopqrstuvwxyz0123456789') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@facts +POSTHOOK: query: insert into facts values ('abcdefghijklmnopqrstuvwxyz0123456789') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@facts +POSTHOOK: Lineage: facts.val SCRIPT [] +FAILED: SemanticException [Error 10411]: Grouping sets size cannot be greater than 64