Author: jvs
Date: Tue Aug 2 04:24:52 2011
New Revision: 1153022
URL: http://svn.apache.org/viewvc?rev=1153022&view=rev
Log:
HIVE-2326. Turn off bitmap indexing when map-side aggregation is turned off
(Syed Albiz via jvs)
Added:
hive/trunk/ql/src/test/queries/clientnegative/index_bitmap_no_map_aggr.q
hive/trunk/ql/src/test/results/clientnegative/index_bitmap_no_map_aggr.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java?rev=1153022&r1=1153021&r2=1153022&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java
Tue Aug 2 04:24:52 2011
@@ -101,7 +101,7 @@ public abstract class TableBasedIndexHan
abstract protected Task<?> getIndexBuilderMapRedTask(Set<ReadEntity> inputs,
Set<WriteEntity> outputs,
List<FieldSchema> indexField, boolean partitioned,
PartitionDesc indexTblPartDesc, String indexTableName,
- PartitionDesc baseTablePartDesc, String baseTableName, String dbName);
+ PartitionDesc baseTablePartDesc, String baseTableName, String dbName)
throws HiveException;
protected List<String> getPartKVPairStringArray(
LinkedHashMap<String, String> partSpec) {
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java?rev=1153022&r1=1153021&r2=1153022&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java
Tue Aug 2 04:24:52 2011
@@ -220,7 +220,7 @@ public class BitmapIndexHandler extends
protected Task<?> getIndexBuilderMapRedTask(Set<ReadEntity> inputs,
Set<WriteEntity> outputs,
List<FieldSchema> indexField, boolean partitioned,
PartitionDesc indexTblPartDesc, String indexTableName,
- PartitionDesc baseTablePartDesc, String baseTableName, String dbName) {
+ PartitionDesc baseTablePartDesc, String baseTableName, String dbName)
throws HiveException {
HiveConf conf = new HiveConf(getConf(), BitmapIndexHandler.class);
HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVEROWOFFSET, true);
@@ -280,9 +280,9 @@ public class BitmapIndexHandler extends
}
// Require clusterby ROWOFFSET if map-size aggregation is off.
+ // TODO: Make this work without map side aggregation
if (!conf.get("hive.map.aggr", null).equals("true")) {
- command.append(" CLUSTER BY ");
- command.append(VirtualColumn.ROWOFFSET.getName());
+ throw new HiveException("Cannot construct index without map-side
aggregation");
}
Driver driver = new Driver(conf);
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java?rev=1153022&r1=1153021&r2=1153022&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
Tue Aug 2 04:24:52 2011
@@ -81,7 +81,7 @@ public class CompactIndexHandler extends
protected Task<?> getIndexBuilderMapRedTask(Set<ReadEntity> inputs,
Set<WriteEntity> outputs,
List<FieldSchema> indexField, boolean partitioned,
PartitionDesc indexTblPartDesc, String indexTableName,
- PartitionDesc baseTablePartDesc, String baseTableName, String dbName) {
+ PartitionDesc baseTablePartDesc, String baseTableName, String dbName)
throws HiveException {
String indexCols =
HiveUtils.getUnparsedColumnNamesFromFieldSchema(indexField);
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java?rev=1153022&r1=1153021&r2=1153022&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java
Tue Aug 2 04:24:52 2011
@@ -92,14 +92,19 @@ public class GenericUDAFEWAHBitmap exten
inputOI = (PrimitiveObjectInspector) parameters[0];
return ObjectInspectorFactory
.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
- } else {
- //no map aggregation
+ } else if (m == Mode.PARTIAL2 || m == Mode.FINAL) {
internalMergeOI = (StandardListObjectInspector) parameters[0];
bitmapLongOI =
PrimitiveObjectInspectorFactory.writableLongObjectInspector;
inputOI = PrimitiveObjectInspectorFactory.writableByteObjectInspector;
loi = (StandardListObjectInspector) ObjectInspectorFactory
.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
return loi;
+ } else { // Mode.COMPLETE, ie. no map-side aggregation, requires ordering
+ bitmapLongOI =
PrimitiveObjectInspectorFactory.writableLongObjectInspector;
+ inputOI = PrimitiveObjectInspectorFactory.writableByteObjectInspector;
+ loi = (StandardListObjectInspector) ObjectInspectorFactory
+
.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
+ return loi;
}
}
Added: hive/trunk/ql/src/test/queries/clientnegative/index_bitmap_no_map_aggr.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/index_bitmap_no_map_aggr.q?rev=1153022&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/index_bitmap_no_map_aggr.q
(added)
+++ hive/trunk/ql/src/test/queries/clientnegative/index_bitmap_no_map_aggr.q
Tue Aug 2 04:24:52 2011
@@ -0,0 +1,7 @@
+EXPLAIN
+CREATE INDEX src1_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD;
+
+SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+SET hive.map.aggr=false;
+CREATE INDEX src1_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD;
+ALTER INDEX src1_index ON src REBUILD;
Added:
hive/trunk/ql/src/test/results/clientnegative/index_bitmap_no_map_aggr.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/index_bitmap_no_map_aggr.q.out?rev=1153022&view=auto
==============================================================================
---
hive/trunk/ql/src/test/results/clientnegative/index_bitmap_no_map_aggr.q.out
(added)
+++
hive/trunk/ql/src/test/results/clientnegative/index_bitmap_no_map_aggr.q.out
Tue Aug 2 04:24:52 2011
@@ -0,0 +1,21 @@
+PREHOOK: query: EXPLAIN
+CREATE INDEX src1_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD
+PREHOOK: type: CREATEINDEX
+POSTHOOK: query: EXPLAIN
+CREATE INDEX src1_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD
+POSTHOOK: type: CREATEINDEX
+ABSTRACT SYNTAX TREE:
+ (TOK_CREATEINDEX src1_index 'BITMAP' (TOK_TABNAME src) (TOK_TABCOLNAME key)
TOK_DEFERRED_REBUILDINDEX)
+
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+
+
+PREHOOK: query: CREATE INDEX src1_index ON TABLE src(key) as 'BITMAP' WITH
DEFERRED REBUILD
+PREHOOK: type: CREATEINDEX
+POSTHOOK: query: CREATE INDEX src1_index ON TABLE src(key) as 'BITMAP' WITH
DEFERRED REBUILD
+POSTHOOK: type: CREATEINDEX
+FAILED: Error in semantic analysis:
org.apache.hadoop.hive.ql.parse.SemanticException:
org.apache.hadoop.hive.ql.metadata.HiveException: Cannot construct index
without map-side aggregation
