svn commit: r1627552 [1/3] - in /hive/branches/spark: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/ ql/src/test/results/clientpositive/spark/

[xuefu]

Author: xuefu
Date: Thu Sep 25 14:39:02 2014
New Revision: 1627552

URL: http://svn.apache.org/r1627552
Log:
HIVE-8106: Enable vectorization for spark [spark branch] (Chinna via Xuefu)

Added:
    
hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_13.q.out
    
hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorized_shufflejoin.q.out
Modified:
    hive/branches/spark/itests/src/test/resources/testconfiguration.properties
    
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java

Modified: 
hive/branches/spark/itests/src/test/resources/testconfiguration.properties
URL: 
http://svn.apache.org/viewvc/hive/branches/spark/itests/src/test/resources/testconfiguration.properties?rev=1627552&r1=1627551&r2=1627552&view=diff
==============================================================================
--- hive/branches/spark/itests/src/test/resources/testconfiguration.properties 
(original)
+++ hive/branches/spark/itests/src/test/resources/testconfiguration.properties 
Thu Sep 25 14:39:02 2014
@@ -547,4 +547,6 @@ spark.query.files=add_part_multiple.q \
   load_dyn_part12.q \
   load_dyn_part13.q \
   load_dyn_part14.q \
-  load_dyn_part15.q
+  load_dyn_part15.q \
+  vectorization_13.q \
+  vectorized_shufflejoin.q

Modified: 
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL: 
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1627552&r1=1627551&r2=1627552&view=diff
==============================================================================
--- 
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
 (original)
+++ 
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
 Thu Sep 25 14:39:02 2014
@@ -35,6 +35,7 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.*;
 import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
+import org.apache.hadoop.hive.ql.exec.spark.SparkTask;
 import org.apache.hadoop.hive.ql.exec.tez.TezTask;
 import org.apache.hadoop.hive.ql.exec.vector.VectorExtractOperator;
 import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
@@ -71,6 +72,7 @@ import org.apache.hadoop.hive.ql.plan.Op
 import org.apache.hadoop.hive.ql.plan.PartitionDesc;
 import org.apache.hadoop.hive.ql.plan.ReduceWork;
 import org.apache.hadoop.hive.ql.plan.SMBJoinDesc;
+import org.apache.hadoop.hive.ql.plan.SparkWork;
 import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.hive.ql.plan.TableScanDesc;
 import org.apache.hadoop.hive.ql.plan.TezWork;
@@ -308,6 +310,15 @@ public class Vectorizer implements Physi
             }
           }
         }
+      } else if (currTask instanceof SparkTask) {
+       SparkWork sparkWork = (SparkWork) currTask.getWork();
+       for (BaseWork baseWork : sparkWork.getAllWork()) {
+         if (baseWork instanceof MapWork) {
+           convertMapWork((MapWork) baseWork, false);
+         } else if (baseWork instanceof ReduceWork) {
+           convertReduceWork((ReduceWork) baseWork);
+         }
+       }
       }
       return null;
     }