http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java index 62ae630..7cc534b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java @@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.persistence.RowContainer; import org.apache.hadoop.hive.ql.io.HiveInputFormat; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -85,7 +86,13 @@ public class SMBMapJoinOperator extends AbstractMapJoinOperator<SMBJoinDesc> imp // performed as a smb join, based on all the tables/partitions being joined. private transient boolean convertedAutomaticallySMBJoin = false; - public SMBMapJoinOperator() { + /** Kryo ctor. */ + protected SMBMapJoinOperator() { + super(); + } + + public SMBMapJoinOperator(CompilationOpContext ctx) { + super(ctx); } public SMBMapJoinOperator(AbstractMapJoinOperator<? extends MapJoinDesc> mapJoinOp) {
http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java index 63837ce..9f311a4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.ScriptDesc; @@ -266,6 +267,15 @@ public class ScriptOperator extends Operator<ScriptDesc> implements } } + /** Kryo ctor. */ + protected ScriptOperator() { + super(); + } + + public ScriptOperator(CompilationOpContext ctx) { + super(ctx); + } + @Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java index b1b8459..e7c23e8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java @@ -25,6 +25,7 @@ import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -43,6 +44,15 @@ public class SelectOperator extends Operator<SelectDesc> implements Serializable private transient boolean isSelectStarNoCompute = false; + /** Kryo ctor. */ + protected SelectOperator() { + super(); + } + + public SelectOperator(CompilationOpContext ctx) { + super(ctx); + } + @Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java index 45f50c3..0e6e2de 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hive.ql.exec; +import java.util.LinkedList; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.InputStream; @@ -33,6 +35,7 @@ import java.util.List; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; @@ -356,26 +359,64 @@ public class SerializationUtilities { // TODO: need proper clone. Meanwhile, let's at least keep this horror in one place PerfLogger perfLogger = SessionState.getPerfLogger(); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.CLONE_PLAN); + Operator<?> op = plan.getAnyOperator(); + CompilationOpContext ctx = (op == null) ? null : op.getCompilationOpContext(); ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); serializePlan(plan, baos, true); MapredWork newPlan = deserializePlan(new ByteArrayInputStream(baos.toByteArray()), MapredWork.class, true); + // Restore the context. + for (Operator<?> newOp : newPlan.getAllOperators()) { + newOp.setCompilationOpContext(ctx); + } perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.CLONE_PLAN); return newPlan; } /** * Clones using the powers of XML. Do not use unless necessary. + * @param roots The roots. + * @return The clone. + */ + public static List<Operator<?>> cloneOperatorTree(List<Operator<?>> roots) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); + CompilationOpContext ctx = roots.isEmpty() ? null : roots.get(0).getCompilationOpContext(); + serializePlan(roots, baos, true); + @SuppressWarnings("unchecked") + List<Operator<?>> result = + deserializePlan(new ByteArrayInputStream(baos.toByteArray()), + roots.getClass(), true); + // Restore the context. + LinkedList<Operator<?>> newOps = new LinkedList<>(result); + while (!newOps.isEmpty()) { + Operator<?> newOp = newOps.poll(); + newOp.setCompilationOpContext(ctx); + List<Operator<?>> children = newOp.getChildOperators(); + if (children != null) { + newOps.addAll(children); + } + } + return result; + } + + /** + * Clones using the powers of XML. Do not use unless necessary. * @param plan The plan. * @return The clone. */ public static BaseWork cloneBaseWork(BaseWork plan) { PerfLogger perfLogger = SessionState.getPerfLogger(); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.CLONE_PLAN); + Operator<?> op = plan.getAnyRootOperator(); + CompilationOpContext ctx = (op == null) ? null : op.getCompilationOpContext(); ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); serializePlan(plan, baos, true); BaseWork newPlan = deserializePlan(new ByteArrayInputStream(baos.toByteArray()), plan.getClass(), true); + // Restore the context. + for (Operator<?> newOp : newPlan.getAllOperators()) { + newOp.setCompilationOpContext(ctx); + } perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.CLONE_PLAN); return newPlan; } @@ -399,16 +440,6 @@ public class SerializationUtilities { return t; } - public static List<Operator<?>> cloneOperatorTree(List<Operator<?>> roots) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); - serializePlan(roots, baos, true); - @SuppressWarnings("unchecked") - List<Operator<?>> result = - deserializePlan(new ByteArrayInputStream(baos.toByteArray()), - roots.getClass(), true); - return result; - } - /** * Serializes expression via Kryo. * @param expr Expression. http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java index 7a8de2d..85344fc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java @@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinPersistableTableContainer; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe; import org.apache.hadoop.hive.ql.log.PerfLogger; @@ -54,8 +55,15 @@ public class SparkHashTableSinkOperator private final HashTableSinkOperator htsOperator; - public SparkHashTableSinkOperator() { - htsOperator = new HashTableSinkOperator(); + /** Kryo ctor. */ + protected SparkHashTableSinkOperator() { + super(); + htsOperator = null; // Kryo will set this; or so we hope. + } + + public SparkHashTableSinkOperator(CompilationOpContext ctx) { + super(ctx); + htsOperator = new HashTableSinkOperator(ctx); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java index bb4bde9..14eacdf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.DriverContext; import org.apache.hadoop.hive.ql.QueryPlan; import org.apache.hadoop.hive.ql.io.StatsProvidingRecordReader; @@ -81,8 +82,9 @@ public class StatsNoJobTask extends Task<StatsNoJobWork> implements Serializable } @Override - public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext driverContext) { - super.initialize(conf, queryPlan, driverContext); + public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext driverContext, + CompilationOpContext opContext) { + super.initialize(conf, queryPlan, driverContext, opContext); jc = new JobConf(conf); } http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java index 32806dc..5253521 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; @@ -209,6 +210,15 @@ public class TableScanOperator extends Operator<TableScanDesc> implements } + /** Kryo ctor. */ + protected TableScanOperator() { + super(); + } + + public TableScanOperator(CompilationOpContext ctx) { + super(ctx); + } + @Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java index 0eab63e..40c89cb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java @@ -30,6 +30,7 @@ import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.DriverContext; import org.apache.hadoop.hive.ql.QueryPlan; import org.apache.hadoop.hive.ql.lib.Node; @@ -120,7 +121,8 @@ public abstract class Task<T extends Serializable> implements Serializable, Node return taskHandle; } - public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext driverContext) { + public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext driverContext, + CompilationOpContext opContext) { this.queryPlan = queryPlan; isdone = false; started = false; http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/TemporaryHashSinkOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TemporaryHashSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TemporaryHashSinkOperator.java index 9e3a84f..bc311ee 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TemporaryHashSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TemporaryHashSinkOperator.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.HashTableSinkDesc; import org.apache.hadoop.hive.ql.plan.MapJoinDesc; @@ -27,7 +28,8 @@ import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import java.io.IOException; public class TemporaryHashSinkOperator extends HashTableSinkOperator { - public TemporaryHashSinkOperator(MapJoinDesc desc) { + public TemporaryHashSinkOperator(CompilationOpContext ctx, MapJoinDesc desc) { + super(ctx); conf = new HashTableSinkDesc(desc); // Sanity check the config. http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/TerminalOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TerminalOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TerminalOperator.java index 069eab2..04d6c9f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TerminalOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TerminalOperator.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.plan.OperatorDesc; /** @@ -29,4 +30,12 @@ public abstract class TerminalOperator<T extends OperatorDesc> extends Operator<T> implements Serializable { private static final long serialVersionUID = 1L; + /** Kryo ctor. */ + protected TerminalOperator() { + super(); + } + + public TerminalOperator(CompilationOpContext ctx) { + super(ctx); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/TezDummyStoreOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TezDummyStoreOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TezDummyStoreOperator.java index e9f65be..f28f928 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TezDummyStoreOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TezDummyStoreOperator.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.exec; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.metadata.HiveException; /** @@ -27,6 +28,15 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; * */ public class TezDummyStoreOperator extends DummyStoreOperator { + /** Kryo ctor. */ + protected TezDummyStoreOperator() { + super(); + } + + public TezDummyStoreOperator(CompilationOpContext ctx) { + super(ctx); + } + private boolean fetchDone = false; /** http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java index b3c6d91..1dae963 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java @@ -29,6 +29,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.UDTFDesc; import org.apache.hadoop.hive.ql.plan.api.OperatorType; @@ -58,6 +59,15 @@ public class UDTFOperator extends Operator<UDTFDesc> implements Serializable { */ transient AutoProgressor autoProgressor; + /** Kryo ctor. */ + protected UDTFOperator() { + super(); + } + + public UDTFOperator(CompilationOpContext ctx) { + super(ctx); + } + @Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java index ddb23ee..3a673e6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java @@ -20,11 +20,10 @@ package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; import java.util.ArrayList; -import java.util.Collection; import java.util.List; -import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.UnionDesc; import org.apache.hadoop.hive.ql.plan.api.OperatorType; @@ -48,6 +47,15 @@ public class UnionOperator extends Operator<UnionDesc> implements Serializable { ArrayList<Object> outputRow; + /** Kryo ctor. */ + protected UnionOperator() { + super(); + } + + public UnionOperator(CompilationOpContext ctx) { + super(ctx); + } + /** * UnionOperator will transform the input rows if the inputObjInspectors from * different parents are different. If one parent has exactly the same http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java index 472e8ed..b184b4e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hive.common.LogUtils; import org.apache.hadoop.hive.common.LogUtils.LogInitializationException; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.DriverContext; import org.apache.hadoop.hive.ql.ErrorMsg; @@ -141,8 +142,9 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop * Initialization when invoked from QL. */ @Override - public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext driverContext) { - super.initialize(conf, queryPlan, driverContext); + public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext driverContext, + CompilationOpContext opContext) { + super.initialize(conf, queryPlan, driverContext, opContext); job = new JobConf(conf, ExecDriver.class); http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java index 23497a9..c34dd1f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java @@ -28,6 +28,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.MapOperator; import org.apache.hadoop.hive.ql.exec.MapredContext; import org.apache.hadoop.hive.ql.exec.Operator; @@ -92,10 +93,11 @@ public class ExecMapper extends MapReduceBase implements Mapper { // create map and fetch operators MapWork mrwork = Utilities.getMapWork(job); + CompilationOpContext runtimeCtx = new CompilationOpContext(); if (mrwork.getVectorMode()) { - mo = new VectorMapOperator(); + mo = new VectorMapOperator(runtimeCtx); } else { - mo = new MapOperator(); + mo = new MapOperator(runtimeCtx); } mo.setConf(mrwork); // initialize map operator http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HashTableLoader.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HashTableLoader.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HashTableLoader.java index 1070384..f5662f0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HashTableLoader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HashTableLoader.java @@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.MapredContext; @@ -135,7 +136,7 @@ public class HashTableLoader implements org.apache.hadoop.hive.ql.exec.HashTable JobConf job = new JobConf(hconf); MapredLocalTask localTask = new MapredLocalTask(localWork, job, false); - HashTableSinkOperator sink = new TemporaryHashSinkOperator(desc); + HashTableSinkOperator sink = new TemporaryHashSinkOperator(new CompilationOpContext(), desc); sink.setParentOperators(new ArrayList<Operator<? extends OperatorDesc>>(directWorks)); for (Operator<?> operator : directWorks) { http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java index ee99269..1d97a44 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java @@ -44,6 +44,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.io.CachingPrintStream; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.DriverContext; import org.apache.hadoop.hive.ql.QueryPlan; @@ -118,8 +119,9 @@ public class MapredLocalTask extends Task<MapredLocalWork> implements Serializab } @Override - public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext driverContext) { - super.initialize(conf, queryPlan, driverContext); + public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext driverContext, + CompilationOpContext opContext) { + super.initialize(conf, queryPlan, driverContext, opContext); job = new JobConf(conf, ExecDriver.class); execContext = new ExecMapperContext(job); //we don't use the HadoopJobExecHooks for local tasks http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java index 7ada611..1634f42 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java @@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.MapredContext; @@ -183,7 +184,7 @@ public class HashTableLoader implements org.apache.hadoop.hive.ql.exec.HashTable JobConf job = new JobConf(hconf); MapredLocalTask localTask = new MapredLocalTask(localWork, job, false); - HashTableSinkOperator sink = new TemporaryHashSinkOperator(desc); + HashTableSinkOperator sink = new TemporaryHashSinkOperator(new CompilationOpContext(), desc); sink.setParentOperators(new ArrayList<Operator<? extends OperatorDesc>>(directWorks)); for (Operator<?> operator : directWorks) { http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java index 62be3f8..d8fe35f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java @@ -24,6 +24,7 @@ import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.MapOperator; import org.apache.hadoop.hive.ql.exec.MapredContext; import org.apache.hadoop.hive.ql.exec.Operator; @@ -72,10 +73,11 @@ public class SparkMapRecordHandler extends SparkRecordHandler { // create map and fetch operators MapWork mrwork = Utilities.getMapWork(job); + CompilationOpContext runtimeCtx = new CompilationOpContext(); if (mrwork.getVectorMode()) { - mo = new VectorMapOperator(); + mo = new VectorMapOperator(runtimeCtx); } else { - mo = new MapOperator(); + mo = new MapOperator(runtimeCtx); } mo.setConf(mrwork); http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java index 31eee45..eb93aca 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.DriverContext; import org.apache.hadoop.hive.ql.QueryPlan; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; @@ -88,8 +89,9 @@ public class SparkTask extends Task<SparkWork> { private SparkCounters sparkCounters; @Override - public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext driverContext) { - super.initialize(conf, queryPlan, driverContext); + public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext driverContext, + CompilationOpContext opContext) { + super.initialize(conf, queryPlan, driverContext, opContext); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java index ee62ab3..a4cf0b1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java @@ -35,6 +35,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.llap.io.api.LlapProxy; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.DummyStoreOperator; import org.apache.hadoop.hive.ql.exec.HashTableDummyOperator; import org.apache.hadoop.hive.ql.exec.MapOperator; @@ -174,10 +175,11 @@ public class MapRecordProcessor extends RecordProcessor { try { + CompilationOpContext runtimeCtx = new CompilationOpContext(); if (mapWork.getVectorMode()) { - mapOp = new VectorMapOperator(); + mapOp = new VectorMapOperator(runtimeCtx); } else { - mapOp = new MapOperator(); + mapOp = new MapOperator(runtimeCtx); } mapOp.clearConnectedOperators(); @@ -188,9 +190,9 @@ public class MapRecordProcessor extends RecordProcessor { for (BaseWork mergeWork : mergeWorkList) { MapWork mergeMapWork = (MapWork) mergeWork; if (mergeMapWork.getVectorMode()) { - mergeMapOp = new VectorMapOperator(); + mergeMapOp = new VectorMapOperator(runtimeCtx); } else { - mergeMapOp = new MapOperator(); + mergeMapOp = new MapOperator(runtimeCtx); } mergeMapOpList.add(mergeMapOp); http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAppMasterEventOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAppMasterEventOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAppMasterEventOperator.java index e4ca2cd..c591288 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAppMasterEventOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAppMasterEventOperator.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.AppMasterEventOperator; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.AppMasterEventDesc; @@ -52,14 +53,20 @@ public class VectorAppMasterEventOperator extends AppMasterEventOperator { protected transient Object[] singleRow; - public VectorAppMasterEventOperator(VectorizationContext vContext, - OperatorDesc conf) { - super(); + public VectorAppMasterEventOperator( + CompilationOpContext ctx, VectorizationContext vContext, OperatorDesc conf) { + super(ctx); this.conf = (AppMasterEventDesc) conf; this.vContext = vContext; } - public VectorAppMasterEventOperator() { + /** Kryo ctor. */ + protected VectorAppMasterEventOperator() { + super(); + } + + public VectorAppMasterEventOperator(CompilationOpContext ctx) { + super(ctx); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFileSinkOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFileSinkOperator.java index 09d4a8e..f09534c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFileSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFileSinkOperator.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; @@ -47,15 +48,20 @@ public class VectorFileSinkOperator extends FileSinkOperator { protected transient Object[] singleRow; - public VectorFileSinkOperator(VectorizationContext vContext, - OperatorDesc conf) { - super(); + public VectorFileSinkOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) { + this(ctx); this.conf = (FileSinkDesc) conf; this.vContext = vContext; } - public VectorFileSinkOperator() { + /** Kryo ctor. */ + protected VectorFileSinkOperator() { + super(); + } + public VectorFileSinkOperator(CompilationOpContext ctx) { + super(ctx); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java index 58f1190..74a0947 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java @@ -23,6 +23,7 @@ import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.FilterOperator; import org.apache.hadoop.hive.ql.exec.vector.expressions.ConstantVectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; @@ -47,18 +48,23 @@ public class VectorFilterOperator extends FilterOperator { // and 0 if condition needs to be computed. transient private int filterMode = 0; - public VectorFilterOperator(VectorizationContext vContext, OperatorDesc conf) - throws HiveException { - this(); + public VectorFilterOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + this(ctx); ExprNodeDesc oldExpression = ((FilterDesc) conf).getPredicate(); conditionEvaluator = vContext.getVectorExpression(oldExpression, VectorExpressionDescriptor.Mode.FILTER); this.conf = (FilterDesc) conf; } - public VectorFilterOperator() { + /** Kryo ctor. */ + protected VectorFilterOperator() { super(); } + public VectorFilterOperator(CompilationOpContext ctx) { + super(ctx); + } + @Override protected void initializeOp(Configuration hconf) throws HiveException { http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java index 0524c08..b7ce309 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java @@ -33,6 +33,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.KeyWrapper; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; @@ -749,9 +750,9 @@ public class VectorGroupByOperator extends Operator<GroupByDesc> implements private static final long serialVersionUID = 1L; - public VectorGroupByOperator(VectorizationContext vContext, OperatorDesc conf) - throws HiveException { - this(); + public VectorGroupByOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + this(ctx); GroupByDesc desc = (GroupByDesc) conf; this.conf = desc; List<ExprNodeDesc> keysDesc = desc.getKeys(); @@ -769,10 +770,16 @@ public class VectorGroupByOperator extends Operator<GroupByDesc> implements vOutContext = new VectorizationContext(getName(), desc.getOutputColumnNames()); } - public VectorGroupByOperator() { + /** Kryo ctor. */ + protected VectorGroupByOperator() { super(); } + public VectorGroupByOperator(CompilationOpContext ctx) { + super(ctx); + } + + @Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorLimitOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorLimitOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorLimitOperator.java index 4cb91d4..154c647 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorLimitOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorLimitOperator.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.exec.vector; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.LimitOperator; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.LimitDesc; @@ -30,11 +31,18 @@ public class VectorLimitOperator extends LimitOperator { private static final long serialVersionUID = 1L; - public VectorLimitOperator() { + /** Kryo ctor. */ + protected VectorLimitOperator() { super(); } - public VectorLimitOperator(VectorizationContext vContext, OperatorDesc conf) { + public VectorLimitOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorLimitOperator( + CompilationOpContext ctx, VectorizationContext vContext, OperatorDesc conf) { + this(ctx); this.conf = (LimitDesc) conf; } http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinBaseOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinBaseOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinBaseOperator.java index 4b1d9ad..6bed52f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinBaseOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinBaseOperator.java @@ -26,6 +26,7 @@ import java.util.concurrent.Future; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer; @@ -64,13 +65,18 @@ public class VectorMapJoinBaseOperator extends MapJoinOperator implements Vector protected transient int tag; // big table alias - public VectorMapJoinBaseOperator() { + /** Kryo ctor. */ + protected VectorMapJoinBaseOperator() { super(); } - public VectorMapJoinBaseOperator (VectorizationContext vContext, OperatorDesc conf) - throws HiveException { - super(); + public VectorMapJoinBaseOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorMapJoinBaseOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + super(ctx); MapJoinDesc desc = (MapJoinDesc) conf; this.conf = desc; http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java index 8bbf020..622f777 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java @@ -27,6 +27,7 @@ import java.util.concurrent.Future; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; import org.apache.hadoop.hive.ql.exec.JoinUtil; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer; @@ -74,15 +75,20 @@ public class VectorMapJoinOperator extends VectorMapJoinBaseOperator { private VectorExpressionWriter[] rowWriters; // Writer for producing row from input batch protected transient Object[] singleRow; - public VectorMapJoinOperator() { + /** Kryo ctor. */ + protected VectorMapJoinOperator() { super(); } + public VectorMapJoinOperator(CompilationOpContext ctx) { + super(ctx); + } + - public VectorMapJoinOperator (VectorizationContext vContext, OperatorDesc conf) - throws HiveException { + public VectorMapJoinOperator (CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { - super(vContext, conf); + super(ctx, vContext, conf); MapJoinDesc desc = (MapJoinDesc) conf; http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOuterFilteredOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOuterFilteredOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOuterFilteredOperator.java index b8b1f88..509a43f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOuterFilteredOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOuterFilteredOperator.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -48,13 +49,18 @@ public class VectorMapJoinOuterFilteredOperator extends VectorMapJoinBaseOperato protected transient Object[] singleRow; - public VectorMapJoinOuterFilteredOperator() { + /** Kryo ctor. */ + protected VectorMapJoinOuterFilteredOperator() { super(); } - public VectorMapJoinOuterFilteredOperator(VectorizationContext vContext, OperatorDesc conf) - throws HiveException { - super(vContext, conf); + public VectorMapJoinOuterFilteredOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorMapJoinOuterFilteredOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + super(ctx, vContext, conf); this.vContext = vContext; } http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java index aa0d5a5..033be38 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.exec.vector; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.MapOperator; import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -27,6 +28,15 @@ public class VectorMapOperator extends MapOperator { private static final long serialVersionUID = 1L; + /** Kryo ctor. */ + protected VectorMapOperator() { + super(); + } + + public VectorMapOperator(CompilationOpContext ctx) { + super(ctx); + } + @Override public void process(Writable value) throws HiveException { // A mapper can span multiple files/partitions. http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java index 41b2874..b79a3d8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -44,18 +45,23 @@ public class VectorReduceSinkOperator extends ReduceSinkOperator { protected transient Object[] singleRow; - public VectorReduceSinkOperator(VectorizationContext vContext, OperatorDesc conf) - throws HiveException { - this(); + public VectorReduceSinkOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + this(ctx); ReduceSinkDesc desc = (ReduceSinkDesc) conf; this.conf = desc; this.vContext = vContext; } - public VectorReduceSinkOperator() { + /** Kryo ctor. */ + protected VectorReduceSinkOperator() { super(); } + public VectorReduceSinkOperator(CompilationOpContext ctx) { + super(ctx); + } + @Override protected void initializeOp(Configuration hconf) throws HiveException { http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java index 9ff9b77..9a263e6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java @@ -28,6 +28,7 @@ import java.util.concurrent.Future; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; @@ -89,13 +90,18 @@ public class VectorSMBMapJoinOperator extends SMBMapJoinOperator implements Vect List<Object> evaluate(VectorHashKeyWrapper kw) throws HiveException; } - public VectorSMBMapJoinOperator() { + /** Kryo ctor. */ + protected VectorSMBMapJoinOperator() { super(); } - public VectorSMBMapJoinOperator(VectorizationContext vContext, OperatorDesc conf) - throws HiveException { - this(); + public VectorSMBMapJoinOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorSMBMapJoinOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + this(ctx); SMBJoinDesc desc = (SMBJoinDesc) conf; this.conf = desc; http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java index 73b905f..8db6eba 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter; @@ -53,8 +54,9 @@ public class VectorSelectOperator extends Operator<SelectDesc> implements // Create a new outgoing vectorization context because column name map will change. private VectorizationContext vOutContext; - public VectorSelectOperator(VectorizationContext vContext, OperatorDesc conf) - throws HiveException { + public VectorSelectOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + this(ctx); this.conf = (SelectDesc) conf; List<ExprNodeDesc> colList = this.conf.getColList(); vExpressions = new VectorExpression[colList.size()]; @@ -79,7 +81,13 @@ public class VectorSelectOperator extends Operator<SelectDesc> implements } } - public VectorSelectOperator() { + /** Kryo ctor. */ + protected VectorSelectOperator() { + super(); + } + + public VectorSelectOperator(CompilationOpContext ctx) { + super(ctx); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkHashTableSinkOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkHashTableSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkHashTableSinkOperator.java index 8486d12..1e550e7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkHashTableSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkHashTableSinkOperator.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.exec.vector; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.SparkHashTableSinkOperator; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -50,11 +51,18 @@ public class VectorSparkHashTableSinkOperator extends SparkHashTableSinkOperator protected transient Object[] singleRow; - public VectorSparkHashTableSinkOperator() { + /** Kryo ctor. */ + protected VectorSparkHashTableSinkOperator() { + super(); } - public VectorSparkHashTableSinkOperator(VectorizationContext vContext, OperatorDesc conf) { - super(); + public VectorSparkHashTableSinkOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorSparkHashTableSinkOperator( + CompilationOpContext ctx, VectorizationContext vContext, OperatorDesc conf) { + this(ctx); this.vContext = vContext; this.conf = (SparkHashTableSinkDesc) conf; } http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkPartitionPruningSinkOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkPartitionPruningSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkPartitionPruningSinkOperator.java index eb0b408..2f02250 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkPartitionPruningSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSparkPartitionPruningSinkOperator.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.optimizer.spark.SparkPartitionPruningSinkDesc; import org.apache.hadoop.hive.ql.parse.spark.SparkPartitionPruningSinkOperator; @@ -46,14 +47,20 @@ public class VectorSparkPartitionPruningSinkOperator extends SparkPartitionPruni protected transient Object[] singleRow; - public VectorSparkPartitionPruningSinkOperator(VectorizationContext context, - OperatorDesc conf) { - super(); + public VectorSparkPartitionPruningSinkOperator(CompilationOpContext ctx, + VectorizationContext context, OperatorDesc conf) { + this(ctx); this.conf = (SparkPartitionPruningSinkDesc) conf; this.vContext = context; } - public VectorSparkPartitionPruningSinkOperator() { + /** Kryo ctor. */ + protected VectorSparkPartitionPruningSinkOperator() { + super(); + } + + public VectorSparkPartitionPruningSinkOperator(CompilationOpContext ctx) { + super(ctx); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java index 2502ae2..e26e31b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java @@ -27,6 +27,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.HashTableLoaderFactory; import org.apache.hadoop.hive.ql.exec.HashTableLoader; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; @@ -176,13 +177,18 @@ public abstract class VectorMapJoinCommonOperator extends MapJoinOperator implem // The small table hash table for the native vectorized map join operator. protected transient VectorMapJoinHashTable vectorMapJoinHashTable; - public VectorMapJoinCommonOperator() { + /** Kryo ctor. */ + protected VectorMapJoinCommonOperator() { super(); } - public VectorMapJoinCommonOperator(VectorizationContext vContext, OperatorDesc conf) - throws HiveException { - super(); + public VectorMapJoinCommonOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorMapJoinCommonOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + super(ctx); MapJoinDesc desc = (MapJoinDesc) conf; this.conf = desc; http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinGenerateResultOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinGenerateResultOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinGenerateResultOperator.java index c1c137b..5cbace4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinGenerateResultOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinGenerateResultOperator.java @@ -25,6 +25,7 @@ import java.util.List; import org.apache.commons.lang.ArrayUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer; import org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer.HashPartition; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer; @@ -86,13 +87,18 @@ public abstract class VectorMapJoinGenerateResultOperator extends VectorMapJoinC // Debug display. protected transient long batchCounter; - public VectorMapJoinGenerateResultOperator() { + /** Kryo ctor. */ + protected VectorMapJoinGenerateResultOperator() { super(); } - public VectorMapJoinGenerateResultOperator(VectorizationContext vContext, OperatorDesc conf) - throws HiveException { - super(vContext, conf); + public VectorMapJoinGenerateResultOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorMapJoinGenerateResultOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + super(ctx, vContext, conf); } protected void commonSetup(VectorizedRowBatch batch) throws HiveException { http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyGenerateResultOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyGenerateResultOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyGenerateResultOperator.java index 6b33a39..dfb5bf8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyGenerateResultOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyGenerateResultOperator.java @@ -22,6 +22,7 @@ import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.JoinUtil; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -84,13 +85,18 @@ public abstract class VectorMapJoinInnerBigOnlyGenerateResultOperator // Pre-allocated member for storing index into the hashMultiSetResults for each spilled row. protected transient int[] spillHashMapResultIndices; - public VectorMapJoinInnerBigOnlyGenerateResultOperator() { + /** Kryo ctor. */ + protected VectorMapJoinInnerBigOnlyGenerateResultOperator() { super(); } - public VectorMapJoinInnerBigOnlyGenerateResultOperator(VectorizationContext vContext, OperatorDesc conf) - throws HiveException { - super(vContext, conf); + public VectorMapJoinInnerBigOnlyGenerateResultOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorMapJoinInnerBigOnlyGenerateResultOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + super(ctx, vContext, conf); } /* http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyLongOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyLongOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyLongOperator.java index 9e77d22..0bba141 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyLongOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyLongOperator.java @@ -23,6 +23,7 @@ import java.util.Arrays; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.JoinUtil; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -72,12 +73,18 @@ public class VectorMapJoinInnerBigOnlyLongOperator extends VectorMapJoinInnerBig // Pass-thru constructors. // - public VectorMapJoinInnerBigOnlyLongOperator() { + /** Kryo ctor. */ + protected VectorMapJoinInnerBigOnlyLongOperator() { super(); } - public VectorMapJoinInnerBigOnlyLongOperator(VectorizationContext vContext, OperatorDesc conf) throws HiveException { - super(vContext, conf); + public VectorMapJoinInnerBigOnlyLongOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorMapJoinInnerBigOnlyLongOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + super(ctx, vContext, conf); } //--------------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyMultiKeyOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyMultiKeyOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyMultiKeyOperator.java index e4f6c5d..621804b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyMultiKeyOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyMultiKeyOperator.java @@ -23,6 +23,7 @@ import java.util.Arrays; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.JoinUtil; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -77,12 +78,18 @@ public class VectorMapJoinInnerBigOnlyMultiKeyOperator extends VectorMapJoinInne // Pass-thru constructors. // - public VectorMapJoinInnerBigOnlyMultiKeyOperator() { + /** Kryo ctor. */ + protected VectorMapJoinInnerBigOnlyMultiKeyOperator() { super(); } - public VectorMapJoinInnerBigOnlyMultiKeyOperator(VectorizationContext vContext, OperatorDesc conf) throws HiveException { - super(vContext, conf); + public VectorMapJoinInnerBigOnlyMultiKeyOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorMapJoinInnerBigOnlyMultiKeyOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + super(ctx, vContext, conf); } //--------------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java index 2711b10..10e75ab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java @@ -23,6 +23,7 @@ import java.util.Arrays; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.JoinUtil; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -68,12 +69,18 @@ public class VectorMapJoinInnerBigOnlyStringOperator extends VectorMapJoinInnerB // Pass-thru constructors. // - public VectorMapJoinInnerBigOnlyStringOperator() { + /** Kryo ctor. */ + protected VectorMapJoinInnerBigOnlyStringOperator() { super(); } - public VectorMapJoinInnerBigOnlyStringOperator(VectorizationContext vContext, OperatorDesc conf) throws HiveException { - super(vContext, conf); + public VectorMapJoinInnerBigOnlyStringOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorMapJoinInnerBigOnlyStringOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + super(ctx, vContext, conf); } //--------------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerGenerateResultOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerGenerateResultOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerGenerateResultOperator.java index 36d0611..319a2b0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerGenerateResultOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerGenerateResultOperator.java @@ -22,6 +22,7 @@ import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.JoinUtil; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; @@ -89,13 +90,18 @@ public abstract class VectorMapJoinInnerGenerateResultOperator // Pre-allocated member for storing index into the hashMapResults for each spilled row. protected transient int[] spillHashMapResultIndices; - public VectorMapJoinInnerGenerateResultOperator() { + /** Kryo ctor. */ + protected VectorMapJoinInnerGenerateResultOperator() { super(); } - public VectorMapJoinInnerGenerateResultOperator(VectorizationContext vContext, OperatorDesc conf) - throws HiveException { - super(vContext, conf); + public VectorMapJoinInnerGenerateResultOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorMapJoinInnerGenerateResultOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + super(ctx, vContext, conf); } /* http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerLongOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerLongOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerLongOperator.java index 0197225..804d69c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerLongOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerLongOperator.java @@ -23,6 +23,7 @@ import java.util.Arrays; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.JoinUtil; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -71,12 +72,18 @@ public class VectorMapJoinInnerLongOperator extends VectorMapJoinInnerGenerateRe // Pass-thru constructors. // - public VectorMapJoinInnerLongOperator() { + /** Kryo ctor. */ + protected VectorMapJoinInnerLongOperator() { super(); } - public VectorMapJoinInnerLongOperator(VectorizationContext vContext, OperatorDesc conf) throws HiveException { - super(vContext, conf); + public VectorMapJoinInnerLongOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorMapJoinInnerLongOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + super(ctx, vContext, conf); } //--------------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerMultiKeyOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerMultiKeyOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerMultiKeyOperator.java index 837d97b..fcfa0bd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerMultiKeyOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerMultiKeyOperator.java @@ -23,6 +23,7 @@ import java.util.Arrays; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.JoinUtil; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -75,12 +76,18 @@ public class VectorMapJoinInnerMultiKeyOperator extends VectorMapJoinInnerGenera // Pass-thru constructors. // - public VectorMapJoinInnerMultiKeyOperator() { + /** Kryo ctor. */ + protected VectorMapJoinInnerMultiKeyOperator() { super(); } - public VectorMapJoinInnerMultiKeyOperator(VectorizationContext vContext, OperatorDesc conf) throws HiveException { - super(vContext, conf); + public VectorMapJoinInnerMultiKeyOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorMapJoinInnerMultiKeyOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + super(ctx, vContext, conf); } //--------------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java index b2711c3..0f9baae 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java @@ -23,6 +23,7 @@ import java.util.Arrays; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.JoinUtil; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -67,12 +68,18 @@ public class VectorMapJoinInnerStringOperator extends VectorMapJoinInnerGenerate // Pass-thru constructors. // - public VectorMapJoinInnerStringOperator() { + /** Kryo ctor. */ + protected VectorMapJoinInnerStringOperator() { super(); } - public VectorMapJoinInnerStringOperator(VectorizationContext vContext, OperatorDesc conf) throws HiveException { - super(vContext, conf); + public VectorMapJoinInnerStringOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorMapJoinInnerStringOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + super(ctx, vContext, conf); } //--------------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiGenerateResultOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiGenerateResultOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiGenerateResultOperator.java index d1d6c42..c71ebba 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiGenerateResultOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiGenerateResultOperator.java @@ -22,6 +22,7 @@ import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.JoinUtil; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -70,13 +71,18 @@ public abstract class VectorMapJoinLeftSemiGenerateResultOperator // Pre-allocated member for storing index into the hashSetResults for each spilled row. protected transient int[] spillHashMapResultIndices; - public VectorMapJoinLeftSemiGenerateResultOperator() { + /** Kryo ctor. */ + protected VectorMapJoinLeftSemiGenerateResultOperator() { super(); } - public VectorMapJoinLeftSemiGenerateResultOperator(VectorizationContext vContext, OperatorDesc conf) - throws HiveException { - super(vContext, conf); + public VectorMapJoinLeftSemiGenerateResultOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorMapJoinLeftSemiGenerateResultOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + super(ctx, vContext, conf); } /* http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiLongOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiLongOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiLongOperator.java index 4b8ab58..1149a9d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiLongOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiLongOperator.java @@ -23,6 +23,7 @@ import java.util.Arrays; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.JoinUtil; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -72,12 +73,18 @@ public class VectorMapJoinLeftSemiLongOperator extends VectorMapJoinLeftSemiGene // Pass-thru constructors. // - public VectorMapJoinLeftSemiLongOperator() { + /** Kryo ctor. */ + protected VectorMapJoinLeftSemiLongOperator() { super(); } - public VectorMapJoinLeftSemiLongOperator(VectorizationContext vContext, OperatorDesc conf) throws HiveException { - super(vContext, conf); + public VectorMapJoinLeftSemiLongOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorMapJoinLeftSemiLongOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + super(ctx, vContext, conf); } //--------------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiMultiKeyOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiMultiKeyOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiMultiKeyOperator.java index bdf7901..e0baebc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiMultiKeyOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiMultiKeyOperator.java @@ -23,6 +23,7 @@ import java.util.Arrays; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.JoinUtil; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -76,12 +77,18 @@ public class VectorMapJoinLeftSemiMultiKeyOperator extends VectorMapJoinLeftSemi // Pass-thru constructors. // - public VectorMapJoinLeftSemiMultiKeyOperator() { + /** Kryo ctor. */ + protected VectorMapJoinLeftSemiMultiKeyOperator() { super(); } - public VectorMapJoinLeftSemiMultiKeyOperator(VectorizationContext vContext, OperatorDesc conf) throws HiveException { - super(vContext, conf); + public VectorMapJoinLeftSemiMultiKeyOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorMapJoinLeftSemiMultiKeyOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + super(ctx, vContext, conf); } //--------------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java index a8d3459..49e1177 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java @@ -23,6 +23,7 @@ import java.util.Arrays; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.JoinUtil; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -68,12 +69,18 @@ public class VectorMapJoinLeftSemiStringOperator extends VectorMapJoinLeftSemiGe // Pass-thru constructors. // - public VectorMapJoinLeftSemiStringOperator() { + /** Kryo ctor. */ + protected VectorMapJoinLeftSemiStringOperator() { super(); } - public VectorMapJoinLeftSemiStringOperator(VectorizationContext vContext, OperatorDesc conf) throws HiveException { - super(vContext, conf); + public VectorMapJoinLeftSemiStringOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorMapJoinLeftSemiStringOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + super(ctx, vContext, conf); } //--------------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterGenerateResultOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterGenerateResultOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterGenerateResultOperator.java index 5a88784..0e2d65a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterGenerateResultOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterGenerateResultOperator.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.exec.vector.mapjoin; import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.JoinUtil; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; @@ -111,13 +112,18 @@ public abstract class VectorMapJoinOuterGenerateResultOperator protected transient int[] noMatchs; protected transient int[] merged; - public VectorMapJoinOuterGenerateResultOperator() { + /** Kryo ctor. */ + protected VectorMapJoinOuterGenerateResultOperator() { super(); } - public VectorMapJoinOuterGenerateResultOperator(VectorizationContext vContext, OperatorDesc conf) - throws HiveException { - super(vContext, conf); + public VectorMapJoinOuterGenerateResultOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorMapJoinOuterGenerateResultOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + super(ctx, vContext, conf); } /* http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterLongOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterLongOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterLongOperator.java index 5b687fd..58bd0ab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterLongOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterLongOperator.java @@ -23,6 +23,7 @@ import java.util.Arrays; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.JoinUtil; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; @@ -72,12 +73,18 @@ public class VectorMapJoinOuterLongOperator extends VectorMapJoinOuterGenerateRe // Pass-thru constructors. // - public VectorMapJoinOuterLongOperator() { + /** Kryo ctor. */ + protected VectorMapJoinOuterLongOperator() { super(); } - public VectorMapJoinOuterLongOperator(VectorizationContext vContext, OperatorDesc conf) throws HiveException { - super(vContext, conf); + public VectorMapJoinOuterLongOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorMapJoinOuterLongOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + super(ctx, vContext, conf); } //--------------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/88fceaca/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterMultiKeyOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterMultiKeyOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterMultiKeyOperator.java index e212a2a..7f9afd2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterMultiKeyOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterMultiKeyOperator.java @@ -23,6 +23,7 @@ import java.util.Arrays; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.JoinUtil; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; @@ -75,12 +76,18 @@ public class VectorMapJoinOuterMultiKeyOperator extends VectorMapJoinOuterGenera // Pass-thru constructors. // - public VectorMapJoinOuterMultiKeyOperator() { + /** Kryo ctor. */ + protected VectorMapJoinOuterMultiKeyOperator() { super(); } - public VectorMapJoinOuterMultiKeyOperator(VectorizationContext vContext, OperatorDesc conf) throws HiveException { - super(vContext, conf); + public VectorMapJoinOuterMultiKeyOperator(CompilationOpContext ctx) { + super(ctx); + } + + public VectorMapJoinOuterMultiKeyOperator(CompilationOpContext ctx, + VectorizationContext vContext, OperatorDesc conf) throws HiveException { + super(ctx, vContext, conf); } //---------------------------------------------------------------------------
