http://git-wip-us.apache.org/repos/asf/hive/blob/aed21d0b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java index 77b44fb..c288731 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java @@ -22,7 +22,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; - import org.apache.commons.lang.ArrayUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,7 +56,6 @@ import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc; import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc.HashTableImplementationType; -import org.apache.hadoop.hive.ql.plan.VectorMapJoinInfo; import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinaryDeserializeRead; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -65,8 +63,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import com.google.common.base.Preconditions; - /** * This class is common operator class for native vectorized map join. * @@ -76,43 +72,7 @@ import com.google.common.base.Preconditions; */ public abstract class VectorMapJoinCommonOperator extends MapJoinOperator implements VectorizationContextRegion { private static final long serialVersionUID = 1L; - - //------------------------------------------------------------------------------------------------ - - private static final String CLASS_NAME = VectorMapJoinCommonOperator.class.getName(); -private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); - - protected abstract String getLoggingPrefix(); - - // For debug tracing: information about the map or reduce task, operator, operator class, etc. - protected transient String loggingPrefix; - - protected String getLoggingPrefix(String className) { - if (loggingPrefix == null) { - initLoggingPrefix(className); - } - return loggingPrefix; - } - - protected void initLoggingPrefix(String className) { - if (hconf == null) { - // Constructor time... - loggingPrefix = className; - } else { - // Determine the name of our map or reduce task for debug tracing. - BaseWork work = Utilities.getMapWork(hconf); - if (work == null) { - work = Utilities.getReduceWork(hconf); - } - loggingPrefix = className + " " + work.getName() + " " + getOperatorId(); - } - } - - //------------------------------------------------------------------------------------------------ - - protected VectorMapJoinDesc vectorDesc; - - protected VectorMapJoinInfo vectorMapJoinInfo; + private static final Logger LOG = LoggerFactory.getLogger(VectorMapJoinCommonOperator.class.getName()); // Whether this operator is an outer join. protected boolean isOuterJoin; @@ -128,10 +88,10 @@ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); // a mixture of input big table columns and new scratch columns. protected VectorizationContext vOutContext; - // The output column projection of the vectorized row batch. And, the type infos of the output + // The output column projection of the vectorized row batch. And, the type names of the output // columns. protected int[] outputProjection; - protected TypeInfo[] outputTypeInfos; + protected String[] outputTypeNames; // These are the vectorized batch expressions for filtering, key expressions, and value // expressions. @@ -141,17 +101,15 @@ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); // This is map of which vectorized row batch columns are the big table key columns. Since // we may have key expressions that produce new scratch columns, we need a mapping. - // And, we have their type infos. + // And, we have their type names. protected int[] bigTableKeyColumnMap; - protected String[] bigTableKeyColumnNames; - protected TypeInfo[] bigTableKeyTypeInfos; + protected ArrayList<String> bigTableKeyTypeNames; // Similarly, this is map of which vectorized row batch columns are the big table value columns. // Since we may have value expressions that produce new scratch columns, we need a mapping. - // And, we have their type infos. + // And, we have their type names. protected int[] bigTableValueColumnMap; - protected String[] bigTableValueColumnNames; - protected TypeInfo[] bigTableValueTypeInfos; + protected ArrayList<String> bigTableValueTypeNames; // This is a mapping of which big table columns (input and key/value expressions) will be // part of the big table portion of the join output result. @@ -166,8 +124,6 @@ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); // to output batch scratch columns for the small table portion. protected VectorColumnSourceMapping smallTableMapping; - protected VectorColumnSourceMapping projectionMapping; - // These are the output columns for the small table and the outer small table keys. protected int[] smallTableOutputVectorColumns; protected int[] bigTableOuterKeyOutputVectorColumns; @@ -181,6 +137,9 @@ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); // transient. //--------------------------------------------------------------------------- + // For debug tracing: the name of the map or reduce task. + protected transient String taskName; + // The threshold where we should use a repeating vectorized row batch optimization for // generating join output results. protected transient boolean useOverflowRepeatedThreshold; @@ -233,9 +192,6 @@ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); MapJoinDesc desc = (MapJoinDesc) conf; this.conf = desc; - vectorDesc = (VectorMapJoinDesc) desc.getVectorDesc(); - vectorMapJoinInfo = vectorDesc.getVectorMapJoinInfo(); - Preconditions.checkState(vectorMapJoinInfo != null); this.vContext = vContext; @@ -254,29 +210,215 @@ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); bigTableFilterExpressions = vContext.getVectorExpressions(filterExpressions.get(posBigTable), VectorExpressionDescriptor.Mode.FILTER); - bigTableKeyColumnMap = vectorMapJoinInfo.getBigTableKeyColumnMap(); - bigTableKeyColumnNames = vectorMapJoinInfo.getBigTableKeyColumnNames(); - bigTableKeyTypeInfos = vectorMapJoinInfo.getBigTableKeyTypeInfos(); - bigTableKeyExpressions = vectorMapJoinInfo.getBigTableKeyExpressions(); - - bigTableValueColumnMap = vectorMapJoinInfo.getBigTableValueColumnMap(); - bigTableValueColumnNames = vectorMapJoinInfo.getBigTableValueColumnNames(); - bigTableValueTypeInfos = vectorMapJoinInfo.getBigTableValueTypeInfos(); - bigTableValueExpressions = vectorMapJoinInfo.getBigTableValueExpressions(); - - bigTableRetainedMapping = vectorMapJoinInfo.getBigTableRetainedMapping(); - - bigTableOuterKeyMapping = vectorMapJoinInfo.getBigTableOuterKeyMapping(); + List<ExprNodeDesc> keyDesc = desc.getKeys().get(posBigTable); + bigTableKeyExpressions = vContext.getVectorExpressions(keyDesc); + + // Since a key expression can be a calculation and the key will go into a scratch column, + // we need the mapping and type information. + bigTableKeyColumnMap = new int[bigTableKeyExpressions.length]; + bigTableKeyTypeNames = new ArrayList<String>(); + boolean onlyColumns = true; + for (int i = 0; i < bigTableKeyColumnMap.length; i++) { + VectorExpression ve = bigTableKeyExpressions[i]; + if (!IdentityExpression.isColumnOnly(ve)) { + onlyColumns = false; + } + bigTableKeyTypeNames.add(keyDesc.get(i).getTypeString()); + bigTableKeyColumnMap[i] = ve.getOutputColumn(); + } + if (onlyColumns) { + bigTableKeyExpressions = null; + } - smallTableMapping = vectorMapJoinInfo.getSmallTableMapping(); + List<ExprNodeDesc> bigTableExprs = desc.getExprs().get(posBigTable); + bigTableValueExpressions = vContext.getVectorExpressions(bigTableExprs); - projectionMapping = vectorMapJoinInfo.getProjectionMapping(); + /* + * Similarly, we need a mapping since a value expression can be a calculation and the value + * will go into a scratch column. + */ + bigTableValueColumnMap = new int[bigTableValueExpressions.length]; + bigTableValueTypeNames = new ArrayList<String>(); + onlyColumns = true; + for (int i = 0; i < bigTableValueColumnMap.length; i++) { + VectorExpression ve = bigTableValueExpressions[i]; + if (!IdentityExpression.isColumnOnly(ve)) { + onlyColumns = false; + } + bigTableValueTypeNames.add(bigTableExprs.get(i).getTypeString()); + bigTableValueColumnMap[i] = ve.getOutputColumn(); + } + if (onlyColumns) { + bigTableValueExpressions = null; + } determineCommonInfo(isOuterJoin); } protected void determineCommonInfo(boolean isOuter) throws HiveException { + bigTableRetainedMapping = new VectorColumnOutputMapping("Big Table Retained Mapping"); + + bigTableOuterKeyMapping = new VectorColumnOutputMapping("Big Table Outer Key Mapping"); + + // The order of the fields in the LazyBinary small table value must be used, so + // we use the source ordering flavor for the mapping. + smallTableMapping = new VectorColumnSourceMapping("Small Table Mapping"); + + // We use a mapping object here so we can build the projection in any order and + // get the ordered by 0 to n-1 output columns at the end. + // + // Also, to avoid copying a big table key into the small table result area for inner joins, + // we reference it with the projection so there can be duplicate output columns + // in the projection. + VectorColumnSourceMapping projectionMapping = new VectorColumnSourceMapping("Projection Mapping"); + + /* + * Gather up big and small table output result information from the MapJoinDesc. + */ + List<Integer> bigTableRetainList = conf.getRetainList().get(posBigTable); + int bigTableRetainSize = bigTableRetainList.size(); + + int[] smallTableIndices; + int smallTableIndicesSize; + List<ExprNodeDesc> smallTableExprs = conf.getExprs().get(posSingleVectorMapJoinSmallTable); + if (conf.getValueIndices() != null && conf.getValueIndices().get(posSingleVectorMapJoinSmallTable) != null) { + smallTableIndices = conf.getValueIndices().get(posSingleVectorMapJoinSmallTable); + smallTableIndicesSize = smallTableIndices.length; + } else { + smallTableIndices = null; + smallTableIndicesSize = 0; + } + + List<Integer> smallTableRetainList = conf.getRetainList().get(posSingleVectorMapJoinSmallTable); + int smallTableRetainSize = smallTableRetainList.size(); + + int smallTableResultSize = 0; + if (smallTableIndicesSize > 0) { + smallTableResultSize = smallTableIndicesSize; + } else if (smallTableRetainSize > 0) { + smallTableResultSize = smallTableRetainSize; + } + + /* + * Determine the big table retained mapping first so we can optimize out (with + * projection) copying inner join big table keys in the subsequent small table results section. + */ + int nextOutputColumn = (order[0] == posBigTable ? 0 : smallTableResultSize); + for (int i = 0; i < bigTableRetainSize; i++) { + + // Since bigTableValueExpressions may do a calculation and produce a scratch column, we + // need to map to the right batch column. + + int retainColumn = bigTableRetainList.get(i); + int batchColumnIndex = bigTableValueColumnMap[retainColumn]; + String typeName = bigTableValueTypeNames.get(i); + + // With this map we project the big table batch to make it look like an output batch. + projectionMapping.add(nextOutputColumn, batchColumnIndex, typeName); + + // Collect columns we copy from the big table batch to the overflow batch. + if (!bigTableRetainedMapping.containsOutputColumn(batchColumnIndex)) { + // Tolerate repeated use of a big table column. + bigTableRetainedMapping.add(batchColumnIndex, batchColumnIndex, typeName); + } + + nextOutputColumn++; + } + + /* + * Now determine the small table results. + */ + int firstSmallTableOutputColumn; + firstSmallTableOutputColumn = (order[0] == posBigTable ? bigTableRetainSize : 0); + int smallTableOutputCount = 0; + nextOutputColumn = firstSmallTableOutputColumn; + + // Small table indices has more information (i.e. keys) than retain, so use it if it exists... + if (smallTableIndicesSize > 0) { + smallTableOutputCount = smallTableIndicesSize; + + for (int i = 0; i < smallTableIndicesSize; i++) { + if (smallTableIndices[i] >= 0) { + + // Zero and above numbers indicate a big table key is needed for + // small table result "area". + + int keyIndex = smallTableIndices[i]; + + // Since bigTableKeyExpressions may do a calculation and produce a scratch column, we + // need to map the right column. + int batchKeyColumn = bigTableKeyColumnMap[keyIndex]; + String typeName = bigTableKeyTypeNames.get(keyIndex); + + if (!isOuter) { + + // Optimize inner join keys of small table results. + + // Project the big table key into the small table result "area". + projectionMapping.add(nextOutputColumn, batchKeyColumn, typeName); + + if (!bigTableRetainedMapping.containsOutputColumn(batchKeyColumn)) { + // If necessary, copy the big table key into the overflow batch's small table + // result "area". + bigTableRetainedMapping.add(batchKeyColumn, batchKeyColumn, typeName); + } + } else { + + // For outer joins, since the small table key can be null when there is no match, + // we must have a physical (scratch) column for those keys. We cannot use the + // projection optimization used by inner joins above. + + int scratchColumn = vOutContext.allocateScratchColumn(typeName); + projectionMapping.add(nextOutputColumn, scratchColumn, typeName); + + bigTableRetainedMapping.add(batchKeyColumn, scratchColumn, typeName); + + bigTableOuterKeyMapping.add(batchKeyColumn, scratchColumn, typeName); + } + } else { + + // Negative numbers indicate a column to be (deserialize) read from the small table's + // LazyBinary value row. + int smallTableValueIndex = -smallTableIndices[i] - 1; + + String typeName = smallTableExprs.get(i).getTypeString(); + + // Make a new big table scratch column for the small table value. + int scratchColumn = vOutContext.allocateScratchColumn(typeName); + projectionMapping.add(nextOutputColumn, scratchColumn, typeName); + + smallTableMapping.add(smallTableValueIndex, scratchColumn, typeName); + } + nextOutputColumn++; + } + } else if (smallTableRetainSize > 0) { + smallTableOutputCount = smallTableRetainSize; + + // Only small table values appear in join output result. + + for (int i = 0; i < smallTableRetainSize; i++) { + int smallTableValueIndex = smallTableRetainList.get(i); + + // Make a new big table scratch column for the small table value. + String typeName = smallTableExprs.get(i).getTypeString(); + int scratchColumn = vOutContext.allocateScratchColumn(typeName); + + projectionMapping.add(nextOutputColumn, scratchColumn, typeName); + + smallTableMapping.add(smallTableValueIndex, scratchColumn, typeName); + nextOutputColumn++; + } + } + + // Convert dynamic arrays and maps to simple arrays. + + bigTableRetainedMapping.finalize(); + + bigTableOuterKeyMapping.finalize(); + + smallTableMapping.finalize(); + bigTableOuterKeyOutputVectorColumns = bigTableOuterKeyMapping.getOutputColumns(); smallTableOutputVectorColumns = smallTableMapping.getOutputColumns(); @@ -287,37 +429,46 @@ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); smallTableByteColumnVectorColumns = getByteColumnVectorColumns(smallTableMapping); + projectionMapping.finalize(); + + // Verify we added an entry for each output. + assert projectionMapping.isSourceSequenceGood(); + outputProjection = projectionMapping.getOutputColumns(); - outputTypeInfos = projectionMapping.getTypeInfos(); + outputTypeNames = projectionMapping.getTypeNames(); if (isLogDebugEnabled) { int[] orderDisplayable = new int[order.length]; for (int i = 0; i < order.length; i++) { orderDisplayable[i] = (int) order[i]; } - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator constructor order " + Arrays.toString(orderDisplayable)); - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator constructor posBigTable " + (int) posBigTable); - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator constructor posSingleVectorMapJoinSmallTable " + (int) posSingleVectorMapJoinSmallTable); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor order " + Arrays.toString(orderDisplayable)); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor posBigTable " + (int) posBigTable); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor posSingleVectorMapJoinSmallTable " + (int) posSingleVectorMapJoinSmallTable); + + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor bigTableKeyColumnMap " + Arrays.toString(bigTableKeyColumnMap)); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor bigTableKeyTypeNames " + bigTableKeyTypeNames); - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator constructor bigTableKeyColumnMap " + Arrays.toString(bigTableKeyColumnMap)); - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator constructor bigTableKeyColumnNames " + Arrays.toString(bigTableKeyColumnNames)); - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator constructor bigTableKeyTypeInfos " + Arrays.toString(bigTableKeyTypeInfos)); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor bigTableValueColumnMap " + Arrays.toString(bigTableValueColumnMap)); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor bigTableValueTypeNames " + bigTableValueTypeNames); - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator constructor bigTableValueColumnMap " + Arrays.toString(bigTableValueColumnMap)); - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator constructor bigTableValueColumnNames " + Arrays.toString(bigTableValueColumnNames)); - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator constructor bigTableValueTypeNames " + Arrays.toString(bigTableValueTypeInfos)); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor smallTableIndices " + Arrays.toString(smallTableIndices)); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor smallTableRetainList " + smallTableRetainList); - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator constructor bigTableRetainedMapping " + bigTableRetainedMapping.toString()); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor firstSmallTableOutputColumn " + firstSmallTableOutputColumn); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor smallTableOutputCount " + smallTableOutputCount); - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator constructor bigTableOuterKeyMapping " + bigTableOuterKeyMapping.toString()); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor bigTableRetainedMapping " + bigTableRetainedMapping.toString()); - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator constructor smallTableMapping " + smallTableMapping.toString()); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor bigTableOuterKeyMapping " + bigTableOuterKeyMapping.toString()); - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator constructor bigTableByteColumnVectorColumns " + Arrays.toString(bigTableByteColumnVectorColumns)); - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator constructor smallTableByteColumnVectorColumns " + Arrays.toString(smallTableByteColumnVectorColumns)); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor smallTableMapping " + smallTableMapping.toString()); - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator constructor outputProjection " + Arrays.toString(outputProjection)); - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator constructor outputTypeInfos " + Arrays.toString(outputTypeInfos)); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor bigTableByteColumnVectorColumns " + Arrays.toString(bigTableByteColumnVectorColumns)); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor smallTableByteColumnVectorColumns " + Arrays.toString(smallTableByteColumnVectorColumns)); + + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor outputProjection " + Arrays.toString(outputProjection)); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor outputTypeNames " + Arrays.toString(outputTypeNames)); } setupVOutContext(conf.getOutputColumnNames()); @@ -331,10 +482,10 @@ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); ArrayList<Integer> list = new ArrayList<Integer>(); int count = mapping.getCount(); int[] outputColumns = mapping.getOutputColumns(); - TypeInfo[] typeInfos = mapping.getTypeInfos(); + String[] typeNames = mapping.getTypeNames(); for (int i = 0; i < count; i++) { int outputColumn = outputColumns[i]; - String typeName = typeInfos[i].getTypeName(); + String typeName = typeNames[i]; if (VectorizationContext.isStringFamily(typeName)) { list.add(outputColumn); } @@ -349,10 +500,10 @@ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); */ protected void setupVOutContext(List<String> outputColumnNames) { if (isLogDebugEnabled) { - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator constructor outputColumnNames " + outputColumnNames); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor outputColumnNames " + outputColumnNames); } if (outputColumnNames.size() != outputProjection.length) { - throw new RuntimeException("Output column names " + outputColumnNames + " length and output projection " + Arrays.toString(outputProjection) + " / " + Arrays.toString(outputTypeInfos) + " length mismatch"); + throw new RuntimeException("Output column names " + outputColumnNames + " length and output projection " + Arrays.toString(outputProjection) + " / " + Arrays.toString(outputTypeNames) + " length mismatch"); } vOutContext.resetProjectionColumns(); for (int i = 0; i < outputColumnNames.size(); ++i) { @@ -361,7 +512,7 @@ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); vOutContext.addProjectionColumn(columnName, outputColumn); if (isLogDebugEnabled) { - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator constructor addProjectionColumn " + i + " columnName " + columnName + " outputColumn " + outputColumn); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor addProjectionColumn " + i + " columnName " + columnName + " outputColumn " + outputColumn); } } } @@ -371,7 +522,7 @@ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); */ @Override protected HashTableLoader getHashTableLoader(Configuration hconf) { - VectorMapJoinDesc vectorDesc = (VectorMapJoinDesc) conf.getVectorDesc(); + VectorMapJoinDesc vectorDesc = conf.getVectorDesc(); HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType(); HashTableLoader hashTableLoader; switch (vectorDesc.hashTableImplementationType()) { @@ -395,6 +546,15 @@ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); + if (isLogDebugEnabled) { + // Determine the name of our map or reduce task for debug tracing. + BaseWork work = Utilities.getMapWork(hconf); + if (work == null) { + work = Utilities.getReduceWork(hconf); + } + taskName = work.getName(); + } + /* * Get configuration parameters. */ @@ -410,8 +570,9 @@ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); smallTableVectorDeserializeRow = new VectorDeserializeRow<LazyBinaryDeserializeRead>( new LazyBinaryDeserializeRead( - smallTableMapping.getTypeInfos(), - /* useExternalBuffer */ true)); + VectorizedBatchUtil.typeInfosFromTypeNames( + smallTableMapping.getTypeNames()), + /* useExternalBuffer */ true)); smallTableVectorDeserializeRow.init(smallTableMapping.getOutputColumns()); } @@ -435,13 +596,13 @@ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); if (isLogDebugEnabled) { int[] currentScratchColumns = vOutContext.currentScratchColumns(); - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator initializeOp currentScratchColumns " + Arrays.toString(currentScratchColumns)); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator initializeOp currentScratchColumns " + Arrays.toString(currentScratchColumns)); StructObjectInspector structOutputObjectInspector = (StructObjectInspector) outputObjInspector; List<? extends StructField> fields = structOutputObjectInspector.getAllStructFieldRefs(); int i = 0; for (StructField field : fields) { - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator initializeOp " + i + " field " + field.getFieldName() + " type " + field.getFieldObjectInspector().getTypeName()); + LOG.debug("VectorMapJoinInnerBigOnlyCommonOperator initializeOp " + i + " field " + field.getFieldName() + " type " + field.getFieldObjectInspector().getTypeName()); i++; } } @@ -452,7 +613,7 @@ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); // setup mapJoinTables and serdes super.completeInitializationOp(os); - VectorMapJoinDesc vectorDesc = (VectorMapJoinDesc) conf.getVectorDesc(); + VectorMapJoinDesc vectorDesc = conf.getVectorDesc(); HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType(); switch (vectorDesc.hashTableImplementationType()) { case OPTIMIZED: @@ -494,7 +655,7 @@ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); // First, just allocate just the projection columns we will be using. for (int i = 0; i < outputProjection.length; i++) { int outputColumn = outputProjection[i]; - String typeName = outputTypeInfos[i].getTypeName(); + String typeName = outputTypeNames[i]; allocateOverflowBatchColumnVector(overflowBatch, outputColumn, typeName); } @@ -526,7 +687,7 @@ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); overflowBatch.cols[outputColumn] = VectorizedBatchUtil.createColumnVector(typeInfo); if (isLogDebugEnabled) { - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator initializeOp overflowBatch outputColumn " + outputColumn + " class " + overflowBatch.cols[outputColumn].getClass().getSimpleName()); + LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator initializeOp overflowBatch outputColumn " + outputColumn + " class " + overflowBatch.cols[outputColumn].getClass().getSimpleName()); } } } @@ -563,9 +724,9 @@ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); } protected void displayBatchColumns(VectorizedRowBatch batch, String batchName) { - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator commonSetup " + batchName + " column count " + batch.numCols); + LOG.debug("commonSetup " + batchName + " column count " + batch.numCols); for (int column = 0; column < batch.numCols; column++) { - LOG.debug(getLoggingPrefix() + " VectorMapJoinCommonOperator commonSetup " + batchName + " column " + column + " type " + (batch.cols[column] == null ? "NULL" : batch.cols[column].getClass().getSimpleName())); + LOG.debug("commonSetup " + batchName + " column " + column + " type " + (batch.cols[column] == null ? "NULL" : batch.cols[column].getClass().getSimpleName())); } }
http://git-wip-us.apache.org/repos/asf/hive/blob/aed21d0b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyLongOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyLongOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyLongOperator.java index 43f3951..0bba141 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyLongOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyLongOperator.java @@ -45,17 +45,8 @@ import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; public class VectorMapJoinInnerBigOnlyLongOperator extends VectorMapJoinInnerBigOnlyGenerateResultOperator { private static final long serialVersionUID = 1L; - - //------------------------------------------------------------------------------------------------ - + private static final Logger LOG = LoggerFactory.getLogger(VectorMapJoinInnerBigOnlyLongOperator.class.getName()); private static final String CLASS_NAME = VectorMapJoinInnerBigOnlyLongOperator.class.getName(); - private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); - - protected String getLoggingPrefix() { - return super.getLoggingPrefix(CLASS_NAME); - } - - //------------------------------------------------------------------------------------------------ // (none) http://git-wip-us.apache.org/repos/asf/hive/blob/aed21d0b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyMultiKeyOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyMultiKeyOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyMultiKeyOperator.java index 95fb0c2..621804b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyMultiKeyOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyMultiKeyOperator.java @@ -40,8 +40,6 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorSerializeRow; import org.apache.hadoop.hive.serde2.ByteStream.Output; import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableSerializeWrite; -import com.google.common.base.Preconditions; - /* * Specialized class for doing a vectorized map join that is an inner join on Multi-Key * and only big table columns appear in the join result so a hash multi-set is used. @@ -50,17 +48,8 @@ import com.google.common.base.Preconditions; public class VectorMapJoinInnerBigOnlyMultiKeyOperator extends VectorMapJoinInnerBigOnlyGenerateResultOperator { private static final long serialVersionUID = 1L; - - //------------------------------------------------------------------------------------------------ - + private static final Logger LOG = LoggerFactory.getLogger(VectorMapJoinInnerBigOnlyMultiKeyOperator.class.getName()); private static final String CLASS_NAME = VectorMapJoinInnerBigOnlyMultiKeyOperator.class.getName(); - private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); - - protected String getLoggingPrefix() { - return super.getLoggingPrefix(CLASS_NAME); - } - - //------------------------------------------------------------------------------------------------ // (none) @@ -125,7 +114,7 @@ public class VectorMapJoinInnerBigOnlyMultiKeyOperator extends VectorMapJoinInne keyVectorSerializeWrite = new VectorSerializeRow( new BinarySortableSerializeWrite(bigTableKeyColumnMap.length)); - keyVectorSerializeWrite.init(bigTableKeyTypeInfos, bigTableKeyColumnMap); + keyVectorSerializeWrite.init(bigTableKeyTypeNames, bigTableKeyColumnMap); currentKeyOutput = new Output(); saveKeyOutput = new Output(); http://git-wip-us.apache.org/repos/asf/hive/blob/aed21d0b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java index 044e3e6..10e75ab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java @@ -46,17 +46,8 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; public class VectorMapJoinInnerBigOnlyStringOperator extends VectorMapJoinInnerBigOnlyGenerateResultOperator { private static final long serialVersionUID = 1L; - - //------------------------------------------------------------------------------------------------ - + private static final Logger LOG = LoggerFactory.getLogger(VectorMapJoinInnerBigOnlyStringOperator.class.getName()); private static final String CLASS_NAME = VectorMapJoinInnerBigOnlyStringOperator.class.getName(); - private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); - - protected String getLoggingPrefix() { - return super.getLoggingPrefix(CLASS_NAME); - } - - //------------------------------------------------------------------------------------------------ // (none) http://git-wip-us.apache.org/repos/asf/hive/blob/aed21d0b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerLongOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerLongOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerLongOperator.java index c85e1d8..804d69c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerLongOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerLongOperator.java @@ -44,17 +44,8 @@ import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; public class VectorMapJoinInnerLongOperator extends VectorMapJoinInnerGenerateResultOperator { private static final long serialVersionUID = 1L; - - //------------------------------------------------------------------------------------------------ - + private static final Logger LOG = LoggerFactory.getLogger(VectorMapJoinInnerLongOperator.class.getName()); private static final String CLASS_NAME = VectorMapJoinInnerLongOperator.class.getName(); - private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); - - protected String getLoggingPrefix() { - return super.getLoggingPrefix(CLASS_NAME); - } - - //------------------------------------------------------------------------------------------------ // (none) http://git-wip-us.apache.org/repos/asf/hive/blob/aed21d0b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerMultiKeyOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerMultiKeyOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerMultiKeyOperator.java index a108cd0..fcfa0bd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerMultiKeyOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerMultiKeyOperator.java @@ -39,8 +39,6 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorSerializeRow; import org.apache.hadoop.hive.serde2.ByteStream.Output; import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableSerializeWrite; -import com.google.common.base.Preconditions; - /* * Specialized class for doing a vectorized map join that is an inner join on a Multi-Key * using a hash map. @@ -48,17 +46,8 @@ import com.google.common.base.Preconditions; public class VectorMapJoinInnerMultiKeyOperator extends VectorMapJoinInnerGenerateResultOperator { private static final long serialVersionUID = 1L; - - //------------------------------------------------------------------------------------------------ - - private static final String CLASS_NAME = VectorMapJoinInnerMultiKeyOperator.class.getName(); private static final Logger LOG = LoggerFactory.getLogger(VectorMapJoinInnerMultiKeyOperator.class.getName()); - - protected String getLoggingPrefix() { - return super.getLoggingPrefix(CLASS_NAME); - } - - //------------------------------------------------------------------------------------------------ + private static final String CLASS_NAME = VectorMapJoinInnerMultiKeyOperator.class.getName(); // (none) @@ -123,7 +112,7 @@ public class VectorMapJoinInnerMultiKeyOperator extends VectorMapJoinInnerGenera keyVectorSerializeWrite = new VectorSerializeRow( new BinarySortableSerializeWrite(bigTableKeyColumnMap.length)); - keyVectorSerializeWrite.init(bigTableKeyTypeInfos, bigTableKeyColumnMap); + keyVectorSerializeWrite.init(bigTableKeyTypeNames, bigTableKeyColumnMap); currentKeyOutput = new Output(); saveKeyOutput = new Output(); http://git-wip-us.apache.org/repos/asf/hive/blob/aed21d0b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java index 3211d7d..0f9baae 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java @@ -45,17 +45,8 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; public class VectorMapJoinInnerStringOperator extends VectorMapJoinInnerGenerateResultOperator { private static final long serialVersionUID = 1L; - - //------------------------------------------------------------------------------------------------ - + private static final Logger LOG = LoggerFactory.getLogger(VectorMapJoinInnerStringOperator.class.getName()); private static final String CLASS_NAME = VectorMapJoinInnerStringOperator.class.getName(); - private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); - - protected String getLoggingPrefix() { - return super.getLoggingPrefix(CLASS_NAME); - } - - //------------------------------------------------------------------------------------------------ // (none) http://git-wip-us.apache.org/repos/asf/hive/blob/aed21d0b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiLongOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiLongOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiLongOperator.java index b02e6fd..1149a9d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiLongOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiLongOperator.java @@ -45,17 +45,8 @@ import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; public class VectorMapJoinLeftSemiLongOperator extends VectorMapJoinLeftSemiGenerateResultOperator { private static final long serialVersionUID = 1L; - - //------------------------------------------------------------------------------------------------ - + private static final Logger LOG = LoggerFactory.getLogger(VectorMapJoinInnerBigOnlyLongOperator.class.getName()); private static final String CLASS_NAME = VectorMapJoinLeftSemiLongOperator.class.getName(); - private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); - - protected String getLoggingPrefix() { - return super.getLoggingPrefix(CLASS_NAME); - } - - //------------------------------------------------------------------------------------------------ // (none) http://git-wip-us.apache.org/repos/asf/hive/blob/aed21d0b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiMultiKeyOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiMultiKeyOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiMultiKeyOperator.java index 36b8f3f..e0baebc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiMultiKeyOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiMultiKeyOperator.java @@ -40,8 +40,6 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorSerializeRow; import org.apache.hadoop.hive.serde2.ByteStream.Output; import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableSerializeWrite; -import com.google.common.base.Preconditions; - /* * Specialized class for doing a vectorized map join that is an left semi join on Multi-Key * using hash set. @@ -49,17 +47,8 @@ import com.google.common.base.Preconditions; public class VectorMapJoinLeftSemiMultiKeyOperator extends VectorMapJoinLeftSemiGenerateResultOperator { private static final long serialVersionUID = 1L; - - //------------------------------------------------------------------------------------------------ - + private static final Logger LOG = LoggerFactory.getLogger(VectorMapJoinInnerBigOnlyLongOperator.class.getName()); private static final String CLASS_NAME = VectorMapJoinLeftSemiMultiKeyOperator.class.getName(); - private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); - - protected String getLoggingPrefix() { - return super.getLoggingPrefix(CLASS_NAME); - } - - //------------------------------------------------------------------------------------------------ // (none) @@ -124,7 +113,7 @@ public class VectorMapJoinLeftSemiMultiKeyOperator extends VectorMapJoinLeftSemi keyVectorSerializeWrite = new VectorSerializeRow( new BinarySortableSerializeWrite(bigTableKeyColumnMap.length)); - keyVectorSerializeWrite.init(bigTableKeyTypeInfos, bigTableKeyColumnMap); + keyVectorSerializeWrite.init(bigTableKeyTypeNames, bigTableKeyColumnMap); currentKeyOutput = new Output(); saveKeyOutput = new Output(); http://git-wip-us.apache.org/repos/asf/hive/blob/aed21d0b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java index 0b3de0a..49e1177 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java @@ -46,17 +46,8 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; public class VectorMapJoinLeftSemiStringOperator extends VectorMapJoinLeftSemiGenerateResultOperator { private static final long serialVersionUID = 1L; - - //------------------------------------------------------------------------------------------------ - + private static final Logger LOG = LoggerFactory.getLogger(VectorMapJoinInnerBigOnlyLongOperator.class.getName()); private static final String CLASS_NAME = VectorMapJoinLeftSemiStringOperator.class.getName(); - private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); - - protected String getLoggingPrefix() { - return super.getLoggingPrefix(CLASS_NAME); - } - - //------------------------------------------------------------------------------------------------ // (none) http://git-wip-us.apache.org/repos/asf/hive/blob/aed21d0b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterLongOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterLongOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterLongOperator.java index 72309e8..58bd0ab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterLongOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterLongOperator.java @@ -45,17 +45,8 @@ import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; */ public class VectorMapJoinOuterLongOperator extends VectorMapJoinOuterGenerateResultOperator { private static final long serialVersionUID = 1L; - - //------------------------------------------------------------------------------------------------ - + private static final Logger LOG = LoggerFactory.getLogger(VectorMapJoinOuterLongOperator.class.getName()); private static final String CLASS_NAME = VectorMapJoinOuterLongOperator.class.getName(); - private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); - - protected String getLoggingPrefix() { - return super.getLoggingPrefix(CLASS_NAME); - } - - //------------------------------------------------------------------------------------------------ // (none) http://git-wip-us.apache.org/repos/asf/hive/blob/aed21d0b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterMultiKeyOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterMultiKeyOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterMultiKeyOperator.java index a4fc7d3..7f9afd2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterMultiKeyOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterMultiKeyOperator.java @@ -40,8 +40,6 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorSerializeRow; import org.apache.hadoop.hive.serde2.ByteStream.Output; import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableSerializeWrite; -import com.google.common.base.Preconditions; - /* * Specialized class for doing a vectorized map join that is an outer join on Multi-Key * using a hash map. @@ -49,17 +47,8 @@ import com.google.common.base.Preconditions; public class VectorMapJoinOuterMultiKeyOperator extends VectorMapJoinOuterGenerateResultOperator { private static final long serialVersionUID = 1L; - - //------------------------------------------------------------------------------------------------ - + private static final Logger LOG = LoggerFactory.getLogger(VectorMapJoinOuterMultiKeyOperator.class.getName()); private static final String CLASS_NAME = VectorMapJoinOuterMultiKeyOperator.class.getName(); - private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); - - protected String getLoggingPrefix() { - return super.getLoggingPrefix(CLASS_NAME); - } - - //------------------------------------------------------------------------------------------------ // (none) @@ -123,7 +112,7 @@ public class VectorMapJoinOuterMultiKeyOperator extends VectorMapJoinOuterGenera keyVectorSerializeWrite = new VectorSerializeRow( new BinarySortableSerializeWrite(bigTableKeyColumnMap.length)); - keyVectorSerializeWrite.init(bigTableKeyTypeInfos, bigTableKeyColumnMap); + keyVectorSerializeWrite.init(bigTableKeyTypeNames, bigTableKeyColumnMap); currentKeyOutput = new Output(); saveKeyOutput = new Output(); http://git-wip-us.apache.org/repos/asf/hive/blob/aed21d0b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterStringOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterStringOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterStringOperator.java index 6e7e5cb..8ed1ed4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterStringOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterStringOperator.java @@ -45,17 +45,8 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; public class VectorMapJoinOuterStringOperator extends VectorMapJoinOuterGenerateResultOperator { private static final long serialVersionUID = 1L; - - //------------------------------------------------------------------------------------------------ - + private static final Logger LOG = LoggerFactory.getLogger(VectorMapJoinOuterStringOperator.class.getName()); private static final String CLASS_NAME = VectorMapJoinOuterStringOperator.class.getName(); - private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); - - protected String getLoggingPrefix() { - return super.getLoggingPrefix(CLASS_NAME); - } - - //------------------------------------------------------------------------------------------------ // (none) http://git-wip-us.apache.org/repos/asf/hive/blob/aed21d0b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java index 069cc9a..9f3b107 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java @@ -94,7 +94,7 @@ public class VectorMapJoinFastTableContainer implements VectorMapJoinTableContai private VectorMapJoinFastHashTable createHashTable(int newThreshold) { boolean isOuterJoin = !desc.isNoOuterJoin(); - VectorMapJoinDesc vectorDesc = (VectorMapJoinDesc) desc.getVectorDesc(); + VectorMapJoinDesc vectorDesc = desc.getVectorDesc(); HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType(); HashTableKind hashTableKind = vectorDesc.hashTableKind(); HashTableKeyType hashTableKeyType = vectorDesc.hashTableKeyType(); http://git-wip-us.apache.org/repos/asf/hive/blob/aed21d0b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedCreateHashTable.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedCreateHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedCreateHashTable.java index 111a6d2..f34b1cd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedCreateHashTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedCreateHashTable.java @@ -40,7 +40,7 @@ public class VectorMapJoinOptimizedCreateHashTable { ReusableGetAdaptor hashMapRowGetter = mapJoinTableContainer.createGetter(refKey); boolean isOuterJoin = !desc.isNoOuterJoin(); - VectorMapJoinDesc vectorDesc = (VectorMapJoinDesc) desc.getVectorDesc(); + VectorMapJoinDesc vectorDesc = desc.getVectorDesc(); HashTableKind hashTableKind = vectorDesc.hashTableKind(); HashTableKeyType hashTableKeyType = vectorDesc.hashTableKeyType(); boolean minMaxEnabled = vectorDesc.minMaxEnabled(); http://git-wip-us.apache.org/repos/asf/hive/blob/aed21d0b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkCommonOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkCommonOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkCommonOperator.java index 42ca4b7..8133aef 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkCommonOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkCommonOperator.java @@ -160,7 +160,7 @@ public abstract class VectorReduceSinkCommonOperator extends TerminalOperator<Re ReduceSinkDesc desc = (ReduceSinkDesc) conf; this.conf = desc; - vectorDesc = (VectorReduceSinkDesc) desc.getVectorDesc(); + vectorDesc = desc.getVectorDesc(); vectorReduceSinkInfo = vectorDesc.getVectorReduceSinkInfo(); this.vContext = vContext; http://git-wip-us.apache.org/repos/asf/hive/blob/aed21d0b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java index 6806ab4..20cfb89 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java @@ -374,11 +374,6 @@ public class VectorUDFAdaptor extends VectorExpression { } @Override - public String vectorExpressionParameters() { - return expr.getExprString(); - } - - @Override public VectorExpressionDescriptor.Descriptor getDescriptor() { return (new VectorExpressionDescriptor.Builder()).build(); }
