HIVE-10328 : Loop optimization for SIMD in IfExprColumnColumn.txt (Teddy Choi via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b98a60df Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b98a60df Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b98a60df Branch: refs/heads/beeline-cli Commit: b98a60df2f3779acf82d94965d11ed951b618fad Parents: 6e8eeb7 Author: Teddy Choi <[email protected]> Authored: Tue Aug 11 16:26:00 2015 -0800 Committer: Ashutosh Chauhan <[email protected]> Committed: Wed Sep 23 10:45:02 2015 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ant/GenVectorCode.java | 31 ---- .../vectorization/VectorizationBench.java | 32 +++- .../ExpressionTemplates/IfExprColumnColumn.txt | 186 ------------------- .../IfExprDoubleColumnDoubleColumn.java | 167 +++++++++++++++++ .../expressions/IfExprLongColumnLongColumn.java | 166 +++++++++++++++++ .../hive/ql/udf/generic/GenericUDFIf.java | 4 +- .../exec/vector/TestVectorizationContext.java | 4 +- .../TestVectorConditionalExpressions.java | 3 +- 8 files changed, 369 insertions(+), 224 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/b98a60df/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java ---------------------------------------------------------------------- diff --git a/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java b/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java index fede273..ba7648c 100644 --- a/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java +++ b/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java @@ -788,8 +788,6 @@ public class GenVectorCode extends Task { // IF conditional expression // fileHeader, resultType, arg2Type, arg3Type - {"IfExprColumnColumn", "long"}, - {"IfExprColumnColumn", "double"}, {"IfExprColumnScalar", "long", "long"}, {"IfExprColumnScalar", "double", "long"}, {"IfExprColumnScalar", "long", "double"}, @@ -1051,8 +1049,6 @@ public class GenVectorCode extends Task { generateFilterStringGroupColumnCompareStringGroupColumn(tdesc); } else if (tdesc[0].equals("StringGroupColumnCompareStringGroupColumn")) { generateStringGroupColumnCompareStringGroupColumn(tdesc); - } else if (tdesc[0].equals("IfExprColumnColumn")) { - generateIfExprColumnColumn(tdesc); } else if (tdesc[0].equals("IfExprColumnScalar")) { generateIfExprColumnScalar(tdesc); } else if (tdesc[0].equals("IfExprScalarColumn")) { @@ -1644,33 +1640,6 @@ public class GenVectorCode extends Task { className, templateString); } - private void generateIfExprColumnColumn(String[] tdesc) throws Exception { - String operandType = tdesc[1]; - String inputColumnVectorType = this.getColumnVectorType(operandType); - String outputColumnVectorType = inputColumnVectorType; - String returnType = operandType; - String className = "IfExpr" + getCamelCaseType(operandType) + "Column" - + getCamelCaseType(operandType) + "Column"; - String outputFile = joinPath(this.expressionOutputDirectory, className + ".java"); - File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); - String templateString = readFile(templateFile); - // Expand, and write result - templateString = templateString.replaceAll("<ClassName>", className); - templateString = templateString.replaceAll("<InputColumnVectorType>", inputColumnVectorType); - templateString = templateString.replaceAll("<OperandType>", operandType); - String vectorExprArgType = operandType; - - // Toss in timestamp and date. - if (operandType.equals("long")) { - // Let comparisons occur for DATE and TIMESTAMP, too. - vectorExprArgType = "int_datetime_interval_family"; - } - templateString = templateString.replaceAll("<VectorExprArgType>", vectorExprArgType); - - writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, - className, templateString); - } - private void generateIfExprColumnScalar(String[] tdesc) throws Exception { String operandType2 = tdesc[1]; String operandType3 = tdesc[2]; http://git-wip-us.apache.org/repos/asf/hive/blob/b98a60df/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/VectorizationBench.java ---------------------------------------------------------------------- diff --git a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/VectorizationBench.java b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/VectorizationBench.java index 0e880c6..dcd9501 100644 --- a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/VectorizationBench.java +++ b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/VectorizationBench.java @@ -17,6 +17,8 @@ import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprLongColumnLongColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprDoubleColumnDoubleColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.ColAndCol; import org.apache.hadoop.hive.ql.exec.vector.expressions.ColOrCol; import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColDivideLongColumn; @@ -40,6 +42,7 @@ import org.openjdk.jmh.runner.RunnerException; import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; +import java.lang.Override; import java.util.Random; import java.util.concurrent.TimeUnit; @@ -302,9 +305,36 @@ public class VectorizationBench { } } + public static class IfExprLongColumnLongColumnBench extends AbstractExpression { + @Override + public void setup() { + rowBatch = buildRowBatch(new LongColumnVector(), 3, getBooleanLongColumnVector(), + getLongColumnVector(), getLongColumnVector()); + expression = new IfExprLongColumnLongColumn(0, 1, 2, 3); + } + } + + public static class IfExprRepeatingLongColumnLongColumnBench extends AbstractExpression { + @Override + public void setup() { + rowBatch = buildRowBatch(new LongColumnVector(), 3, getBooleanLongColumnVector(), + getRepeatingLongColumnVector(), getLongColumnVector()); + expression = new IfExprLongColumnLongColumn(0, 1, 2, 3); + } + } + + public static class IfExprLongColumnRepeatingLongColumnBench extends AbstractExpression { + @Override + public void setup() { + rowBatch = buildRowBatch(new LongColumnVector(), 3, getBooleanLongColumnVector(), + getLongColumnVector(), getRepeatingLongColumnVector()); + expression = new IfExprLongColumnLongColumn(0, 1, 2, 3); + } + } + public static void main(String[] args) throws RunnerException { Options opt = new OptionsBuilder().include(".*" + VectorizationBench.class.getSimpleName() + ".*").build(); new Runner(opt).run(); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hive/blob/b98a60df/ql/src/gen/vectorization/ExpressionTemplates/IfExprColumnColumn.txt ---------------------------------------------------------------------- diff --git a/ql/src/gen/vectorization/ExpressionTemplates/IfExprColumnColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/IfExprColumnColumn.txt deleted file mode 100644 index 27d769c..0000000 --- a/ql/src/gen/vectorization/ExpressionTemplates/IfExprColumnColumn.txt +++ /dev/null @@ -1,186 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; - -/** - * Compute IF(expr1, expr2, expr3) for 3 input column expressions. - * The first is always a boolean (LongColumnVector). - * The second and third are long columns or long expression results. - */ -public class <ClassName> extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private int arg1Column, arg2Column, arg3Column; - private int outputColumn; - - public <ClassName>(int arg1Column, int arg2Column, int arg3Column, int outputColumn) { - this.arg1Column = arg1Column; - this.arg2Column = arg2Column; - this.arg3Column = arg3Column; - this.outputColumn = outputColumn; - } - - public <ClassName>() { - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column]; - <InputColumnVectorType> arg2ColVector = (<InputColumnVectorType>) batch.cols[arg2Column]; - <InputColumnVectorType> arg3ColVector = (<InputColumnVectorType>) batch.cols[arg3Column]; - <InputColumnVectorType> outputColVector = (<InputColumnVectorType>) batch.cols[outputColumn]; - int[] sel = batch.selected; - boolean[] outputIsNull = outputColVector.isNull; - outputColVector.noNulls = arg2ColVector.noNulls && arg3ColVector.noNulls; - outputColVector.isRepeating = false; // may override later - int n = batch.size; - long[] vector1 = arg1ColVector.vector; - <OperandType>[] vector2 = arg2ColVector.vector; - <OperandType>[] vector3 = arg3ColVector.vector; - <OperandType>[] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - /* All the code paths below propagate nulls even if neither arg2 nor arg3 - * have nulls. This is to reduce the number of code paths and shorten the - * code, at the expense of maybe doing unnecessary work if neither input - * has nulls. This could be improved in the future by expanding the number - * of code paths. - */ - if (arg1ColVector.isRepeating) { - if (vector1[0] == 1) { - arg2ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector); - } else { - arg3ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector); - } - return; - } - - // extend any repeating values and noNulls indicator in the inputs - arg2ColVector.flatten(batch.selectedInUse, sel, n); - arg3ColVector.flatten(batch.selectedInUse, sel, n); - - if (arg1ColVector.noNulls) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = (vector1[i] == 1 ? vector2[i] : vector3[i]); - outputIsNull[i] = (vector1[i] == 1 ? - arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = (vector1[i] == 1 ? vector2[i] : vector3[i]); - outputIsNull[i] = (vector1[i] == 1 ? - arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); - } - } - } else /* there are nulls */ { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? - vector2[i] : vector3[i]); - outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? - arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? - vector2[i] : vector3[i]); - outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? - arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); - } - } - } - - // restore repeating and no nulls indicators - arg2ColVector.unFlatten(); - arg3ColVector.unFlatten(); - } - - @Override - public int getOutputColumn() { - return outputColumn; - } - - @Override - public String getOutputType() { - return "<OperandType>"; - } - - public int getArg1Column() { - return arg1Column; - } - - public void setArg1Column(int colNum) { - this.arg1Column = colNum; - } - - public int getArg2Column() { - return arg2Column; - } - - public void setArg2Column(int colNum) { - this.arg2Column = colNum; - } - - public int getArg3Column() { - return arg3Column; - } - - public void setArg3Column(int colNum) { - this.arg3Column = colNum; - } - - public void setOutputColumn(int outputColumn) { - this.outputColumn = outputColumn; - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(3) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("long"), - VectorExpressionDescriptor.ArgumentType.getType("<VectorExprArgType>"), - VectorExpressionDescriptor.ArgumentType.getType("<VectorExprArgType>")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.COLUMN, - VectorExpressionDescriptor.InputExpressionType.COLUMN, - VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/b98a60df/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprDoubleColumnDoubleColumn.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprDoubleColumnDoubleColumn.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprDoubleColumnDoubleColumn.java new file mode 100644 index 0000000..71c99f6 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprDoubleColumnDoubleColumn.java @@ -0,0 +1,167 @@ +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + +/** + * Compute IF(expr1, expr2, expr3) for 3 input column expressions. + * The first is always a boolean (LongColumnVector). + * The second and third are long columns or long expression results. + */ +public class IfExprDoubleColumnDoubleColumn extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private int arg1Column, arg2Column, arg3Column; + private int outputColumn; + + public IfExprDoubleColumnDoubleColumn(int arg1Column, int arg2Column, int arg3Column, int outputColumn) { + this.arg1Column = arg1Column; + this.arg2Column = arg2Column; + this.arg3Column = arg3Column; + this.outputColumn = outputColumn; + } + + public IfExprDoubleColumnDoubleColumn() { + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column]; + DoubleColumnVector arg2ColVector = (DoubleColumnVector) batch.cols[arg2Column]; + DoubleColumnVector arg3ColVector = (DoubleColumnVector) batch.cols[arg3Column]; + DoubleColumnVector outputColVector = (DoubleColumnVector) batch.cols[outputColumn]; + int[] sel = batch.selected; + boolean[] outputIsNull = outputColVector.isNull; + outputColVector.noNulls = arg2ColVector.noNulls && arg3ColVector.noNulls; + outputColVector.isRepeating = false; // may override later + int n = batch.size; + long[] vector1 = arg1ColVector.vector; + double[] vector2 = arg2ColVector.vector; + double[] vector3 = arg3ColVector.vector; + double[] outputVector = outputColVector.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + /* All the code paths below propagate nulls even if neither arg2 nor arg3 + * have nulls. This is to reduce the number of code paths and shorten the + * code, at the expense of maybe doing unnecessary work if neither input + * has nulls. This could be improved in the future by expanding the number + * of code paths. + */ + if (arg1ColVector.isRepeating) { + if (vector1[0] == 1) { + arg2ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector); + } else { + arg3ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector); + } + return; + } + + // extend any repeating values and noNulls indicator in the inputs + arg2ColVector.flatten(batch.selectedInUse, sel, n); + arg3ColVector.flatten(batch.selectedInUse, sel, n); + + if (arg1ColVector.noNulls) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputVector[i] = (vector1[i] == 1 ? vector2[i] : vector3[i]); + outputIsNull[i] = (vector1[i] == 1 ? + arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = (vector1[i] == 1 ? vector2[i] : vector3[i]); + outputIsNull[i] = (vector1[i] == 1 ? + arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); + } + } + } else /* there are nulls */ { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputVector[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? + vector2[i] : vector3[i]); + outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? + arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? + vector2[i] : vector3[i]); + outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? + arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); + } + } + } + + // restore repeating and no nulls indicators + arg2ColVector.unFlatten(); + arg3ColVector.unFlatten(); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return "double"; + } + + public int getArg1Column() { + return arg1Column; + } + + public void setArg1Column(int colNum) { + this.arg1Column = colNum; + } + + public int getArg2Column() { + return arg2Column; + } + + public void setArg2Column(int colNum) { + this.arg2Column = colNum; + } + + public int getArg3Column() { + return arg3Column; + } + + public void setArg3Column(int colNum) { + this.arg3Column = colNum; + } + + public void setOutputColumn(int outputColumn) { + this.outputColumn = outputColumn; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(3) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.getType("long"), + VectorExpressionDescriptor.ArgumentType.getType("double"), + VectorExpressionDescriptor.ArgumentType.getType("double")) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/b98a60df/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprLongColumnLongColumn.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprLongColumnLongColumn.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprLongColumnLongColumn.java new file mode 100644 index 0000000..00485a2 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprLongColumnLongColumn.java @@ -0,0 +1,166 @@ +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + +/** + * Compute IF(expr1, expr2, expr3) for 3 input column expressions. + * The first is always a boolean (LongColumnVector). + * The second and third are long columns or long expression results. + */ +public class IfExprLongColumnLongColumn extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private int arg1Column, arg2Column, arg3Column; + private int outputColumn; + + public IfExprLongColumnLongColumn(int arg1Column, int arg2Column, int arg3Column, int outputColumn) { + this.arg1Column = arg1Column; + this.arg2Column = arg2Column; + this.arg3Column = arg3Column; + this.outputColumn = outputColumn; + } + + public IfExprLongColumnLongColumn() { + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column]; + LongColumnVector arg2ColVector = (LongColumnVector) batch.cols[arg2Column]; + LongColumnVector arg3ColVector = (LongColumnVector) batch.cols[arg3Column]; + LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn]; + int[] sel = batch.selected; + boolean[] outputIsNull = outputColVector.isNull; + outputColVector.noNulls = arg2ColVector.noNulls && arg3ColVector.noNulls; + outputColVector.isRepeating = false; // may override later + int n = batch.size; + long[] vector1 = arg1ColVector.vector; + long[] vector2 = arg2ColVector.vector; + long[] vector3 = arg3ColVector.vector; + long[] outputVector = outputColVector.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + /* All the code paths below propagate nulls even if neither arg2 nor arg3 + * have nulls. This is to reduce the number of code paths and shorten the + * code, at the expense of maybe doing unnecessary work if neither input + * has nulls. This could be improved in the future by expanding the number + * of code paths. + */ + if (arg1ColVector.isRepeating) { + if (vector1[0] == 1) { + arg2ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector); + } else { + arg3ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector); + } + return; + } + + // extend any repeating values and noNulls indicator in the inputs + arg2ColVector.flatten(batch.selectedInUse, sel, n); + arg3ColVector.flatten(batch.selectedInUse, sel, n); + + if (arg1ColVector.noNulls) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputVector[i] = (~(vector1[i] - 1) & vector2[i]) | ((vector1[i] - 1) & vector3[i]); + outputIsNull[i] = (vector1[i] == 1 ? + arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = (~(vector1[i] - 1) & vector2[i]) | ((vector1[i] - 1) & vector3[i]); + outputIsNull[i] = (vector1[i] == 1 ? + arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); + } + } + } else /* there are nulls */ { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputVector[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? + vector2[i] : vector3[i]); + outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? + arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? + vector2[i] : vector3[i]); + outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? + arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); + } + } + } + + // restore repeating and no nulls indicators + arg2ColVector.unFlatten(); + arg3ColVector.unFlatten(); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return "long"; + } + + public int getArg1Column() { + return arg1Column; + } + + public void setArg1Column(int colNum) { + this.arg1Column = colNum; + } + + public int getArg2Column() { + return arg2Column; + } + + public void setArg2Column(int colNum) { + this.arg2Column = colNum; + } + + public int getArg3Column() { + return arg3Column; + } + + public void setArg3Column(int colNum) { + this.arg3Column = colNum; + } + + public void setOutputColumn(int outputColumn) { + this.outputColumn = outputColumn; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(3) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.getType("long"), + VectorExpressionDescriptor.ArgumentType.getType("int_datetime_interval_family"), + VectorExpressionDescriptor.ArgumentType.getType("int_datetime_interval_family")) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/b98a60df/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java index 568fd46..b5e2837 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java @@ -28,8 +28,6 @@ import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; -import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongColumnLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleColumnDoubleColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongColumnLongScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleColumnDoubleScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleColumnLongScalar; @@ -42,6 +40,8 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongScalarLon import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleScalarDoubleScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleScalarLongScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongScalarDoubleScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprDoubleColumnDoubleColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprLongColumnLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringGroupColumnStringGroupColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringGroupColumnStringScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringGroupColumnCharScalar; http://git-wip-us.apache.org/repos/asf/hive/blob/b98a60df/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java index 8470c47..704c654 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java @@ -42,6 +42,8 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.FuncLogWithBaseDoubleTo import org.apache.hadoop.hive.ql.exec.vector.expressions.FuncLogWithBaseLongToDouble; import org.apache.hadoop.hive.ql.exec.vector.expressions.FuncPowerDoubleToDouble; import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprCharScalarStringGroupColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprDoubleColumnDoubleColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprLongColumnLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringGroupColumnCharScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringGroupColumnStringGroupColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringGroupColumnStringScalar; @@ -68,11 +70,9 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFYearLong; import org.apache.hadoop.hive.ql.exec.vector.expressions.FilterStringColumnInList; import org.apache.hadoop.hive.ql.exec.vector.expressions.FilterLongColumnInList; import org.apache.hadoop.hive.ql.exec.vector.expressions.FilterDoubleColumnInList; -import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongColumnLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongColumnLongScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongScalarLongScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongScalarLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleColumnDoubleColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleColumnDoubleScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleScalarDoubleColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleScalarDoubleScalar; http://git-wip-us.apache.org/repos/asf/hive/blob/b98a60df/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorConditionalExpressions.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorConditionalExpressions.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorConditionalExpressions.java index a711b55..47ebe57 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorConditionalExpressions.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorConditionalExpressions.java @@ -24,8 +24,7 @@ import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongColumnLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleColumnDoubleColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprDoubleColumnDoubleColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongColumnLongScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongScalarLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongScalarLongScalar;
