Author: omalley
Date: Wed May 22 21:04:35 2013
New Revision: 1485421
URL: http://svn.apache.org/r1485421
Log:
HIVE-4553 Column Column, and Column Scalar vectorized execution tests
(Tony Murphy via omalley)
Added:
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestClass.txt
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestCodeGen.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnFilterVectorExpressionEvaluation.txt
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnOperationVectorExpressionEvaluation.txt
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarFilterVectorExpressionEvaluation.txt
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarOperationVectorExpressionEvaluation.txt
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnColumnFilterVectorExpressionEvaluation.java
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnColumnOperationVectorExpressionEvaluation.java
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnScalarFilterVectorExpressionEvaluation.java
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnScalarOperationVectorExpressionEvaluation.java
Modified:
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java
Modified:
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java
URL:
http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java?rev=1485421&r1=1485420&r2=1485421&view=diff
==============================================================================
---
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java
(original)
+++
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java
Wed May 22 21:04:35 2013
@@ -216,10 +216,12 @@ public class CodeGen {
};
+
private final String templateDirectory;
private final String outputDirectory;
+ private final TestCodeGen testCodeGen;
- private static String joinPath(String...parts) {
+ static String joinPath(String...parts) {
String path = parts[0];
for (int i=1; i < parts.length; ++i) {
path += File.separatorChar + parts[i];
@@ -231,11 +233,13 @@ public class CodeGen {
templateDirectory = System.getProperty("user.dir");
File f = new File(templateDirectory);
outputDirectory = joinPath(f.getParent(), "gen");
+ testCodeGen = new TestCodeGen(joinPath(f.getParent(),
"test"),templateDirectory);
}
- public CodeGen(String templateDirectory, String outputDirectory) {
+ public CodeGen(String templateDirectory, String outputDirectory, String
testOutputDirectory) {
this.templateDirectory = templateDirectory;
this.outputDirectory = outputDirectory;
+ testCodeGen = new TestCodeGen(testOutputDirectory,templateDirectory);
}
/**
@@ -244,10 +248,13 @@ public class CodeGen {
*/
public static void main(String[] args) throws Exception {
CodeGen gen;
- if (args != null && args[0] != null) {
- gen = new CodeGen(args[0], args[1]);
- } else {
+ if (args == null || args.length==0) {
gen = new CodeGen();
+ } else if (args.length==3) {
+ gen = new CodeGen(args[0], args[1], args[2]);
+ }else{
+ System.out.println("args: <templateDir> <outputDir> <testOutputDir>");
+ return;
}
gen.generate();
}
@@ -287,6 +294,7 @@ public class CodeGen {
continue;
}
}
+ testCodeGen.generateTestSuites();
}
private void generateVectorUDAFMinMax(String[] tdesc) throws Exception {
@@ -417,9 +425,9 @@ public class CodeGen {
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
- String className = "Filter" + this.getCamelCaseType(operandType1)
- + "Col" + operatorName + this.getCamelCaseType(operandType2) +
"Column";
- generateColumnBinaryOperatorColumn(tdesc, "doesn't matter", className);
+ String className = "Filter" + getCamelCaseType(operandType1)
+ + "Col" + operatorName + getCamelCaseType(operandType2) + "Column";
+ generateColumnBinaryOperatorColumn(tdesc, null, className);
}
private void generateColumnUnaryMinus(String[] tdesc) throws IOException {
@@ -427,7 +435,7 @@ public class CodeGen {
String inputColumnVectorType = this.getColumnVectorType(operandType);
String outputColumnVectorType = inputColumnVectorType;
String returnType = operandType;
- String className = this.getCamelCaseType(operandType) + "ColUnaryMinus";
+ String className = getCamelCaseType(operandType) + "ColUnaryMinus";
String outputFile = joinPath(this.outputDirectory, className + ".java");
String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt");
String templateString = readFile(templateFile);
@@ -444,8 +452,8 @@ public class CodeGen {
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
- String className = this.getCamelCaseType(operandType1)
- + "Col" + operatorName + this.getCamelCaseType(operandType2) +
"Column";
+ String className = getCamelCaseType(operandType1)
+ + "Col" + operatorName + getCamelCaseType(operandType2) + "Column";
String returnType = getArithmeticReturnType(operandType1, operandType2);
generateColumnBinaryOperatorColumn(tdesc, returnType, className);
}
@@ -456,9 +464,9 @@ public class CodeGen {
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
- String className = "Filter" + this.getCamelCaseType(operandType1)
- + "Col" + operatorName + this.getCamelCaseType(operandType2) +
"Scalar";
- generateColumnBinaryOperatorScalar(tdesc, "doesn't matter", className);
+ String className = "Filter" + getCamelCaseType(operandType1)
+ + "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar";
+ generateColumnBinaryOperatorScalar(tdesc, null, className);
}
private void generateFilterScalarCompareColumn(String[] tdesc) throws
IOException {
@@ -466,9 +474,9 @@ public class CodeGen {
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
- String className = "Filter" + this.getCamelCaseType(operandType1)
- + "Scalar" + operatorName + this.getCamelCaseType(operandType2) +
"Column";
- generateScalarBinaryOperatorColumn(tdesc, "doesn't matter", className);
+ String className = "Filter" + getCamelCaseType(operandType1)
+ + "Scalar" + operatorName + getCamelCaseType(operandType2) + "Column";
+ generateScalarBinaryOperatorColumn(tdesc, null, className);
}
private void generateColumnCompareScalar(String[] tdesc) throws IOException {
@@ -476,8 +484,8 @@ public class CodeGen {
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
String returnType = "long";
- String className = this.getCamelCaseType(operandType1)
- + "Col" + operatorName + this.getCamelCaseType(operandType2) +
"Scalar";
+ String className = getCamelCaseType(operandType1)
+ + "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar";
generateColumnBinaryOperatorScalar(tdesc, returnType, className);
}
@@ -503,6 +511,20 @@ public class CodeGen {
templateString = templateString.replaceAll("<OperandType2>", operandType2);
templateString = templateString.replaceAll("<ReturnType>", returnType);
writeFile(outputFile, templateString);
+
+ if(returnType==null){
+ testCodeGen.addColumnColumnFilterTestCases(
+ className,
+ inputColumnVectorType1,
+ inputColumnVectorType2,
+ operatorSymbol);
+ }else{
+ testCodeGen.addColumnColumnOperationTestCases(
+ className,
+ inputColumnVectorType1,
+ inputColumnVectorType2,
+ outputColumnVectorType);
+ }
}
private void generateColumnBinaryOperatorScalar(String[] tdesc, String
returnType,
@@ -525,6 +547,25 @@ public class CodeGen {
templateString = templateString.replaceAll("<OperandType2>", operandType2);
templateString = templateString.replaceAll("<ReturnType>", returnType);
writeFile(outputFile, templateString);
+
+ if(returnType==null)
+ {
+ testCodeGen.addColumnScalarFilterTestCases(
+ true,
+ className,
+ inputColumnVectorType,
+ operandType2,
+ operatorSymbol);
+ }else
+ {
+ testCodeGen.addColumnScalarOperationTestCases(
+ true,
+ className,
+ inputColumnVectorType,
+ outputColumnVectorType,
+ operandType2);
+ }
+
}
private void generateScalarBinaryOperatorColumn(String[] tdesc, String
returnType,
@@ -547,6 +588,24 @@ public class CodeGen {
templateString = templateString.replaceAll("<OperandType2>",
operandType2);
templateString = templateString.replaceAll("<ReturnType>", returnType);
writeFile(outputFile, templateString);
+
+ if(returnType==null)
+ {
+ testCodeGen.addColumnScalarFilterTestCases(
+ false,
+ className,
+ inputColumnVectorType,
+ operandType1,
+ operatorSymbol);
+ }else
+ {
+ testCodeGen.addColumnScalarOperationTestCases(
+ false,
+ className,
+ inputColumnVectorType,
+ outputColumnVectorType,
+ operandType1);
+ }
}
//Binary arithmetic operator
@@ -554,8 +613,8 @@ public class CodeGen {
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
- String className = this.getCamelCaseType(operandType1)
- + "Col" + operatorName + this.getCamelCaseType(operandType2) +
"Scalar";
+ String className = getCamelCaseType(operandType1)
+ + "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar";
String returnType = getArithmeticReturnType(operandType1, operandType2);
generateColumnBinaryOperatorScalar(tdesc, returnType, className);
}
@@ -564,19 +623,20 @@ public class CodeGen {
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
- String className = this.getCamelCaseType(operandType1)
- + "Scalar" + operatorName + this.getCamelCaseType(operandType2) +
"Column";
+ String className = getCamelCaseType(operandType1)
+ + "Scalar" + operatorName + getCamelCaseType(operandType2) + "Column";
String returnType = getArithmeticReturnType(operandType1, operandType2);
generateScalarBinaryOperatorColumn(tdesc, returnType, className);
}
- private void writeFile(String outputFile, String str) throws IOException {
+
+ static void writeFile(String outputFile, String str) throws IOException {
BufferedWriter w = new BufferedWriter(new FileWriter(outputFile));
w.write(str);
w.close();
}
- private String readFile(String templateFile) throws IOException {
+ static String readFile(String templateFile) throws IOException {
BufferedReader r = new BufferedReader(new FileReader(templateFile));
String line = r.readLine();
StringBuilder b = new StringBuilder();
@@ -589,7 +649,7 @@ public class CodeGen {
return b.toString();
}
- private String getCamelCaseType(String type) {
+ static String getCamelCaseType(String type) {
if (type.equals("long")) {
return "Long";
} else if (type.equals("double")) {
@@ -610,9 +670,7 @@ public class CodeGen {
}
private String getColumnVectorType(String primitiveType) {
- if (primitiveType.equals("long")) {
- return "LongColumnVector";
- } else if (primitiveType.equals("double")) {
+ if(primitiveType!=null && primitiveType.equals("double")) {
return "DoubleColumnVector";
}
return "LongColumnVector";
Added:
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestClass.txt
URL:
http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestClass.txt?rev=1485421&view=auto
==============================================================================
---
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestClass.txt
(added)
+++
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestClass.txt
Wed May 22 21:04:35 2013
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+
+import static org.junit.Assert.assertEquals;
+import java.util.Random;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.util.VectorizedRowGroupGenUtil;
+import org.junit.Test;
+
+
+/**
+ *
+ * <ClassName>.
+ *
+ */
+public class <ClassName>{
+
+ private static final int BATCH_SIZE = 100;
+ private static final long SEED = 0xfa57;
+
+ <TestCases>
+
+}
+
+
Added:
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestCodeGen.java
URL:
http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestCodeGen.java?rev=1485421&view=auto
==============================================================================
---
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestCodeGen.java
(added)
+++
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestCodeGen.java
Wed May 22 21:04:35 2013
@@ -0,0 +1,256 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.templates;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+/**
+ *
+ * TestCodeGen.
+ * This class is mutable and maintains a hashmap of TestSuiteClassName to test
cases.
+ * The tests cases are added over the course of vectorized expressions class
generation,
+ * with test classes being outputted at the end. For each column vector
(inputs and/or outputs)
+ * a matrix of pairwise covering Booleans is used to generate test cases
across nulls and
+ * repeating dimensions. Based on the input column vector(s) nulls and
repeating states
+ * the states of the output column vector (if there is one) is validated,
along with the null
+ * vector. For filter operations the selection vector is validated against the
generated
+ * data. Each template corresponds to a class representing a test suite.
+ */
+public class TestCodeGen {
+
+ public enum TestSuiteClassName{
+ TestColumnScalarOperationVectorExpressionEvaluation,
+ TestColumnScalarFilterVectorExpressionEvaluation,
+ TestColumnColumnOperationVectorExpressionEvaluation,
+ TestColumnColumnFilterVectorExpressionEvaluation,
+ }
+
+ private final String testOutputDir;
+ private final String templateDirectory;
+ private final HashMap<TestSuiteClassName,StringBuilder> testsuites;
+
+ public TestCodeGen(String testOutputDir, String templateDirectory) {
+ this.testOutputDir = testOutputDir;
+ this.templateDirectory = templateDirectory;
+ testsuites = new HashMap<TestSuiteClassName, StringBuilder>();
+
+ for(TestSuiteClassName className : TestSuiteClassName.values()) {
+ testsuites.put(className,new StringBuilder());
+ }
+
+ }
+
+ public void addColumnScalarOperationTestCases(boolean op1IsCol, String
vectorExpClassName,
+ String inputColumnVectorType, String outputColumnVectorType, String
scalarType)
+ throws IOException {
+
+ TestSuiteClassName template =
+ TestSuiteClassName.TestColumnScalarOperationVectorExpressionEvaluation;
+
+ //Read the template into a string;
+ String templateFile =
CodeGen.joinPath(this.templateDirectory,template.toString()+".txt");
+ String templateString =
removeTemplateComments(CodeGen.readFile(templateFile));
+
+ for(Boolean[] testMatrix :new Boolean[][]{
+ // Pairwise: InitOuputColHasNulls, InitOuputColIsRepeating,
ColumnHasNulls, ColumnIsRepeating
+ {false, true, true, true},
+ {false, false, false, false},
+ {true, false, true, false},
+ {true, true, false, false},
+ {true, false, false, true}}) {
+ String testCase = templateString;
+ testCase = testCase.replaceAll("<TestName>",
+ "test"
+ + vectorExpClassName
+ + createNullRepeatingNameFragment("Out", testMatrix[0],
testMatrix[1])
+ + createNullRepeatingNameFragment("Col", testMatrix[2],
testMatrix[3]));
+ testCase = testCase.replaceAll("<VectorExpClassName>",
vectorExpClassName);
+ testCase = testCase.replaceAll("<InputColumnVectorType>",
inputColumnVectorType);
+ testCase = testCase.replaceAll("<OutputColumnVectorType>",
outputColumnVectorType);
+ testCase = testCase.replaceAll("<ScalarType>", scalarType);
+ testCase = testCase.replaceAll("<CamelCaseScalarType>",
CodeGen.getCamelCaseType(scalarType));
+ testCase = testCase.replaceAll("<InitOuputColHasNulls>",
testMatrix[0].toString());
+ testCase = testCase.replaceAll("<InitOuputColIsRepeating>",
testMatrix[1].toString());
+ testCase = testCase.replaceAll("<ColumnHasNulls>",
testMatrix[2].toString());
+ testCase = testCase.replaceAll("<ColumnIsRepeating>",
testMatrix[3].toString());
+
+ if(op1IsCol){
+ testCase = testCase.replaceAll("<ConstructorParams>","0, scalarValue");
+ }else{
+ testCase = testCase.replaceAll("<ConstructorParams>","scalarValue, 0");
+ }
+
+ testsuites.get(template).append(testCase);
+ }
+ }
+
+ public void addColumnScalarFilterTestCases(boolean op1IsCol, String
vectorExpClassName,
+ String inputColumnVectorType, String scalarType, String operatorSymbol)
+ throws IOException {
+
+ TestSuiteClassName template =
+ TestSuiteClassName.TestColumnScalarFilterVectorExpressionEvaluation;
+
+ //Read the template into a string;
+ String templateFile =
CodeGen.joinPath(this.templateDirectory,template.toString()+".txt");
+ String templateString =
removeTemplateComments(CodeGen.readFile(templateFile));
+
+ for(Boolean[] testMatrix : new Boolean[][]{
+ // Pairwise: ColumnHasNulls, ColumnIsRepeating
+ {true, true},
+ {true, false},
+ {false, false},
+ {false, true}}) {
+ String testCase = templateString;
+ testCase = testCase.replaceAll("<TestName>",
+ "test"
+ + vectorExpClassName
+ + createNullRepeatingNameFragment("Col", testMatrix[0],
testMatrix[1]));
+ testCase = testCase.replaceAll("<VectorExpClassName>",
vectorExpClassName);
+ testCase = testCase.replaceAll("<InputColumnVectorType>",
inputColumnVectorType);
+ testCase = testCase.replaceAll("<ScalarType>", scalarType);
+ testCase = testCase.replaceAll("<CamelCaseScalarType>",
CodeGen.getCamelCaseType(scalarType));
+ testCase = testCase.replaceAll("<ColumnHasNulls>",
testMatrix[0].toString());
+ testCase = testCase.replaceAll("<ColumnIsRepeating>",
testMatrix[1].toString());
+ testCase = testCase.replaceAll("<Operator>", operatorSymbol);
+
+ if(op1IsCol){
+ testCase = testCase.replaceAll("<ConstructorParams>","0, scalarValue");
+ }else{
+ testCase = testCase.replaceAll("<ConstructorParams>","scalarValue, 0");
+ }
+
+ testsuites.get(template).append(testCase);
+ }
+ }
+
+ public void addColumnColumnOperationTestCases(String vectorExpClassName,
+ String inputColumnVectorType1, String inputColumnVectorType2, String
outputColumnVectorType)
+ throws IOException {
+
+ TestSuiteClassName template=
+ TestSuiteClassName.TestColumnColumnOperationVectorExpressionEvaluation;
+
+ //Read the template into a string;
+ String templateFile =
CodeGen.joinPath(this.templateDirectory,template.toString()+".txt");
+ String templateString =
removeTemplateComments(CodeGen.readFile(templateFile));
+
+ for(Boolean[] testMatrix : new Boolean[][]{
+ // Pairwise: InitOuputColHasNulls, InitOuputColIsRepeating,
Column1HasNulls,
+ // Column1IsRepeating, Column2HasNulls, Column2IsRepeating
+ {true, true, false, true, true, true},
+ {false, false, true, false, false, false},
+ {true, false, true, false, true, true},
+ {true, true, true, true, false, false},
+ {false, false, false, true, true, false},
+ {false, true, false, false, false, true}}) {
+ String testCase = templateString;
+ testCase = testCase.replaceAll("<TestName>",
+ "test"
+ + vectorExpClassName
+ + createNullRepeatingNameFragment("Out", testMatrix[0],
testMatrix[1])
+ + createNullRepeatingNameFragment("C1", testMatrix[2], testMatrix[3])
+ + createNullRepeatingNameFragment("C2", testMatrix[4],
testMatrix[5]));
+ testCase = testCase.replaceAll("<VectorExpClassName>",
vectorExpClassName);
+ testCase = testCase.replaceAll("<InputColumnVectorType1>",
inputColumnVectorType1);
+ testCase = testCase.replaceAll("<InputColumnVectorType2>",
inputColumnVectorType2);
+ testCase = testCase.replaceAll("<OutputColumnVectorType>",
outputColumnVectorType);
+ testCase = testCase.replaceAll("<InitOuputColHasNulls>",
testMatrix[0].toString());
+ testCase = testCase.replaceAll("<InitOuputColIsRepeating>",
testMatrix[1].toString());
+ testCase = testCase.replaceAll("<Column1HasNulls>",
testMatrix[2].toString());
+ testCase = testCase.replaceAll("<Column1IsRepeating>",
testMatrix[3].toString());
+ testCase = testCase.replaceAll("<Column2HasNulls>",
testMatrix[4].toString());
+ testCase = testCase.replaceAll("<Column2IsRepeating>",
testMatrix[5].toString());
+
+ testsuites.get(template).append(testCase);
+ }
+ }
+
+ public void addColumnColumnFilterTestCases(String vectorExpClassName,
+ String inputColumnVectorType1, String inputColumnVectorType2, String
operatorSymbol)
+ throws IOException {
+
+ TestSuiteClassName template=
+ TestSuiteClassName.TestColumnColumnFilterVectorExpressionEvaluation;
+
+ //Read the template into a string;
+ String templateFile =
CodeGen.joinPath(this.templateDirectory,template.toString()+".txt");
+ String templateString =
removeTemplateComments(CodeGen.readFile(templateFile));
+
+ for(Boolean[] testMatrix : new Boolean[][]{
+ // Pairwise: Column1HasNulls, Column1IsRepeating, Column2HasNulls,
Column2IsRepeating
+ {false, true, true, true},
+ {false, false, false, false},
+ {true, false, true, false},
+ {true, true, false, false},
+ {true, false, false, true}}) {
+ String testCase = templateString;
+ testCase = testCase.replaceAll("<TestName>",
+ "test"
+ + vectorExpClassName
+ + createNullRepeatingNameFragment("C1", testMatrix[0],
testMatrix[1])
+ + createNullRepeatingNameFragment("C2", testMatrix[2],
testMatrix[3]));
+ testCase = testCase.replaceAll("<VectorExpClassName>",
vectorExpClassName);
+ testCase = testCase.replaceAll("<InputColumnVectorType1>",
inputColumnVectorType1);
+ testCase = testCase.replaceAll("<InputColumnVectorType2>",
inputColumnVectorType2);
+ testCase = testCase.replaceAll("<Column1HasNulls>",
testMatrix[0].toString());
+ testCase = testCase.replaceAll("<Column1IsRepeating>",
testMatrix[1].toString());
+ testCase = testCase.replaceAll("<Column2HasNulls>",
testMatrix[2].toString());
+ testCase = testCase.replaceAll("<Column2IsRepeating>",
testMatrix[3].toString());
+ testCase = testCase.replaceAll("<Operator>", operatorSymbol);
+
+ testsuites.get(template).append(testCase);
+ }
+ }
+
+ public void generateTestSuites() throws IOException {
+
+ String templateFile = CodeGen.joinPath(this.templateDirectory,
"TestClass.txt");
+ for(TestSuiteClassName testClass : testsuites.keySet()) {
+
+ String templateString = CodeGen.readFile(templateFile);
+ templateString = templateString.replaceAll("<ClassName>",
testClass.toString());
+ templateString = templateString.replaceAll("<TestCases>",
testsuites.get(testClass).toString());
+
+ String outputFile = CodeGen.joinPath(this.testOutputDir, testClass +
".java");
+
+ CodeGen.writeFile(outputFile, templateString);
+ }
+ }
+
+ private static String createNullRepeatingNameFragment(String idenitfier,
boolean nulls, boolean repeating)
+ {
+ if(nulls || repeating){
+ if(nulls){
+ idenitfier+="Nulls";
+ }
+ if(repeating){
+ idenitfier+="Repeats";
+ }
+ return idenitfier;
+ }
+
+ return "";
+ }
+
+ private static String removeTemplateComments(String templateString){
+ return templateString.replaceAll("(?s)<!--(.*)-->", "");
+ }
+}
Added:
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnFilterVectorExpressionEvaluation.txt
URL:
http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnFilterVectorExpressionEvaluation.txt?rev=1485421&view=auto
==============================================================================
---
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnFilterVectorExpressionEvaluation.txt
(added)
+++
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnFilterVectorExpressionEvaluation.txt
Wed May 22 21:04:35 2013
@@ -0,0 +1,69 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ @Test
+ public void <TestName>() {
+
+ Random rand = new Random(SEED);
+
+ <InputColumnVectorType1> inputColumnVector1 =
+
VectorizedRowGroupGenUtil.generate<InputColumnVectorType1>(<Column1HasNulls>,
+ <Column1IsRepeating>, BATCH_SIZE, rand);
+
+ <InputColumnVectorType2> inputColumnVector2 =
+
VectorizedRowGroupGenUtil.generate<InputColumnVectorType2>(<Column2HasNulls>,
+ <Column2IsRepeating>, BATCH_SIZE, rand);
+
+ VectorizedRowBatch rowBatch = new VectorizedRowBatch(2, BATCH_SIZE);
+ rowBatch.cols[0] = inputColumnVector1;
+ rowBatch.cols[1] = inputColumnVector2;
+
+ <VectorExpClassName> vectorExpression =
+ new <VectorExpClassName>(0, 1);
+
+ vectorExpression.evaluate(rowBatch);
+
+ int selectedIndex = 0;
+ for(int i = 0; i < BATCH_SIZE; i++) {
+ //null vector is safe to check, as it is always initialized to match the
data vector
+ if(!inputColumnVector1.isNull[i] && !inputColumnVector2.isNull[i]) {
+ if(inputColumnVector1.vector[i] <Operator>
inputColumnVector2.vector[i]) {
+ assertEquals(
+ "Vector index that passes filter "
+ + inputColumnVector1.vector[i] + "<Operator>"
+ + inputColumnVector2.vector[i] + " is not in rowBatch selected
index",
+ i,
+ rowBatch.selected[selectedIndex]);
+ selectedIndex++;
+ }
+ }
+ }
+
+ assertEquals("Row batch size not set to number of selected rows: " +
selectedIndex,
+ selectedIndex, rowBatch.size);
+
+ if(selectedIndex > 0 && selectedIndex < BATCH_SIZE) {
+ assertEquals(
+ "selectedInUse should be set when > 0 and < entire batch(" +
BATCH_SIZE + ") is selected: "
+ + selectedIndex,
+ true, rowBatch.selectedInUse);
+ } else if(selectedIndex == BATCH_SIZE) {
+ assertEquals(
+ "selectedInUse should not be set when entire batch(" + BATCH_SIZE + ")
is selected: "
+ + selectedIndex,
+ false, rowBatch.selectedInUse);
+ }
+ }
\ No newline at end of file
Added:
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnOperationVectorExpressionEvaluation.txt
URL:
http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnOperationVectorExpressionEvaluation.txt?rev=1485421&view=auto
==============================================================================
---
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnOperationVectorExpressionEvaluation.txt
(added)
+++
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnOperationVectorExpressionEvaluation.txt
Wed May 22 21:04:35 2013
@@ -0,0 +1,64 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ @Test
+ public void <TestName>() {
+
+ Random rand = new Random(SEED);
+
+ <OutputColumnVectorType> outputColumnVector =
+
VectorizedRowGroupGenUtil.generate<OutputColumnVectorType>(<InitOuputColHasNulls>,
+ <InitOuputColIsRepeating>, BATCH_SIZE, rand);
+
+ <InputColumnVectorType1> inputColumnVector1 =
+
VectorizedRowGroupGenUtil.generate<InputColumnVectorType1>(<Column1HasNulls>,
+ <Column1IsRepeating>, BATCH_SIZE, rand);
+
+ <InputColumnVectorType2> inputColumnVector2 =
+
VectorizedRowGroupGenUtil.generate<InputColumnVectorType2>(<Column2HasNulls>,
+ <Column2IsRepeating>, BATCH_SIZE, rand);
+
+ VectorizedRowBatch rowBatch = new VectorizedRowBatch(3, BATCH_SIZE);
+ rowBatch.cols[0] = inputColumnVector1;
+ rowBatch.cols[1] = inputColumnVector2;
+ rowBatch.cols[2] = outputColumnVector;
+
+ <VectorExpClassName> vectorExpression =
+ new <VectorExpClassName>(0, 1, 2);
+
+ vectorExpression.evaluate(rowBatch);
+
+ assertEquals(
+ "Output column vector repeating state does not match operand columns",
+ (!inputColumnVector1.noNulls && inputColumnVector1.isRepeating)
+ || (!inputColumnVector2.noNulls && inputColumnVector2.isRepeating)
+ || inputColumnVector1.isRepeating && inputColumnVector2.isRepeating,
+ outputColumnVector.isRepeating);
+
+ assertEquals(
+ "Output column vector no nulls state does not match operand columns",
+ inputColumnVector1.noNulls && inputColumnVector2.noNulls,
outputColumnVector.noNulls);
+
+ //if repeating, only the first value matters
+ if(!outputColumnVector.noNulls && !outputColumnVector.isRepeating) {
+ for(int i = 0; i < BATCH_SIZE; i++) {
+ //null vectors are safe to check, as they are always initialized to
match the data vector
+ assertEquals("Output vector doesn't match input vectors' is null state
for index",
+ inputColumnVector1.isNull[i] || inputColumnVector2.isNull[i],
+ outputColumnVector.isNull[i]);
+ }
+ }
+ }
\ No newline at end of file
Added:
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarFilterVectorExpressionEvaluation.txt
URL:
http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarFilterVectorExpressionEvaluation.txt?rev=1485421&view=auto
==============================================================================
---
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarFilterVectorExpressionEvaluation.txt
(added)
+++
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarFilterVectorExpressionEvaluation.txt
Wed May 22 21:04:35 2013
@@ -0,0 +1,76 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ @Test
+ public void <TestName>() {
+
+ Random rand = new Random(SEED);
+
+ <InputColumnVectorType> inputColumnVector =
+
VectorizedRowGroupGenUtil.generate<InputColumnVectorType>(<ColumnHasNulls>,
+ <ColumnIsRepeating>, BATCH_SIZE, rand);
+
+ VectorizedRowBatch rowBatch = new VectorizedRowBatch(1, BATCH_SIZE);
+ rowBatch.cols[0] = inputColumnVector;
+
+ <ScalarType> scalarValue = 0;
+ do {
+ scalarValue = rand.next<CamelCaseScalarType>();
+ } while(scalarValue == 0);
+
+ <VectorExpClassName> vectorExpression =
+ new <VectorExpClassName>(<ConstructorParams>);
+
+ vectorExpression.evaluate(rowBatch);
+
+ int selectedIndex = 0;
+ //check for isRepeating optimization
+ if(inputColumnVector.isRepeating) {
+ //null vector is safe to check, as it is always initialized to match the
data vector
+ selectedIndex =
+ !inputColumnVector.isNull[0] && inputColumnVector.vector[0] <Operator>
scalarValue
+ ? BATCH_SIZE : 0;
+ } else {
+ for(int i = 0; i < BATCH_SIZE; i++) {
+ if(!inputColumnVector.isNull[i]) {
+ if(inputColumnVector.vector[i] <Operator> scalarValue) {
+ assertEquals(
+ "Vector index that passes filter "
+ + inputColumnVector.vector[i] + "<Operator>"
+ + scalarValue + " is not in rowBatch selected index",
+ i,
+ rowBatch.selected[selectedIndex]);
+ selectedIndex++;
+ }
+ }
+ }
+ }
+
+ assertEquals("Row batch size not set to number of selected rows: " +
selectedIndex,
+ selectedIndex, rowBatch.size);
+
+ if(selectedIndex > 0 && selectedIndex < BATCH_SIZE) {
+ assertEquals(
+ "selectedInUse should be set when > 0 and < entire batch(" +
BATCH_SIZE + ") is selected: "
+ + selectedIndex,
+ true, rowBatch.selectedInUse);
+ } else if(selectedIndex == BATCH_SIZE) {
+ assertEquals(
+ "selectedInUse should not be set when entire batch(" + BATCH_SIZE + ")
is selected: "
+ + selectedIndex,
+ false, rowBatch.selectedInUse);
+ }
+ }
\ No newline at end of file
Added:
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarOperationVectorExpressionEvaluation.txt
URL:
http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarOperationVectorExpressionEvaluation.txt?rev=1485421&view=auto
==============================================================================
---
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarOperationVectorExpressionEvaluation.txt
(added)
+++
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarOperationVectorExpressionEvaluation.txt
Wed May 22 21:04:35 2013
@@ -0,0 +1,59 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ @Test
+ public void <TestName>() {
+
+ Random rand = new Random(SEED);
+
+ <OutputColumnVectorType> outputColumnVector =
+
VectorizedRowGroupGenUtil.generate<OutputColumnVectorType>(<InitOuputColHasNulls>,
+ <InitOuputColIsRepeating>, BATCH_SIZE, rand);
+
+ <InputColumnVectorType> inputColumnVector =
+
VectorizedRowGroupGenUtil.generate<InputColumnVectorType>(<ColumnHasNulls>,
+ <ColumnIsRepeating>, BATCH_SIZE, rand);
+
+ VectorizedRowBatch rowBatch = new VectorizedRowBatch(2, BATCH_SIZE);
+ rowBatch.cols[0] = inputColumnVector;
+ rowBatch.cols[1] = outputColumnVector;
+
+ <ScalarType> scalarValue = 0;
+ do {
+ scalarValue = rand.next<CamelCaseScalarType>();
+ } while(scalarValue == 0);
+
+ <VectorExpClassName> vectorExpression =
+ new <VectorExpClassName>(<ConstructorParams>, 1);
+
+ vectorExpression.evaluate(rowBatch);
+
+ assertEquals(
+ "Output column vector is repeating state does not match operand column",
+ inputColumnVector.isRepeating, outputColumnVector.isRepeating);
+
+ assertEquals(
+ "Output column vector no nulls state does not match operand column",
+ inputColumnVector.noNulls, outputColumnVector.noNulls);
+
+ if(!outputColumnVector.noNulls && !outputColumnVector.isRepeating) {
+ for(int i = 0; i < BATCH_SIZE; i++) {
+ //null vectors are safe to check, as they are always initialized to
match the data vector
+ assertEquals("Output vector doesn't match input vector's is null state
for index",
+ inputColumnVector.isNull[i], outputColumnVector.isNull[i]);
+ }
+ }
+ }
\ No newline at end of file
