Modified: hive/branches/tez/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java URL: http://svn.apache.org/viewvc/hive/branches/tez/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java?rev=1622763&r1=1622762&r2=1622763&view=diff ============================================================================== --- hive/branches/tez/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java (original) +++ hive/branches/tez/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java Fri Sep 5 19:15:44 2014 @@ -232,29 +232,98 @@ public class GenVectorCode extends Task {"FilterScalarCompareColumn", "GreaterEqual", "long", "long", ">="}, {"FilterScalarCompareColumn", "GreaterEqual", "double", "long", ">="}, - {"FilterStringColumnCompareScalar", "Equal", "=="}, - {"FilterStringColumnCompareScalar", "NotEqual", "!="}, - {"FilterStringColumnCompareScalar", "Less", "<"}, - {"FilterStringColumnCompareScalar", "LessEqual", "<="}, - {"FilterStringColumnCompareScalar", "Greater", ">"}, - {"FilterStringColumnCompareScalar", "GreaterEqual", ">="}, + {"FilterStringGroupColumnCompareStringGroupScalarBase", "Equal", "=="}, + {"FilterStringGroupColumnCompareStringGroupScalarBase", "NotEqual", "!="}, + {"FilterStringGroupColumnCompareStringGroupScalarBase", "Less", "<"}, + {"FilterStringGroupColumnCompareStringGroupScalarBase", "LessEqual", "<="}, + {"FilterStringGroupColumnCompareStringGroupScalarBase", "Greater", ">"}, + {"FilterStringGroupColumnCompareStringGroupScalarBase", "GreaterEqual", ">="}, + + {"FilterStringGroupColumnCompareStringScalar", "Equal", "=="}, + {"FilterStringGroupColumnCompareStringScalar", "NotEqual", "!="}, + {"FilterStringGroupColumnCompareStringScalar", "Less", "<"}, + {"FilterStringGroupColumnCompareStringScalar", "LessEqual", "<="}, + {"FilterStringGroupColumnCompareStringScalar", "Greater", ">"}, + {"FilterStringGroupColumnCompareStringScalar", "GreaterEqual", ">="}, + + {"FilterStringGroupColumnCompareTruncStringScalar", "VarChar", "Equal", "=="}, + {"FilterStringGroupColumnCompareTruncStringScalar", "VarChar", "NotEqual", "!="}, + {"FilterStringGroupColumnCompareTruncStringScalar", "VarChar", "Less", "<"}, + {"FilterStringGroupColumnCompareTruncStringScalar", "VarChar", "LessEqual", "<="}, + {"FilterStringGroupColumnCompareTruncStringScalar", "VarChar", "Greater", ">"}, + {"FilterStringGroupColumnCompareTruncStringScalar", "VarChar", "GreaterEqual", ">="}, + + {"FilterStringGroupColumnCompareTruncStringScalar", "Char", "Equal", "=="}, + {"FilterStringGroupColumnCompareTruncStringScalar", "Char", "NotEqual", "!="}, + {"FilterStringGroupColumnCompareTruncStringScalar", "Char", "Less", "<"}, + {"FilterStringGroupColumnCompareTruncStringScalar", "Char", "LessEqual", "<="}, + {"FilterStringGroupColumnCompareTruncStringScalar", "Char", "Greater", ">"}, + {"FilterStringGroupColumnCompareTruncStringScalar", "Char", "GreaterEqual", ">="}, {"FilterStringColumnBetween", ""}, {"FilterStringColumnBetween", "!"}, - {"StringColumnCompareScalar", "Equal", "=="}, - {"StringColumnCompareScalar", "NotEqual", "!="}, - {"StringColumnCompareScalar", "Less", "<"}, - {"StringColumnCompareScalar", "LessEqual", "<="}, - {"StringColumnCompareScalar", "Greater", ">"}, - {"StringColumnCompareScalar", "GreaterEqual", ">="}, - - {"FilterStringScalarCompareColumn", "Equal", "=="}, - {"FilterStringScalarCompareColumn", "NotEqual", "!="}, - {"FilterStringScalarCompareColumn", "Less", "<"}, - {"FilterStringScalarCompareColumn", "LessEqual", "<="}, - {"FilterStringScalarCompareColumn", "Greater", ">"}, - {"FilterStringScalarCompareColumn", "GreaterEqual", ">="}, + {"FilterTruncStringColumnBetween", "VarChar", ""}, + {"FilterTruncStringColumnBetween", "VarChar", "!"}, + + {"FilterTruncStringColumnBetween", "Char", ""}, + {"FilterTruncStringColumnBetween", "Char", "!"}, + + {"StringGroupColumnCompareStringGroupScalarBase", "Equal", "=="}, + {"StringGroupColumnCompareStringGroupScalarBase", "NotEqual", "!="}, + {"StringGroupColumnCompareStringGroupScalarBase", "Less", "<"}, + {"StringGroupColumnCompareStringGroupScalarBase", "LessEqual", "<="}, + {"StringGroupColumnCompareStringGroupScalarBase", "Greater", ">"}, + {"StringGroupColumnCompareStringGroupScalarBase", "GreaterEqual", ">="}, + + {"StringGroupColumnCompareStringScalar", "Equal", "=="}, + {"StringGroupColumnCompareStringScalar", "NotEqual", "!="}, + {"StringGroupColumnCompareStringScalar", "Less", "<"}, + {"StringGroupColumnCompareStringScalar", "LessEqual", "<="}, + {"StringGroupColumnCompareStringScalar", "Greater", ">"}, + {"StringGroupColumnCompareStringScalar", "GreaterEqual", ">="}, + + {"StringGroupColumnCompareTruncStringScalar", "VarChar", "Equal", "=="}, + {"StringGroupColumnCompareTruncStringScalar", "VarChar", "NotEqual", "!="}, + {"StringGroupColumnCompareTruncStringScalar", "VarChar", "Less", "<"}, + {"StringGroupColumnCompareTruncStringScalar", "VarChar", "LessEqual", "<="}, + {"StringGroupColumnCompareTruncStringScalar", "VarChar", "Greater", ">"}, + {"StringGroupColumnCompareTruncStringScalar", "VarChar", "GreaterEqual", ">="}, + + {"StringGroupColumnCompareTruncStringScalar", "Char", "Equal", "=="}, + {"StringGroupColumnCompareTruncStringScalar", "Char", "NotEqual", "!="}, + {"StringGroupColumnCompareTruncStringScalar", "Char", "Less", "<"}, + {"StringGroupColumnCompareTruncStringScalar", "Char", "LessEqual", "<="}, + {"StringGroupColumnCompareTruncStringScalar", "Char", "Greater", ">"}, + {"StringGroupColumnCompareTruncStringScalar", "Char", "GreaterEqual", ">="}, + + {"FilterStringGroupScalarCompareStringGroupColumnBase", "Equal", "=="}, + {"FilterStringGroupScalarCompareStringGroupColumnBase", "NotEqual", "!="}, + {"FilterStringGroupScalarCompareStringGroupColumnBase", "Less", "<"}, + {"FilterStringGroupScalarCompareStringGroupColumnBase", "LessEqual", "<="}, + {"FilterStringGroupScalarCompareStringGroupColumnBase", "Greater", ">"}, + {"FilterStringGroupScalarCompareStringGroupColumnBase", "GreaterEqual", ">="}, + + {"FilterStringScalarCompareStringGroupColumn", "Equal", "=="}, + {"FilterStringScalarCompareStringGroupColumn", "NotEqual", "!="}, + {"FilterStringScalarCompareStringGroupColumn", "Less", "<"}, + {"FilterStringScalarCompareStringGroupColumn", "LessEqual", "<="}, + {"FilterStringScalarCompareStringGroupColumn", "Greater", ">"}, + {"FilterStringScalarCompareStringGroupColumn", "GreaterEqual", ">="}, + + {"FilterTruncStringScalarCompareStringGroupColumn", "VarChar", "Equal", "=="}, + {"FilterTruncStringScalarCompareStringGroupColumn", "VarChar", "NotEqual", "!="}, + {"FilterTruncStringScalarCompareStringGroupColumn", "VarChar", "Less", "<"}, + {"FilterTruncStringScalarCompareStringGroupColumn", "VarChar", "LessEqual", "<="}, + {"FilterTruncStringScalarCompareStringGroupColumn", "VarChar", "Greater", ">"}, + {"FilterTruncStringScalarCompareStringGroupColumn", "VarChar", "GreaterEqual", ">="}, + + {"FilterTruncStringScalarCompareStringGroupColumn", "Char", "Equal", "=="}, + {"FilterTruncStringScalarCompareStringGroupColumn", "Char", "NotEqual", "!="}, + {"FilterTruncStringScalarCompareStringGroupColumn", "Char", "Less", "<"}, + {"FilterTruncStringScalarCompareStringGroupColumn", "Char", "LessEqual", "<="}, + {"FilterTruncStringScalarCompareStringGroupColumn", "Char", "Greater", ">"}, + {"FilterTruncStringScalarCompareStringGroupColumn", "Char", "GreaterEqual", ">="}, {"FilterDecimalColumnCompareScalar", "Equal", "=="}, {"FilterDecimalColumnCompareScalar", "NotEqual", "!="}, @@ -277,26 +346,47 @@ public class GenVectorCode extends Task {"FilterDecimalColumnCompareColumn", "Greater", ">"}, {"FilterDecimalColumnCompareColumn", "GreaterEqual", ">="}, - {"StringScalarCompareColumn", "Equal", "=="}, - {"StringScalarCompareColumn", "NotEqual", "!="}, - {"StringScalarCompareColumn", "Less", "<"}, - {"StringScalarCompareColumn", "LessEqual", "<="}, - {"StringScalarCompareColumn", "Greater", ">"}, - {"StringScalarCompareColumn", "GreaterEqual", ">="}, - - {"FilterStringColumnCompareColumn", "Equal", "=="}, - {"FilterStringColumnCompareColumn", "NotEqual", "!="}, - {"FilterStringColumnCompareColumn", "Less", "<"}, - {"FilterStringColumnCompareColumn", "LessEqual", "<="}, - {"FilterStringColumnCompareColumn", "Greater", ">"}, - {"FilterStringColumnCompareColumn", "GreaterEqual", ">="}, - - {"StringColumnCompareColumn", "Equal", "=="}, - {"StringColumnCompareColumn", "NotEqual", "!="}, - {"StringColumnCompareColumn", "Less", "<"}, - {"StringColumnCompareColumn", "LessEqual", "<="}, - {"StringColumnCompareColumn", "Greater", ">"}, - {"StringColumnCompareColumn", "GreaterEqual", ">="}, + {"StringGroupScalarCompareStringGroupColumnBase", "Equal", "=="}, + {"StringGroupScalarCompareStringGroupColumnBase", "NotEqual", "!="}, + {"StringGroupScalarCompareStringGroupColumnBase", "Less", "<"}, + {"StringGroupScalarCompareStringGroupColumnBase", "LessEqual", "<="}, + {"StringGroupScalarCompareStringGroupColumnBase", "Greater", ">"}, + {"StringGroupScalarCompareStringGroupColumnBase", "GreaterEqual", ">="}, + + {"StringScalarCompareStringGroupColumn", "Equal", "=="}, + {"StringScalarCompareStringGroupColumn", "NotEqual", "!="}, + {"StringScalarCompareStringGroupColumn", "Less", "<"}, + {"StringScalarCompareStringGroupColumn", "LessEqual", "<="}, + {"StringScalarCompareStringGroupColumn", "Greater", ">"}, + {"StringScalarCompareStringGroupColumn", "GreaterEqual", ">="}, + + {"TruncStringScalarCompareStringGroupColumn", "VarChar", "Equal", "=="}, + {"TruncStringScalarCompareStringGroupColumn", "VarChar", "NotEqual", "!="}, + {"TruncStringScalarCompareStringGroupColumn", "VarChar", "Less", "<"}, + {"TruncStringScalarCompareStringGroupColumn", "VarChar", "LessEqual", "<="}, + {"TruncStringScalarCompareStringGroupColumn", "VarChar", "Greater", ">"}, + {"TruncStringScalarCompareStringGroupColumn", "VarChar", "GreaterEqual", ">="}, + + {"TruncStringScalarCompareStringGroupColumn", "Char", "Equal", "=="}, + {"TruncStringScalarCompareStringGroupColumn", "Char", "NotEqual", "!="}, + {"TruncStringScalarCompareStringGroupColumn", "Char", "Less", "<"}, + {"TruncStringScalarCompareStringGroupColumn", "Char", "LessEqual", "<="}, + {"TruncStringScalarCompareStringGroupColumn", "Char", "Greater", ">"}, + {"TruncStringScalarCompareStringGroupColumn", "Char", "GreaterEqual", ">="}, + + {"FilterStringGroupColumnCompareStringGroupColumn", "Equal", "=="}, + {"FilterStringGroupColumnCompareStringGroupColumn", "NotEqual", "!="}, + {"FilterStringGroupColumnCompareStringGroupColumn", "Less", "<"}, + {"FilterStringGroupColumnCompareStringGroupColumn", "LessEqual", "<="}, + {"FilterStringGroupColumnCompareStringGroupColumn", "Greater", ">"}, + {"FilterStringGroupColumnCompareStringGroupColumn", "GreaterEqual", ">="}, + + {"StringGroupColumnCompareStringGroupColumn", "Equal", "=="}, + {"StringGroupColumnCompareStringGroupColumn", "NotEqual", "!="}, + {"StringGroupColumnCompareStringGroupColumn", "Less", "<"}, + {"StringGroupColumnCompareStringGroupColumn", "LessEqual", "<="}, + {"StringGroupColumnCompareStringGroupColumn", "Greater", ">"}, + {"StringGroupColumnCompareStringGroupColumn", "GreaterEqual", ">="}, {"FilterColumnCompareColumn", "Equal", "long", "double", "=="}, {"FilterColumnCompareColumn", "Equal", "double", "double", "=="}, @@ -359,58 +449,58 @@ public class GenVectorCode extends Task {"ColumnCompareColumn", "GreaterEqual", "double", "long", ">="}, // template, <ClassNamePrefix>, <ReturnType>, <OperandType>, <FuncName>, <OperandCast>, - // <ResultCast>, <Cleanup> - {"ColumnUnaryFunc", "FuncRound", "double", "double", "MathExpr.round", "", "", ""}, + // <ResultCast>, <Cleanup> <VectorExprArgType> + {"ColumnUnaryFunc", "FuncRound", "double", "double", "MathExpr.round", "", "", "", ""}, // round(longCol) returns a long and is a no-op. So it will not be implemented here. // round(Col, N) is a special case and will be implemented separately from this template - {"ColumnUnaryFunc", "FuncFloor", "long", "double", "Math.floor", "", "(long)", ""}, + {"ColumnUnaryFunc", "FuncFloor", "long", "double", "Math.floor", "", "(long)", "", ""}, // Floor on an integer argument is a noop, but it is less code to handle it this way. - {"ColumnUnaryFunc", "FuncFloor", "long", "long", "Math.floor", "", "(long)", ""}, - {"ColumnUnaryFunc", "FuncCeil", "long", "double", "Math.ceil", "", "(long)", ""}, + {"ColumnUnaryFunc", "FuncFloor", "long", "long", "Math.floor", "", "(long)", "", ""}, + {"ColumnUnaryFunc", "FuncCeil", "long", "double", "Math.ceil", "", "(long)", "", ""}, // Ceil on an integer argument is a noop, but it is less code to handle it this way. - {"ColumnUnaryFunc", "FuncCeil", "long", "long", "Math.ceil", "", "(long)", ""}, - {"ColumnUnaryFunc", "FuncExp", "double", "double", "Math.exp", "", "", ""}, - {"ColumnUnaryFunc", "FuncExp", "double", "long", "Math.exp", "(double)", "", ""}, + {"ColumnUnaryFunc", "FuncCeil", "long", "long", "Math.ceil", "", "(long)", "", ""}, + {"ColumnUnaryFunc", "FuncExp", "double", "double", "Math.exp", "", "", "", ""}, + {"ColumnUnaryFunc", "FuncExp", "double", "long", "Math.exp", "(double)", "", "", ""}, {"ColumnUnaryFunc", "FuncLn", "double", "double", "Math.log", "", "", - "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);"}, + "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);", ""}, {"ColumnUnaryFunc", "FuncLn", "double", "long", "Math.log", "(double)", "", - "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);"}, + "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);", ""}, {"ColumnUnaryFunc", "FuncLog10", "double", "double", "Math.log10", "", "", - "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);"}, + "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);", ""}, {"ColumnUnaryFunc", "FuncLog10", "double", "long", "Math.log10", "(double)", "", - "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);"}, + "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);", ""}, // The MathExpr class contains helper functions for cases when existing library // routines can't be used directly. {"ColumnUnaryFunc", "FuncLog2", "double", "double", "MathExpr.log2", "", "", - "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);"}, + "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);", ""}, {"ColumnUnaryFunc", "FuncLog2", "double", "long", "MathExpr.log2", "(double)", "", - "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);"}, + "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);", ""}, // Log(base, Col) is a special case and will be implemented separately from this template // Pow(col, P) and Power(col, P) are special cases implemented separately from this template {"ColumnUnaryFunc", "FuncSqrt", "double", "double", "Math.sqrt", "", "", - "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n);"}, + "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n);", ""}, {"ColumnUnaryFunc", "FuncSqrt", "double", "long", "Math.sqrt", "(double)", "", - "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n);"}, - {"ColumnUnaryFunc", "FuncAbs", "double", "double", "Math.abs", "", "", ""}, - {"ColumnUnaryFunc", "FuncAbs", "long", "long", "MathExpr.abs", "", "", ""}, - {"ColumnUnaryFunc", "FuncSin", "double", "double", "Math.sin", "", "", ""}, - {"ColumnUnaryFunc", "FuncSin", "double", "long", "Math.sin", "(double)", "", ""}, - {"ColumnUnaryFunc", "FuncASin", "double", "double", "Math.asin", "", "", ""}, - {"ColumnUnaryFunc", "FuncASin", "double", "long", "Math.asin", "(double)", "", ""}, - {"ColumnUnaryFunc", "FuncCos", "double", "double", "Math.cos", "", "", ""}, - {"ColumnUnaryFunc", "FuncCos", "double", "long", "Math.cos", "(double)", "", ""}, - {"ColumnUnaryFunc", "FuncACos", "double", "double", "Math.acos", "", "", ""}, - {"ColumnUnaryFunc", "FuncACos", "double", "long", "Math.acos", "(double)", "", ""}, - {"ColumnUnaryFunc", "FuncTan", "double", "double", "Math.tan", "", "", ""}, - {"ColumnUnaryFunc", "FuncTan", "double", "long", "Math.tan", "(double)", "", ""}, - {"ColumnUnaryFunc", "FuncATan", "double", "double", "Math.atan", "", "", ""}, - {"ColumnUnaryFunc", "FuncATan", "double", "long", "Math.atan", "(double)", "", ""}, - {"ColumnUnaryFunc", "FuncDegrees", "double", "double", "Math.toDegrees", "", "", ""}, - {"ColumnUnaryFunc", "FuncDegrees", "double", "long", "Math.toDegrees", "(double)", "", ""}, - {"ColumnUnaryFunc", "FuncRadians", "double", "double", "Math.toRadians", "", "", ""}, - {"ColumnUnaryFunc", "FuncRadians", "double", "long", "Math.toRadians", "(double)", "", ""}, - {"ColumnUnaryFunc", "FuncSign", "double", "double", "MathExpr.sign", "", "", ""}, - {"ColumnUnaryFunc", "FuncSign", "double", "long", "MathExpr.sign", "(double)", "", ""}, + "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n);", ""}, + {"ColumnUnaryFunc", "FuncAbs", "double", "double", "Math.abs", "", "", "", ""}, + {"ColumnUnaryFunc", "FuncAbs", "long", "long", "MathExpr.abs", "", "", "", ""}, + {"ColumnUnaryFunc", "FuncSin", "double", "double", "Math.sin", "", "", "", ""}, + {"ColumnUnaryFunc", "FuncSin", "double", "long", "Math.sin", "(double)", "", "", ""}, + {"ColumnUnaryFunc", "FuncASin", "double", "double", "Math.asin", "", "", "", ""}, + {"ColumnUnaryFunc", "FuncASin", "double", "long", "Math.asin", "(double)", "", "", ""}, + {"ColumnUnaryFunc", "FuncCos", "double", "double", "Math.cos", "", "", "", ""}, + {"ColumnUnaryFunc", "FuncCos", "double", "long", "Math.cos", "(double)", "", "", ""}, + {"ColumnUnaryFunc", "FuncACos", "double", "double", "Math.acos", "", "", "", ""}, + {"ColumnUnaryFunc", "FuncACos", "double", "long", "Math.acos", "(double)", "", "", ""}, + {"ColumnUnaryFunc", "FuncTan", "double", "double", "Math.tan", "", "", "", ""}, + {"ColumnUnaryFunc", "FuncTan", "double", "long", "Math.tan", "(double)", "", "", ""}, + {"ColumnUnaryFunc", "FuncATan", "double", "double", "Math.atan", "", "", "", ""}, + {"ColumnUnaryFunc", "FuncATan", "double", "long", "Math.atan", "(double)", "", "", ""}, + {"ColumnUnaryFunc", "FuncDegrees", "double", "double", "Math.toDegrees", "", "", "", ""}, + {"ColumnUnaryFunc", "FuncDegrees", "double", "long", "Math.toDegrees", "(double)", "", "", ""}, + {"ColumnUnaryFunc", "FuncRadians", "double", "double", "Math.toRadians", "", "", "", ""}, + {"ColumnUnaryFunc", "FuncRadians", "double", "long", "Math.toRadians", "(double)", "", "", ""}, + {"ColumnUnaryFunc", "FuncSign", "double", "double", "MathExpr.sign", "", "", "", ""}, + {"ColumnUnaryFunc", "FuncSign", "double", "long", "MathExpr.sign", "(double)", "", "", ""}, {"DecimalColumnUnaryFunc", "FuncFloor", "decimal", "DecimalUtil.floor"}, {"DecimalColumnUnaryFunc", "FuncCeil", "decimal", "DecimalUtil.ceiling"}, @@ -420,20 +510,24 @@ public class GenVectorCode extends Task {"DecimalColumnUnaryFunc", "FuncNegate", "decimal", "DecimalUtil.negate"}, // Casts - {"ColumnUnaryFunc", "Cast", "long", "double", "", "", "(long)", ""}, - {"ColumnUnaryFunc", "Cast", "double", "long", "", "", "(double)", ""}, + {"ColumnUnaryFunc", "Cast", "long", "double", "", "", "(long)", "", ""}, + {"ColumnUnaryFunc", "Cast", "double", "long", "", "", "(double)", "", ""}, {"ColumnUnaryFunc", "CastTimestampToLongVia", "long", "long", "MathExpr.fromTimestamp", "", - "", ""}, + "", "", "timestamp"}, {"ColumnUnaryFunc", "CastTimestampToDoubleVia", "double", "long", - "MathExpr.fromTimestampToDouble", "", "", ""}, + "MathExpr.fromTimestampToDouble", "", "", "", "timestamp"}, {"ColumnUnaryFunc", "CastDoubleToBooleanVia", "long", "double", "MathExpr.toBool", "", - "", ""}, + "", "", ""}, {"ColumnUnaryFunc", "CastLongToBooleanVia", "long", "long", "MathExpr.toBool", "", - "", ""}, + "", "", ""}, + {"ColumnUnaryFunc", "CastDateToBooleanVia", "long", "long", "MathExpr.toBool", "", + "", "", "date"}, + {"ColumnUnaryFunc", "CastTimestampToBooleanVia", "long", "long", "MathExpr.toBool", "", + "", "", "timestamp"}, {"ColumnUnaryFunc", "CastLongToTimestampVia", "long", "long", "MathExpr.longToTimestamp", "", - "", ""}, + "", "", ""}, {"ColumnUnaryFunc", "CastDoubleToTimestampVia", "long", "double", - "MathExpr.doubleToTimestamp", "", "", ""}, + "MathExpr.doubleToTimestamp", "", "", "", ""}, // Boolean to long is done with an IdentityExpression // Boolean to double is done with standard Long to Double cast @@ -658,22 +752,40 @@ public class GenVectorCode extends Task generateVectorUDAFVar(tdesc); } else if (tdesc[0].equals("VectorUDAFVarDecimal")) { generateVectorUDAFVarDecimal(tdesc); - } else if (tdesc[0].equals("FilterStringColumnCompareScalar")) { - generateFilterStringColumnCompareScalar(tdesc); + } else if (tdesc[0].equals("FilterStringGroupColumnCompareStringGroupScalarBase")) { + generateFilterStringGroupColumnCompareStringGroupScalarBase(tdesc); + } else if (tdesc[0].equals("FilterStringGroupColumnCompareStringScalar")) { + generateFilterStringGroupColumnCompareStringScalar(tdesc); + } else if (tdesc[0].equals("FilterStringGroupColumnCompareTruncStringScalar")) { + generateFilterStringGroupColumnCompareTruncStringScalar(tdesc); } else if (tdesc[0].equals("FilterStringColumnBetween")) { generateFilterStringColumnBetween(tdesc); + } else if (tdesc[0].equals("FilterTruncStringColumnBetween")) { + generateFilterTruncStringColumnBetween(tdesc); } else if (tdesc[0].equals("FilterDecimalColumnBetween")) { generateFilterDecimalColumnBetween(tdesc); - } else if (tdesc[0].equals("StringColumnCompareScalar")) { - generateStringColumnCompareScalar(tdesc); - } else if (tdesc[0].equals("FilterStringScalarCompareColumn")) { - generateFilterStringScalarCompareColumn(tdesc); - } else if (tdesc[0].equals("StringScalarCompareColumn")) { - generateStringScalarCompareColumn(tdesc); - } else if (tdesc[0].equals("FilterStringColumnCompareColumn")) { - generateFilterStringColumnCompareColumn(tdesc); - } else if (tdesc[0].equals("StringColumnCompareColumn")) { - generateStringColumnCompareColumn(tdesc); + } else if (tdesc[0].equals("StringGroupColumnCompareStringGroupScalarBase")) { + generateStringGroupColumnCompareStringGroupScalarBase(tdesc); + } else if (tdesc[0].equals("StringGroupColumnCompareStringScalar")) { + generateStringGroupColumnCompareStringScalar(tdesc); + } else if (tdesc[0].equals("StringGroupColumnCompareTruncStringScalar")) { + generateStringGroupColumnCompareTruncStringScalar(tdesc); + } else if (tdesc[0].equals("FilterStringGroupScalarCompareStringGroupColumnBase")) { + generateFilterStringGroupScalarCompareStringGroupColumnBase(tdesc); + } else if (tdesc[0].equals("FilterStringScalarCompareStringGroupColumn")) { + generateFilterStringScalarCompareStringGroupColumn(tdesc); + } else if (tdesc[0].equals("FilterTruncStringScalarCompareStringGroupColumn")) { + generateFilterTruncStringScalarCompareStringGroupColumn(tdesc); + } else if (tdesc[0].equals("StringGroupScalarCompareStringGroupColumnBase")) { + generateStringGroupScalarCompareStringGroupColumnBase(tdesc); + } else if (tdesc[0].equals("StringScalarCompareStringGroupColumn")) { + generateStringScalarCompareStringGroupColumn(tdesc); + } else if (tdesc[0].equals("TruncStringScalarCompareStringGroupColumn")) { + generateTruncStringScalarCompareStringGroupColumn(tdesc); + } else if (tdesc[0].equals("FilterStringGroupColumnCompareStringGroupColumn")) { + generateFilterStringGroupColumnCompareStringGroupColumn(tdesc); + } else if (tdesc[0].equals("StringGroupColumnCompareStringGroupColumn")) { + generateStringGroupColumnCompareStringGroupColumn(tdesc); } else if (tdesc[0].equals("IfExprColumnColumn")) { generateIfExprColumnColumn(tdesc); } else if (tdesc[0].equals("IfExprColumnScalar")) { @@ -710,6 +822,35 @@ public class GenVectorCode extends Task className, templateString); } + private void generateFilterTruncStringColumnBetween(String[] tdesc) throws IOException { + String truncStringTypeName = tdesc[1]; + String truncStringHiveType; + String truncStringHiveGetBytes; + if (truncStringTypeName == "Char") { + truncStringHiveType = "HiveChar"; + truncStringHiveGetBytes = "getStrippedValue().getBytes()"; + } else if (truncStringTypeName == "VarChar") { + truncStringHiveType = "HiveVarchar"; + truncStringHiveGetBytes = "getValue().getBytes()"; + } else { + throw new Error("Unsupported string type: " + truncStringTypeName); + } + String optionalNot = tdesc[2]; + String className = "Filter" + truncStringTypeName + "Column" + (optionalNot.equals("!") ? "Not" : "") + + "Between"; + // Read the template into a string, expand it, and write it. + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("<TruncStringTypeName>", truncStringTypeName); + templateString = templateString.replaceAll("<TruncStringHiveType>", truncStringHiveType); + templateString = templateString.replaceAll("<TruncStringHiveGetBytes>", truncStringHiveGetBytes); + templateString = templateString.replaceAll("<ClassName>", className); + templateString = templateString.replaceAll("<OptionalNot>", optionalNot); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + private void generateFilterDecimalColumnBetween(String[] tdesc) throws IOException { String optionalNot = tdesc[1]; String className = "FilterDecimalColumn" + (optionalNot.equals("!") ? "Not" : "") @@ -745,14 +886,12 @@ public class GenVectorCode extends Task } private void generateColumnCompareColumn(String[] tdesc) throws Exception { - //The variables are all same as ColumnCompareScalar except that - //this template doesn't need a return type. Pass anything as return type. String operatorName = tdesc[1]; String operandType1 = tdesc[2]; String operandType2 = tdesc[3]; String className = getCamelCaseType(operandType1) + "Col" + operatorName + getCamelCaseType(operandType2) + "Column"; - generateColumnBinaryOperatorColumn(tdesc, "long", className); + generateColumnCompareOperatorColumn(tdesc, false, className); } private void generateVectorUDAFMinMax(String[] tdesc) throws Exception { @@ -886,45 +1025,147 @@ public class GenVectorCode extends Task writeFile(templateFile.lastModified(), udafOutputDirectory, udafClassesDirectory, className, templateString); } - - - private void generateFilterStringScalarCompareColumn(String[] tdesc) throws IOException { + + private void generateFilterStringGroupScalarCompareStringGroupColumnBase(String[] tdesc) throws IOException { String operatorName = tdesc[1]; - String className = "FilterStringScalar" + operatorName + "StringColumn"; + String className = "FilterStringGroupScalar" + operatorName + "StringGroupColumnBase"; // Template expansion logic is the same for both column-scalar and scalar-column cases. generateStringColumnCompareScalar(tdesc, className); } - private void generateStringScalarCompareColumn(String[] tdesc) throws IOException { + private void generateFilterStringScalarCompareStringGroupColumn(String[] tdesc) throws IOException { + String operatorName = tdesc[1]; + String className = "FilterStringScalar" + operatorName + "StringGroupColumn"; + String baseClassName = "FilterStringGroupScalar" + operatorName + "StringGroupColumnBase"; + String operatorSymbol = tdesc[2]; + // Read the template into a string; + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + // Expand, and write result + templateString = templateString.replaceAll("<ClassName>", className); + templateString = templateString.replaceAll("<BaseClassName>", baseClassName); + templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol); + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + + private void generateFilterTruncStringScalarCompareStringGroupColumn(String[] tdesc) throws IOException { + String truncStringTypeName = tdesc[1]; + String operatorName = tdesc[2]; + String className = "Filter" + truncStringTypeName + "Scalar" + operatorName + "StringGroupColumn"; + String baseClassName = "FilterStringGroupScalar" + operatorName + "StringGroupColumnBase"; + generateStringCompareTruncStringScalar(tdesc, className, baseClassName); + } + + private void generateStringGroupScalarCompareStringGroupColumnBase(String[] tdesc) throws IOException { String operatorName = tdesc[1]; - String className = "StringScalar" + operatorName + "StringColumn"; + String className = "StringGroupScalar" + operatorName + "StringGroupColumnBase"; // Template expansion logic is the same for both column-scalar and scalar-column cases. generateStringColumnCompareScalar(tdesc, className); } - private void generateFilterStringColumnCompareScalar(String[] tdesc) throws IOException { + private void generateStringScalarCompareStringGroupColumn(String[] tdesc) throws IOException { String operatorName = tdesc[1]; - String className = "FilterStringCol" + operatorName + "StringScalar"; - generateStringColumnCompareScalar(tdesc, className); + String className = "StringScalar" + operatorName + "StringGroupColumn"; + String baseClassName = "StringGroupScalar" + operatorName + "StringGroupColumnBase"; + String operatorSymbol = tdesc[2]; + // Read the template into a string; + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + // Expand, and write result + templateString = templateString.replaceAll("<ClassName>", className); + templateString = templateString.replaceAll("<BaseClassName>", baseClassName); + templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol); + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); } + + private void generateTruncStringScalarCompareStringGroupColumn(String[] tdesc) throws IOException { + String truncStringTypeName = tdesc[1]; + String operatorName = tdesc[2]; + String className = truncStringTypeName + "Scalar" + operatorName + "StringGroupColumn"; + String baseClassName = "StringGroupScalar" + operatorName + "StringGroupColumnBase"; + generateStringCompareTruncStringScalar(tdesc, className, baseClassName); } - private void generateStringColumnCompareScalar(String[] tdesc) throws IOException { + private void generateFilterStringGroupColumnCompareStringGroupScalarBase(String[] tdesc) throws IOException { String operatorName = tdesc[1]; - String className = "StringCol" + operatorName + "StringScalar"; + String className = "FilterStringGroupCol" + operatorName + "StringGroupScalarBase"; generateStringColumnCompareScalar(tdesc, className); } - private void generateFilterStringColumnCompareColumn(String[] tdesc) throws IOException { + private void generateFilterStringGroupColumnCompareStringScalar(String[] tdesc) throws IOException { + String operatorName = tdesc[1]; + String className = "FilterStringGroupCol" + operatorName + "StringScalar"; + String baseClassName = "FilterStringGroupCol" + operatorName + "StringGroupScalarBase"; + String operatorSymbol = tdesc[2]; + // Read the template into a string; + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + // Expand, and write result + templateString = templateString.replaceAll("<ClassName>", className); + templateString = templateString.replaceAll("<BaseClassName>", baseClassName); + templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol); + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + + private void generateFilterStringGroupColumnCompareTruncStringScalar(String[] tdesc) throws IOException { + String truncStringTypeName = tdesc[1]; + String operatorName = tdesc[2]; + String className = "FilterStringGroupCol" + operatorName + truncStringTypeName + "Scalar"; + String baseClassName = "FilterStringGroupCol" + operatorName + "StringGroupScalarBase"; + generateStringCompareTruncStringScalar(tdesc, className, baseClassName); + } + + private void generateStringGroupColumnCompareStringGroupScalarBase(String[] tdesc) throws IOException { + String operatorName = tdesc[1]; + String className = "StringGroupCol" + operatorName + "StringGroupScalarBase"; + String operatorSymbol = tdesc[2]; + // Read the template into a string; + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + // Expand, and write result + templateString = templateString.replaceAll("<ClassName>", className); + templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol); + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + + private void generateStringGroupColumnCompareStringScalar(String[] tdesc) throws IOException { + String operatorName = tdesc[1]; + String className = "StringGroupCol" + operatorName + "StringScalar"; + String baseClassName = "StringGroupCol" + operatorName + "StringGroupScalarBase"; + String operatorSymbol = tdesc[2]; + // Read the template into a string; + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + // Expand, and write result + templateString = templateString.replaceAll("<ClassName>", className); + templateString = templateString.replaceAll("<BaseClassName>", baseClassName); + templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol); + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + + private void generateStringGroupColumnCompareTruncStringScalar(String[] tdesc) throws IOException { + String truncStringTypeName = tdesc[1]; + String operatorName = tdesc[2]; + String className = "StringGroupCol" + operatorName + truncStringTypeName + "Scalar"; + String baseClassName = "StringGroupCol" + operatorName + "StringGroupScalarBase"; + generateStringCompareTruncStringScalar(tdesc, className, baseClassName); + } + + private void generateFilterStringGroupColumnCompareStringGroupColumn(String[] tdesc) throws IOException { String operatorName = tdesc[1]; - String className = "FilterStringCol" + operatorName + "StringColumn"; + String className = "FilterStringGroupCol" + operatorName + "StringGroupColumn"; generateStringColumnCompareScalar(tdesc, className); } - private void generateStringColumnCompareColumn(String[] tdesc) throws IOException { + private void generateStringGroupColumnCompareStringGroupColumn(String[] tdesc) throws IOException { String operatorName = tdesc[1]; - String className = "StringCol" + operatorName + "StringColumn"; + String className = "StringGroupCol" + operatorName + "StringGroupColumn"; generateStringColumnCompareScalar(tdesc, className); } @@ -941,15 +1182,42 @@ public class GenVectorCode extends Task className, templateString); } + private void generateStringCompareTruncStringScalar(String[] tdesc, String className, String baseClassName) + throws IOException { + String truncStringTypeName = tdesc[1]; + String truncStringHiveType; + String truncStringHiveGetBytes; + if (truncStringTypeName == "Char") { + truncStringHiveType = "HiveChar"; + truncStringHiveGetBytes = "getStrippedValue().getBytes()"; + } else if (truncStringTypeName == "VarChar") { + truncStringHiveType = "HiveVarchar"; + truncStringHiveGetBytes = "getValue().getBytes()"; + } else { + throw new Error("Unsupported string type: " + truncStringTypeName); + } + String operatorSymbol = tdesc[3]; + // Read the template into a string; + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + // Expand, and write result + templateString = templateString.replaceAll("<ClassName>", className); + templateString = templateString.replaceAll("<BaseClassName>", baseClassName); + templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol); + templateString = templateString.replaceAll("<TruncStringTypeName>", truncStringTypeName); + templateString = templateString.replaceAll("<TruncStringHiveType>", truncStringHiveType); + templateString = templateString.replaceAll("<TruncStringHiveGetBytes>", truncStringHiveGetBytes); + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + private void generateFilterColumnCompareColumn(String[] tdesc) throws Exception { - //The variables are all same as ColumnCompareScalar except that - //this template doesn't need a return type. Pass anything as return type. String operatorName = tdesc[1]; String operandType1 = tdesc[2]; String operandType2 = tdesc[3]; String className = "Filter" + getCamelCaseType(operandType1) + "Col" + operatorName + getCamelCaseType(operandType2) + "Column"; - generateColumnBinaryOperatorColumn(tdesc, null, className); + generateColumnCompareOperatorColumn(tdesc, true, className); } private void generateColumnUnaryMinus(String[] tdesc) throws Exception { @@ -984,6 +1252,13 @@ public class GenVectorCode extends Task templateString = templateString.replaceAll("<ClassName>", className); templateString = templateString.replaceAll("<InputColumnVectorType>", inputColumnVectorType); templateString = templateString.replaceAll("<OperandType>", operandType); + String vectorExprArgType = operandType; + if (operandType.equals("long")) { + // Let comparisons occur for DATE and TIMESTAMP, too. + vectorExprArgType = "int_datetime_family"; + } + templateString = templateString.replaceAll("<VectorExprArgType>", vectorExprArgType); + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, className, templateString); } @@ -1006,6 +1281,15 @@ public class GenVectorCode extends Task templateString = templateString.replaceAll("<OperandType2>", operandType2); templateString = templateString.replaceAll("<OperandType3>", operandType3); templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType); + String vectorExprArgType2 = operandType2; + String vectorExprArgType3 = operandType3; + if (operandType2.equals("long") && operandType3.equals("long")) { + // Let this work occur for DATE and TIMESTAMP, too. + vectorExprArgType2 = "int_datetime_family"; + vectorExprArgType3 = "int_datetime_family"; + } + templateString = templateString.replaceAll("<VectorExprArgType2>", vectorExprArgType2); + templateString = templateString.replaceAll("<VectorExprArgType3>", vectorExprArgType3); writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, className, templateString); } @@ -1028,6 +1312,15 @@ public class GenVectorCode extends Task templateString = templateString.replaceAll("<OperandType2>", operandType2); templateString = templateString.replaceAll("<OperandType3>", operandType3); templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType); + String vectorExprArgType2 = operandType2; + String vectorExprArgType3 = operandType3; + if (operandType2.equals("long") && operandType3.equals("long")) { + // Let this work occur for DATE and TIMESTAMP, too. + vectorExprArgType2 = "int_datetime_family"; + vectorExprArgType3 = "int_datetime_family"; + } + templateString = templateString.replaceAll("<VectorExprArgType2>", vectorExprArgType2); + templateString = templateString.replaceAll("<VectorExprArgType3>", vectorExprArgType3); writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, className, templateString); } @@ -1049,6 +1342,15 @@ public class GenVectorCode extends Task templateString = templateString.replaceAll("<OperandType2>", operandType2); templateString = templateString.replaceAll("<OperandType3>", operandType3); templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType); + String vectorExprArgType2 = operandType2; + String vectorExprArgType3 = operandType3; + if (operandType2.equals("long") && operandType3.equals("long")) { + // Let this work occur for DATE and TIMESTAMP, too. + vectorExprArgType2 = "int_datetime_family"; + vectorExprArgType3 = "int_datetime_family"; + } + templateString = templateString.replaceAll("<VectorExprArgType2>", vectorExprArgType2); + templateString = templateString.replaceAll("<VectorExprArgType3>", vectorExprArgType3); writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, className, templateString); } @@ -1073,6 +1375,7 @@ public class GenVectorCode extends Task } // template, <ClassNamePrefix>, <ReturnType>, <OperandType>, <FuncName>, <OperandCast>, <ResultCast> + // <Cleanup> <VectorExprArgType> private void generateColumnUnaryFunc(String[] tdesc) throws Exception { String classNamePrefix = tdesc[1]; String operandType = tdesc[3]; @@ -1087,6 +1390,8 @@ public class GenVectorCode extends Task String operandCast = tdesc[5]; String resultCast = tdesc[6]; String cleanup = tdesc[7]; + String vectorExprArgType = tdesc[8].isEmpty() ? operandType : tdesc[8]; + // Expand, and write result templateString = templateString.replaceAll("<ClassName>", className); templateString = templateString.replaceAll("<InputColumnVectorType>", inputColumnVectorType); @@ -1097,6 +1402,7 @@ public class GenVectorCode extends Task templateString = templateString.replaceAll("<OperandCast>", operandCast); templateString = templateString.replaceAll("<ResultCast>", resultCast); templateString = templateString.replaceAll("<Cleanup>", cleanup); + templateString = templateString.replaceAll("<VectorExprArgType>", vectorExprArgType); writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, className, templateString); } @@ -1108,58 +1414,53 @@ public class GenVectorCode extends Task String className = getCamelCaseType(operandType1) + "Col" + operatorName + getCamelCaseType(operandType2) + "Column"; String returnType = getArithmeticReturnType(operandType1, operandType2); - generateColumnBinaryOperatorColumn(tdesc, returnType, className); + generateColumnArithmeticOperatorColumn(tdesc, returnType, className); } private void generateFilterColumnCompareScalar(String[] tdesc) throws Exception { - //The variables are all same as ColumnCompareScalar except that - //this template doesn't need a return type. Pass anything as return type. String operatorName = tdesc[1]; String operandType1 = tdesc[2]; String operandType2 = tdesc[3]; String className = "Filter" + getCamelCaseType(operandType1) + "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar"; - generateColumnBinaryOperatorScalar(tdesc, null, className); + generateColumnCompareOperatorScalar(tdesc, true, className); } private void generateFilterScalarCompareColumn(String[] tdesc) throws Exception { - //this template doesn't need a return type. Pass anything as return type. String operatorName = tdesc[1]; String operandType1 = tdesc[2]; String operandType2 = tdesc[3]; String className = "Filter" + getCamelCaseType(operandType1) + "Scalar" + operatorName + getCamelCaseType(operandType2) + "Column"; - generateScalarBinaryOperatorColumn(tdesc, null, className); + generateScalarCompareOperatorColumn(tdesc, true, className); } private void generateColumnCompareScalar(String[] tdesc) throws Exception { String operatorName = tdesc[1]; String operandType1 = tdesc[2]; String operandType2 = tdesc[3]; - String returnType = "long"; String className = getCamelCaseType(operandType1) + "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar"; - generateColumnBinaryOperatorScalar(tdesc, returnType, className); + generateColumnCompareOperatorScalar(tdesc, false, className); } private void generateScalarCompareColumn(String[] tdesc) throws Exception { String operatorName = tdesc[1]; String operandType1 = tdesc[2]; String operandType2 = tdesc[3]; - String returnType = "long"; String className = getCamelCaseType(operandType1) + "Scalar" + operatorName + getCamelCaseType(operandType2) + "Column"; - generateScalarBinaryOperatorColumn(tdesc, returnType, className); + generateScalarCompareOperatorColumn(tdesc, false, className); } - private void generateColumnBinaryOperatorColumn(String[] tdesc, String returnType, + private void generateColumnCompareOperatorColumn(String[] tdesc, boolean filter, String className) throws Exception { String operandType1 = tdesc[2]; String operandType2 = tdesc[3]; - String outputColumnVectorType = this.getColumnVectorType( - returnType == null ? "long" : returnType); String inputColumnVectorType1 = this.getColumnVectorType(operandType1); String inputColumnVectorType2 = this.getColumnVectorType(operandType2); + String returnType = "long"; + String outputColumnVectorType = this.getColumnVectorType(returnType); String operatorSymbol = tdesc[4]; //Read the template into a string; @@ -1174,16 +1475,25 @@ public class GenVectorCode extends Task templateString = templateString.replaceAll("<OperandType2>", operandType2); templateString = templateString.replaceAll("<ReturnType>", returnType); templateString = templateString.replaceAll("<CamelReturnType>", getCamelCaseType(returnType)); + String vectorExprArgType1 = operandType1; + String vectorExprArgType2 = operandType2; + if (operandType1.equals("long") && operandType2.equals("long")) { + // Let comparisons occur for DATE and TIMESTAMP, too. + vectorExprArgType1 = "int_datetime_family"; + vectorExprArgType2 = "int_datetime_family"; + } + templateString = templateString.replaceAll("<VectorExprArgType1>", vectorExprArgType1); + templateString = templateString.replaceAll("<VectorExprArgType2>", vectorExprArgType2); writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, className, templateString); - if(returnType==null){ + if (filter) { testCodeGen.addColumnColumnFilterTestCases( className, inputColumnVectorType1, inputColumnVectorType2, operatorSymbol); - }else{ + } else { testCodeGen.addColumnColumnOperationTestCases( className, inputColumnVectorType1, @@ -1192,13 +1502,44 @@ public class GenVectorCode extends Task } } - private void generateColumnBinaryOperatorScalar(String[] tdesc, String returnType, + private void generateColumnArithmeticOperatorColumn(String[] tdesc, String returnType, + String className) throws Exception { + String operandType1 = tdesc[2]; + String operandType2 = tdesc[3]; + String outputColumnVectorType = this.getColumnVectorType(returnType); + String inputColumnVectorType1 = this.getColumnVectorType(operandType1); + String inputColumnVectorType2 = this.getColumnVectorType(operandType2); + String operatorSymbol = tdesc[4]; + + //Read the template into a string; + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("<ClassName>", className); + templateString = templateString.replaceAll("<InputColumnVectorType1>", inputColumnVectorType1); + templateString = templateString.replaceAll("<InputColumnVectorType2>", inputColumnVectorType2); + templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType); + templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol); + templateString = templateString.replaceAll("<OperandType1>", operandType1); + templateString = templateString.replaceAll("<OperandType2>", operandType2); + templateString = templateString.replaceAll("<ReturnType>", returnType); + templateString = templateString.replaceAll("<CamelReturnType>", getCamelCaseType(returnType)); + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + + testCodeGen.addColumnColumnOperationTestCases( + className, + inputColumnVectorType1, + inputColumnVectorType2, + outputColumnVectorType); + } + + private void generateColumnCompareOperatorScalar(String[] tdesc, boolean filter, String className) throws Exception { String operandType1 = tdesc[2]; String operandType2 = tdesc[3]; - String outputColumnVectorType = this.getColumnVectorType( - returnType == null ? "long" : returnType); String inputColumnVectorType = this.getColumnVectorType(operandType1); + String returnType = "long"; + String outputColumnVectorType = this.getColumnVectorType(returnType); String operatorSymbol = tdesc[4]; //Read the template into a string; @@ -1211,10 +1552,20 @@ public class GenVectorCode extends Task templateString = templateString.replaceAll("<OperandType1>", operandType1); templateString = templateString.replaceAll("<OperandType2>", operandType2); templateString = templateString.replaceAll("<ReturnType>", returnType); + templateString = templateString.replaceAll("<CamelReturnType>", getCamelCaseType(returnType)); + String vectorExprArgType1 = operandType1; + String vectorExprArgType2 = operandType2; + if (operandType1.equals("long") && operandType2.equals("long")) { + // Let comparisons occur for DATE and TIMESTAMP, too. + vectorExprArgType1 = "int_datetime_family"; + vectorExprArgType2 = "int_datetime_family"; + } + templateString = templateString.replaceAll("<VectorExprArgType1>", vectorExprArgType1); + templateString = templateString.replaceAll("<VectorExprArgType2>", vectorExprArgType2); writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, className, templateString); - if(returnType==null) { + if (filter) { testCodeGen.addColumnScalarFilterTestCases( true, className, @@ -1231,13 +1582,42 @@ public class GenVectorCode extends Task } } - private void generateScalarBinaryOperatorColumn(String[] tdesc, String returnType, + private void generateColumnArithmeticOperatorScalar(String[] tdesc, String returnType, + String className) throws Exception { + String operandType1 = tdesc[2]; + String operandType2 = tdesc[3]; + String outputColumnVectorType = this.getColumnVectorType(returnType); + String inputColumnVectorType = this.getColumnVectorType(operandType1); + String operatorSymbol = tdesc[4]; + + //Read the template into a string; + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("<ClassName>", className); + templateString = templateString.replaceAll("<InputColumnVectorType>", inputColumnVectorType); + templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType); + templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol); + templateString = templateString.replaceAll("<OperandType1>", operandType1); + templateString = templateString.replaceAll("<OperandType2>", operandType2); + templateString = templateString.replaceAll("<ReturnType>", returnType); + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + + testCodeGen.addColumnScalarOperationTestCases( + true, + className, + inputColumnVectorType, + outputColumnVectorType, + operandType2); + } + + private void generateScalarCompareOperatorColumn(String[] tdesc, boolean filter, String className) throws Exception { String operandType1 = tdesc[2]; String operandType2 = tdesc[3]; - String outputColumnVectorType = this.getColumnVectorType( - returnType == null ? "long" : returnType); + String returnType = "long"; String inputColumnVectorType = this.getColumnVectorType(operandType2); + String outputColumnVectorType = this.getColumnVectorType(returnType); String operatorSymbol = tdesc[4]; //Read the template into a string; @@ -1251,10 +1631,19 @@ public class GenVectorCode extends Task templateString = templateString.replaceAll("<OperandType2>", operandType2); templateString = templateString.replaceAll("<ReturnType>", returnType); templateString = templateString.replaceAll("<CamelReturnType>", getCamelCaseType(returnType)); + String vectorExprArgType1 = operandType1; + String vectorExprArgType2 = operandType2; + if (operandType1.equals("long") && operandType2.equals("long")) { + // Let comparisons occur for DATE and TIMESTAMP, too. + vectorExprArgType1 = "int_datetime_family"; + vectorExprArgType2 = "int_datetime_family"; + } + templateString = templateString.replaceAll("<VectorExprArgType1>", vectorExprArgType1); + templateString = templateString.replaceAll("<VectorExprArgType2>", vectorExprArgType2); writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, className, templateString); - if(returnType==null) { + if (filter) { testCodeGen.addColumnScalarFilterTestCases( false, className, @@ -1269,7 +1658,38 @@ public class GenVectorCode extends Task outputColumnVectorType, operandType1); } - } + } + + private void generateScalarArithmeticOperatorColumn(String[] tdesc, String returnType, + String className) throws Exception { + String operandType1 = tdesc[2]; + String operandType2 = tdesc[3]; + String outputColumnVectorType = this.getColumnVectorType( + returnType == null ? "long" : returnType); + String inputColumnVectorType = this.getColumnVectorType(operandType2); + String operatorSymbol = tdesc[4]; + + //Read the template into a string; + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("<ClassName>", className); + templateString = templateString.replaceAll("<InputColumnVectorType>", inputColumnVectorType); + templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType); + templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol); + templateString = templateString.replaceAll("<OperandType1>", operandType1); + templateString = templateString.replaceAll("<OperandType2>", operandType2); + templateString = templateString.replaceAll("<ReturnType>", returnType); + templateString = templateString.replaceAll("<CamelReturnType>", getCamelCaseType(returnType)); + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + + testCodeGen.addColumnScalarOperationTestCases( + false, + className, + inputColumnVectorType, + outputColumnVectorType, + operandType1); + } //Binary arithmetic operator private void generateColumnArithmeticScalar(String[] tdesc) throws Exception { @@ -1279,7 +1699,7 @@ public class GenVectorCode extends Task String className = getCamelCaseType(operandType1) + "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar"; String returnType = getArithmeticReturnType(operandType1, operandType2); - generateColumnBinaryOperatorScalar(tdesc, returnType, className); + generateColumnArithmeticOperatorScalar(tdesc, returnType, className); } private void generateColumnArithmeticScalarDecimal(String[] tdesc) throws IOException { @@ -1373,7 +1793,7 @@ public class GenVectorCode extends Task String className = getCamelCaseType(operandType1) + "Scalar" + operatorName + getCamelCaseType(operandType2) + "Column"; String returnType = getArithmeticReturnType(operandType1, operandType2); - generateScalarBinaryOperatorColumn(tdesc, returnType, className); + generateScalarArithmeticOperatorColumn(tdesc, returnType, className); } private void generateFilterDecimalColumnCompareScalar(String[] tdesc) throws IOException {
Modified: hive/branches/tez/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java?rev=1622763&r1=1622762&r2=1622763&view=diff ============================================================================== --- hive/branches/tez/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java (original) +++ hive/branches/tez/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java Fri Sep 5 19:15:44 2014 @@ -343,7 +343,10 @@ public class HiveEndPoint { if (ep.partitionVals.isEmpty()) { return; } - SessionState state = SessionState.start(new CliSessionState(conf)); + SessionState localSession = null; + if(SessionState.get() == null) { + localSession = SessionState.start(new CliSessionState(conf)); + } Driver driver = new Driver(conf); try { @@ -372,7 +375,9 @@ public class HiveEndPoint { } finally { driver.close(); try { - state.close(); + if(localSession != null) { + localSession.close(); + } } catch (IOException e) { LOG.warn("Error closing SessionState used to run Hive DDL."); } @@ -563,11 +568,14 @@ public class HiveEndPoint { /** * Get Id of currently open transaction - * @return + * @return -1 if there is no open TX */ @Override public Long getCurrentTxnId() { - return txnIds.get(currentTxnIndex); + if(currentTxnIndex >= 0) { + return txnIds.get(currentTxnIndex); + } + return -1L; } /** Modified: hive/branches/tez/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java?rev=1622763&r1=1622762&r2=1622763&view=diff ============================================================================== --- hive/branches/tez/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java (original) +++ hive/branches/tez/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java Fri Sep 5 19:15:44 2014 @@ -54,6 +54,8 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileNotFoundException; @@ -67,6 +69,7 @@ import java.util.Map; public class TestStreaming { + private static final Logger LOG = LoggerFactory.getLogger(TestStreaming.class); public static class RawFileSystem extends RawLocalFileSystem { private static final URI NAME; @@ -636,18 +639,25 @@ public class TestStreaming { connection.close(); } - class WriterThd extends Thread { + private static class WriterThd extends Thread { - private StreamingConnection conn; - private HiveEndPoint ep; - private DelimitedInputWriter writer; - private String data; + private final StreamingConnection conn; + private final DelimitedInputWriter writer; + private final String data; + private Throwable error; WriterThd(HiveEndPoint ep, String data) throws Exception { - this.ep = ep; + super("Writer_" + data); writer = new DelimitedInputWriter(fieldNames, ",", ep); conn = ep.newConnection(false); this.data = data; + setUncaughtExceptionHandler(new UncaughtExceptionHandler() { + @Override + public void uncaughtException(Thread thread, Throwable throwable) { + error = throwable; + LOG.error("Thread " + thread.getName() + " died: " + throwable.getMessage(), throwable); + } + }); } @Override @@ -668,14 +678,14 @@ public class TestStreaming { try { txnBatch.close(); } catch (Exception e) { + LOG.error("txnBatch.close() failed: " + e.getMessage(), e); conn.close(); - throw new RuntimeException(e); } } try { conn.close(); } catch (Exception e) { - throw new RuntimeException(e); + LOG.error("conn.close() failed: " + e.getMessage(), e); } } @@ -685,18 +695,23 @@ public class TestStreaming { @Test public void testConcurrentTransactionBatchCommits() throws Exception { final HiveEndPoint ep = new HiveEndPoint(metaStoreURI, dbName, tblName, partitionVals); - WriterThd t1 = new WriterThd(ep, "1,Matrix"); - WriterThd t2 = new WriterThd(ep, "2,Gandhi"); - WriterThd t3 = new WriterThd(ep, "3,Silence"); - - t1.start(); - t2.start(); - t3.start(); - - t1.join(); - t2.join(); - t3.join(); - + List<WriterThd> writers = new ArrayList<WriterThd>(3); + writers.add(new WriterThd(ep, "1,Matrix")); + writers.add(new WriterThd(ep, "2,Gandhi")); + writers.add(new WriterThd(ep, "3,Silence")); + + for(WriterThd w : writers) { + w.start(); + } + for(WriterThd w : writers) { + w.join(); + } + for(WriterThd w : writers) { + if(w.error != null) { + Assert.assertFalse("Writer thread" + w.getName() + " died: " + w.error.getMessage() + + " See log file for stack trace", true); + } + } } // delete db and all tables in it Modified: hive/branches/tez/itests/hive-unit/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/hive-unit/pom.xml?rev=1622763&r1=1622762&r2=1622763&view=diff ============================================================================== --- hive/branches/tez/itests/hive-unit/pom.xml (original) +++ hive/branches/tez/itests/hive-unit/pom.xml Fri Sep 5 19:15:44 2014 @@ -53,6 +53,16 @@ <artifactId>hive-exec</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.hive.hcatalog</groupId> + <artifactId>hive-hcatalog-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hive.hcatalog</groupId> + <artifactId>hive-hcatalog-streaming</artifactId> + <version>${project.version}</version> + </dependency> <!-- dependencies are always listed in sorted order by groupId, artifectId --> <!-- test intra-project --> Modified: hive/branches/tez/metastore/if/hive_metastore.thrift URL: http://svn.apache.org/viewvc/hive/branches/tez/metastore/if/hive_metastore.thrift?rev=1622763&r1=1622762&r2=1622763&view=diff ============================================================================== --- hive/branches/tez/metastore/if/hive_metastore.thrift (original) +++ hive/branches/tez/metastore/if/hive_metastore.thrift Fri Sep 5 19:15:44 2014 @@ -273,6 +273,32 @@ struct Partition { 8: optional PrincipalPrivilegeSet privileges } +struct PartitionWithoutSD { + 1: list<string> values // string value is converted to appropriate partition key type + 2: i32 createTime, + 3: i32 lastAccessTime, + 4: string relativePath, + 5: map<string, string> parameters, + 6: optional PrincipalPrivilegeSet privileges +} + +struct PartitionSpecWithSharedSD { + 1: list<PartitionWithoutSD> partitions, + 2: StorageDescriptor sd, +} + +struct PartitionListComposingSpec { + 1: list<Partition> partitions +} + +struct PartitionSpec { + 1: string dbName, + 2: string tableName, + 3: string rootPath, + 4: optional PartitionSpecWithSharedSD sharedSDPartitionSpec, + 5: optional PartitionListComposingSpec partitionList +} + struct Index { 1: string indexName, // unique with in the whole database namespace 2: string indexHandlerClass, // reserved @@ -793,6 +819,8 @@ service ThriftHiveMetastore extends fb30 3:MetaException o3) i32 add_partitions(1:list<Partition> new_parts) throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3) + i32 add_partitions_pspec(1:list<PartitionSpec> new_parts) + throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3) Partition append_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals) throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3) AddPartitionsResult add_partitions_req(1:AddPartitionsRequest request) @@ -838,6 +866,9 @@ service ThriftHiveMetastore extends fb30 list<Partition> get_partitions_with_auth(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1, 4: string user_name, 5: list<string> group_names) throws(1:NoSuchObjectException o1, 2:MetaException o2) + list<PartitionSpec> get_partitions_pspec(1:string db_name, 2:string tbl_name, 3:i32 max_parts=-1) + throws(1:NoSuchObjectException o1, 2:MetaException o2) + list<string> get_partition_names(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1) throws(1:MetaException o2) @@ -862,6 +893,11 @@ service ThriftHiveMetastore extends fb30 3:string filter, 4:i16 max_parts=-1) throws(1:MetaException o1, 2:NoSuchObjectException o2) + // List partitions as PartitionSpec instances. + list<PartitionSpec> get_part_specs_by_filter(1:string db_name 2:string tbl_name + 3:string filter, 4:i32 max_parts=-1) + throws(1:MetaException o1, 2:NoSuchObjectException o2) + // get the partitions matching the given partition filter // unlike get_partitions_by_filter, takes serialized hive expression, and with that can work // with any filter (get_partitions_by_filter only works if the filter can be pushed down to JDOQL.
