HIVE-13111: Fix timestamp / interval_day_time wrong results with HIVE-9862 (Matt McCline, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/52016296 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/52016296 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/52016296 Branch: refs/heads/llap Commit: 52016296cf7de89d868e1073a3376b330fee0955 Parents: 761b547 Author: Matt McCline <mmccl...@hortonworks.com> Authored: Mon Mar 28 14:14:37 2016 -0700 Committer: Matt McCline <mmccl...@hortonworks.com> Committed: Mon Mar 28 14:14:37 2016 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ant/GenVectorCode.java | 531 ++++----- .../hive/common/type/HiveIntervalDayTime.java | 245 ---- .../org/apache/hive/common/util/DateUtils.java | 19 - data/files/timestamps.txt | 50 + .../test/resources/testconfiguration.properties | 2 + .../java/org/apache/orc/impl/WriterImpl.java | 23 +- ...eColumnArithmeticIntervalYearMonthColumn.txt | 56 +- ...eColumnArithmeticIntervalYearMonthScalar.txt | 55 +- .../DateColumnArithmeticTimestampColumn.txt | 141 ++- .../DateColumnArithmeticTimestampColumnBase.txt | 171 --- .../DateColumnArithmeticTimestampScalar.txt | 113 +- .../DateColumnArithmeticTimestampScalarBase.txt | 137 --- ...eScalarArithmeticIntervalYearMonthColumn.txt | 53 +- .../DateScalarArithmeticTimestampColumn.txt | 108 +- .../DateScalarArithmeticTimestampColumnBase.txt | 147 --- ...ayTimeColumnCompareIntervalDayTimeColumn.txt | 52 - ...ayTimeColumnCompareIntervalDayTimeScalar.txt | 55 - ...ayTimeScalarCompareIntervalDayTimeColumn.txt | 55 - ...erLongDoubleColumnCompareTimestampColumn.txt | 2 +- ...erLongDoubleColumnCompareTimestampScalar.txt | 4 +- ...erLongDoubleScalarCompareTimestampColumn.txt | 4 + .../FilterTimestampColumnBetween.txt | 11 +- ...terTimestampColumnCompareTimestampColumn.txt | 417 ++++++- ...imestampColumnCompareTimestampColumnBase.txt | 429 ------- ...terTimestampColumnCompareTimestampScalar.txt | 128 ++- ...imestampColumnCompareTimestampScalarBase.txt | 145 --- ...erTimestampScalarCompareLongDoubleColumn.txt | 3 +- ...terTimestampScalarCompareTimestampColumn.txt | 132 ++- ...imestampScalarCompareTimestampColumnBase.txt | 147 --- ...ayTimeColumnCompareIntervalDayTimeColumn.txt | 54 - ...ayTimeColumnCompareIntervalDayTimeScalar.txt | 57 - ...ayTimeScalarCompareIntervalDayTimeColumn.txt | 57 - ...ervalYearMonthColumnArithmeticDateColumn.txt | 55 +- ...ervalYearMonthColumnArithmeticDateScalar.txt | 51 +- ...YearMonthColumnArithmeticTimestampColumn.txt | 63 +- ...YearMonthColumnArithmeticTimestampScalar.txt | 48 +- ...ervalYearMonthScalarArithmeticDateColumn.txt | 51 +- ...YearMonthScalarArithmeticTimestampColumn.txt | 55 +- .../LongDoubleColumnCompareTimestampColumn.txt | 1 - .../LongDoubleColumnCompareTimestampScalar.txt | 3 +- .../LongDoubleScalarCompareTimestampColumn.txt | 1 + .../TimestampColumnArithmeticDateColumn.txt | 138 ++- .../TimestampColumnArithmeticDateColumnBase.txt | 172 --- .../TimestampColumnArithmeticDateScalar.txt | 98 +- .../TimestampColumnArithmeticDateScalarBase.txt | 126 -- ...pColumnArithmeticIntervalYearMonthColumn.txt | 59 +- ...pColumnArithmeticIntervalYearMonthScalar.txt | 41 +- ...TimestampColumnArithmeticTimestampColumn.txt | 128 ++- ...stampColumnArithmeticTimestampColumnBase.txt | 152 --- ...TimestampColumnArithmeticTimestampScalar.txt | 96 +- ...stampColumnArithmeticTimestampScalarBase.txt | 125 -- .../TimestampColumnCompareLongDoubleScalar.txt | 1 + .../TimestampColumnCompareTimestampColumn.txt | 122 +- ...imestampColumnCompareTimestampColumnBase.txt | 140 --- .../TimestampColumnCompareTimestampScalar.txt | 114 +- ...imestampColumnCompareTimestampScalarBase.txt | 131 --- .../TimestampScalarArithmeticDateColumn.txt | 117 +- .../TimestampScalarArithmeticDateColumnBase.txt | 151 --- ...pScalarArithmeticIntervalYearMonthColumn.txt | 62 +- ...TimestampScalarArithmeticTimestampColumn.txt | 103 +- ...stampScalarArithmeticTimestampColumnBase.txt | 139 --- .../TimestampScalarCompareLongDoubleColumn.txt | 4 +- .../TimestampScalarCompareTimestampColumn.txt | 115 +- ...imestampScalarCompareTimestampColumnBase.txt | 132 --- .../VectorUDAFMinMaxIntervalDayTime.txt | 454 ++++++++ .../UDAFTemplates/VectorUDAFMinMaxTimestamp.txt | 31 +- .../hive/ql/exec/vector/TimestampUtils.java | 8 + .../hive/ql/exec/vector/VectorAssignRow.java | 23 +- .../exec/vector/VectorColumnAssignFactory.java | 19 +- .../ql/exec/vector/VectorColumnSetInfo.java | 23 +- .../hive/ql/exec/vector/VectorCopyRow.java | 32 + .../ql/exec/vector/VectorDeserializeRow.java | 13 +- .../exec/vector/VectorExpressionDescriptor.java | 6 +- .../hive/ql/exec/vector/VectorExtractRow.java | 24 +- .../ql/exec/vector/VectorGroupKeyHelper.java | 12 + .../ql/exec/vector/VectorHashKeyWrapper.java | 82 +- .../exec/vector/VectorHashKeyWrapperBatch.java | 112 +- .../hive/ql/exec/vector/VectorSerializeRow.java | 7 +- .../ql/exec/vector/VectorizationContext.java | 13 +- .../ql/exec/vector/VectorizedBatchUtil.java | 20 +- .../ql/exec/vector/VectorizedRowBatchCtx.java | 12 +- .../expressions/CastDecimalToTimestamp.java | 3 +- .../expressions/CastDoubleToTimestamp.java | 17 +- .../vector/expressions/CastLongToTimestamp.java | 8 +- .../CastMillisecondsLongToTimestamp.java | 22 +- .../CastStringToIntervalDayTime.java | 8 +- .../expressions/CastTimestampToBoolean.java | 4 +- .../vector/expressions/CastTimestampToDate.java | 2 +- .../expressions/CastTimestampToDecimal.java | 9 +- .../expressions/CastTimestampToDouble.java | 13 +- .../vector/expressions/CastTimestampToLong.java | 12 +- .../expressions/ConstantVectorExpression.java | 36 +- .../expressions/DateColSubtractDateColumn.java | 80 +- .../expressions/DateColSubtractDateScalar.java | 51 +- .../DateScalarSubtractDateColumn.java | 52 +- .../FilterTimestampColumnInList.java | 27 +- .../IfExprIntervalDayTimeColumnColumn.java | 103 +- .../IfExprIntervalDayTimeColumnScalar.java | 94 +- .../IfExprIntervalDayTimeScalarColumn.java | 96 +- .../IfExprIntervalDayTimeScalarScalar.java | 84 +- .../IfExprTimestampColumnColumnBase.java | 8 +- .../IfExprTimestampColumnScalar.java | 3 +- .../IfExprTimestampColumnScalarBase.java | 14 +- .../IfExprTimestampScalarColumn.java | 3 +- .../IfExprTimestampScalarColumnBase.java | 15 +- .../IfExprTimestampScalarScalar.java | 3 +- .../IfExprTimestampScalarScalarBase.java | 13 +- .../ql/exec/vector/expressions/NullUtil.java | 26 + .../expressions/TimestampColumnInList.java | 29 +- .../expressions/VectorExpressionWriter.java | 6 +- .../VectorExpressionWriterFactory.java | 124 +- .../expressions/VectorUDFDateAddColCol.java | 2 +- .../expressions/VectorUDFDateAddColScalar.java | 2 +- .../expressions/VectorUDFDateDiffColCol.java | 10 +- .../expressions/VectorUDFDateDiffColScalar.java | 2 +- .../expressions/VectorUDFDateDiffScalarCol.java | 2 +- .../expressions/VectorUDFDateTimestamp.java | 2 +- .../expressions/VectorUDFUnixTimeStampDate.java | 7 +- .../VectorUDFUnixTimeStampTimestamp.java | 5 +- .../aggregates/VectorUDAFAvgTimestamp.java | 40 +- .../aggregates/VectorUDAFStdPopTimestamp.java | 24 +- .../aggregates/VectorUDAFStdSampTimestamp.java | 27 +- .../aggregates/VectorUDAFVarPopTimestamp.java | 24 +- .../aggregates/VectorUDAFVarSampTimestamp.java | 24 +- .../ql/exec/vector/udf/VectorUDFAdaptor.java | 13 +- .../hadoop/hive/ql/util/DateTimeMath.java | 214 +++- .../TestTimestampWritableAndColumnVector.java | 68 ++ .../TestVectorExpressionWriters.java | 23 +- .../TestVectorFilterExpressions.java | 1 - .../expressions/TestVectorMathFunctions.java | 53 +- .../TestVectorTimestampExpressions.java | 26 +- .../vector/expressions/TestVectorTypeCasts.java | 216 ++-- .../FakeVectorRowBatchFromObjectIterables.java | 3 +- .../vector/util/VectorizedRowGroupGenUtil.java | 14 +- .../hive/ql/io/orc/TestInputOutputFormat.java | 2 +- .../hadoop/hive/ql/io/orc/TestOrcFile.java | 5 +- .../hive/ql/io/orc/TestVectorOrcFile.java | 22 +- .../clientpositive/vector_interval_arithmetic.q | 174 +++ .../tez/vector_interval_arithmetic.q.out | 1086 ++++++++++++++++++ .../clientpositive/tez/vectorized_casts.q.out | 18 +- .../tez/vectorized_timestamp.q.out | 157 +++ .../vector_interval_arithmetic.q.out | 1027 +++++++++++++++++ .../clientpositive/vectorized_casts.q.out | 18 +- .../hive/serde2/io/TimestampWritable.java | 71 +- .../hive/common/type/HiveIntervalDayTime.java | 253 ++++ .../hadoop/hive/common/type/PisaTimestamp.java | 609 ---------- .../hadoop/hive/common/type/RandomTypeUtil.java | 70 +- .../hive/ql/exec/vector/ColumnVector.java | 2 +- .../vector/IntervalDayTimeColumnVector.java | 348 ++++++ .../ql/exec/vector/TimestampColumnVector.java | 341 ++---- .../hive/common/util/IntervalDayTimeUtils.java | 77 ++ .../hive/common/type/TestPisaTimestamp.java | 118 -- .../exec/vector/TestTimestampColumnVector.java | 117 ++ 153 files changed, 8221 insertions(+), 5349 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java ---------------------------------------------------------------------- diff --git a/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java b/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java index 2e369ec..6c6cc63 100644 --- a/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java +++ b/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java @@ -77,15 +77,6 @@ public class GenVectorCode extends Task { {"DTIScalarArithmeticDTIColumnNoConvert", "Subtract", "interval_year_month", "interval_year_month", "-"}, {"DTIColumnArithmeticDTIColumnNoConvert", "Subtract", "interval_year_month", "interval_year_month", "-"}, - // Arithmetic on two TimestampColumnVector base classes. - {"TimestampArithmeticTimestampBase", "Add", "Col", "Column"}, - {"TimestampArithmeticTimestampBase", "Add", "Scalar", "Column"}, - {"TimestampArithmeticTimestampBase", "Add", "Col", "Scalar"}, - - {"TimestampArithmeticTimestampBase", "Subtract", "Col", "Column"}, - {"TimestampArithmeticTimestampBase", "Subtract", "Scalar", "Column"}, - {"TimestampArithmeticTimestampBase", "Subtract", "Col", "Scalar"}, - // Arithmetic on two type interval_day_time (TimestampColumnVector storing nanosecond interval // in 2 longs) produces a interval_day_time. {"TimestampArithmeticTimestamp", "Add", "interval_day_time", "Col", "interval_day_time", "Scalar"}, @@ -111,30 +102,13 @@ public class GenVectorCode extends Task { {"TimestampArithmeticTimestamp", "Subtract", "timestamp", "Col", "interval_day_time", "Column"}, // A type timestamp (TimestampColumnVector) minus a type timestamp produces a - // type interval_day_time (TimestampColumnVector storing nanosecond interval in 2 longs). + // type interval_day_time (IntervalDayTimeColumnVector storing nanosecond interval in 2 primitives). {"TimestampArithmeticTimestamp", "Subtract", "timestamp", "Col", "timestamp", "Scalar"}, {"TimestampArithmeticTimestamp", "Subtract", "timestamp", "Scalar", "timestamp", "Column"}, {"TimestampArithmeticTimestamp", "Subtract", "timestamp", "Col", "timestamp", "Column"}, - // Arithmetic on a TimestampColumnVector and date base classes. - {"DateArithmeticTimestampBase", "Add", "Col", "Column"}, - {"DateArithmeticTimestampBase", "Add", "Scalar", "Column"}, - {"DateArithmeticTimestampBase", "Add", "Col", "Scalar"}, - - {"DateArithmeticTimestampBase", "Subtract", "Col", "Column"}, - {"DateArithmeticTimestampBase", "Subtract", "Scalar", "Column"}, - {"DateArithmeticTimestampBase", "Subtract", "Col", "Scalar"}, - - {"TimestampArithmeticDateBase", "Add", "Col", "Column"}, - {"TimestampArithmeticDateBase", "Add", "Scalar", "Column"}, - {"TimestampArithmeticDateBase", "Add", "Col", "Scalar"}, - - {"TimestampArithmeticDateBase", "Subtract", "Col", "Column"}, - {"TimestampArithmeticDateBase", "Subtract", "Scalar", "Column"}, - {"TimestampArithmeticDateBase", "Subtract", "Col", "Scalar"}, - - // Arithmetic with a type date (LongColumnVector storing epoch days) and type interval_day_time (TimestampColumnVector storing - // nanosecond interval in 2 longs) produces a type timestamp (TimestampColumnVector). + // Arithmetic with a type date (LongColumnVector storing epoch days) and type interval_day_time (IntervalDayTimeColumnVector storing + // nanosecond interval in 2 primitives) produces a type timestamp (TimestampColumnVector). {"DateArithmeticTimestamp", "Add", "date", "Col", "interval_day_time", "Column"}, {"DateArithmeticTimestamp", "Add", "date", "Scalar", "interval_day_time", "Column"}, {"DateArithmeticTimestamp", "Add", "date", "Col", "interval_day_time", "Scalar"}, @@ -147,7 +121,8 @@ public class GenVectorCode extends Task { {"TimestampArithmeticDate", "Add", "interval_day_time", "Scalar", "date", "Column"}, {"TimestampArithmeticDate", "Add", "interval_day_time", "Col", "date", "Scalar"}, - // Subtraction with a type date (LongColumnVector storing epoch days) and type timestamp produces a type timestamp (TimestampColumnVector). + // Subtraction with a type date (LongColumnVector storing days) and type timestamp produces a + // type interval_day_time (IntervalDayTimeColumnVector). {"DateArithmeticTimestamp", "Subtract", "date", "Col", "timestamp", "Column"}, {"DateArithmeticTimestamp", "Subtract", "date", "Scalar", "timestamp", "Column"}, {"DateArithmeticTimestamp", "Subtract", "date", "Col", "timestamp", "Scalar"}, @@ -318,70 +293,48 @@ public class GenVectorCode extends Task { {"ScalarCompareColumn", "Greater", "double", "long", ">"}, {"ScalarCompareColumn", "GreaterEqual", "double", "long", ">="}, - // Base compare timestamp to timestamp used by Timestamp and IntervalDayTime. - {"TimestampCompareTimestampBase", "Equal", "==", "Col", "Column"}, - {"TimestampCompareTimestampBase", "NotEqual", "!=", "Col", "Column"}, - {"TimestampCompareTimestampBase", "Less", "<", "Col", "Column"}, - {"TimestampCompareTimestampBase", "LessEqual", "<=", "Col", "Column"}, - {"TimestampCompareTimestampBase", "Greater", ">", "Col", "Column"}, - {"TimestampCompareTimestampBase", "GreaterEqual", ">=", "Col", "Column"}, - - {"TimestampCompareTimestampBase", "Equal", "==", "Col", "Scalar"}, - {"TimestampCompareTimestampBase", "NotEqual", "!=", "Col", "Scalar"}, - {"TimestampCompareTimestampBase", "Less", "<", "Col", "Scalar"}, - {"TimestampCompareTimestampBase", "LessEqual", "<=", "Col", "Scalar"}, - {"TimestampCompareTimestampBase", "Greater", ">", "Col", "Scalar"}, - {"TimestampCompareTimestampBase", "GreaterEqual", ">=", "Col", "Scalar"}, - - {"TimestampCompareTimestampBase", "Equal", "==", "Scalar", "Column"}, - {"TimestampCompareTimestampBase", "NotEqual", "!=", "Scalar", "Column"}, - {"TimestampCompareTimestampBase", "Less", "<", "Scalar", "Column"}, - {"TimestampCompareTimestampBase", "LessEqual", "<=", "Scalar", "Column"}, - {"TimestampCompareTimestampBase", "Greater", ">", "Scalar", "Column"}, - {"TimestampCompareTimestampBase", "GreaterEqual", ">=", "Scalar", "Column"}, - // Compare timestamp to timestamp. - {"TimestampCompareTimestamp", "Equal", "timestamp", "Col", "Column"}, - {"TimestampCompareTimestamp", "NotEqual", "timestamp", "Col", "Column"}, - {"TimestampCompareTimestamp", "Less", "timestamp", "Col", "Column"}, - {"TimestampCompareTimestamp", "LessEqual", "timestamp", "Col", "Column"}, - {"TimestampCompareTimestamp", "Greater", "timestamp", "Col", "Column"}, - {"TimestampCompareTimestamp", "GreaterEqual", "timestamp", "Col", "Column"}, - - {"TimestampCompareTimestamp", "Equal", "timestamp", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "NotEqual", "timestamp", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "Less", "timestamp", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "LessEqual", "timestamp", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "Greater", "timestamp", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "GreaterEqual", "timestamp", "Col", "Scalar"}, - - {"TimestampCompareTimestamp", "Equal", "timestamp", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "NotEqual", "timestamp", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "Less", "timestamp", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "LessEqual", "timestamp", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "Greater", "timestamp", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "GreaterEqual", "timestamp", "Scalar", "Column"}, - - {"TimestampCompareTimestamp", "Equal", "interval_day_time", "Col", "Column"}, - {"TimestampCompareTimestamp", "NotEqual", "interval_day_time", "Col", "Column"}, - {"TimestampCompareTimestamp", "Less", "interval_day_time", "Col", "Column"}, - {"TimestampCompareTimestamp", "LessEqual", "interval_day_time", "Col", "Column"}, - {"TimestampCompareTimestamp", "Greater", "interval_day_time", "Col", "Column"}, - {"TimestampCompareTimestamp", "GreaterEqual", "interval_day_time", "Col", "Column"}, - - {"TimestampCompareTimestamp", "Equal", "interval_day_time", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "NotEqual", "interval_day_time", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "Less", "interval_day_time", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "LessEqual", "interval_day_time", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "Greater", "interval_day_time", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "GreaterEqual", "interval_day_time", "Col", "Scalar"}, - - {"TimestampCompareTimestamp", "Equal", "interval_day_time", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "NotEqual", "interval_day_time", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "Less", "interval_day_time", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "LessEqual", "interval_day_time", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "Greater", "interval_day_time", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "GreaterEqual", "interval_day_time", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "Equal", "==", "timestamp", "Col", "Column"}, + {"TimestampCompareTimestamp", "NotEqual", "!=", "timestamp", "Col", "Column"}, + {"TimestampCompareTimestamp", "Less", "<", "timestamp", "Col", "Column"}, + {"TimestampCompareTimestamp", "LessEqual", "<=", "timestamp", "Col", "Column"}, + {"TimestampCompareTimestamp", "Greater", ">", "timestamp", "Col", "Column"}, + {"TimestampCompareTimestamp", "GreaterEqual", ">=", "timestamp", "Col", "Column"}, + + {"TimestampCompareTimestamp", "Equal", "==", "timestamp", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "NotEqual", "!=", "timestamp", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "Less", "<", "timestamp", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "LessEqual", "<=", "timestamp", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "Greater", ">", "timestamp", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "GreaterEqual", ">=", "timestamp", "Col", "Scalar"}, + + {"TimestampCompareTimestamp", "Equal", "==", "timestamp", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "NotEqual", "!=", "timestamp", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "Less", "<", "timestamp", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "LessEqual", "<=", "timestamp", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "Greater", ">", "timestamp", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "GreaterEqual", ">=", "timestamp", "Scalar", "Column"}, + + {"TimestampCompareTimestamp", "Equal", "==", "interval_day_time", "Col", "Column"}, + {"TimestampCompareTimestamp", "NotEqual", "!=", "interval_day_time", "Col", "Column"}, + {"TimestampCompareTimestamp", "Less", "<", "interval_day_time", "Col", "Column"}, + {"TimestampCompareTimestamp", "LessEqual", "<=", "interval_day_time", "Col", "Column"}, + {"TimestampCompareTimestamp", "Greater", ">", "interval_day_time", "Col", "Column"}, + {"TimestampCompareTimestamp", "GreaterEqual", ">=", "interval_day_time", "Col", "Column"}, + + {"TimestampCompareTimestamp", "Equal", "==", "interval_day_time", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "NotEqual", "!=", "interval_day_time", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "Less", "<", "interval_day_time", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "LessEqual", "<=", "interval_day_time", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "Greater", ">", "interval_day_time", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "GreaterEqual", ">=", "interval_day_time", "Col", "Scalar"}, + + {"TimestampCompareTimestamp", "Equal", "==", "interval_day_time", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "NotEqual", "!=", "interval_day_time", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "Less", "<", "interval_day_time", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "LessEqual", "<=", "interval_day_time", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "Greater", ">", "interval_day_time", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "GreaterEqual", ">=", "interval_day_time", "Scalar", "Column"}, // Compare timestamp to integer seconds or double seconds with fractional nanoseonds. {"TimestampCompareLongDouble", "Equal", "long", "==", "Col", "Column"}, @@ -515,71 +468,49 @@ public class GenVectorCode extends Task { {"FilterScalarCompareColumn", "GreaterEqual", "long", "long", ">="}, {"FilterScalarCompareColumn", "GreaterEqual", "double", "long", ">="}, - // Base filter timestamp against timestamp used by Timestamp and IntervalDayTime. - {"FilterTimestampCompareTimestampBase", "Equal", "==", "Col", "Column"}, - {"FilterTimestampCompareTimestampBase", "NotEqual", "!=", "Col", "Column"}, - {"FilterTimestampCompareTimestampBase", "Less", "<", "Col", "Column"}, - {"FilterTimestampCompareTimestampBase", "LessEqual", "<=", "Col", "Column"}, - {"FilterTimestampCompareTimestampBase", "Greater", ">", "Col", "Column"}, - {"FilterTimestampCompareTimestampBase", "GreaterEqual", ">=", "Col", "Column"}, - - {"FilterTimestampCompareTimestampBase", "Equal", "==", "Col", "Scalar"}, - {"FilterTimestampCompareTimestampBase", "NotEqual", "!=", "Col", "Scalar"}, - {"FilterTimestampCompareTimestampBase", "Less", "<", "Col", "Scalar"}, - {"FilterTimestampCompareTimestampBase", "LessEqual", "<=", "Col", "Scalar"}, - {"FilterTimestampCompareTimestampBase", "Greater", ">", "Col", "Scalar"}, - {"FilterTimestampCompareTimestampBase", "GreaterEqual", ">=", "Col", "Scalar"}, - - {"FilterTimestampCompareTimestampBase", "Equal", "==", "Scalar", "Column"}, - {"FilterTimestampCompareTimestampBase", "NotEqual", "!=", "Scalar", "Column"}, - {"FilterTimestampCompareTimestampBase", "Less", "<", "Scalar", "Column"}, - {"FilterTimestampCompareTimestampBase", "LessEqual", "<=", "Scalar", "Column"}, - {"FilterTimestampCompareTimestampBase", "Greater", ">", "Scalar", "Column"}, - {"FilterTimestampCompareTimestampBase", "GreaterEqual", ">=", "Scalar", "Column"}, - // Filter timestamp against timestamp, or interval day time against interval day time. - {"FilterTimestampCompareTimestamp", "Equal", "timestamp", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "NotEqual", "timestamp", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "Less", "timestamp", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "LessEqual", "timestamp", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "Greater", "timestamp", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "GreaterEqual", "timestamp", "Col", "Column"}, - - {"FilterTimestampCompareTimestamp", "Equal", "timestamp", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "NotEqual", "timestamp", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "Less", "timestamp", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "LessEqual", "timestamp", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "Greater", "timestamp", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "GreaterEqual", "timestamp", "Col", "Scalar"}, - - {"FilterTimestampCompareTimestamp", "Equal", "timestamp", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "NotEqual", "timestamp", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "Less", "timestamp", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "LessEqual", "timestamp", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "Greater", "timestamp", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "GreaterEqual", "timestamp", "Scalar", "Column"}, - - {"FilterTimestampCompareTimestamp", "Equal", "interval_day_time", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "NotEqual", "interval_day_time", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "Less", "interval_day_time", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "LessEqual", "interval_day_time", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "Greater", "interval_day_time", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "GreaterEqual", "interval_day_time", "Col", "Column"}, - - {"FilterTimestampCompareTimestamp", "Equal", "interval_day_time", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "NotEqual", "interval_day_time", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "Less", "interval_day_time", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "LessEqual", "interval_day_time", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "Greater", "interval_day_time", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "GreaterEqual", "interval_day_time", "Col", "Scalar"}, - - {"FilterTimestampCompareTimestamp", "Equal", "interval_day_time", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "NotEqual", "interval_day_time", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "Less", "interval_day_time", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "LessEqual", "interval_day_time", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "Greater", "interval_day_time", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "GreaterEqual", "interval_day_time", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "Equal", "==", "timestamp", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "NotEqual", "!=", "timestamp", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "Less", "<", "timestamp", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "LessEqual", "<=", "timestamp", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "Greater", ">", "timestamp", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "GreaterEqual", ">=", "timestamp", "Col", "Column"}, + + {"FilterTimestampCompareTimestamp", "Equal", "==", "timestamp", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "NotEqual", "!=", "timestamp", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "Less", "<", "timestamp", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "LessEqual", "<=", "timestamp", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "Greater", ">", "timestamp", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "GreaterEqual", ">=", "timestamp", "Col", "Scalar"}, + + {"FilterTimestampCompareTimestamp", "Equal", "==", "timestamp", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "NotEqual", "!=", "timestamp", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "Less", "<", "timestamp", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "LessEqual", "<=", "timestamp", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "Greater", ">", "timestamp", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "GreaterEqual", ">=", "timestamp", "Scalar", "Column"}, + + {"FilterTimestampCompareTimestamp", "Equal", "==", "interval_day_time", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "NotEqual", "!=", "interval_day_time", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "Less", "<", "interval_day_time", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "LessEqual", "<=", "interval_day_time", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "Greater", ">", "interval_day_time", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "GreaterEqual", ">=", "interval_day_time", "Col", "Column"}, + + {"FilterTimestampCompareTimestamp", "Equal", "==", "interval_day_time", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "NotEqual", "!=", "interval_day_time", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "Less", "<", "interval_day_time", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "LessEqual", "<=", "interval_day_time", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "Greater", ">", "interval_day_time", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "GreaterEqual", ">=", "interval_day_time", "Col", "Scalar"}, + + {"FilterTimestampCompareTimestamp", "Equal", "==", "interval_day_time", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "NotEqual", "!=", "interval_day_time", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "Less", "<", "interval_day_time", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "LessEqual", "<=", "interval_day_time", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "Greater", ">", "interval_day_time", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "GreaterEqual", ">=", "interval_day_time", "Scalar", "Column"}, // Filter timestamp against long (seconds) or double (seconds with fractional // nanoseconds). @@ -1057,6 +988,11 @@ public class GenVectorCode extends Task { {"VectorUDAFMinMaxTimestamp", "VectorUDAFMinTimestamp", ">", "min", "_FUNC_(expr) - Returns the minimum value of expr (vectorized, type: timestamp)"}, + {"VectorUDAFMinMaxIntervalDayTime", "VectorUDAFMaxIntervalDayTime", "<", "max", + "_FUNC_(expr) - Returns the maximum value of expr (vectorized, type: interval_day_time)"}, + {"VectorUDAFMinMaxIntervalDayTime", "VectorUDAFMinIntervalDayTime", ">", "min", + "_FUNC_(expr) - Returns the minimum value of expr (vectorized, type: interval_day_time)"}, + //template, <ClassName>, <ValueType> {"VectorUDAFSum", "VectorUDAFSumLong", "long"}, {"VectorUDAFSum", "VectorUDAFSumDouble", "double"}, @@ -1202,9 +1138,6 @@ public class GenVectorCode extends Task { } else if (tdesc[0].equals("ScalarCompareColumn")) { generateScalarCompareColumn(tdesc); - } else if (tdesc[0].equals("TimestampCompareTimestampBase")) { - generateTimestampCompareTimestampBase(tdesc); - } else if (tdesc[0].equals("TimestampCompareTimestamp")) { generateTimestampCompareTimestamp(tdesc); @@ -1219,9 +1152,6 @@ public class GenVectorCode extends Task { } else if (tdesc[0].equals("FilterScalarCompareColumn")) { generateFilterScalarCompareColumn(tdesc); - } else if (tdesc[0].equals("FilterTimestampCompareTimestampBase")) { - generateFilterTimestampCompareTimestampBase(tdesc); - } else if (tdesc[0].equals("FilterTimestampCompareTimestamp")) { generateFilterTimestampCompareTimestamp(tdesc); @@ -1255,6 +1185,8 @@ public class GenVectorCode extends Task { generateVectorUDAFMinMaxObject(tdesc); } else if (tdesc[0].equals("VectorUDAFMinMaxTimestamp")) { generateVectorUDAFMinMaxObject(tdesc); + } else if (tdesc[0].equals("VectorUDAFMinMaxIntervalDayTime")) { + generateVectorUDAFMinMaxObject(tdesc); } else if (tdesc[0].equals("VectorUDAFSum")) { generateVectorUDAFSum(tdesc); } else if (tdesc[0].equals("VectorUDAFAvg")) { @@ -1338,21 +1270,12 @@ public class GenVectorCode extends Task { } else if (tdesc[0].equals("IntervalYearMonthArithmeticTimestamp")) { generateDateTimeArithmeticIntervalYearMonth(tdesc); - } else if (tdesc[0].equals("TimestampArithmeticTimestampBase")) { - generateTimestampArithmeticTimestampBase(tdesc); - } else if (tdesc[0].equals("TimestampArithmeticTimestamp")) { generateTimestampArithmeticTimestamp(tdesc); - } else if (tdesc[0].equals("DateArithmeticTimestampBase")) { - generateDateArithmeticTimestampBase(tdesc); - } else if (tdesc[0].equals("DateArithmeticTimestamp")) { generateDateArithmeticTimestamp(tdesc); - } else if (tdesc[0].equals("TimestampArithmeticDateBase")) { - generateTimestampArithmeticDateBase(tdesc); - } else if (tdesc[0].equals("TimestampArithmeticDate")) { generateTimestampArithmeticDate(tdesc); @@ -2182,35 +2105,28 @@ public class GenVectorCode extends Task { // // ----------------------------------------------------------------------------------------------- - private void generateFilterTimestampCompareTimestampBase(String[] tdesc) throws Exception { + private void generateFilterTimestampCompareTimestamp(String[] tdesc) throws Exception { String operatorName = tdesc[1]; String operatorSymbol = tdesc[2]; - String className = "FilterTimestamp" + tdesc[3] + operatorName + "Timestamp" + tdesc[4] + "Base"; + String operandType = tdesc[3]; + String camelOperandType = getCamelCaseType(operandType); + String className = "Filter" + camelOperandType + tdesc[4] + operatorName + camelOperandType + tdesc[5]; + String baseClassName = "FilterTimestamp" + tdesc[4] + operatorName + "Timestamp" + tdesc[5] + "Base"; //Read the template into a string; - String fileName = "FilterTimestamp" + (tdesc[3].equals("Col") ? "Column" : tdesc[3]) + "CompareTimestamp" + - tdesc[4] + "Base"; + String fileName = "FilterTimestamp" + (tdesc[4].equals("Col") ? "Column" : tdesc[4]) + "CompareTimestamp" + + tdesc[5]; File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt")); String templateString = readFile(templateFile); templateString = templateString.replaceAll("<ClassName>", className); templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol); - writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, - className, templateString); - } + templateString = templateString.replaceAll("<OperandType>", operandType); + templateString = templateString.replaceAll("<CamelOperandType>", camelOperandType); + templateString = templateString.replaceAll("<HiveOperandType>", getTimestampHiveType(operandType)); + + String inputColumnVectorType = this.getColumnVectorType(operandType); + templateString = templateString.replaceAll("<InputColumnVectorType>", inputColumnVectorType); - private void generateFilterTimestampCompareTimestamp(String[] tdesc) throws Exception { - String operatorName = tdesc[1]; - String operandType = tdesc[2]; - String camelCaseOperandType = getCamelCaseType(operandType); - String className = "Filter" + camelCaseOperandType + tdesc[3] + operatorName + camelCaseOperandType + tdesc[4]; - String baseClassName = "FilterTimestamp" + tdesc[3] + operatorName + "Timestamp" + tdesc[4] + "Base"; - //Read the template into a string; - String fileName = "Filter" + camelCaseOperandType + (tdesc[3].equals("Col") ? "Column" : tdesc[3]) + "Compare" + camelCaseOperandType + - tdesc[4]; - File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt")); - String templateString = readFile(templateFile); - templateString = templateString.replaceAll("<ClassName>", className); - templateString = templateString.replaceAll("<BaseClassName>", baseClassName); writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, className, templateString); } @@ -2287,9 +2203,9 @@ public class GenVectorCode extends Task { private String timestampLongDoubleMethod(String operandType) { if (operandType.equals("long")) { - return "getTimestampSeconds"; + return "getTimestampAsLong"; } else if (operandType.equals("double")) { - return "getTimestampSecondsWithFractionalNanos"; + return "getDouble"; } else { return "unknown"; } @@ -2314,35 +2230,26 @@ public class GenVectorCode extends Task { // // ----------------------------------------------------------------------------------------------- - private void generateTimestampCompareTimestampBase(String[] tdesc) throws Exception { + + private void generateTimestampCompareTimestamp(String[] tdesc) throws Exception { String operatorName = tdesc[1]; String operatorSymbol = tdesc[2]; - String className = "Timestamp" + tdesc[3] + operatorName + "Timestamp" + tdesc[4] + "Base"; + String operandType = tdesc[3]; + String camelOperandType = getCamelCaseType(operandType); + String className = camelOperandType + tdesc[4] + operatorName + camelOperandType + tdesc[5]; //Read the template into a string; - String fileName = "Timestamp" + (tdesc[3].equals("Col") ? "Column" : tdesc[3]) + "CompareTimestamp" + - tdesc[4] + "Base"; + String fileName = "Timestamp" + (tdesc[4].equals("Col") ? "Column" : tdesc[4]) + "CompareTimestamp" + + (tdesc[5].equals("Col") ? "Column" : tdesc[5]); File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt")); String templateString = readFile(templateFile); templateString = templateString.replaceAll("<ClassName>", className); templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol); - writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, - className, templateString); - } + templateString = templateString.replaceAll("<OperandType>", operandType); + templateString = templateString.replaceAll("<CamelOperandType>", camelOperandType); + templateString = templateString.replaceAll("<HiveOperandType>", getTimestampHiveType(operandType)); + templateString = templateString.replaceAll("<InputColumnVectorType>", getColumnVectorType(operandType)); - private void generateTimestampCompareTimestamp(String[] tdesc) throws Exception { - String operatorName = tdesc[1]; - String operandType = tdesc[2]; - String camelCaseOperandType = getCamelCaseType(operandType); - String className = camelCaseOperandType + tdesc[3] + operatorName + camelCaseOperandType + tdesc[4]; - String baseClassName = "Timestamp" + tdesc[3] + operatorName + "Timestamp" + tdesc[4] + "Base"; - //Read the template into a string; - String fileName = camelCaseOperandType + (tdesc[3].equals("Col") ? "Column" : tdesc[3]) + "Compare" + camelCaseOperandType + - tdesc[4]; - File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt")); - String templateString = readFile(templateFile); - templateString = templateString.replaceAll("<ClassName>", className); - templateString = templateString.replaceAll("<BaseClassName>", baseClassName); writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, className, templateString); } @@ -2851,6 +2758,7 @@ public class GenVectorCode extends Task { String templateString = readFile(templateFile); templateString = templateString.replaceAll("<ClassName>", className); templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol); + templateString = templateString.replaceAll("<OperatorMethod>", operatorName.toLowerCase()); writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, className, templateString); @@ -2924,30 +2832,6 @@ public class GenVectorCode extends Task { return templateString; } - // TimestampColumnArithmeticTimestampColumnBase.txt - // TimestampScalarArithmeticTimestampColumnBase.txt - // TimestampColumnArithmeticTimestampScalarBase.txt - // - private void generateTimestampArithmeticTimestampBase(String[] tdesc) throws Exception { - String operatorName = tdesc[1]; - String colOrScalar1 = tdesc[2]; - String colOrScalar2 = tdesc[3]; - - String baseClassName = "Timestamp" + colOrScalar1 + operatorName + - "Timestamp" + colOrScalar2 + "Base"; - - //Read the template into a string; - String fileName = "Timestamp" + (colOrScalar1.equals("Col") ? "Column" : colOrScalar1) + "Arithmetic" + - "Timestamp" + colOrScalar2 + "Base"; - File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt")); - String templateString = readFile(templateFile); - templateString = templateString.replaceAll("<BaseClassName>", baseClassName); - templateString = templateString.replaceAll("<OperatorMethod>", operatorName.toLowerCase()); - - writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, - baseClassName, templateString); - } - // TimestampColumnArithmeticTimestampColumn.txt // TimestampScalarArithmeticTimestampColumn.txt // TimestampColumnArithmeticTimestampScalar.txt @@ -2955,10 +2839,23 @@ public class GenVectorCode extends Task { private void generateTimestampArithmeticTimestamp(String[] tdesc) throws Exception { String operatorName = tdesc[1]; String operandType1 = tdesc[2]; + String camelOperandType1 = getCamelCaseType(operandType1); String colOrScalar1 = tdesc[3]; String operandType2 = tdesc[4]; + String camelOperandType2 = getCamelCaseType(operandType2); String colOrScalar2 = tdesc[5]; + String returnType; + if (operandType1.equals(operandType2)) { + // timestamp - timestamp + // interval_day_time +/- interval_day_time + returnType = "interval_day_time"; + } else { + // timestamp +/- interval_day_time + // interval_day_time + timestamp + returnType = "timestamp"; + } + String className = getCamelCaseType(operandType1) + colOrScalar1 + operatorName + getCamelCaseType(operandType2) + colOrScalar2; String baseClassName = "Timestamp" + colOrScalar1 + operatorName + @@ -2971,20 +2868,26 @@ public class GenVectorCode extends Task { String templateString = readFile(templateFile); templateString = templateString.replaceAll("<ClassName>", className); templateString = templateString.replaceAll("<BaseClassName>", baseClassName); + templateString = templateString.replaceAll("<OperatorMethod>", operatorName.toLowerCase()); templateString = templateString.replaceAll("<OperandType1>", operandType1); templateString = templateString.replaceAll("<OperandType2>", operandType2); - if (colOrScalar1.equals("Scalar")) { - templateString = replaceTimestampScalar(templateString, 1, operandType1); - } - if (colOrScalar2.equals("Scalar")) { - templateString = replaceTimestampScalar(templateString, 2, operandType2); - } - - writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, - className, templateString); + templateString = templateString.replaceAll("<CamelOperandType1>", camelOperandType1); + templateString = templateString.replaceAll("<CamelOperandType2>", camelOperandType2); + templateString = templateString.replaceAll("<HiveOperandType1>", getTimestampHiveType(operandType1)); + templateString = templateString.replaceAll("<HiveOperandType2>", getTimestampHiveType(operandType2)); String inputColumnVectorType1 = this.getColumnVectorType(operandType1); + templateString = templateString.replaceAll("<InputColumnVectorType1>", inputColumnVectorType1); String inputColumnVectorType2 = this.getColumnVectorType(operandType2); + templateString = templateString.replaceAll("<InputColumnVectorType2>", inputColumnVectorType2); + + String outputColumnVectorType = this.getColumnVectorType(returnType); + templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType); + templateString = templateString.replaceAll("<CamelReturnType>", getCamelCaseType(returnType)); + templateString = templateString.replaceAll("<ReturnType>", returnType); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); /* UNDONE: Col Col, vs Scalar Col vs Col Scalar testCodeGen.addColumnColumnOperationTestCases( @@ -2995,30 +2898,6 @@ public class GenVectorCode extends Task { */ } - // DateColumnArithmeticTimestampColumnBase.txt - // DateScalarArithmeticTimestampColumnBase.txt - // DateColumnArithmeticTimestampScalarBase.txt - // - private void generateDateArithmeticTimestampBase(String[] tdesc) throws Exception { - String operatorName = tdesc[1]; - String colOrScalar1 = tdesc[2]; - String colOrScalar2 = tdesc[3]; - - String baseClassName = "Date" + colOrScalar1 + operatorName + - "Timestamp" + colOrScalar2 + "Base"; - - //Read the template into a string; - String fileName = "Date" + (colOrScalar1.equals("Col") ? "Column" : colOrScalar1) + "Arithmetic" + - "Timestamp" + colOrScalar2 + "Base"; - File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt")); - String templateString = readFile(templateFile); - templateString = templateString.replaceAll("<BaseClassName>", baseClassName); - templateString = templateString.replaceAll("<OperatorMethod>", operatorName.toLowerCase()); - - writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, - baseClassName, templateString); - } - // DateColumnArithmeticTimestampColumn.txt // DateScalarArithmeticTimestampColumn.txt // DateColumnArithmeticTimestampScalar.txt @@ -3026,14 +2905,23 @@ public class GenVectorCode extends Task { private void generateDateArithmeticTimestamp(String[] tdesc) throws Exception { String operatorName = tdesc[1]; String operandType1 = tdesc[2]; + String camelOperandType1 = getCamelCaseType(operandType1); String colOrScalar1 = tdesc[3]; String operandType2 = tdesc[4]; + String camelOperandType2 = getCamelCaseType(operandType2); String colOrScalar2 = tdesc[5]; - String className = getCamelCaseType(operandType1) + colOrScalar1 + operatorName + - getCamelCaseType(operandType2) + colOrScalar2; - String baseClassName = "Date" + colOrScalar1 + operatorName + - "Timestamp" + colOrScalar2 + "Base"; + String returnType; + if (operandType1.equals("interval_day_time") || operandType2.equals("interval_day_time")) { + returnType = "timestamp"; + } else if (operandType1.equals("timestamp") || operandType2.equals("timestamp")) { + returnType = "interval_day_time"; + } else { + returnType = "unknown"; + } + + String className = camelOperandType1 + colOrScalar1 + operatorName + + camelOperandType2 + colOrScalar2; //Read the template into a string; String fileName = "Date" + (colOrScalar1.equals("Col") ? "Column" : colOrScalar1) + "Arithmetic" + @@ -3041,21 +2929,26 @@ public class GenVectorCode extends Task { File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt")); String templateString = readFile(templateFile); templateString = templateString.replaceAll("<ClassName>", className); - templateString = templateString.replaceAll("<BaseClassName>", baseClassName); + templateString = templateString.replaceAll("<OperatorMethod>", operatorName.toLowerCase()); templateString = templateString.replaceAll("<OperandType1>", operandType1); templateString = templateString.replaceAll("<OperandType2>", operandType2); - if (colOrScalar1.equals("Scalar")) { - templateString = replaceTimestampScalar(templateString, 1, operandType1); - } - if (colOrScalar2.equals("Scalar")) { - templateString = replaceTimestampScalar(templateString, 2, operandType2); - } - - writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, - className, templateString); + templateString = templateString.replaceAll("<CamelOperandType1>", camelOperandType1); + templateString = templateString.replaceAll("<CamelOperandType2>", camelOperandType2); + templateString = templateString.replaceAll("<HiveOperandType1>", getTimestampHiveType(operandType1)); + templateString = templateString.replaceAll("<HiveOperandType2>", getTimestampHiveType(operandType2)); String inputColumnVectorType1 = this.getColumnVectorType(operandType1); + templateString = templateString.replaceAll("<InputColumnVectorType1>", inputColumnVectorType1); String inputColumnVectorType2 = this.getColumnVectorType(operandType2); + templateString = templateString.replaceAll("<InputColumnVectorType2>", inputColumnVectorType2); + + String outputColumnVectorType = this.getColumnVectorType(returnType); + templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType); + templateString = templateString.replaceAll("<CamelReturnType>", getCamelCaseType(returnType)); + templateString = templateString.replaceAll("<ReturnType>", returnType); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); /* UNDONE: Col Col, vs Scalar Col vs Col Scalar testCodeGen.addColumnColumnOperationTestCases( @@ -3066,30 +2959,6 @@ public class GenVectorCode extends Task { */ } - // TimestampColumnArithmeticDateColumnBase.txt - // TimestampScalarArithmeticDateColumnBase.txt - // TimestampColumnArithmeticDateScalarBase.txt - // - private void generateTimestampArithmeticDateBase(String[] tdesc) throws Exception { - String operatorName = tdesc[1]; - String colOrScalar1 = tdesc[2]; - String colOrScalar2 = tdesc[3]; - - String baseClassName = "Timestamp" + colOrScalar1 + operatorName + - "Date" + colOrScalar2 + "Base"; - - //Read the template into a string; - String fileName = "Timestamp" + (colOrScalar1.equals("Col") ? "Column" : colOrScalar1) + "Arithmetic" + - "Date" + colOrScalar2 + "Base"; - File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt")); - String templateString = readFile(templateFile); - templateString = templateString.replaceAll("<BaseClassName>", baseClassName); - templateString = templateString.replaceAll("<OperatorMethod>", operatorName.toLowerCase()); - - writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, - baseClassName, templateString); - } - // TimestampColumnArithmeticDateColumn.txt // TimestampScalarArithmeticDateColumn.txt // TimestampColumnArithmeticDateScalar.txt @@ -3097,14 +2966,23 @@ public class GenVectorCode extends Task { private void generateTimestampArithmeticDate(String[] tdesc) throws Exception { String operatorName = tdesc[1]; String operandType1 = tdesc[2]; + String camelOperandType1 = getCamelCaseType(operandType1); String colOrScalar1 = tdesc[3]; String operandType2 = tdesc[4]; + String camelOperandType2 = getCamelCaseType(operandType2); String colOrScalar2 = tdesc[5]; - String className = getCamelCaseType(operandType1) + colOrScalar1 + operatorName + - getCamelCaseType(operandType2) + colOrScalar2; - String baseClassName = "Timestamp" + colOrScalar1 + operatorName + - "Date" + colOrScalar2 + "Base"; + String returnType; + if (operandType1.equals("interval_day_time") || operandType2.equals("interval_day_time")) { + returnType = "timestamp"; + } else if (operandType1.equals("timestamp") || operandType2.equals("timestamp")) { + returnType = "interval_day_time"; + } else { + returnType = "unknown"; + } + + String className = camelOperandType1 + colOrScalar1 + operatorName + + camelOperandType2 + colOrScalar2; //Read the template into a string; String fileName = "Timestamp" + (colOrScalar1.equals("Col") ? "Column" : colOrScalar1) + "Arithmetic" + @@ -3112,21 +2990,26 @@ public class GenVectorCode extends Task { File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt")); String templateString = readFile(templateFile); templateString = templateString.replaceAll("<ClassName>", className); - templateString = templateString.replaceAll("<BaseClassName>", baseClassName); + templateString = templateString.replaceAll("<OperatorMethod>", operatorName.toLowerCase()); templateString = templateString.replaceAll("<OperandType1>", operandType1); templateString = templateString.replaceAll("<OperandType2>", operandType2); - if (colOrScalar1.equals("Scalar")) { - templateString = replaceTimestampScalar(templateString, 1, operandType1); - } - if (colOrScalar2.equals("Scalar")) { - templateString = replaceTimestampScalar(templateString, 2, operandType2); - } - - writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, - className, templateString); + templateString = templateString.replaceAll("<CamelOperandType1>", camelOperandType1); + templateString = templateString.replaceAll("<CamelOperandType2>", camelOperandType2); + templateString = templateString.replaceAll("<HiveOperandType1>", getTimestampHiveType(operandType1)); + templateString = templateString.replaceAll("<HiveOperandType2>", getTimestampHiveType(operandType2)); String inputColumnVectorType1 = this.getColumnVectorType(operandType1); + templateString = templateString.replaceAll("<InputColumnVectorType1>", inputColumnVectorType1); String inputColumnVectorType2 = this.getColumnVectorType(operandType2); + templateString = templateString.replaceAll("<InputColumnVectorType2>", inputColumnVectorType2); + + String outputColumnVectorType = this.getColumnVectorType(returnType); + templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType); + templateString = templateString.replaceAll("<CamelReturnType>", getCamelCaseType(returnType)); + templateString = templateString.replaceAll("<ReturnType>", returnType); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); /* UNDONE: Col Col, vs Scalar Col vs Col Scalar testCodeGen.addColumnColumnOperationTestCases( @@ -3272,8 +3155,10 @@ public class GenVectorCode extends Task { return "DecimalColumnVector"; } else if (primitiveType.equals("string")) { return "BytesColumnVector"; - } else if (isTimestampIntervalType(primitiveType)) { + } else if (primitiveType.equals("timestamp")) { return "TimestampColumnVector"; + } else if (primitiveType.equals("interval_day_time")) { + return "IntervalDayTimeColumnVector"; } throw new Exception("Unimplemented primitive column vector type: " + primitiveType); } http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/common/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java deleted file mode 100644 index e262f01..0000000 --- a/common/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java +++ /dev/null @@ -1,245 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.type; - -import java.math.BigDecimal; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.commons.lang.builder.HashCodeBuilder; -import org.apache.hive.common.util.DateUtils; - -/** - * Day-time interval type representing an offset in days/hours/minutes/seconds, - * with nanosecond precision. - * 1 day = 24 hours = 1440 minutes = 86400 seconds - */ -public class HiveIntervalDayTime implements Comparable<HiveIntervalDayTime> { - - // days/hours/minutes/seconds all represented as seconds - protected long totalSeconds; - protected int nanos; - - public HiveIntervalDayTime() { - } - - public HiveIntervalDayTime(int days, int hours, int minutes, int seconds, int nanos) { - set(days, hours, minutes, seconds, nanos); - } - - public HiveIntervalDayTime(long seconds, int nanos) { - set(seconds, nanos); - } - - public HiveIntervalDayTime(BigDecimal seconds) { - set(seconds); - } - - public HiveIntervalDayTime(HiveIntervalDayTime other) { - set(other.totalSeconds, other.nanos); - } - - public int getDays() { - return (int) TimeUnit.SECONDS.toDays(totalSeconds); - } - - public int getHours() { - return (int) (TimeUnit.SECONDS.toHours(totalSeconds) % TimeUnit.DAYS.toHours(1)); - } - - public int getMinutes() { - return (int) (TimeUnit.SECONDS.toMinutes(totalSeconds) % TimeUnit.HOURS.toMinutes(1)); - } - - public int getSeconds() { - return (int) (totalSeconds % TimeUnit.MINUTES.toSeconds(1)); - } - - public int getNanos() { - return nanos; - } - - /** - * Returns days/hours/minutes all converted into seconds. - * Nanos still need to be retrieved using getNanos() - * @return - */ - public long getTotalSeconds() { - return totalSeconds; - } - - /** - * Ensures that the seconds and nanoseconds fields have consistent sign - */ - protected void normalizeSecondsAndNanos() { - if (totalSeconds > 0 && nanos < 0) { - --totalSeconds; - nanos += DateUtils.NANOS_PER_SEC; - } else if (totalSeconds < 0 && nanos > 0) { - ++totalSeconds; - nanos -= DateUtils.NANOS_PER_SEC; - } - } - - public void set(int days, int hours, int minutes, int seconds, int nanos) { - long totalSeconds = seconds; - totalSeconds += TimeUnit.DAYS.toSeconds(days); - totalSeconds += TimeUnit.HOURS.toSeconds(hours); - totalSeconds += TimeUnit.MINUTES.toSeconds(minutes); - totalSeconds += TimeUnit.NANOSECONDS.toSeconds(nanos); - nanos = nanos % DateUtils.NANOS_PER_SEC; - - this.totalSeconds = totalSeconds; - this.nanos = nanos; - - normalizeSecondsAndNanos(); - } - - public void set(long seconds, int nanos) { - this.totalSeconds = seconds; - this.nanos = nanos; - normalizeSecondsAndNanos(); - } - - public void set(PisaTimestamp pisaTimestamp) { - this.totalSeconds = pisaTimestamp.getEpochSeconds(); - this.nanos = pisaTimestamp.getSignedNanos(); - normalizeSecondsAndNanos(); - } - - public void set(BigDecimal totalSecondsBd) { - long totalSeconds = totalSecondsBd.longValue(); - BigDecimal fractionalSecs = totalSecondsBd.remainder(BigDecimal.ONE); - int nanos = fractionalSecs.multiply(DateUtils.NANOS_PER_SEC_BD).intValue(); - set(totalSeconds, nanos); - } - - public void set(HiveIntervalDayTime other) { - set(other.getTotalSeconds(), other.getNanos()); - } - - public HiveIntervalDayTime negate() { - return new HiveIntervalDayTime(-getTotalSeconds(), -getNanos()); - } - - public PisaTimestamp pisaTimestampUpdate(PisaTimestamp pisaTimestamp) { - // NOTE: Our nanos here are *SIGNED*. - return pisaTimestamp.updateFromEpochSecondsAndSignedNanos(totalSeconds, nanos); - } - - @Override - public int compareTo(HiveIntervalDayTime other) { - long cmp = this.totalSeconds - other.totalSeconds; - if (cmp == 0) { - cmp = this.nanos - other.nanos; - } - if (cmp != 0) { - cmp = cmp > 0 ? 1 : -1; - } - return (int) cmp; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof HiveIntervalDayTime)) { - return false; - } - return 0 == compareTo((HiveIntervalDayTime) obj); - } - - @Override - public int hashCode() { - return new HashCodeBuilder().append(totalSeconds).append(nanos).toHashCode(); - } - - @Override - public String toString() { - // If normalize() was used, then day-hour-minute-second-nanos should have the same sign. - // This is currently working with that assumption. - boolean isNegative = (totalSeconds < 0 || nanos < 0); - String daySecondSignStr = isNegative ? "-" : ""; - - return String.format("%s%d %02d:%02d:%02d.%09d", - daySecondSignStr, Math.abs(getDays()), - Math.abs(getHours()), Math.abs(getMinutes()), - Math.abs(getSeconds()), Math.abs(getNanos())); - } - - public static HiveIntervalDayTime valueOf(String strVal) { - HiveIntervalDayTime result = null; - if (strVal == null) { - throw new IllegalArgumentException("Interval day-time string was null"); - } - Matcher patternMatcher = PATTERN_MATCHER.get(); - patternMatcher.reset(strVal); - if (patternMatcher.matches()) { - // Parse out the individual parts - try { - // Sign - whether interval is positive or negative - int sign = 1; - String field = patternMatcher.group(1); - if (field != null && field.equals("-")) { - sign = -1; - } - int days = sign * - DateUtils.parseNumericValueWithRange("day", patternMatcher.group(2), - 0, Integer.MAX_VALUE); - byte hours = (byte) (sign * - DateUtils.parseNumericValueWithRange("hour", patternMatcher.group(3), 0, 23)); - byte minutes = (byte) (sign * - DateUtils.parseNumericValueWithRange("minute", patternMatcher.group(4), 0, 59)); - int seconds = 0; - int nanos = 0; - field = patternMatcher.group(5); - if (field != null) { - BigDecimal bdSeconds = new BigDecimal(field); - if (bdSeconds.compareTo(DateUtils.MAX_INT_BD) > 0) { - throw new IllegalArgumentException("seconds value of " + bdSeconds + " too large"); - } - seconds = sign * bdSeconds.intValue(); - nanos = sign * bdSeconds.subtract(new BigDecimal(bdSeconds.toBigInteger())) - .multiply(DateUtils.NANOS_PER_SEC_BD).intValue(); - } - - result = new HiveIntervalDayTime(days, hours, minutes, seconds, nanos); - } catch (Exception err) { - throw new IllegalArgumentException("Error parsing interval day-time string: " + strVal, err); - } - } else { - throw new IllegalArgumentException( - "Interval string does not match day-time format of 'd h:m:s.n': " + strVal); - } - - return result; - } - - // Simple pattern: D H:M:S.nnnnnnnnn - private final static String PARSE_PATTERN = - "([+|-])?(\\d+) (\\d+):(\\d+):((\\d+)(\\.(\\d+))?)"; - - private static final ThreadLocal<Matcher> PATTERN_MATCHER = new ThreadLocal<Matcher>() { - @Override - protected Matcher initialValue() { - return Pattern.compile(PARSE_PATTERN).matcher(""); - } - }; -} http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/common/src/java/org/apache/hive/common/util/DateUtils.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hive/common/util/DateUtils.java b/common/src/java/org/apache/hive/common/util/DateUtils.java index c749bcb..959a542 100644 --- a/common/src/java/org/apache/hive/common/util/DateUtils.java +++ b/common/src/java/org/apache/hive/common/util/DateUtils.java @@ -21,8 +21,6 @@ package org.apache.hive.common.util; import java.math.BigDecimal; import java.text.SimpleDateFormat; -import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; - /** * DateUtils. Thread-safe class * @@ -56,21 +54,4 @@ public class DateUtils { } return result; } - - public static long getIntervalDayTimeTotalNanos(HiveIntervalDayTime intervalDayTime) { - return intervalDayTime.getTotalSeconds() * NANOS_PER_SEC + intervalDayTime.getNanos(); - } - - public static void setIntervalDayTimeTotalNanos(HiveIntervalDayTime intervalDayTime, - long totalNanos) { - intervalDayTime.set(totalNanos / NANOS_PER_SEC, (int) (totalNanos % NANOS_PER_SEC)); - } - - public static long getIntervalDayTimeTotalSecondsFromTotalNanos(long totalNanos) { - return totalNanos / NANOS_PER_SEC; - } - - public static int getIntervalDayTimeNanosFromTotalNanos(long totalNanos) { - return (int) (totalNanos % NANOS_PER_SEC); - } } http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/data/files/timestamps.txt ---------------------------------------------------------------------- diff --git a/data/files/timestamps.txt b/data/files/timestamps.txt new file mode 100644 index 0000000..36ffd23 --- /dev/null +++ b/data/files/timestamps.txt @@ -0,0 +1,50 @@ +6631-11-13 16:31:29.702202248 +6731-02-12 08:12:48.287783702 +6705-09-28 18:27:28.000845672 +5397-07-13 07:12:32.000896438 +9209-11-11 04:08:58.223768453 +9403-01-09 18:12:33.547 +6482-04-27 12:07:38.073915413 +7503-06-23 23:14:17.486 +1883-04-17 04:14:34.647766229 +0004-09-22 18:26:29.519542222 +7160-12-02 06:00:24.81200852 +8422-07-22 03:21:45.745036084 +4143-07-08 10:53:27.252802259 +5344-10-04 18:40:08.165 +5966-07-09 03:30:50.597 +9075-06-13 16:20:09.218517797 +1815-05-06 00:12:37.543584705 +7409-09-07 23:33:32.459349602 +5339-02-01 14:10:01.085678691 +4966-12-04 09:30:55.202 +1319-02-02 16:31:57.778 +1404-07-23 15:32:16.059185026 +6229-06-28 02:54:28.970117179 +0528-10-27 08:15:18.941718273 +8521-01-16 20:42:05.668832388 +1976-05-06 00:42:30.910786948 +2003-09-23 22:33:17.00003252 +2007-02-09 05:17:29.368756876 +1998-10-16 20:05:29.397591987 +1976-03-03 04:54:33.000895162 +1985-07-20 09:30:11.0 +2021-09-24 03:18:32.413655165 +2013-04-07 02:44:43.00086821 +2002-05-10 05:29:48.990818073 +1973-04-17 06:30:38.596784156 +1987-02-21 19:48:29.0 +1981-11-15 23:03:10.999338387 +2000-12-18 08:42:30.000595596 +1999-10-03 16:59:10.396903939 +2024-11-11 16:42:41.101 +2013-04-10 00:43:46.854731546 +2010-04-08 02:43:35.861742727 +2004-03-07 20:14:13.0 +1987-05-28 13:52:07.900916635 +1978-08-05 14:41:05.501 +1966-08-16 13:36:50.183618031 +2009-01-21 10:49:07.108 +1981-04-25 09:01:12.077192689 +1985-11-18 16:37:54.0 +1974-10-04 17:21:03.989 http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index f8e8bda..0672e0e 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -289,6 +289,7 @@ minitez.query.files.shared=acid_globallimit.q,\ vector_inner_join.q,\ vector_interval_1.q,\ vector_interval_2.q,\ + vector_interval_arithmetic.q,\ vector_interval_mapjoin.q,\ vector_join30.q,\ vector_join_filters.q,\ @@ -319,6 +320,7 @@ minitez.query.files.shared=acid_globallimit.q,\ vector_reduce3.q,\ vector_string_concat.q,\ vector_struct_in.q,\ + vectorized_timestamp.q,\ vector_varchar_4.q,\ vector_varchar_mapjoin1.q,\ vector_varchar_simple.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/orc/src/java/org/apache/orc/impl/WriterImpl.java ---------------------------------------------------------------------- diff --git a/orc/src/java/org/apache/orc/impl/WriterImpl.java b/orc/src/java/org/apache/orc/impl/WriterImpl.java index 6497ecf..d4b9a14 100644 --- a/orc/src/java/org/apache/orc/impl/WriterImpl.java +++ b/orc/src/java/org/apache/orc/impl/WriterImpl.java @@ -1737,19 +1737,17 @@ public class WriterImpl implements Writer, MemoryManager.Callback { int length) throws IOException { super.writeBatch(vector, offset, length); TimestampColumnVector vec = (TimestampColumnVector) vector; + Timestamp val; if (vector.isRepeating) { if (vector.noNulls || !vector.isNull[0]) { - long millis = vec.getEpochMilliseconds(0); - int adjustedNanos = vec.getSignedNanos(0); - if (adjustedNanos < 0) { - adjustedNanos += NANOS_PER_SECOND; - } + val = vec.asScratchTimestamp(0); + long millis = val.getTime(); indexStatistics.updateTimestamp(millis); if (createBloomFilter) { bloomFilter.addLong(millis); } - final long secs = vec.getEpochSeconds(0) - base_timestamp; - final long nano = formatNanos(adjustedNanos); + final long secs = millis / MILLIS_PER_SECOND - base_timestamp; + final long nano = formatNanos(val.getNanos()); for(int i=0; i < length; ++i) { seconds.write(secs); nanos.write(nano); @@ -1758,14 +1756,11 @@ public class WriterImpl implements Writer, MemoryManager.Callback { } else { for(int i=0; i < length; ++i) { if (vec.noNulls || !vec.isNull[i + offset]) { - long secs = vec.getEpochSeconds(i + offset) - base_timestamp; - long millis = vec.getEpochMilliseconds(i + offset); - int adjustedNanos = vec.getSignedNanos(i + offset); - if (adjustedNanos < 0) { - adjustedNanos += NANOS_PER_SECOND; - } + val = vec.asScratchTimestamp(i + offset); + long millis = val.getTime(); + long secs = millis / MILLIS_PER_SECOND - base_timestamp; seconds.write(secs); - nanos.write(formatNanos(adjustedNanos)); + nanos.write(formatNanos(val.getNanos())); indexStatistics.updateTimestamp(millis); if (createBloomFilter) { bloomFilter.addLong(millis); http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthColumn.txt ---------------------------------------------------------------------- diff --git a/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthColumn.txt index 845bc5f..c3d8d7e 100644 --- a/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthColumn.txt +++ b/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthColumn.txt @@ -18,15 +18,18 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; +import java.sql.Date; +import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.util.DateTimeMath; +import org.apache.hadoop.hive.serde2.io.DateWritable; /** - * Generated from template DateColumnArithmeticIntervalYearMonthColumn.txt, which covers binary arithmetic + * Generated from template DateColumnArithmeticIntervalYearMonthColumn.txt, which covers binary arithmetic * expressions between date and interval year month columns. */ public class <ClassName> extends VectorExpression { @@ -36,12 +39,18 @@ public class <ClassName> extends VectorExpression { private int colNum1; private int colNum2; private int outputColumn; + private Date scratchDate1; + private HiveIntervalYearMonth scratchIntervalYearMonth2; + private Date outputDate; private DateTimeMath dtm = new DateTimeMath(); public <ClassName>(int colNum1, int colNum2, int outputColumn) { this.colNum1 = colNum1; this.colNum2 = colNum2; this.outputColumn = outputColumn; + scratchDate1 = new Date(0); + scratchIntervalYearMonth2 = new HiveIntervalYearMonth(); + outputDate = new Date(0); } public <ClassName>() { @@ -54,10 +63,10 @@ public class <ClassName> extends VectorExpression { super.evaluateChildren(batch); } - // Input #1 is type date (epochDays). + // Input #1 is type date. LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum1]; - // Input #2 is type interval_year_month (months). + // Input #2 is type interval_year_month. LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum2]; // Output is type date. @@ -89,38 +98,65 @@ public class <ClassName> extends VectorExpression { * conditional checks in the inner loop. */ if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - outputVector[0] = dtm.addMonthsToDays(vector1[0], <OperatorSymbol> (int) vector2[0]); + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[0])); + scratchIntervalYearMonth2.set((int) vector2[0]); + dtm.<OperatorMethod>( + scratchDate1, scratchIntervalYearMonth2, outputDate); + outputVector[0] = DateWritable.dateToDays(outputDate); } else if (inputColVector1.isRepeating) { + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[0])); if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = dtm.addMonthsToDays(vector1[0], <OperatorSymbol> (int) vector2[i]); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.<OperatorMethod>( + scratchDate1, scratchIntervalYearMonth2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } else { for(int i = 0; i != n; i++) { - outputVector[i] = dtm.addMonthsToDays(vector1[0], <OperatorSymbol> (int) vector2[i]); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.<OperatorMethod>( + scratchDate1, scratchIntervalYearMonth2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } } else if (inputColVector2.isRepeating) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = dtm.addMonthsToDays(vector1[i], <OperatorSymbol> (int) vector2[0]); + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i])); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.<OperatorMethod>( + scratchDate1, scratchIntervalYearMonth2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } else { for(int i = 0; i != n; i++) { - outputVector[i] = dtm.addMonthsToDays(vector1[i], <OperatorSymbol> (int) vector2[0]); + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i])); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.<OperatorMethod>( + scratchDate1, scratchIntervalYearMonth2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } } else { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = dtm.addMonthsToDays(vector1[i], <OperatorSymbol> (int) vector2[i]); + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i])); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.<OperatorMethod>( + scratchDate1, scratchIntervalYearMonth2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } else { for(int i = 0; i != n; i++) { - outputVector[i] = dtm.addMonthsToDays(vector1[i], <OperatorSymbol> (int) vector2[i]); + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i])); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.<OperatorMethod>( + scratchDate1, scratchIntervalYearMonth2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } } http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthScalar.txt ---------------------------------------------------------------------- diff --git a/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthScalar.txt b/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthScalar.txt index 86a95c9..d1474fb 100644 --- a/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthScalar.txt +++ b/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthScalar.txt @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; +import java.sql.Date; +import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -25,6 +27,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.ql.util.DateTimeMath; +import org.apache.hadoop.hive.serde2.io.DateWritable; /** * Generated from template DateColumnArithmeticIntervalYearMonthScalar.txt, which covers binary arithmetic @@ -35,14 +38,18 @@ public class <ClassName> extends VectorExpression { private static final long serialVersionUID = 1L; private int colNum; - private long value; + private HiveIntervalYearMonth value; private int outputColumn; + private Date scratchDate1; + private Date outputDate; private DateTimeMath dtm = new DateTimeMath(); public <ClassName>(int colNum, long value, int outputColumn) { this.colNum = colNum; - this.value = value; + this.value = new HiveIntervalYearMonth((int) value); this.outputColumn = outputColumn; + scratchDate1 = new Date(0); + outputDate = new Date(0); } public <ClassName>() { @@ -55,19 +62,19 @@ public class <ClassName> extends VectorExpression { super.evaluateChildren(batch); } - // Input #1 is type date (epochDays). - LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum]; + // Input #1 is type date. + LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum]; // Output is type date. LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn]; int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; + boolean[] inputIsNull = inputColVector1.isNull; boolean[] outputIsNull = outputColVector.isNull; - outputColVector.noNulls = inputColVector.noNulls; - outputColVector.isRepeating = inputColVector.isRepeating; + outputColVector.noNulls = inputColVector1.noNulls; + outputColVector.isRepeating = inputColVector1.isRepeating; int n = batch.size; - long[] vector = inputColVector.vector; + long[] vector1 = inputColVector1.vector; long[] outputVector = outputColVector.vector; // return immediately if batch is empty @@ -75,32 +82,46 @@ public class <ClassName> extends VectorExpression { return; } - if (inputColVector.isRepeating) { - outputVector[0] = dtm.addMonthsToDays(vector[0], <OperatorSymbol> (int) value); - - // Even if there are no nulls, we always copy over entry 0. Simplifies code. + if (inputColVector1.isRepeating) { + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[0])); + dtm.<OperatorMethod>( + scratchDate1, value, outputDate); + outputVector[0] = DateWritable.dateToDays(outputDate); + // Even if there are no nulls, we always copy over entry 0. Simplifies code. outputIsNull[0] = inputIsNull[0]; - } else if (inputColVector.noNulls) { + } else if (inputColVector1.noNulls) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = dtm.addMonthsToDays(vector[i], <OperatorSymbol> (int) value); + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.<OperatorMethod>( + scratchDate1, value, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } else { for(int i = 0; i != n; i++) { - outputVector[i] = dtm.addMonthsToDays(vector[i], <OperatorSymbol> (int) value); + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.<OperatorMethod>( + scratchDate1, value, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } } else /* there are nulls */ { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = dtm.addMonthsToDays(vector[i], <OperatorSymbol> (int) value); + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.<OperatorMethod>( + scratchDate1, value, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); outputIsNull[i] = inputIsNull[i]; } } else { for(int i = 0; i != n; i++) { - outputVector[i] = dtm.addMonthsToDays(vector[i], <OperatorSymbol> (int) value); + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.<OperatorMethod>( + scratchDate1, value, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); }