http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateScalar.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateScalar.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateScalar.java index 8942b78..a1745c8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateScalar.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateScalar.java @@ -21,15 +21,13 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; import java.sql.Timestamp; import java.util.Arrays; -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.util.DateTimeMath; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; // A type date (LongColumnVector storing epoch days) minus a type date produces a // type interval_day_time (TimestampColumnVector storing nanosecond interval in 2 longs). @@ -47,7 +45,7 @@ public class DateColSubtractDateScalar extends VectorExpression { super(outputColumnNum); this.colNum = colNum; this.value = new Timestamp(0); - this.value.setTime(DateWritable.daysToMillis((int) value)); + this.value.setTime(DateWritableV2.daysToMillis((int) value)); } public DateColSubtractDateScalar() { @@ -88,7 +86,7 @@ public class DateColSubtractDateScalar extends VectorExpression { if (inputColVector1.isRepeating) { if (inputColVector1.noNulls || !inputIsNull[0]) { outputIsNull[0] = false; - scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[0])); + scratchTimestamp1.setTime(DateWritableV2.daysToMillis((int) vector1[0])); dtm.subtract(scratchTimestamp1, value, outputColVector.getScratchIntervalDayTime()); outputColVector.setFromScratchIntervalDayTime(0); @@ -111,14 +109,14 @@ public class DateColSubtractDateScalar extends VectorExpression { final int i = sel[j]; // Set isNull before call in case it changes it mind. outputIsNull[i] = false; - scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + scratchTimestamp1.setTime(DateWritableV2.daysToMillis((int) vector1[i])); dtm.subtract(scratchTimestamp1, value, outputColVector.getScratchIntervalDayTime()); outputColVector.setFromScratchIntervalDayTime(i); } } else { for(int j = 0; j != n; j++) { final int i = sel[j]; - scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + scratchTimestamp1.setTime(DateWritableV2.daysToMillis((int) vector1[i])); dtm.subtract(scratchTimestamp1, value, outputColVector.getScratchIntervalDayTime()); outputColVector.setFromScratchIntervalDayTime(i); } @@ -132,7 +130,7 @@ public class DateColSubtractDateScalar extends VectorExpression { outputColVector.noNulls = true; } for(int i = 0; i != n; i++) { - scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + scratchTimestamp1.setTime(DateWritableV2.daysToMillis((int) vector1[i])); dtm.subtract(scratchTimestamp1, value, outputColVector.getScratchIntervalDayTime()); outputColVector.setFromScratchIntervalDayTime(i); } @@ -146,14 +144,14 @@ public class DateColSubtractDateScalar extends VectorExpression { for(int j = 0; j != n; j++) { int i = sel[j]; outputIsNull[i] = inputIsNull[i]; - scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + scratchTimestamp1.setTime(DateWritableV2.daysToMillis((int) vector1[i])); dtm.subtract(scratchTimestamp1, value, outputColVector.getScratchIntervalDayTime()); outputColVector.setFromScratchIntervalDayTime(i); } } else { System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); for(int i = 0; i != n; i++) { - scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + scratchTimestamp1.setTime(DateWritableV2.daysToMillis((int) vector1[i])); dtm.subtract(scratchTimestamp1, value, outputColVector.getScratchIntervalDayTime()); outputColVector.setFromScratchIntervalDayTime(i); } @@ -165,7 +163,7 @@ public class DateColSubtractDateScalar extends VectorExpression { @Override public String vectorExpressionParameters() { - return getColumnParamString(0, colNum) + ", val " + value; + return getColumnParamString(0, colNum) + ", val " + Date.ofEpochMilli(value.getTime()); } @Override
http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateScalarSubtractDateColumn.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateScalarSubtractDateColumn.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateScalarSubtractDateColumn.java index 68b038f..cedbba2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateScalarSubtractDateColumn.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateScalarSubtractDateColumn.java @@ -21,12 +21,10 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; import java.sql.Timestamp; import java.util.Arrays; -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.util.DateTimeMath; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; // A type date (LongColumnVector storing epoch days) minus a type date produces a // type interval_day_time (TimestampColumnVector storing nanosecond interval in 2 longs). @@ -44,7 +42,7 @@ public class DateScalarSubtractDateColumn extends VectorExpression { super(outputColumnNum); this.colNum = colNum; this.value = new Timestamp(0); - this.value.setTime(DateWritable.daysToMillis((int) value)); + this.value.setTime(DateWritableV2.daysToMillis((int) value)); } public DateScalarSubtractDateColumn() { @@ -91,7 +89,7 @@ public class DateScalarSubtractDateColumn extends VectorExpression { if (inputColVector2.isRepeating) { if (inputColVector2.noNulls || !inputIsNull[0]) { outputIsNull[0] = false; - scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[0])); + scratchTimestamp2.setTime(DateWritableV2.daysToMillis((int) vector2[0])); dtm.subtract(value, scratchTimestamp2, outputColVector.getScratchIntervalDayTime()); outputColVector.setFromScratchIntervalDayTime(0); } else { @@ -108,14 +106,14 @@ public class DateScalarSubtractDateColumn extends VectorExpression { for(int j = 0; j != n; j++) { int i = sel[j]; outputIsNull[i] = false; - scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + scratchTimestamp2.setTime(DateWritableV2.daysToMillis((int) vector2[i])); dtm.subtract(value, scratchTimestamp2, outputColVector.getScratchIntervalDayTime()); outputColVector.setFromScratchIntervalDayTime(i); } } else { Arrays.fill(outputIsNull, 0, n, false); for(int i = 0; i != n; i++) { - scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + scratchTimestamp2.setTime(DateWritableV2.daysToMillis((int) vector2[i])); dtm.subtract(value, scratchTimestamp2, outputColVector.getScratchIntervalDayTime()); outputColVector.setFromScratchIntervalDayTime(i); } @@ -129,14 +127,14 @@ public class DateScalarSubtractDateColumn extends VectorExpression { for(int j = 0; j != n; j++) { int i = sel[j]; outputIsNull[i] = inputIsNull[i]; - scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + scratchTimestamp2.setTime(DateWritableV2.daysToMillis((int) vector2[i])); dtm.subtract(value, scratchTimestamp2, outputColVector.getScratchIntervalDayTime()); outputColVector.setFromScratchIntervalDayTime(i); } } else { System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); for(int i = 0; i != n; i++) { - scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + scratchTimestamp2.setTime(DateWritableV2.daysToMillis((int) vector2[i])); dtm.subtract(value, scratchTimestamp2, outputColVector.getScratchIntervalDayTime()); outputColVector.setFromScratchIntervalDayTime(i); } @@ -148,7 +146,7 @@ public class DateScalarSubtractDateColumn extends VectorExpression { @Override public String vectorExpressionParameters() { - return "val " + value + ", " + getColumnParamString(1, colNum); + return "val " + org.apache.hadoop.hive.common.type.Date.ofEpochMilli(value.getTime()) + ", " + getColumnParamString(1, colNum); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColumnInList.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColumnInList.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColumnInList.java index e150789..e444b68 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColumnInList.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColumnInList.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.Descript import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import java.util.Arrays; http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DynamicValueVectorExpression.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DynamicValueVectorExpression.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DynamicValueVectorExpression.java index 252a816..eff20c9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DynamicValueVectorExpression.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DynamicValueVectorExpression.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.DynamicValue; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -181,9 +181,9 @@ public class DynamicValueVectorExpression extends VectorExpression { decimalValue = PrimitiveObjectInspectorUtils.getHiveDecimal(val, poi); break; case DATE: - longValue = DateWritable.dateToDays(PrimitiveObjectInspectorUtils.getDate(val, poi)); + longValue = DateWritableV2.dateToDays(PrimitiveObjectInspectorUtils.getDate(val, poi)); case TIMESTAMP: - timestampValue = PrimitiveObjectInspectorUtils.getTimestamp(val, poi); + timestampValue = PrimitiveObjectInspectorUtils.getTimestamp(val, poi).toSqlTimestamp(); break; case INTERVAL_YEAR_MONTH: longValue = PrimitiveObjectInspectorUtils.getHiveIntervalYearMonth(val, poi).getTotalMonths(); http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/TimestampToStringUnaryUDF.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/TimestampToStringUnaryUDF.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/TimestampToStringUnaryUDF.java index 0d9f9f7..f924e2e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/TimestampToStringUnaryUDF.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/TimestampToStringUnaryUDF.java @@ -33,7 +33,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; abstract public class TimestampToStringUnaryUDF extends VectorExpression { private static final long serialVersionUID = 1L; - private final int inputColumn; + protected final int inputColumn; public TimestampToStringUnaryUDF(int inputColumn, int outputColumnNum) { super(outputColumnNum); http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java index 48638b7..929f845 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; /** @@ -41,7 +41,7 @@ public interface VectorExpressionWriter { Object writeValue(byte[] value, int start, int length) throws HiveException; Object writeValue(HiveDecimalWritable value) throws HiveException; Object writeValue(HiveDecimal value) throws HiveException; - Object writeValue(TimestampWritable value) throws HiveException; + Object writeValue(TimestampWritableV2 value) throws HiveException; Object writeValue(Timestamp value) throws HiveException; Object writeValue(HiveIntervalDayTimeWritable value) throws HiveException; Object writeValue(HiveIntervalDayTime value) throws HiveException; http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java index a086535..6a87927 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java @@ -70,7 +70,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; import org.apache.hadoop.io.Text; -import org.apache.hive.common.util.DateUtils; /** * VectorExpressionWritableFactory helper class for generating VectorExpressionWritable objects. @@ -185,14 +184,14 @@ public final class VectorExpressionWriterFactory { * The base implementation must be overridden by the Timestamp specialization */ @Override - public Object writeValue(TimestampWritable value) throws HiveException { + public Object writeValue(TimestampWritableV2 value) throws HiveException { throw new HiveException("Internal error: should not reach here"); } /** * The base implementation must be overridden by the Timestamp specialization */ - public Object setValue(Object field, TimestampWritable value) throws HiveException { + public Object setValue(Object field, TimestampWritableV2 value) throws HiveException { throw new HiveException("Internal error: should not reach here"); } @@ -471,9 +470,9 @@ public final class VectorExpressionWriterFactory { @Override public Object writeValue(ColumnVector column, int row) throws HiveException { TimestampColumnVector dcv = (TimestampColumnVector) column; - TimestampWritable timestampWritable = (TimestampWritable) dcv.getScratchWritable(); + TimestampWritableV2 timestampWritable = (TimestampWritableV2) dcv.getScratchWritable(); if (timestampWritable == null) { - timestampWritable = new TimestampWritable(); + timestampWritable = new TimestampWritableV2(); dcv.setScratchWritable(timestampWritable); } if (dcv.noNulls && !dcv.isRepeating) { @@ -498,9 +497,9 @@ public final class VectorExpressionWriterFactory { @Override public Object setValue(Object field, ColumnVector column, int row) throws HiveException { TimestampColumnVector dcv = (TimestampColumnVector) column; - TimestampWritable timestampWritable = (TimestampWritable) dcv.getScratchWritable(); + TimestampWritableV2 timestampWritable = (TimestampWritableV2) dcv.getScratchWritable(); if (timestampWritable == null) { - timestampWritable = new TimestampWritable(); + timestampWritable = new TimestampWritableV2(); dcv.setScratchWritable(timestampWritable); } if (dcv.noNulls && !dcv.isRepeating) { @@ -786,7 +785,7 @@ public final class VectorExpressionWriterFactory { } @Override - public Object setValue(Object field, TimestampWritable value) { + public Object setValue(Object field, TimestampWritableV2 value) { if (null == field) { field = initValue(null); } @@ -824,7 +823,7 @@ public final class VectorExpressionWriterFactory { @Override public Object writeValue(long value) { - dt.setTime(DateWritable.daysToMillis((int) value)); + dt.setTime(DateWritableV2.daysToMillis((int) value)); ((SettableDateObjectInspector) this.objectInspector).set(obj, dt); return obj; } @@ -834,7 +833,7 @@ public final class VectorExpressionWriterFactory { if (null == field) { field = initValue(null); } - dt.setTime(DateWritable.daysToMillis((int) value)); + dt.setTime(DateWritableV2.daysToMillis((int) value)); ((SettableDateObjectInspector) this.objectInspector).set(field, dt); return field; } @@ -860,7 +859,7 @@ public final class VectorExpressionWriterFactory { } @Override - public Object writeValue(TimestampWritable value) throws HiveException { + public Object writeValue(TimestampWritableV2 value) throws HiveException { return ((SettableTimestampObjectInspector) this.objectInspector).set(obj, value); } @@ -880,7 +879,7 @@ public final class VectorExpressionWriterFactory { } @Override - public Object setValue(Object field, TimestampWritable value) { + public Object setValue(Object field, TimestampWritableV2 value) { if (null == field) { field = initValue(null); } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColCol.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColCol.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColCol.java index 0c83494..998b6b1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColCol.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColCol.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; +import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; @@ -25,15 +26,12 @@ import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.io.Text; import org.apache.hive.common.util.DateParser; -import java.util.Arrays; -import java.sql.Date; - public class VectorUDFDateAddColCol extends VectorExpression { private static final long serialVersionUID = 1L; @@ -43,7 +41,6 @@ public class VectorUDFDateAddColCol extends VectorExpression { protected boolean isPositive = true; private transient final Text text = new Text(); - private transient final Date date = new Date(0); private transient final DateParser dateParser = new DateParser(); // Transient members initialized by transientInit method. @@ -261,7 +258,7 @@ public class VectorUDFDateAddColCol extends VectorExpression { protected long evaluateTimestamp(ColumnVector columnVector, int index, long numDays) { TimestampColumnVector tcv = (TimestampColumnVector) columnVector; // Convert to date value (in days) - long days = DateWritable.millisToDays(tcv.getTime(index)); + long days = DateWritableV2.millisToDays(tcv.getTime(index)); if (isPositive) { days += numDays; } else { @@ -281,7 +278,7 @@ public class VectorUDFDateAddColCol extends VectorExpression { } TimestampColumnVector tcv = (TimestampColumnVector) columnVector; // Convert to date value (in days) - long days = DateWritable.millisToDays(tcv.getTime(0)); + long days = DateWritableV2.millisToDays(tcv.getTime(0)); evaluateRepeatedCommon(days, vector2, outputVector, selectedInUse, selected, n); } @@ -292,13 +289,14 @@ public class VectorUDFDateAddColCol extends VectorExpression { outputVector.isNull[index] = true; } else { text.set(inputColumnVector1.vector[index], inputColumnVector1.start[index], inputColumnVector1.length[index]); - boolean parsed = dateParser.parseDate(text.toString(), date); + Date hDate = new Date(); + boolean parsed = dateParser.parseDate(text.toString(), hDate); if (!parsed) { outputVector.noNulls = false; outputVector.isNull[index] = true; return; } - long days = DateWritable.millisToDays(date.getTime()); + long days = DateWritableV2.millisToDays(hDate.toEpochMilli()); if (isPositive) { days += numDays; } else { @@ -319,6 +317,7 @@ public class VectorUDFDateAddColCol extends VectorExpression { } text.set( inputColumnVector1.vector[0], inputColumnVector1.start[0], inputColumnVector1.length[0]); + Date date = new Date(); boolean parsed = dateParser.parseDate(text.toString(), date); if (!parsed) { outputVector.noNulls = false; @@ -326,7 +325,7 @@ public class VectorUDFDateAddColCol extends VectorExpression { outputVector.isRepeating = true; return; } - long days = DateWritable.millisToDays(date.getTime()); + long days = DateWritableV2.millisToDays(date.toEpochMilli()); evaluateRepeatedCommon(days, vector2, outputVector, selectedInUse, selected, n); } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColScalar.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColScalar.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColScalar.java index a73d2e6..30b20c8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColScalar.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColScalar.java @@ -25,13 +25,12 @@ import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.io.Text; import org.apache.hive.common.util.DateParser; -import java.sql.Date; import java.util.Arrays; public class VectorUDFDateAddColScalar extends VectorExpression { @@ -44,7 +43,6 @@ public class VectorUDFDateAddColScalar extends VectorExpression { private transient final Text text = new Text(); private transient final DateParser dateParser = new DateParser(); - private transient final Date date = new Date(0); // Transient members initialized by transientInit method. private transient PrimitiveCategory primitiveCategory; @@ -305,7 +303,7 @@ public class VectorUDFDateAddColScalar extends VectorExpression { protected long evaluateTimestamp(ColumnVector columnVector, int index) { TimestampColumnVector tcv = (TimestampColumnVector) columnVector; // Convert to date value (in days) - long days = DateWritable.millisToDays(tcv.getTime(index)); + long days = DateWritableV2.millisToDays(tcv.getTime(index)); if (isPositive) { days += numDays; } else { @@ -328,13 +326,14 @@ public class VectorUDFDateAddColScalar extends VectorExpression { protected void evaluateString(ColumnVector columnVector, LongColumnVector outputVector, int i) { BytesColumnVector bcv = (BytesColumnVector) columnVector; text.set(bcv.vector[i], bcv.start[i], bcv.length[i]); - boolean parsed = dateParser.parseDate(text.toString(), date); + org.apache.hadoop.hive.common.type.Date hDate = new org.apache.hadoop.hive.common.type.Date(); + boolean parsed = dateParser.parseDate(text.toString(), hDate); if (!parsed) { outputVector.noNulls = false; outputVector.isNull[i] = true; return; } - long days = DateWritable.millisToDays(date.getTime()); + long days = DateWritableV2.millisToDays(hDate.toEpochMilli()); if (isPositive) { days += numDays; } else { http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddScalarCol.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddScalarCol.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddScalarCol.java index c861321..669ca55 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddScalarCol.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddScalarCol.java @@ -18,17 +18,17 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; +import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hive.common.util.DateParser; import java.nio.charset.StandardCharsets; -import java.sql.Date; import java.sql.Timestamp; import java.util.Arrays; @@ -46,7 +46,7 @@ public class VectorUDFDateAddScalarCol extends VectorExpression { protected boolean isPositive = true; private transient final DateParser dateParser = new DateParser(); - private transient final Date baseDate = new Date(0); + private transient final Date baseDate = new Date(); // Transient members initialized by transientInit method. private transient PrimitiveCategory primitiveCategory; @@ -99,11 +99,11 @@ public class VectorUDFDateAddScalarCol extends VectorExpression { switch (primitiveCategory) { case DATE: - baseDate.setTime(DateWritable.daysToMillis((int) longValue)); + baseDate.setTimeInMillis(DateWritableV2.daysToMillis((int) longValue)); break; case TIMESTAMP: - baseDate.setTime(timestampValue.getTime()); + baseDate.setTimeInMillis(timestampValue.getTime()); break; case STRING: @@ -137,7 +137,7 @@ public class VectorUDFDateAddScalarCol extends VectorExpression { // We do not need to do a column reset since we are carefully changing the output. outputColVector.isRepeating = false; - long baseDateDays = DateWritable.millisToDays(baseDate.getTime()); + long baseDateDays = DateWritableV2.millisToDays(baseDate.toEpochMilli()); if (inputCol.isRepeating) { if (inputCol.noNulls || !inputCol.isNull[0]) { outputColVector.isNull[0] = false; @@ -245,11 +245,12 @@ public class VectorUDFDateAddScalarCol extends VectorExpression { public String vectorExpressionParameters() { String value; if (object instanceof Long) { - Date tempDate = new Date(0); - tempDate.setTime(DateWritable.daysToMillis((int) longValue)); + Date tempDate = new Date(); + tempDate.setTimeInMillis(DateWritableV2.daysToMillis((int) longValue)); value = tempDate.toString(); } else if (object instanceof Timestamp) { - value = this.timestampValue.toString(); + value = org.apache.hadoop.hive.common.type.Timestamp.ofEpochMilli( + timestampValue.getTime(), timestampValue.getNanos()).toString(); } else if (object instanceof byte []) { value = new String(this.stringValue, StandardCharsets.UTF_8); } else { http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColCol.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColCol.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColCol.java index bb12fcb..ff29ee3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColCol.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColCol.java @@ -20,13 +20,12 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -217,7 +216,7 @@ public class VectorUDFDateDiffColCol extends VectorExpression { String string = new String(input.vector[0], input.start[0], input.length[0]); try { date.setTime(formatter.parse(string).getTime()); - output.vector[0] = DateWritable.dateToDays(date); + output.vector[0] = DateWritableV2.dateToDays(date); output.isNull[0] = false; } catch (ParseException e) { output.isNull[0] = true; @@ -296,7 +295,7 @@ public class VectorUDFDateDiffColCol extends VectorExpression { String string = new String(input.vector[i], input.start[i], input.length[i]); try { date.setTime(formatter.parse(string).getTime()); - output.vector[i] = DateWritable.dateToDays(date); + output.vector[i] = DateWritableV2.dateToDays(date); } catch (ParseException e) { output.isNull[i] = true; output.noNulls = false; @@ -314,7 +313,7 @@ public class VectorUDFDateDiffColCol extends VectorExpression { if (input.isRepeating) { if (input.noNulls || !input.isNull[0]) { date.setTime(input.getTime(0)); - output.vector[0] = DateWritable.dateToDays(date); + output.vector[0] = DateWritableV2.dateToDays(date); output.isNull[0] = false; } else { output.isNull[0] = true; @@ -334,14 +333,14 @@ public class VectorUDFDateDiffColCol extends VectorExpression { for (int j=0; j < size; j++) { int i = sel[j]; date.setTime(input.getTime(i)); - output.vector[i] = DateWritable.dateToDays(date); + output.vector[i] = DateWritableV2.dateToDays(date); output.isNull[i] = false; } } else { for (int j=0; j < size; j++) { int i = sel[j]; date.setTime(input.getTime(i)); - output.vector[i] = DateWritable.dateToDays(date); + output.vector[i] = DateWritableV2.dateToDays(date); } } } else { @@ -352,7 +351,7 @@ public class VectorUDFDateDiffColCol extends VectorExpression { } for (int i = 0; i < size; i++) { date.setTime(input.getTime(i)); - output.vector[i] = DateWritable.dateToDays(date); + output.vector[i] = DateWritableV2.dateToDays(date); } } } else /* there are nulls in our column */ { @@ -373,14 +372,14 @@ public class VectorUDFDateDiffColCol extends VectorExpression { int i = sel[j]; if (!input.isNull[i]) { date.setTime(input.getTime(i)); - output.vector[i] = DateWritable.dateToDays(date); + output.vector[i] = DateWritableV2.dateToDays(date); } } } else { for (int i = 0; i < size; i++) { if (!input.isNull[i]) { date.setTime(input.getTime(i)); - output.vector[i] = DateWritable.dateToDays(date); + output.vector[i] = DateWritableV2.dateToDays(date); } } } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColScalar.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColScalar.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColScalar.java index 55af413..caedc80 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColScalar.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColScalar.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; -import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; @@ -26,12 +25,11 @@ import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.io.Text; -import java.nio.charset.StandardCharsets; import java.sql.Date; import java.sql.Timestamp; import java.text.ParseException; @@ -106,7 +104,7 @@ public class VectorUDFDateDiffColScalar extends VectorExpression { case TIMESTAMP: date.setTime(timestampValue.getTime()); - baseDate = DateWritable.dateToDays(date); + baseDate = DateWritableV2.dateToDays(date); break; case STRING: @@ -114,7 +112,7 @@ public class VectorUDFDateDiffColScalar extends VectorExpression { case VARCHAR: try { date.setTime(formatter.parse(new String(bytesValue, "UTF-8")).getTime()); - baseDate = DateWritable.dateToDays(date); + baseDate = DateWritableV2.dateToDays(date); break; } catch (Exception e) { outputColVector.noNulls = false; @@ -346,7 +344,7 @@ public class VectorUDFDateDiffColScalar extends VectorExpression { protected int evaluateTimestamp(ColumnVector columnVector, int index) { TimestampColumnVector tcv = (TimestampColumnVector) columnVector; date.setTime(tcv.getTime(index)); - return DateWritable.dateToDays(date) - baseDate; + return DateWritableV2.dateToDays(date) - baseDate; } protected int evaluateDate(ColumnVector columnVector, int index) { @@ -359,7 +357,7 @@ public class VectorUDFDateDiffColScalar extends VectorExpression { text.set(bcv.vector[i], bcv.start[i], bcv.length[i]); try { date.setTime(formatter.parse(text.toString()).getTime()); - output.vector[i] = DateWritable.dateToDays(date) - baseDate; + output.vector[i] = DateWritableV2.dateToDays(date) - baseDate; } catch (ParseException e) { output.vector[i] = 1; output.isNull[i] = true; http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffScalarCol.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffScalarCol.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffScalarCol.java index c51d3cd..28addf7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffScalarCol.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffScalarCol.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.io.Text; @@ -105,7 +105,7 @@ public class VectorUDFDateDiffScalarCol extends VectorExpression { case TIMESTAMP: date.setTime(timestampValue.getTime()); - baseDate = DateWritable.dateToDays(date); + baseDate = DateWritableV2.dateToDays(date); break; case STRING: @@ -113,7 +113,7 @@ public class VectorUDFDateDiffScalarCol extends VectorExpression { case VARCHAR: try { date.setTime(formatter.parse(new String(stringValue, "UTF-8")).getTime()); - baseDate = DateWritable.dateToDays(date); + baseDate = DateWritableV2.dateToDays(date); break; } catch (Exception e) { outputColVector.noNulls = false; @@ -346,7 +346,7 @@ public class VectorUDFDateDiffScalarCol extends VectorExpression { protected int evaluateTimestamp(ColumnVector columnVector, int index) { TimestampColumnVector tcv = (TimestampColumnVector) columnVector; date.setTime(tcv.getTime(index)); - return baseDate - DateWritable.dateToDays(date); + return baseDate - DateWritableV2.dateToDays(date); } protected int evaluateDate(ColumnVector columnVector, int index) { @@ -359,7 +359,7 @@ public class VectorUDFDateDiffScalarCol extends VectorExpression { text.set(bcv.vector[i], bcv.start[i], bcv.length[i]); try { date.setTime(formatter.parse(text.toString()).getTime()); - output.vector[i] = baseDate - DateWritable.dateToDays(date); + output.vector[i] = baseDate - DateWritableV2.dateToDays(date); } catch (ParseException e) { output.vector[i] = 1; output.isNull[i] = true; http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateLong.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateLong.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateLong.java index 202f5d8..8e5f9da 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateLong.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateLong.java @@ -18,14 +18,6 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; -import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.serde2.io.DateWritable; - -import java.io.UnsupportedEncodingException; -import java.sql.Date; -import java.text.SimpleDateFormat; - /** * Vectorized version of TO_DATE(TIMESTAMP)/TO_DATE(DATE). * As TO_DATE() now returns DATE type, this should be the same behavior as the DATE cast operator. http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateString.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateString.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateString.java index 8d87ef7..accf32c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateString.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateString.java @@ -18,17 +18,6 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; -import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.serde2.io.DateWritable; -import org.apache.hive.common.util.DateParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.sql.Date; - /** * Vectorized version of TO_DATE(STRING) * As TO_DATE() now returns DATE type, this should be the same behavior as the DATE cast operator. http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateTimestamp.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateTimestamp.java index 7e7080c..5b497e5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateTimestamp.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateTimestamp.java @@ -18,15 +18,6 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; -import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.serde2.io.DateWritable; - -import java.io.UnsupportedEncodingException; -import java.sql.Date; -import java.text.SimpleDateFormat; - /** * Vectorized version of TO_DATE(timestamp). * As TO_DATE() now returns DATE type, this should be the same behavior as the DATE cast operator. http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDayOfWeekString.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDayOfWeekString.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDayOfWeekString.java index a701d70..e8f1454 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDayOfWeekString.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDayOfWeekString.java @@ -23,6 +23,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; +import java.util.TimeZone; import org.apache.hadoop.io.Text; @@ -35,7 +36,8 @@ public final class VectorUDFDayOfWeekString extends VectorUDFTimestampFieldStrin private static final long serialVersionUID = 1L; private transient final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); - private transient final Calendar calendar = Calendar.getInstance(); + private transient final Calendar calendar = Calendar.getInstance( + TimeZone.getTimeZone("UTC")); public VectorUDFDayOfWeekString(int colNum, int outputColumnNum) { super(colNum, outputColumnNum, -1, -1); http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldDate.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldDate.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldDate.java index a1167b2..837de9d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldDate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldDate.java @@ -20,13 +20,14 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; import java.util.Arrays; import java.util.Calendar; +import java.util.TimeZone; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hive.common.util.DateUtils; @@ -43,7 +44,8 @@ public abstract class VectorUDFTimestampFieldDate extends VectorExpression { protected final int colNum; protected final int field; - protected transient final Calendar calendar = Calendar.getInstance(); + protected transient final Calendar calendar = Calendar.getInstance( + TimeZone.getTimeZone("UTC")); public VectorUDFTimestampFieldDate(int field, int colNum, int outputColumnNum) { super(outputColumnNum); @@ -69,7 +71,7 @@ public abstract class VectorUDFTimestampFieldDate extends VectorExpression { } protected long getDateField(long days) { - calendar.setTimeInMillis(DateWritable.daysToMillis((int) days)); + calendar.setTimeInMillis(DateWritableV2.daysToMillis((int) days)); return calendar.get(field); } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldString.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldString.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldString.java index 931cc98..9acfa86 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldString.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldString.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import java.text.ParseException; import java.util.Calendar; +import java.util.TimeZone; /** * Abstract class to return various fields from a String. @@ -39,7 +40,8 @@ public abstract class VectorUDFTimestampFieldString extends VectorExpression { protected final int fieldLength; private static final String patternMin = "0000-00-00 00:00:00.000000000"; private static final String patternMax = "9999-19-99 29:59:59.999999999"; - protected transient final Calendar calendar = Calendar.getInstance(); + protected transient final Calendar calendar = Calendar.getInstance( + TimeZone.getTimeZone("UTC")); public VectorUDFTimestampFieldString(int colNum, int outputColumnNum, int fieldStart, int fieldLength) { super(outputColumnNum); http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldTimestamp.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldTimestamp.java index a9ea1f6..94e8b47 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldTimestamp.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldTimestamp.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; import java.util.Arrays; import java.util.Calendar; +import java.util.TimeZone; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; @@ -43,7 +44,8 @@ public abstract class VectorUDFTimestampFieldTimestamp extends VectorExpression protected final int colNum; protected final int field; - protected transient final Calendar calendar = Calendar.getInstance(); + protected transient final Calendar calendar = Calendar.getInstance( + TimeZone.getTimeZone("UTC")); public VectorUDFTimestampFieldTimestamp(int field, int colNum, int outputColumnNum) { super(outputColumnNum); http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampDate.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampDate.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampDate.java index 3515329..1f83ead 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampDate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampDate.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; /** * Return Unix Timestamp. @@ -28,7 +28,7 @@ public final class VectorUDFUnixTimeStampDate extends VectorUDFTimestampFieldDat private static final long serialVersionUID = 1L; - private DateWritable dateWritable; + private DateWritableV2 dateWritable; @Override protected long getDateField(long days) { @@ -39,7 +39,7 @@ public final class VectorUDFUnixTimeStampDate extends VectorUDFTimestampFieldDat public VectorUDFUnixTimeStampDate(int colNum, int outputColumnNum) { /* not a real field */ super(-1, colNum, outputColumnNum); - dateWritable = new DateWritable(); + dateWritable = new DateWritableV2(); } public VectorUDFUnixTimeStampDate() { http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampString.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampString.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampString.java index aaea433..3b5b33b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampString.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampString.java @@ -25,6 +25,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; +import java.util.TimeZone; /** * Return Unix Timestamp. @@ -34,8 +35,9 @@ public final class VectorUDFUnixTimeStampString extends VectorUDFTimestampFieldS private static final long serialVersionUID = 1L; - private transient final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - private transient final Calendar calendar = Calendar.getInstance(); + private transient final SimpleDateFormat format = getFormatter(); + private transient final Calendar calendar = Calendar.getInstance( + TimeZone.getTimeZone("UTC")); public VectorUDFUnixTimeStampString(int colNum, int outputColumnNum) { super(colNum, outputColumnNum, -1, -1); @@ -56,4 +58,10 @@ public final class VectorUDFUnixTimeStampString extends VectorUDFTimestampFieldS calendar.setTime(date); return calendar.getTimeInMillis() / 1000; } + + private static SimpleDateFormat getFormatter() { + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + format.setTimeZone(TimeZone.getTimeZone("UTC")); + return format; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java index e011657..31ef6ad 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java @@ -17,37 +17,17 @@ */ package org.apache.hadoop.hive.ql.exec.vector.udf; -import java.sql.Date; -import java.sql.Timestamp; -import java.util.Map; - -import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; -import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; -import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.ql.exec.MapredContext; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; -import org.apache.hadoop.hive.serde2.io.DateWritable; -import org.apache.hadoop.hive.serde2.io.HiveCharWritable; -import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.SettableMapObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.*; -import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBinaryObjectInspector; -import org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable; -import org.apache.hadoop.hive.serde2.io.HiveIntervalYearMonthWritable; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.Text; /** * A VectorUDFAdaptor is a vectorized expression for invoking a custom http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/io/BatchToRowReader.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/BatchToRowReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/BatchToRowReader.java index c88ee99..434a5b8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/BatchToRowReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/BatchToRowReader.java @@ -24,7 +24,7 @@ import org.apache.hadoop.hive.llap.DebugUtils; import java.util.Arrays; -import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector; +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +37,7 @@ import java.util.Map; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; @@ -47,13 +48,12 @@ import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; import org.apache.hadoop.hive.serde2.io.ByteWritable; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveCharWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; @@ -429,18 +429,18 @@ public abstract class BatchToRowReader<StructType, UnionType> } } - public static DateWritable nextDate(ColumnVector vector, - int row, - Object previous) { + public static DateWritableV2 nextDate(ColumnVector vector, + int row, + Object previous) { if (vector.isRepeating) { row = 0; } if (vector.noNulls || !vector.isNull[row]) { - DateWritable result; - if (previous == null || previous.getClass() != DateWritable.class) { - result = new DateWritable(); + DateWritableV2 result; + if (previous == null || previous.getClass() != DateWritableV2.class) { + result = new DateWritableV2(); } else { - result = (DateWritable) previous; + result = (DateWritableV2) previous; } int date = (int) ((LongColumnVector) vector).vector[row]; result.set(date); @@ -450,18 +450,18 @@ public abstract class BatchToRowReader<StructType, UnionType> } } - public static TimestampWritable nextTimestamp(ColumnVector vector, - int row, - Object previous) { + public static TimestampWritableV2 nextTimestamp(ColumnVector vector, + int row, + Object previous) { if (vector.isRepeating) { row = 0; } if (vector.noNulls || !vector.isNull[row]) { - TimestampWritable result; - if (previous == null || previous.getClass() != TimestampWritable.class) { - result = new TimestampWritable(); + TimestampWritableV2 result; + if (previous == null || previous.getClass() != TimestampWritableV2.class) { + result = new TimestampWritableV2(); } else { - result = (TimestampWritable) previous; + result = (TimestampWritableV2) previous; } TimestampColumnVector tcv = (TimestampColumnVector) vector; result.setInternal(tcv.time[row], tcv.nanos[row]); http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java index 56d590c..e7dfb05 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java @@ -58,6 +58,7 @@ public final class OrcFile extends org.apache.orc.OrcFile { public static class ReaderOptions extends org.apache.orc.OrcFile.ReaderOptions { public ReaderOptions(Configuration conf) { super(conf); + useUTCTimestamp(true); } public ReaderOptions filesystem(FileSystem fs) { @@ -79,6 +80,11 @@ public final class OrcFile extends org.apache.orc.OrcFile { super.orcTail(orcTail); return this; } + + public ReaderOptions useUTCTimestamp(boolean value) { + super.useUTCTimestamp(value); + return this; + } } public static ReaderOptions readerOptions(Configuration conf) { @@ -104,6 +110,7 @@ public final class OrcFile extends org.apache.orc.OrcFile { WriterOptions(Properties tableProperties, Configuration conf) { super(tableProperties, conf); + useUTCTimestamp(true); } /** @@ -275,6 +282,11 @@ public final class OrcFile extends org.apache.orc.OrcFile { return this; } + public WriterOptions useUTCTimestamp(boolean value) { + super.useUTCTimestamp(value); + return this; + } + ObjectInspector getInspector() { return inspector; } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java index c6fe4fc..d177e3f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java @@ -38,20 +38,19 @@ import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.serde2.io.ByteWritable; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveCharWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.FloatWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; -import org.apache.orc.OrcFile; import org.apache.orc.TypeDescription; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -415,18 +414,18 @@ public class RecordReaderImpl extends org.apache.orc.impl.RecordReaderImpl } } - static DateWritable nextDate(ColumnVector vector, - int row, - Object previous) { + static DateWritableV2 nextDate(ColumnVector vector, + int row, + Object previous) { if (vector.isRepeating) { row = 0; } if (vector.noNulls || !vector.isNull[row]) { - DateWritable result; - if (previous == null || previous.getClass() != DateWritable.class) { - result = new DateWritable(); + DateWritableV2 result; + if (previous == null || previous.getClass() != DateWritableV2.class) { + result = new DateWritableV2(); } else { - result = (DateWritable) previous; + result = (DateWritableV2) previous; } int date = (int) ((LongColumnVector) vector).vector[row]; result.set(date); @@ -436,18 +435,18 @@ public class RecordReaderImpl extends org.apache.orc.impl.RecordReaderImpl } } - static TimestampWritable nextTimestamp(ColumnVector vector, - int row, - Object previous) { + static TimestampWritableV2 nextTimestamp(ColumnVector vector, + int row, + Object previous) { if (vector.isRepeating) { row = 0; } if (vector.noNulls || !vector.isNull[row]) { - TimestampWritable result; - if (previous == null || previous.getClass() != TimestampWritable.class) { - result = new TimestampWritable(); + TimestampWritableV2 result; + if (previous == null || previous.getClass() != TimestampWritableV2.class) { + result = new TimestampWritableV2(); } else { - result = (TimestampWritable) previous; + result = (TimestampWritableV2) previous; } TimestampColumnVector tcv = (TimestampColumnVector) vector; result.setInternal(tcv.time[row], tcv.nanos[row]); http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java index 91a01e9..4082c61 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java @@ -19,19 +19,17 @@ package org.apache.hadoop.hive.ql.io.orc; import java.io.IOException; -import java.sql.Timestamp; import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; @@ -43,6 +41,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; @@ -63,8 +62,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectIn import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; -import com.google.common.annotations.VisibleForTesting; -import org.apache.orc.PhysicalWriter; import org.apache.orc.TypeDescription; /** @@ -206,9 +203,9 @@ public class WriterImpl extends org.apache.orc.impl.WriterImpl implements Writer } case TIMESTAMP: { TimestampColumnVector vector = (TimestampColumnVector) column; - Timestamp ts = ((TimestampObjectInspector) inspector) - .getPrimitiveJavaObject(obj); - vector.set(rowId, ts); + vector.setIsUTC(true); + vector.set(rowId, ((TimestampObjectInspector) inspector) + .getPrimitiveJavaObject(obj).toSqlTimestamp()); break; } case DATE: { http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java index 8be8d13..08788cf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java @@ -14,18 +14,18 @@ package org.apache.hadoop.hive.ql.io.parquet.convert; import java.math.BigDecimal; -import java.sql.Timestamp; import java.util.ArrayList; import java.util.Map; +import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTime; import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTimeUtils; import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -340,12 +340,12 @@ public enum ETypeConverter { }; } }, - ETIMESTAMP_CONVERTER(TimestampWritable.class) { + ETIMESTAMP_CONVERTER(TimestampWritableV2.class) { @Override PrimitiveConverter getConverter(final PrimitiveType type, final int index, final ConverterParent parent, TypeInfo hiveTypeInfo) { - return new BinaryConverter<TimestampWritable>(type, parent, index) { + return new BinaryConverter<TimestampWritableV2>(type, parent, index) { @Override - protected TimestampWritable convert(Binary binary) { + protected TimestampWritableV2 convert(Binary binary) { NanoTime nt = NanoTime.fromBinary(binary); Map<String, String> metadata = parent.getMetadata(); //Current Hive parquet timestamp implementation stores it in UTC, but other components do not do that. @@ -353,18 +353,18 @@ public enum ETypeConverter { boolean skipConversion = Boolean.parseBoolean( metadata.get(HiveConf.ConfVars.HIVE_PARQUET_TIMESTAMP_SKIP_CONVERSION.varname)); Timestamp ts = NanoTimeUtils.getTimestamp(nt, skipConversion); - return new TimestampWritable(ts); + return new TimestampWritableV2(ts); } }; } }, - EDATE_CONVERTER(DateWritable.class) { + EDATE_CONVERTER(DateWritableV2.class) { @Override PrimitiveConverter getConverter(final PrimitiveType type, final int index, final ConverterParent parent, TypeInfo hiveTypeInfo) { return new PrimitiveConverter() { @Override public void addInt(final int value) { - parent.set(index, new DateWritable(value)); + parent.set(index, new DateWritableV2(value)); } }; } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/ParquetStringInspector.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/ParquetStringInspector.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/ParquetStringInspector.java index dead324..106ace0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/ParquetStringInspector.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/ParquetStringInspector.java @@ -18,7 +18,7 @@ import java.nio.charset.CharacterCodingException; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaStringObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableStringObjectInspector; import org.apache.hadoop.io.BytesWritable; @@ -29,7 +29,7 @@ import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.BooleanWritable; /** - * The ParquetStringInspector inspects a BytesWritable, TimestampWritable, HiveDecimalWritable, + * The ParquetStringInspector inspects a BytesWritable, TimestampWritableV2, HiveDecimalWritable, * DoubleWritable, FloatWritable, LongWritable, IntWritable, and BooleanWritable to give a Text * or String. * @@ -58,7 +58,7 @@ public class ParquetStringInspector extends JavaStringObjectInspector implements return new Text((String) o); } - if ((o instanceof TimestampWritable) || (o instanceof HiveDecimalWritable) + if ((o instanceof TimestampWritableV2) || (o instanceof HiveDecimalWritable) || (o instanceof DoubleWritable) || (o instanceof FloatWritable) || (o instanceof LongWritable) || (o instanceof IntWritable) || (o instanceof BooleanWritable)) { @@ -90,7 +90,7 @@ public class ParquetStringInspector extends JavaStringObjectInspector implements return (String) o; } - if ((o instanceof TimestampWritable) || (o instanceof HiveDecimalWritable) + if ((o instanceof TimestampWritableV2) || (o instanceof HiveDecimalWritable) || (o instanceof DoubleWritable) || (o instanceof FloatWritable) || (o instanceof LongWritable) || (o instanceof IntWritable) || (o instanceof BooleanWritable)) { http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTimeUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTimeUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTimeUtils.java index 677fb53..bf78d8c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTimeUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTimeUtils.java @@ -13,12 +13,13 @@ */ package org.apache.hadoop.hive.ql.io.parquet.timestamp; -import java.sql.Timestamp; import java.util.Calendar; import java.util.GregorianCalendar; import java.util.TimeZone; import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hive.common.type.Timestamp; + import jodd.datetime.JDateTime; /** @@ -58,7 +59,7 @@ public class NanoTimeUtils { public static NanoTime getNanoTime(Timestamp ts, boolean skipConversion) { Calendar calendar = getCalendar(skipConversion); - calendar.setTime(ts); + calendar.setTimeInMillis(ts.toEpochMilli()); int year = calendar.get(Calendar.YEAR); if (calendar.get(Calendar.ERA) == GregorianCalendar.BC) { year = 1 - year; @@ -106,8 +107,7 @@ public class NanoTimeUtils { calendar.set(Calendar.HOUR_OF_DAY, hour); calendar.set(Calendar.MINUTE, minutes); calendar.set(Calendar.SECOND, seconds); - Timestamp ts = new Timestamp(calendar.getTimeInMillis()); - ts.setNanos((int) nanos); + Timestamp ts = Timestamp.ofEpochMilli(calendar.getTimeInMillis(), (int) nanos); return ts; } } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReader.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReader.java index 4fd4cfd..26a4511 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReader.java @@ -19,10 +19,10 @@ package org.apache.hadoop.hive.ql.io.parquet.vector; import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.parquet.column.Dictionary; import java.io.IOException; -import java.sql.Timestamp; /** * The interface to wrap the underlying Parquet dictionary and non dictionary encoded page reader. http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java index f5f19e1..9170e9f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.io.parquet.vector; import org.apache.hadoop.hive.common.type.HiveBaseChar; +import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTime; import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTimeUtils; @@ -41,7 +42,6 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.sql.Timestamp; import java.util.Arrays; /** http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java index e89a736..e4c6156 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java @@ -13,6 +13,7 @@ */ package org.apache.hadoop.hive.ql.io.parquet.vector; +import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; @@ -387,7 +388,7 @@ public class VectorizedPrimitiveColumnReader extends BaseVectorizedColumnReader switch (descriptor.getType()) { //INT64 is not yet supported case INT96: - c.set(rowId, dataColumn.readTimestamp()); + c.set(rowId, dataColumn.readTimestamp().toSqlTimestamp()); break; default: throw new IOException( @@ -510,7 +511,7 @@ public class VectorizedPrimitiveColumnReader extends BaseVectorizedColumnReader case TIMESTAMP: for (int i = rowId; i < rowId + num; ++i) { ((TimestampColumnVector) column) - .set(i, dictionary.readTimestamp((int) dictionaryIds.vector[i])); + .set(i, dictionary.readTimestamp((int) dictionaryIds.vector[i]).toSqlTimestamp()); } break; case INTERVAL_DAY_TIME: http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java index cf1210b..3d61c33 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java @@ -13,12 +13,12 @@ */ package org.apache.hadoop.hive.ql.io.parquet.write; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTimeUtils; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.io.ParquetHiveRecord; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; @@ -46,9 +46,9 @@ import org.apache.parquet.io.api.RecordConsumer; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.OriginalType; import org.apache.parquet.schema.Type; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.sql.Date; -import java.sql.Timestamp; import java.util.List; import java.util.Map; @@ -550,7 +550,7 @@ public class DataWritableWriter { @Override public void write(Object value) { Date vDate = inspector.getPrimitiveJavaObject(value); - recordConsumer.addInteger(DateWritable.dateToDays(vDate)); + recordConsumer.addInteger(DateWritableV2.dateToDays(vDate)); } } } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java index 44687ef..7af6dab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java @@ -54,7 +54,7 @@ import org.apache.hadoop.hive.ql.metadata.UniqueConstraint.UniqueConstraintCol; import org.apache.hadoop.hive.ql.metadata.ForeignKeyInfo.ForeignKeyCol; import org.apache.hadoop.hive.ql.plan.DescTableDesc; import org.apache.hadoop.hive.ql.plan.PlanUtils; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hive.common.util.HiveStringUtils; import com.google.common.collect.Lists; @@ -107,7 +107,7 @@ public final class MetaDataFormatUtils { return ""; } - DateWritable writableValue = new DateWritable((int) val.getDaysSinceEpoch()); + DateWritableV2 writableValue = new DateWritableV2((int) val.getDaysSinceEpoch()); return writableValue.toString(); } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java index 857f300..f3ec709 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java @@ -66,7 +66,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMin; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum; import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; @@ -180,7 +180,7 @@ public class StatsOptimizer extends Transform { enum DateSubType { DAYS {@Override - Object cast(long longValue) { return (new DateWritable((int)longValue)).get();} + Object cast(long longValue) { return (new DateWritableV2((int)longValue)).get();} }; abstract Object cast(long longValue); }
