http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java index 12af94e..d950991 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java @@ -18,8 +18,6 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.translator; import java.math.BigDecimal; -import java.sql.Date; -import java.sql.Timestamp; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; @@ -45,9 +43,11 @@ import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.util.DateString; import org.apache.calcite.util.TimeString; import org.apache.calcite.util.TimestampString; +import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; +import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.metadata.Hive;
http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java index 7cedab6..2ae015a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java @@ -46,12 +46,14 @@ import org.apache.calcite.util.ConversionUtil; import org.apache.calcite.util.DateString; import org.apache.calcite.util.NlsString; import org.apache.calcite.util.TimestampString; +import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.common.type.Decimal128; import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.common.type.TimestampTZ; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; @@ -99,14 +101,11 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import java.math.BigDecimal; import java.math.BigInteger; -import java.sql.Timestamp; import java.time.Instant; import java.util.ArrayList; import java.util.Calendar; -import java.util.Date; import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; import java.util.Map; public class RexNodeConverter { @@ -394,8 +393,6 @@ public class RexNodeConverter { GenericUDF udf = func.getGenericUDF(); if ((udf instanceof GenericUDFToChar) || (udf instanceof GenericUDFToVarchar) || (udf instanceof GenericUDFToDecimal) || (udf instanceof GenericUDFToDate) - // Calcite can not specify the scale for timestamp. As a result, all - // the millisecond part will be lost || (udf instanceof GenericUDFTimestamp) || (udf instanceof GenericUDFToTimestampLocalTZ) || (udf instanceof GenericUDFToBinary) || castExprUsingUDFBridge(udf)) { castExpr = cluster.getRexBuilder().makeAbstractCast( @@ -701,9 +698,9 @@ public class RexNodeConverter { calciteLiteral = rexBuilder.makeCharLiteral(asUnicodeString((String) value)); break; case DATE: - final Calendar cal = Calendar.getInstance(Locale.getDefault()); - cal.setTime((Date) value); - calciteLiteral = rexBuilder.makeDateLiteral(DateString.fromCalendarFields(cal)); + final Date date = (Date) value; + calciteLiteral = rexBuilder.makeDateLiteral( + DateString.fromDaysSinceEpoch(date.toEpochDay())); break; case TIMESTAMP: final TimestampString tsString; @@ -711,9 +708,7 @@ public class RexNodeConverter { tsString = TimestampString.fromCalendarFields((Calendar) value); } else { final Timestamp ts = (Timestamp) value; - final Calendar calt = Calendar.getInstance(Locale.getDefault()); - calt.setTimeInMillis(ts.getTime()); - tsString = TimestampString.fromCalendarFields(calt).withNanos(ts.getNanos()); + tsString = TimestampString.fromMillisSinceEpoch(ts.toEpochMilli()).withNanos(ts.getNanos()); } // Must call makeLiteral, not makeTimestampLiteral // to have the RexBuilder.roundTime logic kick in http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 596edde..ebea31d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -21,7 +21,7 @@ package org.apache.hadoop.hive.ql.parse; import java.io.IOException; import java.io.Serializable; import java.io.UnsupportedEncodingException; -import java.sql.Date; +import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -43,6 +43,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.FileUtils; +import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -60,7 +61,6 @@ import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryProperties; import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.cache.results.CacheUsage; -import org.apache.hadoop.hive.ql.cache.results.QueryResultsCache; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.FetchTask; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; @@ -96,7 +96,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCurrentTimestamp; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCurrentUser; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull; import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; @@ -109,9 +109,6 @@ import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; -import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; -import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; - /** * BaseSemanticAnalyzer. * @@ -2098,14 +2095,19 @@ public abstract class BaseSemanticAnalyzer { private static String normalizeDateCol( Object colValue, String originalColSpec) throws SemanticException { Date value; - if (colValue instanceof DateWritable) { - value = ((DateWritable) colValue).get(false); // Time doesn't matter. + if (colValue instanceof DateWritableV2) { + value = ((DateWritableV2) colValue).get(); // Time doesn't matter. } else if (colValue instanceof Date) { value = (Date) colValue; } else { throw new SemanticException("Unexpected date type " + colValue.getClass()); } - return MetaStoreUtils.PARTITION_DATE_FORMAT.get().format(value); + try { + return MetaStoreUtils.PARTITION_DATE_FORMAT.get().format( + MetaStoreUtils.PARTITION_DATE_FORMAT.get().parse(value.toString())); + } catch (ParseException e) { + throw new SemanticException(e); + } } protected WriteEntity toWriteEntity(String location) throws SemanticException { http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index 2506172..d8c7d7f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -19,8 +19,7 @@ package org.apache.hadoop.hive.ql.parse; import java.math.BigDecimal; -import java.sql.Date; -import java.sql.Timestamp; + import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -33,10 +32,12 @@ import java.util.Stack; import org.apache.calcite.rel.RelNode; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.math.NumberUtils; +import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; +import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.common.type.TimestampTZUtil; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.ErrorMsg; http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java index 607545d..31c9682 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java @@ -35,7 +35,7 @@ import org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataI import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector; import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -197,10 +197,10 @@ public class ColumnStatisticsObjTranslator { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDateStats().setNumDVs(v); } else if (fName.equals("max")) { - DateWritable v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); + DateWritableV2 v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); statsObj.getStatsData().getDateStats().setHighValue(new Date(v.getDays())); } else if (fName.equals("min")) { - DateWritable v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); + DateWritableV2 v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); statsObj.getStatsData().getDateStats().setLowValue(new Date(v.getDays())); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateFloor.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateFloor.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateFloor.java index 21164b7..2fb8844 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateFloor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateFloor.java @@ -25,10 +25,11 @@ import java.util.Iterator; import java.util.Map; import java.util.NoSuchElementException; +import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.common.type.TimestampTZ; import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.joda.time.Chronology; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -46,26 +47,22 @@ import com.google.common.collect.ImmutableMap; public abstract class UDFDateFloor extends UDF { private final QueryGranularity granularity; - private final TimestampWritable resultTS; + private final TimestampWritableV2 resultTS; private final TimestampLocalTZWritable resultTSLTZ; public UDFDateFloor(String granularity) { this.granularity = QueryGranularity.fromString(granularity); - this.resultTS = new TimestampWritable(); + this.resultTS = new TimestampWritableV2(); this.resultTSLTZ = new TimestampLocalTZWritable(); } - public TimestampWritable evaluate(TimestampWritable t) { + public TimestampWritableV2 evaluate(TimestampWritableV2 t) { if (t == null) { return null; } - final long originalTimestamp = t.getTimestamp().getTime(); // default - final long originalTimestampUTC = new DateTime(originalTimestamp) - .withZoneRetainFields(DateTimeZone.UTC).getMillis(); // default -> utc - final long newTimestampUTC = granularity.truncate(originalTimestampUTC); // utc - final long newTimestamp = new DateTime(newTimestampUTC, DateTimeZone.UTC) - .withZoneRetainFields(DateTimeZone.getDefault()).getMillis(); // utc -> default - resultTS.setTime(newTimestamp); + final long originalTimestamp = t.getTimestamp().toEpochMilli(); + final long newTimestamp = granularity.truncate(originalTimestamp); + resultTS.set(Timestamp.ofEpochMilli(newTimestamp)); return resultTS; } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfMonth.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfMonth.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfMonth.java index f774954..72fa263 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfMonth.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfMonth.java @@ -18,23 +18,26 @@ package org.apache.hadoop.hive.ql.udf; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; - +import org.apache.hadoop.hive.common.type.Date; +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.ql.exec.Description; -import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFDayOfMonthDate; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFDayOfMonthString; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFDayOfMonthTimestamp; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.NDV; -import org.apache.hadoop.hive.serde2.io.DateWritable; -import org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; + +import java.util.Calendar; +import java.util.TimeZone; /** * UDFDayOfMonth. @@ -51,66 +54,81 @@ import org.apache.hadoop.io.Text; + " > SELECT _FUNC_('2009-07-30') FROM src LIMIT 1;\n" + " 30") @VectorizedExpressions({VectorUDFDayOfMonthDate.class, VectorUDFDayOfMonthString.class, VectorUDFDayOfMonthTimestamp.class}) @NDV(maxNdv = 31) -public class UDFDayOfMonth extends UDF { - private final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); - private final Calendar calendar = Calendar.getInstance(); - - private final IntWritable result = new IntWritable(); +public class UDFDayOfMonth extends GenericUDF { - public UDFDayOfMonth() { - } + private transient ObjectInspectorConverters.Converter[] converters = new ObjectInspectorConverters.Converter[1]; + private transient PrimitiveObjectInspector.PrimitiveCategory[] inputTypes = new PrimitiveObjectInspector.PrimitiveCategory[1]; + private final IntWritable output = new IntWritable(); - /** - * Get the day of month from a date string. - * - * @param dateString - * the dateString in the format of "yyyy-MM-dd HH:mm:ss" or - * "yyyy-MM-dd". - * @return an int from 1 to 31. null if the dateString is not a valid date - * string. - */ - public IntWritable evaluate(Text dateString) { + private final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - if (dateString == null) { - return null; + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + checkArgsSize(arguments, 1, 1); + checkArgPrimitive(arguments, 0); + switch (((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory()) { + case INTERVAL_DAY_TIME: + inputTypes[0] = PrimitiveObjectInspector.PrimitiveCategory.INTERVAL_DAY_TIME; + converters[0] = ObjectInspectorConverters.getConverter( + arguments[0], PrimitiveObjectInspectorFactory.writableHiveIntervalDayTimeObjectInspector); + break; + case STRING: + case CHAR: + case VARCHAR: + case DATE: + case TIMESTAMP: + case TIMESTAMPLOCALTZ: + case VOID: + obtainDateConverter(arguments, 0, inputTypes, converters); + break; + default: + // build error message + StringBuilder sb = new StringBuilder(); + sb.append(getFuncName()); + sb.append(" does not take "); + sb.append(((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory()); + sb.append(" type"); + throw new UDFArgumentTypeException(0, sb.toString()); } - try { - Date date = formatter.parse(dateString.toString()); - calendar.setTime(date); - result.set(calendar.get(Calendar.DAY_OF_MONTH)); - return result; - } catch (ParseException e) { - return null; - } + ObjectInspector outputOI = PrimitiveObjectInspectorFactory.writableIntObjectInspector; + return outputOI; } - public IntWritable evaluate(DateWritable d) { - if (d == null) { - return null; + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + switch (inputTypes[0]) { + case INTERVAL_DAY_TIME: + HiveIntervalDayTime intervalDayTime = getIntervalDayTimeValue(arguments, 0, inputTypes, converters); + if (intervalDayTime == null) { + return null; + } + output.set(intervalDayTime.getDays()); + break; + case STRING: + case CHAR: + case VARCHAR: + case DATE: + case TIMESTAMP: + case TIMESTAMPLOCALTZ: + case VOID: + Date date = getDateValue(arguments, 0, inputTypes, converters); + if (date == null) { + return null; + } + calendar.setTimeInMillis(date.toEpochMilli()); + output.set(calendar.get(Calendar.DAY_OF_MONTH)); } - - calendar.setTime(d.get(false)); // Time doesn't matter. - result.set(calendar.get(Calendar.DAY_OF_MONTH)); - return result; + return output; } - public IntWritable evaluate(TimestampWritable t) { - if (t == null) { - return null; - } - - calendar.setTime(t.getTimestamp()); - result.set(calendar.get(Calendar.DAY_OF_MONTH)); - return result; + @Override + protected String getFuncName() { + return "day"; } - public IntWritable evaluate(HiveIntervalDayTimeWritable i) { - if (i == null) { - return null; - } - - result.set(i.getHiveIntervalDayTime().getDays()); - return result; + @Override + public String getDisplayString(String[] children) { + return getStandardDisplayString(getFuncName(), children); } } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfWeek.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfWeek.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfWeek.java index 88e6d94..defa9d1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfWeek.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfWeek.java @@ -18,11 +18,7 @@ package org.apache.hadoop.hive.ql.udf; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; - +import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; @@ -30,8 +26,8 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFDayOfWeekDate; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFDayOfWeekString; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFDayOfWeekTimestamp; import org.apache.hadoop.hive.ql.udf.generic.NDV; -import org.apache.hadoop.hive.serde2.io.DateWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; @@ -51,8 +47,6 @@ import org.apache.hadoop.io.Text; @VectorizedExpressions({VectorUDFDayOfWeekDate.class, VectorUDFDayOfWeekString.class, VectorUDFDayOfWeekTimestamp.class}) @NDV(maxNdv = 7) public class UDFDayOfWeek extends UDF { - private final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); - private final Calendar calendar = Calendar.getInstance(); private final IntWritable result = new IntWritable(); @@ -73,32 +67,29 @@ public class UDFDayOfWeek extends UDF { return null; } try { - Date date = formatter.parse(dateString.toString()); - calendar.setTime(date); - result.set(calendar.get(Calendar.DAY_OF_WEEK)); + Date date = Date.valueOf(dateString.toString()); + result.set(date.getDayOfWeek()); return result; - } catch (ParseException e) { + } catch (IllegalArgumentException e) { return null; } } - public IntWritable evaluate(DateWritable d) { + public IntWritable evaluate(DateWritableV2 d) { if (d == null) { return null; } - calendar.setTime(d.get(false)); // Time doesn't matter. - result.set(calendar.get(Calendar.DAY_OF_WEEK)); + result.set(d.get().getDayOfWeek()); return result; } - public IntWritable evaluate(TimestampWritable t) { + public IntWritable evaluate(TimestampWritableV2 t) { if (t == null) { return null; } - calendar.setTime(t.getTimestamp()); - result.set(calendar.get(Calendar.DAY_OF_WEEK)); + result.set(t.getTimestamp().getDayOfWeek()); return result; } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFFromUnixTime.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFFromUnixTime.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFFromUnixTime.java index 8f531fd..3cee0c1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFFromUnixTime.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFFromUnixTime.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.udf; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.TimeZone; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; @@ -119,6 +120,7 @@ public class UDFFromUnixTime extends UDF { private Text eval(long unixtime, Text format) { if (!format.equals(lastFormat)) { formatter = new SimpleDateFormat(format.toString()); + formatter.setTimeZone(TimeZone.getTimeZone("UTC")); lastFormat.set(format); } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFHour.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFHour.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFHour.java index a0c4e96..f906f36 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFHour.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFHour.java @@ -18,22 +18,26 @@ package org.apache.hadoop.hive.ql.udf; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; - +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; +import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.ql.exec.Description; -import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFHourDate; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFHourString; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFHourTimestamp; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.NDV; -import org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; + +import java.util.Calendar; +import java.util.TimeZone; /** * UDFHour. @@ -51,62 +55,82 @@ import org.apache.hadoop.io.Text; + " > SELECT _FUNC_('12:58:59') FROM src LIMIT 1;\n" + " 12") @VectorizedExpressions({VectorUDFHourDate.class, VectorUDFHourString.class, VectorUDFHourTimestamp.class}) @NDV(maxNdv = 24) -public class UDFHour extends UDF { - private final SimpleDateFormat formatter1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - private final SimpleDateFormat formatter2 = new SimpleDateFormat("HH:mm:ss"); - private final Calendar calendar = Calendar.getInstance(); +public class UDFHour extends GenericUDF { - private final IntWritable result = new IntWritable(); + private transient ObjectInspectorConverters.Converter[] converters = new ObjectInspectorConverters.Converter[1]; + private transient PrimitiveObjectInspector.PrimitiveCategory[] inputTypes = new PrimitiveObjectInspector.PrimitiveCategory[1]; + private final IntWritable output = new IntWritable(); - public UDFHour() { - } + private final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - /** - * Get the hour from a date string. - * - * @param dateString - * the dateString in the format of "yyyy-MM-dd HH:mm:ss" or - * "yyyy-MM-dd". - * @return an int from 0 to 23. null if the dateString is not a valid date - * string. - */ - public IntWritable evaluate(Text dateString) { - if (dateString == null) { - return null; + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + checkArgsSize(arguments, 1, 1); + checkArgPrimitive(arguments, 0); + switch (((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory()) { + case INTERVAL_DAY_TIME: + inputTypes[0] = PrimitiveObjectInspector.PrimitiveCategory.INTERVAL_DAY_TIME; + converters[0] = ObjectInspectorConverters.getConverter( + arguments[0], PrimitiveObjectInspectorFactory.writableHiveIntervalDayTimeObjectInspector); + break; + case STRING: + case CHAR: + case VARCHAR: + case DATE: + case TIMESTAMP: + case TIMESTAMPLOCALTZ: + case VOID: + obtainTimestampConverter(arguments, 0, inputTypes, converters); + break; + default: + // build error message + StringBuilder sb = new StringBuilder(); + sb.append(getFuncName()); + sb.append(" does not take "); + sb.append(((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory()); + sb.append(" type"); + throw new UDFArgumentTypeException(0, sb.toString()); } - try { - Date date = null; - try { - date = formatter1.parse(dateString.toString()); - } catch (ParseException e) { - date = formatter2.parse(dateString.toString()); - } - calendar.setTime(date); - result.set(calendar.get(Calendar.HOUR_OF_DAY)); - return result; - } catch (ParseException e) { - return null; - } + ObjectInspector outputOI = PrimitiveObjectInspectorFactory.writableIntObjectInspector; + return outputOI; } - public IntWritable evaluate(TimestampWritable t) { - if (t == null) { - return null; + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + switch (inputTypes[0]) { + case INTERVAL_DAY_TIME: + HiveIntervalDayTime intervalDayTime = getIntervalDayTimeValue(arguments, 0, inputTypes, converters); + if (intervalDayTime == null) { + return null; + } + output.set(intervalDayTime.getHours()); + break; + case STRING: + case CHAR: + case VARCHAR: + case DATE: + case TIMESTAMP: + case TIMESTAMPLOCALTZ: + case VOID: + Timestamp ts = getTimestampValue(arguments, 0, converters); + if (ts == null) { + return null; + } + calendar.setTimeInMillis(ts.toEpochMilli()); + output.set(calendar.get(Calendar.HOUR_OF_DAY)); } - - calendar.setTime(t.getTimestamp()); - result.set(calendar.get(Calendar.HOUR_OF_DAY)); - return result; + return output; } - public IntWritable evaluate(HiveIntervalDayTimeWritable i) { - if (i == null) { - return null; - } + @Override + protected String getFuncName() { + return "hour"; + } - result.set(i.getHiveIntervalDayTime().getHours()); - return result; + @Override + public String getDisplayString(String[] children) { + return getStandardDisplayString(getFuncName(), children); } } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMinute.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMinute.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMinute.java index 306d458..2e62173 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMinute.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMinute.java @@ -18,22 +18,26 @@ package org.apache.hadoop.hive.ql.udf; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; - +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; +import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.ql.exec.Description; -import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFMinuteDate; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFMinuteString; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFMinuteTimestamp; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.NDV; -import org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; + +import java.util.Calendar; +import java.util.TimeZone; /** * UDFMinute. @@ -51,62 +55,82 @@ import org.apache.hadoop.io.Text; + " > SELECT _FUNC_('12:58:59') FROM src LIMIT 1;\n" + " 58") @VectorizedExpressions({VectorUDFMinuteDate.class, VectorUDFMinuteString.class, VectorUDFMinuteTimestamp.class}) @NDV(maxNdv = 60) -public class UDFMinute extends UDF { - private final SimpleDateFormat formatter1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - private final SimpleDateFormat formatter2 = new SimpleDateFormat("HH:mm:ss"); - private final Calendar calendar = Calendar.getInstance(); +public class UDFMinute extends GenericUDF { - private final IntWritable result = new IntWritable(); + private transient ObjectInspectorConverters.Converter[] converters = new ObjectInspectorConverters.Converter[1]; + private transient PrimitiveObjectInspector.PrimitiveCategory[] inputTypes = new PrimitiveObjectInspector.PrimitiveCategory[1]; + private final IntWritable output = new IntWritable(); - public UDFMinute() { - } + private final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - /** - * Get the minute from a date string. - * - * @param dateString - * the dateString in the format of "yyyy-MM-dd HH:mm:ss" or - * "yyyy-MM-dd". - * @return an int from 0 to 59. null if the dateString is not a valid date - * string. - */ - public IntWritable evaluate(Text dateString) { - if (dateString == null) { - return null; + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + checkArgsSize(arguments, 1, 1); + checkArgPrimitive(arguments, 0); + switch (((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory()) { + case INTERVAL_DAY_TIME: + inputTypes[0] = PrimitiveObjectInspector.PrimitiveCategory.INTERVAL_DAY_TIME; + converters[0] = ObjectInspectorConverters.getConverter( + arguments[0], PrimitiveObjectInspectorFactory.writableHiveIntervalDayTimeObjectInspector); + break; + case STRING: + case CHAR: + case VARCHAR: + case DATE: + case TIMESTAMP: + case TIMESTAMPLOCALTZ: + case VOID: + obtainTimestampConverter(arguments, 0, inputTypes, converters); + break; + default: + // build error message + StringBuilder sb = new StringBuilder(); + sb.append(getFuncName()); + sb.append(" does not take "); + sb.append(((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory()); + sb.append(" type"); + throw new UDFArgumentTypeException(0, sb.toString()); } - try { - Date date = null; - try { - date = formatter1.parse(dateString.toString()); - } catch (ParseException e) { - date = formatter2.parse(dateString.toString()); - } - calendar.setTime(date); - result.set(calendar.get(Calendar.MINUTE)); - return result; - } catch (ParseException e) { - return null; - } + ObjectInspector outputOI = PrimitiveObjectInspectorFactory.writableIntObjectInspector; + return outputOI; } - public IntWritable evaluate(TimestampWritable t) { - if (t == null) { - return null; + @Override + public Object evaluate(GenericUDF.DeferredObject[] arguments) throws HiveException { + switch (inputTypes[0]) { + case INTERVAL_DAY_TIME: + HiveIntervalDayTime intervalDayTime = getIntervalDayTimeValue(arguments, 0, inputTypes, converters); + if (intervalDayTime == null) { + return null; + } + output.set(intervalDayTime.getMinutes()); + break; + case STRING: + case CHAR: + case VARCHAR: + case DATE: + case TIMESTAMP: + case TIMESTAMPLOCALTZ: + case VOID: + Timestamp ts = getTimestampValue(arguments, 0, converters); + if (ts == null) { + return null; + } + calendar.setTimeInMillis(ts.toEpochMilli()); + output.set(calendar.get(Calendar.MINUTE)); } - - calendar.setTime(t.getTimestamp()); - result.set(calendar.get(Calendar.MINUTE)); - return result; + return output; } - public IntWritable evaluate(HiveIntervalDayTimeWritable i) { - if (i == null) { - return null; - } + @Override + protected String getFuncName() { + return "minute"; + } - result.set(i.getHiveIntervalDayTime().getMinutes()); - return result; + @Override + public String getDisplayString(String[] children) { + return getStandardDisplayString(getFuncName(), children); } } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java index 7995934..1a8b2da 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java @@ -18,23 +18,26 @@ package org.apache.hadoop.hive.ql.udf; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; - +import org.apache.hadoop.hive.common.type.Date; +import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.ql.exec.Description; -import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFMonthDate; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFMonthString; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFMonthTimestamp; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.NDV; -import org.apache.hadoop.hive.serde2.io.DateWritable; -import org.apache.hadoop.hive.serde2.io.HiveIntervalYearMonthWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; + +import java.util.Calendar; +import java.util.TimeZone; /** * UDFMonth. @@ -51,64 +54,82 @@ import org.apache.hadoop.io.Text; + " > SELECT _FUNC_('2009-07-30') FROM src LIMIT 1;\n" + " 7") @VectorizedExpressions({VectorUDFMonthDate.class, VectorUDFMonthString.class, VectorUDFMonthTimestamp.class}) @NDV(maxNdv = 31) -public class UDFMonth extends UDF { - private final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); - private final Calendar calendar = Calendar.getInstance(); +public class UDFMonth extends GenericUDF { - private final IntWritable result = new IntWritable(); + private transient ObjectInspectorConverters.Converter[] converters = new ObjectInspectorConverters.Converter[1]; + private transient PrimitiveObjectInspector.PrimitiveCategory[] inputTypes = new PrimitiveObjectInspector.PrimitiveCategory[1]; + private final IntWritable output = new IntWritable(); - public UDFMonth() { - } + private final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - /** - * Get the month from a date string. - * - * @param dateString - * the dateString in the format of "yyyy-MM-dd HH:mm:ss" or - * "yyyy-MM-dd". - * @return an int from 1 to 12. null if the dateString is not a valid date - * string. - */ - public IntWritable evaluate(Text dateString) { - if (dateString == null) { - return null; - } - try { - Date date = formatter.parse(dateString.toString()); - calendar.setTime(date); - result.set(1 + calendar.get(Calendar.MONTH)); - return result; - } catch (ParseException e) { - return null; - } - } - public IntWritable evaluate(DateWritable d) { - if (d == null) { - return null; + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + checkArgsSize(arguments, 1, 1); + checkArgPrimitive(arguments, 0); + switch (((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory()) { + case INTERVAL_YEAR_MONTH: + inputTypes[0] = PrimitiveObjectInspector.PrimitiveCategory.INTERVAL_YEAR_MONTH; + converters[0] = ObjectInspectorConverters.getConverter( + arguments[0], PrimitiveObjectInspectorFactory.writableHiveIntervalYearMonthObjectInspector); + break; + case STRING: + case CHAR: + case VARCHAR: + case DATE: + case TIMESTAMP: + case TIMESTAMPLOCALTZ: + case VOID: + obtainDateConverter(arguments, 0, inputTypes, converters); + break; + default: + // build error message + StringBuilder sb = new StringBuilder(); + sb.append(getFuncName()); + sb.append(" does not take "); + sb.append(((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory()); + sb.append(" type"); + throw new UDFArgumentTypeException(0, sb.toString()); } - calendar.setTime(d.get(false)); // Time doesn't matter. - result.set(1 + calendar.get(Calendar.MONTH)); - return result; + ObjectInspector outputOI = PrimitiveObjectInspectorFactory.writableIntObjectInspector; + return outputOI; } - public IntWritable evaluate(TimestampWritable t) { - if (t == null) { - return null; + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + switch (inputTypes[0]) { + case INTERVAL_YEAR_MONTH: + HiveIntervalYearMonth intervalYearMonth = getIntervalYearMonthValue(arguments, 0, inputTypes, converters); + if (intervalYearMonth == null) { + return null; + } + output.set(intervalYearMonth.getMonths()); + break; + case STRING: + case CHAR: + case VARCHAR: + case DATE: + case TIMESTAMP: + case TIMESTAMPLOCALTZ: + case VOID: + Date date = getDateValue(arguments, 0, inputTypes, converters); + if (date == null) { + return null; + } + calendar.setTimeInMillis(date.toEpochMilli()); + output.set(1 + calendar.get(Calendar.MONTH)); } - - calendar.setTime(t.getTimestamp()); - result.set(1 + calendar.get(Calendar.MONTH)); - return result; + return output; } - public IntWritable evaluate(HiveIntervalYearMonthWritable i) { - if (i == null) { - return null; - } + @Override + protected String getFuncName() { + return "month"; + } - result.set(i.getHiveIntervalYearMonth().getMonths()); - return result; + @Override + public String getDisplayString(String[] children) { + return getStandardDisplayString(getFuncName(), children); } } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSecond.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSecond.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSecond.java index 5bf8b24..31a8529 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSecond.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSecond.java @@ -18,25 +18,27 @@ package org.apache.hadoop.hive.ql.udf; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; - import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; +import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.ql.exec.Description; -import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFSecondDate; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFSecondString; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFSecondTimestamp; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.NDV; -import org.apache.hadoop.hive.serde2.io.DoubleWritable; -import org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; -import org.apache.hive.common.util.DateUtils; + +import java.util.Calendar; +import java.util.TimeZone; + /** * UDFSecond. @@ -54,64 +56,82 @@ import org.apache.hive.common.util.DateUtils; + " > SELECT _FUNC_('12:58:59') FROM src LIMIT 1;\n" + " 59") @VectorizedExpressions({VectorUDFSecondDate.class, VectorUDFSecondString.class, VectorUDFSecondTimestamp.class}) @NDV(maxNdv = 60) -public class UDFSecond extends UDF { - private final SimpleDateFormat formatter1 = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - private final SimpleDateFormat formatter2 = new SimpleDateFormat("HH:mm:ss"); - private final Calendar calendar = Calendar.getInstance(); +public class UDFSecond extends GenericUDF { - private final IntWritable result = new IntWritable(); + private transient ObjectInspectorConverters.Converter[] converters = new ObjectInspectorConverters.Converter[1]; + private transient PrimitiveObjectInspector.PrimitiveCategory[] inputTypes = new PrimitiveObjectInspector.PrimitiveCategory[1]; + private final IntWritable output = new IntWritable(); - public UDFSecond() { - } + private final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - /** - * Get the minute from a date string. - * - * @param dateString - * the dateString in the format of "yyyy-MM-dd HH:mm:ss" or - * "yyyy-MM-dd". - * @return an int from 0 to 59. null if the dateString is not a valid date - * string. - */ - public IntWritable evaluate(Text dateString) { - if (dateString == null) { - return null; + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + checkArgsSize(arguments, 1, 1); + checkArgPrimitive(arguments, 0); + switch (((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory()) { + case INTERVAL_DAY_TIME: + inputTypes[0] = PrimitiveObjectInspector.PrimitiveCategory.INTERVAL_DAY_TIME; + converters[0] = ObjectInspectorConverters.getConverter( + arguments[0], PrimitiveObjectInspectorFactory.writableHiveIntervalDayTimeObjectInspector); + break; + case STRING: + case CHAR: + case VARCHAR: + case DATE: + case TIMESTAMP: + case TIMESTAMPLOCALTZ: + case VOID: + obtainTimestampConverter(arguments, 0, inputTypes, converters); + break; + default: + // build error message + StringBuilder sb = new StringBuilder(); + sb.append(getFuncName()); + sb.append(" does not take "); + sb.append(((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory()); + sb.append(" type"); + throw new UDFArgumentTypeException(0, sb.toString()); } - try { - Date date = null; - try { - date = formatter1.parse(dateString.toString()); - } catch (ParseException e) { - date = formatter2.parse(dateString.toString()); - } - calendar.setTime(date); - result.set(calendar.get(Calendar.SECOND)); - return result; - } catch (ParseException e) { - return null; - } + ObjectInspector outputOI = PrimitiveObjectInspectorFactory.writableIntObjectInspector; + return outputOI; } - public IntWritable evaluate(TimestampWritable t) { - if (t == null) { - return null; + @Override + public Object evaluate(GenericUDF.DeferredObject[] arguments) throws HiveException { + switch (inputTypes[0]) { + case INTERVAL_DAY_TIME: + HiveIntervalDayTime intervalDayTime = getIntervalDayTimeValue(arguments, 0, inputTypes, converters); + if (intervalDayTime == null) { + return null; + } + output.set(intervalDayTime.getSeconds()); + break; + case STRING: + case CHAR: + case VARCHAR: + case DATE: + case TIMESTAMP: + case TIMESTAMPLOCALTZ: + case VOID: + Timestamp ts = getTimestampValue(arguments, 0, converters); + if (ts == null) { + return null; + } + calendar.setTimeInMillis(ts.toEpochMilli()); + output.set(calendar.get(Calendar.SECOND)); } - - calendar.setTime(t.getTimestamp()); - result.set(calendar.get(Calendar.SECOND)); - return result; + return output; } - public IntWritable evaluate(HiveIntervalDayTimeWritable i) { - if (i == null) { - return null; - } + @Override + protected String getFuncName() { + return "second"; + } - HiveIntervalDayTime idt = i.getHiveIntervalDayTime(); - result.set(idt.getSeconds()); - return result; + @Override + public String getDisplayString(String[] children) { + return getStandardDisplayString(getFuncName(), children); } } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java index 3ac7a06..a7f4bf1 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java @@ -30,11 +30,11 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastLongToBooleanVi import org.apache.hadoop.hive.ql.exec.vector.expressions.CastDateToBoolean; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastTimestampToBoolean; import org.apache.hadoop.hive.serde2.io.ByteWritable; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.FloatWritable; @@ -179,12 +179,12 @@ public class UDFToBoolean extends UDF { return booleanWritable; } - public BooleanWritable evaluate(DateWritable d) { + public BooleanWritable evaluate(DateWritableV2 d) { // date value to boolean doesn't make any sense. return null; } - public BooleanWritable evaluate(TimestampWritable i) { + public BooleanWritable evaluate(TimestampWritableV2 i) { if (i == null) { return null; } else { http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java index 1128b32..8f4ec3b 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java @@ -28,7 +28,7 @@ import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.apache.hadoop.hive.serde2.lazy.LazyByte; import org.apache.hadoop.hive.serde2.lazy.LazyUtils; import org.apache.hadoop.io.BooleanWritable; @@ -183,7 +183,7 @@ public class UDFToByte extends UDF { } } - public ByteWritable evaluate(TimestampWritable i) { + public ByteWritable evaluate(TimestampWritableV2 i) { if (i == null) { return null; } else { http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java index a8de3d5..7a01452 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java @@ -28,7 +28,7 @@ import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.apache.hadoop.hive.serde2.lazy.LazyUtils; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.FloatWritable; @@ -180,7 +180,7 @@ public class UDFToDouble extends UDF { } } - public DoubleWritable evaluate(TimestampWritable i) { + public DoubleWritable evaluate(TimestampWritableV2 i) { if (i == null) { return null; } else { http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java index 2872ff2..451b45f 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java @@ -28,7 +28,7 @@ import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.apache.hadoop.hive.serde2.lazy.LazyUtils; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.FloatWritable; @@ -181,7 +181,7 @@ public class UDFToFloat extends UDF { } } - public FloatWritable evaluate(TimestampWritable i) { + public FloatWritable evaluate(TimestampWritableV2 i) { if (i == null) { return null; } else { http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java index 748a688..4fe9c32 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java @@ -30,7 +30,7 @@ import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.apache.hadoop.hive.serde2.lazy.LazyInteger; import org.apache.hadoop.hive.serde2.lazy.LazyUtils; import org.apache.hadoop.io.BooleanWritable; @@ -193,7 +193,7 @@ public class UDFToInteger extends UDF { * The Timestamp value to convert * @return IntWritable */ - public IntWritable evaluate(TimestampWritable i) { + public IntWritable evaluate(TimestampWritableV2 i) { if (i == null) { return null; } else { http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java index e286652..b31eeb0 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java @@ -28,7 +28,7 @@ import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.apache.hadoop.hive.serde2.lazy.LazyLong; import org.apache.hadoop.hive.serde2.lazy.LazyUtils; import org.apache.hadoop.io.BooleanWritable; @@ -195,7 +195,7 @@ public class UDFToLong extends UDF { } } - public LongWritable evaluate(TimestampWritable i) { + public LongWritable evaluate(TimestampWritableV2 i) { if (i == null) { return null; } else { http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java index e003ff3..315789c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java @@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; -import org.apache.hadoop.hive.ql.exec.vector.expressions.CastDecimalToDouble; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastDecimalToLong; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastStringToLong; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastDoubleToLong; @@ -29,7 +28,7 @@ import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.apache.hadoop.hive.serde2.lazy.LazyShort; import org.apache.hadoop.hive.serde2.lazy.LazyUtils; import org.apache.hadoop.io.BooleanWritable; @@ -185,7 +184,7 @@ public class UDFToShort extends UDF { } } - public ShortWritable evaluate(TimestampWritable i) { + public ShortWritable evaluate(TimestampWritableV2 i) { if (i == null) { return null; } else { http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java index 557cb1c..a16d429 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java @@ -21,12 +21,12 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.serde2.ByteStream; import org.apache.hadoop.hive.serde2.io.ByteWritable; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.apache.hadoop.hive.serde2.lazy.LazyInteger; import org.apache.hadoop.hive.serde2.lazy.LazyLong; import org.apache.hadoop.io.BooleanWritable; @@ -135,7 +135,7 @@ public class UDFToString extends UDF { return i; } - public Text evaluate(DateWritable d) { + public Text evaluate(DateWritableV2 d) { if (d == null) { return null; } else { @@ -144,7 +144,7 @@ public class UDFToString extends UDF { } } - public Text evaluate(TimestampWritable i) { + public Text evaluate(TimestampWritableV2 i) { if (i == null) { return null; } else { http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFWeekOfYear.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFWeekOfYear.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFWeekOfYear.java index 18ed52d..18ca9a7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFWeekOfYear.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFWeekOfYear.java @@ -18,11 +18,11 @@ package org.apache.hadoop.hive.ql.udf; -import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.Calendar; -import java.util.Date; +import java.util.TimeZone; +import org.apache.hadoop.hive.common.type.Date; +import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; @@ -30,8 +30,8 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFWeekOfYearDate import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFWeekOfYearString; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFWeekOfYearTimestamp; import org.apache.hadoop.hive.ql.udf.generic.NDV; -import org.apache.hadoop.hive.serde2.io.DateWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; @@ -49,11 +49,12 @@ import org.apache.hadoop.io.Text; @VectorizedExpressions({VectorUDFWeekOfYearDate.class, VectorUDFWeekOfYearString.class, VectorUDFWeekOfYearTimestamp.class}) @NDV(maxNdv = 52) public class UDFWeekOfYear extends UDF { - private final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); - private final Calendar calendar = Calendar.getInstance(); private final IntWritable result = new IntWritable(); + private final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + + public UDFWeekOfYear() { calendar.setFirstDayOfWeek(Calendar.MONDAY); calendar.setMinimalDaysInFirstWeek(4); @@ -73,31 +74,32 @@ public class UDFWeekOfYear extends UDF { return null; } try { - Date date = formatter.parse(dateString.toString()); - calendar.setTime(date); + Date date = Date.valueOf(dateString.toString()); + calendar.setTimeInMillis(date.toEpochMilli()); result.set(calendar.get(Calendar.WEEK_OF_YEAR)); return result; - } catch (ParseException e) { + } catch (IllegalArgumentException e) { return null; } } - public IntWritable evaluate(DateWritable d) { + public IntWritable evaluate(DateWritableV2 d) { if (d == null) { return null; } - - calendar.setTime(d.get(false)); // Time doesn't matter. + Date date = d.get(); + calendar.setTimeInMillis(date.toEpochMilli()); result.set(calendar.get(Calendar.WEEK_OF_YEAR)); return result; } - public IntWritable evaluate(TimestampWritable t) { + public IntWritable evaluate(TimestampWritableV2 t) { if (t == null) { return null; } - calendar.setTime(t.getTimestamp()); + Timestamp ts = t.getTimestamp(); + calendar.setTimeInMillis(ts.toEpochMilli()); result.set(calendar.get(Calendar.WEEK_OF_YEAR)); return result; } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java index 8417591..fcbb57f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java @@ -18,23 +18,26 @@ package org.apache.hadoop.hive.ql.udf; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; - +import org.apache.hadoop.hive.common.type.Date; +import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.ql.exec.Description; -import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFYearDate; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFYearString; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFYearTimestamp; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.NDV; -import org.apache.hadoop.hive.serde2.io.DateWritable; -import org.apache.hadoop.hive.serde2.io.HiveIntervalYearMonthWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; + +import java.util.Calendar; +import java.util.TimeZone; /** * UDFYear. @@ -51,66 +54,82 @@ import org.apache.hadoop.io.Text; + " > SELECT _FUNC_('2009-07-30') FROM src LIMIT 1;\n" + " 2009") @VectorizedExpressions({VectorUDFYearDate.class, VectorUDFYearString.class, VectorUDFYearTimestamp.class}) @NDV(maxNdv = 20) // although technically its unbounded, its unlikely we will ever see ndv > 20 -public class UDFYear extends UDF { - private final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); - private final Calendar calendar = Calendar.getInstance(); +public class UDFYear extends GenericUDF { - private final IntWritable result = new IntWritable(); + private transient ObjectInspectorConverters.Converter[] converters = new ObjectInspectorConverters.Converter[1]; + private transient PrimitiveObjectInspector.PrimitiveCategory[] inputTypes = new PrimitiveObjectInspector.PrimitiveCategory[1]; + private final IntWritable output = new IntWritable(); - public UDFYear() { - } + private final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - /** - * Get the year from a date string. - * - * @param dateString - * the dateString in the format of "yyyy-MM-dd HH:mm:ss" or - * "yyyy-MM-dd". - * @return an int from 1 to 12. null if the dateString is not a valid date - * string. - */ - public IntWritable evaluate(Text dateString) { - if (dateString == null) { - return null; + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + checkArgsSize(arguments, 1, 1); + checkArgPrimitive(arguments, 0); + switch (((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory()) { + case INTERVAL_YEAR_MONTH: + inputTypes[0] = PrimitiveObjectInspector.PrimitiveCategory.INTERVAL_YEAR_MONTH; + converters[0] = ObjectInspectorConverters.getConverter( + arguments[0], PrimitiveObjectInspectorFactory.writableHiveIntervalYearMonthObjectInspector); + break; + case STRING: + case CHAR: + case VARCHAR: + case DATE: + case TIMESTAMP: + case TIMESTAMPLOCALTZ: + case VOID: + obtainDateConverter(arguments, 0, inputTypes, converters); + break; + default: + // build error message + StringBuilder sb = new StringBuilder(); + sb.append(getFuncName()); + sb.append(" does not take "); + sb.append(((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory()); + sb.append(" type"); + throw new UDFArgumentTypeException(0, sb.toString()); } - try { - Date date = formatter.parse(dateString.toString()); - calendar.setTime(date); - result.set(calendar.get(Calendar.YEAR)); - return result; - } catch (ParseException e) { - return null; - } + ObjectInspector outputOI = PrimitiveObjectInspectorFactory.writableIntObjectInspector; + return outputOI; } - public IntWritable evaluate(DateWritable d) { - if (d == null) { - return null; + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + switch (inputTypes[0]) { + case INTERVAL_YEAR_MONTH: + HiveIntervalYearMonth intervalYearMonth = getIntervalYearMonthValue(arguments, 0, inputTypes, converters); + if (intervalYearMonth == null) { + return null; + } + output.set(intervalYearMonth.getYears()); + break; + case STRING: + case CHAR: + case VARCHAR: + case DATE: + case TIMESTAMP: + case TIMESTAMPLOCALTZ: + case VOID: + Date date = getDateValue(arguments, 0, inputTypes, converters); + if (date == null) { + return null; + } + calendar.setTimeInMillis(date.toEpochMilli()); + output.set(calendar.get(Calendar.YEAR)); } - - calendar.setTime(d.get(false)); // Time doesn't matter. - result.set(calendar.get(Calendar.YEAR)); - return result; + return output; } - public IntWritable evaluate(TimestampWritable t) { - if (t == null) { - return null; - } - - calendar.setTime(t.getTimestamp()); - result.set(calendar.get(Calendar.YEAR)); - return result; + @Override + protected String getFuncName() { + return "year"; } - public IntWritable evaluate(HiveIntervalYearMonthWritable i) { - if (i == null) { - return null; - } - - result.set(i.getHiveIntervalYearMonth().getYears()); - return result; + @Override + public String getDisplayString(String[] children) { + return getStandardDisplayString(getFuncName(), children); } } http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/BaseMaskUDF.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/BaseMaskUDF.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/BaseMaskUDF.java index 5c67242..d69a4f7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/BaseMaskUDF.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/BaseMaskUDF.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.udf.generic; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; @@ -34,8 +35,6 @@ import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; -import java.sql.Date; - public abstract class BaseMaskUDF extends GenericUDF { private static final Log LOG = LogFactory.getLog(BaseMaskUDF.class); @@ -228,13 +227,13 @@ class ByteTransformerAdapter extends AbstractTransformerAdapter { class DateTransformerAdapter extends AbstractTransformerAdapter { final DateObjectInspector columnType; - final DateWritable writable; + final DateWritableV2 writable; public DateTransformerAdapter(DateObjectInspector columnType, AbstractTransformer transformer) { - this(columnType, transformer, new DateWritable()); + this(columnType, transformer, new DateWritableV2()); } - public DateTransformerAdapter(DateObjectInspector columnType, AbstractTransformer transformer, DateWritable writable) { + public DateTransformerAdapter(DateObjectInspector columnType, AbstractTransformer transformer, DateWritableV2 writable) { super(transformer); this.columnType = columnType; http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBloomFilter.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBloomFilter.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBloomFilter.java index ca8bc8f..a8bcc97 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBloomFilter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBloomFilter.java @@ -20,19 +20,19 @@ package org.apache.hadoop.hive.ql.udf.generic; import org.apache.hadoop.hive.common.io.NonSyncByteArrayInputStream; import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.exec.vector.VectorizedUDAFs; import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.*; -import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.*; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ColStatistics; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; import org.apache.hadoop.hive.ql.plan.Statistics; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -46,7 +46,6 @@ import org.apache.hive.common.util.BloomKFilter; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.sql.Timestamp; import java.util.List; /** @@ -197,14 +196,14 @@ public class GenericUDAFBloomFilter implements GenericUDAFResolver2 { bf.addBytes(scratchBuffer, startIdx, scratchBuffer.length - startIdx); break; case DATE: - DateWritable vDate = ((DateObjectInspector)inputOI). + DateWritableV2 vDate = ((DateObjectInspector)inputOI). getPrimitiveWritableObject(parameters[0]); bf.addLong(vDate.getDays()); break; case TIMESTAMP: Timestamp vTimeStamp = ((TimestampObjectInspector)inputOI). getPrimitiveJavaObject(parameters[0]); - bf.addLong(vTimeStamp.getTime()); + bf.addLong(vTimeStamp.toEpochMilli()); break; case CHAR: Text vChar = ((HiveCharObjectInspector)inputOI). http://git-wip-us.apache.org/repos/asf/hive/blob/c95136a0/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java index 2267589..dd365dd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java @@ -29,7 +29,7 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.util.JavaDataModel; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; @@ -1297,7 +1297,7 @@ public class GenericUDAFComputeStats extends AbstractGenericUDAFResolver { * High/low value will be saved in stats DB as long value representing days since epoch. */ public static class GenericUDAFDateStatsEvaluator - extends GenericUDAFNumericStatsEvaluator<DateWritable, DateObjectInspector> { + extends GenericUDAFNumericStatsEvaluator<DateWritableV2, DateObjectInspector> { @Override protected DateObjectInspector getValueObjectInspector() { @@ -1319,8 +1319,8 @@ public class GenericUDAFComputeStats extends AbstractGenericUDAFResolver { @Override protected void update(Object p, PrimitiveObjectInspector inputOI) { - // DateWritable is mutable, DateStatsAgg needs its own copy - DateWritable v = new DateWritable((DateWritable) inputOI.getPrimitiveWritableObject(p)); + // DateWritableV2 is mutable, DateStatsAgg needs its own copy + DateWritableV2 v = new DateWritableV2((DateWritableV2) inputOI.getPrimitiveWritableObject(p)); //Update min counter if new value is less than min seen so far if (min == null || v.compareTo(min) < 0) { @@ -1338,8 +1338,8 @@ public class GenericUDAFComputeStats extends AbstractGenericUDAFResolver { protected void updateMin(Object minValue, DateObjectInspector minFieldOI) { if ((minValue != null) && (min == null || min.compareTo(minFieldOI.getPrimitiveWritableObject(minValue)) > 0)) { - // DateWritable is mutable, DateStatsAgg needs its own copy - min = new DateWritable(minFieldOI.getPrimitiveWritableObject(minValue)); + // DateWritableV2 is mutable, DateStatsAgg needs its own copy + min = new DateWritableV2(minFieldOI.getPrimitiveWritableObject(minValue)); } } @@ -1347,8 +1347,8 @@ public class GenericUDAFComputeStats extends AbstractGenericUDAFResolver { protected void updateMax(Object maxValue, DateObjectInspector maxFieldOI) { if ((maxValue != null) && (max == null || max.compareTo(maxFieldOI.getPrimitiveWritableObject(maxValue)) < 0)) { - // DateWritable is mutable, DateStatsAgg needs its own copy - max = new DateWritable(maxFieldOI.getPrimitiveWritableObject(maxValue)); + // DateWritableV2 is mutable, DateStatsAgg needs its own copy + max = new DateWritableV2(maxFieldOI.getPrimitiveWritableObject(maxValue)); } } };
