http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritable.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritable.java index fdc64e7..305fdbe 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritable.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritable.java @@ -28,7 +28,6 @@ import java.text.SimpleDateFormat; import java.util.Date; import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.common.type.PisaTimestamp; import org.apache.hadoop.hive.serde2.ByteStream.RandomAccessOutput; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.VInt; @@ -323,7 +322,20 @@ public class TimestampWritable implements WritableComparable<TimestampWritable> return seconds + nanos / 1000000000; } + public static long getLong(Timestamp timestamp) { + return timestamp.getTime() / 1000; + } + /** + * + * @return double representation of the timestamp, accurate to nanoseconds + */ + public static double getDouble(Timestamp timestamp) { + double seconds, nanos; + seconds = millisToSeconds(timestamp.getTime()); + nanos = timestamp.getNanos(); + return seconds + nanos / 1000000000; + } public void readFields(DataInput in) throws IOException { in.readFully(internalBytes, 0, 4); @@ -543,6 +555,21 @@ public class TimestampWritable implements WritableComparable<TimestampWritable> return t; } + public HiveDecimal getHiveDecimal() { + if (timestampEmpty) { + populateTimestamp(); + } + return getHiveDecimal(timestamp); + } + + public static HiveDecimal getHiveDecimal(Timestamp timestamp) { + // The BigDecimal class recommends not converting directly from double to BigDecimal, + // so we convert through a string... + Double timestampDouble = TimestampWritable.getDouble(timestamp); + HiveDecimal result = HiveDecimal.create(timestampDouble.toString()); + return result; + } + /** * Converts the time in seconds or milliseconds to a timestamp. * @param time time in seconds or in milliseconds @@ -553,6 +580,17 @@ public class TimestampWritable implements WritableComparable<TimestampWritable> return new Timestamp(intToTimestampInSeconds ? time * 1000 : time); } + /** + * Converts the time in seconds or milliseconds to a timestamp. + * @param time time in seconds or in milliseconds + * @return the timestamp + */ + public static void setTimestampFromLong(Timestamp timestamp, long time, + boolean intToTimestampInSeconds) { + // If the time is in seconds, converts it to milliseconds first. + timestamp.setTime(intToTimestampInSeconds ? time * 1000 : time); + } + public static Timestamp doubleToTimestamp(double f) { long seconds = (long) f; @@ -576,6 +614,37 @@ public class TimestampWritable implements WritableComparable<TimestampWritable> return t; } + public static void setTimestampFromDouble(Timestamp timestamp, double f) { + // Otherwise, BigDecimal throws an exception. (Support vector operations that sometimes + // do work on double Not-a-Number NaN values). + if (Double.isNaN(f)) { + timestamp.setTime(0); + return; + } + // Algorithm used by TimestampWritable.doubleToTimestamp method. + // Allocates a BigDecimal object! + + long seconds = (long) f; + + // We must ensure the exactness of the double's fractional portion. + // 0.6 as the fraction part will be converted to 0.59999... and + // significantly reduce the savings from binary serialization + BigDecimal bd = new BigDecimal(String.valueOf(f)); + bd = bd.subtract(new BigDecimal(seconds)).multiply(new BigDecimal(1000000000)); + int nanos = bd.intValue(); + + // Convert to millis + long millis = seconds * 1000; + if (nanos < 0) { + millis -= 1000; + nanos += 1000000000; + } + timestamp.setTime(millis); + + // Set remaining fractional portion to nanos + timestamp.setNanos(nanos); + } + public static void setTimestamp(Timestamp t, byte[] bytes, int offset) { boolean hasDecimalOrSecondVInt = hasDecimalOrSecondVInt(bytes[offset]); long seconds = (long) TimestampWritable.getSeconds(bytes, offset);
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java ---------------------------------------------------------------------- diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java new file mode 100644 index 0000000..b891e27 --- /dev/null +++ b/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java @@ -0,0 +1,253 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.util.Date; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.hive.common.util.IntervalDayTimeUtils; + +import sun.util.calendar.BaseCalendar; + +/** + * Day-time interval type representing an offset in days/hours/minutes/seconds, + * with nanosecond precision. + * 1 day = 24 hours = 1440 minutes = 86400 seconds + */ +public class HiveIntervalDayTime implements Comparable<HiveIntervalDayTime> { + + // days/hours/minutes/seconds all represented as seconds + protected long totalSeconds; + protected int nanos; + + public HiveIntervalDayTime() { + } + + public HiveIntervalDayTime(int days, int hours, int minutes, int seconds, int nanos) { + set(days, hours, minutes, seconds, nanos); + } + + public HiveIntervalDayTime(long seconds, int nanos) { + set(seconds, nanos); + } + + public HiveIntervalDayTime(BigDecimal seconds) { + set(seconds); + } + + public HiveIntervalDayTime(HiveIntervalDayTime other) { + set(other.totalSeconds, other.nanos); + } + + public int getDays() { + return (int) TimeUnit.SECONDS.toDays(totalSeconds); + } + + public int getHours() { + return (int) (TimeUnit.SECONDS.toHours(totalSeconds) % TimeUnit.DAYS.toHours(1)); + } + + public int getMinutes() { + return (int) (TimeUnit.SECONDS.toMinutes(totalSeconds) % TimeUnit.HOURS.toMinutes(1)); + } + + public int getSeconds() { + return (int) (totalSeconds % TimeUnit.MINUTES.toSeconds(1)); + } + + public int getNanos() { + return nanos; + } + + /** + * Returns days/hours/minutes all converted into seconds. + * Nanos still need to be retrieved using getNanos() + * @return + */ + public long getTotalSeconds() { + return totalSeconds; + } + + /** + * + * @return double representation of the interval day time, accurate to nanoseconds + */ + public double getDouble() { + return totalSeconds + nanos / 1000000000; + } + + /** + * Ensures that the seconds and nanoseconds fields have consistent sign + */ + protected void normalizeSecondsAndNanos() { + if (totalSeconds > 0 && nanos < 0) { + --totalSeconds; + nanos += IntervalDayTimeUtils.NANOS_PER_SEC; + } else if (totalSeconds < 0 && nanos > 0) { + ++totalSeconds; + nanos -= IntervalDayTimeUtils.NANOS_PER_SEC; + } + } + + public void set(int days, int hours, int minutes, int seconds, int nanos) { + long totalSeconds = seconds; + totalSeconds += TimeUnit.DAYS.toSeconds(days); + totalSeconds += TimeUnit.HOURS.toSeconds(hours); + totalSeconds += TimeUnit.MINUTES.toSeconds(minutes); + totalSeconds += TimeUnit.NANOSECONDS.toSeconds(nanos); + nanos = nanos % IntervalDayTimeUtils.NANOS_PER_SEC; + + this.totalSeconds = totalSeconds; + this.nanos = nanos; + + normalizeSecondsAndNanos(); + } + + public void set(long seconds, int nanos) { + this.totalSeconds = seconds; + this.nanos = nanos; + normalizeSecondsAndNanos(); + } + + public void set(BigDecimal totalSecondsBd) { + long totalSeconds = totalSecondsBd.longValue(); + BigDecimal fractionalSecs = totalSecondsBd.remainder(BigDecimal.ONE); + int nanos = fractionalSecs.multiply(IntervalDayTimeUtils.NANOS_PER_SEC_BD).intValue(); + set(totalSeconds, nanos); + } + + public void set(HiveIntervalDayTime other) { + set(other.getTotalSeconds(), other.getNanos()); + } + + public HiveIntervalDayTime negate() { + return new HiveIntervalDayTime(-getTotalSeconds(), -getNanos()); + } + + @Override + public int compareTo(HiveIntervalDayTime other) { + long cmp = this.totalSeconds - other.totalSeconds; + if (cmp == 0) { + cmp = this.nanos - other.nanos; + } + if (cmp != 0) { + cmp = cmp > 0 ? 1 : -1; + } + return (int) cmp; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof HiveIntervalDayTime)) { + return false; + } + return 0 == compareTo((HiveIntervalDayTime) obj); + } + + /** + * Return a copy of this object. + */ + public Object clone() { + return new HiveIntervalDayTime(totalSeconds, nanos); + } + + @Override + public int hashCode() { + return new HashCodeBuilder().append(totalSeconds).append(nanos).toHashCode(); + } + + @Override + public String toString() { + // If normalize() was used, then day-hour-minute-second-nanos should have the same sign. + // This is currently working with that assumption. + boolean isNegative = (totalSeconds < 0 || nanos < 0); + String daySecondSignStr = isNegative ? "-" : ""; + + return String.format("%s%d %02d:%02d:%02d.%09d", + daySecondSignStr, Math.abs(getDays()), + Math.abs(getHours()), Math.abs(getMinutes()), + Math.abs(getSeconds()), Math.abs(getNanos())); + } + + public static HiveIntervalDayTime valueOf(String strVal) { + HiveIntervalDayTime result = null; + if (strVal == null) { + throw new IllegalArgumentException("Interval day-time string was null"); + } + Matcher patternMatcher = PATTERN_MATCHER.get(); + patternMatcher.reset(strVal); + if (patternMatcher.matches()) { + // Parse out the individual parts + try { + // Sign - whether interval is positive or negative + int sign = 1; + String field = patternMatcher.group(1); + if (field != null && field.equals("-")) { + sign = -1; + } + int days = sign * + IntervalDayTimeUtils.parseNumericValueWithRange("day", patternMatcher.group(2), + 0, Integer.MAX_VALUE); + byte hours = (byte) (sign * + IntervalDayTimeUtils.parseNumericValueWithRange("hour", patternMatcher.group(3), 0, 23)); + byte minutes = (byte) (sign * + IntervalDayTimeUtils.parseNumericValueWithRange("minute", patternMatcher.group(4), 0, 59)); + int seconds = 0; + int nanos = 0; + field = patternMatcher.group(5); + if (field != null) { + BigDecimal bdSeconds = new BigDecimal(field); + if (bdSeconds.compareTo(IntervalDayTimeUtils.MAX_INT_BD) > 0) { + throw new IllegalArgumentException("seconds value of " + bdSeconds + " too large"); + } + seconds = sign * bdSeconds.intValue(); + nanos = sign * bdSeconds.subtract(new BigDecimal(bdSeconds.toBigInteger())) + .multiply(IntervalDayTimeUtils.NANOS_PER_SEC_BD).intValue(); + } + + result = new HiveIntervalDayTime(days, hours, minutes, seconds, nanos); + } catch (Exception err) { + throw new IllegalArgumentException("Error parsing interval day-time string: " + strVal, err); + } + } else { + throw new IllegalArgumentException( + "Interval string does not match day-time format of 'd h:m:s.n': " + strVal); + } + + return result; + } + + // Simple pattern: D H:M:S.nnnnnnnnn + private final static String PARSE_PATTERN = + "([+|-])?(\\d+) (\\d+):(\\d+):((\\d+)(\\.(\\d+))?)"; + + private static final ThreadLocal<Matcher> PATTERN_MATCHER = new ThreadLocal<Matcher>() { + @Override + protected Matcher initialValue() { + return Pattern.compile(PARSE_PATTERN).matcher(""); + } + }; +} http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/storage-api/src/java/org/apache/hadoop/hive/common/type/PisaTimestamp.java ---------------------------------------------------------------------- diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/PisaTimestamp.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/PisaTimestamp.java deleted file mode 100644 index ac1e38a..0000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/type/PisaTimestamp.java +++ /dev/null @@ -1,609 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.type; - -import java.math.BigDecimal; -import java.sql.Timestamp; -import java.util.Random; -import java.util.concurrent.TimeUnit; - -import com.google.common.base.Preconditions; - -/** - * Pisa project is named after the famous Leonardo of Pisa, or better known as Fibanacci. - * - * A Pisa timestamp is a timestamp without a time-zone (i.e. local) in the ISO-8601 calendar system, - * such as 2007-12-03 10:15:30.0123456789, with accuracy to the nanosecond (1 billionth of a - * second). - * - * Pisa timestamps use the same starting point as a java.sql.Timestamp -- the number of nanoseconds - * since the epoch (1970-01-01, or the day Unix roared awake) where negative numbers represent - * earlier days. - * - * However, we use the PisaTimestamp class which has different design requirements than - * java.sql.Timestamp. It is designed to be mutable and NOT thread-safe to avoid high memory - * allocation / garbage collection costs. And, provides for ease of use by our vectorization - * code to avoid the high CPU data cache miss cost for small objects, too. We do this by allowing - * the epoch day and nano of day to be stored externally (i.e. vector arrays). - * - * And, importantly, PisaTimestamp is a light-weight class similar to the epochDay/NanoOfDay of - * the newer Java 8 LocalDateTime class, except the timestamp is *indifferent* to timezone. - * - * A common usage would be to treat it as UTC. - * - * You can work with days, seconds, milliseconds, nanoseconds, etc. But to work with months you - * will need to convert to an external timestamp object and use calendars, etc. - * * - * The storage for a PisaTimestamp is: - * - * long epochDay - * // The number of days since 1970-01-01 (==> similar to Java 8 LocalDate). - * long nanoOfDay - * // The number of nanoseconds within the day, with the range of - * // 0 to 24 * 60 * 60 * 1,000,000,000 - 1 (==> similar to Java 8 LocalTime). - * - * Both epochDay and nanoOfDay are signed. - * - * We when both epochDay and nanoOfDay are non-zero, we will maintain them so they have the - * same sign. - * - */ - -public class PisaTimestamp { - - private static final long serialVersionUID = 1L; - - private long epochDay; - private long nanoOfDay; - - private Timestamp scratchTimestamp; - - public static final long NANOSECONDS_PER_SECOND = TimeUnit.SECONDS.toNanos(1); - public static final long NANOSECONDS_PER_MILLISECOND = TimeUnit.MILLISECONDS.toNanos(1); - public static final long NANOSECONDS_PER_DAY = TimeUnit.DAYS.toNanos(1); - - public static final long MILLISECONDS_PER_SECOND = TimeUnit.SECONDS.toMillis(1); - public static final long MILLISECONDS_PER_DAY = TimeUnit.DAYS.toMillis(1); - - public static final long SECONDS_PER_DAY = TimeUnit.DAYS.toSeconds(1); - - public static final long MIN_NANO_OF_DAY = -NANOSECONDS_PER_DAY; - public static final long MAX_NANO_OF_DAY = NANOSECONDS_PER_DAY; - - public static final BigDecimal BIG_NANOSECONDS_PER_SECOND = new BigDecimal(NANOSECONDS_PER_SECOND); - - public long getEpochDay() { - return epochDay; - } - - public long getNanoOfDay() { - return nanoOfDay; - } - - public PisaTimestamp() { - epochDay = 0; - nanoOfDay = 0; - scratchTimestamp = new Timestamp(0); - } - - public PisaTimestamp(long epochDay, long nanoOfDay) { - - Preconditions.checkState(validateIntegrity(epochDay, nanoOfDay), - "epochDay " + epochDay + ", nanoOfDay " + nanoOfDay + " not valid"); - - this.epochDay = epochDay; - this.nanoOfDay = nanoOfDay; - scratchTimestamp = new Timestamp(0); - } - - public PisaTimestamp(Timestamp timestamp) { - super(); - updateFromTimestamp(timestamp); - } - - public void reset() { - epochDay = 0; - nanoOfDay = 0; - } - - /** - * NOTE: This method validates the integrity rules between epoch day and nano of day, - * but not overflow/underflow of epoch day. Since epoch day overflow/underflow can result - * from to client data input, that must be checked manually with <undone> as this - * class do not throw data range exceptions as a rule. It leaves that choice to the caller. - * @param epochDay - * @param nanoOfDay - * @return true if epoch day and nano of day have integrity. - */ - public static boolean validateIntegrity(long epochDay, long nanoOfDay) { - - // Range check nano per day as invariant. - if (nanoOfDay >= NANOSECONDS_PER_DAY || nanoOfDay <= -NANOSECONDS_PER_DAY) { - return false; - } - - // Signs of epoch day and nano of day must match. - if (!(epochDay >= 0 && nanoOfDay >= 0 || - epochDay <= 0 && nanoOfDay <= 0)) { - return false; - } - - return true; - } - - /** - * Set this PisaTimestamp from another PisaTimestamp. - * @param source - * @return this - */ - public PisaTimestamp update(PisaTimestamp source) { - this.epochDay = source.epochDay; - this.nanoOfDay = source.nanoOfDay; - return this; - } - - /** - * Set this PisaTimestamp from a epoch day and nano of day. - * @param epochDay - * @param nanoOfDay - * @return this - */ - public PisaTimestamp update(long epochDay, long nanoOfDay) { - - Preconditions.checkState(validateIntegrity(epochDay, nanoOfDay), - "epochDay " + epochDay + ", nanoOfDay " + nanoOfDay + " not valid"); - - this.epochDay = epochDay; - this.nanoOfDay = nanoOfDay; - return this; - } - - /** - * Set the PisaTimestamp from a Timestamp object. - * @param timestamp - * @return this - */ - public PisaTimestamp updateFromTimestamp(Timestamp timestamp) { - - long timestampTime = timestamp.getTime(); - int nanos = timestamp.getNanos(); - - /** - * Since the Timestamp class always stores nanos as a positive quantity (0 .. 999,999,999), - * we have to adjust back the time (subtract) by 1,000,000,000 to get right quantity for - * our calculations below. One thing it ensures is nanoOfDay will be negative. - */ - if (timestampTime < 0 && nanos > 0) { - timestampTime -= MILLISECONDS_PER_SECOND; - } - - // The Timestamp class does not use the milliseconds part (always 0). It is covered by nanos. - long epochSeconds = timestampTime / MILLISECONDS_PER_SECOND; - - nanoOfDay = (epochSeconds % SECONDS_PER_DAY) * NANOSECONDS_PER_SECOND + nanos; - epochDay = epochSeconds / SECONDS_PER_DAY + (nanoOfDay / NANOSECONDS_PER_DAY); - - Preconditions.checkState(validateIntegrity(epochDay, nanoOfDay)); - return this; - } - - /** - * Set this PisaTimestamp from a timestamp milliseconds. - * @param epochMilliseconds - * @return this - */ - public PisaTimestamp updateFromTimestampMilliseconds(long timestampMilliseconds) { - /** - * The Timestamp class setTime sets both the time (seconds stored as milliseconds) and - * the nanos. - */ - scratchTimestamp.setTime(timestampMilliseconds); - updateFromTimestamp(scratchTimestamp); - return this; - } - - /** - * Set this PisaTimestamp from a timestamp seconds. - * @param epochMilliseconds - * @return this - */ - public PisaTimestamp updateFromTimestampSeconds(long timestampSeconds) { - return updateFromTimestampMilliseconds(timestampSeconds * MILLISECONDS_PER_SECOND); - } - - /** - * Set this PisaTimestamp from a timestamp seconds. - * @param epochMilliseconds - * @return this - */ - public PisaTimestamp updateFromTimestampSecondsWithFractionalNanoseconds( - double timestampSecondsWithFractionalNanoseconds) { - - // Otherwise, BigDecimal throws an exception. (Support vector operations that sometimes - // do work on double Not-a-Number NaN values). - if (Double.isNaN(timestampSecondsWithFractionalNanoseconds)) { - timestampSecondsWithFractionalNanoseconds = 0; - } - // Algorithm used by TimestampWritable.doubleToTimestamp method. - // Allocates a BigDecimal object! - - long seconds = (long) timestampSecondsWithFractionalNanoseconds; - - // We must ensure the exactness of the double's fractional portion. - // 0.6 as the fraction part will be converted to 0.59999... and - // significantly reduce the savings from binary serialization. - BigDecimal bd; - - bd = new BigDecimal(String.valueOf(timestampSecondsWithFractionalNanoseconds)); - bd = bd.subtract(new BigDecimal(seconds)); // Get the nanos fraction. - bd = bd.multiply(BIG_NANOSECONDS_PER_SECOND); // Make nanos an integer. - - int nanos = bd.intValue(); - - // Convert to millis - long millis = seconds * 1000; - if (nanos < 0) { - millis -= 1000; - nanos += 1000000000; - } - - scratchTimestamp.setTime(millis); - scratchTimestamp.setNanos(nanos); - updateFromTimestamp(scratchTimestamp); - return this; - } - - /** - * Set this PisaTimestamp from a epoch seconds and signed nanos (-999999999 to 999999999). - * @param epochSeconds - * @param signedNanos - * @return this - */ - public PisaTimestamp updateFromEpochSecondsAndSignedNanos(long epochSeconds, int signedNanos) { - - long nanoOfDay = (epochSeconds % SECONDS_PER_DAY) * NANOSECONDS_PER_SECOND + signedNanos; - long epochDay = epochSeconds / SECONDS_PER_DAY + nanoOfDay / NANOSECONDS_PER_DAY; - - Preconditions.checkState(validateIntegrity(epochDay, nanoOfDay)); - - this.epochDay = epochDay; - this.nanoOfDay = nanoOfDay; - return this; - } - - /** - * Set a scratch PisaTimestamp with this PisaTimestamp's values and return the scratch object. - * @param epochDay - * @param nanoOfDay - */ - public PisaTimestamp scratchCopy(PisaTimestamp scratch) { - - scratch.epochDay = epochDay; - scratch.nanoOfDay = nanoOfDay; - return scratch; - } - - /** - * Set a Timestamp object from this PisaTimestamp. - * @param timestamp - */ - public void timestampUpdate(Timestamp timestamp) { - - /* - * java.sql.Timestamp consists of a long variable to store milliseconds and an integer variable for nanoseconds. - * The long variable is used to store only the full seconds converted to millis. For example for 1234 milliseconds, - * 1000 is stored in the long variable, and 234000000 (234 converted to nanoseconds) is stored as nanoseconds. - * The negative timestamps are also supported, but nanoseconds must be positive therefore millisecond part is - * reduced by one second. - */ - - long epochSeconds = epochDay * SECONDS_PER_DAY + nanoOfDay / NANOSECONDS_PER_SECOND; - long integralSecInMillis; - int nanos = (int) (nanoOfDay % NANOSECONDS_PER_SECOND); // The nanoseconds. - if (nanos < 0) { - nanos = (int) NANOSECONDS_PER_SECOND + nanos; // The positive nano-part that will be added to milliseconds. - integralSecInMillis = (epochSeconds - 1) * MILLISECONDS_PER_SECOND; // Reduce by one second. - } else { - integralSecInMillis = epochSeconds * MILLISECONDS_PER_SECOND; // Full seconds converted to millis. - } - - timestamp.setTime(integralSecInMillis); - timestamp.setNanos(nanos); - } - - /** - * Return the scratch timestamp with values from Pisa timestamp. - * @return - */ - public Timestamp asScratchTimestamp() { - timestampUpdate(scratchTimestamp); - return scratchTimestamp; - } - - /** - * Return the scratch timestamp for use by the caller. - * @return - */ - public Timestamp useScratchTimestamp() { - return scratchTimestamp; - } - - public int compareTo(PisaTimestamp another) { - - if (epochDay == another.epochDay) { - if (nanoOfDay == another.nanoOfDay){ - return 0; - } else { - return (nanoOfDay < another.nanoOfDay ? -1 : 1); - } - } else { - return (epochDay < another.epochDay ? -1: 1); - } - } - - public static int compareTo(long epochDay1, long nanoOfDay1, PisaTimestamp another) { - - if (epochDay1 == another.epochDay) { - if (nanoOfDay1 == another.nanoOfDay){ - return 0; - } else { - return (nanoOfDay1 < another.nanoOfDay ? -1 : 1); - } - } else { - return (epochDay1 < another.epochDay ? -1: 1); - } - } - - public static int compareTo(PisaTimestamp pisaTimestamp1, long epochDay2, long nanoOfDay2) { - - if (pisaTimestamp1.epochDay == epochDay2) { - if (pisaTimestamp1.nanoOfDay == nanoOfDay2){ - return 0; - } else { - return (pisaTimestamp1.nanoOfDay < nanoOfDay2 ? -1 : 1); - } - } else { - return (pisaTimestamp1.epochDay < epochDay2 ? -1: 1); - } - } - - public static int compareTo(long epochDay1, long nanoOfDay1, long epochDay2, long nanoOfDay2) { - - if (epochDay1 == epochDay2) { - if (nanoOfDay1 == nanoOfDay2){ - return 0; - } else { - return (nanoOfDay1 < nanoOfDay2 ? -1 : 1); - } - } else { - return (epochDay1 < epochDay2 ? -1: 1); - } - } - - - /** - * Standard equals method override. - */ - @Override - public boolean equals(Object obj) { - if (obj == null || obj.getClass() != getClass()) { - return false; - } - return equals((PisaTimestamp) obj); - } - - public boolean equals(PisaTimestamp other) { - - if (epochDay == other.epochDay) { - if (nanoOfDay == other.nanoOfDay) { - return true; - } else { - return false; - } - } else { - return false; - } - } - - public static void add(PisaTimestamp pisaTimestamp1, PisaTimestamp pisaTimestamp2, - PisaTimestamp result) { - add(pisaTimestamp1.epochDay, pisaTimestamp1.nanoOfDay, - pisaTimestamp2.epochDay, pisaTimestamp2.nanoOfDay, - result); - } - - public static void add(long epochDay1, long nanoOfDay1, - long epochDay2, long nanoOfDay2, - PisaTimestamp result) { - - // Validate integrity rules between epoch day and nano of day. - Preconditions.checkState(PisaTimestamp.validateIntegrity(epochDay1, nanoOfDay1)); - Preconditions.checkState(PisaTimestamp.validateIntegrity(epochDay2, nanoOfDay2)); - - long intermediateEpochDay = epochDay1 + epochDay2; - long intermediateNanoOfDay = nanoOfDay1 + nanoOfDay2; - - // Normalize so both are positive or both are negative. - long normalizedEpochDay; - long normalizedNanoOfDay; - if (intermediateEpochDay > 0 && intermediateNanoOfDay < 0) { - normalizedEpochDay = intermediateEpochDay - 1; - normalizedNanoOfDay = intermediateNanoOfDay + NANOSECONDS_PER_DAY; - } else if (intermediateEpochDay < 0 && intermediateNanoOfDay > 0) { - normalizedEpochDay = intermediateEpochDay + 1; - normalizedNanoOfDay = intermediateNanoOfDay - NANOSECONDS_PER_DAY; - } else { - normalizedEpochDay = intermediateEpochDay; - normalizedNanoOfDay = intermediateNanoOfDay; - } - - long resultEpochDay; - long resultNanoOfDay; - if (normalizedNanoOfDay >= NANOSECONDS_PER_DAY || normalizedNanoOfDay <= -NANOSECONDS_PER_DAY) { - // Adjust for carry or overflow... - - resultEpochDay = normalizedEpochDay + normalizedNanoOfDay / NANOSECONDS_PER_DAY; - resultNanoOfDay = normalizedNanoOfDay % NANOSECONDS_PER_DAY; - - } else { - resultEpochDay = normalizedEpochDay; - resultNanoOfDay = normalizedNanoOfDay; - } - - // The update method will validate integrity rules between epoch day and nano of day, - // but not overflow/underflow of epoch day. - result.update(resultEpochDay, resultNanoOfDay); - } - - public static void addSeconds(PisaTimestamp timestamp1, long epochSeconds, PisaTimestamp result) { - long epochDay = epochSeconds / SECONDS_PER_DAY; - long nanoOfDay = (epochSeconds % SECONDS_PER_DAY) * NANOSECONDS_PER_SECOND; - add(timestamp1.epochDay, timestamp1.nanoOfDay, epochDay, nanoOfDay, result); - } - - public static void subtract(PisaTimestamp timestamp1, PisaTimestamp timestamp2, - PisaTimestamp result) { - - add(timestamp1.epochDay, timestamp1.nanoOfDay, -timestamp2.epochDay, -timestamp2.nanoOfDay, - result); - } - - public static void subtract(long epochDay1, long nanoOfDay1, - long epochDay2, long nanoOfDay2, - PisaTimestamp result) { - - add(epochDay1, nanoOfDay1, -epochDay2, -nanoOfDay2, result); - } - - public static void subtractSeconds(PisaTimestamp timestamp1, long epochSeconds, - PisaTimestamp result) { - long epochDay = epochSeconds / SECONDS_PER_DAY; - long nanoOfDay = (epochSeconds % SECONDS_PER_DAY) * NANOSECONDS_PER_SECOND; - add(timestamp1.epochDay, timestamp1.nanoOfDay, -epochDay, -nanoOfDay, result); - } - - /** - * Rounds the number of milliseconds relative to the epoch down to the nearest whole number of - * seconds. 500 would round to 0, -500 would round to -1. - */ - public static long timestampMillisToSeconds(long millis) { - if (millis >= 0) { - return millis / 1000; - } else { - return (millis - 999) / 1000; - } - } - - /** - * Return a double with the integer part as the seconds and the fractional part as - * the nanoseconds the way the Timestamp class does it. - * @return seconds.nanoseconds - */ - public double getTimestampSecondsWithFractionalNanos() { - // Algorithm must be the same as TimestampWritable.getDouble method. - timestampUpdate(scratchTimestamp); - double seconds = timestampMillisToSeconds(scratchTimestamp.getTime()); - double nanos = scratchTimestamp.getNanos(); - BigDecimal bigSeconds = new BigDecimal(seconds); - BigDecimal bigNanos = new BigDecimal(nanos).divide(BIG_NANOSECONDS_PER_SECOND); - return bigSeconds.add(bigNanos).doubleValue(); - } - - /** - * Return an integer as the seconds the way the Timestamp class does it. - * @return seconds.nanoseconds - */ - public long getTimestampSeconds() { - // Algorithm must be the same as TimestampWritable.getSeconds method. - timestampUpdate(scratchTimestamp); - return timestampMillisToSeconds(scratchTimestamp.getTime()); - } - - /** - * Return an integer as the milliseconds the way the Timestamp class does it. - * @return seconds.nanoseconds - */ - public long getTimestampMilliseconds() { - timestampUpdate(scratchTimestamp); - return scratchTimestamp.getTime(); - } - - /** - * Return the epoch seconds. - * @return - */ - public long getEpochSeconds() { - return epochDay * SECONDS_PER_DAY + nanoOfDay / NANOSECONDS_PER_SECOND; - } - - /** - * Return the epoch seconds, given the epoch day and nano of day. - * @param epochDay - * @param nanoOfDay - * @return - */ - public static long getEpochSecondsFromEpochDayAndNanoOfDay(long epochDay, long nanoOfDay) { - return epochDay * SECONDS_PER_DAY + nanoOfDay / NANOSECONDS_PER_SECOND; - } - - /** - * Return the signed nanos (-999999999 to 999999999). - * NOTE: Not the same as Timestamp class nanos (which are always positive). - */ - public int getSignedNanos() { - return (int) (nanoOfDay % NANOSECONDS_PER_SECOND); - } - - /** - * Return the signed nanos (-999999999 to 999999999). - * NOTE: Not the same as Timestamp class nanos (which are always positive). - */ - public static int getSignedNanos(long nanoOfDay) { - return (int) (nanoOfDay % NANOSECONDS_PER_SECOND); - } - - /** - * Return the epoch milliseconds. - * @return - */ - public long getEpochMilliseconds() { - return epochDay * MILLISECONDS_PER_DAY + nanoOfDay / NANOSECONDS_PER_MILLISECOND; - } - - /** - * Return the epoch seconds, given the epoch day and nano of day. - * @param epochDay - * @param nanoOfDay - * @return - */ - public static long getEpochMillisecondsFromEpochDayAndNanoOfDay(long epochDay, long nanoOfDay) { - return epochDay * MILLISECONDS_PER_DAY + nanoOfDay / NANOSECONDS_PER_MILLISECOND; - } - - @Override - public int hashCode() { - // UNDONE: We don't want to box the longs just to get the hash codes... - return new Long(epochDay).hashCode() ^ new Long(nanoOfDay).hashCode(); - } - - @Override - public String toString() { - timestampUpdate(scratchTimestamp); - return scratchTimestamp.toString(); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java ---------------------------------------------------------------------- diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java index 13baff4..3fb0cfd 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java +++ b/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java @@ -18,21 +18,67 @@ package org.apache.hadoop.hive.common.type; import java.sql.Timestamp; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.Random; import java.util.concurrent.TimeUnit; public class RandomTypeUtil { public static final long NANOSECONDS_PER_SECOND = TimeUnit.SECONDS.toNanos(1); + public static final long MILLISECONDS_PER_SECOND = TimeUnit.SECONDS.toMillis(1); + public static final long NANOSECONDS_PER_MILLISSECOND = TimeUnit.MILLISECONDS.toNanos(1); + + private static ThreadLocal<DateFormat> DATE_FORMAT = + new ThreadLocal<DateFormat>() { + @Override + protected DateFormat initialValue() { + return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + } + }; + + // We've switched to Joda/Java Calendar which has a more limited time range.... + public static int MIN_YEAR = 1900; + public static int MAX_YEAR = 3000; + private static long MIN_FOUR_DIGIT_YEAR_MILLIS = parseToMillis("1900-01-01 00:00:00"); + private static long MAX_FOUR_DIGIT_YEAR_MILLIS = parseToMillis("3000-01-01 00:00:00"); + + private static long parseToMillis(String s) { + try { + return DATE_FORMAT.get().parse(s).getTime(); + } catch (ParseException ex) { + throw new RuntimeException(ex); + } + } public static Timestamp getRandTimestamp(Random r) { + return getRandTimestamp(r, MIN_YEAR, MAX_YEAR); + } + + public static Timestamp getRandTimestamp(Random r, int minYear, int maxYear) { String optionalNanos = ""; - if (r.nextInt(2) == 1) { + switch (r.nextInt(4)) { + case 0: + // No nanos. + break; + case 1: + optionalNanos = String.format(".%09d", + Integer.valueOf(r.nextInt((int) NANOSECONDS_PER_SECOND))); + break; + case 2: + // Limit to milliseconds only... optionalNanos = String.format(".%09d", - Integer.valueOf(0 + r.nextInt((int) NANOSECONDS_PER_SECOND))); + Integer.valueOf(r.nextInt((int) MILLISECONDS_PER_SECOND)) * NANOSECONDS_PER_MILLISSECOND); + break; + case 3: + // Limit to below milliseconds only... + optionalNanos = String.format(".%09d", + Integer.valueOf(r.nextInt((int) NANOSECONDS_PER_MILLISSECOND))); + break; } String timestampStr = String.format("%04d-%02d-%02d %02d:%02d:%02d%s", - Integer.valueOf(0 + r.nextInt(10000)), // year + Integer.valueOf(minYear + r.nextInt(maxYear - minYear + 1)), // year Integer.valueOf(1 + r.nextInt(12)), // month Integer.valueOf(1 + r.nextInt(28)), // day Integer.valueOf(0 + r.nextInt(24)), // hour @@ -48,4 +94,22 @@ public class RandomTypeUtil { } return timestampVal; } + + public static long randomMillis(long minMillis, long maxMillis, Random rand) { + return minMillis + (long) ((maxMillis - minMillis) * rand.nextDouble()); + } + + public static long randomMillis(Random rand) { + return randomMillis(MIN_FOUR_DIGIT_YEAR_MILLIS, MAX_FOUR_DIGIT_YEAR_MILLIS, rand); + } + + public static int randomNanos(Random rand, int decimalDigits) { + // Only keep the most significant decimalDigits digits. + int nanos = rand.nextInt((int) NANOSECONDS_PER_SECOND); + return nanos - nanos % (int) Math.pow(10, 9 - decimalDigits); + } + + public static int randomNanos(Random rand) { + return randomNanos(rand, 9); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java ---------------------------------------------------------------------- diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java index 4ae9c47..c069a5f 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java +++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.ql.exec.vector; -import java.io.IOException; import java.util.Arrays; /** @@ -43,6 +42,7 @@ public abstract class ColumnVector { BYTES, DECIMAL, TIMESTAMP, + INTERVAL_DAY_TIME, STRUCT, LIST, MAP, http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java ---------------------------------------------------------------------- diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java new file mode 100644 index 0000000..39ccea8 --- /dev/null +++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java @@ -0,0 +1,348 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.exec.vector; + +import java.util.Arrays; + +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; +import org.apache.hadoop.io.Writable; + +/** + * This class represents a nullable interval day time column vector capable of handing a + * wide range of interval day time values. + * + * We store the 2 (value) fields of a HiveIntervalDayTime class in primitive arrays. + * + * We do this to avoid an array of Java HiveIntervalDayTime objects which would have poor storage + * and memory access characteristics. + * + * Generally, the caller will fill in a scratch HiveIntervalDayTime object with values from a row, + * work using the scratch HiveIntervalDayTime, and then perhaps update the column vector row + * with a result. + */ +public class IntervalDayTimeColumnVector extends ColumnVector { + + /* + * The storage arrays for this column vector corresponds to the storage of a HiveIntervalDayTime: + */ + private long[] totalSeconds; + // The values from HiveIntervalDayTime.getTotalSeconds(). + + private int[] nanos; + // The values from HiveIntervalDayTime.getNanos(). + + /* + * Scratch objects. + */ + private final HiveIntervalDayTime scratchIntervalDayTime; + + private Writable scratchWritable; + // Supports keeping a HiveIntervalDayTimeWritable object without having to import + // that definition... + + /** + * Use this constructor by default. All column vectors + * should normally be the default size. + */ + public IntervalDayTimeColumnVector() { + this(VectorizedRowBatch.DEFAULT_SIZE); + } + + /** + * Don't use this except for testing purposes. + * + * @param len the number of rows + */ + public IntervalDayTimeColumnVector(int len) { + super(len); + + totalSeconds = new long[len]; + nanos = new int[len]; + + scratchIntervalDayTime = new HiveIntervalDayTime(); + + scratchWritable = null; // Allocated by caller. + } + + /** + * Return the number of rows. + * @return + */ + public int getLength() { + return totalSeconds.length; + } + + /** + * Return a row's HiveIntervalDayTime.getTotalSeconds() value. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param elementNum + * @return + */ + public long getTotalSeconds(int elementNum) { + return totalSeconds[elementNum]; + } + + /** + * Return a row's HiveIntervalDayTime.getNanos() value. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param elementNum + * @return + */ + public long getNanos(int elementNum) { + return nanos[elementNum]; + } + + /** + * Return a row's HiveIntervalDayTime.getDouble() value. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param elementNum + * @return + */ + public double getDouble(int elementNum) { + return asScratchIntervalDayTime(elementNum).getDouble(); + } + + /** + * Set a HiveIntervalDayTime object from a row of the column. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param intervalDayTime + * @param elementNum + */ + public void intervalDayTimeUpdate(HiveIntervalDayTime intervalDayTime, int elementNum) { + intervalDayTime.set(totalSeconds[elementNum], nanos[elementNum]); + } + + + /** + * Return the scratch HiveIntervalDayTime object set from a row. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param elementNum + * @return + */ + public HiveIntervalDayTime asScratchIntervalDayTime(int elementNum) { + scratchIntervalDayTime.set(totalSeconds[elementNum], nanos[elementNum]); + return scratchIntervalDayTime; + } + + /** + * Return the scratch HiveIntervalDayTime (contents undefined). + * @return + */ + public HiveIntervalDayTime getScratchIntervalDayTime() { + return scratchIntervalDayTime; + } + + /** + * Compare row to HiveIntervalDayTime. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param elementNum + * @param intervalDayTime + * @return -1, 0, 1 standard compareTo values. + */ + public int compareTo(int elementNum, HiveIntervalDayTime intervalDayTime) { + return asScratchIntervalDayTime(elementNum).compareTo(intervalDayTime); + } + + /** + * Compare HiveIntervalDayTime to row. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param intervalDayTime + * @param elementNum + * @return -1, 0, 1 standard compareTo values. + */ + public int compareTo(HiveIntervalDayTime intervalDayTime, int elementNum) { + return intervalDayTime.compareTo(asScratchIntervalDayTime(elementNum)); + } + + /** + * Compare a row to another TimestampColumnVector's row. + * @param elementNum1 + * @param intervalDayTimeColVector2 + * @param elementNum2 + * @return + */ + public int compareTo(int elementNum1, IntervalDayTimeColumnVector intervalDayTimeColVector2, + int elementNum2) { + return asScratchIntervalDayTime(elementNum1).compareTo( + intervalDayTimeColVector2.asScratchIntervalDayTime(elementNum2)); + } + + /** + * Compare another TimestampColumnVector's row to a row. + * @param intervalDayTimeColVector1 + * @param elementNum1 + * @param elementNum2 + * @return + */ + public int compareTo(IntervalDayTimeColumnVector intervalDayTimeColVector1, int elementNum1, + int elementNum2) { + return intervalDayTimeColVector1.asScratchIntervalDayTime(elementNum1).compareTo( + asScratchIntervalDayTime(elementNum2)); + } + + @Override + public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) { + + IntervalDayTimeColumnVector timestampColVector = (IntervalDayTimeColumnVector) inputVector; + + totalSeconds[outElementNum] = timestampColVector.totalSeconds[inputElementNum]; + nanos[outElementNum] = timestampColVector.nanos[inputElementNum]; + } + + // Simplify vector by brute-force flattening noNulls and isRepeating + // This can be used to reduce combinatorial explosion of code paths in VectorExpressions + // with many arguments. + public void flatten(boolean selectedInUse, int[] sel, int size) { + flattenPush(); + if (isRepeating) { + isRepeating = false; + long repeatFastTime = totalSeconds[0]; + int repeatNanos = nanos[0]; + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + totalSeconds[i] = repeatFastTime; + nanos[i] = repeatNanos; + } + } else { + Arrays.fill(totalSeconds, 0, size, repeatFastTime); + Arrays.fill(nanos, 0, size, repeatNanos); + } + flattenRepeatingNulls(selectedInUse, sel, size); + } + flattenNoNulls(selectedInUse, sel, size); + } + + /** + * Set a row from a HiveIntervalDayTime. + * We assume the entry has already been isRepeated adjusted. + * @param elementNum + * @param intervalDayTime + */ + public void set(int elementNum, HiveIntervalDayTime intervalDayTime) { + this.totalSeconds[elementNum] = intervalDayTime.getTotalSeconds(); + this.nanos[elementNum] = intervalDayTime.getNanos(); + } + + /** + * Set a row from the current value in the scratch interval day time. + * @param elementNum + */ + public void setFromScratchIntervalDayTime(int elementNum) { + this.totalSeconds[elementNum] = scratchIntervalDayTime.getTotalSeconds(); + this.nanos[elementNum] = scratchIntervalDayTime.getNanos(); + } + + /** + * Set row to standard null value(s). + * We assume the entry has already been isRepeated adjusted. + * @param elementNum + */ + public void setNullValue(int elementNum) { + totalSeconds[elementNum] = 0; + nanos[elementNum] = 1; + } + + // Copy the current object contents into the output. Only copy selected entries, + // as indicated by selectedInUse and the sel array. + public void copySelected( + boolean selectedInUse, int[] sel, int size, IntervalDayTimeColumnVector output) { + + // Output has nulls if and only if input has nulls. + output.noNulls = noNulls; + output.isRepeating = false; + + // Handle repeating case + if (isRepeating) { + output.totalSeconds[0] = totalSeconds[0]; + output.nanos[0] = nanos[0]; + output.isNull[0] = isNull[0]; + output.isRepeating = true; + return; + } + + // Handle normal case + + // Copy data values over + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.totalSeconds[i] = totalSeconds[i]; + output.nanos[i] = nanos[i]; + } + } + else { + System.arraycopy(totalSeconds, 0, output.totalSeconds, 0, size); + System.arraycopy(nanos, 0, output.nanos, 0, size); + } + + // Copy nulls over if needed + if (!noNulls) { + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.isNull[i] = isNull[i]; + } + } + else { + System.arraycopy(isNull, 0, output.isNull, 0, size); + } + } + } + + /** + * Fill all the vector entries with a HiveIntervalDayTime. + * @param intervalDayTime + */ + public void fill(HiveIntervalDayTime intervalDayTime) { + noNulls = true; + isRepeating = true; + totalSeconds[0] = intervalDayTime.getTotalSeconds(); + nanos[0] = intervalDayTime.getNanos(); + } + + /** + * Return a convenience writable object stored by this column vector. + * Supports keeping a TimestampWritable object without having to import that definition... + * @return + */ + public Writable getScratchWritable() { + return scratchWritable; + } + + /** + * Set the convenience writable object stored by this column vector + * @param scratchWritable + */ + public void setScratchWritable(Writable scratchWritable) { + this.scratchWritable = scratchWritable; + } + + @Override + public void stringifyValue(StringBuilder buffer, int row) { + if (isRepeating) { + row = 0; + } + if (noNulls || !isNull[row]) { + scratchIntervalDayTime.set(totalSeconds[row], nanos[row]); + buffer.append(scratchIntervalDayTime.toString()); + } else { + buffer.append("null"); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java ---------------------------------------------------------------------- diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java index b73a0d2..c0dd5ed 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java +++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java @@ -20,37 +20,35 @@ package org.apache.hadoop.hive.ql.exec.vector; import java.sql.Timestamp; import java.util.Arrays; -import org.apache.hadoop.hive.common.type.PisaTimestamp; -import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.io.Writable; -import com.google.common.base.Preconditions; - /** * This class represents a nullable timestamp column vector capable of handing a wide range of * timestamp values. * - * We use the PisaTimestamp which is designed to be mutable and avoid the heavy memory allocation - * and CPU data cache miss costs. + * We store the 2 (value) fields of a Timestamp class in primitive arrays. + * + * We do this to avoid an array of Java Timestamp objects which would have poor storage + * and memory access characteristics. + * + * Generally, the caller will fill in a scratch timestamp object with values from a row, work + * using the scratch timestamp, and then perhaps update the column vector row with a result. */ public class TimestampColumnVector extends ColumnVector { /* - * The storage arrays for this column vector corresponds to the storage of a PisaTimestamp: + * The storage arrays for this column vector corresponds to the storage of a Timestamp: */ - private long[] epochDay; - // An array of the number of days since 1970-01-01 (similar to Java 8 LocalDate). + public long[] time; + // The values from Timestamp.getTime(). - private long[] nanoOfDay; - // An array of the number of nanoseconds within the day, with the range of - // 0 to 24 * 60 * 60 * 1,000,000,000 - 1 (similar to Java 8 LocalTime). + public int[] nanos; + // The values from Timestamp.getNanos(). /* * Scratch objects. */ - private PisaTimestamp scratchPisaTimestamp; - // Convenience scratch Pisa timestamp object. + private final Timestamp scratchTimestamp; private Writable scratchWritable; // Supports keeping a TimestampWritable object without having to import that definition... @@ -71,10 +69,10 @@ public class TimestampColumnVector extends ColumnVector { public TimestampColumnVector(int len) { super(len); - epochDay = new long[len]; - nanoOfDay = new long[len]; + time = new long[len]; + nanos = new int[len]; - scratchPisaTimestamp = new PisaTimestamp(); + scratchTimestamp = new Timestamp(0); scratchWritable = null; // Allocated by caller. } @@ -84,48 +82,27 @@ public class TimestampColumnVector extends ColumnVector { * @return */ public int getLength() { - return epochDay.length; + return time.length; } /** - * Returnt a row's epoch day. + * Return a row's Timestamp.getTime() value. * We assume the entry has already been NULL checked and isRepeated adjusted. * @param elementNum * @return */ - public long getEpochDay(int elementNum) { - return epochDay[elementNum]; + public long getTime(int elementNum) { + return time[elementNum]; } /** - * Return a row's nano of day. + * Return a row's Timestamp.getNanos() value. * We assume the entry has already been NULL checked and isRepeated adjusted. * @param elementNum * @return */ - public long getNanoOfDay(int elementNum) { - return nanoOfDay[elementNum]; - } - - /** - * Get a scratch PisaTimestamp object from a row of the column. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param elementNum - * @return scratch - */ - public PisaTimestamp asScratchPisaTimestamp(int elementNum) { - scratchPisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]); - return scratchPisaTimestamp; - } - - /** - * Set a PisaTimestamp object from a row of the column. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param pisaTimestamp - * @param elementNum - */ - public void pisaTimestampUpdate(PisaTimestamp pisaTimestamp, int elementNum) { - pisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]); + public int getNanos(int elementNum) { + return nanos[elementNum]; } /** @@ -135,154 +112,133 @@ public class TimestampColumnVector extends ColumnVector { * @param elementNum */ public void timestampUpdate(Timestamp timestamp, int elementNum) { - scratchPisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]); - scratchPisaTimestamp.timestampUpdate(timestamp); + timestamp.setTime(time[elementNum]); + timestamp.setNanos(nanos[elementNum]); } /** - * Compare row to PisaTimestamp. + * Return the scratch Timestamp object set from a row. * We assume the entry has already been NULL checked and isRepeated adjusted. * @param elementNum - * @param pisaTimestamp - * @return -1, 0, 1 standard compareTo values. - */ - public int compareTo(int elementNum, PisaTimestamp pisaTimestamp) { - return PisaTimestamp.compareTo(epochDay[elementNum], nanoOfDay[elementNum], pisaTimestamp); - } - - /** - * Compare PisaTimestamp to row. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param pisaTimestamp - * @param elementNum - * @return -1, 0, 1 standard compareTo values. + * @return */ - public int compareTo(PisaTimestamp pisaTimestamp, int elementNum) { - return PisaTimestamp.compareTo(pisaTimestamp, epochDay[elementNum], nanoOfDay[elementNum]); + public Timestamp asScratchTimestamp(int elementNum) { + scratchTimestamp.setTime(time[elementNum]); + scratchTimestamp.setNanos(nanos[elementNum]); + return scratchTimestamp; } /** - * Compare a row to another TimestampColumnVector's row. - * @param elementNum1 - * @param timestampColVector2 - * @param elementNum2 + * Return the scratch timestamp (contents undefined). * @return */ - public int compareTo(int elementNum1, TimestampColumnVector timestampColVector2, - int elementNum2) { - return PisaTimestamp.compareTo( - epochDay[elementNum1], nanoOfDay[elementNum1], - timestampColVector2.epochDay[elementNum2], timestampColVector2.nanoOfDay[elementNum2]); + public Timestamp getScratchTimestamp() { + return scratchTimestamp; } /** - * Compare another TimestampColumnVector's row to a row. - * @param timestampColVector1 - * @param elementNum1 - * @param elementNum2 + * Return a long representation of a Timestamp. + * @param elementNum * @return */ - public int compareTo(TimestampColumnVector timestampColVector1, int elementNum1, - int elementNum2) { - return PisaTimestamp.compareTo( - timestampColVector1.epochDay[elementNum1], timestampColVector1.nanoOfDay[elementNum1], - epochDay[elementNum2], nanoOfDay[elementNum2]); - } - - public void add(PisaTimestamp timestamp1, PisaTimestamp timestamp2, int resultElementNum) { - PisaTimestamp.add(timestamp1, timestamp2, scratchPisaTimestamp); - epochDay[resultElementNum] = scratchPisaTimestamp.getEpochDay(); - nanoOfDay[resultElementNum] = scratchPisaTimestamp.getNanoOfDay(); - } - - public void subtract(PisaTimestamp timestamp1, PisaTimestamp timestamp2, int resultElementNum) { - PisaTimestamp.subtract(timestamp1, timestamp2, scratchPisaTimestamp); - epochDay[resultElementNum] = scratchPisaTimestamp.getEpochDay(); - nanoOfDay[resultElementNum] = scratchPisaTimestamp.getNanoOfDay(); + public long getTimestampAsLong(int elementNum) { + scratchTimestamp.setTime(time[elementNum]); + scratchTimestamp.setNanos(nanos[elementNum]); + return getTimestampAsLong(scratchTimestamp); } /** - * Return row as a double with the integer part as the seconds and the fractional part as - * the nanoseconds the way the Timestamp class does it. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param elementNum - * @return seconds.nanoseconds + * Return a long representation of a Timestamp. + * @param timestamp + * @return */ - public double getTimestampSecondsWithFractionalNanos(int elementNum) { - scratchPisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]); - return scratchPisaTimestamp.getTimestampSecondsWithFractionalNanos(); + public static long getTimestampAsLong(Timestamp timestamp) { + return millisToSeconds(timestamp.getTime()); } + // Copy of TimestampWritable.millisToSeconds /** - * Return row as integer as the seconds the way the Timestamp class does it. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param elementNum - * @return seconds + * Rounds the number of milliseconds relative to the epoch down to the nearest whole number of + * seconds. 500 would round to 0, -500 would round to -1. */ - public long getTimestampSeconds(int elementNum) { - scratchPisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]); - return scratchPisaTimestamp.getTimestampSeconds(); + private static long millisToSeconds(long millis) { + if (millis >= 0) { + return millis / 1000; + } else { + return (millis - 999) / 1000; + } } - /** - * Return row as milliseconds the way the Timestamp class does it. - * We assume the entry has already been NULL checked and isRepeated adjusted. + * Return a double representation of a Timestamp. * @param elementNum * @return */ - public long getTimestampMilliseconds(int elementNum) { - scratchPisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]); - return scratchPisaTimestamp.getTimestampMilliseconds(); + public double getDouble(int elementNum) { + scratchTimestamp.setTime(time[elementNum]); + scratchTimestamp.setNanos(nanos[elementNum]); + return getDouble(scratchTimestamp); } /** - * Return row as epoch seconds. - * We assume the entry has already been NULL checked and isRepeated adjusted. + * Return a double representation of a Timestamp. * @param elementNum * @return */ - public long getEpochSeconds(int elementNum) { - return PisaTimestamp.getEpochSecondsFromEpochDayAndNanoOfDay(epochDay[elementNum], nanoOfDay[elementNum]); + public static double getDouble(Timestamp timestamp) { + // Same algorithm as TimestampWritable (not currently import-able here). + double seconds, nanos; + seconds = millisToSeconds(timestamp.getTime()); + nanos = timestamp.getNanos(); + return seconds + nanos / 1000000000; } /** - * Return row as epoch milliseconds. + * Compare row to Timestamp. * We assume the entry has already been NULL checked and isRepeated adjusted. * @param elementNum - * @return + * @param timestamp + * @return -1, 0, 1 standard compareTo values. */ - public long getEpochMilliseconds(int elementNum) { - return PisaTimestamp.getEpochMillisecondsFromEpochDayAndNanoOfDay(epochDay[elementNum], nanoOfDay[elementNum]); + public int compareTo(int elementNum, Timestamp timestamp) { + return asScratchTimestamp(elementNum).compareTo(timestamp); } /** - * Return row as signed nanos (-999999999 to 999999999). - * NOTE: This is not the same as the Timestamp class nanos (which is always positive). + * Compare Timestamp to row. * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param timestamp * @param elementNum - * @return + * @return -1, 0, 1 standard compareTo values. */ - public int getSignedNanos(int elementNum) { - return PisaTimestamp.getSignedNanos(nanoOfDay[elementNum]); + public int compareTo(Timestamp timestamp, int elementNum) { + return timestamp.compareTo(asScratchTimestamp(elementNum)); } /** - * Get scratch timestamp with value of a row. - * @param elementNum + * Compare a row to another TimestampColumnVector's row. + * @param elementNum1 + * @param timestampColVector2 + * @param elementNum2 * @return */ - public Timestamp asScratchTimestamp(int elementNum) { - scratchPisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]); - return scratchPisaTimestamp.asScratchTimestamp(); + public int compareTo(int elementNum1, TimestampColumnVector timestampColVector2, + int elementNum2) { + return asScratchTimestamp(elementNum1).compareTo( + timestampColVector2.asScratchTimestamp(elementNum2)); } /** - * Get scratch Pisa timestamp for use by the caller. + * Compare another TimestampColumnVector's row to a row. + * @param timestampColVector1 + * @param elementNum1 + * @param elementNum2 * @return */ - public PisaTimestamp useScratchPisaTimestamp() { - return scratchPisaTimestamp; + public int compareTo(TimestampColumnVector timestampColVector1, int elementNum1, + int elementNum2) { + return timestampColVector1.asScratchTimestamp(elementNum1).compareTo( + asScratchTimestamp(elementNum2)); } @Override @@ -290,8 +246,8 @@ public class TimestampColumnVector extends ColumnVector { TimestampColumnVector timestampColVector = (TimestampColumnVector) inputVector; - epochDay[outElementNum] = timestampColVector.epochDay[inputElementNum]; - nanoOfDay[outElementNum] = timestampColVector.nanoOfDay[inputElementNum]; + time[outElementNum] = timestampColVector.time[inputElementNum]; + nanos[outElementNum] = timestampColVector.nanos[inputElementNum]; } // Simplify vector by brute-force flattening noNulls and isRepeating @@ -301,17 +257,17 @@ public class TimestampColumnVector extends ColumnVector { flattenPush(); if (isRepeating) { isRepeating = false; - long repeatEpochDay = epochDay[0]; - long repeatNanoOfDay = nanoOfDay[0]; + long repeatFastTime = time[0]; + int repeatNanos = nanos[0]; if (selectedInUse) { for (int j = 0; j < size; j++) { int i = sel[j]; - epochDay[i] = repeatEpochDay; - nanoOfDay[i] = repeatNanoOfDay; + time[i] = repeatFastTime; + nanos[i] = repeatNanos; } } else { - Arrays.fill(epochDay, 0, size, repeatEpochDay); - Arrays.fill(nanoOfDay, 0, size, repeatNanoOfDay); + Arrays.fill(time, 0, size, repeatFastTime); + Arrays.fill(nanos, 0, size, repeatNanos); } flattenRepeatingNulls(selectedInUse, sel, size); } @@ -319,71 +275,23 @@ public class TimestampColumnVector extends ColumnVector { } /** - * Set a row from a PisaTimestamp. - * We assume the entry has already been isRepeated adjusted. - * @param elementNum - * @param pisaTimestamp - */ - public void set(int elementNum, PisaTimestamp pisaTimestamp) { - this.epochDay[elementNum] = pisaTimestamp.getEpochDay(); - this.nanoOfDay[elementNum] = pisaTimestamp.getNanoOfDay(); - } - - /** * Set a row from a timestamp. * We assume the entry has already been isRepeated adjusted. * @param elementNum * @param timestamp */ public void set(int elementNum, Timestamp timestamp) { - scratchPisaTimestamp.updateFromTimestamp(timestamp); - this.epochDay[elementNum] = scratchPisaTimestamp.getEpochDay(); - this.nanoOfDay[elementNum] = scratchPisaTimestamp.getNanoOfDay(); + this.time[elementNum] = timestamp.getTime(); + this.nanos[elementNum] = timestamp.getNanos(); } /** - * Set a row from a epoch seconds and signed nanos (-999999999 to 999999999). + * Set a row from the current value in the scratch timestamp. * @param elementNum - * @param epochSeconds - * @param signedNanos */ - public void setEpochSecondsAndSignedNanos(int elementNum, long epochSeconds, int signedNanos) { - scratchPisaTimestamp.updateFromEpochSecondsAndSignedNanos(epochSeconds, signedNanos); - set(elementNum, scratchPisaTimestamp); - } - - /** - * Set a row from timestamp milliseconds. - * We assume the entry has already been isRepeated adjusted. - * @param elementNum - * @param timestampMilliseconds - */ - public void setTimestampMilliseconds(int elementNum, long timestampMilliseconds) { - scratchPisaTimestamp.updateFromTimestampMilliseconds(timestampMilliseconds); - set(elementNum, scratchPisaTimestamp.useScratchTimestamp()); - } - - /** - * Set a row from timestamp seconds. - * We assume the entry has already been isRepeated adjusted. - * @param elementNum - * @param timestamp - */ - public void setTimestampSeconds(int elementNum, long timestampSeconds) { - scratchPisaTimestamp.updateFromTimestampSeconds(timestampSeconds); - set(elementNum, scratchPisaTimestamp); - } - - /** - * Set a row from a double timestamp seconds with fractional nanoseconds. - * We assume the entry has already been isRepeated adjusted. - * @param elementNum - * @param timestamp - */ - public void setTimestampSecondsWithFractionalNanoseconds(int elementNum, - double secondsWithFractionalNanoseconds) { - scratchPisaTimestamp.updateFromTimestampSecondsWithFractionalNanoseconds(secondsWithFractionalNanoseconds); - set(elementNum, scratchPisaTimestamp); + public void setFromScratchTimestamp(int elementNum) { + this.time[elementNum] = scratchTimestamp.getTime(); + this.nanos[elementNum] = scratchTimestamp.getNanos(); } /** @@ -392,8 +300,8 @@ public class TimestampColumnVector extends ColumnVector { * @param elementNum */ public void setNullValue(int elementNum) { - epochDay[elementNum] = 0; - nanoOfDay[elementNum] = 1; + time[elementNum] = 0; + nanos[elementNum] = 1; } // Copy the current object contents into the output. Only copy selected entries, @@ -407,8 +315,8 @@ public class TimestampColumnVector extends ColumnVector { // Handle repeating case if (isRepeating) { - output.epochDay[0] = epochDay[0]; - output.nanoOfDay[0] = nanoOfDay[0]; + output.time[0] = time[0]; + output.nanos[0] = nanos[0]; output.isNull[0] = isNull[0]; output.isRepeating = true; return; @@ -420,13 +328,13 @@ public class TimestampColumnVector extends ColumnVector { if (selectedInUse) { for (int j = 0; j < size; j++) { int i = sel[j]; - output.epochDay[i] = epochDay[i]; - output.nanoOfDay[i] = nanoOfDay[i]; + output.time[i] = time[i]; + output.nanos[i] = nanos[i]; } } else { - System.arraycopy(epochDay, 0, output.epochDay, 0, size); - System.arraycopy(nanoOfDay, 0, output.nanoOfDay, 0, size); + System.arraycopy(time, 0, output.time, 0, size); + System.arraycopy(nanos, 0, output.nanos, 0, size); } // Copy nulls over if needed @@ -444,26 +352,14 @@ public class TimestampColumnVector extends ColumnVector { } /** - * Fill all the vector entries with a PisaTimestamp. - * @param pisaTimestamp - */ - public void fill(PisaTimestamp pisaTimestamp) { - noNulls = true; - isRepeating = true; - epochDay[0] = pisaTimestamp.getEpochDay(); - nanoOfDay[0] = pisaTimestamp.getNanoOfDay(); - } - - /** * Fill all the vector entries with a timestamp. * @param timestamp */ public void fill(Timestamp timestamp) { noNulls = true; isRepeating = true; - scratchPisaTimestamp.updateFromTimestamp(timestamp); - epochDay[0] = scratchPisaTimestamp.getEpochDay(); - nanoOfDay[0] = scratchPisaTimestamp.getNanoOfDay(); + time[0] = timestamp.getTime(); + nanos[0] = timestamp.getNanos(); } /** @@ -489,8 +385,9 @@ public class TimestampColumnVector extends ColumnVector { row = 0; } if (noNulls || !isNull[row]) { - scratchPisaTimestamp.update(epochDay[row], nanoOfDay[row]); - buffer.append(scratchPisaTimestamp.toString()); + scratchTimestamp.setTime(time[row]); + scratchTimestamp.setNanos(nanos[row]); + buffer.append(scratchTimestamp.toString()); } else { buffer.append("null"); } http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/storage-api/src/java/org/apache/hive/common/util/IntervalDayTimeUtils.java ---------------------------------------------------------------------- diff --git a/storage-api/src/java/org/apache/hive/common/util/IntervalDayTimeUtils.java b/storage-api/src/java/org/apache/hive/common/util/IntervalDayTimeUtils.java new file mode 100644 index 0000000..727c1e6 --- /dev/null +++ b/storage-api/src/java/org/apache/hive/common/util/IntervalDayTimeUtils.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.common.util; + +import java.math.BigDecimal; +import java.text.SimpleDateFormat; + +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; + + +/** + * DateUtils. Thread-safe class + * + */ +public class IntervalDayTimeUtils { + + private static final ThreadLocal<SimpleDateFormat> dateFormatLocal = new ThreadLocal<SimpleDateFormat>() { + @Override + protected SimpleDateFormat initialValue() { + return new SimpleDateFormat("yyyy-MM-dd"); + } + }; + + public static SimpleDateFormat getDateFormat() { + return dateFormatLocal.get(); + } + + public static final int NANOS_PER_SEC = 1000000000; + public static final BigDecimal MAX_INT_BD = new BigDecimal(Integer.MAX_VALUE); + public static final BigDecimal NANOS_PER_SEC_BD = new BigDecimal(NANOS_PER_SEC); + + public static int parseNumericValueWithRange(String fieldName, + String strVal, int minValue, int maxValue) throws IllegalArgumentException { + int result = 0; + if (strVal != null) { + result = Integer.parseInt(strVal); + if (result < minValue || result > maxValue) { + throw new IllegalArgumentException(String.format("%s value %d outside range [%d, %d]", + fieldName, result, minValue, maxValue)); + } + } + return result; + } + + public static long getIntervalDayTimeTotalNanos(HiveIntervalDayTime intervalDayTime) { + return intervalDayTime.getTotalSeconds() * NANOS_PER_SEC + intervalDayTime.getNanos(); + } + + public static void setIntervalDayTimeTotalNanos(HiveIntervalDayTime intervalDayTime, + long totalNanos) { + intervalDayTime.set(totalNanos / NANOS_PER_SEC, (int) (totalNanos % NANOS_PER_SEC)); + } + + public static long getIntervalDayTimeTotalSecondsFromTotalNanos(long totalNanos) { + return totalNanos / NANOS_PER_SEC; + } + + public static int getIntervalDayTimeNanosFromTotalNanos(long totalNanos) { + return (int) (totalNanos % NANOS_PER_SEC); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/storage-api/src/test/org/apache/hadoop/hive/common/type/TestPisaTimestamp.java ---------------------------------------------------------------------- diff --git a/storage-api/src/test/org/apache/hadoop/hive/common/type/TestPisaTimestamp.java b/storage-api/src/test/org/apache/hadoop/hive/common/type/TestPisaTimestamp.java deleted file mode 100644 index 8e7395c..0000000 --- a/storage-api/src/test/org/apache/hadoop/hive/common/type/TestPisaTimestamp.java +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common.type; - -import org.junit.Test; - -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.sql.Timestamp; -import java.util.Random; -import org.apache.hadoop.hive.common.type.RandomTypeUtil; - -import static org.junit.Assert.*; - -/** - * Test for ListColumnVector - */ -public class TestPisaTimestamp { - - private static int TEST_COUNT = 5000; - - @Test - public void testPisaTimestampCreate() throws Exception { - - Random r = new Random(1234); - - for (int i = 0; i < TEST_COUNT; i++) { - Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r); - PisaTimestamp pisaTimestamp = new PisaTimestamp(randTimestamp); - Timestamp reconstructedTimestamp = new Timestamp(0); - pisaTimestamp.timestampUpdate(reconstructedTimestamp); - if (!randTimestamp.equals(reconstructedTimestamp)) { - assertTrue(false); - } - } - } - - static BigDecimal BIG_MAX_LONG = new BigDecimal(Long.MAX_VALUE); - static BigDecimal BIG_MIN_LONG = new BigDecimal(Long.MIN_VALUE); - static BigDecimal BIG_NANOSECONDS_PER_DAY = new BigDecimal(PisaTimestamp.NANOSECONDS_PER_DAY); - - static boolean beyondLongRange = false; - - private BigDecimal[] randomEpochDayAndNanoOfDay(Random r) { - double randDouble = (r.nextDouble() - 0.5D) * 2.0D; - randDouble *= PisaTimestamp.NANOSECONDS_PER_DAY; - randDouble *= 365 * 10000; - BigDecimal bigDecimal = new BigDecimal(randDouble); - bigDecimal = bigDecimal.setScale(0, RoundingMode.HALF_UP); - - if (bigDecimal.compareTo(BIG_MAX_LONG) > 0 || bigDecimal.compareTo(BIG_MIN_LONG) < 0) { - beyondLongRange = true; - } - - BigDecimal[] divideAndRemainder = bigDecimal.divideAndRemainder(BIG_NANOSECONDS_PER_DAY); - - return new BigDecimal[] {divideAndRemainder[0], divideAndRemainder[1], bigDecimal}; - } - - private BigDecimal pisaTimestampToBig(PisaTimestamp pisaTimestamp) { - BigDecimal bigNanoOfDay = new BigDecimal(pisaTimestamp.getNanoOfDay()); - - BigDecimal bigEpochDay = new BigDecimal(pisaTimestamp.getEpochDay()); - BigDecimal result = bigEpochDay.multiply(BIG_NANOSECONDS_PER_DAY); - result = result.add(bigNanoOfDay); - return result; - } - - @Test - public void testPisaTimestampArithmetic() throws Exception { - - Random r = new Random(1234); - - - for (int i = 0; i < TEST_COUNT; i++) { - BigDecimal[] random1 = randomEpochDayAndNanoOfDay(r); - long epochDay1 = random1[0].longValue(); - long nanoOfDay1 = random1[1].longValue(); - PisaTimestamp pisa1 = new PisaTimestamp(epochDay1, nanoOfDay1); - BigDecimal big1 = random1[2]; - - BigDecimal[] random2 = randomEpochDayAndNanoOfDay(r); - long epochDay2 = random2[0].longValue(); - long nanoOfDay2 = random2[1].longValue(); - PisaTimestamp pisa2 = new PisaTimestamp(epochDay2, nanoOfDay2); - BigDecimal big2 = random2[2]; - - BigDecimal expectedBig; - PisaTimestamp pisaResult = new PisaTimestamp(); - if (i % 2 == 0) { - expectedBig = big1.add(big2); - PisaTimestamp.add(pisa1, pisa2, pisaResult); - } else { - expectedBig = big1.add(big2.negate()); - PisaTimestamp.subtract(pisa1, pisa2, pisaResult); - } - BigDecimal resultBig = pisaTimestampToBig(pisaResult); - assertEquals(expectedBig, resultBig); - - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampColumnVector.java ---------------------------------------------------------------------- diff --git a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampColumnVector.java b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampColumnVector.java new file mode 100644 index 0000000..6e5d5c8 --- /dev/null +++ b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampColumnVector.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector; + +import org.junit.Test; + +import java.io.PrintWriter; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.sql.Timestamp; +import java.util.Date; +import java.util.Random; + +import org.apache.hadoop.hive.common.type.RandomTypeUtil; + +import static org.junit.Assert.*; + +/** + * Test for ListColumnVector + */ +public class TestTimestampColumnVector { + + private static int TEST_COUNT = 5000; + + private static int fake = 0; + + @Test + public void testSaveAndRetrieve() throws Exception { + + Random r = new Random(1234); + TimestampColumnVector timestampColVector = new TimestampColumnVector(); + Timestamp[] randTimestamps = new Timestamp[VectorizedRowBatch.DEFAULT_SIZE]; + + for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) { + Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r); + randTimestamps[i] = randTimestamp; + timestampColVector.set(i, randTimestamp); + } + for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) { + Timestamp retrievedTimestamp = timestampColVector.asScratchTimestamp(i); + Timestamp randTimestamp = randTimestamps[i]; + if (!retrievedTimestamp.equals(randTimestamp)) { + assertTrue(false); + } + } + } + + @Test + public void testTimestampCompare() throws Exception { + Random r = new Random(1234); + TimestampColumnVector timestampColVector = new TimestampColumnVector(); + Timestamp[] randTimestamps = new Timestamp[VectorizedRowBatch.DEFAULT_SIZE]; + Timestamp[] candTimestamps = new Timestamp[VectorizedRowBatch.DEFAULT_SIZE]; + int[] compareToLeftRights = new int[VectorizedRowBatch.DEFAULT_SIZE]; + int[] compareToRightLefts = new int[VectorizedRowBatch.DEFAULT_SIZE]; + + for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) { + Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r); + randTimestamps[i] = randTimestamp; + timestampColVector.set(i, randTimestamp); + Timestamp candTimestamp = RandomTypeUtil.getRandTimestamp(r); + candTimestamps[i] = candTimestamp; + compareToLeftRights[i] = candTimestamp.compareTo(randTimestamp); + compareToRightLefts[i] = randTimestamp.compareTo(candTimestamp); + } + + for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) { + Timestamp retrievedTimestamp = timestampColVector.asScratchTimestamp(i); + Timestamp randTimestamp = randTimestamps[i]; + if (!retrievedTimestamp.equals(randTimestamp)) { + assertTrue(false); + } + Timestamp candTimestamp = candTimestamps[i]; + int compareToLeftRight = timestampColVector.compareTo(candTimestamp, i); + if (compareToLeftRight != compareToLeftRights[i]) { + assertTrue(false); + } + int compareToRightLeft = timestampColVector.compareTo(i, candTimestamp); + if (compareToRightLeft != compareToRightLefts[i]) { + assertTrue(false); + } + } + } + + /* + @Test + public void testGenerate() throws Exception { + PrintWriter writer = new PrintWriter("/Users/you/timestamps.txt"); + Random r = new Random(18485); + for (int i = 0; i < 25; i++) { + Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r); + writer.println(randTimestamp.toString()); + } + for (int i = 0; i < 25; i++) { + Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r, 1965, 2025); + writer.println(randTimestamp.toString()); + } + writer.close(); + } + */ +}