This is an automated email from the ASF dual-hosted git repository. sunchao pushed a commit to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-2.3 by this push: new ac0e85715af HIVE-25054: (2.3) Drop vulnerable jodd-core dependency (#5151) ac0e85715af is described below commit ac0e85715af20cd023895ba85e86b018282c8c10 Author: Cheng Pan <cheng...@apache.org> AuthorDate: Thu Mar 28 12:24:47 2024 +0800 HIVE-25054: (2.3) Drop vulnerable jodd-core dependency (#5151) --- pom.xml | 2 +- ql/pom.xml | 6 - .../ql/io/parquet/timestamp/NanoTimeUtils.java | 2 +- .../parquet/timestamp/datetime/DateTimeStamp.java | 53 +++ .../io/parquet/timestamp/datetime/JDateTime.java | 458 +++++++++++++++++++++ .../timestamp/datetime/JDateTimeDefault.java | 24 ++ .../timestamp/datetime/JulianDateStamp.java | 140 +++++++ .../ql/io/parquet/timestamp/datetime/TimeUtil.java | 258 ++++++++++++ .../hive-webapps/hiveserver2/hiveserver2.jsp | 6 +- 9 files changed, 938 insertions(+), 11 deletions(-) diff --git a/pom.xml b/pom.xml index 52f9edccc3e..4636582c507 100644 --- a/pom.xml +++ b/pom.xml @@ -165,7 +165,6 @@ <jline.version>2.12</jline.version> <jms.version>2.0.2</jms.version> <joda.version>2.8.1</joda.version> - <jodd.version>3.5.2</jodd.version> <json.version>1.8</json.version> <junit.version>4.11</junit.version> <kryo.version>3.0.3</kryo.version> @@ -1120,6 +1119,7 @@ <exclude>**/sit</exclude> <exclude>**/test/queries/**/*.sql</exclude> <exclude>**/PriorityBlockingDeque.java</exclude> + <exclude>**/ql/io/parquet/timestamp/datetime/**</exclude> </excludes> </configuration> </plugin> diff --git a/ql/pom.xml b/ql/pom.xml index 36574a4bc16..b4c8ded04a5 100644 --- a/ql/pom.xml +++ b/ql/pom.xml @@ -325,11 +325,6 @@ <artifactId>groovy-all</artifactId> <version>${groovy.version}</version> </dependency> - <dependency> - <groupId>org.jodd</groupId> - <artifactId>jodd-core</artifactId> - <version>${jodd.version}</version> - </dependency> <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-annotations</artifactId> @@ -879,7 +874,6 @@ <include>javax.jdo:jdo-api</include> <include>commons-lang:commons-lang</include> <include>org.apache.commons:commons-lang3</include> - <include>org.jodd:jodd-core</include> <include>com.tdunning:json</include> <include>org.apache.avro:avro</include> <include>org.apache.avro:avro-mapred</include> diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTimeUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTimeUtils.java index 3fd75d24f3f..e8a2adc9e4d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTimeUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTimeUtils.java @@ -19,7 +19,7 @@ import java.util.GregorianCalendar; import java.util.TimeZone; import java.util.concurrent.TimeUnit; -import jodd.datetime.JDateTime; +import org.apache.hadoop.hive.ql.io.parquet.timestamp.datetime.JDateTime; /** * Utilities for converting from java.sql.Timestamp to parquet timestamp. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/datetime/DateTimeStamp.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/datetime/DateTimeStamp.java new file mode 100644 index 00000000000..8ef45b8e37a --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/datetime/DateTimeStamp.java @@ -0,0 +1,53 @@ +// Copyright (c) 2003-2014, Jodd Team (jodd.org). All Rights Reserved. + +package org.apache.hadoop.hive.ql.io.parquet.timestamp.datetime; + + +import java.io.Serializable; + + +/** + * Generic date time stamp just stores and holds date and time information. + * This class does not contain any date/time logic, neither guarantees + * that date is valid. + * + * @see JDateTime + * @see JulianDateStamp + */ +public class DateTimeStamp implements Serializable { + + /** + * Year. + */ + public int year; + + /** + * Month, range: [1 - 12] + */ + public int month = 1; + + /** + * Day, range: [1 - 31] + */ + public int day = 1; + + /** + * Hour, range: [0 - 23] + */ + public int hour; + + /** + * Minute, range [0 - 59] + */ + public int minute; + + /** + * Second, range: [0 - 59] + */ + public int second; + + /** + * Millisecond, range: [0 - 1000] + */ + public int millisecond; +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/datetime/JDateTime.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/datetime/JDateTime.java new file mode 100644 index 00000000000..5b50dd6cd62 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/datetime/JDateTime.java @@ -0,0 +1,458 @@ +// Copyright (c) 2003-2014, Jodd Team (jodd.org). All Rights Reserved. + +package org.apache.hadoop.hive.ql.io.parquet.timestamp.datetime; + +import java.util.TimeZone; + +/** + * Universal all-in-one date and time class that uses Astronomical Julian + * Dates for time calculations. Guaranteed precision for all manipulations + * and calculations is up to 1 ms (0.001 sec). + * + * <p> + * The Julian day or Julian day number (JDN) is the (integer) number of days that + * have elapsed since Monday, January 1, 4713 BC in the proleptic Julian calendar 1. + * That day is counted as Julian day zero. Thus the multiples of 7 are Mondays. + * Negative values can also be used. + * + * <p> + * The Julian Date (JD) is the number of days (with decimal fraction of the day) that + * have elapsed since 12 noon Greenwich Mean Time (UT or TT) of that day. + * Rounding to the nearest integer gives the Julian day number. + * + * <p> + * <code>JDateTime</code> contains date/time information for current day. By + * default, behaviour and formats are set to ISO standard, although this may + * be changed. + * + * <p> + * <code>JDateTime</code> can be set in many different ways by using <code>setXxx()</code> + * methods or equivalent constructors. Moreover, date time information may be loaded from an instance + * of any available java date-time class. This functionality can be easily + * enhanced for any custom date/time class. Furthermore, <code>JDateTime</code> + * can be converted to any such date/time class. + * + * <p> + * Rolling dates with <code>JDateTime</code> is easy. For this + * <code>JDateTime</code> contains many <code>addXxx()</code> methods. Time can be added + * or subtracted with any value or more values at once. All combinations are + * valid. Calculations also performs month fixes by default. + * + * <p> + * <code>JDateTime</code> behaviour is set by several attributes (or + * parameters). Each one contains 2 values: one is the default value, used by + * all instances of <code>JDateTime</code> and the other one is just for a + * specific instance of <code>JDateTime</code>. This means that it is + * possible to set behaviour of all instances at once or of one particular + * instance. + * + * <p> + * Bellow is the list of behaviour attributes: + * + * <ul> + * + * <li>monthFix - since months do not have the same number of days, adding + * months and years may be calculated in two different ways: with or + * without month fix. when month fix is on, <code>JDateTime</code> will + * additionally fix all time adding and fix the date. For example, adding + * one month to 2003-01-31 will give 2003-02-28 and not 2003-03-03. + * By default, monthFix is turned on and set to <code>true</code>. + * </li> + * + * <li>locale - current locale, used for getting names during formatting the + * output string. + * </li> + * + * <li>timezone - current timezone</li> + * + * <li>format - is String that describes how time is converted and parsed to + * and from a String. Default format matches ISO standard. An instance of + * <code>JdtFormatter</code> parses and uses this template.</li> + * + * <li>week definition - is used for specifying the definition of the week. + * Week is defined with first day of the week and with the must have day. A + * must have day is a day that must exist in the 1st week of the year. For + * example, default value is Thursday (4) as specified by ISO standard. + * Alternatively, instead of must have day, minimal days of week may be used, + * since this two numbers are in relation. + * </li> + * + * </ul> + * + * Optimization: although based on heavy calculations, <code>JDateTime</code> + * works significantly faster then java's <code>Calendar</code>s. Since + * <code>JDateTime</code> doesn't use lazy initialization, setXxx() method is + * slower. However, this doesn't have much effect to the global performances: + * settings are usually not used without gettings:) As soon as any other method is + * used (getXxx() or addXxx()) performances of <code>JDateTime</code> becomes + * significantly better. + * + * <p> + * Year 1582 is (almost:) working: after 1582-10-04 (Thursday) is 1582-10-15 (Friday). + * Users must be aware of this when doing time rolling before across this period. + * + * <p> + * More info: <a href="http://en.wikipedia.org/wiki/Julian_Date">Julian Date on Wikipedia</a> + */ +public class JDateTime { + + // day of week names + public static final int MONDAY = 1; + + /** + * {@link DateTimeStamp} for current date. + */ + protected DateTimeStamp time = new DateTimeStamp(); + + /** + * Day of week, range: [1-7] == [Monday - Sunday] + */ + protected int dayofweek; + + /** + * Day of year, range: [1-365] or [1-366] + */ + protected int dayofyear; + + /** + * Leap year flag. + */ + protected boolean leap; + + /** + * Week of year, range: [1-52] or [1-53] + */ + protected int weekofyear; + + /** + * Week of month. + */ + protected int weekofmonth; + + /** + * Current Julian Date. + */ + protected JulianDateStamp jdate; + + + // ---------------------------------------------------------------- some precalculated times + + /** + * Sets current Julian Date. This is the core of the JDateTime class and it + * is used by all other classes. This method performs all calculations + * required for whole class. + * + * @param jds current julian date + */ + public void setJulianDate(JulianDateStamp jds) { + setJdOnly(jds.clone()); + calculateAdditionalData(); + } + + /** + * Returns JDN. Note that JDN is not equal to integer part of julian date. It is calculated by + * rounding to the nearest integer. + */ + public int getJulianDayNumber() { + return jdate.getJulianDayNumber(); + } + + /** + * Internal method for calculating various data other then date/time. + */ + private void calculateAdditionalData() { + this.leap = TimeUtil.isLeapYear(time.year); + this.dayofweek = calcDayOfWeek(); + this.dayofyear = calcDayOfYear(); + this.weekofyear = calcWeekOfYear(firstDayOfWeek, mustHaveDayOfFirstWeek); + this.weekofmonth = calcWeekNumber(time.day, this.dayofweek); + } + + /** + * Internal method that just sets the time stamp and not all other additional + * parameters. Used for faster calculations only and only by main core + * set/add methods. + * + * @param jds julian date + */ + private void setJdOnly(JulianDateStamp jds) { + jdate = jds; + time = TimeUtil.fromJulianDate(jds); + } + + /** + * Core method that sets date and time. All others set() methods use this + * one. Milliseconds are truncated after 3rd digit. + * + * @param year year to set + * @param month month to set + * @param day day to set + * @param hour hour to set + * @param minute minute to set + * @param second second to set + */ + public void set(int year, int month, int day, int hour, int minute, int second, int millisecond) { + + // fix seconds fractions because of float point arithmetic + //second = ((int) second) + ((int) ((second - (int)second) * 1000 + 1e-9) / 1000.0); +/* + double ms = (second - (int)second) * 1000; + if (ms > 999) { + ms = 999; + } else { + ms += 1.0e-9; + } + second = ((int) second) + ((int) ms / 1000.0); +*/ + jdate = TimeUtil.toJulianDate(year, month, day, hour, minute, second, millisecond); + + // if given time is valid it means that there is no need to calculate it + // again from already calculated Julian date. however, it is still + // necessary to fix milliseconds to match the value that would be + // calculated as setJulianDate() is used. This fix only deals with the + // time, not doing the complete and more extensive date calculation. + // this means that method works faster when regular date is specified. +/* + if (TimeUtil.isValidDateTime(year, month, day, hour, minute, second, millisecond)) { + + int ka = (int)(jdate.fraction + 0.5); + double frac = jdate.fraction + 0.5 - ka + 1.0e-10; + + // hour with minute and second included as fraction + double d_hour = frac * 24.0; + + // minute with second included as a fraction + double d_minute = (d_hour - (int)d_hour) * 60.0; + + second = (d_minute - (int)d_minute) * 60.0; + + // fix calculation errors + second = ((int) (second * 10000) + 0.5) / 10000.0; + + time.year = year; time.month = month; time.day = day; + time.hour = hour; time.minute = minute; time.second = second; + setParams(); + } else { + setJulianDate(jdate); + } +*/ + if (TimeUtil.isValidDateTime(year, month, day, hour, minute, second, millisecond)) { + time.year = year; time.month = month; time.day = day; + time.hour = hour; time.minute = minute; time.second = second; + time.millisecond = millisecond; + calculateAdditionalData(); + } else { + setJulianDate(jdate); + } + + } + + // ---------------------------------------------------------------- core calculations + + /** + * Calculates day of week. + */ + private int calcDayOfWeek() { + int jd = (int)(jdate.doubleValue() + 0.5); + return (jd % 7) + 1; + //return (jd + 1) % 7; // return 0 (Sunday), 1 (Monday),... + } + + private static final int NUM_DAYS[] = {-1, 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334}; // 1-based + private static final int LEAP_NUM_DAYS[] = {-1, 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335}; // 1-based + + /** + * Calculates day of year. + */ + private int calcDayOfYear() { + if (leap == true) { + return LEAP_NUM_DAYS[time.month] + time.day; + } + return NUM_DAYS[time.month] + time.day; + } + + + /** + * Calculates week of year. Based on: + * "Algorithm for Converting Gregorian Dates to ISO 8601 Week Date" + * by Rick McCarty, 1999 + * + * @param start first day of week + * @param must must have day of week + * + * @return week of year number + */ + private int calcWeekOfYear(int start, int must) { + + // is modification required? + // modification is a fix for the days of year because of the different + // starting day of week. when modification is required, one week is added + // or subtracted to the current day, so calculation of the week of year + // would be correct. + int delta = 0; + if (start <= this.dayofweek) { + if (must < start) { + delta = 7; + } + } else { + if (must >= start) { + delta = -7; + } + } + + int jd = (int)(jdate.doubleValue() + 0.5) + delta; + int WeekDay = (jd % 7) + 1; + + int time_year = time.year; + int DayOfYearNumber = this.dayofyear + delta; + if (DayOfYearNumber < 1) { + time_year--; + DayOfYearNumber = TimeUtil.isLeapYear(time_year) ? 366 + DayOfYearNumber: 365 + DayOfYearNumber; + } else if (DayOfYearNumber > (this.leap ? 366 : 365)) { + DayOfYearNumber = this.leap ? DayOfYearNumber - 366: DayOfYearNumber - 365; + time_year++; + } + + // modification, if required, is finished. proceed to the calculation. + + int firstDay = jd - DayOfYearNumber + 1; + int Jan1WeekDay = (firstDay % 7) + 1; + + // find if the date falls in YearNumber Y - 1 set WeekNumber to 52 or 53 + int YearNumber = time_year; + int WeekNumber = 52; + if ((DayOfYearNumber <= (8 - Jan1WeekDay)) && (Jan1WeekDay > must)) { + YearNumber--; + if ((Jan1WeekDay == must + 1) || ( (Jan1WeekDay == must + 2) && (TimeUtil.isLeapYear(YearNumber)) ) ) { + WeekNumber = 53; + } + } + + // set WeekNumber to 1 to 53 if date falls in YearNumber + int m = 365; + if (YearNumber == time_year) { + if (TimeUtil.isLeapYear(time_year) == true) { + m = 366; + } + if ((m - DayOfYearNumber) < (must - WeekDay)) { + YearNumber = time_year + 1; + WeekNumber = 1; + } + } + + if (YearNumber == time_year) { + int n = DayOfYearNumber + (7 - WeekDay) + (Jan1WeekDay - 1); + WeekNumber = n / 7; + if (Jan1WeekDay > must) { + WeekNumber -= 1; + } + } + return WeekNumber; + } + + /** + * Return the week number of a day, within a period. This may be the week number in + * a year, or the week number in a month. Usually this will be a value >= 1, but if + * some initial days of the period are excluded from week 1, because + * minimalDaysInFirstWeek is > 1, then the week number will be zero for those + * initial days. Requires the day of week for the given date in order to determine + * the day of week of the first day of the period. + * + * @param dayOfPeriod + * Day-of-year or day-of-month. Should be 1 for first day of period. + * @param dayOfWeek day of week + * + * @return Week number, one-based, or zero if the day falls in part of the + * month before the first week, when there are days before the first + * week because the minimum days in the first week is more than one. + */ + private int calcWeekNumber(int dayOfPeriod, int dayOfWeek) { + // Determine the day of the week of the first day of the period + // in question (either a year or a month). Zero represents the + // first day of the week on this calendar. + int periodStartDayOfWeek = (dayOfWeek - firstDayOfWeek - dayOfPeriod + 1) % 7; + if (periodStartDayOfWeek < 0) { + periodStartDayOfWeek += 7; + } + + // Compute the week number. Initially, ignore the first week, which + // may be fractional (or may not be). We add periodStartDayOfWeek in + // order to fill out the first week, if it is fractional. + int weekNo = (dayOfPeriod + periodStartDayOfWeek - 1) / 7; + + // If the first week is long enough, then count it. If + // the minimal days in the first week is one, or if the period start + // is zero, we always increment weekNo. + if ((7 - periodStartDayOfWeek) >= minDaysInFirstWeek) { + ++weekNo; + } + + return weekNo; + } + + // ---------------------------------------------------------------- add/sub time + + /** + * Sets date, time is set to midnight (00:00:00.000). + * + * @param year year to set + * @param month month to set + * @param day day to set + */ + public void set(int year, int month, int day) { + set(year, month, day, 0, 0, 0, 0); + } + + /** + * Constructor that sets just date. Time is set to 00:00:00. + * + * @param year year to set + * @param month month to set + * @param day day to set + * + * @see #set(int, int, int) + */ + public JDateTime(int year, int month, int day) { + this.set(year, month, day); + } + + /** + * Returns current year. + */ + public int getYear() { + return time.year; + } + + /** + * Returns current month. + */ + public int getMonth() { + return time.month; + } + + /** + * Returns current day of month. + */ + public int getDay() { + return time.day; + } + + // ---------------------------------------------------------------- other gets + /** + * Creates <code>JDateTime</code> from <code>double</code> that represents JD. + */ + public JDateTime(double jd) { + setJulianDate(new JulianDateStamp(jd)); + } + + // ---------------------------------------------------------------- week definitions + + protected int firstDayOfWeek = JDateTimeDefault.firstDayOfWeek; + protected int mustHaveDayOfFirstWeek = JDateTimeDefault.mustHaveDayOfFirstWeek; + + + // ---------------------------------------------------------------- week definitions (alt) + + protected int minDaysInFirstWeek = JDateTimeDefault.minDaysInFirstWeek; + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/datetime/JDateTimeDefault.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/datetime/JDateTimeDefault.java new file mode 100644 index 00000000000..8bd5ff5297e --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/datetime/JDateTimeDefault.java @@ -0,0 +1,24 @@ +// Copyright (c) 2003-2014, Jodd Team (jodd.org). All Rights Reserved. + +package org.apache.hadoop.hive.ql.io.parquet.timestamp.datetime; + +/** + * Defaults for {@link JDateTime}. + */ +public class JDateTimeDefault { + + /** + * Default definition of first day of week. + */ + public static int firstDayOfWeek = JDateTime.MONDAY; + + /** + * Default number of days first week of year must have. + */ + public static int mustHaveDayOfFirstWeek = 4; + + /** + * Default minimal number of days firs week of year must have. + */ + public static int minDaysInFirstWeek = 4; +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/datetime/JulianDateStamp.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/datetime/JulianDateStamp.java new file mode 100644 index 00000000000..c13d7462652 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/datetime/JulianDateStamp.java @@ -0,0 +1,140 @@ +// Copyright (c) 2003-2014, Jodd Team (jodd.org). All Rights Reserved. + +package org.apache.hadoop.hive.ql.io.parquet.timestamp.datetime; + +import java.io.Serializable; + +/** + * Julian Date stamp, for high precision calculations. Julian date is a real + * number and it basically consist of two parts: integer and fraction. Integer + * part carries date information, fraction carries time information. + * + * <p> + * The Julian day or Julian day number (JDN) is the (integer) number of days that + * have elapsed since Monday, January 1, 4713 BC in the proleptic Julian calendar 1. + * That day is counted as Julian day zero. Thus the multiples of 7 are Mondays. + * Negative values can also be used. + * + * <p> + * The Julian Date (JD) is the number of days (with decimal fraction of the day) that + * have elapsed since 12 noon Greenwich Mean Time (UT or TT) of that day. + * Rounding to the nearest integer gives the Julian day number. + * <p> + * For calculations that will have time precision of 1e-3 seconds, both + * fraction and integer part must have enough digits in it. The problem is + * that integer part is big and, on the other hand fractional is small, and + * since final julian date is a sum of this two values, some fraction + * numerals may be lost. Therefore, for higher precision both + * fractional and integer part of julian date real number has to be + * preserved. + * <p> + * This class stores the unmodified fraction part, but not all digits + * are significant! For 1e-3 seconds precision, only 8 digits after + * the decimal point are significant. + * + * @see TimeUtil + * @see JDateTime + * @see DateTimeStamp + */ +public class JulianDateStamp implements Serializable, Cloneable { + + /** + * Integer part of the Julian Date (JD). + */ + protected int integer; + + /** + * Fraction part of the Julian Date (JD). + * Should be always in [0.0, 1.0) range. + */ + protected double fraction; + + /** + * Returns JDN. Note that JDN is not equal to {@link #integer}. It is calculated by + * rounding to the nearest integer. + */ + public int getJulianDayNumber() { + if (fraction >= 0.5) { + return integer + 1; + } + return integer; + } + + // ---------------------------------------------------------------- ctors + + /** + * Creates JD from a <code>double</code>. + */ + public JulianDateStamp(double jd) { + set(jd); + } + + /** + * Creates JD from both integer and fractional part using normalization. + * Normalization occurs when fractional part is out of range. + * + * @see #set(int, double) + * + * @param i integer part + * @param f fractional part should be in range [0.0, 1.0) + */ + public JulianDateStamp(int i, double f) { + set(i, f); + } + + // ---------------------------------------------------------------- conversion + + + /** + * Returns <code>double</code> value of JD. + * <b>CAUTION</b>: double values may not be suit for precision math due to + * loss of precision. + */ + public double doubleValue() { + return (double)integer + fraction; + } + + /** + * Returns string representation of JD. + * + * @return julian integer as string + */ + @Override + public String toString() { + String s = Double.toString(fraction); + int i = s.indexOf('.'); + s = s.substring(i); + return integer + s; + } + + + // ---------------------------------------------------------------- math + + + /** + * Sets integer and fractional part with normalization. + * Normalization means that if double is out of range, + * values will be correctly fixed. + */ + public void set(int i, double f) { + integer = i; + int fi = (int) f; + f -= fi; + integer += fi; + if (f < 0) { + f += 1; + integer--; + } + this.fraction = f; + } + + public void set(double jd) { + integer = (int)jd; + fraction = jd - (double)integer; + } + + @Override + protected JulianDateStamp clone() { + return new JulianDateStamp(this.integer, this.fraction); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/datetime/TimeUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/datetime/TimeUtil.java new file mode 100644 index 00000000000..e0c40ba8b07 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/datetime/TimeUtil.java @@ -0,0 +1,258 @@ +// Copyright (c) 2003-2014, Jodd Team (jodd.org). All Rights Reserved. + +package org.apache.hadoop.hive.ql.io.parquet.timestamp.datetime; + + +/** + * Date time calculations and utilities. <code>TimeUtil</code> is used by + * {@link JDateTime} and it contains few utilities that may be used + * elsewhere, although {@link JDateTime} is recommended for all time + * manipulation. + */ +public class TimeUtil { + // ---------------------------------------------------------------- misc calc + + /** + * Check if the given year is leap year. + * + * @return <code>true</code> if the year is a leap year + */ + public static boolean isLeapYear(int y) { + boolean result = false; + + if (((y % 4) == 0) && // must be divisible by 4... + ((y < 1582) || // and either before reform year... + ((y % 100) != 0) || // or not a century... + ((y % 400) == 0))) { // or a multiple of 400... + result = true; // for leap year. + } + return result; + } + + private static final int[] MONTH_LENGTH = {0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + private static final int[] LEAP_MONTH_LENGTH = {0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + + /** + * Returns the length of the specified month in days. Month is 1 for January + * and 12 for December. It works only for years after 1582. + * + * @return length of the specified month in days + */ + public static int getMonthLength(int year, int m) { + if ((m < 1) || (m > 12)) { + return -1; + } + if (isLeapYear(year)) { + return LEAP_MONTH_LENGTH[m]; + } + return MONTH_LENGTH[m]; + } + + + // ---------------------------------------------------------------- valid + + /** + * Checks if date is valid. + * + * @return <code>true</code> if date is valid, otherwise <code>false</code> + */ + public static boolean isValidDate(int year, int month, int day) { + if ((month < 1) || (month > 12)) { + return false; + } + int ml = getMonthLength(year, month); + //noinspection RedundantIfStatement + if ((day < 1) || (day > ml)) { + return false; + } + return true; + } + + /** + * Checks if time is valid. + * + * @param hour hour to check + * @param minute minute to check + * @param second second to check + * + * @return <code>true</code> if time is valid, otherwise <code>false</code> + */ + public static boolean isValidTime(int hour, int minute, int second, int millisecond) { + if ((hour < 0) || (hour >= 24)) { + return false; + } + if ((minute < 0) || (minute >= 60)) { + return false; + } + if ((second < 0) || (second >= 60)) { + return false; + } + //noinspection RedundantIfStatement + if ((millisecond < 0) || (millisecond >= 1000)) { + return false; + } + return true; + } + + /** + * Checks if date and time are valid. + * + * @param year year to check + * @param month month to check + * @param day day to check + * @param hour hour to check + * @param minute minute to check + * @param second second to check + * + * @return <code>true</code> if date and time are valid, otherwise <code>false</code> + */ + public static boolean isValidDateTime(int year, int month, int day, int hour, int minute, int second, int millisecond) { + return (isValidDate(year, month, day) && isValidTime(hour, minute, second, millisecond)); + } + + + /** + * Calculates Astronomical Julian Date from given time.<p> + * + * Astronomical Julian Dates are counting from noon of the January 1st, -4712 + * (julian date 0 is -4712/01/01 12:00:00). Zero year exist. Julian Date + * is always GMT, there are no timezones. + * <p> + * + * Algorithm based on Numerical Recipesin C, 2nd ed., Cambridge University + * Press 1992, modified and enhanced by Igor Spasic. + * + * @param year year + * @param month month + * @param day day + * @param hour hour + * @param minute minute + * @param second second + * + * @return julian time stamp + */ + public static JulianDateStamp toJulianDate(int year, int month, int day, int hour, int minute, int second, int millisecond) { + + // month range fix + if ((month > 12) || (month < -12)) { + month--; + int delta = month / 12; + year += delta; + month -= delta * 12; + month++; + } + if (month < 0) { + year--; + month += 12; + } + + // decimal day fraction + double frac = (hour / 24.0) + (minute / 1440.0) + (second / 86400.0) + (millisecond / 86400000.0); + if (frac < 0) { // negative time fix + int delta = ((int)(-frac)) + 1; + frac += delta; + day -= delta; + } + //double gyr = year + (0.01 * month) + (0.0001 * day) + (0.0001 * frac) + 1.0e-9; + double gyr = year + (0.01 * month) + (0.0001 * (day + frac)) + 1.0e-9; + + // conversion factors + int iy0; + int im0; + if (month <= 2) { + iy0 = year - 1; + im0 = month + 12; + } else { + iy0 = year; + im0 = month; + } + int ia = iy0 / 100; + int ib = 2 - ia + (ia >> 2); + + // calculate julian date + int jd; + if (year <= 0) { + jd = (int)((365.25 * iy0) - 0.75) + (int)(30.6001 * (im0 + 1)) + day + 1720994; + } else { + jd = (int)(365.25 * iy0) + (int)(30.6001 * (im0 + 1)) + day + 1720994; + } + if (gyr >= 1582.1015) { // on or after 15 October 1582 + jd += ib; + } + //return jd + frac + 0.5; + + return new JulianDateStamp(jd, frac + 0.5); + } + + + /** + * Calculates time stamp from Astronomical Julian Date. + * Algorithm based on Numerical Recipesin C, 2nd ed., Cambridge University + * Press 1992. + * + * @param jds julian date stamp + * + * @return time stamp + */ + public static DateTimeStamp fromJulianDate(JulianDateStamp jds) { + DateTimeStamp time = new DateTimeStamp(); + int year, month, day; + double frac; + int jd, ka, kb, kc, kd, ke, ialp; + + //double JD = jds.doubleValue();//jdate; + //jd = (int)(JD + 0.5); // integer julian date + //frac = JD + 0.5 - (double)jd + 1.0e-10; // day fraction + + ka = (int)(jds.fraction + 0.5); + jd = jds.integer + ka; + frac = jds.fraction + 0.5 - ka + 1.0e-10; + + ka = jd; + if (jd >= 2299161) { + ialp = (int)(((double)jd - 1867216.25) / (36524.25)); + ka = jd + 1 + ialp - (ialp >> 2); + } + kb = ka + 1524; + kc = (int)(((double)kb - 122.1) / 365.25); + kd = (int)((double)kc * 365.25); + ke = (int)((double)(kb - kd) / 30.6001); + day = kb - kd - ((int)((double)ke * 30.6001)); + if (ke > 13) { + month = ke - 13; + } else { + month = ke - 1; + } + if ((month == 2) && (day > 28)){ + day = 29; + } + if ((month == 2) && (day == 29) && (ke == 3)) { + year = kc - 4716; + } else if (month > 2) { + year = kc - 4716; + } else { + year = kc - 4715; + } + time.year = year; + time.month = month; + time.day = day; + + // hour with minute and second included as fraction + double d_hour = frac * 24.0; + time.hour = (int) d_hour; // integer hour + + // minute with second included as a fraction + double d_minute = (d_hour - (double)time.hour) * 60.0; + time.minute = (int) d_minute; // integer minute + + double d_second = (d_minute - (double)time.minute) * 60.0; + time.second = (int) d_second; // integer seconds + + double d_millis = (d_second - (double)time.second) * 1000.0; + + // fix calculation errors + time.millisecond = (int) (((d_millis * 10) + 0.5) / 10); + + return time; + } +} diff --git a/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp b/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp index 33797edc03e..79d4a73a2e4 100644 --- a/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp +++ b/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp @@ -18,6 +18,7 @@ */ --%> <%@ page contentType="text/html;charset=UTF-8" + import="org.apache.commons.lang3.StringEscapeUtils" import="org.apache.hadoop.conf.Configuration" import="org.apache.hadoop.hive.conf.HiveConf" import="org.apache.hadoop.hive.conf.HiveConf.ConfVars" @@ -31,7 +32,6 @@ import="java.util.Collection" import="java.util.Date" import="java.util.List" - import="jodd.util.HtmlEncoder" %> <% @@ -147,7 +147,7 @@ for (HiveSession hiveSession: hiveSessions) { %> <tr> <td><%= operation.getUserName() %></td> - <td><%= HtmlEncoder.strict(operation.getQueryDisplay() == null ? "Unknown" : operation.getQueryDisplay().getQueryString()) %></td> + <td><%= StringEscapeUtils.escapeHtml4(operation.getQueryDisplay() == null ? "Unknown" : operation.getQueryDisplay().getQueryString()) %></td> <td><%= operation.getExecutionEngine() %> <td><%= operation.getState() %></td> <td><%= new Date(operation.getBeginTime()) %></td> @@ -188,7 +188,7 @@ for (HiveSession hiveSession: hiveSessions) { %> <tr> <td><%= operation.getUserName() %></td> - <td><%= HtmlEncoder.strict(operation.getQueryDisplay() == null ? "Unknown" : operation.getQueryDisplay().getQueryString()) %></td> + <td><%= StringEscapeUtils.escapeHtml4(operation.getQueryDisplay() == null ? "Unknown" : operation.getQueryDisplay().getQueryString()) %></td> <td><%= operation.getExecutionEngine() %> <td><%= operation.getState() %></td> <td><%= operation.getElapsedTime()/1000 %></td>