Author: jdere
Date: Mon Feb 2 19:19:01 2015
New Revision: 1656544
URL: http://svn.apache.org/r1656544
Log:
HIVE-5472: support a simple scalar which returns the current timestamp (Jason
Dere, reviewed by Thejas Nair)
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentDate.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentTimestamp.java
hive/trunk/ql/src/test/queries/clientpositive/current_date_timestamp.q
hive/trunk/ql/src/test/results/clientpositive/current_date_timestamp.q.out
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL:
http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1656544&r1=1656543&r2=1656544&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
(original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Mon
Feb 2 19:19:01 2015
@@ -842,20 +842,22 @@ public class HiveConf extends Configurat
// test mode in hive mode
HIVETESTMODE("hive.test.mode", false,
- "Whether Hive is running in test mode. If yes, it turns on sampling
and prefixes the output tablename."),
+ "Whether Hive is running in test mode. If yes, it turns on sampling
and prefixes the output tablename.",
+ false),
HIVETESTMODEPREFIX("hive.test.mode.prefix", "test_",
- "In test mode, specfies prefixes for the output table"),
+ "In test mode, specfies prefixes for the output table", false),
HIVETESTMODESAMPLEFREQ("hive.test.mode.samplefreq", 32,
"In test mode, specfies sampling frequency for table, which is not
bucketed,\n" +
"For example, the following query:\n" +
" INSERT OVERWRITE TABLE dest SELECT col1 from src\n" +
"would be converted to\n" +
" INSERT OVERWRITE TABLE test_dest\n" +
- " SELECT col1 from src TABLESAMPLE (BUCKET 1 out of 32 on rand(1))"),
+ " SELECT col1 from src TABLESAMPLE (BUCKET 1 out of 32 on rand(1))",
false),
HIVETESTMODENOSAMPLE("hive.test.mode.nosamplelist", "",
- "In test mode, specifies comma separated table names which would not
apply sampling"),
- HIVETESTMODEDUMMYSTATAGGR("hive.test.dummystats.aggregator", "", "internal
variable for test"),
- HIVETESTMODEDUMMYSTATPUB("hive.test.dummystats.publisher", "", "internal
variable for test"),
+ "In test mode, specifies comma separated table names which would not
apply sampling", false),
+ HIVETESTMODEDUMMYSTATAGGR("hive.test.dummystats.aggregator", "", "internal
variable for test", false),
+ HIVETESTMODEDUMMYSTATPUB("hive.test.dummystats.publisher", "", "internal
variable for test", false),
+ HIVETESTCURRENTTIMESTAMP("hive.test.currenttimestamp", null, "current
timestamp for test", false),
HIVEMERGEMAPFILES("hive.merge.mapfiles", true,
"Merge small files at the end of a map-only job"),
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1656544&r1=1656543&r2=1656544&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Mon Feb 2
19:19:01 2015
@@ -381,6 +381,8 @@ public class Driver implements CommandPr
String queryId = QueryPlan.makeQueryId();
conf.setVar(HiveConf.ConfVars.HIVEQUERYID, queryId);
+ SessionState.get().setupQueryCurrentTimestamp();
+
try {
command = new VariableSubstitution().substitute(conf,command);
ctx = new Context(conf);
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1656544&r1=1656543&r2=1656544&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
Mon Feb 2 19:19:01 2015
@@ -304,6 +304,8 @@ public final class FunctionRegistry {
registerUDF("~", UDFOPBitNot.class, true);
registerGenericUDF("current_database", UDFCurrentDB.class);
+ registerGenericUDF("current_date", GenericUDFCurrentDate.class);
+ registerGenericUDF("current_timestamp", GenericUDFCurrentTimestamp.class);
registerGenericUDF("isnull", GenericUDFOPNull.class);
registerGenericUDF("isnotnull", GenericUDFOPNotNull.class);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g?rev=1656544&r1=1656543&r2=1656544&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g Mon Feb
2 19:19:01 2015
@@ -270,6 +270,8 @@ KW_UNBOUNDED: 'UNBOUNDED';
KW_PRECEDING: 'PRECEDING';
KW_FOLLOWING: 'FOLLOWING';
KW_CURRENT: 'CURRENT';
+KW_CURRENT_DATE: 'CURRENT_DATE';
+KW_CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP';
KW_LESS: 'LESS';
KW_MORE: 'MORE';
KW_OVER: 'OVER';
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g?rev=1656544&r1=1656543&r2=1656544&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
Mon Feb 2 19:19:01 2015
@@ -173,11 +173,29 @@ function
-> ^(TOK_FUNCTIONDI functionName
(selectExpression+)?)
;
+nonParenthesizedFunction
+@init { gParent.pushMsg("non-parenthesized function name", state); }
+@after { gParent.popMsg(state); }
+ :
+ nonParenthesizedFunctionName
+ -> ^(TOK_FUNCTION nonParenthesizedFunctionName)
+ ;
+
+nonParenthesizedFunctionName
+@init { gParent.pushMsg("non-parenthesized function name", state); }
+@after { gParent.popMsg(state); }
+ :
+ KW_CURRENT_DATE | KW_CURRENT_TIMESTAMP
+ ;
+
functionName
@init { gParent.pushMsg("function name", state); }
@after { gParent.popMsg(state); }
: // Keyword IF is also a function name
KW_IF | KW_ARRAY | KW_MAP | KW_STRUCT | KW_UNIONTYPE | functionIdentifier
+ |
+ // This allows current_timestamp() to work as well as current_timestamp
+ nonParenthesizedFunctionName
;
castExpression
@@ -273,6 +291,7 @@ atomExpression
| castExpression
| caseExpression
| whenExpression
+ | nonParenthesizedFunction
| (functionName LPAREN) => function
| tableOrColumn
| LPAREN! expression RPAREN!
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1656544&r1=1656543&r2=1656544&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
Mon Feb 2 19:19:01 2015
@@ -25,6 +25,7 @@ import java.io.InputStream;
import java.io.PrintStream;
import java.net.URI;
import java.net.URLClassLoader;
+import java.sql.Timestamp;
import java.util.*;
import org.apache.commons.io.FileUtils;
@@ -252,6 +253,11 @@ public class SessionState {
private final Set<String> preReloadableAuxJars = new HashSet<String>();
/**
+ * CURRENT_TIMESTAMP value for query
+ */
+ private Timestamp queryCurrentTimestamp;
+
+ /**
* Get the lineage state stored in this session.
*
* @return LineageState
@@ -1410,4 +1416,27 @@ public class SessionState {
return Integer.toString(nextValueTempTableSuffix++);
}
+ /**
+ * Initialize current timestamp, other necessary query initialization.
+ */
+ public void setupQueryCurrentTimestamp() {
+ queryCurrentTimestamp = new Timestamp(System.currentTimeMillis());
+
+ // Provide a facility to set current timestamp during tests
+ if (conf.getBoolVar(ConfVars.HIVE_IN_TEST)) {
+ String overrideTimestampString =
+ HiveConf.getVar(conf, HiveConf.ConfVars.HIVETESTCURRENTTIMESTAMP,
null);
+ if (overrideTimestampString != null && overrideTimestampString.length()
> 0) {
+ queryCurrentTimestamp = Timestamp.valueOf(overrideTimestampString);
+ }
+ }
+ }
+
+ /**
+ * Get query current timestamp
+ * @return
+ */
+ public Timestamp getQueryCurrentTimestamp() {
+ return queryCurrentTimestamp;
+ }
}
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentDate.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentDate.java?rev=1656544&view=auto
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentDate.java
(added)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentDate.java
Mon Feb 2 19:19:01 2015
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import java.sql.Date;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+
+// If there is a new UDFType to describe a function that is deterministic
within a query
+// but changes value between queries, this function would fall into that
category.
+@UDFType(deterministic = true)
+@Description(name = "current_date",
+ value = "_FUNC_() - Returns the current date at the start of query
evaluation."
+ + " All calls of current_date within the same query return the same
value.")
+public class GenericUDFCurrentDate extends GenericUDF {
+
+ protected DateWritable currentDate;
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments)
+ throws UDFArgumentException {
+ if (arguments.length != 0) {
+ throw new UDFArgumentLengthException(
+ "The function CURRENT_DATE does not take any arguments, but found "
+ + arguments.length);
+ }
+
+ if (currentDate == null) {
+ Date dateVal =
+
Date.valueOf(SessionState.get().getQueryCurrentTimestamp().toString().substring(0,
10));
+ currentDate = new DateWritable(dateVal);
+ }
+
+ return PrimitiveObjectInspectorFactory.writableDateObjectInspector;
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ return currentDate;
+ }
+
+ public DateWritable getCurrentDate() {
+ return currentDate;
+ }
+
+ public void setCurrentDate(DateWritable currentDate) {
+ this.currentDate = currentDate;
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ return "CURRENT_DATE()";
+ }
+
+
+ @Override
+ public void copyToNewInstance(Object newInstance) throws
UDFArgumentException {
+ super.copyToNewInstance(newInstance);
+ // Need to preserve currentDate
+ GenericUDFCurrentDate other = (GenericUDFCurrentDate) newInstance;
+ if (this.currentDate != null) {
+ other.currentDate = new DateWritable(this.currentDate);
+ }
+ }
+}
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentTimestamp.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentTimestamp.java?rev=1656544&view=auto
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentTimestamp.java
(added)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentTimestamp.java
Mon Feb 2 19:19:01 2015
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+
+// If there is a new UDFType to describe a function that is deterministic
within a query
+// but changes value between queries, this function would fall into that
category.
+@UDFType(deterministic = true)
+@Description(name = "current_timestamp",
+ value = "_FUNC_() - Returns the current timestamp at the start of query
evaluation."
+ + " All calls of current_timestamp within the same query return the same
value.")
+public class GenericUDFCurrentTimestamp extends GenericUDF {
+
+ protected TimestampWritable currentTimestamp;
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments)
+ throws UDFArgumentException {
+ if (arguments.length != 0) {
+ throw new UDFArgumentLengthException(
+ "The function CURRENT_TIMESTAMP does not take any arguments, but
found "
+ + arguments.length);
+ }
+
+ if (currentTimestamp == null) {
+ currentTimestamp = new
TimestampWritable(SessionState.get().getQueryCurrentTimestamp());
+ }
+
+ return PrimitiveObjectInspectorFactory.writableTimestampObjectInspector;
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ return currentTimestamp;
+ }
+
+ public TimestampWritable getCurrentTimestamp() {
+ return currentTimestamp;
+ }
+
+ public void setCurrentTimestamp(TimestampWritable currentTimestamp) {
+ this.currentTimestamp = currentTimestamp;
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ return "CURRENT_TIMESTAMP()";
+ }
+
+ @Override
+ public void copyToNewInstance(Object newInstance) throws
UDFArgumentException {
+ super.copyToNewInstance(newInstance);
+ // Need to preserve currentTimestamp
+ GenericUDFCurrentTimestamp other = (GenericUDFCurrentTimestamp)
newInstance;
+ if (this.currentTimestamp != null) {
+ other.currentTimestamp = new TimestampWritable(this.currentTimestamp);
+ }
+ }
+}
Added: hive/trunk/ql/src/test/queries/clientpositive/current_date_timestamp.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/current_date_timestamp.q?rev=1656544&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/current_date_timestamp.q
(added)
+++ hive/trunk/ql/src/test/queries/clientpositive/current_date_timestamp.q Mon
Feb 2 19:19:01 2015
@@ -0,0 +1,4 @@
+select current_timestamp = current_timestamp(), current_date = current_date()
from src limit 5;
+
+set hive.test.currenttimestamp =2012-01-01 01:02:03;
+select current_date, current_timestamp from src limit 5;
Added:
hive/trunk/ql/src/test/results/clientpositive/current_date_timestamp.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/current_date_timestamp.q.out?rev=1656544&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/current_date_timestamp.q.out
(added)
+++ hive/trunk/ql/src/test/results/clientpositive/current_date_timestamp.q.out
Mon Feb 2 19:19:01 2015
@@ -0,0 +1,26 @@
+PREHOOK: query: select current_timestamp = current_timestamp(), current_date =
current_date() from src limit 5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select current_timestamp = current_timestamp(), current_date
= current_date() from src limit 5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+true true
+true true
+true true
+true true
+true true
+PREHOOK: query: select current_date, current_timestamp from src limit 5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select current_date, current_timestamp from src limit 5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+2012-01-01 2012-01-01 01:02:03
+2012-01-01 2012-01-01 01:02:03
+2012-01-01 2012-01-01 01:02:03
+2012-01-01 2012-01-01 01:02:03
+2012-01-01 2012-01-01 01:02:03
Modified: hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out?rev=1656544&r1=1656543&r2=1656544&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
(original)
+++ hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out Mon Feb
2 19:19:01 2015
@@ -52,6 +52,8 @@ covar_samp
create_union
cume_dist
current_database
+current_date
+current_timestamp
date_add
date_sub
datediff
@@ -228,6 +230,8 @@ covar_samp
create_union
cume_dist
current_database
+current_date
+current_timestamp
PREHOOK: query: SHOW FUNCTIONS '.*e$'
PREHOOK: type: SHOWFUNCTIONS
POSTHOOK: query: SHOW FUNCTIONS '.*e$'
@@ -236,6 +240,7 @@ assert_true
case
coalesce
current_database
+current_date
decode
e
encode
@@ -277,6 +282,7 @@ PREHOOK: query: SHOW FUNCTIONS '.*date.*
PREHOOK: type: SHOWFUNCTIONS
POSTHOOK: query: SHOW FUNCTIONS '.*date.*'
POSTHOOK: type: SHOWFUNCTIONS
+current_date
date_add
date_sub
datediff
