IMPALA-3504: UDF for current timestamp in UTC This change adds a UDF "utc_timestamp" which returns the current date and time in UTC. Example query:
select utc_timestamp(); +-------------------------------+ | utc_timestamp() | +-------------------------------+ | 2017-06-15 17:36:39.290773000 | +-------------------------------+ Change-Id: I969fc805922f2bb9c8101e84f85ff2cc3b1b6729 Reviewed-on: http://gerrit.cloudera.org:8080/7203 Tested-by: Impala Public Jenkins Reviewed-by: Matthew Jacobs <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/9037b8e3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/9037b8e3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/9037b8e3 Branch: refs/heads/master Commit: 9037b8e38598a9710cd2dfcf30b9fd353bdf6191 Parents: 4e17839 Author: Bikramjeet Vig <[email protected]> Authored: Thu Jun 15 10:29:02 2017 -0700 Committer: Matthew Jacobs <[email protected]> Committed: Thu Jul 6 23:04:28 2017 +0000 ---------------------------------------------------------------------- be/src/exprs/expr-test.cc | 31 ++++++++++++++++++++ be/src/exprs/timestamp-functions-ir.cc | 7 +++++ be/src/exprs/timestamp-functions.h | 1 + be/src/runtime/runtime-state.cc | 3 ++ be/src/runtime/runtime-state.h | 8 +++-- be/src/runtime/timestamp-value.h | 8 +++++ be/src/service/impala-server.cc | 6 +++- common/function-registry/impala_functions.py | 1 + common/thrift/ImpalaInternalService.thrift | 6 +++- .../org/apache/impala/testutil/TestUtils.java | 7 ++++- 10 files changed, 73 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/exprs/expr-test.cc ---------------------------------------------------------------------- diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc index a8998ec..a1fd9a1 100644 --- a/be/src/exprs/expr-test.cc +++ b/be/src/exprs/expr-test.cc @@ -3864,6 +3864,7 @@ TEST_F(ExprTest, UtilityFunctions) { TestStringValue("typeOf(cast(10 as DOUBLE))", "DOUBLE"); TestStringValue("typeOf(current_database())", "STRING"); TestStringValue("typeOf(now())", "TIMESTAMP"); + TestStringValue("typeOf(utc_timestamp())", "TIMESTAMP"); TestStringValue("typeOf(cast(10 as DECIMAL))", "DECIMAL(9,0)"); TestStringValue("typeOf(0.0)", "DECIMAL(1,1)"); TestStringValue("typeOf(3.14)", "DECIMAL(3,2)"); @@ -5229,6 +5230,7 @@ TEST_F(ExprTest, TimestampFunctions) { // Test functions with unknown expected value. TestValidTimestampValue("now()"); + TestValidTimestampValue("utc_timestamp()"); TestValidTimestampValue("current_timestamp()"); TestValidTimestampValue("cast(unix_timestamp() as timestamp)"); @@ -5264,6 +5266,10 @@ TEST_F(ExprTest, TimestampFunctions) { timestamp_result = ConvertValue<TimestampValue>(GetValue("current_timestamp()", TYPE_TIMESTAMP)); EXPECT_BETWEEN(start_time, timestamp_result, TimestampValue::LocalTime()); + const TimestampValue utc_start_time = TimestampValue::UtcTime(); + timestamp_result = ConvertValue<TimestampValue>(GetValue("utc_timestamp()", + TYPE_TIMESTAMP)); + EXPECT_BETWEEN(utc_start_time, timestamp_result, TimestampValue::UtcTime()); // UNIX_TIMESTAMP() has second precision so the comparison start time is shifted back // a second to ensure an earlier value. unix_start_time = @@ -5273,6 +5279,31 @@ TEST_F(ExprTest, TimestampFunctions) { EXPECT_BETWEEN(TimestampValue::FromUnixTime(unix_start_time - 1), timestamp_result, TimestampValue::LocalTime()); + // Test that UTC and local time represent the same point in time + { + const string stmt = "select now(), utc_timestamp()"; + vector<FieldSchema> result_types; + Status status = executor_->Exec(stmt, &result_types); + EXPECT_TRUE(status.ok()) << "stmt: " << stmt << "\nerror: " << status.GetDetail(); + DCHECK(result_types.size() == 2); + EXPECT_EQ(TypeToOdbcString(TYPE_TIMESTAMP), result_types[0].type) + << "invalid type returned by now()"; + EXPECT_EQ(TypeToOdbcString(TYPE_TIMESTAMP), result_types[1].type) + << "invalid type returned by utc_timestamp()"; + string result_row; + status = executor_->FetchResult(&result_row); + EXPECT_TRUE(status.ok()) << "stmt: " << stmt << "\nerror: " << status.GetDetail(); + vector<string> result_cols; + boost::split(result_cols, result_row, boost::is_any_of("\t")); + // To ensure this fails if columns are not tab separated + DCHECK(result_cols.size() == 2); + const TimestampValue local_time = ConvertValue<TimestampValue>(result_cols[0]); + const TimestampValue utc_timestamp = ConvertValue<TimestampValue>(result_cols[1]); + TimestampValue utc_converted_to_local(utc_timestamp); + utc_converted_to_local.UtcToLocal(); + EXPECT_EQ(utc_converted_to_local, local_time); + } + // Test alias TestValue("now() = current_timestamp()", TYPE_BOOLEAN, true); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/exprs/timestamp-functions-ir.cc ---------------------------------------------------------------------- diff --git a/be/src/exprs/timestamp-functions-ir.cc b/be/src/exprs/timestamp-functions-ir.cc index 6a25ced..0cbf19e 100644 --- a/be/src/exprs/timestamp-functions-ir.cc +++ b/be/src/exprs/timestamp-functions-ir.cc @@ -304,6 +304,13 @@ TimestampVal TimestampFunctions::Now(FunctionContext* context) { return return_val; } +TimestampVal TimestampFunctions::UtcTimestamp(FunctionContext* context) { + const TimestampValue* utc_timestamp = context->impl()->state()->utc_timestamp(); + TimestampVal return_val; + utc_timestamp->ToTimestampVal(&return_val); + return return_val; +} + // Writes 'num' as ASCII into 'dst'. If necessary, adds leading zeros to make the ASCII // representation exactly 'len' characters. Both 'num' and 'len' must be >= 0. static inline void IntToChar(uint8_t* dst, int num, int len) { http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/exprs/timestamp-functions.h ---------------------------------------------------------------------- diff --git a/be/src/exprs/timestamp-functions.h b/be/src/exprs/timestamp-functions.h index cbd8b98..9e447b8 100644 --- a/be/src/exprs/timestamp-functions.h +++ b/be/src/exprs/timestamp-functions.h @@ -140,6 +140,7 @@ class TimestampFunctions { /// Date/time functions. static TimestampVal Now(FunctionContext* context); + static TimestampVal UtcTimestamp(FunctionContext* context); static StringVal ToDate(FunctionContext* context, const TimestampVal& ts_val); static IntVal DateDiff(FunctionContext* context, const TimestampVal& ts_val1, const TimestampVal& ts_val2); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/runtime/runtime-state.cc ---------------------------------------------------------------------- diff --git a/be/src/runtime/runtime-state.cc b/be/src/runtime/runtime-state.cc index b05fc6b..89eec29 100644 --- a/be/src/runtime/runtime-state.cc +++ b/be/src/runtime/runtime-state.cc @@ -78,6 +78,8 @@ RuntimeState::RuntimeState(QueryState* query_state, const TPlanFragmentCtx& frag fragment_ctx_(&fragment_ctx), instance_ctx_(&instance_ctx), now_(new TimestampValue(TimestampValue::Parse(query_state->query_ctx().now_string))), + utc_timestamp_(new TimestampValue(TimestampValue::Parse( + query_state->query_ctx().utc_timestamp_string))), exec_env_(exec_env), profile_(obj_pool(), "Fragment " + PrintId(instance_ctx.fragment_instance_id)), instance_buffer_reservation_(nullptr), @@ -93,6 +95,7 @@ RuntimeState::RuntimeState( instance_ctx_(nullptr), local_query_state_(query_state_), now_(new TimestampValue(TimestampValue::Parse(qctx.now_string))), + utc_timestamp_(new TimestampValue(TimestampValue::Parse(qctx.utc_timestamp_string))), exec_env_(exec_env), profile_(obj_pool(), "<unnamed>"), instance_buffer_reservation_(nullptr), http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/runtime/runtime-state.h ---------------------------------------------------------------------- diff --git a/be/src/runtime/runtime-state.h b/be/src/runtime/runtime-state.h index 186d540..9a1d0b2 100644 --- a/be/src/runtime/runtime-state.h +++ b/be/src/runtime/runtime-state.h @@ -115,6 +115,7 @@ class RuntimeState { return query_ctx().session.connected_user; } const TimestampValue* now() const { return now_.get(); } + const TimestampValue* utc_timestamp() const { return utc_timestamp_.get(); } void set_now(const TimestampValue* now); const TUniqueId& query_id() const { return query_ctx().query_id; } const TUniqueId& fragment_instance_id() const { @@ -338,9 +339,12 @@ class RuntimeState { /// Provides instance id if instance_ctx_ == nullptr TUniqueId no_instance_id_; - /// Query-global timestamp, e.g., for implementing now(). Set from query_globals_. - /// Use pointer to avoid inclusion of timestampvalue.h and avoid clang issues. + /// Query-global timestamps for implementing now() and utc_timestamp(). Both represent + /// the same point in time but now_ is in local time and utc_timestamp_ is in UTC. + /// Set from query_globals_. Use pointer to avoid inclusion of timestampvalue.h and + /// avoid clang issues. boost::scoped_ptr<TimestampValue> now_; + boost::scoped_ptr<TimestampValue> utc_timestamp_; /// TODO: get rid of this and use ExecEnv::GetInstance() instead ExecEnv* exec_env_; http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/runtime/timestamp-value.h ---------------------------------------------------------------------- diff --git a/be/src/runtime/timestamp-value.h b/be/src/runtime/timestamp-value.h index 83c38e9..c84eb7f 100644 --- a/be/src/runtime/timestamp-value.h +++ b/be/src/runtime/timestamp-value.h @@ -123,6 +123,14 @@ class TimestampValue { return TimestampValue(boost::posix_time::microsec_clock::local_time()); } + /// Returns the current Coordinated Universal Time ("UTC") with microsecond accuracy. + /// This should not be used to time something because it is affected by adjustments to + /// the system clock such as a manual correction by a system admin. For timings, + /// use functions in util/time.h. + static TimestampValue UtcTime() { + return TimestampValue(boost::posix_time::microsec_clock::universal_time()); + } + /// Returns a TimestampValue converted from a TimestampVal. The caller must ensure /// the TimestampVal does not represent a NULL. static TimestampValue FromTimestampVal(const impala_udf::TimestampVal& udf_value) { http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/service/impala-server.cc ---------------------------------------------------------------------- diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc index 37e06ac..cf91ba3 100644 --- a/be/src/service/impala-server.cc +++ b/be/src/service/impala-server.cc @@ -875,7 +875,11 @@ Status ImpalaServer::ExecuteInternal( void ImpalaServer::PrepareQueryContext(TQueryCtx* query_ctx) { query_ctx->__set_pid(getpid()); - query_ctx->__set_now_string(TimestampValue::LocalTime().ToString()); + TimestampValue utc_timestamp = TimestampValue::UtcTime(); + query_ctx->__set_utc_timestamp_string(utc_timestamp.ToString()); + TimestampValue local_timestamp(utc_timestamp); + local_timestamp.UtcToLocal(); + query_ctx->__set_now_string(local_timestamp.ToString()); query_ctx->__set_start_unix_millis(UnixMillis()); query_ctx->__set_coord_address(MakeNetworkAddress(FLAGS_hostname, FLAGS_be_port)); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/common/function-registry/impala_functions.py ---------------------------------------------------------------------- diff --git a/common/function-registry/impala_functions.py b/common/function-registry/impala_functions.py index b80261c..4201553 100644 --- a/common/function-registry/impala_functions.py +++ b/common/function-registry/impala_functions.py @@ -227,6 +227,7 @@ visible_functions = [ '_ZN6impala18TimestampFunctions22UnixAndFromUnixPrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE', '_ZN6impala18TimestampFunctions20UnixAndFromUnixCloseEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE'], [['now', 'current_timestamp'], 'TIMESTAMP', [], '_ZN6impala18TimestampFunctions3NowEPN10impala_udf15FunctionContextE'], + [['utc_timestamp'], 'TIMESTAMP', [], '_ZN6impala18TimestampFunctions12UtcTimestampEPN10impala_udf15FunctionContextE'], [['from_utc_timestamp'], 'TIMESTAMP', ['TIMESTAMP', 'STRING'], "impala::TimestampFunctions::FromUtc"], [['to_utc_timestamp'], 'TIMESTAMP', ['TIMESTAMP', 'STRING'], http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/common/thrift/ImpalaInternalService.thrift ---------------------------------------------------------------------- diff --git a/common/thrift/ImpalaInternalService.thrift b/common/thrift/ImpalaInternalService.thrift index c31cd7f..4aefe55 100644 --- a/common/thrift/ImpalaInternalService.thrift +++ b/common/thrift/ImpalaInternalService.thrift @@ -322,7 +322,7 @@ struct TQueryCtx { // Session state including user. 3: required TSessionState session - // String containing a timestamp set as the query submission time. + // String containing a timestamp (in local timezone) set as the query submission time. 4: required string now_string // Process ID of the impalad to which the user is connected. @@ -371,6 +371,10 @@ struct TQueryCtx { // The pool to which this request has been submitted. Used to update pool statistics // for admission control. 16: optional string request_pool + + // String containing a timestamp (in UTC) set as the query submission time. It + // represents the same point in time as now_string + 17: required string utc_timestamp_string } // Specification of one output destination of a plan fragment http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/fe/src/test/java/org/apache/impala/testutil/TestUtils.java ---------------------------------------------------------------------- diff --git a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java index ce93984..5404455 100644 --- a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java +++ b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java @@ -22,8 +22,10 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; import java.util.Collections; +import java.util.Date; import java.util.Map; import java.util.Scanner; +import java.util.TimeZone; import javax.json.Json; import javax.json.JsonObject; @@ -258,7 +260,10 @@ public class TestUtils { queryCtx.setSession(new TSessionState(new TUniqueId(), TSessionType.BEESWAX, defaultDb, user, new TNetworkAddress("localhost", 0))); SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSSSSS"); - queryCtx.setNow_string(formatter.format(Calendar.getInstance().getTime())); + Date now = Calendar.getInstance().getTime(); + queryCtx.setNow_string(formatter.format(now)); + formatter.setTimeZone(TimeZone.getTimeZone("GMT")); + queryCtx.setUtc_timestamp_string(formatter.format(now)); queryCtx.setStart_unix_millis(System.currentTimeMillis()); queryCtx.setPid(1000); // Disable rewrites by default because some analyzer tests have non-executable
