Repository: incubator-impala Updated Branches: refs/heads/master 7d9e76620 -> 31025ab10
IMPALA-4549: consistently treat 9999 as upper bound for timestamp year Previously Impala was inconsistent about whether the year 10000 was supported, as a result of inconsistency in boost, which reported the maximum year as 9999 but sometimes allowed 10000. This meant that Impala sometimes accepted the year 10000 and sometimes not. Use the patched boost version and update tests accordingly. Testing: Ran an exhaustive build. Change-Id: Iaf23b40833017789d879e5da7bb10384129e2d10 Reviewed-on: http://gerrit.cloudera.org:8080/5665 Reviewed-by: Tim Armstrong <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/69859bdd Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/69859bdd Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/69859bdd Branch: refs/heads/master Commit: 69859bddfb058d742cece7d7c2e3e3f07f29bb6a Parents: 7d9e766 Author: Tim Armstrong <[email protected]> Authored: Mon Jan 9 14:11:19 2017 -0800 Committer: Impala Public Jenkins <[email protected]> Committed: Thu Jan 19 00:04:27 2017 +0000 ---------------------------------------------------------------------- CMakeLists.txt | 3 +- be/src/exprs/expr-test.cc | 33 +++++++++++++------- be/src/runtime/timestamp-test.cc | 12 +++---- bin/impala-config.sh | 4 +-- .../queries/QueryTest/exprs.test | 6 ++-- tests/comparison/discrepancy_searcher.py | 2 +- 6 files changed, 33 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/69859bdd/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index 6a17714..1a28143 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,8 +113,7 @@ if (CMAKE_DEBUG) set(Boost_DEBUG TRUE) endif() -find_package(Boost $ENV{IMPALA_BOOST_VERSION} REQUIRED - COMPONENTS thread regex system filesystem date_time) +find_package(Boost REQUIRED COMPONENTS thread regex system filesystem date_time) include_directories(${Boost_INCLUDE_DIRS}) set(LIBS ${LIBS} ${Boost_LIBRARIES}) message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIRS}) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/69859bdd/be/src/exprs/expr-test.cc ---------------------------------------------------------------------- diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc index 4185ea1..3672101 100644 --- a/be/src/exprs/expr-test.cc +++ b/be/src/exprs/expr-test.cc @@ -1453,6 +1453,10 @@ TEST_F(ExprTest, CastExprs) { TestIsNull("cast(cast(1180591620717411303425 as decimal(38, 0)) as timestamp)", TYPE_TIMESTAMP); + // Out of range String <--> Timestamp - invalid boundary cases. + TestIsNull("cast('1399-12-31 23:59:59' as timestamp)", TYPE_TIMESTAMP); + TestIsNull("cast('10000-01-01 00:00:00' as timestamp)", TYPE_TIMESTAMP); + // Timestamp <--> Int TestIsNull("cast(cast('09:10:11.000000' as timestamp) as int)", TYPE_INT); TestValue("cast(cast('2000-01-01' as timestamp) as int)", TYPE_INT, 946684800); @@ -1464,9 +1468,19 @@ TEST_F(ExprTest, CastExprs) { 946717811); TestTimestampValue("cast(946717811 as timestamp)", TimestampValue("2000-01-01 09:10:11", 19)); - TestValue("cast(cast('1400-01-01' as timestamp) as bigint)", TYPE_BIGINT, -17987443200); - TestTimestampValue("cast(-17987443200 as timestamp)", TimestampValue("1400-01-01", 10)); + + // Timestamp <--> Int conversions boundary cases + TestValue("cast(cast('1400-01-01 00:00:00' as timestamp) as bigint)", + TYPE_BIGINT, -17987443200); + TestTimestampValue("cast(-17987443200 as timestamp)", + TimestampValue("1400-01-01 00:00:00", 19)); TestIsNull("cast(-17987443201 as timestamp)", TYPE_TIMESTAMP); + TestValue("cast(cast('9999-12-31 23:59:59' as timestamp) as bigint)", + TYPE_BIGINT, 253402300799); + TestTimestampValue("cast(253402300799 as timestamp)", + TimestampValue("9999-12-31 23:59:59", 19)); + TestIsNull("cast(253402300800 as timestamp)", TYPE_TIMESTAMP); + // Timestamp <--> Float TestIsNull("cast(cast('09:10:11.000000' as timestamp) as float)", TYPE_FLOAT); TestValue("cast(cast('2000-01-01' as timestamp) as double)", TYPE_DOUBLE, 946684800); @@ -1478,10 +1492,7 @@ TEST_F(ExprTest, CastExprs) { TestValue("cast(cast('1400-01-01' as timestamp) as double)", TYPE_DOUBLE, -17987443200); TestIsNull("cast(cast(-17987443201.03 as double) as timestamp)", TYPE_TIMESTAMP); - // Use 4 digit years otherwise string parsing will fail. - TestValue("cast(cast('9999-12-31 23:59:59' as timestamp) + interval 1 year as bigint)", - TYPE_BIGINT, 253433923199); - TestTimestampValue("cast(253433923199 as timestamp) - interval 1 year", + TestTimestampValue("cast(253402300799 as timestamp)", TimestampValue("9999-12-31 23:59:59", 19)); TestIsNull("cast(253433923200 as timestamp)", TYPE_TIMESTAMP); TestIsNull("cast(cast(null as bigint) as timestamp)", TYPE_TIMESTAMP); @@ -3594,7 +3605,7 @@ TEST_F(ExprTest, TimestampFunctions) { TYPE_TIMESTAMP); TestIsNull( "CAST('1400-01-01 00:12:00' AS TIMESTAMP) - INTERVAL 13 MINUTES", TYPE_TIMESTAMP); - TestIsNull("CAST('9999-12-31 21:00:00' AS TIMESTAMP) + INTERVAL 100000000 SECONDS", + TestIsNull("CAST('9999-12-31 23:59:59' AS TIMESTAMP) + INTERVAL 1 SECONDS", TYPE_TIMESTAMP); TestIsNull( "CAST('1400-01-01 00:00:00' AS TIMESTAMP) - INTERVAL 1 SECONDS", TYPE_TIMESTAMP); @@ -3804,7 +3815,7 @@ TEST_F(ExprTest, TimestampFunctions) { const string& lt_max_interval = lexical_cast<string>(static_cast<int64_t>(0.9 * it->second)); // Test that pushing a value beyond the max/min values results in a NULL. - TestIsNull(unit + "_add(cast('9999-12-31 23:59:59' as timestamp) + interval 1 year, " + TestIsNull(unit + "_add(cast('9999-12-31 23:59:59' as timestamp), " + lt_max_interval + ")", TYPE_TIMESTAMP); TestIsNull(unit + "_sub(cast('1400-01-01 00:00:00' as timestamp), " + lt_max_interval + ")", TYPE_TIMESTAMP); @@ -4109,11 +4120,9 @@ TEST_F(ExprTest, TimestampFunctions) { TYPE_TIMESTAMP); TestIsNull("from_utc_timestamp(CAST(\"1400-01-01 05:00:00\" as TIMESTAMP), \"PST\")", TYPE_TIMESTAMP); - // TODO: IMPALA-4549: these should return NULL, but validation doesn't catch year 10000. - // Just check that we don't crash. - GetValue("from_utc_timestamp(CAST(\"10000-12-31 21:00:00\" as TIMESTAMP), \"JST\")", + TestIsNull("from_utc_timestamp(CAST(\"9999-12-31 21:00:00\" as TIMESTAMP), \"JST\")", TYPE_TIMESTAMP); - GetValue("to_utc_timestamp(CAST(\"10000-12-31 21:00:00\" as TIMESTAMP), \"PST\")", + TestIsNull("to_utc_timestamp(CAST(\"9999-12-31 21:00:00\" as TIMESTAMP), \"PST\")", TYPE_TIMESTAMP); // With support of date strings this generates a date and 0 time. http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/69859bdd/be/src/runtime/timestamp-test.cc ---------------------------------------------------------------------- diff --git a/be/src/runtime/timestamp-test.cc b/be/src/runtime/timestamp-test.cc index b3002a9..367fbd7 100644 --- a/be/src/runtime/timestamp-test.cc +++ b/be/src/runtime/timestamp-test.cc @@ -597,18 +597,16 @@ TEST(TimestampTest, Basic) { TimestampValue too_early(-17987443201); EXPECT_FALSE(too_early.HasDate()); EXPECT_FALSE(too_early.HasTime()); - // Apparently 5 digit years don't parse (at least by default) but can be printed. - // Boost's documented says the max year supported is 9,999 but 10K seems to be - // the actual limit. + // Boost's max supported year is 9999. TimestampValue max_date = - TimestampValue(date(10000, Dec, 31), time_duration(23, 59, 59)); + TimestampValue(date(9999, Dec, 31), time_duration(23, 59, 59)); EXPECT_TRUE(max_date.HasDate()); EXPECT_TRUE(max_date.HasTime()); time_t tm_max; EXPECT_TRUE(max_date.ToUnixTime(&tm_max)); - EXPECT_EQ(253433923199, tm_max); - EXPECT_EQ("10000-12-31 23:59:59", TimestampValue(253433923199).DebugString()); - TimestampValue too_late(253433923200); + EXPECT_EQ(253402300799, tm_max); + EXPECT_EQ("9999-12-31 23:59:59", TimestampValue(253402300799).DebugString()); + TimestampValue too_late(253402300800); EXPECT_FALSE(too_late.HasDate()); EXPECT_FALSE(too_late.HasTime()); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/69859bdd/bin/impala-config.sh ---------------------------------------------------------------------- diff --git a/bin/impala-config.sh b/bin/impala-config.sh index 84f2880..7a0e58e 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -72,13 +72,13 @@ fi # moving to a different build of the toolchain, e.g. when a version is bumped or a # compile option is changed. The build id can be found in the output of the toolchain # build jobs, it is constructed from the build number and toolchain git hash prefix. -export IMPALA_TOOLCHAIN_BUILD_ID=333-f7c19a394a +export IMPALA_TOOLCHAIN_BUILD_ID=337-ccbb7e04a1 # Versions of toolchain dependencies. # ----------------------------------- export IMPALA_AVRO_VERSION=1.7.4-p4 export IMPALA_BINUTILS_VERSION=2.26-p1 -export IMPALA_BOOST_VERSION=1.57.0 +export IMPALA_BOOST_VERSION=1.57.0-p1 export IMPALA_BREAKPAD_VERSION=20150612-p1 export IMPALA_BZIP2_VERSION=1.0.6-p2 export IMPALA_CMAKE_VERSION=3.2.3-p1 http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/69859bdd/testdata/workloads/functional-query/queries/QueryTest/exprs.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/exprs.test b/testdata/workloads/functional-query/queries/QueryTest/exprs.test index ed3c9d9..9e5d3dc 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/exprs.test +++ b/testdata/workloads/functional-query/queries/QueryTest/exprs.test @@ -2612,7 +2612,7 @@ UDF WARNING: Timestamp '1400-01-01 05:00:00' in timezone 'JST' could not be conv ==== ---- QUERY # Test out-of-range value handling when adding dates. -select CAST('9999-12-31 21:00:00' AS TIMESTAMP) + INTERVAL 367 DAYS +select CAST('9999-12-31 21:00:00' AS TIMESTAMP) + INTERVAL 1 DAYS from alltypes limit 1 ---- RESULTS @@ -2620,7 +2620,7 @@ NULL ---- TYPES TIMESTAMP ---- ERRORS -UDF WARNING: Cannot add interval 367: Year is out of valid range: 1400..10000 +UDF WARNING: Cannot add interval 1: Year is out of valid range: 1400..9999 ==== ---- QUERY # Test out-of-range value handling when subtracting dates. @@ -2632,7 +2632,7 @@ NULL ---- TYPES TIMESTAMP ---- ERRORS -UDF WARNING: Cannot subtract interval 1: Year is out of valid range: 1400..10000 +UDF WARNING: Cannot subtract interval 1: Year is out of valid range: 1400..9999 ==== ---- QUERY # IMPALA-4574: UUID() is not a constant expression. http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/69859bdd/tests/comparison/discrepancy_searcher.py ---------------------------------------------------------------------- diff --git a/tests/comparison/discrepancy_searcher.py b/tests/comparison/discrepancy_searcher.py index 3bd8d23..a88dd2c 100755 --- a/tests/comparison/discrepancy_searcher.py +++ b/tests/comparison/discrepancy_searcher.py @@ -112,7 +112,7 @@ class QueryResultComparator(object): if ref_exception: comparison_result.exception = ref_exception error_message = str(ref_exception) - if 'Year is out of valid range: 1400..10000' in error_message: + if 'Year is out of valid range: 1400..9999' in error_message: # This comes from Postgresql. Overflow errors will be ignored. comparison_result.exception = TypeOverflow(error_message) LOG.debug('%s encountered an error running query: %s',
