Repository: incubator-impala Updated Branches: refs/heads/master cb37be893 -> a16a0fa84
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a16a0fa8/common/thrift/generate_error_codes.py ---------------------------------------------------------------------- diff --git a/common/thrift/generate_error_codes.py b/common/thrift/generate_error_codes.py index 84a0dad..89abea4 100755 --- a/common/thrift/generate_error_codes.py +++ b/common/thrift/generate_error_codes.py @@ -318,6 +318,10 @@ error_codes = ( ("PARQUET_MR_TIMESTAMP_CONVERSION_FAILED", 103, "Failed to convert timestamp '$0' to " "timezone '$1' for a Parquet file in table '$2'."), + + ("KUDU_TIMESTAMP_OUT_OF_RANGE", 104, + "Kudu table '$0' column '$1' contains an out of range timestamp. " + "The valid date range is 1400-01-01..9999-12-31."), ) import sys http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a16a0fa8/fe/src/main/java/org/apache/impala/util/KuduUtil.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/util/KuduUtil.java b/fe/src/main/java/org/apache/impala/util/KuduUtil.java index 073cf70..4aa1b2a 100644 --- a/fe/src/main/java/org/apache/impala/util/KuduUtil.java +++ b/fe/src/main/java/org/apache/impala/util/KuduUtil.java @@ -341,10 +341,10 @@ public class KuduUtil { case STRING: return org.apache.kudu.Type.STRING; case DOUBLE: return org.apache.kudu.Type.DOUBLE; case FLOAT: return org.apache.kudu.Type.FLOAT; + case TIMESTAMP: return org.apache.kudu.Type.UNIXTIME_MICROS; /* Fall through below */ case INVALID_TYPE: case NULL_TYPE: - case TIMESTAMP: case BINARY: case DATE: case DATETIME: @@ -368,6 +368,7 @@ public class KuduUtil { case INT32: return Type.INT; case INT64: return Type.BIGINT; case STRING: return Type.STRING; + case UNIXTIME_MICROS: return Type.TIMESTAMP; default: throw new ImpalaRuntimeException(String.format( "Kudu type '%s' is not supported in Impala", t.getName())); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a16a0fa8/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java ---------------------------------------------------------------------- diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java index 4a662f3..2380609 100644 --- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java +++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java @@ -1585,9 +1585,6 @@ public class AnalyzeDDLTest extends FrontendTestBase { "external Kudu tables."); // CTAS into Kudu tables with unsupported types - AnalysisError("create table t primary key (id) partition by hash partitions 3" + - " stored as kudu as select id, timestamp_col from functional.alltypestiny", - "Cannot create table 't': Type TIMESTAMP is not supported in Kudu"); AnalysisError("create table t primary key (cs) partition by hash partitions 3" + " stored as kudu as select cs from functional.chars_tiny", "Cannot create table 't': Type CHAR(5) is not supported in Kudu"); @@ -2007,6 +2004,8 @@ public class AnalyzeDDLTest extends FrontendTestBase { "partition by hash(x) partitions 8 stored as kudu"); AnalyzesOk("create table tab (x int, y int, primary key(x, y)) " + "partition by hash(y) partitions 8 stored as kudu"); + AnalyzesOk("create table tab (x timestamp, y timestamp, primary key(x)) " + + "partition by hash(x) partitions 8 stored as kudu"); AnalyzesOk("create table tab (x int, y string, primary key (x)) partition by " + "hash (x) partitions 3, range (x) (partition values < 1, partition " + "1 <= values < 10, partition 10 <= values < 20, partition value = 30) " + @@ -2181,7 +2180,7 @@ public class AnalyzeDDLTest extends FrontendTestBase { // Test unsupported Kudu types List<String> unsupportedTypes = Lists.newArrayList( - "DECIMAL(9,0)", "TIMESTAMP", "VARCHAR(20)", "CHAR(20)", + "DECIMAL(9,0)", "VARCHAR(20)", "CHAR(20)", "STRUCT<F1:INT,F2:STRING>", "ARRAY<INT>", "MAP<STRING,STRING>"); for (String t: unsupportedTypes) { String expectedError = String.format( http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a16a0fa8/testdata/datasets/functional/functional_schema_template.sql ---------------------------------------------------------------------- diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql index e7b8a07..3e068f1 100644 --- a/testdata/datasets/functional/functional_schema_template.sql +++ b/testdata/datasets/functional/functional_schema_template.sql @@ -89,7 +89,7 @@ CREATE TABLE {db_name}{db_suffix}.{table_name} ( double_col DOUBLE, date_string_col STRING, string_col STRING, - timestamp_col STRING, + timestamp_col TIMESTAMP, year INT, month INT ) @@ -97,7 +97,7 @@ PARTITION BY HASH (id) PARTITIONS 3 STORED AS KUDU; ---- DEPENDENT_LOAD_KUDU INSERT into TABLE {db_name}{db_suffix}.{table_name} SELECT id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, - cast(timestamp_col as string), year, month + timestamp_col, year, month FROM {db_name}.{table_name}; ==== ---- DATASET @@ -167,7 +167,7 @@ CREATE TABLE {db_name}{db_suffix}.{table_name} ( double_col DOUBLE, date_string_col STRING, string_col STRING, - timestamp_col STRING, + timestamp_col TIMESTAMP, year INT, month INT ) @@ -175,7 +175,7 @@ PARTITION BY HASH (id) PARTITIONS 3 STORED AS KUDU; ---- DEPENDENT_LOAD_KUDU INSERT into TABLE {db_name}{db_suffix}.{table_name} SELECT id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, - cast(timestamp_col as string), year, month + timestamp_col, year, month FROM {db_name}.{table_name}; ==== ---- DATASET @@ -226,7 +226,7 @@ CREATE TABLE {db_name}{db_suffix}.{table_name} ( double_col DOUBLE, date_string_col STRING, string_col STRING, - timestamp_col STRING, + timestamp_col TIMESTAMP, year INT, month INT ) @@ -234,7 +234,7 @@ PARTITION BY HASH (id) PARTITIONS 3 STORED AS KUDU; ---- DEPENDENT_LOAD_KUDU INSERT INTO TABLE {db_name}{db_suffix}.{table_name} SELECT id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, - cast(timestamp_col as string), year, month + timestamp_col, year, month FROM {db_name}.{table_name}; ==== ---- DATASET @@ -319,7 +319,7 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name}_tmp ( double_col STRING, date_string_col STRING, string_col STRING, - timestamp_col STRING) + timestamp_col TIMESTAMP) PARTITIONED BY (year INT, month INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' @@ -443,7 +443,7 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name}_tmp ( double_col STRING, date_string_col STRING, string_col STRING, - timestamp_col STRING) + timestamp_col TIMESTAMP) PARTITIONED BY (year INT, month INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' @@ -561,7 +561,7 @@ CREATE TABLE {db_name}{db_suffix}.{table_name}_idx ( double_col DOUBLE NULL, date_string_col STRING NULL, string_col STRING NULL, - timestamp_col STRING NULL, + timestamp_col TIMESTAMP NULL, year INT NULL, month INT NULL, day INT NULL @@ -576,7 +576,7 @@ INSERT into TABLE {db_name}{db_suffix}.{table_name}_idx SELECT row_number() over (order by year, month, id, day), id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, - cast(timestamp_col as string), year, month, day + timestamp_col, year, month, day FROM {db_name}.{table_name}; ==== ---- DATASET @@ -640,7 +640,7 @@ CREATE TABLE {db_name}{db_suffix}.{table_name} ( double_col DOUBLE, date_string_col STRING, string_col STRING, - timestamp_col STRING, + timestamp_col TIMESTAMP, year INT, month INT, day INT @@ -650,7 +650,7 @@ PARTITION BY HASH (id) PARTITIONS 3 STORED AS KUDU; INSERT into TABLE {db_name}{db_suffix}.{table_name} SELECT id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, - cast(timestamp_col as string), year, month, day + timestamp_col, year, month, day FROM {db_name}.{table_name}; ==== ---- DATASET http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a16a0fa8/testdata/workloads/functional-query/queries/QueryTest/kudu-overflow-ts-abort-on-error.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu-overflow-ts-abort-on-error.test b/testdata/workloads/functional-query/queries/QueryTest/kudu-overflow-ts-abort-on-error.test new file mode 100644 index 0000000..402a44a --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/kudu-overflow-ts-abort-on-error.test @@ -0,0 +1,6 @@ +==== +---- QUERY +select * from times +---- CATCH +Kudu table 'impala::$DATABASE.times' column 'ts' contains an out of range timestamp. The valid date range is 1400-01-01..9999-12-31. +==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a16a0fa8/testdata/workloads/functional-query/queries/QueryTest/kudu-overflow-ts.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu-overflow-ts.test b/testdata/workloads/functional-query/queries/QueryTest/kudu-overflow-ts.test new file mode 100644 index 0000000..5fcbca5 --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/kudu-overflow-ts.test @@ -0,0 +1,11 @@ +==== +---- QUERY +select * from times +---- RESULTS +0,1987-05-19 00:00:00 +1,NULL +---- TYPES +INT,TIMESTAMP +---- ERRORS +Kudu table 'impala::$DATABASE.times' column 'ts' contains an out of range timestamp. The valid date range is 1400-01-01..9999-12-31. +==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a16a0fa8/testdata/workloads/functional-query/queries/QueryTest/kudu_describe.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu_describe.test b/testdata/workloads/functional-query/queries/QueryTest/kudu_describe.test index 878c079..687ca31 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/kudu_describe.test +++ b/testdata/workloads/functional-query/queries/QueryTest/kudu_describe.test @@ -14,7 +14,7 @@ NAME,TYPE,COMMENT,PRIMARY_KEY,NULLABLE,DEFAULT_VALUE,ENCODING,COMPRESSION,BLOCK_ 'month','int','','false','true','','AUTO_ENCODING','DEFAULT_COMPRESSION','0' 'smallint_col','smallint','','false','true','','AUTO_ENCODING','DEFAULT_COMPRESSION','0' 'string_col','string','','false','true','','AUTO_ENCODING','DEFAULT_COMPRESSION','0' -'timestamp_col','string','','false','true','','AUTO_ENCODING','DEFAULT_COMPRESSION','0' +'timestamp_col','timestamp','','false','true','','AUTO_ENCODING','DEFAULT_COMPRESSION','0' 'tinyint_col','tinyint','','false','true','','AUTO_ENCODING','DEFAULT_COMPRESSION','0' 'year','int','','false','true','','AUTO_ENCODING','DEFAULT_COMPRESSION','0' ---- TYPES http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a16a0fa8/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test b/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test index 9779c7f..7572fe2 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test +++ b/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test @@ -2,54 +2,59 @@ ---- QUERY create table tdata (id int primary key, valf float null, vali bigint null, valv string null, - valb boolean null, valt tinyint null, vals smallint null, vald double null) + valb boolean null, valt tinyint null, vals smallint null, vald double null, + ts timestamp) PARTITION BY RANGE (PARTITION VALUES < 10, PARTITION 10 <= VALUES < 30, PARTITION 30 <= VALUES) STORED AS KUDU ---- RESULTS ==== ---- QUERY # VALUES, single row, all target cols, no errors -insert into tdata values (1, 1, 1, 'one', true, 1, 1, 1) +insert into tdata values (1, 1, 1, 'one', true, 1, 1, 1, + cast('1987-05-19 00:00:00' as timestamp)) ---- RUNTIME_PROFILE NumModifiedRows: 1 NumRowErrors: 0 ---- LABELS -ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD +ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS ---- DML_RESULTS: tdata -1,1,1,'one',true,1,1,1 +1,1,1,'one',true,1,1,1,1987-05-19 00:00:00 ---- TYPES -INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE +INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP ==== ---- QUERY # VALUES, single row, all target cols, NULL -insert into tdata values (2, NULL, NULL, NULL, NULL, NULL, NULL, NULL) +insert into tdata values (2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) ---- RUNTIME_PROFILE NumModifiedRows: 1 NumRowErrors: 0 ---- LABELS -ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD +ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS ---- DML_RESULTS: tdata -1,1,1,'one',true,1,1,1 -2,NULL,NULL,'NULL',NULL,NULL,NULL,NULL +1,1,1,'one',true,1,1,1,1987-05-19 00:00:00 +2,NULL,NULL,'NULL',NULL,NULL,NULL,NULL,NULL ---- TYPES -INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE +INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP ==== ---- QUERY -# VALUES, single row, all target cols, bounday values +# VALUES, single row, all target cols, boundary values. The timestamp value is the max +# possible value that Impala can represent; it gets truncated rather than rounded up to +# the nearest microsecond. If it were rounded up, it wouldn't be readable by Impala. insert into tdata values (3, cast('nan' as float), max_bigint(), '', true, min_tinyint(), max_smallint(), - cast('-inf' as double)) + cast('-inf' as double), + nanoseconds_add(cast('9999-12-31 23:59:59' as timestamp), 999999999)) ---- RUNTIME_PROFILE NumModifiedRows: 1 NumRowErrors: 0 ---- LABELS -ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD +ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS ---- DML_RESULTS: tdata -1,1,1,'one',true,1,1,1 -2,NULL,NULL,'NULL',NULL,NULL,NULL,NULL -3,NaN,9223372036854775807,'',true,-128,32767,-Infinity +1,1,1,'one',true,1,1,1,1987-05-19 00:00:00 +2,NULL,NULL,'NULL',NULL,NULL,NULL,NULL,NULL +3,NaN,9223372036854775807,'',true,-128,32767,-Infinity,9999-12-31 23:59:59.999999000 ---- TYPES -INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE +INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP ==== ---- QUERY # VALUES, single row, subset of target cols @@ -58,36 +63,36 @@ insert into tdata (valb, vald, id) values (true, 0, 4) NumModifiedRows: 1 NumRowErrors: 0 ---- LABELS -ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD +ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS ---- DML_RESULTS: tdata -1,1,1,'one',true,1,1,1 -2,NULL,NULL,'NULL',NULL,NULL,NULL,NULL -3,NaN,9223372036854775807,'',true,-128,32767,-Infinity -4,NULL,NULL,'NULL',true,NULL,NULL,0 +1,1,1,'one',true,1,1,1,1987-05-19 00:00:00 +2,NULL,NULL,'NULL',NULL,NULL,NULL,NULL,NULL +3,NaN,9223372036854775807,'',true,-128,32767,-Infinity,9999-12-31 23:59:59.999999000 +4,NULL,NULL,'NULL',true,NULL,NULL,0,NULL ---- TYPES -INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE +INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP ==== ---- QUERY # VALUES, multiple rows, all target cols insert into tdata values -(5, 5.0, 5, 'five', false, NULL, NULL, NULL), -(6, 16, 60, '', true, 0, -1, -6), -(7, NULL, 10, NULL, false, max_tinyint(), -7, 2) +(5, 5.0, 5, 'five', false, NULL, NULL, NULL, NULL), +(6, 16, 60, '', true, 0, -1, -6, cast('2010-12-31 23:59:59' as timestamp)), +(7, NULL, 10, NULL, false, max_tinyint(), -7, 2, cast('1400-01-01 00:00:00' as timestamp)) ---- RUNTIME_PROFILE NumModifiedRows: 3 NumRowErrors: 0 ---- LABELS -ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD +ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS ---- DML_RESULTS: tdata -1,1,1,'one',true,1,1,1 -2,NULL,NULL,'NULL',NULL,NULL,NULL,NULL -3,NaN,9223372036854775807,'',true,-128,32767,-Infinity -4,NULL,NULL,'NULL',true,NULL,NULL,0 -5,5.0,5,'five',false,NULL,NULL,NULL -6,16,60,'',true,0,-1,-6 -7,NULL,10,'NULL',false,127,-7,2 +1,1,1,'one',true,1,1,1,1987-05-19 00:00:00 +2,NULL,NULL,'NULL',NULL,NULL,NULL,NULL,NULL +3,NaN,9223372036854775807,'',true,-128,32767,-Infinity,9999-12-31 23:59:59.999999000 +4,NULL,NULL,'NULL',true,NULL,NULL,0,NULL +5,5.0,5,'five',false,NULL,NULL,NULL,NULL +6,16,60,'',true,0,-1,-6,2010-12-31 23:59:59 +7,NULL,10,'NULL',false,127,-7,2,1400-01-01 00:00:00 ---- TYPES -INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE +INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP ==== ---- QUERY # VALUES, multiple rows, subset of cols @@ -98,68 +103,69 @@ insert into tdata (valv, valf, vali, id) values NumModifiedRows: 2 NumRowErrors: 0 ---- LABELS -ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD +ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS ---- DML_RESULTS: tdata -1,1,1,'one',true,1,1,1 -2,NULL,NULL,'NULL',NULL,NULL,NULL,NULL -3,NaN,9223372036854775807,'',true,-128,32767,-Infinity -4,NULL,NULL,'NULL',true,NULL,NULL,0 -5,5.0,5,'five',false,NULL,NULL,NULL -6,16,60,'',true,0,-1,-6 -7,NULL,10,'NULL',false,127,-7,2 -8,88,888,'eight',NULL,NULL,NULL,NULL -9,-9,-99,'NULL',NULL,NULL,NULL,NULL +1,1,1,'one',true,1,1,1,1987-05-19 00:00:00 +2,NULL,NULL,'NULL',NULL,NULL,NULL,NULL,NULL +3,NaN,9223372036854775807,'',true,-128,32767,-Infinity,9999-12-31 23:59:59.999999000 +4,NULL,NULL,'NULL',true,NULL,NULL,0,NULL +5,5.0,5,'five',false,NULL,NULL,NULL,NULL +6,16,60,'',true,0,-1,-6,2010-12-31 23:59:59 +7,NULL,10,'NULL',false,127,-7,2,1400-01-01 00:00:00 +8,88,888,'eight',NULL,NULL,NULL,NULL,NULL +9,-9,-99,'NULL',NULL,NULL,NULL,NULL,NULL ---- TYPES -INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE +INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP ==== ---- QUERY # SELECT, single row, all target cols insert into tdata -select id, float_col, bigint_col, string_col, bool_col, tinyint_col, smallint_col, double_col +select id, float_col, bigint_col, string_col, bool_col, tinyint_col, smallint_col, +double_col, timestamp_col from functional.alltypes where id = 10 ---- RUNTIME_PROFILE NumModifiedRows: 1 NumRowErrors: 0 ---- LABELS -ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD +ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS ---- DML_RESULTS: tdata -1,1,1,'one',true,1,1,1 -2,NULL,NULL,'NULL',NULL,NULL,NULL,NULL -3,NaN,9223372036854775807,'',true,-128,32767,-Infinity -4,NULL,NULL,'NULL',true,NULL,NULL,0 -5,5.0,5,'five',false,NULL,NULL,NULL -6,16,60,'',true,0,-1,-6 -7,NULL,10,'NULL',false,127,-7,2 -8,88,888,'eight',NULL,NULL,NULL,NULL -9,-9,-99,'NULL',NULL,NULL,NULL,NULL -10,0,0,'0',true,0,0,0 +1,1,1,'one',true,1,1,1,1987-05-19 00:00:00 +2,NULL,NULL,'NULL',NULL,NULL,NULL,NULL,NULL +3,NaN,9223372036854775807,'',true,-128,32767,-Infinity,9999-12-31 23:59:59.999999000 +4,NULL,NULL,'NULL',true,NULL,NULL,0,NULL +5,5.0,5,'five',false,NULL,NULL,NULL,NULL +6,16,60,'',true,0,-1,-6,2010-12-31 23:59:59 +7,NULL,10,'NULL',false,127,-7,2,1400-01-01 00:00:00 +8,88,888,'eight',NULL,NULL,NULL,NULL,NULL +9,-9,-99,'NULL',NULL,NULL,NULL,NULL,NULL +10,0,0,'0',true,0,0,0,2009-01-02 00:10:00.450000000 ---- TYPES -INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE +INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP ==== ---- QUERY # SELECT, single row, subset of cols -insert into tdata (id, vald, valb, vali) -select id, double_col, bool_col, bigint_col +insert into tdata (id, vald, valb, vali, ts) +select id, double_col, bool_col, bigint_col, timestamp_col from functional.alltypes where id = 11 ---- RUNTIME_PROFILE NumModifiedRows: 1 NumRowErrors: 0 ---- LABELS -ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD +ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS ---- DML_RESULTS: tdata -1,1,1,'one',true,1,1,1 -2,NULL,NULL,'NULL',NULL,NULL,NULL,NULL -3,NaN,9223372036854775807,'',true,-128,32767,-Infinity -4,NULL,NULL,'NULL',true,NULL,NULL,0 -5,5.0,5,'five',false,NULL,NULL,NULL -6,16,60,'',true,0,-1,-6 -7,NULL,10,'NULL',false,127,-7,2 -8,88,888,'eight',NULL,NULL,NULL,NULL -9,-9,-99,'NULL',NULL,NULL,NULL,NULL -10,0,0,'0',true,0,0,0 -11,NULL,10,'NULL',false,NULL,NULL,10.1 +1,1,1,'one',true,1,1,1,1987-05-19 00:00:00 +2,NULL,NULL,'NULL',NULL,NULL,NULL,NULL,NULL +3,NaN,9223372036854775807,'',true,-128,32767,-Infinity,9999-12-31 23:59:59.999999000 +4,NULL,NULL,'NULL',true,NULL,NULL,0,NULL +5,5.0,5,'five',false,NULL,NULL,NULL,NULL +6,16,60,'',true,0,-1,-6,2010-12-31 23:59:59 +7,NULL,10,'NULL',false,127,-7,2,1400-01-01 00:00:00 +8,88,888,'eight',NULL,NULL,NULL,NULL,NULL +9,-9,-99,'NULL',NULL,NULL,NULL,NULL,NULL +10,0,0,'0',true,0,0,0,2009-01-02 00:10:00.450000000 +11,NULL,10,'NULL',false,NULL,NULL,10.1,2009-01-02 00:11:00.450000000 ---- TYPES -INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE +INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP ==== ---- QUERY delete tdata @@ -168,42 +174,43 @@ delete tdata ---- QUERY # SELECT, multiple rows, all target cols insert into tdata -select id, float_col, bigint_col, string_col, bool_col, tinyint_col, smallint_col, double_col +select id, float_col, bigint_col, string_col, bool_col, tinyint_col, smallint_col, +double_col, timestamp_col from functional.alltypes where id < 2 ---- RUNTIME_PROFILE NumModifiedRows: 2 NumRowErrors: 0 ---- LABELS -ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD +ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS ---- DML_RESULTS: tdata -0,0,0,'0',true,0,0,0 -1,1.100000023841858,10,'1',false,1,1,10.1 +0,0,0,'0',true,0,0,0,2009-01-01 00:00:00 +1,1.100000023841858,10,'1',false,1,1,10.1,2009-01-01 00:01:00 ---- TYPES -INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE +INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP ==== ---- QUERY # SELECT, multiple rows, subset of cols -insert into tdata (vals, id, valt, vald) -select smallint_col, id, tinyint_col, double_col +insert into tdata (vals, id, valt, vald, ts) +select smallint_col, id, tinyint_col, double_col, timestamp_col from functional.alltypes where id > 2 and id < 6 ---- RUNTIME_PROFILE NumModifiedRows: 3 NumRowErrors: 0 ---- LABELS -ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD +ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS ---- DML_RESULTS: tdata -0,0,0,'0',true,0,0,0 -1,1.100000023841858,10,'1',false,1,1,10.1 -3,NULL,NULL,'NULL',NULL,3,3,30.3 -4,NULL,NULL,'NULL',NULL,4,4,40.4 -5,NULL,NULL,'NULL',NULL,5,5,50.5 +0,0,0,'0',true,0,0,0,2009-01-01 00:00:00 +1,1.100000023841858,10,'1',false,1,1,10.1,2009-01-01 00:01:00 +3,NULL,NULL,'NULL',NULL,3,3,30.3,2009-01-01 00:03:00.300000000 +4,NULL,NULL,'NULL',NULL,4,4,40.4,2009-01-01 00:04:00.600000000 +5,NULL,NULL,'NULL',NULL,5,5,50.5,2009-01-01 00:05:00.100000000 ---- TYPES -INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE +INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP ==== ---- QUERY # Make sure we can insert empty strings into string columns and that we can scan them # back. -insert into tdata values (320, 2.0, 932, cast('' as string), false, 0, 0, 0) +insert into tdata values (320, 2.0, 932, cast('' as string), false, 0, 0, 0, NULL) ---- RESULTS : 1 ---- RUNTIME_PROFILE @@ -219,7 +226,7 @@ INT,STRING,BOOLEAN ==== ---- QUERY insert into tdata values -(666, cast(1.2 as float), 43, cast('z' as string), true, 0, 0, 0) +(666, cast(1.2 as float), 43, cast('z' as string), true, 0, 0, 0, NULL) ---- RESULTS : 1 ---- RUNTIME_PROFILE @@ -229,7 +236,7 @@ NumRowErrors: 0 ---- QUERY # insert row with primary key that already exists insert into tdata values -(666, cast(1.2 as float), 43, cast('z' as VARCHAR(20)), true, 0, 0, 0) +(666, cast(1.2 as float), 43, cast('z' as VARCHAR(20)), true, 0, 0, 0, NULL) ---- RESULTS : 0 ---- RUNTIME_PROFILE @@ -422,4 +429,4 @@ insert into multiple_partition_cols select bigint_col, null, string_col ---- RUNTIME_PROFILE NumModifiedRows: 0 NumRowErrors: 1 -==== \ No newline at end of file +==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a16a0fa8/tests/query_test/test_kudu.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_kudu.py b/tests/query_test/test_kudu.py index 86a6fa7..8c80ad8 100644 --- a/tests/query_test/test_kudu.py +++ b/tests/query_test/test_kudu.py @@ -31,6 +31,8 @@ from kudu.client import Partitioning import logging import pytest import textwrap +from datetime import datetime +from pytz import utc from tests.common.kudu_test_suite import KuduTestSuite from tests.common.impala_cluster import ImpalaCluster @@ -45,6 +47,33 @@ class TestKuduOperations(KuduTestSuite): This suite tests the different modification operations when using a kudu table. """ + def test_out_of_range_timestamps(self, vector, cursor, kudu_client, unique_database): + """Test timestamp values that are outside of Impala's supported date range.""" + cursor.execute("""CREATE TABLE %s.times (a INT PRIMARY KEY, ts TIMESTAMP) + PARTITION BY HASH(a) PARTITIONS 3 STORED AS KUDU""" % unique_database) + assert kudu_client.table_exists( + KuduTestSuite.to_kudu_table_name(unique_database, "times")) + + table = kudu_client.table(KuduTestSuite.to_kudu_table_name(unique_database, "times")) + session = kudu_client.new_session() + session.apply(table.new_insert((0, datetime(1987, 5, 19, 0, 0, tzinfo=utc)))) + # Add a date before 1400 + session.apply(table.new_insert((1, datetime(1300, 1, 1, 0, 0, tzinfo=utc)))) + # TODO: Add a date after 9999. There isn't a way to represent a date greater than + # 9999 in Python datetime. + #session.apply(table.new_insert((2, datetime(12000, 1, 1, 0, 0, tzinfo=utc)))) + session.flush() + + # TODO: The test driver should have a way to specify query options in an 'options' + # section rather than having to split abort_on_error cases into separate files. + vector.get_value('exec_option')['abort_on_error'] = 0 + self.run_test_case('QueryTest/kudu-overflow-ts', vector, + use_db=unique_database) + + vector.get_value('exec_option')['abort_on_error'] = 1 + self.run_test_case('QueryTest/kudu-overflow-ts-abort-on-error', vector, + use_db=unique_database) + def test_kudu_scan_node(self, vector, unique_database): self.run_test_case('QueryTest/kudu-scan-node', vector, use_db=unique_database) @@ -380,23 +409,10 @@ class TestCreateExternalTable(KuduTestSuite): STORED AS KUDU TBLPROPERTIES('kudu.table_name' = '%s')""" % (impala_table_name, kudu_table.name)) + assert False except Exception as e: assert "Kudu type 'binary' is not supported in Impala" in str(e) - def test_unsupported_unixtime_col(self, cursor, kudu_client): - """Check that external tables with UNIXTIME_MICROS columns fail gracefully. - """ - with self.temp_kudu_table(kudu_client, [INT32, UNIXTIME_MICROS]) as kudu_table: - impala_table_name = self.random_table_name() - try: - cursor.execute(""" - CREATE EXTERNAL TABLE %s - STORED AS KUDU - TBLPROPERTIES('kudu.table_name' = '%s')""" % (impala_table_name, - kudu_table.name)) - except Exception as e: - assert "Kudu type 'unixtime_micros' is not supported in Impala" in str(e) - def test_drop_external_table(self, cursor, kudu_client): """Check that dropping an external table only affects the catalog and does not delete the table in Kudu. @@ -454,6 +470,7 @@ class TestCreateExternalTable(KuduTestSuite): STORED AS KUDU TBLPROPERTIES('kudu.table_name' = '%s')""" % ( self.random_table_name(), kudu_table_name)) + assert False except Exception as e: assert "Table does not exist in Kudu: '%s'" % kudu_table_name in str(e) @@ -469,6 +486,7 @@ class TestCreateExternalTable(KuduTestSuite): STORED AS KUDU TBLPROPERTIES('kudu.table_name' = '%s')""" % ( self.get_kudu_table_base_name(kudu_table.name), table_name)) + assert False except Exception as e: assert "Table does not exist in Kudu: '%s'" % table_name in str(e) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a16a0fa8/tests/query_test/test_queries.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_queries.py b/tests/query_test/test_queries.py index d875cda..1099330 100644 --- a/tests/query_test/test_queries.py +++ b/tests/query_test/test_queries.py @@ -50,11 +50,11 @@ class TestQueries(ImpalaTestSuite): return 'functional-query' def test_analytic_fns(self, vector): - # TODO: Enable some of these tests for Avro/Kudu if possible - # Don't attempt to evaluate timestamp expressions with Avro/Kudu tables which don't - # support a timestamp type yet + # TODO: Enable some of these tests for Avro if possible + # Don't attempt to evaluate timestamp expressions with Avro tables which don't + # support a timestamp type table_format = vector.get_value('table_format') - if table_format.file_format in ['avro', 'kudu']: + if table_format.file_format == 'avro': pytest.xfail("%s doesn't support TIMESTAMP" % (table_format.file_format)) if table_format.file_format == 'hbase': pytest.xfail("A lot of queries check for NULLs, which hbase does not recognize")
