IMPALA-4502: test_partition_ddl_predicates breaks on non-HDFS filesystems This is because that test uses 'set cached' and 'set uncached' which are not supported on non-HDFS filesystems. This patch creates a separate test file for non-HDFS filesystems with only supported queries and invokes the right file based on the filesystem.
Change-Id: I8606aa427cb6e50be3395cdde246abb53db5172c Reviewed-on: http://gerrit.cloudera.org:8080/5164 Reviewed-by: Sailesh Mukil <[email protected]> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/178fd591 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/178fd591 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/178fd591 Branch: refs/heads/master Commit: 178fd59142db5f401d1426fa4daa11432e1e2f50 Parents: 7bfa6e3 Author: Sailesh Mukil <[email protected]> Authored: Mon Nov 21 08:27:21 2016 -0800 Committer: Internal Jenkins <[email protected]> Committed: Tue Nov 22 00:42:57 2016 +0000 ---------------------------------------------------------------------- .../partition-ddl-predicates-all-fs.test | 135 ++++++++++++++++ .../partition-ddl-predicates-hdfs-only.test | 38 +++++ .../QueryTest/partition-ddl-predicates.test | 156 ------------------- tests/metadata/test_ddl.py | 7 +- 4 files changed, 178 insertions(+), 158 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/178fd591/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test b/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test new file mode 100644 index 0000000..36104c3 --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test @@ -0,0 +1,135 @@ +==== +---- QUERY +# First create a partitioned table +create table p1 (i int) partitioned by(j int, k string); +# Add some partitions +alter table p1 add partition (j=1,k="a"); +alter table p1 add partition (j=1,k="b"); +alter table p1 add partition (j=1,k="c"); +alter table p1 add partition (j=2,k="d"); +alter table p1 add partition (j=2,k="e"); +alter table p1 add partition (j=2,k="f"); +alter table p1 add partition (j=2,k=NULL); +alter table p1 add partition (j=NULL,k="g"); +alter table p1 add partition (j=NULL,k=NULL); +# Populate the table +insert into p1 partition (j, k) values (100, 1, "a"), (200, 1, "b"), (300, 1, "c"); +==== +---- QUERY +show partitions p1 +---- RESULTS +'NULL','NULL',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=__HIVE_DEFAULT_PARTITION__/k=__HIVE_DEFAULT_PARTITION__ +'NULL','g',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=__HIVE_DEFAULT_PARTITION__/k=g +'1','a',-1,1,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=1/k=a +'1','b',-1,1,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=1/k=b +'1','c',-1,1,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=1/k=c +'2','NULL',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=__HIVE_DEFAULT_PARTITION__ +'2','d',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=d +'2','e',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=e +'2','f',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=f +'Total','',-1,3,regex:.+,regex:.+,'','','','' +---- TYPES +STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING +==== +---- QUERY +show files in p1 partition (j<2, k="a") +---- RESULTS +regex:.+,regex:.+,'j=1/k=a' +---- TYPES +STRING, STRING, STRING +==== +---- QUERY +alter table p1 partition (j=2) set fileformat parquet +---- RESULTS +'Updated 4 partition(s).' +---- TYPES +STRING +==== +---- QUERY +alter table p1 partition (k rlike ".*") set serdeproperties ('k'='v') +---- RESULTS +'Updated 7 partition(s).' +---- TYPES +STRING +==== +---- QUERY +alter table p1 partition (k="") set serdeproperties ('k'='v') +---- RESULTS +'Updated 2 partition(s).' +---- TYPES +STRING +==== +---- QUERY +alter table p1 partition (k=null) set serdeproperties ('k'='v') +---- RESULTS +'Updated 2 partition(s).' +---- TYPES +STRING +==== +---- QUERY +alter table p1 partition (j is not null) set tblproperties ('k'='v') +---- RESULTS +'Updated 7 partition(s).' +---- TYPES +STRING +==== +---- QUERY +compute incremental stats p1 partition(j<2) +---- RESULTS +'Updated 3 partition(s) and 1 column(s).' +---- TYPES +STRING +==== +---- QUERY +alter table p1 drop partition (j<2) +---- RESULTS +'Dropped 3 partition(s).' +---- TYPES +STRING +==== +---- QUERY +alter table p1 drop partition (j<2) +---- RESULTS +'Dropped 0 partition(s).' +---- TYPES +STRING +==== +---- QUERY +alter table p1 drop if exists partition (j<2) +---- RESULTS +'Dropped 0 partition(s).' +---- TYPES +STRING +==== +---- QUERY +alter table p1 drop if exists partition (j=2, k="bla") +---- RESULTS +'Dropped 0 partition(s).' +---- TYPES +STRING +==== +---- QUERY +alter table p1 drop partition (j=2, k=NULL) +---- RESULTS +'Dropped 1 partition(s).' +---- TYPES +STRING +==== +---- QUERY +alter table p1 drop partition (j IS NULL AND k IS NULL); +---- RESULTS +'Dropped 1 partition(s).' +---- TYPES +STRING +==== +---- QUERY +show partitions p1 +---- RESULTS +'NULL','g',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=__HIVE_DEFAULT_PARTITION__/k=g +'2','d',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=d +'2','e',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=e +'2','f',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=f +'Total','',0,0,regex:.+,regex:.+,'','','','' +---- TYPES +STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING +==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/178fd591/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-hdfs-only.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-hdfs-only.test b/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-hdfs-only.test new file mode 100644 index 0000000..5ca2a06 --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-hdfs-only.test @@ -0,0 +1,38 @@ +==== +---- QUERY +# First create a partitioned table +create table p1_hdfs (i int) partitioned by(j int, k string); +# Add some partitions +alter table p1_hdfs add partition (j=1,k="a"); +alter table p1_hdfs add partition (j=1,k="b"); +alter table p1_hdfs add partition (j=1,k="c"); +alter table p1_hdfs add partition (j=2,k="d"); +alter table p1_hdfs add partition (j=2,k="e"); +alter table p1_hdfs add partition (j=2,k="f"); +alter table p1_hdfs add partition (j=2,k=NULL); +alter table p1_hdfs add partition (j=NULL,k="g"); +alter table p1_hdfs add partition (j=NULL,k=NULL); +# Populate the table +insert into p1_hdfs partition (j, k) values (100, 1, "a"), (200, 1, "b"), (300, 1, "c"); +==== +---- QUERY +alter table p1_hdfs partition (j<2, k in ("b", "c")) set cached in 'testPool' +---- RESULTS +'Cached 2 partition(s).' +---- TYPES +STRING +==== +---- QUERY +alter table p1_hdfs partition (j<2, j>0, k<>"d") set uncached +---- RESULTS +'Uncached 2 partition(s).' +---- TYPES +STRING +==== +---- QUERY +alter table p1_hdfs partition (j=3 or j=2, k like "%") set uncached +---- RESULTS +'Uncached 0 partition(s).' +---- TYPES +STRING +==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/178fd591/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates.test b/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates.test deleted file mode 100644 index cc7e865..0000000 --- a/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates.test +++ /dev/null @@ -1,156 +0,0 @@ -==== ----- QUERY -# First create a partitioned table -create table p1 (i int) partitioned by(j int, k string); -# Add some partitions -alter table p1 add partition (j=1,k="a"); -alter table p1 add partition (j=1,k="b"); -alter table p1 add partition (j=1,k="c"); -alter table p1 add partition (j=2,k="d"); -alter table p1 add partition (j=2,k="e"); -alter table p1 add partition (j=2,k="f"); -alter table p1 add partition (j=2,k=NULL); -alter table p1 add partition (j=NULL,k="g"); -alter table p1 add partition (j=NULL,k=NULL); -# Populate the table -insert into p1 partition (j, k) values (100, 1, "a"), (200, 1, "b"), (300, 1, "c"); -==== ----- QUERY -show partitions p1 ----- RESULTS -'NULL','NULL',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=__HIVE_DEFAULT_PARTITION__/k=__HIVE_DEFAULT_PARTITION__ -'NULL','g',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=__HIVE_DEFAULT_PARTITION__/k=g -'1','a',-1,1,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=1/k=a -'1','b',-1,1,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=1/k=b -'1','c',-1,1,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=1/k=c -'2','NULL',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=__HIVE_DEFAULT_PARTITION__ -'2','d',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=d -'2','e',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=e -'2','f',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=f -'Total','',-1,3,regex:.+,regex:.+,'','','','' ----- TYPES -STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING -==== ----- QUERY -show files in p1 partition (j<2, k="a") ----- RESULTS -regex:.+,regex:.+,'j=1/k=a' ----- TYPES -STRING, STRING, STRING -==== ----- QUERY -alter table p1 partition (j<2, k in ("b", "c")) set cached in 'testPool' ----- RESULTS -'Cached 2 partition(s).' ----- TYPES -STRING -==== ----- QUERY -alter table p1 partition (j<2, j>0, k<>"d") set uncached ----- RESULTS -'Uncached 2 partition(s).' ----- TYPES -STRING -==== ----- QUERY -alter table p1 partition (j=3 or j=2, k like "%") set uncached ----- RESULTS -'Uncached 0 partition(s).' ----- TYPES -STRING -==== ----- QUERY -alter table p1 partition (j=2) set fileformat parquet ----- RESULTS -'Updated 4 partition(s).' ----- TYPES -STRING -==== ----- QUERY -alter table p1 partition (k rlike ".*") set serdeproperties ('k'='v') ----- RESULTS -'Updated 7 partition(s).' ----- TYPES -STRING -==== ----- QUERY -alter table p1 partition (k="") set serdeproperties ('k'='v') ----- RESULTS -'Updated 2 partition(s).' ----- TYPES -STRING -==== ----- QUERY -alter table p1 partition (k=null) set serdeproperties ('k'='v') ----- RESULTS -'Updated 2 partition(s).' ----- TYPES -STRING -==== ----- QUERY -alter table p1 partition (j is not null) set tblproperties ('k'='v') ----- RESULTS -'Updated 7 partition(s).' ----- TYPES -STRING -==== ----- QUERY -compute incremental stats p1 partition(j<2) ----- RESULTS -'Updated 3 partition(s) and 1 column(s).' ----- TYPES -STRING -==== ----- QUERY -alter table p1 drop partition (j<2) ----- RESULTS -'Dropped 3 partition(s).' ----- TYPES -STRING -==== ----- QUERY -alter table p1 drop partition (j<2) ----- RESULTS -'Dropped 0 partition(s).' ----- TYPES -STRING -==== ----- QUERY -alter table p1 drop if exists partition (j<2) ----- RESULTS -'Dropped 0 partition(s).' ----- TYPES -STRING -==== ----- QUERY -alter table p1 drop if exists partition (j=2, k="bla") ----- RESULTS -'Dropped 0 partition(s).' ----- TYPES -STRING -==== ----- QUERY -alter table p1 drop partition (j=2, k=NULL) ----- RESULTS -'Dropped 1 partition(s).' ----- TYPES -STRING -==== ----- QUERY -alter table p1 drop partition (j IS NULL AND k IS NULL); ----- RESULTS -'Dropped 1 partition(s).' ----- TYPES -STRING -==== ----- QUERY -show partitions p1 ----- RESULTS -'NULL','g',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=__HIVE_DEFAULT_PARTITION__/k=g -'2','d',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=d -'2','e',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=e -'2','f',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=f -'Total','',0,0,regex:.+,regex:.+,'','','','' ----- TYPES -STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING -==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/178fd591/tests/metadata/test_ddl.py ---------------------------------------------------------------------- diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py index 15fe4c7..58f42eb 100644 --- a/tests/metadata/test_ddl.py +++ b/tests/metadata/test_ddl.py @@ -24,7 +24,7 @@ from tests.common.impala_test_suite import LOG from tests.common.parametrize import UniqueDatabase from tests.common.skip import SkipIf, SkipIfLocal, SkipIfOldAggsJoins from tests.common.test_dimensions import create_single_exec_option_dimension -from tests.util.filesystem_utils import WAREHOUSE, IS_LOCAL, IS_S3 +from tests.util.filesystem_utils import WAREHOUSE, IS_HDFS, IS_LOCAL, IS_S3 # Validates DDL statements (create, drop) class TestDdlStatements(TestDdlBase): @@ -393,8 +393,11 @@ class TestDdlStatements(TestDdlBase): @UniqueDatabase.parametrize(sync_ddl=True) def test_partition_ddl_predicates(self, vector, unique_database): - self.run_test_case('QueryTest/partition-ddl-predicates', vector, + self.run_test_case('QueryTest/partition-ddl-predicates-all-fs', vector, use_db=unique_database, multiple_impalad=self._use_multiple_impalad(vector)) + if IS_HDFS: + self.run_test_case('QueryTest/partition-ddl-predicates-hdfs-only', vector, + use_db=unique_database, multiple_impalad=self._use_multiple_impalad(vector)) # IMPALA-2002: Tests repeated adding/dropping of .jar and .so in the lib cache. class TestLibCache(TestDdlBase):
