IMPALA-4897: AnalysisException: specified cache pool does not exist A few tests were added with IMPALA-1670 that made use of HDFS caching. This patch moves these tests to a new file and only executes them when the default filesystem is HDFS.
There was also a bug where the tests used absolute locations instead of locations relative to the table they were in which could easily collide with locations of other tables if they raced. That has been fixed too. Also added a testcase for testing alter table ADD multiple PARTITIONS for non-HDFS filesystems. Change-Id: Iefe61556bc28ae320f3f41fdc930d37b258d970a Reviewed-on: http://gerrit.cloudera.org:8080/5972 Reviewed-by: Sailesh Mukil <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/1f80396b Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/1f80396b Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/1f80396b Branch: refs/heads/master Commit: 1f80396b20529e6e77d4edbd7d3d62b29a106568 Parents: cd153d6 Author: Sailesh Mukil <[email protected]> Authored: Fri Feb 10 16:12:06 2017 -0800 Committer: Impala Public Jenkins <[email protected]> Committed: Tue Feb 14 05:56:33 2017 +0000 ---------------------------------------------------------------------- .../QueryTest/alter-table-hdfs-caching.test | 110 +++++++++++++++++++ .../queries/QueryTest/alter-table.test | 80 +++----------- tests/metadata/test_ddl.py | 5 + 3 files changed, 129 insertions(+), 66 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1f80396b/testdata/workloads/functional-query/queries/QueryTest/alter-table-hdfs-caching.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/alter-table-hdfs-caching.test b/testdata/workloads/functional-query/queries/QueryTest/alter-table-hdfs-caching.test new file mode 100644 index 0000000..71b61f9 --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/alter-table-hdfs-caching.test @@ -0,0 +1,110 @@ +# This test file covers alter table statements that add multiple partitions and use hdfs caching +==== +---- QUERY +create table i1670B_alter (s string) partitioned by (i integer); +alter table i1670B_alter add +partition (i=1) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670B_alter/i1' +cached in 'testPool' with replication=3 +partition (i=2) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670B_alter/i2' +partition (i=3) uncached; +show partitions i1670B_alter; +---- RESULTS +'1',-1,0,'0B','0B','3','TEXT','false',regex:.*/i1 +'2',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i2 +'3',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i=3 +'Total',-1,0,'0B','0B','','','','' +---- TYPES +STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING +==== +---- QUERY +# IMPALA-1670: Set up i1670C_alter table for the next test case. +create table i1670C_alter (s string) partitioned by (i integer); +alter table i1670C_alter add +partition (i=2) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670C_alter/i2A' +cached in 'testPool' with replication=2 +partition (i=4) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670C_alter/i4A' uncached; +show partitions i1670C_alter; +---- RESULTS +'2',-1,0,'0B','0B','2','TEXT','false',regex:.*/i2A +'4',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i4A +'Total',-1,0,'0B','0B','','','','' +---- TYPES +STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING +==== +---- QUERY +# IMPALA-1670: If 'IF NOT EXISTS' is used ALTER TABLE ADD PARTITION works with preexisting +# partitions. Location and caching options of existing partitions are not modified. +alter table i1670C_alter add if not exists +partition (i=1) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670C_alter/i1B' +partition (i=2) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670C_alter/i2B' uncached +partition (i=3) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670C_alter/i3B' +cached in 'testPool' with replication=3 +partition (i=4) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670C_alter/i4B' +cached in 'testPool' with replication=4; +show partitions i1670C_alter; +---- RESULTS +'1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i1B +'2',-1,0,'0B','0B','2','TEXT','false',regex:.*/i2A +'3',-1,0,'0B','0B','3','TEXT','false',regex:.*/i3B +'4',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i4A +'Total',-1,0,'0B','0B','','','','' +---- TYPES +STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING +==== +---- QUERY +# IMPALA-1670: Partitions without explicit CACHED IN/UNCACHED clause inherit cacheop from +# the parent table +create table i1670D_alter (s string) partitioned by (i integer) +cached in 'testPool' with replication=7; +alter table i1670D_alter add +partition (i=1) cached in 'testPool' with replication=5 +partition (i=2) +partition (i=3) uncached +partition (i=4); +show partitions i1670D_alter; +---- RESULTS +'1',-1,0,'0B','0B','5','TEXT','false',regex:.*/i=1 +'2',-1,0,'0B','0B','7','TEXT','false',regex:.*/i=2 +'3',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i=3 +'4',-1,0,'0B','0B','7','TEXT','false',regex:.*/i=4 +'Total',-1,0,'0B','0B','','','','' +---- TYPES +STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING +==== +---- QUERY +# IMPALA-1670: After INVALIDATE METADATA Impala can access previously added partitions and +# partition data. +create table i1670E_alter (a int) partitioned by (x int); +alter table i1670E_alter add partition (x=1) +partition (x=2) uncached +partition (x=3) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670E_alter/x3' +cached in 'testPool' with replication=7; +insert into i1670E_alter partition(x=1) values (1), (2), (3); +insert into i1670E_alter partition(x=2) values (1), (2), (3), (4); +insert into i1670E_alter partition(x=3) values (1); +invalidate metadata i1670E_alter; +==== +---- QUERY +show partitions i1670E_alter; +---- RESULTS +'1',-1,1,regex:.*,'NOT CACHED','NOT CACHED','TEXT','false',regex:.*/x=1 +'2',-1,1,regex:.*,'NOT CACHED','NOT CACHED','TEXT','false',regex:.*/x=2 +'3',-1,1,regex:.*,regex:.*,'7','TEXT','false',regex:.*/x3 +'Total',-1,3,regex:.*,regex:.*,'','','','' +---- TYPES +STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING +==== +---- QUERY +select x, a from i1670E_alter order by x, a; +---- RESULTS +1,1 +1,2 +1,3 +2,1 +2,2 +2,3 +2,4 +3,1 +---- TYPES +INT, INT +==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1f80396b/testdata/workloads/functional-query/queries/QueryTest/alter-table.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/alter-table.test b/testdata/workloads/functional-query/queries/QueryTest/alter-table.test index e067bc1..ecb6d6e 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/alter-table.test +++ b/testdata/workloads/functional-query/queries/QueryTest/alter-table.test @@ -1007,12 +1007,12 @@ INT, STRING # IMPALA-1670: Support adding multiple partitions in ALTER TABLE ADD PARTITION create table i1670A_alter (s string) partitioned by (i integer); alter table i1670A_alter add -partition (i=1) location '/i1' cached in 'testPool' with replication=3 -partition (i=2) location '/i2' +partition (i=1) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670A_alter/i1' +partition (i=2) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670A_alter/i2' partition (i=3) uncached; show partitions i1670A_alter; ---- RESULTS -'1',-1,0,'0B','0B','3','TEXT','false',regex:.*/i1 +'1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i1 '2',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i2 '3',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i=3 'Total',-1,0,'0B','0B','','','','' @@ -1020,81 +1020,29 @@ show partitions i1670A_alter; STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING ==== ---- QUERY -# IMPALA-1670: Set up i1670C_alter table for the next test case. -create table i1670C_alter (s string) partitioned by (i integer); -alter table i1670C_alter add -partition (i=2) location '/i2A' cached in 'testPool' with replication=2 -partition (i=4) location '/i4A' uncached; -show partitions i1670C_alter; ----- RESULTS -'2',-1,0,'0B','0B','2','TEXT','false',regex:.*/i2A -'4',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i4A -'Total',-1,0,'0B','0B','','','','' ----- TYPES -STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING -==== ----- QUERY -# IMPALA-1670: If 'IF NOT EXISTS' is used ALTER TABLE ADD PARTITION works with preexisting -# partitions. Location and caching options of existing partitions are not modified. -alter table i1670C_alter add if not exists -partition (i=1) location '/i1B' -partition (i=2) location '/i2B' uncached -partition (i=3) location '/i3B' cached in 'testPool' with replication=3 -partition (i=4) location '/i4B' cached in 'testPool' with replication=4; -show partitions i1670C_alter; ----- RESULTS -'1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i1B -'2',-1,0,'0B','0B','2','TEXT','false',regex:.*/i2A -'3',-1,0,'0B','0B','3','TEXT','false',regex:.*/i3B -'4',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i4A -'Total',-1,0,'0B','0B','','','','' ----- TYPES -STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING -==== ----- QUERY -# IMPALA-1670: Partitions without explicit CACHED IN/UNCACHED clause inherit cacheop from -# the parent table -create table i1670D_alter (s string) partitioned by (i integer) -cached in 'testPool' with replication=7; -alter table i1670D_alter add -partition (i=1) cached in 'testPool' with replication=5 -partition (i=2) -partition (i=3) uncached -partition (i=4); -show partitions i1670D_alter; ----- RESULTS -'1',-1,0,'0B','0B','5','TEXT','false',regex:.*/i=1 -'2',-1,0,'0B','0B','7','TEXT','false',regex:.*/i=2 -'3',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i=3 -'4',-1,0,'0B','0B','7','TEXT','false',regex:.*/i=4 -'Total',-1,0,'0B','0B','','','','' ----- TYPES -STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING -==== ----- QUERY # IMPALA-1670: After INVALIDATE METADATA Impala can access previously added partitions and # partition data. -create table i1670E_alter (a int) partitioned by (x int); -alter table i1670E_alter add partition (x=1) +create table i1670A_alter_1 (a int) partitioned by (x int); +alter table i1670A_alter_1 add partition (x=1) partition (x=2) uncached -partition (x=3) location '/x3' cached in 'testPool' with replication=7; -insert into i1670E_alter partition(x=1) values (1), (2), (3); -insert into i1670E_alter partition(x=2) values (1), (2), (3), (4); -insert into i1670E_alter partition(x=3) values (1); -invalidate metadata i1670E_alter; +partition (x=3) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670A_alter_1/x3'; +insert into i1670A_alter_1 partition(x=1) values (1), (2), (3); +insert into i1670A_alter_1 partition(x=2) values (1), (2), (3), (4); +insert into i1670A_alter_1 partition(x=3) values (1); +invalidate metadata i1670A_alter_1; ==== ---- QUERY -show partitions i1670E_alter; +show partitions i1670A_alter_1; ---- RESULTS '1',-1,1,regex:.*,'NOT CACHED','NOT CACHED','TEXT','false',regex:.*/x=1 '2',-1,1,regex:.*,'NOT CACHED','NOT CACHED','TEXT','false',regex:.*/x=2 -'3',-1,1,regex:.*,regex:.*,'7','TEXT','false',regex:.*/x3 -'Total',-1,3,regex:.*,regex:.*,'','','','' +'3',-1,1,regex:.*,'NOT CACHED','NOT CACHED','TEXT','false',regex:.*/x3 +'Total',-1,3,regex:.*,'0B','','','','' ---- TYPES STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING ==== ---- QUERY -select x, a from i1670E_alter order by x, a; +select x, a from i1670A_alter_1 order by x, a; ---- RESULTS 1,1 1,2 http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1f80396b/tests/metadata/test_ddl.py ---------------------------------------------------------------------- diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py index 40c7a99..37ca7cd 100644 --- a/tests/metadata/test_ddl.py +++ b/tests/metadata/test_ddl.py @@ -253,6 +253,11 @@ class TestDdlStatements(TestDdlBase): file_data='1984') self.run_test_case('QueryTest/alter-table', vector, use_db=unique_database, multiple_impalad=self._use_multiple_impalad(vector)) + # The following tests require HDFS caching which is supported only in the HDFS + # filesystem. + if IS_HDFS: + self.run_test_case('QueryTest/alter-table-hdfs-caching', vector, + use_db=unique_database, multiple_impalad=self._use_multiple_impalad(vector)) @UniqueDatabase.parametrize(sync_ddl=True) def test_alter_set_column_stats(self, vector, unique_database):
