This is an automated email from the ASF dual-hosted git repository. tarmstrong pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 3e9cac0cac0063dda5d73ce00ccc8bd2332f50a1 Author: Zoltan Borok-Nagy <borokna...@cloudera.com> AuthorDate: Wed Aug 14 18:26:13 2019 +0200 IMPALA-8854: fix acid insert tests test_acid_nonacid_insert has been failing lately. HMS became more strict about checking the capabilities of its clients. Seems like the Python client doesn't set any capabilities for itself therefore HMS rejects its attempts of creating and dropping tables. Now instead of using the RESET utility from the e2e test framework (to drop and re-create tables), the test is using a unique database and creates the tables through Impala. Different file formats are exercised with the help of the DEFAULT_FILE_FORMAT query option. Change-Id: I3a82338a7820d0ee748c961c8656fa3319c3929c Reviewed-on: http://gerrit.cloudera.org:8080/14064 Reviewed-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> --- .../functional/functional_schema_template.sql | 23 -------- .../queries/QueryTest/acid-insert.test | 63 +++++++++++----------- tests/query_test/test_insert.py | 29 ++++------ 3 files changed, 41 insertions(+), 74 deletions(-) diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql index ab59a5c..d460eaa 100644 --- a/testdata/datasets/functional/functional_schema_template.sql +++ b/testdata/datasets/functional/functional_schema_template.sql @@ -2649,26 +2649,3 @@ date_part DATE id_col INT date_col DATE ==== ----- DATASET -functional ----- BASE_TABLE_NAME -insertonly_nopart_insert ----- HIVE_MAJOR_VERSION -3 ----- CREATE -CREATE TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} (i int) -STORED AS {file_format} -TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only'); -==== ----- DATASET -functional ----- BASE_TABLE_NAME -insertonly_part_insert ----- HIVE_MAJOR_VERSION -3 ----- CREATE -CREATE TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} (i int) -PARTITIONED BY (p int) -STORED AS {file_format} -TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only'); -==== diff --git a/testdata/workloads/functional-query/queries/QueryTest/acid-insert.test b/testdata/workloads/functional-query/queries/QueryTest/acid-insert.test index 3e0baae..094b463 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/acid-insert.test +++ b/testdata/workloads/functional-query/queries/QueryTest/acid-insert.test @@ -1,11 +1,11 @@ ==== ----- SETUP -RESET insertonly_nopart_insert ---- QUERY -insert into insertonly_nopart_insert values (1), (2); +create table insertonly_nopart (i int) +tblproperties('transactional'='true', 'transactional_properties'='insert_only'); +insert into insertonly_nopart values (1), (2); ==== ---- QUERY -select i from insertonly_nopart_insert order by i; +select i from insertonly_nopart order by i; ---- RESULTS 1 2 @@ -13,10 +13,10 @@ select i from insertonly_nopart_insert order by i; INT ==== ---- QUERY -insert into insertonly_nopart_insert values (3); +insert into insertonly_nopart values (3); ==== ---- QUERY -select i from insertonly_nopart_insert order by i; +select i from insertonly_nopart order by i; ---- RESULTS 1 2 @@ -25,43 +25,44 @@ select i from insertonly_nopart_insert order by i; INT ==== ---- QUERY -insert overwrite insertonly_nopart_insert values (10); +insert overwrite insertonly_nopart values (10); ==== ---- QUERY -select i from insertonly_nopart_insert order by i; +select i from insertonly_nopart order by i; ---- RESULTS 10 ---- TYPES INT ==== ---- QUERY -insert overwrite insertonly_nopart_insert select 100; +insert overwrite insertonly_nopart select 100; ==== ---- QUERY -select i from insertonly_nopart_insert order by i; +select i from insertonly_nopart order by i; ---- RESULTS 100 ---- TYPES INT ==== ---- QUERY -insert overwrite insertonly_nopart_insert -select * from insertonly_nopart_insert limit 0; +insert overwrite insertonly_nopart +select * from insertonly_nopart limit 0; ==== ---- QUERY -select i from insertonly_nopart_insert order by i; +select i from insertonly_nopart order by i; ---- RESULTS ---- TYPES INT ==== ----- SETUP -RESET insertonly_part_insert ---- QUERY -insert into insertonly_part_insert partition (p=1) values (10), (11); -insert into insertonly_part_insert partition (p=2) values (20); +create table if not exists insertonly_part (i int) +partitioned by (p int) +tblproperties('transactional'='true', 'transactional_properties'='insert_only'); +insert into insertonly_part partition (p=1) values (10), (11); +insert into insertonly_part partition (p=2) values (20); ==== ---- QUERY -select p, i from insertonly_part_insert order by i; +select p, i from insertonly_part order by i; ---- RESULTS 1,10 1,11 @@ -70,11 +71,11 @@ select p, i from insertonly_part_insert order by i; INT,INT ==== ---- QUERY -insert into insertonly_part_insert partition (p=2) values (21); -insert into insertonly_part_insert partition (p=3) values (30); +insert into insertonly_part partition (p=2) values (21); +insert into insertonly_part partition (p=3) values (30); ==== ---- QUERY -select p, i from insertonly_part_insert order by i; +select p, i from insertonly_part order by i; ---- RESULTS 1,10 1,11 @@ -85,11 +86,11 @@ select p, i from insertonly_part_insert order by i; INT,INT ==== ---- QUERY -insert overwrite insertonly_part_insert partition (p=2) values (22); -insert overwrite insertonly_part_insert partition (p=3) values (31); +insert overwrite insertonly_part partition (p=2) values (22); +insert overwrite insertonly_part partition (p=3) values (31); ==== ---- QUERY -select p, i from insertonly_part_insert order by i; +select p, i from insertonly_part order by i; ---- RESULTS 1,10 1,11 @@ -99,24 +100,24 @@ select p, i from insertonly_part_insert order by i; INT,INT ==== ---- QUERY -insert overwrite insertonly_part_insert partition (p=1) -select * from insertonly_nopart_insert limit 0; -insert overwrite insertonly_part_insert partition (p=2) -select * from insertonly_nopart_insert limit 0; +insert overwrite insertonly_part partition (p=1) +select * from insertonly_nopart limit 0; +insert overwrite insertonly_part partition (p=2) +select * from insertonly_nopart limit 0; ==== ---- QUERY -select p, i from insertonly_part_insert order by i; +select p, i from insertonly_part order by i; ---- RESULTS 3,31 ---- TYPES INT,INT ==== ---- QUERY -insert overwrite insertonly_part_insert partition (p) +insert overwrite insertonly_part partition (p) values (1000, 1), (2000, 2), (4000, 4), (5000, 5), (5001, 5); ==== ---- QUERY -select p, i from insertonly_part_insert order by p, i; +select p, i from insertonly_part order by p, i; ---- RESULTS 1,1000 2,2000 diff --git a/tests/query_test/test_insert.py b/tests/query_test/test_insert.py index a630b3e..17ee7d8 100644 --- a/tests/query_test/test_insert.py +++ b/tests/query_test/test_insert.py @@ -136,27 +136,16 @@ class TestInsertQueries(ImpalaTestSuite): self.run_test_case('QueryTest/insert', vector, multiple_impalad=vector.get_value('exec_option')['sync_ddl'] == 1) - @pytest.mark.execute_serially @SkipIfHive2.acid - def test_acid_insert(self, vector): - pytest.skip("IMPALA-8854: skipping to unbreak builds") - if (vector.get_value('table_format').file_format == 'parquet'): - vector.get_value('exec_option')['COMPRESSION_CODEC'] = \ - vector.get_value('compression_codec') - # We need to turn off capability checks. Otherwise we get an error from HMS because - # this python client doesn't have the capability for handling ACID tables. But we only - # need to drop and create such tables, and table properties are preserved during - # those operations and this is enough for the tests (A table is ACID if it has the - # relevant table properties). - CAPABILITY_CHECK_CONF = "hive.metastore.client.capability.check" - capability_check = self.hive_client.getMetaConf(CAPABILITY_CHECK_CONF) - try: - self.hive_client.setMetaConf(CAPABILITY_CHECK_CONF, "false") - self.run_test_case('QueryTest/acid-insert', vector, - multiple_impalad=vector.get_value('exec_option')['sync_ddl'] == 1) - finally: - # Reset original state. - self.hive_client.setMetaConf(CAPABILITY_CHECK_CONF, capability_check) + @UniqueDatabase.parametrize(sync_ddl=True) + def test_acid_insert(self, vector, unique_database): + exec_options = vector.get_value('exec_option') + file_format = vector.get_value('table_format').file_format + if (file_format == 'parquet'): + exec_options['COMPRESSION_CODEC'] = vector.get_value('compression_codec') + exec_options['DEFAULT_FILE_FORMAT'] = file_format + self.run_test_case('QueryTest/acid-insert', vector, unique_database, + multiple_impalad=exec_options['sync_ddl'] == 1) @SkipIfHive2.acid @UniqueDatabase.parametrize(sync_ddl=True)