IMPALA-3532: S3: test_truncate_cleans_hdfs_files fails because we skip INSERT staging
On the introduction of IMPALA-3452, we defaulted to skipping the INSERT staging for S3. test_truncate_cleans_hdfs_files assumes that it will always see the _impala_insert_staging folder but we will not see that on S3 runs. This patch deletes the staging folder if it exists and continues the test without taking into account the staging folder. Change-Id: I3580f03690e29fe99f441b26bc9baa4c0964d79c Reviewed-on: http://gerrit.cloudera.org:8080/3049 Reviewed-by: Sailesh Mukil <[email protected]> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/4c9c74dd Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/4c9c74dd Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/4c9c74dd Branch: refs/heads/master Commit: 4c9c74dd33e0b84c3a3328df0a1721d627267192 Parents: b4558d3 Author: Sailesh Mukil <[email protected]> Authored: Thu May 12 13:07:58 2016 -0700 Committer: Tim Armstrong <[email protected]> Committed: Sat May 14 01:30:01 2016 -0700 ---------------------------------------------------------------------- tests/metadata/test_ddl.py | 54 +++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4c9c74dd/tests/metadata/test_ddl.py ---------------------------------------------------------------------- diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py index 0a1900c..6179d1a 100644 --- a/tests/metadata/test_ddl.py +++ b/tests/metadata/test_ddl.py @@ -183,30 +183,36 @@ class TestDdlStatements(ImpalaTestSuite): # Verify the table directory exists assert self.filesystem_client.exists("test-warehouse/truncate_table_test_db.db/t1/") - # Should have created one file in the table's dir - self.client.execute("insert into %s.t1 values (1)" % TRUNCATE_DB) - assert len(self.filesystem_client.ls("test-warehouse/%s.db/t1/" % TRUNCATE_DB)) == 2 - - # Truncating the table removes the data files and preserves the table's directory - self.client.execute("truncate table %s.t1" % TRUNCATE_DB) - assert len(self.filesystem_client.ls("test-warehouse/%s.db/t1/" % TRUNCATE_DB)) == 1 - - self.client.execute( - "create table %s.t2(i int) partitioned by (p int)" % TRUNCATE_DB) - # Verify the table directory exists - assert self.filesystem_client.exists("test-warehouse/%s.db/t2/" % TRUNCATE_DB) - - # Should have created the partition dir, which should contain exactly one file - self.client.execute( - "insert into %s.t2 partition(p=1) values (1)" % TRUNCATE_DB) - assert len(self.filesystem_client.ls( - "test-warehouse/%s.db/t2/p=1" % TRUNCATE_DB)) == 1 - - # Truncating the table removes the data files and preserves the partition's directory - self.client.execute("truncate table %s.t2" % TRUNCATE_DB) - assert self.filesystem_client.exists("test-warehouse/%s.db/t2/p=1" % TRUNCATE_DB) - assert len(self.filesystem_client.ls( - "test-warehouse/%s.db/t2/p=1" % TRUNCATE_DB)) == 0 + try: + # If we're testing S3, we want the staging directory to be created. + self.client.execute("set s3_skip_insert_staging=false") + # Should have created one file in the table's dir + self.client.execute("insert into %s.t1 values (1)" % TRUNCATE_DB) + assert len(self.filesystem_client.ls("test-warehouse/%s.db/t1/" % TRUNCATE_DB)) == 2 + + # Truncating the table removes the data files and preserves the table's directory + self.client.execute("truncate table %s.t1" % TRUNCATE_DB) + assert len(self.filesystem_client.ls("test-warehouse/%s.db/t1/" % TRUNCATE_DB)) == 1 + + self.client.execute( + "create table %s.t2(i int) partitioned by (p int)" % TRUNCATE_DB) + # Verify the table directory exists + assert self.filesystem_client.exists("test-warehouse/%s.db/t2/" % TRUNCATE_DB) + + # Should have created the partition dir, which should contain exactly one file + self.client.execute( + "insert into %s.t2 partition(p=1) values (1)" % TRUNCATE_DB) + assert len(self.filesystem_client.ls( + "test-warehouse/%s.db/t2/p=1" % TRUNCATE_DB)) == 1 + + # Truncating the table removes the data files and preserves the partition's directory + self.client.execute("truncate table %s.t2" % TRUNCATE_DB) + assert self.filesystem_client.exists("test-warehouse/%s.db/t2/p=1" % TRUNCATE_DB) + assert len(self.filesystem_client.ls( + "test-warehouse/%s.db/t2/p=1" % TRUNCATE_DB)) == 0 + finally: + # Reset to its default value. + self.client.execute("set s3_skip_insert_staging=true") @pytest.mark.execute_serially def test_truncate_table(self, vector):
