IMPALA-6544: S3 tests fail because S3 is eventually consistent. This skips test_insert_parquet on S3, as S3's eventually consistent behavior is exposing us to "file not found" errors.
To remove this band-aid, we'll need to either handle s3 consistency natively in some way, or to retry certain tests. Note that this may be only a drop in a big bucket at the moment; I think we see these issues in a variety of places. Change-Id: I22e7620e97366bece69a20885f5c3b75de05fab6 Reviewed-on: http://gerrit.cloudera.org:8080/12127 Reviewed-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/6f6d1f64 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/6f6d1f64 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/6f6d1f64 Branch: refs/heads/master Commit: 6f6d1f64a10f9a958dfa6162ec8bdab4cdcc6d92 Parents: 378f8c2 Author: Philip Zeyliger <phi...@cloudera.com> Authored: Fri Dec 21 13:36:54 2018 -0800 Committer: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Committed: Fri Dec 28 01:50:39 2018 +0000 ---------------------------------------------------------------------- tests/common/skip.py | 2 ++ tests/query_test/test_insert_parquet.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/6f6d1f64/tests/common/skip.py ---------------------------------------------------------------------- diff --git a/tests/common/skip.py b/tests/common/skip.py index 8ba7a06..b58f057 100644 --- a/tests/common/skip.py +++ b/tests/common/skip.py @@ -56,6 +56,8 @@ class SkipIfS3: hbase = pytest.mark.skipif(IS_S3, reason="HBase not started with S3") qualified_path = pytest.mark.skipif(IS_S3, reason="Tests rely on HDFS qualified paths, IMPALA-1872") + eventually_consistent = pytest.mark.skipif(IS_S3, + reason="Flakiness on account of S3 eventual consistency.") class SkipIfABFS: http://git-wip-us.apache.org/repos/asf/impala/blob/6f6d1f64/tests/query_test/test_insert_parquet.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_insert_parquet.py b/tests/query_test/test_insert_parquet.py index 0bf28fe..6924d26 100644 --- a/tests/query_test/test_insert_parquet.py +++ b/tests/query_test/test_insert_parquet.py @@ -22,7 +22,6 @@ import os from collections import namedtuple from datetime import datetime from decimal import Decimal -from shutil import rmtree from subprocess import check_call from parquet.ttypes import ColumnOrder, SortingColumn, TypeDefinedOrder @@ -104,6 +103,7 @@ class TestInsertParquetQueries(ImpalaTestSuite): @SkipIfEC.oom @SkipIfLocal.multiple_impalad + @SkipIfS3.eventually_consistent @UniqueDatabase.parametrize(sync_ddl=True) def test_insert_parquet(self, vector, unique_database): vector.get_value('exec_option')['PARQUET_FILE_SIZE'] = \
