Repository: incubator-impala Updated Branches: refs/heads/master c68734ad6 -> cac02d6b7
IMPALA-4454: test_kudu.TestShowCreateTable flaky The cause of the flakiness is Kudu CREATE TABLE operations that are sometimes taking a long time, leading to timeouts in the hiveserver2 connection. This patch adds the ability for tests using the 'conn' pytest fixture to specify a timeout to connect(), and sets a timeout of 5 minutes for this test. Change-Id: I2727c27ff66140ac4043bcad332cd4e1d72b255f Reviewed-on: http://gerrit.cloudera.org:8080/5040 Reviewed-by: Michael Brown <[email protected]> Reviewed-by: Alex Behm <[email protected]> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/d15f86cb Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/d15f86cb Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/d15f86cb Branch: refs/heads/master Commit: d15f86cb6fa5eb482b039f036dab3d979c7513c8 Parents: c68734a Author: Thomas Tauber-Marshall <[email protected]> Authored: Thu Nov 10 15:13:56 2016 -0800 Committer: Internal Jenkins <[email protected]> Committed: Fri Nov 11 20:04:01 2016 +0000 ---------------------------------------------------------------------- tests/conftest.py | 14 +++++++++----- tests/query_test/test_kudu.py | 5 +++++ 2 files changed, 14 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d15f86cb/tests/conftest.py ---------------------------------------------------------------------- diff --git a/tests/conftest.py b/tests/conftest.py index c75f117..6bbbf78 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -36,6 +36,7 @@ from tests.util.filesystem_utils import FILESYSTEM, ISILON_WEBHDFS_PORT logging.basicConfig(level=logging.INFO, format='%(threadName)s: %(message)s') LOG = logging.getLogger('test_configuration') +DEFAULT_CONN_TIMEOUT = 45 def _get_default_nn_http_addr(): """Return the namenode ip and webhdfs port if the default shouldn't be used""" @@ -320,6 +321,7 @@ def conn(request): provided by get_db_name(), it must not exist. Classes that use both auto_create_db() and get_db_name() should generate a random name in get_db_name() and cache it. + - get_conn_timeout(): The timeout, in seconds, to use for this connection. The returned connection will have a 'db_name' property. See the 'unique_database' fixture above if you want to use Impala's custom python @@ -327,8 +329,10 @@ def conn(request): """ db_name = __call_cls_method_if_exists(request.cls, "get_db_name") use_unique_conn = __call_cls_method_if_exists(request.cls, "auto_create_db") + timeout = __call_cls_method_if_exists(request.cls, "get_conn_timeout") or \ + DEFAULT_CONN_TIMEOUT if use_unique_conn: - with __unique_conn(db_name=db_name) as conn: + with __unique_conn(db_name=db_name, timeout=timeout) as conn: yield conn else: with __auto_closed_conn(db_name=db_name) as conn: @@ -345,7 +349,7 @@ def __call_cls_method_if_exists(cls, method_name): @contextlib.contextmanager -def __unique_conn(db_name=None): +def __unique_conn(db_name=None, timeout=DEFAULT_CONN_TIMEOUT): """Connects to Impala and creates a new database, then returns a connection to it. This is intended to be used in a "with" block. Upon exit, the database will be dropped. A database name can be provided by 'db_name', a database by that name @@ -362,7 +366,7 @@ def __unique_conn(db_name=None): with __auto_closed_conn() as conn: with __auto_closed_cursor(conn) as cur: cur.execute("CREATE DATABASE %s" % db_name) - with __auto_closed_conn(db_name=db_name) as conn: + with __auto_closed_conn(db_name=db_name, timeout=timeout) as conn: try: yield conn finally: @@ -378,13 +382,13 @@ def __unique_conn(db_name=None): @contextlib.contextmanager -def __auto_closed_conn(db_name=None): +def __auto_closed_conn(db_name=None, timeout=DEFAULT_CONN_TIMEOUT): """Returns a connection to Impala. This is intended to be used in a "with" block. The connection will be closed upon exiting the block. The returned connection will have a 'db_name' property. """ - conn = impala_connect(database=db_name) + conn = impala_connect(database=db_name, timeout=timeout) try: conn.db_name = db_name yield conn http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d15f86cb/tests/query_test/test_kudu.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_kudu.py b/tests/query_test/test_kudu.py index 2a88137..f5579ed 100644 --- a/tests/query_test/test_kudu.py +++ b/tests/query_test/test_kudu.py @@ -200,6 +200,11 @@ class TestCreateExternalTable(KuduTestSuite): class TestShowCreateTable(KuduTestSuite): + @classmethod + def get_conn_timeout(cls): + # For IMPALA-4454 + return 60 * 5 # 5 minutes + def assert_show_create_equals(self, cursor, create_sql, show_create_sql): """Executes 'create_sql' to create a table, then runs "SHOW CREATE TABLE" and checks that the output is the same as 'show_create_sql'. 'create_sql' and
