IMPALA-7790: Skip some Kudu tests if use_hybrid_clock=false Since IMPALA-6812, we've run many of our tests against Kudu at the READ_AT_SNAPSHOT scan level, which ensures consistent results. This scan level is only supported if Kudu is run with the flag --use_hybrid_clock=true (which is the default).
This patch uses the Kudu master webui to detect when use_hybrid_clock is false and skips these tests. Follow up work will address allowing these tests to run regardless of the value of the flag. Testing: - Ran a full exhaustive build with use_hybrid_clock=false set in the minicluster. Change-Id: I4c9ed4a4ea0720760d65c98acfc394247ab2f1a2 Reviewed-on: http://gerrit.cloudera.org:8080/11851 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/a1407adf Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/a1407adf Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/a1407adf Branch: refs/heads/master Commit: a1407adf61cec9bdd2ebb5bb7a7ccbbf5d2edd41 Parents: a39c847 Author: Thomas Tauber-Marshall <[email protected]> Authored: Mon Oct 29 17:04:17 2018 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Wed Nov 28 02:48:09 2018 +0000 ---------------------------------------------------------------------- bin/impala-config.sh | 1 + tests/common/kudu_test_suite.py | 28 ++++++++++++++++++++++++++-- tests/common/skip.py | 4 ++++ tests/custom_cluster/test_kudu.py | 2 ++ tests/metadata/test_ddl.py | 3 ++- tests/query_test/test_kudu.py | 19 +++++++++++++++++-- 6 files changed, 52 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/a1407adf/bin/impala-config.sh ---------------------------------------------------------------------- diff --git a/bin/impala-config.sh b/bin/impala-config.sh index 32df25d..ff13919 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -438,6 +438,7 @@ export NUM_CONCURRENT_TESTS="${NUM_CONCURRENT_TESTS-${CORES}}" export KUDU_MASTER_HOSTS="${KUDU_MASTER_HOSTS:-127.0.0.1}" export KUDU_MASTER_PORT="${KUDU_MASTER_PORT:-7051}" +export KUDU_MASTER_WEBUI_PORT="${KUDU_MASTER_WEBUI_PORT:-8051}" export IMPALA_FE_DIR="$IMPALA_HOME/fe" export IMPALA_BE_DIR="$IMPALA_HOME/be" http://git-wip-us.apache.org/repos/asf/impala/blob/a1407adf/tests/common/kudu_test_suite.py ---------------------------------------------------------------------- diff --git a/tests/common/kudu_test_suite.py b/tests/common/kudu_test_suite.py index 876752b..5826c74 100644 --- a/tests/common/kudu_test_suite.py +++ b/tests/common/kudu_test_suite.py @@ -16,8 +16,9 @@ # under the License. import os -import string import pytest +import requests +import string from contextlib import contextmanager from kudu.schema import ( BOOL, @@ -36,9 +37,32 @@ from random import choice, sample from string import ascii_lowercase, digits from tests.common.impala_test_suite import ImpalaTestSuite -from tests.common.skip import SkipIf from tests.common.test_dimensions import create_uncompressed_text_dimension +DEFAULT_KUDU_MASTER_WEBUI_PORT = os.getenv('KUDU_MASTER_WEBUI_PORT', '8051') + + +def get_kudu_master_webpage(page_name): + kudu_master = pytest.config.option.kudu_master_hosts + + if "," in kudu_master: + raise NotImplementedError("Multi-master not supported yet") + if ":" in kudu_master: + kudu_master_host = kudu_master.split(":")[0] + else: + kudu_master_host = kudu_master + url = "http://%s:%s/%s"; % (kudu_master_host, DEFAULT_KUDU_MASTER_WEBUI_PORT, page_name) + return requests.get(url).text + + +def get_kudu_master_flag(flag): + varz = get_kudu_master_webpage("varz") + for line in varz.split("\n"): + split = line.split("=") + if len(split) == 2 and split[0] == flag: + return split[1] + assert False, "Failed to find Kudu master flag: %s" % flag + class KuduTestSuite(ImpalaTestSuite): # Lazily set. http://git-wip-us.apache.org/repos/asf/impala/blob/a1407adf/tests/common/skip.py ---------------------------------------------------------------------- diff --git a/tests/common/skip.py b/tests/common/skip.py index 41e119a..8ba7a06 100644 --- a/tests/common/skip.py +++ b/tests/common/skip.py @@ -25,6 +25,7 @@ import pytest from functools import partial from tests.common.environ import IMPALA_TEST_CLUSTER_PROPERTIES +from tests.common.kudu_test_suite import get_kudu_master_flag from tests.util.filesystem_utils import ( IS_ABFS, IS_ADLS, @@ -100,6 +101,9 @@ class SkipIfADLS: class SkipIfKudu: unsupported_env = pytest.mark.skipif(os.environ["KUDU_IS_SUPPORTED"] == "false", reason="Kudu is not supported in this environment") + no_hybrid_clock = pytest.mark.skipif( + get_kudu_master_flag("--use_hybrid_clock") == "false", + reason="Test relies on --use_hybrid_clock=true in Kudu.") class SkipIf: skip_hbase = pytest.mark.skipif(pytest.config.option.skip_hbase, http://git-wip-us.apache.org/repos/asf/impala/blob/a1407adf/tests/custom_cluster/test_kudu.py ---------------------------------------------------------------------- diff --git a/tests/custom_cluster/test_kudu.py b/tests/custom_cluster/test_kudu.py index d79703d..ce86f3c 100644 --- a/tests/custom_cluster/test_kudu.py +++ b/tests/custom_cluster/test_kudu.py @@ -21,6 +21,7 @@ from kudu.schema import INT32 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.common.kudu_test_suite import KuduTestSuite +from tests.common.skip import SkipIfKudu from tests.common.test_dimensions import add_exec_option_dimension KUDU_MASTER_HOSTS = pytest.config.option.kudu_master_hosts @@ -42,6 +43,7 @@ class TestKuduOperations(CustomClusterTestSuite, KuduTestSuite): @pytest.mark.execute_serially @CustomClusterTestSuite.with_args(impalad_args=\ "--use_local_tz_for_unix_timestamp_conversions=true") + @SkipIfKudu.no_hybrid_clock def test_local_tz_conversion_ops(self, vector, unique_database): """IMPALA-5539: Test Kudu timestamp reads/writes are correct with the use_local_tz_for_unix_timestamp_conversions flag.""" http://git-wip-us.apache.org/repos/asf/impala/blob/a1407adf/tests/metadata/test_ddl.py ---------------------------------------------------------------------- diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py index b510af1..9d35f32 100644 --- a/tests/metadata/test_ddl.py +++ b/tests/metadata/test_ddl.py @@ -25,7 +25,7 @@ from test_ddl_base import TestDdlBase from tests.beeswax.impala_beeswax import ImpalaBeeswaxException from tests.common.impala_test_suite import LOG from tests.common.parametrize import UniqueDatabase -from tests.common.skip import SkipIf, SkipIfABFS, SkipIfADLS, SkipIfLocal +from tests.common.skip import SkipIf, SkipIfABFS, SkipIfADLS, SkipIfKudu, SkipIfLocal from tests.common.test_dimensions import create_single_exec_option_dimension from tests.util.filesystem_utils import WAREHOUSE, IS_HDFS, IS_S3, IS_ADLS from tests.common.impala_cluster import ImpalaCluster @@ -282,6 +282,7 @@ class TestDdlStatements(TestDdlBase): @SkipIf.kudu_not_supported @UniqueDatabase.parametrize(sync_ddl=True) + @SkipIfKudu.no_hybrid_clock def test_create_kudu(self, vector, unique_database): vector.get_value('exec_option')['abort_on_error'] = False vector.get_value('exec_option')['kudu_read_mode'] = "READ_AT_SNAPSHOT" http://git-wip-us.apache.org/repos/asf/impala/blob/a1407adf/tests/query_test/test_kudu.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_kudu.py b/tests/query_test/test_kudu.py index 5936649..f5edac4 100644 --- a/tests/query_test/test_kudu.py +++ b/tests/query_test/test_kudu.py @@ -39,7 +39,7 @@ from pytz import utc from tests.common.kudu_test_suite import KuduTestSuite from tests.common.impala_cluster import ImpalaCluster -from tests.common.skip import SkipIfNotHdfsMinicluster +from tests.common.skip import SkipIfNotHdfsMinicluster, SkipIfKudu from tests.common.test_dimensions import add_exec_option_dimension from tests.verifiers.metric_verifier import MetricVerifier @@ -59,6 +59,7 @@ class TestKuduOperations(KuduTestSuite): # these tests. add_exec_option_dimension(cls, "kudu_read_mode", "READ_AT_SNAPSHOT") + @SkipIfKudu.no_hybrid_clock def test_out_of_range_timestamps(self, vector, cursor, kudu_client, unique_database): """Test timestamp values that are outside of Impala's supported date range.""" cursor.execute("""CREATE TABLE %s.times (a INT PRIMARY KEY, ts TIMESTAMP) @@ -86,40 +87,51 @@ class TestKuduOperations(KuduTestSuite): self.run_test_case('QueryTest/kudu-overflow-ts-abort-on-error', vector, use_db=unique_database) + @SkipIfKudu.no_hybrid_clock def test_kudu_scan_node(self, vector, unique_database): self.run_test_case('QueryTest/kudu-scan-node', vector, use_db=unique_database) + @SkipIfKudu.no_hybrid_clock def test_kudu_insert(self, vector, unique_database): self.run_test_case('QueryTest/kudu_insert', vector, use_db=unique_database) @SkipIfNotHdfsMinicluster.tuned_for_minicluster + @SkipIfKudu.no_hybrid_clock def test_kudu_insert_mem_limit(self, vector, unique_database): self.run_test_case('QueryTest/kudu_insert_mem_limit', vector, use_db=unique_database) + @SkipIfKudu.no_hybrid_clock def test_kudu_update(self, vector, unique_database): self.run_test_case('QueryTest/kudu_update', vector, use_db=unique_database) + @SkipIfKudu.no_hybrid_clock def test_kudu_upsert(self, vector, unique_database): self.run_test_case('QueryTest/kudu_upsert', vector, use_db=unique_database) + @SkipIfKudu.no_hybrid_clock def test_kudu_delete(self, vector, unique_database): self.run_test_case('QueryTest/kudu_delete', vector, use_db=unique_database) + @SkipIfKudu.no_hybrid_clock def test_kudu_partition_ddl(self, vector, unique_database): self.run_test_case('QueryTest/kudu_partition_ddl', vector, use_db=unique_database) @pytest.mark.skipif(pytest.config.option.testing_remote_cluster, reason="Test references hardcoded hostnames: IMPALA-4873") @pytest.mark.execute_serially + @SkipIfKudu.no_hybrid_clock def test_kudu_alter_table(self, vector, unique_database): self.run_test_case('QueryTest/kudu_alter', vector, use_db=unique_database) + @SkipIfKudu.no_hybrid_clock def test_kudu_stats(self, vector, unique_database): self.run_test_case('QueryTest/kudu_stats', vector, use_db=unique_database) + @SkipIfKudu.no_hybrid_clock def test_kudu_describe(self, vector, unique_database): self.run_test_case('QueryTest/kudu_describe', vector, use_db=unique_database) + @SkipIfKudu.no_hybrid_clock def test_kudu_limit(self, vector, unique_database): self.run_test_case('QueryTest/kudu_limit', vector, use_db=unique_database) @@ -305,6 +317,7 @@ class TestKuduOperations(KuduTestSuite): cursor.execute("SELECT * FROM %s.foo" % (unique_database)) assert cursor.fetchall() == [(0, 0)] + @SkipIfKudu.no_hybrid_clock def test_kudu_col_removed(self, cursor, kudu_client, unique_database): """Test removing a Kudu column outside of Impala.""" cursor.execute("set kudu_read_mode=READ_AT_SNAPSHOT") @@ -365,7 +378,7 @@ class TestKuduOperations(KuduTestSuite): if kudu_client.table_exists(name): kudu_client.delete_table(name) - + @SkipIfKudu.no_hybrid_clock def test_column_storage_attributes(self, cursor, unique_database): """Tests that for every valid combination of column type, encoding, and compression, we can insert a value and scan it back from Kudu.""" @@ -642,6 +655,7 @@ class TestCreateExternalTable(KuduTestSuite): except Exception as e: assert "Table does not exist in Kudu: '%s'" % table_name in str(e) + @SkipIfKudu.no_hybrid_clock def test_table_without_partitioning(self, cursor, kudu_client, unique_database): """Test a Kudu table created without partitioning (i.e. equivalent to a single unbounded partition). It is not possible to create such a table in Impala, but @@ -676,6 +690,7 @@ class TestCreateExternalTable(KuduTestSuite): if kudu_client.table_exists(name): kudu_client.delete_table(name) + @SkipIfKudu.no_hybrid_clock def test_column_name_case(self, cursor, kudu_client, unique_database): """IMPALA-5286: Tests that an external Kudu table that was created with a column name containing upper case letters is handled correctly."""
