IMPALA-4122: qgen: fix bitrotted cluster unit tests There's a small set of pytest-style tests and associated conftest for testing some of the cluster-related test infrastructure Python objects. Going forward, I want unit tests for the query generator to be run as part of patch acceptance (CI isn't necessary at this time).
This patch fixes a bitrotted test and moves the tests into the tests-for-qgen directory. The moves were performed thusly: $ git mv conftest.py tests/ $ git mv cluster_tests.py tests/test_cluster.py Change-Id: I3e855e265ae245ebe3691d077284ac5761909e00 Reviewed-on: http://gerrit.cloudera.org:8080/4404 Reviewed-by: Michael Brown <[email protected]> Reviewed-by: Tim Armstrong <[email protected]> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/0c874189 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/0c874189 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/0c874189 Branch: refs/heads/master Commit: 0c874189ed902d566b3df37959c937ff6a7bfee8 Parents: d379875 Author: Michael Brown <[email protected]> Authored: Tue Sep 13 12:30:26 2016 -0700 Committer: Internal Jenkins <[email protected]> Committed: Wed Sep 14 00:24:06 2016 +0000 ---------------------------------------------------------------------- tests/comparison/cluster_tests.py | 94 ----------------------------- tests/comparison/conftest.py | 46 -------------- tests/comparison/tests/conftest.py | 62 +++++++++++++++++++ tests/comparison/tests/test_cluster.py | 94 +++++++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 140 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0c874189/tests/comparison/cluster_tests.py ---------------------------------------------------------------------- diff --git a/tests/comparison/cluster_tests.py b/tests/comparison/cluster_tests.py deleted file mode 100644 index 976f423..0000000 --- a/tests/comparison/cluster_tests.py +++ /dev/null @@ -1,94 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# These are unit tests for cluster.py. - -from time import time - -from tests.common.errors import Timeout - -from common import Column, Table -from db_types import BigInt, String - -class TestCluster(object): - - def test_cmd(self, cluster): - host = cluster.impala.impalads[0].host_name - assert cluster.shell("echo -n HI", host) == "HI" - - try: - cluster.shell("bad", host) - assert False - except Exception as e: - assert "command not found" in str(e) - - start = time() - try: - cluster.shell("echo HI; sleep 60", host, timeout_secs=3) - assert False - except Timeout as e: - assert "HI" in str(e) - assert 3 <= time() - start <= 6 - - -class TestHdfs(object): - - def test_ls(self, cluster): - ls = cluster.hdfs.create_client().list("/") - assert "tmp" in ls - assert "etc" not in ls - - -class TestHive(object): - - def test_list_databases(self, hive_cursor): - assert "default" in hive_cursor.list_db_names() - - def test_non_mr_exec(self, hive_cursor): - hive_cursor.execute("SELECT 1") - rows = hive_cursor.fetchall() - assert rows - assert rows[0][0] == 1 - - -class TestImpala(object): - - def test_list_databases(self, cursor): - assert "default" in cursor.list_db_names() - - def test_exec(self, cursor): - cursor.execute("SELECT 1") - rows = cursor.fetchall() - assert rows - assert rows[0][0] == 1 - - -class TestModel(object): - - def test_table_model(self, cursor, hive_cursor): - table = Table("some_test_table") - cursor.drop_table(table.name, if_exists=True) - table.storage_format = 'textfile' - table.cols.append(Column(table, "bigint_col", BigInt)) - table.cols.append(Column(table, "string_col", String)) - cursor.create_table(table) - try: - other = hive_cursor.describe_table(table.name) - assert other.name == table.name - assert other.cols == table.cols - finally: - cursor.drop_table(table.name) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0c874189/tests/comparison/conftest.py ---------------------------------------------------------------------- diff --git a/tests/comparison/conftest.py b/tests/comparison/conftest.py deleted file mode 100644 index 8126a84..0000000 --- a/tests/comparison/conftest.py +++ /dev/null @@ -1,46 +0,0 @@ -import pytest - -import cli_options -from cluster import CmCluster, MiniCluster - -"""This module provides pytest 'fixtures'. See cluster_tests.py for usage.""" - -__cluster = None - -def pytest_addoption(parser): - if not hasattr(parser, "add_argument"): - parser.add_argument = parser.addoption - cli_options.add_cm_options(parser) - - [email protected] -def cluster(request): - global __cluster - if not __cluster: - cm_host = get_option_value(request, "cm_host") - if cm_host: - __cluster = CmCluster(cm_host, port=get_option_value(request, "cm_port"), - user=get_option_value(request, "cm_user"), - password=get_option_value(request, "cm_password"), - cluster_name=get_option_value(request, "cm_cluster_name")) - else: - __cluster = MiniCluster() - return __cluster - - [email protected]_fixture -def hive_cursor(request): - with cluster(request).hive.connect() as conn: - with conn.cursor() as cur: - yield cur - - [email protected]_fixture -def cursor(request): - with cluster(request).impala.connect() as conn: - with conn.cursor() as cur: - yield cur - - -def get_option_value(request, dest_var_name): - return request.config.getoption("--" + dest_var_name.replace("_", "-")) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0c874189/tests/comparison/tests/conftest.py ---------------------------------------------------------------------- diff --git a/tests/comparison/tests/conftest.py b/tests/comparison/tests/conftest.py new file mode 100644 index 0000000..dd39fde --- /dev/null +++ b/tests/comparison/tests/conftest.py @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest + +from tests.comparison import cli_options +from tests.comparison.cluster import CmCluster, MiniCluster + + +__cluster = None + +def pytest_addoption(parser): + if not hasattr(parser, "add_argument"): + parser.add_argument = parser.addoption + cli_options.add_cm_options(parser) + + [email protected] +def cluster(request): + global __cluster + if not __cluster: + cm_host = get_option_value(request, "cm_host") + if cm_host: + __cluster = CmCluster(cm_host, port=get_option_value(request, "cm_port"), + user=get_option_value(request, "cm_user"), + password=get_option_value(request, "cm_password"), + cluster_name=get_option_value(request, "cm_cluster_name")) + else: + __cluster = MiniCluster() + return __cluster + + [email protected]_fixture +def hive_cursor(request): + with cluster(request).hive.connect() as conn: + with conn.cursor() as cur: + yield cur + + [email protected]_fixture +def cursor(request): + with cluster(request).impala.connect() as conn: + with conn.cursor() as cur: + yield cur + + +def get_option_value(request, dest_var_name): + return request.config.getoption("--" + dest_var_name.replace("_", "-")) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0c874189/tests/comparison/tests/test_cluster.py ---------------------------------------------------------------------- diff --git a/tests/comparison/tests/test_cluster.py b/tests/comparison/tests/test_cluster.py new file mode 100644 index 0000000..2e3bc54 --- /dev/null +++ b/tests/comparison/tests/test_cluster.py @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# These are unit tests for cluster.py. + +from time import time + +from tests.common.errors import Timeout + +from tests.comparison.common import Column, Table +from tests.comparison.db_types import BigInt, String + +class TestCluster(object): + + def test_cmd(self, cluster): + host = cluster.impala.impalads[0].host_name + assert cluster.shell("echo -n HI", host) == "HI" + + try: + cluster.shell("bad", host) + assert False + except Exception as e: + assert "command not found" in str(e) + + start = time() + try: + cluster.shell("echo HI; sleep 60", host, timeout_secs=3) + assert False + except Timeout as e: + assert "HI" in str(e) + assert 3 <= time() - start <= 6 + + +class TestHdfs(object): + + def test_ls(self, cluster): + ls = cluster.hdfs.create_client().list("/") + assert "tmp" in ls + assert "etc" not in ls + + +class TestHive(object): + + def test_list_databases(self, hive_cursor): + assert "default" in hive_cursor.list_db_names() + + def test_non_mr_exec(self, hive_cursor): + hive_cursor.execute("SELECT 1") + rows = hive_cursor.fetchall() + assert rows + assert rows[0][0] == 1 + + +class TestImpala(object): + + def test_list_databases(self, cursor): + assert "default" in cursor.list_db_names() + + def test_exec(self, cursor): + cursor.execute("SELECT 1") + rows = cursor.fetchall() + assert rows + assert rows[0][0] == 1 + + +class TestModel(object): + + def test_table_model(self, cursor, hive_cursor): + table = Table("some_test_table") + cursor.drop_table(table.name, if_exists=True) + table.storage_format = 'textfile' + table.add_col(Column(table, "bigint_col", BigInt)) + table.add_col(Column(table, "string_col", String)) + cursor.create_table(table) + try: + other = hive_cursor.describe_table(table.name) + assert other.name == table.name + assert other.cols == table.cols + finally: + cursor.drop_table(table.name)
