Repository: incubator-impala Updated Branches: refs/heads/master 36b524f68 -> 08eff2bc0
Revert "Add Kudu test helpers" This reverts commit 9248dcb70478b8f93f022893776a0960f45fdc28. Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/08eff2bc Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/08eff2bc Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/08eff2bc Branch: refs/heads/master Commit: 08eff2bc0984b09508b49f8b3520af4204347927 Parents: 36b524f Author: Shiraz Ali <[email protected]> Authored: Fri May 20 00:13:12 2016 -0700 Committer: Tim Armstrong <[email protected]> Committed: Fri May 20 08:46:00 2016 -0700 ---------------------------------------------------------------------- bin/impala-ipython | 9 +- bin/impala-py.test | 17 ++- bin/impala-python | 4 +- bin/impala-python-common.sh | 25 ----- infra/python/bootstrap_virtualenv.py | 152 +++------------------------ infra/python/deps/requirements.txt | 15 --- tests/common/__init__.py | 3 +- tests/conftest.py | 167 +----------------------------- 8 files changed, 44 insertions(+), 348 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/08eff2bc/bin/impala-ipython ---------------------------------------------------------------------- diff --git a/bin/impala-ipython b/bin/impala-ipython index e7f9806..8833384 100755 --- a/bin/impala-ipython +++ b/bin/impala-ipython @@ -1,3 +1,10 @@ #!/bin/bash -source $(dirname "$0")/impala-python-common.sh + +set -eu -o pipefail + +PY_DIR=$(dirname "$0")/../infra/python + +# impala-python will build or upgrade the python virtualenv automatically. +impala-python -c "" + exec "$PY_DIR/env/bin/ipython" "$@" http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/08eff2bc/bin/impala-py.test ---------------------------------------------------------------------- diff --git a/bin/impala-py.test b/bin/impala-py.test index 48d01e9..55756a0 100755 --- a/bin/impala-py.test +++ b/bin/impala-py.test @@ -1,3 +1,14 @@ -#!/bin/bash -source $(dirname "$0")/impala-python-common.sh -exec "$PY_DIR/env/bin/py.test" "$@" +#!/usr/bin/env impala-python +# EASY-INSTALL-ENTRY-SCRIPT: 'pytest==2.7.2','console_scripts','py.test' + +# This was copied straight from py.test, the only modification is to use impala-python +# instead of python above + +__requires__ = 'pytest==2.7.2' +import sys +from pkg_resources import load_entry_point + +if __name__ == '__main__': + sys.exit( + load_entry_point('pytest==2.7.2', 'console_scripts', 'py.test')() + ) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/08eff2bc/bin/impala-python ---------------------------------------------------------------------- diff --git a/bin/impala-python b/bin/impala-python index 5ff4190..67c7be3 100755 --- a/bin/impala-python +++ b/bin/impala-python @@ -1,3 +1,5 @@ #!/bin/bash -source $(dirname "$0")/impala-python-common.sh +set -eu -o pipefail +PY_DIR=$(dirname "$0")/../infra/python +python "$PY_DIR/bootstrap_virtualenv.py" exec "$PY_DIR/env/bin/python" "$@" http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/08eff2bc/bin/impala-python-common.sh ---------------------------------------------------------------------- diff --git a/bin/impala-python-common.sh b/bin/impala-python-common.sh deleted file mode 100644 index 298a0ca..0000000 --- a/bin/impala-python-common.sh +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2016 Cloudera Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file is intended to be sourced to perform common setup for -# $IMPALA_HOME/bin/impala-py* executables. - -set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR - -LD_LIBRARY_PATH+=":$(python $IMPALA_HOME/infra/python/bootstrap_virtualenv.py \ - --print-ld-library-path)" - -PY_DIR=$(dirname "$0")/../infra/python -python "$PY_DIR/bootstrap_virtualenv.py" http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/08eff2bc/infra/python/bootstrap_virtualenv.py ---------------------------------------------------------------------- diff --git a/infra/python/bootstrap_virtualenv.py b/infra/python/bootstrap_virtualenv.py index 6f20064..11065db 100644 --- a/infra/python/bootstrap_virtualenv.py +++ b/infra/python/bootstrap_virtualenv.py @@ -62,28 +62,17 @@ def create_virtualenv(): shutil.rmtree(build_dir) -def exec_cmd(args, **kwargs): +def exec_cmd(args): '''Executes a command and waits for it to finish, raises an exception if the return - status is not zero. The command output is returned. + status is not zero. - 'args' and 'kwargs' use the same format as subprocess.Popen(). + 'args' uses the same format as subprocess.Popen(). ''' - process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - **kwargs) + process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = process.communicate()[0] if process.returncode != 0: raise Exception("Command returned non-zero status\nCommand: %s\nOutput: %s" % (args, output)) - return output - - -def exec_pip_install(args, **popen_kwargs): - # Don't call the virtualenv pip directly, it uses a hashbang to to call the python - # virtualenv using an absolute path. If the path to the virtualenv is very long, the - # hashbang won't work. - exec_cmd([os.path.join(ENV_DIR, "bin", "python"), os.path.join(ENV_DIR, "bin", "pip"), - "install", "--no-index", "--find-links", - "file://%s" % urllib.pathname2url(os.path.abspath(DEPS_DIR))] + args, **popen_kwargs) def find_file(*paths): @@ -119,118 +108,19 @@ def detect_python_cmd(): def install_deps(): - LOG.info("Installing packages into the virtualenv") - exec_pip_install(["-r", REQS_PATH]) + LOG.info("Installing packages into virtualenv") + # Don't call the virtualenv pip directly, it uses a hashbang to to call the python + # virtualenv using an absolute path. If the path to the virtualenv is very long, the + # hashbang won't work. + # --no-cache-dir is used because the dev version of Impyla may be the same even though + # the contents are different. Since the version doesn't change, pip may use its cached + # build. + exec_cmd([os.path.join(ENV_DIR, "bin", "python"), os.path.join(ENV_DIR, "bin", "pip"), + "install", "--no-cache-dir", "--no-index", "--find-links", + "file://%s" % urllib.pathname2url(os.path.abspath(DEPS_DIR)), "-r", REQS_PATH]) shutil.copyfile(REQS_PATH, INSTALLED_REQS_PATH) -def install_kudu_client_if_possible(): - """Installs the Kudu python module if possible. The Kudu module is the only one that - requires the toolchain. If the toolchain isn't in use or hasn't been populated - yet, nothing will be done. Also nothing will be done if the Kudu client lib required - by the module isn't available (as determined by KUDU_IS_SUPPORTED). - """ - if os.environ["KUDU_IS_SUPPORTED"] != "true": - LOG.debug("Skipping Kudu: Kudu is not supported") - return - impala_toolchain_dir = os.environ.get("IMPALA_TOOLCHAIN") - if not impala_toolchain_dir: - LOG.debug("Skipping Kudu: IMPALA_TOOLCHAIN not set") - return - toolchain_kudu_dir = os.path.join( - impala_toolchain_dir, "kudu-" + os.environ["IMPALA_KUDU_VERSION"]) - if not os.path.exists(toolchain_kudu_dir): - LOG.debug("Skipping Kudu: %s doesn't exist" % toolchain_kudu_dir) - return - - # The "pip" command could be used to provide the version of Kudu installed (if any) - # but it's a little too slow. Running the virtualenv python to detect the installed - # version is faster. - actual_version_string = exec_cmd([os.path.join(ENV_DIR, "bin", "python"), "-c", - textwrap.dedent(""" - try: - import kudu - print kudu.__version__ - except ImportError: - pass""")]).strip() - actual_version = [int(v) for v in actual_version_string.split(".") if v] - - reqs_file = open(REQS_PATH) - try: - for line in reqs_file: - if not line.startswith("# kudu-python=="): - continue - expected_version_string = line.split()[1].split("==")[1] - break - else: - raise Exception("Unable to find kudu-python version in requirements file") - finally: - reqs_file.close() - expected_version = [int(v) for v in expected_version_string.split(".")] - - if actual_version and actual_version == expected_version: - LOG.debug("Skipping Kudu: Installed %s == required %s" - % (actual_version_string, expected_version_string)) - return - LOG.debug("Kudu installation required. Actual version %s. Required version %s.", - actual_version, expected_version) - - LOG.info("Installing Kudu into the virtualenv") - # The installation requires that KUDU_HOME/build/latest exists. An empty directory - # structure will be made to satisfy that. The Kudu client headers and lib will be made - # available through GCC environment variables. - fake_kudu_build_dir = os.path.join(tempfile.gettempdir(), "virtualenv-kudu") - try: - artifact_dir = os.path.join(fake_kudu_build_dir, "build", "latest") - if not os.path.exists(artifact_dir): - os.makedirs(artifact_dir) - env = dict(os.environ) - env["KUDU_HOME"] = fake_kudu_build_dir - kudu_client_dir = find_kudu_client_install_dir() - env["CPLUS_INCLUDE_PATH"] = os.path.join(kudu_client_dir, "include") - env["LIBRARY_PATH"] = "{0}{1}lib{2}{0}{1}lib64".format( - kudu_client_dir, os.path.sep, os.path.pathsep) - exec_pip_install(["kudu-python==" + expected_version_string], env=env) - finally: - try: - shutil.rmtree(fake_kudu_build_dir) - except Exception: - LOG.debug("Error removing temp Kudu build dir", exc_info=True) - - -def find_kudu_client_install_dir(): - custom_client_dir = os.environ["KUDU_CLIENT_DIR"] - if custom_client_dir: - install_dir = os.path.join(custom_client_dir, "usr", "local") - error_if_kudu_client_not_found(install_dir) - else: - # If the toolchain appears to have been setup already, then the Kudu client is - # required to exist. It's possible that the toolchain won't be setup yet though - # since the toolchain bootstrap script depends on the virtualenv. - kudu_base_dir = os.path.join(os.environ["IMPALA_TOOLCHAIN"], - "kudu-%s" % os.environ["IMPALA_KUDU_VERSION"]) - install_dir = os.path.join(kudu_base_dir, "debug") - if os.path.exists(kudu_base_dir): - error_if_kudu_client_not_found(install_dir) - return install_dir - - -def error_if_kudu_client_not_found(install_dir): - header_path = os.path.join(install_dir, "include", "kudu", "client", "client.h") - if not os.path.exists(header_path): - raise Exception("Kudu client header not found at %s" % header_path) - - kudu_client_lib = "libkudu_client.so" - lib_dir = os.path.join(install_dir, "lib64") - if not os.path.exists(lib_dir): - lib_dir = os.path.join(install_dir, "lib") - for _, _, files in os.walk(lib_dir): - for file in files: - if file == kudu_client_lib: - return - raise Exception("%s not found at %s" % (kudu_client_lib, lib_dir)) - - def deps_are_installed(): if not os.path.exists(INSTALLED_REQS_PATH): return False @@ -258,23 +148,11 @@ def setup_virtualenv_if_not_exists(): if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) parser = optparse.OptionParser() - parser.add_option("-l", "--log-level", default="INFO", - choices=("DEBUG", "INFO", "WARN", "ERROR")) parser.add_option("-r", "--rebuild", action="store_true", help="Force a rebuild of" " the virtualenv even if it exists and appears to be completely up-to-date.") - parser.add_option("--print-ld-library-path", action="store_true", help="Print the" - " LD_LIBRARY_PATH that should be used when running python from the virtualenv.") options, args = parser.parse_args() - - if options.print_ld_library_path: - kudu_client_dir = find_kudu_client_install_dir() - print("{0}{1}lib{2}{0}{1}lib64".format( - kudu_client_dir, os.path.sep, os.path.pathsep)) - exit() - - logging.basicConfig(level=getattr(logging, options.log_level)) if options.rebuild: delete_virtualenv_if_exist() setup_virtualenv_if_not_exists() - install_kudu_client_if_possible() http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/08eff2bc/infra/python/deps/requirements.txt ---------------------------------------------------------------------- diff --git a/infra/python/deps/requirements.txt b/infra/python/deps/requirements.txt index 5409027..2725344 100644 --- a/infra/python/deps/requirements.txt +++ b/infra/python/deps/requirements.txt @@ -53,20 +53,5 @@ requests == 2.7.0 sh == 1.11 sqlparse == 0.1.15 texttable == 0.8.3 - -# kudu-python is needed but cannot be listed as usual. The Kudu client lib (.so file) -# is needed for compilation/installation but the client lib is provided by the toolchain. -# The virtualenv may need to be functional even if the toolchain isn't present. The -# bootstap_virtualenv.py script special-cases kudu-python, the line below is actually -# functional and determines the expected kudu-python version. The version must be listed -# in the format below including # and spacing. Keep this formatting! -# kudu-python==0.1.1 - Cython == 0.23.4 - numpy == 1.10.4 - # These should eventually be removed https://issues.apache.org/jira/browse/KUDU-1456 - unittest2 == 1.1.0 - linecache2 == 1.0.0 - traceback2 == 1.4.0 - # For dev purposes, not used in scripting. Version 1.2.1 is the latest that supports 2.6. ipython == 1.2.1 http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/08eff2bc/tests/common/__init__.py ---------------------------------------------------------------------- diff --git a/tests/common/__init__.py b/tests/common/__init__.py index 010f973..946a474 100644 --- a/tests/common/__init__.py +++ b/tests/common/__init__.py @@ -1,2 +1 @@ -KUDU_MASTER_HOST = "127.0.0.1" -KUDU_MASTER_PORT = 7051 +# This file is needed to make the files in this directory a python module http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/08eff2bc/tests/conftest.py ---------------------------------------------------------------------- diff --git a/tests/conftest.py b/tests/conftest.py index 3fb7f04..c6a22c0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,21 +1,17 @@ # Copyright (c) 2012 Cloudera, Inc. All rights reserved. # py.test configuration module # -from impala.dbapi import connect as impala_connect -from kudu import connect as kudu_connect -from random import choice, sample -from string import ascii_lowercase, digits -from zlib import crc32 -import contextlib import logging import os import pytest -from common import KUDU_MASTER_HOST, KUDU_MASTER_PORT +from zlib import crc32 + from common.test_result_verifier import QueryTestResult from tests.common.patterns import is_valid_impala_identifier from tests.util.filesystem_utils import FILESYSTEM, ISILON_WEBHDFS_PORT + logging.basicConfig(level=logging.INFO, format='%(threadName)s: %(message)s') LOG = logging.getLogger('test_configuration') @@ -207,9 +203,6 @@ def unique_database(request, testid_checksum): name_prefix: string (defaults to test function __name__) - prefix to be used for the database name - - For a similar DB-API 2 compliant connection/cursor that use HS2 see the 'conn' and - 'unique_cursor' fixtures below. """ # Test cases are at the function level, so no one should "accidentally" re-scope this. @@ -242,157 +235,3 @@ def unique_database(request, testid_checksum): LOG.info('Created database "{0}" for test ID "{1}"'.format(db_name, str(request.node.nodeid))) return db_name - - [email protected]_fixture -def kudu_client(): - """Provides a new Kudu client as a pytest fixture. The client only exists for the - duration of the method it is used in. - """ - kudu_client = kudu_connect(KUDU_MASTER_HOST, KUDU_MASTER_PORT) - try: - yield kudu_client - finally: - try: - kudu_client.close() - except Exception as e: - LOG.warn("Error closing Kudu client: %s", e) - - [email protected]_fixture(scope="class") -def conn(request): - """Provides a new DB-API compliant connection to Impala as a pytest fixture. The - same connection is used for all test methods in a class. The class may provide the - following customizations: - - get_db_name(): The name of the database to connect to. - - auto_create_db(): If declared and the method returns True, the database will - be created before tests run and dropped afterwards. If a database name is - provided by get_db_name(), it must not exist. Classes that use both - auto_create_db() and get_db_name() should generate a random name in - get_db_name() and cache it. - The returned connection will have a 'db_name' property. - - See the 'unique_database' fixture above if you want to use Impala's custom python - API instead of DB-API. - """ - db_name = __call_cls_method_if_exists(request.cls, "get_db_name") - use_unique_conn = __call_cls_method_if_exists(request.cls, "auto_create_db") - if use_unique_conn: - with __unique_conn(db_name=db_name) as conn: - yield conn - else: - with __auto_closed_conn(db_name=db_name) as conn: - yield conn - - -def __call_cls_method_if_exists(cls, method_name): - """Returns the result of calling the method 'method_name' on class 'class' if the class - defined such a method, otherwise returns None. - """ - method = getattr(cls, method_name, None) - if method: - return method() - - [email protected] -def __unique_conn(db_name=None): - """Connects to Impala and creates a new database, then returns a connection to it. - This is intended to be used in a "with" block. Upon exit, the database will be - dropped. A database name can be provided by 'db_name', a database by that name - must not exist prior to calling this method. - - with __unique_conn() as conn: - # Use conn - # The database no longer exists and the conn is closed. - - The returned connection will have a 'db_name' property. - """ - if not db_name: - db_name = choice(ascii_lowercase) + "".join(sample(ascii_lowercase + digits, 5)) - with __auto_closed_conn() as conn: - with __auto_closed_cursor(conn) as cur: - cur.execute("CREATE DATABASE %s" % db_name) - with __auto_closed_conn(db_name=db_name) as conn: - try: - yield conn - finally: - try: - with __auto_closed_cursor(conn) as cur: - try: - cur.execute("USE DEFAULT") - cur.execute("DROP DATABASE IF EXISTS %s CASCADE" % db_name) - except Exception as e: - LOG.warn("Error dropping database: %s", e) - except Exception as e: - LOG.warn("Error creating a cursor: %s", e) - - [email protected] -def __auto_closed_conn(db_name=None): - """Returns a connection to Impala. This is intended to be used in a "with" block. The - connection will be closed upon exiting the block. - - The returned connection will have a 'db_name' property. - """ - conn = impala_connect(database=db_name) - try: - conn.db_name = db_name - yield conn - finally: - try: - conn.close() - except Exception as e: - LOG.warn("Error closing Impala connection: %s", e) - - [email protected]_fixture -def cursor(conn): - """Provides a new DB-API compliant cursor from a connection provided by the conn() - fixture. The cursor only exists for the duration of the method it is used in. - - The returned cursor will have a 'conn' property. The 'conn' will have a 'db_name' - property. - """ - with __auto_closed_cursor(conn) as cur: - yield cur - [email protected]_fixture(scope="class") -def cls_cursor(conn): - """Provides a new DB-API compliant cursor from a connection provided by the conn() - fixture. The cursor exists for the duration of the class it is used in. - - The returned cursor will have a 'conn' property. The 'conn' will have a 'db_name' - property. - """ - with __auto_closed_cursor(conn) as cur: - yield cur - - [email protected]_fixture -def unique_cursor(): - """Provides a new DB-API compliant cursor to a newly created Impala database. The - cursor only exists for the duration of the method it is used in. The database will - be dropped after the test executes. - - The returned cursor will have a 'conn' property. The 'conn' will have a 'db_name' - property. - """ - with __unique_conn() as conn: - with __auto_closed_cursor(conn) as cur: - yield cur - - [email protected] -def __auto_closed_cursor(conn): - """Returns a cursor created from conn. This is intended to be used in a "with" block. - The cursor will be closed upon exiting the block. - """ - cursor = conn.cursor() - cursor.conn = conn - try: - yield cursor - finally: - try: - cursor.close() - except Exception as e: - LOG.warn("Error closing Impala cursor: %s", e)
