This is an automated email from the ASF dual-hosted git repository. joemcdonnell pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit c5a0ec8bdffd958984c0f6d34e0fcb8acc138457 Author: Joe McDonnell <[email protected]> AuthorDate: Sat Jul 1 11:02:55 2023 -0700 IMPALA-11980 (part 1): Put all thrift-generated python code into the impala_thrift_gen package This puts all of the thrift-generated python code into the impala_thrift_gen package. This is similar to what Impyla does for its thrift-generated python code, except that it uses the impala_thrift_gen package rather than impala._thrift_gen. This is a preparatory patch for fixing the absolute import issues. This patches all of the thrift files to add the python namespace. This has code to apply the patching to the thirdparty thrift files (hive_metastore.thrift, fb303.thrift) to do the same. Putting all the generated python into a package makes it easier to understand where the imports are getting code. When the subsequent change rearranges the shell code, the thrift generated code can stay in a separate directory. This uses isort to sort the imports for the affected Python files with the provided .isort.cfg file. This also adds an impala-isort shell script to make it easy to run. Testing: - Ran a core job Change-Id: Ie2927f22c7257aa38a78084efe5bd76d566493c0 Reviewed-on: http://gerrit.cloudera.org:8080/20169 Tested-by: Impala Public Jenkins <[email protected]> Reviewed-by: Riza Suminto <[email protected]> --- .../python/deps/py3-requirements.txt => .isort.cfg | 31 +++++------- bin/cmake_aux/add_thrift_python_namespace.sh | 55 ++++++++++++++++++++ .../deps/py3-requirements.txt => bin/impala-isort | 23 ++------- common/thrift/.gitignore | 1 + common/thrift/BackendGflags.thrift | 1 + common/thrift/CMakeLists.txt | 59 +++++++++++++++++++--- common/thrift/CatalogInternalService.thrift | 1 + common/thrift/CatalogObjects.thrift | 1 + common/thrift/CatalogService.thrift | 1 + common/thrift/Data.thrift | 1 + common/thrift/DataSinks.thrift | 1 + common/thrift/Descriptors.thrift | 1 + common/thrift/ExecStats.thrift | 1 + common/thrift/Exprs.thrift | 1 + common/thrift/ExternalDataSource.thrift | 1 + common/thrift/Frontend.thrift | 1 + common/thrift/ImpalaInternalService.thrift | 1 + common/thrift/ImpalaService.thrift | 1 + common/thrift/JniCatalog.thrift | 1 + common/thrift/LineageGraph.thrift | 1 + common/thrift/Logging.thrift | 1 + common/thrift/Metrics.thrift | 1 + common/thrift/NetworkTest.thrift | 1 + common/thrift/Partitions.thrift | 1 + common/thrift/PlanNodes.thrift | 1 + common/thrift/Planner.thrift | 1 + common/thrift/Query.thrift | 1 + common/thrift/ResourceProfile.thrift | 1 + common/thrift/Results.thrift | 1 + common/thrift/RuntimeProfile.thrift | 1 + common/thrift/SqlConstraints.thrift | 3 +- common/thrift/StatestoreService.thrift | 1 + common/thrift/Status.thrift | 1 + common/thrift/SystemTables.thrift | 1 + common/thrift/Types.thrift | 1 + common/thrift/Zip.thrift | 1 + common/thrift/beeswax.thrift | 2 +- common/thrift/generate_error_codes.py | 1 + common/thrift/generate_metrics.py | 1 + common/thrift/hive-1-api/TCLIService.thrift | 1 + common/thrift/parquet.thrift | 1 + infra/python/deps/py3-requirements.txt | 3 +- lib/python/impala_py_lib/profiles.py | 4 +- shell/exec_summary.py | 4 +- shell/impala_client.py | 24 +++++---- shell/value_converter.py | 2 +- testdata/bin/wait-for-hiveserver2.py | 2 +- testdata/bin/wait-for-metastore.py | 2 +- tests/authorization/test_authorization.py | 11 ++-- tests/authorization/test_authorized_proxy.py | 11 ++-- tests/beeswax/impala_beeswax.py | 21 ++++---- .../catalog_service/test_catalog_service_client.py | 9 ++-- tests/common/impala_connection.py | 23 ++++----- tests/common/impala_test_suite.py | 4 +- tests/custom_cluster/test_admission_controller.py | 38 ++++++++------ tests/custom_cluster/test_custom_statestore.py | 25 +++++---- tests/custom_cluster/test_delegation.py | 6 ++- .../custom_cluster/test_event_processing_error.py | 4 +- tests/custom_cluster/test_events_custom_configs.py | 6 +-- tests/custom_cluster/test_hs2.py | 6 +-- tests/custom_cluster/test_metastore_service.py | 28 +++++----- tests/custom_cluster/test_query_log.py | 19 +++---- tests/custom_cluster/test_query_retries.py | 26 ++++++---- tests/custom_cluster/test_restart_services.py | 23 +++++---- tests/custom_cluster/test_set_and_unset.py | 6 ++- tests/custom_cluster/test_statestored_ha.py | 22 ++++---- tests/custom_cluster/test_workload_mgmt_init.py | 18 +++---- .../test_workload_mgmt_sql_details.py | 2 +- tests/hs2/hs2_test_suite.py | 16 +++--- tests/hs2/test_fetch.py | 14 +++-- tests/hs2/test_fetch_first.py | 7 +-- tests/hs2/test_fetch_timeout.py | 9 ++-- tests/hs2/test_hs2.py | 26 ++++++---- tests/hs2/test_json_endpoints.py | 12 +++-- tests/metadata/test_compute_stats.py | 21 +++++--- tests/query_test/test_acid.py | 13 +++-- tests/query_test/test_cancellation.py | 13 ++--- tests/query_test/test_iceberg.py | 30 +++++------ tests/query_test/test_insert_parquet.py | 24 ++++++--- tests/query_test/test_observability.py | 10 ++-- tests/query_test/test_parquet_bloom_filter.py | 13 ++--- tests/query_test/test_parquet_page_index.py | 13 +++-- tests/query_test/test_query_opts.py | 3 +- tests/query_test/test_scanners.py | 29 +++++------ tests/statestore/test_statestore.py | 36 ++++++------- tests/util/acid_txn.py | 26 +++++++--- tests/util/event_processor_utils.py | 9 ++-- tests/util/get_parquet_metadata.py | 13 ++--- tests/util/parse_util.py | 4 +- tests/util/workload_management.py | 2 +- 90 files changed, 548 insertions(+), 352 deletions(-) diff --git a/infra/python/deps/py3-requirements.txt b/.isort.cfg similarity index 65% copy from infra/python/deps/py3-requirements.txt copy to .isort.cfg index c21bb3e92..5574ffd31 100644 --- a/infra/python/deps/py3-requirements.txt +++ b/.isort.cfg @@ -15,22 +15,15 @@ # specific language governing permissions and limitations # under the License. -# Python3-only requirements - -pylint == 2.10.2 - astroid == 2.7.3 - lazy-object-proxy == 1.6.0 - wrapt == 1.12.1 - typed-ast == 1.4.3 - configparser == 4.0.2 - isort == 4.3.21 - futures == 3.3.0; python_version == "2.7" - singledispatch == 3.6.1 - toml == 0.10.2 - platformdirs == 2.4.1 - typing-extensions == 3.10.0.2 -k5test==0.10.3 - -# Below are needed only for dev -flake8==3.9.2 -flake8-unused-arguments==0.0.13 +[settings] +force_alphabetical_sort_within_sections=true +force_sort_within_sections=true +include_trailing_comma=true +known_third_party=builtins +known_first_party=impala_py_lib,impala_shell,impala_thrift_gen,tests +line_length=90 +multi_line_output=3 +no_lines_before=STDLIB +order_by_type=false +skip_gitignore=true +skip_glob=testdata/cluster/cdh* \ No newline at end of file diff --git a/bin/cmake_aux/add_thrift_python_namespace.sh b/bin/cmake_aux/add_thrift_python_namespace.sh new file mode 100755 index 000000000..7ec482d2e --- /dev/null +++ b/bin/cmake_aux/add_thrift_python_namespace.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# add_thrift_python_namespace.sh IN_THRIFT_FILE OUT_THRIFT_FILE +# +# This script reads $IN_THRIFT_FILE and adds a python namespace +# (replacing any existing python namespace) with +# impala_thrift_gen.${BASE_NAME} where the BASE_NAME is the +# thrift filename without the ".thrift". i.e. Foo.thrift uses +# impala_thrift_gen.Foo python namespace. It writes the resulting +# thrift file to $OUT_THRIFT_FILE. +# +# This logic is taken from Impyla's impala/thrift/process_thrift.sh +# script with minor changes. This requires that the source thrift +# file have at least one preexisting non-python namespace. That is +# true for all of the Thrift files that we care about. + +set -eou pipefail + +THRIFT_FILE_IN=$1 +THRIFT_FILE_OUT=$2 + +FILE_NAME=$(basename $THRIFT_FILE_IN) +BASE_NAME=${FILE_NAME%.*} +# Awk script to add the python namespace before the first namespace +# in the thrift file. +ADD_NAMESPACE_PY=" + BEGIN { + n = 0 + } + { + if (\$0 ~ /^namespace/ && n == 0) { + print \"namespace py impala_thrift_gen.$BASE_NAME\"; + n += 1; + } + print \$0; + }" + +# Remove any existing python namespace, then add our namespace +cat $THRIFT_FILE_IN | grep -v "^namespace py" | awk "$ADD_NAMESPACE_PY" > $THRIFT_FILE_OUT diff --git a/infra/python/deps/py3-requirements.txt b/bin/impala-isort old mode 100644 new mode 100755 similarity index 65% copy from infra/python/deps/py3-requirements.txt copy to bin/impala-isort index c21bb3e92..8dab98af6 --- a/infra/python/deps/py3-requirements.txt +++ b/bin/impala-isort @@ -1,3 +1,5 @@ +#!/bin/bash + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,22 +17,5 @@ # specific language governing permissions and limitations # under the License. -# Python3-only requirements - -pylint == 2.10.2 - astroid == 2.7.3 - lazy-object-proxy == 1.6.0 - wrapt == 1.12.1 - typed-ast == 1.4.3 - configparser == 4.0.2 - isort == 4.3.21 - futures == 3.3.0; python_version == "2.7" - singledispatch == 3.6.1 - toml == 0.10.2 - platformdirs == 2.4.1 - typing-extensions == 3.10.0.2 -k5test==0.10.3 - -# Below are needed only for dev -flake8==3.9.2 -flake8-unused-arguments==0.0.13 +source "$(dirname "$0")/impala-python3-common.sh" +exec "$PY_ENV_DIR/bin/isort" "$@" diff --git a/common/thrift/.gitignore b/common/thrift/.gitignore index 0f6c0c55e..cd3eb435e 100644 --- a/common/thrift/.gitignore +++ b/common/thrift/.gitignore @@ -3,3 +3,4 @@ ErrorCodes.thrift MetricDefs.thrift hive-2-api/TCLIService.thrift hive-3-api/TCLIService.thrift +/thirdparty_thrift/ diff --git a/common/thrift/BackendGflags.thrift b/common/thrift/BackendGflags.thrift index dd2fc423d..065c25ec2 100644 --- a/common/thrift/BackendGflags.thrift +++ b/common/thrift/BackendGflags.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.BackendGflags namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/CMakeLists.txt b/common/thrift/CMakeLists.txt index f86036606..fd77cf82a 100644 --- a/common/thrift/CMakeLists.txt +++ b/common/thrift/CMakeLists.txt @@ -135,18 +135,19 @@ function(THRIFT_GEN_DS VAR) set(${VAR} ${${VAR}} PARENT_SCOPE) endfunction(THRIFT_GEN_DS) +set(THIRDPARTY_THRIFT_DIR "thirdparty_thrift") set(HIVE_THRIFT_SOURCE_DIR "hive-$ENV{IMPALA_HIVE_MAJOR_VERSION}-api") set(TCLI_SERVICE_THRIFT "${HIVE_THRIFT_SOURCE_DIR}/TCLIService.thrift") message("Using Thrift CPP compiler: ${THRIFT_CPP_COMPILER}") message("Using Thrift JAVA compiler: ${THRIFT_JAVA_COMPILER}") message("Using Thrift PY compiler: ${THRIFT_PY_COMPILER}") set(THRIFT_QUIET_WRAPPER "${CMAKE_SOURCE_DIR}/bin/thrift-quiet-wrapper.sh") -set(THRIFT_CPP_INCLUDE_DIR_OPTION -I ${THRIFT_CPP_CONTRIB_DIR} - -I $ENV{HIVE_METASTORE_THRIFT_DIR} -I ${HIVE_THRIFT_SOURCE_DIR}) -set(THRIFT_JAVA_INCLUDE_DIR_OPTION -I ${THRIFT_JAVA_CONTRIB_DIR} - -I $ENV{HIVE_METASTORE_THRIFT_DIR} -I ${HIVE_THRIFT_SOURCE_DIR}) -set(THRIFT_PY_INCLUDE_DIR_OPTION -I ${THRIFT_PY_CONTRIB_DIR} - -I $ENV{HIVE_METASTORE_THRIFT_DIR} -I ${HIVE_THRIFT_SOURCE_DIR}) +set(THRIFT_CPP_INCLUDE_DIR_OPTION -I ${THIRDPARTY_THRIFT_DIR} + -I ${HIVE_THRIFT_SOURCE_DIR}) +set(THRIFT_JAVA_INCLUDE_DIR_OPTION -I ${THIRDPARTY_THRIFT_DIR} + -I ${HIVE_THRIFT_SOURCE_DIR}) +set(THRIFT_PY_INCLUDE_DIR_OPTION -I ${THIRDPARTY_THRIFT_DIR} + -I ${HIVE_THRIFT_SOURCE_DIR}) set(BE_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/be/generated-sources) set(FE_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/fe/generated-sources) # TODO: avoid duplicating generated java classes @@ -158,12 +159,13 @@ file(MAKE_DIRECTORY ${FE_OUTPUT_DIR}) file(MAKE_DIRECTORY ${EXT_DS_OUTPUT_DIR}) file(MAKE_DIRECTORY ${PYTHON_OUTPUT_DIR}) file(MAKE_DIRECTORY ${HIVE_THRIFT_SOURCE_DIR}) +file(MAKE_DIRECTORY ${THIRDPARTY_THRIFT_DIR}) # Args passed to thrift for Java gen set(JAVA_FE_ARGS ${THRIFT_JAVA_INCLUDE_DIR_OPTION} --gen java -o ${FE_OUTPUT_DIR}) set(JAVA_EXT_DS_ARGS ${THRIFT_JAVA_INCLUDE_DIR_OPTION} --gen java -o ${EXT_DS_OUTPUT_DIR}) set(PYTHON_ARGS ${THRIFT_PY_INCLUDE_DIR_OPTION} -r --gen py:no_utf8strings -o - ${PYTHON_OUTPUT_DIR}) + ${PYTHON_OUTPUT_DIR}) set (EXT_DATA_SRC_FILES ErrorCodes.thrift @@ -211,6 +213,16 @@ set (SRC_FILES ) SET_SOURCE_FILES_PROPERTIES(Status.thrift PROPERTIES OBJECT_DEPENDS ErrorCodes.thrift) +SET_SOURCE_FILES_PROPERTIES(CatalogObjects.thrift PROPERTIES OBJECT_DEPENDS + ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift) +SET_SOURCE_FILES_PROPERTIES(CatalogService.thrift PROPERTIES OBJECT_DEPENDS + ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift) +SET_SOURCE_FILES_PROPERTIES(JniCatalog.thrift PROPERTIES OBJECT_DEPENDS + ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift) +SET_SOURCE_FILES_PROPERTIES(SqlConstraints.thrift PROPERTIES OBJECT_DEPENDS + ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift) +SET_SOURCE_FILES_PROPERTIES(beeswax.thrift PROPERTIES OBJECT_DEPENDS + ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift) add_custom_command(OUTPUT ErrorCodes.thrift COMMAND python generate_error_codes.py @@ -232,6 +244,33 @@ add_custom_command(OUTPUT hive-$ENV{IMPALA_HIVE_MAJOR_VERSION}-api/TCLIService.t DEPENDS hive-1-api/TCLIService.thrift ) +# This generates hive_metastore.thrift in the $THIRDPARTY_THRIFT_DIR. The two +# modification are: +# 1. Set the impala_thrift_gen python namespace +# 2. Rearranges the fb303 reference so that it doesn't have the share/fb303/if +# directory structure +add_custom_command(OUTPUT ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift + COMMAND ${CMAKE_SOURCE_DIR}/bin/cmake_aux/add_thrift_python_namespace.sh + $ENV{HIVE_METASTORE_THRIFT_DIR}/hive_metastore.thrift + ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift.tmp + COMMAND cat ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift.tmp | + sed 's|share/fb303/if/||' > ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift + COMMAND rm ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift.tmp + DEPENDS $ENV{HIVE_METASTORE_THRIFT_DIR}/hive_metastore.thrift +) + +# Generate fb303.thrift in the $THIRDPARTY_THRIFT_DIR with the appropriate +# impala_thrift_gen python namespace. +add_custom_command(OUTPUT ${THIRDPARTY_THRIFT_DIR}/fb303.thrift + COMMAND ${CMAKE_SOURCE_DIR}/bin/cmake_aux/add_thrift_python_namespace.sh + ${THRIFT_PY_CONTRIB_DIR}/share/fb303/if/fb303.thrift + ${THIRDPARTY_THRIFT_DIR}/fb303.thrift + DEPENDS ${THRIFT_PY_CONTRIB_DIR}/share/fb303/if/fb303.thrift +) + +SET_SOURCE_FILES_PROPERTIES(${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift + PROPERTIES OBJECT_DEPENDS ${THIRDPARTY_THRIFT_DIR}/fb303.thrift) + # Create a build command for each of the thrift src files and generate # a list of files they produce THRIFT_GEN(THRIFT_ALL_FILES ${SRC_FILES}) @@ -240,11 +279,15 @@ THRIFT_GEN_DS(THRIFT_DATA_SRC_FILES ${EXT_DATA_SRC_FILES}) add_custom_target(thrift-generated-files-error DEPENDS ErrorCodes.thrift) add_custom_target(thrift-generated-files-metrics DEPENDS MetricDefs.thrift) add_custom_target(thrift-generated-files-tcli-service DEPENDS ${TCLI_SERVICE_THRIFT}) +add_custom_target(thrift-generated-files-hive-metastore + DEPENDS ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift) +add_custom_target(thrift-generated-files-fb303 DEPENDS ${THIRDPARTY_THRIFT_DIR}/fb303.thrift) # Add a custom target that generates all the thrift files add_custom_target(thrift-cpp ALL DEPENDS ${THRIFT_ALL_FILES}) add_dependencies(thrift-cpp thrift-generated-files-metrics thrift-generated-files-error - thrift-generated-files-tcli-service) + thrift-generated-files-tcli-service thrift-generated-files-hive-metastore + thrift-generated-files-fb303) add_custom_target(thrift-ext-data-src ALL DEPENDS ${THRIFT_DATA_SRC_FILES}) add_dependencies(thrift-ext-data-src thrift-cpp) diff --git a/common/thrift/CatalogInternalService.thrift b/common/thrift/CatalogInternalService.thrift index 6007f0227..03a1208ad 100644 --- a/common/thrift/CatalogInternalService.thrift +++ b/common/thrift/CatalogInternalService.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.CatalogInternalService namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/CatalogObjects.thrift b/common/thrift/CatalogObjects.thrift index 6e186d6b9..0afc4f3a1 100644 --- a/common/thrift/CatalogObjects.thrift +++ b/common/thrift/CatalogObjects.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.CatalogObjects namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/CatalogService.thrift b/common/thrift/CatalogService.thrift index f57f62ec9..6702e3c62 100644 --- a/common/thrift/CatalogService.thrift +++ b/common/thrift/CatalogService.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.CatalogService namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Data.thrift b/common/thrift/Data.thrift index b68657f3e..999f9330b 100644 --- a/common/thrift/Data.thrift +++ b/common/thrift/Data.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Data namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/DataSinks.thrift b/common/thrift/DataSinks.thrift index 882fe0bee..942863b2c 100644 --- a/common/thrift/DataSinks.thrift +++ b/common/thrift/DataSinks.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.DataSinks namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Descriptors.thrift b/common/thrift/Descriptors.thrift index 2acc31264..83cc597c6 100644 --- a/common/thrift/Descriptors.thrift +++ b/common/thrift/Descriptors.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Descriptors namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/ExecStats.thrift b/common/thrift/ExecStats.thrift index f629eaf0a..0a6073c86 100644 --- a/common/thrift/ExecStats.thrift +++ b/common/thrift/ExecStats.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.ExecStats namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Exprs.thrift b/common/thrift/Exprs.thrift index fb02a298f..c2b342956 100644 --- a/common/thrift/Exprs.thrift +++ b/common/thrift/Exprs.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Exprs namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/ExternalDataSource.thrift b/common/thrift/ExternalDataSource.thrift index 7ec0d8079..4e8e6d593 100644 --- a/common/thrift/ExternalDataSource.thrift +++ b/common/thrift/ExternalDataSource.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.ExternalDataSource namespace cpp impala.extdatasource namespace java org.apache.impala.extdatasource.thrift diff --git a/common/thrift/Frontend.thrift b/common/thrift/Frontend.thrift index 770017505..5426ba7e6 100644 --- a/common/thrift/Frontend.thrift +++ b/common/thrift/Frontend.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Frontend namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/ImpalaInternalService.thrift b/common/thrift/ImpalaInternalService.thrift index c01e537a4..5f324d668 100644 --- a/common/thrift/ImpalaInternalService.thrift +++ b/common/thrift/ImpalaInternalService.thrift @@ -18,6 +18,7 @@ // // This file contains the details of the protocol between coordinators and backends. +namespace py impala_thrift_gen.ImpalaInternalService namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/ImpalaService.thrift b/common/thrift/ImpalaService.thrift index 88c4de5c4..43886c001 100644 --- a/common/thrift/ImpalaService.thrift +++ b/common/thrift/ImpalaService.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.ImpalaService namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/JniCatalog.thrift b/common/thrift/JniCatalog.thrift old mode 100755 new mode 100644 index 1b91bec65..62dc72fcd --- a/common/thrift/JniCatalog.thrift +++ b/common/thrift/JniCatalog.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.JniCatalog namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/LineageGraph.thrift b/common/thrift/LineageGraph.thrift index 61b2f5610..5d0f48202 100644 --- a/common/thrift/LineageGraph.thrift +++ b/common/thrift/LineageGraph.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.LineageGraph namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Logging.thrift b/common/thrift/Logging.thrift index 6335667f5..10c7c7837 100644 --- a/common/thrift/Logging.thrift +++ b/common/thrift/Logging.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Logging namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Metrics.thrift b/common/thrift/Metrics.thrift index 8d92bd74b..754cd44d3 100644 --- a/common/thrift/Metrics.thrift +++ b/common/thrift/Metrics.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Metrics namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/NetworkTest.thrift b/common/thrift/NetworkTest.thrift index d52751092..585bf28f5 100644 --- a/common/thrift/NetworkTest.thrift +++ b/common/thrift/NetworkTest.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.NetworkTest namespace cpp impalatest struct ThriftDataParams { diff --git a/common/thrift/Partitions.thrift b/common/thrift/Partitions.thrift index 0d1c860ae..b3fdef8fb 100644 --- a/common/thrift/Partitions.thrift +++ b/common/thrift/Partitions.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Partitions namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/PlanNodes.thrift b/common/thrift/PlanNodes.thrift index 3760a6b9f..929dd0008 100644 --- a/common/thrift/PlanNodes.thrift +++ b/common/thrift/PlanNodes.thrift @@ -21,6 +21,7 @@ // of the execution parameters of any one of the backends on which it is running // (those are recorded in TPlanFragmentInstanceCtx). +namespace py impala_thrift_gen.PlanNodes namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Planner.thrift b/common/thrift/Planner.thrift index 36622f97f..a600e6219 100644 --- a/common/thrift/Planner.thrift +++ b/common/thrift/Planner.thrift @@ -18,6 +18,7 @@ // // This file contains structures produced by the planner. +namespace py impala_thrift_gen.Planner namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Query.thrift b/common/thrift/Query.thrift index b1ce5fd62..8e3fc5969 100644 --- a/common/thrift/Query.thrift +++ b/common/thrift/Query.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Query namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/ResourceProfile.thrift b/common/thrift/ResourceProfile.thrift index be76edd53..8fe092119 100644 --- a/common/thrift/ResourceProfile.thrift +++ b/common/thrift/ResourceProfile.thrift @@ -19,6 +19,7 @@ // and DataSinks to configure their memory usage. See ResourceProfile.java for more // details. +namespace py impala_thrift_gen.ResourceProfile namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Results.thrift b/common/thrift/Results.thrift index ff5ffd4ff..da639bf57 100644 --- a/common/thrift/Results.thrift +++ b/common/thrift/Results.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Results namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/RuntimeProfile.thrift b/common/thrift/RuntimeProfile.thrift index 02dd1e7ea..1f632b88e 100644 --- a/common/thrift/RuntimeProfile.thrift +++ b/common/thrift/RuntimeProfile.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.RuntimeProfile namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/SqlConstraints.thrift b/common/thrift/SqlConstraints.thrift index ee4a9cf30..7d3882e42 100644 --- a/common/thrift/SqlConstraints.thrift +++ b/common/thrift/SqlConstraints.thrift @@ -16,6 +16,7 @@ // under the License. +namespace py impala_thrift_gen.SqlConstraints namespace cpp impala namespace java org.apache.impala.thrift @@ -28,4 +29,4 @@ struct TSqlConstraints { // Foreign Keys information for the given table. 2: required list<hive_metastore.SQLForeignKey> foreign_keys -} \ No newline at end of file +} diff --git a/common/thrift/StatestoreService.thrift b/common/thrift/StatestoreService.thrift index 1ae961718..e25ea18d4 100644 --- a/common/thrift/StatestoreService.thrift +++ b/common/thrift/StatestoreService.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.StatestoreService namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Status.thrift b/common/thrift/Status.thrift index 3997e4070..ef74c4965 100644 --- a/common/thrift/Status.thrift +++ b/common/thrift/Status.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Status namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/SystemTables.thrift b/common/thrift/SystemTables.thrift index e1cd1ca06..d1b84d2e0 100644 --- a/common/thrift/SystemTables.thrift +++ b/common/thrift/SystemTables.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.SystemTables namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Types.thrift b/common/thrift/Types.thrift index 0c951ee73..4903dfac0 100644 --- a/common/thrift/Types.thrift +++ b/common/thrift/Types.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Types namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Zip.thrift b/common/thrift/Zip.thrift index b5f02d1b6..7d97e6c8e 100644 --- a/common/thrift/Zip.thrift +++ b/common/thrift/Zip.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Zip namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/beeswax.thrift b/common/thrift/beeswax.thrift index 8fdb6c328..f66d06228 100644 --- a/common/thrift/beeswax.thrift +++ b/common/thrift/beeswax.thrift @@ -17,8 +17,8 @@ // Interface for interacting with Beeswax Server +namespace py impala_thrift_gen.beeswax namespace java com.cloudera.beeswax.api -namespace py beeswaxd namespace cpp beeswax include "hive_metastore.thrift" diff --git a/common/thrift/generate_error_codes.py b/common/thrift/generate_error_codes.py index 80a275f90..b2c00b702 100755 --- a/common/thrift/generate_error_codes.py +++ b/common/thrift/generate_error_codes.py @@ -539,6 +539,7 @@ preamble = """ // IT BY HAND. // +namespace py impala_thrift_gen.ErrorCodes namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/generate_metrics.py b/common/thrift/generate_metrics.py index 6aa02822f..66125ed14 100755 --- a/common/thrift/generate_metrics.py +++ b/common/thrift/generate_metrics.py @@ -88,6 +88,7 @@ THRIFT_PREAMBLE = """ // THIS FILE IS AUTO GENERATED BY generate_metrics.py DO NOT MODIFY IT BY HAND. // +namespace py impala_thrift_gen.Metrics namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/hive-1-api/TCLIService.thrift b/common/thrift/hive-1-api/TCLIService.thrift index 09f272e52..777fcae24 100644 --- a/common/thrift/hive-1-api/TCLIService.thrift +++ b/common/thrift/hive-1-api/TCLIService.thrift @@ -32,6 +32,7 @@ // * Service names begin with the letter "T", use a capital letter for each // new word (with no underscores), and end with the word "Service". +namespace py impala_thrift_gen.TCLIService namespace java org.apache.hive.service.cli.thrift namespace cpp apache.hive.service.cli.thrift diff --git a/common/thrift/parquet.thrift b/common/thrift/parquet.thrift index 443493518..675cf7261 100644 --- a/common/thrift/parquet.thrift +++ b/common/thrift/parquet.thrift @@ -20,6 +20,7 @@ /** * File format description for the parquet file format */ +namespace py impala_thrift_gen.parquet namespace cpp parquet namespace java org.apache.parquet.format diff --git a/infra/python/deps/py3-requirements.txt b/infra/python/deps/py3-requirements.txt index c21bb3e92..051226125 100644 --- a/infra/python/deps/py3-requirements.txt +++ b/infra/python/deps/py3-requirements.txt @@ -23,8 +23,9 @@ pylint == 2.10.2 wrapt == 1.12.1 typed-ast == 1.4.3 configparser == 4.0.2 - isort == 4.3.21 + isort == 5.13.2 futures == 3.3.0; python_version == "2.7" + poetry-core == 1.9.1 singledispatch == 3.6.1 toml == 0.10.2 platformdirs == 2.4.1 diff --git a/lib/python/impala_py_lib/profiles.py b/lib/python/impala_py_lib/profiles.py index 4d5ecb731..5425e06ef 100644 --- a/lib/python/impala_py_lib/profiles.py +++ b/lib/python/impala_py_lib/profiles.py @@ -22,9 +22,11 @@ from __future__ import absolute_import, division, print_function import base64 import datetime import zlib + from thrift.protocol import TCompactProtocol from thrift.TSerialization import deserialize -from RuntimeProfile.ttypes import TRuntimeProfileTree + +from impala_thrift_gen.RuntimeProfile.ttypes import TRuntimeProfileTree def decode_profile_line(line): diff --git a/shell/exec_summary.py b/shell/exec_summary.py index a7897315c..7431cc704 100755 --- a/shell/exec_summary.py +++ b/shell/exec_summary.py @@ -18,7 +18,9 @@ # specific language governing permissions and limitations # under the License. -from ExecStats.ttypes import TExecStats +from __future__ import absolute_import, print_function, unicode_literals + +from impala_thrift_gen.ExecStats.ttypes import TExecStats def build_exec_summary_table(summary, idx, indent_level, new_indent_level, output, diff --git a/shell/impala_client.py b/shell/impala_client.py index 3fc1891e1..96b2554c7 100755 --- a/shell/impala_client.py +++ b/shell/impala_client.py @@ -33,17 +33,19 @@ import traceback from datetime import datetime import uuid -from beeswaxd import BeeswaxService -from beeswaxd.BeeswaxService import QueryState -from ImpalaService import ImpalaService, ImpalaHiveServer2Service -from ImpalaService.ImpalaHiveServer2Service import (TGetRuntimeProfileReq, - TGetExecSummaryReq, TPingImpalaHS2ServiceReq, TCloseImpalaOperationReq) -from ErrorCodes.ttypes import TErrorCode -from Status.ttypes import TStatus -from TCLIService.TCLIService import (TExecuteStatementReq, TOpenSessionReq, - TCloseSessionReq, TProtocolVersion, TStatusCode, TGetOperationStatusReq, - TOperationState, TFetchResultsReq, TFetchOrientation, TGetLogReq, - TGetResultSetMetadataReq, TTypeId, TCancelOperationReq, TCloseOperationReq) +from impala_thrift_gen.beeswax import BeeswaxService +from impala_thrift_gen.beeswax.BeeswaxService import QueryState +from impala_thrift_gen.ImpalaService import ImpalaService, ImpalaHiveServer2Service +from impala_thrift_gen.ImpalaService.ImpalaHiveServer2Service import ( + TGetRuntimeProfileReq, TGetExecSummaryReq, TPingImpalaHS2ServiceReq, + TCloseImpalaOperationReq) +from impala_thrift_gen.ErrorCodes.ttypes import TErrorCode +from impala_thrift_gen.Status.ttypes import TStatus +from impala_thrift_gen.TCLIService.TCLIService import (TExecuteStatementReq, + TOpenSessionReq, TCloseSessionReq, TProtocolVersion, TStatusCode, + TGetOperationStatusReq, TOperationState, TFetchResultsReq, TFetchOrientation, + TGetLogReq, TGetResultSetMetadataReq, TTypeId, TCancelOperationReq, + TCloseOperationReq) from ImpalaHttpClient import ImpalaHttpClient from exec_summary import build_exec_summary_table from kerberos_util import get_kerb_host_from_kerberos_host_fqdn diff --git a/shell/value_converter.py b/shell/value_converter.py index 4c33fb004..4c8e5e0a7 100644 --- a/shell/value_converter.py +++ b/shell/value_converter.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -from TCLIService.TCLIService import TTypeId +from impala_thrift_gen.TCLIService.TCLIService import TTypeId import sys diff --git a/testdata/bin/wait-for-hiveserver2.py b/testdata/bin/wait-for-hiveserver2.py index 37a2898f2..1b7344596 100755 --- a/testdata/bin/wait-for-hiveserver2.py +++ b/testdata/bin/wait-for-hiveserver2.py @@ -30,7 +30,7 @@ from optparse import OptionParser from tests.util.thrift_util import create_transport # Imports required for HiveServer2 Client -from TCLIService import TCLIService +from impala_thrift_gen.TCLIService import TCLIService from thrift.transport import TTransport, TSocket from thrift.protocol import TBinaryProtocol diff --git a/testdata/bin/wait-for-metastore.py b/testdata/bin/wait-for-metastore.py index ab898fa27..983c40763 100755 --- a/testdata/bin/wait-for-metastore.py +++ b/testdata/bin/wait-for-metastore.py @@ -28,7 +28,7 @@ from optparse import OptionParser from tests.util.thrift_util import create_transport # Imports required for Hive Metastore Client -from hive_metastore import ThriftHiveMetastore +from impala_thrift_gen.hive_metastore import ThriftHiveMetastore from thrift.transport import TTransport, TSocket from thrift.protocol import TBinaryProtocol diff --git a/tests/authorization/test_authorization.py b/tests/authorization/test_authorization.py index 07ce8ad18..594e5ab6e 100644 --- a/tests/authorization/test_authorization.py +++ b/tests/authorization/test_authorization.py @@ -18,17 +18,18 @@ # Client tests for SQL statement authorization from __future__ import absolute_import, division, print_function -import pytest +from getpass import getuser import random import threading import time -from getpass import getuser -from ImpalaService import ImpalaHiveServer2Service -from TCLIService import TCLIService +import pytest +from thrift.protocol import TBinaryProtocol from thrift.transport.TSocket import TSocket from thrift.transport.TTransport import TBufferedTransport -from thrift.protocol import TBinaryProtocol + +from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service +from impala_thrift_gen.TCLIService import TCLIService from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.common.file_utils import assert_file_in_dir_contains from tests.common.test_result_verifier import error_msg_equal diff --git a/tests/authorization/test_authorized_proxy.py b/tests/authorization/test_authorized_proxy.py index 20005139b..5e624b3b8 100644 --- a/tests/authorization/test_authorized_proxy.py +++ b/tests/authorization/test_authorized_proxy.py @@ -16,16 +16,17 @@ # under the License. from __future__ import absolute_import, division, print_function -import pytest +import json import os import time -import json -from ImpalaService import ImpalaHiveServer2Service -from TCLIService import TCLIService +import pytest +from thrift.protocol import TBinaryProtocol from thrift.transport.TSocket import TSocket from thrift.transport.TTransport import TBufferedTransport -from thrift.protocol import TBinaryProtocol + +from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service +from impala_thrift_gen.TCLIService import TCLIService from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.hs2.hs2_test_suite import operation_id_to_query_id diff --git a/tests/beeswax/impala_beeswax.py b/tests/beeswax/impala_beeswax.py index cd1f29bd5..2a7e5b8de 100644 --- a/tests/beeswax/impala_beeswax.py +++ b/tests/beeswax/impala_beeswax.py @@ -26,22 +26,23 @@ # result = client.execute(query_string) # where result is an object of the class ImpalaBeeswaxResult. from __future__ import absolute_import, division, print_function -from builtins import filter, map -import logging -import time -import shlex +from datetime import datetime import getpass +import logging import re +import shlex import sys +import time -from beeswaxd import BeeswaxService -from beeswaxd.BeeswaxService import QueryState -from datetime import datetime -from ImpalaService import ImpalaService -from tests.util.thrift_util import create_transport -from thrift.transport.TTransport import TTransportException +from builtins import filter, map from thrift.protocol import TBinaryProtocol from thrift.Thrift import TApplicationException +from thrift.transport.TTransport import TTransportException + +from impala_thrift_gen.beeswax import BeeswaxService +from impala_thrift_gen.beeswax.BeeswaxService import QueryState +from impala_thrift_gen.ImpalaService import ImpalaService +from tests.util.thrift_util import create_transport LOG = logging.getLogger('impala_beeswax') # time to sleep in seconds before polling again. This uses a fixed diff --git a/tests/catalog_service/test_catalog_service_client.py b/tests/catalog_service/test_catalog_service_client.py index 22520c4c9..eaf244dc0 100644 --- a/tests/catalog_service/test_catalog_service_client.py +++ b/tests/catalog_service/test_catalog_service_client.py @@ -19,13 +19,13 @@ from __future__ import absolute_import, division, print_function import logging -import pytest -from CatalogService import CatalogService -from CatalogService.CatalogService import TGetFunctionsRequest -from ErrorCodes.ttypes import TErrorCode +import pytest from thrift.protocol import TBinaryProtocol +from impala_thrift_gen.CatalogService import CatalogService +from impala_thrift_gen.CatalogService.CatalogService import TGetFunctionsRequest +from impala_thrift_gen.ErrorCodes.ttypes import TErrorCode from tests.common.impala_cluster import ImpalaCluster from tests.common.impala_test_suite import ImpalaTestSuite from tests.common.skip import SkipIfDockerizedCluster @@ -33,7 +33,6 @@ from tests.common.test_dimensions import create_single_exec_option_dimension from tests.util.filesystem_utils import WAREHOUSE from tests.util.thrift_util import create_transport - LOG = logging.getLogger('test_catalog_service_client') # TODO: Add a test that asserts correct/compatible responses diff --git a/tests/common/impala_connection.py b/tests/common/impala_connection.py index 50cddfe4d..9ac6bafb0 100644 --- a/tests/common/impala_connection.py +++ b/tests/common/impala_connection.py @@ -21,29 +21,28 @@ from __future__ import absolute_import, division, print_function import abc -from future.utils import with_metaclass import getpass import logging import re import time -from beeswaxd.BeeswaxService import QueryState +from future.utils import with_metaclass import impala.dbapi as impyla import impala.error as impyla_error import impala.hiveserver2 as hs2 -import tests.common -from Query.ttypes import TQueryOptions -from RuntimeProfile.ttypes import TRuntimeProfileFormat + +from impala_thrift_gen.beeswax.BeeswaxService import QueryState +from impala_thrift_gen.Query.ttypes import TQueryOptions +from impala_thrift_gen.RuntimeProfile.ttypes import TRuntimeProfileFormat from tests.beeswax.impala_beeswax import ( - DEFAULT_SLEEP_INTERVAL, - ImpalaBeeswaxClient, - ImpalaBeeswaxException) + DEFAULT_SLEEP_INTERVAL, + ImpalaBeeswaxClient, + ImpalaBeeswaxException, +) +import tests.common from tests.common.patterns import LOG_FORMAT from tests.common.test_vector import BEESWAX, HS2, HS2_HTTP -from tests.util.thrift_util import ( - op_handle_to_query_id, - session_handle_to_session_id) - +from tests.util.thrift_util import op_handle_to_query_id, session_handle_to_session_id LOG = logging.getLogger(__name__) console_handler = logging.StreamHandler() diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py index fe4f4ac50..72126a39a 100644 --- a/tests/common/impala_test_suite.py +++ b/tests/common/impala_test_suite.py @@ -102,11 +102,11 @@ from tests.util.thrift_util import create_transport from tests.util.retry import retry # Imports required for Hive Metastore Client -from hive_metastore import ThriftHiveMetastore +from impala_thrift_gen.hive_metastore import ThriftHiveMetastore from thrift.protocol import TBinaryProtocol # Import to validate query option names -from ImpalaService.ttypes import TImpalaQueryOptions +from impala_thrift_gen.ImpalaService.ttypes import TImpalaQueryOptions # Initializing the logger before conditional imports, since we will need it # for them. diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py index ad19590a5..5cee71bd3 100644 --- a/tests/custom_cluster/test_admission_controller.py +++ b/tests/custom_cluster/test_admission_controller.py @@ -18,50 +18,58 @@ # Tests admission control from __future__ import absolute_import, division, print_function -from builtins import int, range, round +from copy import deepcopy import itertools import logging import os -import pytest import re import signal import sys import threading -from copy import deepcopy from time import sleep, time +from builtins import int, range, round +import pytest + +from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service +from impala_thrift_gen.TCLIService import TCLIService from tests.common.cluster_config import ( - impalad_admission_ctrl_flags, impalad_admission_ctrl_config_args, - RESOURCES_DIR) + impalad_admission_ctrl_flags, + RESOURCES_DIR, +) from tests.common.custom_cluster_test_suite import ( ADMISSIOND_ARGS, + CustomClusterTestSuite, IMPALAD_ARGS, START_ARGS, WORKLOAD_MGMT_IMPALAD_FLAGS, - CustomClusterTestSuite) +) from tests.common.environ import build_flavor_timeout, ImpalaTestClusterProperties from tests.common.impala_connection import ( - RUNNING, FINISHED, ERROR, - IMPALA_CONNECTION_EXCEPTION) + ERROR, + FINISHED, + IMPALA_CONNECTION_EXCEPTION, + RUNNING, +) from tests.common.resource_pool_config import ResourcePoolConfig -from tests.common.skip import SkipIfFS, SkipIfEC, SkipIfNotHdfsMinicluster +from tests.common.skip import SkipIfEC, SkipIfFS, SkipIfNotHdfsMinicluster from tests.common.test_dimensions import ( - HS2, add_mandatory_exec_option, create_exec_option_dimension, create_single_exec_option_dimension, - create_uncompressed_text_dimension) + create_uncompressed_text_dimension, + HS2, +) from tests.common.test_vector import ImpalaTestDimension from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session -from tests.util.workload_management import QUERY_TBL_LIVE from tests.util.web_pages_util import ( + get_mem_admitted_backends_debug_page, get_num_completed_backends, - get_mem_admitted_backends_debug_page) +) +from tests.util.workload_management import QUERY_TBL_LIVE from tests.verifiers.mem_usage_verifier import MemUsageVerifier from tests.verifiers.metric_verifier import MetricVerifier -from ImpalaService import ImpalaHiveServer2Service -from TCLIService import TCLIService LOG = logging.getLogger('admission_test') diff --git a/tests/custom_cluster/test_custom_statestore.py b/tests/custom_cluster/test_custom_statestore.py index 8e14f6a82..4a701d8cd 100644 --- a/tests/custom_cluster/test_custom_statestore.py +++ b/tests/custom_cluster/test_custom_statestore.py @@ -19,26 +19,25 @@ # Tests statestore with non-default startup options from __future__ import absolute_import, division, print_function -from builtins import range import logging -import pytest -import uuid import socket - -from tests.common.custom_cluster_test_suite import CustomClusterTestSuite -from tests.common.environ import build_flavor_timeout -from tests.common.skip import SkipIfBuildType -from tests.common.patterns import print_id from time import sleep +import uuid -from Types.ttypes import TNetworkAddress +from builtins import range +import pytest from thrift.protocol import TBinaryProtocol from thrift.transport import TSocket, TTransport -import StatestoreService.StatestoreSubscriber as Subscriber -import StatestoreService.StatestoreService as Statestore -import CatalogService.CatalogService as Catalog -from ErrorCodes.ttypes import TErrorCode +import impala_thrift_gen.CatalogService.CatalogService as Catalog +from impala_thrift_gen.ErrorCodes.ttypes import TErrorCode +import impala_thrift_gen.StatestoreService.StatestoreService as Statestore +import impala_thrift_gen.StatestoreService.StatestoreSubscriber as Subscriber +from impala_thrift_gen.Types.ttypes import TNetworkAddress +from tests.common.custom_cluster_test_suite import CustomClusterTestSuite +from tests.common.environ import build_flavor_timeout +from tests.common.patterns import print_id +from tests.common.skip import SkipIfBuildType LOG = logging.getLogger('custom_statestore_test') STATESTORE_SERVICE_PORT = 24000 diff --git a/tests/custom_cluster/test_delegation.py b/tests/custom_cluster/test_delegation.py index 72660b394..3b3c988ab 100644 --- a/tests/custom_cluster/test_delegation.py +++ b/tests/custom_cluster/test_delegation.py @@ -17,10 +17,12 @@ # from __future__ import absolute_import, division, print_function import getpass + import pytest -from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session -from TCLIService import TCLIService + +from impala_thrift_gen.TCLIService import TCLIService from tests.common.custom_cluster_test_suite import CustomClusterTestSuite +from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session USER_NAME = getpass.getuser() PROXY_USER = "proxy_user_name" diff --git a/tests/custom_cluster/test_event_processing_error.py b/tests/custom_cluster/test_event_processing_error.py index cd9a11547..7dfe1d195 100644 --- a/tests/custom_cluster/test_event_processing_error.py +++ b/tests/custom_cluster/test_event_processing_error.py @@ -15,10 +15,10 @@ # specific language governing permissions and limitations # under the License. from __future__ import absolute_import, division, print_function + from builtins import range -from hive_metastore.ttypes import FireEventRequest -from hive_metastore.ttypes import FireEventRequestData +from impala_thrift_gen.hive_metastore.ttypes import FireEventRequest, FireEventRequestData from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.common.skip import SkipIfCatalogV2, SkipIfFS from tests.metadata.test_event_processing_base import TestEventProcessingBase diff --git a/tests/custom_cluster/test_events_custom_configs.py b/tests/custom_cluster/test_events_custom_configs.py index fe728a747..1b6f29670 100644 --- a/tests/custom_cluster/test_events_custom_configs.py +++ b/tests/custom_cluster/test_events_custom_configs.py @@ -23,9 +23,9 @@ from os import getenv from time import sleep -from hive_metastore.ttypes import FireEventRequest -from hive_metastore.ttypes import FireEventRequestData -from hive_metastore.ttypes import InsertEventRequestData +from impala_thrift_gen.hive_metastore.ttypes import FireEventRequest +from impala_thrift_gen.hive_metastore.ttypes import FireEventRequestData +from impala_thrift_gen.hive_metastore.ttypes import InsertEventRequestData from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.common.impala_connection import ERROR, FINISHED from tests.common.impala_test_suite import ImpalaTestSuite diff --git a/tests/custom_cluster/test_hs2.py b/tests/custom_cluster/test_hs2.py index 7c64b885a..e7d24a6d3 100644 --- a/tests/custom_cluster/test_hs2.py +++ b/tests/custom_cluster/test_hs2.py @@ -16,13 +16,13 @@ # under the License. from __future__ import absolute_import, division, print_function -from tests.common.custom_cluster_test_suite import CustomClusterTestSuite +from time import sleep import pytest +from impala_thrift_gen.TCLIService import TCLIService +from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.hs2.hs2_test_suite import HS2TestSuite, operation_id_to_query_id -from time import sleep -from TCLIService import TCLIService class TestHS2(CustomClusterTestSuite): diff --git a/tests/custom_cluster/test_metastore_service.py b/tests/custom_cluster/test_metastore_service.py index eadb58d24..f185db076 100644 --- a/tests/custom_cluster/test_metastore_service.py +++ b/tests/custom_cluster/test_metastore_service.py @@ -16,24 +16,26 @@ # under the License. from __future__ import absolute_import, division, print_function + from builtins import range import pytest -from hive_metastore.ttypes import Database -from hive_metastore.ttypes import FieldSchema -from hive_metastore.ttypes import FindNextCompactRequest -from hive_metastore.ttypes import GetTableRequest -from hive_metastore.ttypes import GetPartitionsByNamesRequest -from hive_metastore.ttypes import TruncateTableRequest -from hive_metastore.ttypes import Table -from hive_metastore.ttypes import StorageDescriptor -from hive_metastore.ttypes import SerDeInfo -from hive_metastore.ttypes import UpdateTransactionalStatsRequest -from hive_metastore.ttypes import WriteNotificationLogBatchRequest - -from tests.util.event_processor_utils import EventProcessorUtils +from impala_thrift_gen.hive_metastore.ttypes import ( + Database, + FieldSchema, + FindNextCompactRequest, + GetPartitionsByNamesRequest, + GetTableRequest, + SerDeInfo, + StorageDescriptor, + Table, + TruncateTableRequest, + UpdateTransactionalStatsRequest, + WriteNotificationLogBatchRequest, +) from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.common.impala_test_suite import ImpalaTestSuite +from tests.util.event_processor_utils import EventProcessorUtils from tests.util.filesystem_utils import IS_HDFS, IS_OZONE diff --git a/tests/custom_cluster/test_query_log.py b/tests/custom_cluster/test_query_log.py index 4095e0d7b..23513fbd4 100644 --- a/tests/custom_cluster/test_query_log.py +++ b/tests/custom_cluster/test_query_log.py @@ -16,19 +16,19 @@ # under the License. from __future__ import absolute_import, division, print_function - +from getpass import getuser import os +from random import choice, randint +from signal import SIGRTMIN import string from time import sleep, time -from getpass import getuser -from ImpalaService import ImpalaHiveServer2Service -from random import choice, randint -from signal import SIGRTMIN -from TCLIService import TCLIService +from thrift.protocol import TBinaryProtocol from thrift.transport.TSocket import TSocket from thrift.transport.TTransport import TBufferedTransport -from thrift.protocol import TBinaryProtocol + +from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service +from impala_thrift_gen.TCLIService import TCLIService from tests.common.cluster_config import impalad_admission_ctrl_config_args from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.common.impala_connection import FINISHED @@ -39,9 +39,10 @@ from tests.common.wm_test_suite import WorkloadManagementTestSuite from tests.util.retry import retry from tests.util.workload_management import ( assert_query, - WM_DB, QUERY_TBL_LOG, - redaction_rules_file) + redaction_rules_file, + WM_DB, +) class TestQueryLogTableBasic(WorkloadManagementTestSuite): diff --git a/tests/custom_cluster/test_query_retries.py b/tests/custom_cluster/test_query_retries.py index 3357dfe5e..c94bc53b5 100644 --- a/tests/custom_cluster/test_query_retries.py +++ b/tests/custom_cluster/test_query_retries.py @@ -22,21 +22,29 @@ # TODO: Add a test that cancels queries while a retry is running from __future__ import absolute_import, division, print_function -from builtins import map, range -import pytest +from random import randint import re import time -from random import randint +from builtins import map, range +import pytest -from RuntimeProfile.ttypes import TRuntimeProfileFormat -from tests.common.impala_connection import ( - ERROR, FINISHED, IMPALA_CONNECTION_EXCEPTION, RUNNING) -from tests.common.impala_test_suite import ImpalaTestSuite, LOG +from impala_thrift_gen.RuntimeProfile.ttypes import TRuntimeProfileFormat from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.common.errors import Timeout -from tests.common.skip import (SkipIfEC, SkipIfBuildType, SkipIfFS, - SkipIfNotHdfsMinicluster) +from tests.common.impala_connection import ( + ERROR, + FINISHED, + IMPALA_CONNECTION_EXCEPTION, + RUNNING, +) +from tests.common.impala_test_suite import ImpalaTestSuite, LOG +from tests.common.skip import ( + SkipIfBuildType, + SkipIfEC, + SkipIfFS, + SkipIfNotHdfsMinicluster, +) from tests.common.test_dimensions import add_mandatory_exec_option # The BE krpc port of the impalad to simulate rpc or disk errors in tests. diff --git a/tests/custom_cluster/test_restart_services.py b/tests/custom_cluster/test_restart_services.py index 2f798a5c8..f2b3e7a48 100644 --- a/tests/custom_cluster/test_restart_services.py +++ b/tests/custom_cluster/test_restart_services.py @@ -16,28 +16,31 @@ # under the License. from __future__ import absolute_import, division, print_function -from builtins import range import logging import os -import pytest -import psutil import re import signal import socket -import time -import threading - from subprocess import check_call -from tests.common.environ import build_flavor_timeout +import threading +import time from time import sleep +from builtins import range from impala.error import HiveServer2Error -from TCLIService import TCLIService +import psutil +import pytest +from impala_thrift_gen.TCLIService import TCLIService from tests.common.custom_cluster_test_suite import CustomClusterTestSuite +from tests.common.environ import build_flavor_timeout from tests.common.impala_connection import ( - ERROR, FINISHED, IMPALA_CONNECTION_EXCEPTION, RUNNING) -from tests.common.skip import SkipIfNotHdfsMinicluster, SkipIfFS + ERROR, + FINISHED, + IMPALA_CONNECTION_EXCEPTION, + RUNNING, +) +from tests.common.skip import SkipIfFS, SkipIfNotHdfsMinicluster from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session LOG = logging.getLogger(__name__) diff --git a/tests/custom_cluster/test_set_and_unset.py b/tests/custom_cluster/test_set_and_unset.py index 2abe2586b..52c128c19 100644 --- a/tests/custom_cluster/test_set_and_unset.py +++ b/tests/custom_cluster/test_set_and_unset.py @@ -16,12 +16,14 @@ # under the License. from __future__ import absolute_import, division, print_function + import pytest +from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service +from impala_thrift_gen.TCLIService import TCLIService from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session -from TCLIService import TCLIService -from ImpalaService import ImpalaHiveServer2Service + class TestSetAndUnset(CustomClusterTestSuite, HS2TestSuite): """ diff --git a/tests/custom_cluster/test_statestored_ha.py b/tests/custom_cluster/test_statestored_ha.py index 665ba4d57..9f5f747c1 100644 --- a/tests/custom_cluster/test_statestored_ha.py +++ b/tests/custom_cluster/test_statestored_ha.py @@ -17,23 +17,23 @@ from __future__ import absolute_import, division, print_function import logging -import pytest import time - -from tests.common.custom_cluster_test_suite import CustomClusterTestSuite -from tests.common.environ import build_flavor_timeout, ImpalaTestClusterProperties -from tests.common.impala_cluster import ( - DEFAULT_CATALOG_SERVICE_PORT, DEFAULT_STATESTORE_SERVICE_PORT) -from tests.common.impala_connection import ( - ERROR, IMPALA_CONNECTION_EXCEPTION, RUNNING) -from tests.common.skip import SkipIfBuildType, SkipIfNotHdfsMinicluster from time import sleep +import pytest from thrift.protocol import TBinaryProtocol from thrift.transport import TSocket, TTransport -import StatestoreService.StatestoreSubscriber as Subscriber -import StatestoreService.StatestoreService as Statestore +import impala_thrift_gen.StatestoreService.StatestoreService as Statestore +import impala_thrift_gen.StatestoreService.StatestoreSubscriber as Subscriber +from tests.common.custom_cluster_test_suite import CustomClusterTestSuite +from tests.common.environ import build_flavor_timeout, ImpalaTestClusterProperties +from tests.common.impala_cluster import ( + DEFAULT_CATALOG_SERVICE_PORT, + DEFAULT_STATESTORE_SERVICE_PORT, +) +from tests.common.impala_connection import ERROR, IMPALA_CONNECTION_EXCEPTION, RUNNING +from tests.common.skip import SkipIfBuildType, SkipIfNotHdfsMinicluster LOG = logging.getLogger('statestored_ha_test') diff --git a/tests/custom_cluster/test_workload_mgmt_init.py b/tests/custom_cluster/test_workload_mgmt_init.py index 7991dcd32..4191ff979 100644 --- a/tests/custom_cluster/test_workload_mgmt_init.py +++ b/tests/custom_cluster/test_workload_mgmt_init.py @@ -16,26 +16,26 @@ # under the License. from __future__ import absolute_import, division, print_function - +from logging import getLogger import os import re - from subprocess import CalledProcessError -from logging import getLogger -from SystemTables.ttypes import TQueryTableColumn +from impala_thrift_gen.SystemTables.ttypes import TQueryTableColumn from tests.common.custom_cluster_test_suite import ( + CustomClusterTestSuite, WORKLOAD_MGMT_IMPALAD_FLAGS, - CustomClusterTestSuite) +) from tests.common.test_dimensions import hs2_client_protocol_dimension from tests.common.test_vector import HS2 from tests.util.workload_management import ( assert_query, - WM_DB, - QUERY_TBL_LOG_NAME, - QUERY_TBL_LOG, + QUERY_TBL_LIVE, QUERY_TBL_LIVE_NAME, - QUERY_TBL_LIVE) + QUERY_TBL_LOG, + QUERY_TBL_LOG_NAME, + WM_DB, +) LOG = getLogger(__name__) QUERY_TBL_ALL = "{},{}".format(QUERY_TBL_LOG_NAME, QUERY_TBL_LIVE_NAME) diff --git a/tests/custom_cluster/test_workload_mgmt_sql_details.py b/tests/custom_cluster/test_workload_mgmt_sql_details.py index 7938fc03c..b2839d8d1 100644 --- a/tests/custom_cluster/test_workload_mgmt_sql_details.py +++ b/tests/custom_cluster/test_workload_mgmt_sql_details.py @@ -19,7 +19,7 @@ from __future__ import absolute_import, division, print_function import os -from SystemTables.ttypes import TQueryTableColumn +from impala_thrift_gen.SystemTables.ttypes import TQueryTableColumn from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.common.test_dimensions import hs2_client_protocol_dimension from tests.common.wm_test_suite import WorkloadManagementTestSuite diff --git a/tests/hs2/hs2_test_suite.py b/tests/hs2/hs2_test_suite.py index a79743749..a1e50b162 100644 --- a/tests/hs2/hs2_test_suite.py +++ b/tests/hs2/hs2_test_suite.py @@ -18,17 +18,19 @@ # Superclass of all HS2 tests containing commonly used functions. from __future__ import absolute_import, division, print_function -from builtins import range from getpass import getuser -from TCLIService import TCLIService -from ImpalaService import ImpalaHiveServer2Service +import sys +from time import sleep, time + +from builtins import range +from thrift.protocol import TBinaryProtocol from thrift.transport.TSocket import TSocket from thrift.transport.TTransport import TBufferedTransport -from thrift.protocol import TBinaryProtocol -from tests.common.impala_test_suite import ImpalaTestSuite, IMPALAD_HS2_HOST_PORT + +from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service +from impala_thrift_gen.TCLIService import TCLIService +from tests.common.impala_test_suite import IMPALAD_HS2_HOST_PORT, ImpalaTestSuite from tests.common.test_result_verifier import error_msg_startswith -from time import sleep, time -import sys def add_session_helper(self, protocol_version, conf_overlay, close_session, fn): diff --git a/tests/hs2/test_fetch.py b/tests/hs2/test_fetch.py index 4392ed5eb..7b40ef502 100644 --- a/tests/hs2/test_fetch.py +++ b/tests/hs2/test_fetch.py @@ -17,13 +17,17 @@ # from __future__ import absolute_import, division, print_function -import pytest import re -from tests.hs2.hs2_test_suite import (HS2TestSuite, needs_session, - create_op_handle_without_secret) -from TCLIService import TCLIService, constants -from TCLIService.ttypes import TTypeId +import pytest + +from impala_thrift_gen.TCLIService import constants, TCLIService +from impala_thrift_gen.TCLIService.ttypes import TTypeId +from tests.hs2.hs2_test_suite import ( + create_op_handle_without_secret, + HS2TestSuite, + needs_session, +) # Simple test to make sure all the HS2 types are supported for both the row and diff --git a/tests/hs2/test_fetch_first.py b/tests/hs2/test_fetch_first.py index 93f428a9b..c597538cc 100644 --- a/tests/hs2/test_fetch_first.py +++ b/tests/hs2/test_fetch_first.py @@ -21,13 +21,14 @@ # succeed as long all previously fetched rows fit into the bounded result cache. from __future__ import absolute_import, division, print_function + from builtins import range import pytest -from ImpalaService import ImpalaHiveServer2Service -from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session -from TCLIService import TCLIService +from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service +from impala_thrift_gen.TCLIService import TCLIService from tests.common.impala_cluster import ImpalaCluster +from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session class TestFetchFirst(HS2TestSuite): diff --git a/tests/hs2/test_fetch_timeout.py b/tests/hs2/test_fetch_timeout.py index ea0ec5929..148ebf31d 100644 --- a/tests/hs2/test_fetch_timeout.py +++ b/tests/hs2/test_fetch_timeout.py @@ -16,12 +16,11 @@ # under the License. from __future__ import absolute_import, division, print_function -from time import sleep -from time import time -from tests.common.errors import Timeout -from tests.hs2.hs2_test_suite import (HS2TestSuite, needs_session) -from TCLIService import TCLIService +from time import sleep, time +from impala_thrift_gen.TCLIService import TCLIService +from tests.common.errors import Timeout +from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session # Tests for the query option FETCH_ROWS_TIMEOUT_MS, which is the maximum amount of # time, in milliseconds, a fetch rows request (TFetchResultsReq) from the client should diff --git a/tests/hs2/test_hs2.py b/tests/hs2/test_hs2.py index 672cb4f52..c5206346b 100644 --- a/tests/hs2/test_hs2.py +++ b/tests/hs2/test_hs2.py @@ -18,33 +18,37 @@ # Client tests for Impala's HiveServer2 interface from __future__ import absolute_import, division, print_function -from builtins import range -from getpass import getuser from contextlib import contextmanager +from getpass import getuser import json import logging -import pytest import random import threading import time import uuid + +from builtins import range import impala.dbapi as impyla +import pytest +from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service +from impala_thrift_gen.TCLIService import TCLIService from tests.common.impala_test_suite import IMPALAD_HOSTNAME, IMPALAD_HS2_HTTP_PORT +from tests.common.skip import SkipIfDockerizedCluster +from tests.hs2.hs2_test_suite import ( + create_op_handle_without_secret, + create_session_handle_without_secret, + HS2TestSuite, + needs_session, + needs_session_cluster_properties, + operation_id_to_query_id, +) try: from urllib.request import urlopen except ImportError: from urllib2 import urlopen -from ImpalaService import ImpalaHiveServer2Service -from tests.common.environ import ImpalaTestClusterProperties -from tests.common.skip import SkipIfDockerizedCluster -from tests.hs2.hs2_test_suite import (HS2TestSuite, needs_session, - operation_id_to_query_id, create_session_handle_without_secret, - create_op_handle_without_secret, needs_session_cluster_properties) -from TCLIService import TCLIService - LOG = logging.getLogger('test_hs2') SQLSTATE_GENERAL_ERROR = "HY000" diff --git a/tests/hs2/test_json_endpoints.py b/tests/hs2/test_json_endpoints.py index b531fcc45..30c0c1fff 100644 --- a/tests/hs2/test_json_endpoints.py +++ b/tests/hs2/test_json_endpoints.py @@ -19,18 +19,20 @@ from __future__ import absolute_import, division, print_function import json -import pytest from time import time +import pytest + +from impala_thrift_gen.TCLIService import TCLIService +from tests.common.environ import IS_DOCKERIZED_TEST_CLUSTER +from tests.common.impala_cluster import ImpalaCluster +from tests.hs2.hs2_test_suite import HS2TestSuite + try: from urllib.request import urlopen except ImportError: from urllib2 import urlopen -from tests.common.environ import IS_DOCKERIZED_TEST_CLUSTER -from tests.common.impala_cluster import ImpalaCluster -from tests.hs2.hs2_test_suite import HS2TestSuite -from TCLIService import TCLIService class TestJsonEndpoints(HS2TestSuite): def _get_json_queries(self, http_addr): diff --git a/tests/metadata/test_compute_stats.py b/tests/metadata/test_compute_stats.py index 62b91c587..41763e5fa 100644 --- a/tests/metadata/test_compute_stats.py +++ b/tests/metadata/test_compute_stats.py @@ -16,23 +16,28 @@ # under the License. from __future__ import absolute_import, division, print_function -from builtins import range import os + +from builtins import range import pytest -from hive_metastore.ttypes import ( - ColumnStatistics, ColumnStatisticsDesc, ColumnStatisticsData, - ColumnStatisticsObj, StringColumnStatsData) +from impala_thrift_gen.CatalogObjects.ttypes import THdfsCompression +from impala_thrift_gen.hive_metastore.ttypes import ( + ColumnStatistics, + ColumnStatisticsData, + ColumnStatisticsDesc, + ColumnStatisticsObj, + StringColumnStatsData, +) from tests.common.environ import ImpalaTestClusterProperties from tests.common.impala_cluster import ImpalaCluster from tests.common.impala_test_suite import ImpalaTestSuite -from tests.common.skip import SkipIfFS, SkipIfLocal, SkipIfCatalogV2 +from tests.common.skip import SkipIfCatalogV2, SkipIfFS, SkipIfLocal from tests.common.test_dimensions import ( create_exec_option_dimension, create_single_exec_option_dimension, - create_uncompressed_text_dimension) -from CatalogObjects.ttypes import THdfsCompression - + create_uncompressed_text_dimension, +) IMPALA_TEST_CLUSTER_PROPERTIES = ImpalaTestClusterProperties.get_instance() diff --git a/tests/query_test/test_acid.py b/tests/query_test/test_acid.py index a25cb3867..277bd2da1 100644 --- a/tests/query_test/test_acid.py +++ b/tests/query_test/test_acid.py @@ -19,13 +19,18 @@ from __future__ import absolute_import, division, print_function import os -import pytest +from subprocess import check_call import time -from hive_metastore.ttypes import CommitTxnRequest, LockType, OpenTxnRequest -from subprocess import check_call +import pytest + +from impala_thrift_gen.hive_metastore.ttypes import ( + CommitTxnRequest, + LockType, + OpenTxnRequest, +) from tests.common.impala_test_suite import ImpalaTestSuite -from tests.common.skip import SkipIf, SkipIfHive2, SkipIfCatalogV2, SkipIfFS +from tests.common.skip import SkipIf, SkipIfCatalogV2, SkipIfFS, SkipIfHive2 from tests.common.test_dimensions import create_single_exec_option_dimension from tests.util.acid_txn import AcidTxn diff --git a/tests/query_test/test_cancellation.py b/tests/query_test/test_cancellation.py index f6ce4c75b..efda79e8b 100644 --- a/tests/query_test/test_cancellation.py +++ b/tests/query_test/test_cancellation.py @@ -19,18 +19,19 @@ # from __future__ import absolute_import, division, print_function -from builtins import range -import pytest from threading import Thread from time import sleep -from RuntimeProfile.ttypes import TRuntimeProfileFormat + +from builtins import range +import pytest + +from impala_thrift_gen.RuntimeProfile.ttypes import TRuntimeProfileFormat +from tests.common.impala_connection import MinimalHS2Connection +from tests.common.impala_test_suite import IMPALAD_HS2_HOST_PORT, ImpalaTestSuite from tests.common.test_dimensions import add_mandatory_exec_option from tests.common.test_vector import ImpalaTestDimension -from tests.common.impala_test_suite import ImpalaTestSuite -from tests.common.impala_test_suite import IMPALAD_HS2_HOST_PORT from tests.util.cancel_util import cancel_query_and_validate_state from tests.verifiers.metric_verifier import MetricVerifier -from tests.common.impala_connection import MinimalHS2Connection # PRIMARY KEY for lineitem LINEITEM_PK = 'l_orderkey, l_partkey, l_suppkey, l_linenumber' diff --git a/tests/query_test/test_iceberg.py b/tests/query_test/test_iceberg.py index ad13b1dc0..abc92996b 100644 --- a/tests/query_test/test_iceberg.py +++ b/tests/query_test/test_iceberg.py @@ -16,37 +16,37 @@ # under the License. from __future__ import absolute_import, division, print_function -from builtins import range from collections import defaultdict, namedtuple import datetime +import json import logging import os -import pytest -import pytz import random import re -import time - from subprocess import check_call, check_output -# noinspection PyUnresolvedReferences -from parquet.ttypes import ConvertedType +import time from avro.datafile import DataFileReader from avro.io import DatumReader -import json +from builtins import range +import pytest +import pytz -from tests.common.impala_connection import IMPALA_CONNECTION_EXCEPTION +# noinspection PyUnresolvedReferences +from impala_thrift_gen.parquet.ttypes import ConvertedType +from tests.common.file_utils import ( + create_iceberg_table_from_directory, + create_table_from_parquet, +) from tests.common.iceberg_test_suite import IcebergTestSuite -from tests.common.skip import SkipIf, SkipIfFS, SkipIfDockerizedCluster +from tests.common.impala_connection import IMPALA_CONNECTION_EXCEPTION +from tests.common.skip import SkipIf, SkipIfDockerizedCluster, SkipIfFS from tests.common.test_dimensions import add_exec_option_dimension from tests.common.test_result_verifier import error_msg_startswith -from tests.common.file_utils import ( - create_iceberg_table_from_directory, - create_table_from_parquet) from tests.shell.util import run_impala_shell_cmd -from tests.util.filesystem_utils import get_fs_path, IS_HDFS, WAREHOUSE, FILESYSTEM_PREFIX +from tests.util.filesystem_utils import FILESYSTEM_PREFIX, get_fs_path, IS_HDFS, WAREHOUSE from tests.util.get_parquet_metadata import get_parquet_metadata -from tests.util.iceberg_util import cast_ts, quote, get_snapshots, IcebergCatalogs +from tests.util.iceberg_util import cast_ts, get_snapshots, IcebergCatalogs, quote LOG = logging.getLogger(__name__) diff --git a/tests/query_test/test_insert_parquet.py b/tests/query_test/test_insert_parquet.py index a90f456f7..36399feb7 100644 --- a/tests/query_test/test_insert_parquet.py +++ b/tests/query_test/test_insert_parquet.py @@ -18,27 +18,35 @@ # Targeted Impala insert tests from __future__ import absolute_import, division, print_function -from builtins import map, range, round -import os - from collections import namedtuple -from datetime import datetime, date +from datetime import date, datetime from decimal import Decimal +import os from subprocess import check_call -from parquet.ttypes import ColumnOrder, SortingColumn, TypeDefinedOrder, ConvertedType +from builtins import map, range, round + +from impala_thrift_gen.parquet.ttypes import ( + ColumnOrder, + ConvertedType, + SortingColumn, + TypeDefinedOrder, +) from tests.common.environ import impalad_basedir from tests.common.impala_test_suite import ImpalaTestSuite from tests.common.parametrize import UniqueDatabase from tests.common.skip import SkipIfFS, SkipIfLocal from tests.common.test_dimensions import ( add_exec_option_dimension, - create_exec_option_dimension) + create_exec_option_dimension, +) from tests.common.test_result_verifier import verify_query_result_is_equal from tests.common.test_vector import ImpalaTestDimension from tests.util.filesystem_utils import get_fs_path, WAREHOUSE -from tests.util.get_parquet_metadata import (decode_stats_value, - get_parquet_metadata_from_hdfs_folder) +from tests.util.get_parquet_metadata import ( + decode_stats_value, + get_parquet_metadata_from_hdfs_folder, +) PARQUET_CODECS = ['none', 'snappy', 'gzip', 'zstd', 'lz4'] IMPALA_HOME = os.environ['IMPALA_HOME'] diff --git a/tests/query_test/test_observability.py b/tests/query_test/test_observability.py index 5394e22f5..cfdad05ec 100644 --- a/tests/query_test/test_observability.py +++ b/tests/query_test/test_observability.py @@ -18,6 +18,12 @@ from __future__ import absolute_import, division, print_function from collections import defaultdict from datetime import datetime +import re +from time import sleep, time + +import pytest + +from impala_thrift_gen.RuntimeProfile.ttypes import TRuntimeProfileFormat from tests.common.impala_cluster import ImpalaCluster from tests.common.impala_connection import IMPALA_CONNECTION_EXCEPTION from tests.common.impala_test_suite import ImpalaTestSuite @@ -25,10 +31,6 @@ from tests.common.skip import SkipIfFS, SkipIfLocal, SkipIfNotHdfsMinicluster from tests.common.test_vector import HS2 from tests.util.filesystem_utils import WAREHOUSE from tests.util.parse_util import get_duration_us_from_str -from time import sleep, time -from RuntimeProfile.ttypes import TRuntimeProfileFormat -import pytest -import re class TestObservability(ImpalaTestSuite): diff --git a/tests/query_test/test_parquet_bloom_filter.py b/tests/query_test/test_parquet_bloom_filter.py index 1db10efe5..112fba2b3 100644 --- a/tests/query_test/test_parquet_bloom_filter.py +++ b/tests/query_test/test_parquet_bloom_filter.py @@ -16,21 +16,18 @@ # under the License. from __future__ import absolute_import, division, print_function -from builtins import range +from collections import namedtuple import math import os - -from collections import namedtuple -from parquet.ttypes import BloomFilterHeader from subprocess import check_call +from builtins import range + +from impala_thrift_gen.parquet.ttypes import BloomFilterHeader from tests.common.file_utils import create_table_and_copy_files from tests.common.impala_test_suite import ImpalaTestSuite from tests.util.filesystem_utils import get_fs_path -from tests.util.get_parquet_metadata import ( - get_parquet_metadata, - read_serialized_object -) +from tests.util.get_parquet_metadata import get_parquet_metadata, read_serialized_object class TestParquetBloomFilter(ImpalaTestSuite): diff --git a/tests/query_test/test_parquet_page_index.py b/tests/query_test/test_parquet_page_index.py index 1d0f1f6b4..e1f735fd0 100644 --- a/tests/query_test/test_parquet_page_index.py +++ b/tests/query_test/test_parquet_page_index.py @@ -18,21 +18,26 @@ # Targeted Impala insert tests from __future__ import absolute_import, division, print_function +from collections import namedtuple import os import random import string - -from collections import namedtuple from subprocess import check_call -from parquet.ttypes import BoundaryOrder, ColumnIndex, OffsetIndex, PageHeader, PageType +from impala_thrift_gen.parquet.ttypes import ( + BoundaryOrder, + ColumnIndex, + OffsetIndex, + PageHeader, + PageType, +) from tests.common.impala_test_suite import ImpalaTestSuite from tests.common.skip import SkipIfLocal from tests.util.filesystem_utils import get_fs_path from tests.util.get_parquet_metadata import ( decode_stats_value, get_parquet_metadata, - read_serialized_object + read_serialized_object, ) PAGE_INDEX_MAX_STRING_LENGTH = 64 diff --git a/tests/query_test/test_query_opts.py b/tests/query_test/test_query_opts.py index 71a0b73f2..4d81ecfec 100644 --- a/tests/query_test/test_query_opts.py +++ b/tests/query_test/test_query_opts.py @@ -21,12 +21,13 @@ # timeout). from __future__ import absolute_import, division, print_function -from TCLIService import TCLIService +from impala_thrift_gen.TCLIService import TCLIService from tests.common.impala_test_suite import ImpalaTestSuite from tests.common.test_dimensions import create_exec_option_dimension from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session + class TestQueryOptions(ImpalaTestSuite): @classmethod def add_test_dimensions(cls): diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py index ad4bdbe15..410980d8f 100644 --- a/tests/query_test/test_scanners.py +++ b/tests/query_test/test_scanners.py @@ -22,17 +22,19 @@ # explode. from __future__ import absolute_import, division, print_function -from builtins import range +from copy import deepcopy import os -import pytest import random import re -import tempfile -from copy import deepcopy -from parquet.ttypes import ConvertedType from subprocess import check_call +import tempfile +from builtins import range +import pytest + +from impala_thrift_gen.parquet.ttypes import ConvertedType from testdata.common import widetable +from tests.common.file_utils import create_table_and_copy_files, create_table_from_parquet from tests.common.impala_test_suite import ImpalaTestSuite, LOG from tests.common.skip import ( SkipIf, @@ -41,21 +43,18 @@ from tests.common.skip import ( SkipIfHive2, SkipIfHive3, SkipIfLocal, - SkipIfNotHdfsMinicluster) + SkipIfNotHdfsMinicluster, +) from tests.common.test_dimensions import ( add_exec_option_dimension, add_mandatory_exec_option, - create_single_exec_option_dimension, create_exec_option_dimension, - create_uncompressed_text_dimension) -from tests.common.file_utils import ( - create_table_from_parquet, - create_table_and_copy_files) -from tests.common.test_result_verifier import ( - QueryTestResult, - parse_result_rows) + create_single_exec_option_dimension, + create_uncompressed_text_dimension, +) +from tests.common.test_result_verifier import parse_result_rows, QueryTestResult from tests.common.test_vector import ImpalaTestDimension -from tests.util.filesystem_utils import IS_HDFS, get_fs_path +from tests.util.filesystem_utils import get_fs_path, IS_HDFS from tests.util.get_parquet_metadata import get_parquet_metadata from tests.util.parse_util import get_bytes_summary_stats_counter from tests.util.test_file_parser import QueryTestSectionReader diff --git a/tests/statestore/test_statestore.py b/tests/statestore/test_statestore.py index b7a6b28a3..e6417721a 100644 --- a/tests/statestore/test_statestore.py +++ b/tests/statestore/test_statestore.py @@ -16,38 +16,38 @@ # under the License. from __future__ import absolute_import, division, print_function -from builtins import range from collections import defaultdict import json import logging import socket import threading -import traceback import time +import traceback import uuid -try: - from urllib.request import urlopen -except ImportError: - from urllib2 import urlopen - -from Types.ttypes import TNetworkAddress +from builtins import range from thrift.protocol import TBinaryProtocol from thrift.server.TServer import TServer -from thrift.transport import TSocket -from thrift.transport import TTransport - -import StatestoreService.StatestoreSubscriber as Subscriber -import StatestoreService.StatestoreService as Statestore -from StatestoreService.StatestoreSubscriber import TUpdateStateResponse -from StatestoreService.StatestoreSubscriber import TTopicRegistration -from ErrorCodes.ttypes import TErrorCode -from Status.ttypes import TStatus - +from thrift.transport import TSocket, TTransport + +from impala_thrift_gen.ErrorCodes.ttypes import TErrorCode +import impala_thrift_gen.StatestoreService.StatestoreService as Statestore +import impala_thrift_gen.StatestoreService.StatestoreSubscriber as Subscriber +from impala_thrift_gen.StatestoreService.StatestoreSubscriber import ( + TTopicRegistration, + TUpdateStateResponse, +) +from impala_thrift_gen.Status.ttypes import TStatus +from impala_thrift_gen.Types.ttypes import TNetworkAddress from tests.common.base_test_suite import BaseTestSuite from tests.common.environ import build_flavor_timeout from tests.common.skip import SkipIfDockerizedCluster +try: + from urllib.request import urlopen +except ImportError: + from urllib2 import urlopen + LOG = logging.getLogger('test_statestore') # Tests for the statestore. The StatestoreSubscriber class is a skeleton implementation of diff --git a/tests/util/acid_txn.py b/tests/util/acid_txn.py index f1830f2db..3e28598a7 100644 --- a/tests/util/acid_txn.py +++ b/tests/util/acid_txn.py @@ -16,14 +16,28 @@ # under the License. from __future__ import absolute_import, division, print_function -from tests.util.thrift_util import create_transport -from hive_metastore import ThriftHiveMetastore -from hive_metastore.ttypes import (AbortTxnRequest, AllocateTableWriteIdsRequest, - CheckLockRequest, CommitTxnRequest, GetValidWriteIdsRequest, HeartbeatRequest, - LockComponent, LockLevel, LockType, LockRequest, OpenTxnRequest, ShowLocksRequest, - TruncateTableRequest, UnlockRequest) + from thrift.protocol import TBinaryProtocol +from impala_thrift_gen.hive_metastore import ThriftHiveMetastore +from impala_thrift_gen.hive_metastore.ttypes import ( + AbortTxnRequest, + AllocateTableWriteIdsRequest, + CheckLockRequest, + CommitTxnRequest, + GetValidWriteIdsRequest, + HeartbeatRequest, + LockComponent, + LockLevel, + LockRequest, + LockType, + OpenTxnRequest, + ShowLocksRequest, + TruncateTableRequest, + UnlockRequest, +) +from tests.util.thrift_util import create_transport + # HMS config metastore_host = "localhost" metastore_port = "9083" diff --git a/tests/util/event_processor_utils.py b/tests/util/event_processor_utils.py index 1b1a23c8e..a270f3eb3 100644 --- a/tests/util/event_processor_utils.py +++ b/tests/util/event_processor_utils.py @@ -20,14 +20,15 @@ # succeeded by querying Impala, or vice versa. from __future__ import absolute_import, division, print_function +import json import logging -import requests import time -import json -from hive_metastore.ttypes import NotificationEventRequest -from tests.common.impala_cluster import ImpalaCluster +import requests + +from impala_thrift_gen.hive_metastore.ttypes import NotificationEventRequest from tests.common.custom_cluster_test_suite import CustomClusterTestSuite +from tests.common.impala_cluster import ImpalaCluster LOG = logging.getLogger('event_processor_utils') LOG.setLevel(level=logging.DEBUG) diff --git a/tests/util/get_parquet_metadata.py b/tests/util/get_parquet_metadata.py index 876ffff08..b25f08ede 100644 --- a/tests/util/get_parquet_metadata.py +++ b/tests/util/get_parquet_metadata.py @@ -16,19 +16,20 @@ # under the License. from __future__ import absolute_import, division, print_function -from builtins import map -import os -import struct -import sys - from datetime import date, datetime, time, timedelta from decimal import Decimal from functools import reduce -from parquet.ttypes import ColumnIndex, FileMetaData, OffsetIndex, PageHeader, Type +import os +import struct from subprocess import check_call +import sys + +from builtins import map from thrift.protocol import TCompactProtocol from thrift.transport import TTransport +from impala_thrift_gen.parquet.ttypes import FileMetaData, Type + PARQUET_VERSION_NUMBER = b'PAR1' diff --git a/tests/util/parse_util.py b/tests/util/parse_util.py index 0d2156f51..370ac1d03 100644 --- a/tests/util/parse_util.py +++ b/tests/util/parse_util.py @@ -203,7 +203,7 @@ def get_bytes_summary_stats_counter(counter_name, runtime_profile): """ # This requires the Thrift definitions to be generated. We limit the scope of the import # to allow tools like the stress test to import this file without building Impala. - from RuntimeProfile.ttypes import TSummaryStatsCounter + from impala_thrift_gen.RuntimeProfile.ttypes import TSummaryStatsCounter regex_summary_stat = re.compile(r"""\( Avg:[^\(]*\((?P<avg>[0-9]+)\)\s;\s # Matches Avg: [?].[?] [?]B (?) @@ -256,7 +256,7 @@ def get_time_summary_stats_counter(counter_name, runtime_profile): """ # This requires the Thrift definitions to be generated. We limit the scope of the import # to allow tools like the stress test to import this file without building Impala. - from RuntimeProfile.ttypes import TSummaryStatsCounter + from impala_thrift_gen.RuntimeProfile.ttypes import TSummaryStatsCounter regex_summary_stat = re.compile(r"""\( Avg:\s(?P<avg>.*)\s;\s # Matches Avg: ? ; diff --git a/tests/util/workload_management.py b/tests/util/workload_management.py index 5053a2a2c..e98249efa 100644 --- a/tests/util/workload_management.py +++ b/tests/util/workload_management.py @@ -24,7 +24,7 @@ import requests from datetime import datetime from time import sleep, time -from SystemTables.ttypes import TQueryTableColumn +from impala_thrift_gen.SystemTables.ttypes import TQueryTableColumn from tests.util.assert_time import assert_time_str, convert_to_milliseconds from tests.util.memory import assert_byte_str, convert_to_bytes
