This is an automated email from the ASF dual-hosted git repository.
boroknagyz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 5071f54a4ca9456efb302f737a13a9eb14710f5f
Author: Joe McDonnell <joemcdonn...@cloudera.com>
AuthorDate: Wed Feb 21 10:47:57 2024 -0800
IMPALA-12825: Install thrift into the impala-python virtualenv
impala-python currently gets its Thrift from the toolchain
by adding the appropriate Thrift toolchain directories to
the PYTHONPATH. This is a problem when switching to Python 3,
because the toolchain Thrift was built with Python 2 and
this can produce complicated bugs. In general, it is also
not a good idea to get Python dependencies from the toolchain.
This switches to installing Thrift into the impala-python
virtualenv, which lets the different Python versions have
their own copy of compiled files.
Testing:
- Ran a core job
Change-Id: Ib36e8a1ce8d446b69b08e81ea458f95c158e28f5
Reviewed-on: http://gerrit.cloudera.org:8080/21046
Reviewed-by: Michael Smith <michael.sm...@cloudera.com>
Reviewed-by: Wenzhe Zhou <wz...@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com>
---
README-build.md | 2 +-
bin/impala-config.sh | 9 ++++++---
bin/set-pythonpath.sh | 14 --------------
infra/python/deps/requirements.txt | 1 +
4 files changed, 8 insertions(+), 18 deletions(-)
diff --git a/README-build.md b/README-build.md
index d96827006..798b7f7d0 100644
--- a/README-build.md
+++ b/README-build.md
@@ -35,7 +35,7 @@ can do so through the environment variables and scripts
listed below.
| IMPALA_JDK_VERSION | "system" | Set to 8 or 11 to select a system Java
version. Default will set JAVA_HOME based on the javac symlink in PATH. |
| JAVA | "${JAVA_HOME}/bin/java" | Java binary location. |
| CLASSPATH | | See bin/set-classpath.sh for details. |
-| PYTHONPATH | Will be changed to include: "${IMPALA_HOME}/shell/gen-py"
"${IMPALA_HOME}/testdata"
"${THRIFT_PY_HOME}/python/lib/python2.7/site-packages" |
+| PYTHONPATH | | See bin/set-pythonpath.sh for details. |
## Source Directories for Impala
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 4543bc169..4e104d911 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -214,9 +214,12 @@ export IMPALA_SIMBA_JDBC_DRIVER_VERSION=42-2.6.32.1041
# Thrift related environment variables.
# IMPALA_THRIFT_POM_VERSION is used to populate IMPALA_THRIFT_JAVA_VERSION and
# thrift.version in java/pom.xml.
-# If upgrading IMPALA_THRIFT_PY_VERSION, remember to also upgrade thrift
version in
-# shell/ext-py and shell/packaging/requirements.txt. IMPALA_THRIFT_PY_VERSION
is used
-# with Impyla and for the thrift compiler.
+# IMPALA_THRIFT_PY_VERSION is used to find the thrift compiler to produce
+# the generated Python code. The code that uses the generated Python code gets
+# the corresponding Thrift runtime library by pip installing thrift (and does
not
+# respect this version). If upgrading IMPALA_THRIFT_PY_VERSION, also upgrade
the
+# thrift version in shell/ext-py, shell/packaging/requirements.txt, and
+# infra/python/deps/requirements.txt.
export IMPALA_THRIFT_CPP_VERSION=0.16.0-p6
unset IMPALA_THRIFT_CPP_URL
if $USE_APACHE_HIVE; then
diff --git a/bin/set-pythonpath.sh b/bin/set-pythonpath.sh
index 9e1c933d1..183c8d6c9 100755
--- a/bin/set-pythonpath.sh
+++ b/bin/set-pythonpath.sh
@@ -28,17 +28,3 @@ export PYTHONPATH=${IMPALA_HOME}:${IMPALA_HOME}/bin
PYTHONPATH=${PYTHONPATH}:${IMPALA_HOME}/shell/gen-py
PYTHONPATH=${PYTHONPATH}:${IMPALA_HOME}/infra/python/env-gcc${IMPALA_GCC_VERSION}/lib
-
-# There should be just a single version of python that created the
-# site-packages directory. We find it by performing shell independent expansion
-# of the following pattern:
-# ${THRIFT_PY_HOME}/python/lib{64,}/python*/site-packages
-# Note: this could go wrong if we have used two different versions of
-# Python to build Thrift on this machine, and the first version is not
-# compatible with the second.
-for PYTHON_DIR in ${THRIFT_PY_HOME}/python/lib{64,}; do
- [[ -d ${PYTHON_DIR} ]] || continue
- for PKG_DIR in ${PYTHON_DIR}/python*/site-packages; do
- PYTHONPATH=${PYTHONPATH}:${PKG_DIR}/
- done
-done
diff --git a/infra/python/deps/requirements.txt
b/infra/python/deps/requirements.txt
index fe61de14b..c73adcfcc 100644
--- a/infra/python/deps/requirements.txt
+++ b/infra/python/deps/requirements.txt
@@ -64,6 +64,7 @@ sqlparse == 0.3.1
texttable == 0.8.3
virtualenv == 16.7.10
avro==1.10.2
+thrift==0.16.0
# Required for Kudu:
Cython == 0.29.14
