Repository: kudu Updated Branches: refs/heads/master da28d0aee -> 131c5d7af
Remove Pandas as a Python Client Requirement This is to remove the pandas dependency for users. We will put the pandas dependency in the jenkins script build-and-test.sh as a workaround. This is based on the initial work done here https://gerrit.cloudera.org/#/c/10809/4 Change-Id: I2cce7d529a804e712ccc32fb2774f5d0ee9dbdf8 Reviewed-on: http://gerrit.cloudera.org:8080/11557 Tested-by: Kudu Jenkins Reviewed-by: Jordan Birdsell <[email protected]> Reviewed-by: Adar Dembo <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/131c5d7a Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/131c5d7a Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/131c5d7a Branch: refs/heads/master Commit: 131c5d7af0a7c4b398ef01aaaea8378f18f84014 Parents: da28d0a Author: Andy Stadtler <[email protected]> Authored: Mon Oct 1 16:13:36 2018 -0500 Committer: Adar Dembo <[email protected]> Committed: Thu Oct 4 18:17:25 2018 +0000 ---------------------------------------------------------------------- build-support/jenkins/build-and-test.sh | 12 ++++++++++++ python/kudu/tests/test_scanner.py | 11 +++++++++-- python/requirements.txt | 6 ------ python/setup.py | 6 ++---- 4 files changed, 23 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/131c5d7a/build-support/jenkins/build-and-test.sh ---------------------------------------------------------------------- diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh index df0336c..6ba91c9 100755 --- a/build-support/jenkins/build-and-test.sh +++ b/build-support/jenkins/build-and-test.sh @@ -503,6 +503,14 @@ if [ "$BUILD_PYTHON" == "1" ]; then # pass in the current values of CC and CXX. CC=$CLANG CXX=$CLANG++ pip install $PIP_INSTALL_FLAGS -r requirements.txt + # We need to install Pandas manually since its not a required package but is needed + # to run all of the tests. + # pandas 0.18 dropped support for python 2.6. + # + # See https://pandas.pydata.org/pandas-docs/version/0.23.0/whatsnew.html#v0-18-0-march-13-2016 + # for more details. + pip install $PIP_INSTALL_FLAGS "pandas<0.18" + # Delete old Cython extensions to force them to be rebuilt. rm -Rf build kudu_python.egg-info kudu/*.so @@ -561,6 +569,10 @@ if [ "$BUILD_PYTHON3" == "1" ]; then # pass in the current values of CC and CXX. CC=$CLANG CXX=$CLANG++ pip install $PIP_INSTALL_FLAGS -r requirements.txt + # We need to install Pandas manually since its not a required package but is needed + # to run all of the tests. + pip install $PIP_INSTALL_FLAGS pandas + # Delete old Cython extensions to force them to be rebuilt. rm -Rf build kudu_python.egg-info kudu/*.so http://git-wip-us.apache.org/repos/asf/kudu/blob/131c5d7a/python/kudu/tests/test_scanner.py ---------------------------------------------------------------------- diff --git a/python/kudu/tests/test_scanner.py b/python/kudu/tests/test_scanner.py index 6d23d96..3c7fe57 100644 --- a/python/kudu/tests/test_scanner.py +++ b/python/kudu/tests/test_scanner.py @@ -328,6 +328,8 @@ class TestScanner(TestScanBase): self.assertEqual(sorted(scanner.read_all_tuples()), sorted(self.tuples)) + @pytest.mark.skipif(not (kudu.CLIENT_SUPPORTS_PANDAS), + reason="Pandas required to run this test.") def test_scanner_to_pandas_types(self): """ This test confirms that data types are converted as expected to Pandas. @@ -357,6 +359,8 @@ class TestScanner(TestScanBase): self.assertEqual(types[6], np.object) self.assertEqual(types[7], np.float32) + @pytest.mark.skipif(not (kudu.CLIENT_SUPPORTS_PANDAS), + reason="Pandas required to run this test.") def test_scanner_to_pandas_row_count(self): """ This test confirms that the record counts match between Pandas and the scanner. @@ -367,6 +371,8 @@ class TestScanner(TestScanBase): df = scanner.to_pandas() self.assertEqual(scanner_count, df.shape[0]) + @pytest.mark.skipif(not (kudu.CLIENT_SUPPORTS_PANDAS), + reason="Pandas required to run this test.") def test_scanner_to_pandas_index(self): """ This test confirms that an index is correctly applied. @@ -376,8 +382,9 @@ class TestScanner(TestScanBase): self.assertEqual(df.index.name, 'key') self.assertEqual(list(df.index), [1, 2]) - @pytest.mark.skipif(not(kudu.CLIENT_SUPPORTS_DECIMAL), - reason="Decimal is not supported on this client.") + @pytest.mark.skipif((not(kudu.CLIENT_SUPPORTS_PANDAS) and + (not(kudu.CLIENT_SUPPORTS_DECIMAL))), + reason="Pandas and Decimal support required to run this test.") def test_scanner_to_pandas_index(self): """ This test confirms that a decimal column is coerced to a double when specified. http://git-wip-us.apache.org/repos/asf/kudu/blob/131c5d7a/python/requirements.txt ---------------------------------------------------------------------- diff --git a/python/requirements.txt b/python/requirements.txt index eeb91af..5b6cbc1 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -8,12 +8,6 @@ pytest >=2.8,<3.3 # See https://pypi.org/project/pytest-timeout/#id5 for more details. pytest-timeout >=1.1.0,<1.2.1 -# pandas 0.18 dropped support for python 2.6. -# -# See https://pandas.pydata.org/pandas-docs/version/0.23.0/whatsnew.html#v0-18-0-march-13-2016 -# for more details. -pandas <0.18 - cython ==0.26.1 six unittest2 http://git-wip-us.apache.org/repos/asf/kudu/blob/131c5d7a/python/setup.py ---------------------------------------------------------------------- diff --git a/python/setup.py b/python/setup.py index 330f253..ef0013a 100644 --- a/python/setup.py +++ b/python/setup.py @@ -194,15 +194,13 @@ setup( # requirements.txt so that dependencies aren't downloaded at test-time # (when it's more difficult to override various pip installation options). # - # pytest 3.3 [1], pytest-timeout 1.2.1 [2], and pandas 0.18 [3] dropped + # pytest 3.3 [1] and pytest-timeout 1.2.1 [2] dropped # support for python 2.6. # # 1. https://docs.pytest.org/en/latest/changelog.html#id164 # 2. https://pypi.org/project/pytest-timeout/#id5 - # 3. https://pandas.pydata.org/pandas-docs/version/0.23.0/whatsnew.html#v0-18-0-march-13-2016 tests_require=['pytest >=2.8,<3.3', - 'pytest-timeout >=1.1.0,<1.2.1', - 'pandas <0.18'], + 'pytest-timeout >=1.1.0,<1.2.1'], install_requires=['cython >= 0.21', 'pytz', 'six'], description=DESCRIPTION,
