Repository: arrow Updated Branches: refs/heads/master 16c97592b -> dc103feaf
ARROW-557: [Python] Add option to explicitly opt in to HDFS tests, do not implicitly skip I have ``` $ py.test pyarrow/tests/test_hdfs.py ================================== test session starts ================================== platform linux2 -- Python 2.7.11, pytest-2.9.0, py-1.4.31, pluggy-0.3.1 rootdir: /home/wesm/code/arrow/python, inifile: collected 15 items pyarrow/tests/test_hdfs.py sssssssssssssss ``` But ``` $ py.test pyarrow/tests/test_hdfs.py --hdfs -v ================================== test session starts ================================== platform linux2 -- Python 2.7.11, pytest-2.9.0, py-1.4.31, pluggy-0.3.1 -- /home/wesm/anaconda3/envs/py27/bin/python cachedir: .cache rootdir: /home/wesm/code/arrow/python, inifile: collected 15 items pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_close PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_download_upload PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_file_context_manager PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_ls PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_mkdir PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_orphaned_file PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_read_multiple_parquet_files SKIPPED pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_read_whole_file PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_close PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_download_upload PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_file_context_manager PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_ls PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_mkdir PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_read_multiple_parquet_files SKIPPED pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_read_whole_file PASSED ``` The `py.test pyarrow --only-hdfs` option will run only the HDFS tests. Author: Wes McKinney <[email protected]> Closes #353 from wesm/ARROW-557 and squashes the following commits: 52e03db [Wes McKinney] Add conftest.py file, hdfs group to opt in to HDFS tests with --hdfs Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/dc103fea Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/dc103fea Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/dc103fea Branch: refs/heads/master Commit: dc103feaf0bb07b95f0c81afe0e342f239319dec Parents: 16c9759 Author: Wes McKinney <[email protected]> Authored: Mon Feb 27 08:13:29 2017 +0100 Committer: Uwe L. Korn <[email protected]> Committed: Mon Feb 27 08:13:29 2017 +0100 ---------------------------------------------------------------------- LICENSE.txt | 12 ------- NOTICE.txt | 4 +++ python/pyarrow/tests/conftest.py | 62 ++++++++++++++++++++++++++++++++++ python/pyarrow/tests/test_hdfs.py | 5 +-- 4 files changed, 69 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/dc103fea/LICENSE.txt ---------------------------------------------------------------------- diff --git a/LICENSE.txt b/LICENSE.txt index c3bec43..d645695 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -200,15 +200,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - --------------------------------------------------------------------------------- - -This product includes code from Apache Kudu. - - * cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake - -Copyright: 2016 The Apache Software Foundation. -Home page: https://kudu.apache.org/ -License: http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/dc103fea/NOTICE.txt ---------------------------------------------------------------------- diff --git a/NOTICE.txt b/NOTICE.txt index 02cb4dd..e71835c 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -42,6 +42,10 @@ This product includes software from the CMake project This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause) * Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved. +This product includes software from the Ibis project (Apache 2.0) + * Copyright (c) 2015 Cloudera, Inc. + * https://github.com/cloudera/ibis + -------------------------------------------------------------------------------- This product includes code from Apache Kudu, which includes the following in http://git-wip-us.apache.org/repos/asf/arrow/blob/dc103fea/python/pyarrow/tests/conftest.py ---------------------------------------------------------------------- diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py new file mode 100644 index 0000000..d5b4b69 --- /dev/null +++ b/python/pyarrow/tests/conftest.py @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pytest import skip + + +groups = ['hdfs'] + + +def pytest_configure(config): + pass + + +def pytest_addoption(parser): + for group in groups: + parser.addoption('--{0}'.format(group), action='store_true', + default=False, + help=('Enable the {0} test group'.format(group))) + + for group in groups: + parser.addoption('--only-{0}'.format(group), action='store_true', + default=False, + help=('Run only the {0} test group'.format(group))) + + +def pytest_runtest_setup(item): + only_set = False + + for group in groups: + only_flag = '--only-{0}'.format(group) + flag = '--{0}'.format(group) + + if item.config.getoption(only_flag): + only_set = True + elif getattr(item.obj, group, None): + if not item.config.getoption(flag): + skip('{0} NOT enabled'.format(flag)) + + if only_set: + skip_item = True + for group in groups: + only_flag = '--only-{0}'.format(group) + if (getattr(item.obj, group, False) and + item.config.getoption(only_flag)): + skip_item = False + + if skip_item: + skip('Only running some groups with only flags') http://git-wip-us.apache.org/repos/asf/arrow/blob/dc103fea/python/pyarrow/tests/test_hdfs.py ---------------------------------------------------------------------- diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py index cb24adb..b8f7e25 100644 --- a/python/pyarrow/tests/test_hdfs.py +++ b/python/pyarrow/tests/test_hdfs.py @@ -48,6 +48,7 @@ def hdfs_test_client(driver='libhdfs'): return HdfsClient(host, port, user, driver=driver) [email protected] class HdfsTestCases(object): def _make_test_file(self, hdfs, test_name, test_path, test_data): @@ -190,7 +191,7 @@ class TestLibHdfs(HdfsTestCases, unittest.TestCase): @classmethod def check_driver(cls): if not io.have_libhdfs(): - pytest.skip('No libhdfs available on system') + pytest.fail('No libhdfs available on system') def test_hdfs_orphaned_file(self): hdfs = hdfs_test_client() @@ -209,4 +210,4 @@ class TestLibHdfs3(HdfsTestCases, unittest.TestCase): @classmethod def check_driver(cls): if not io.have_libhdfs3(): - pytest.skip('No libhdfs3 available on system') + pytest.fail('No libhdfs3 available on system')
