[jira] [Commented] (ARROW-2263) [Python] test_cython.py fails if pyarrow is not in import path (e.g. with inplace builds)
[
https://issues.apache.org/jira/browse/ARROW-2263?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16392372#comment-16392372
]
ASF GitHub Bot commented on ARROW-2263:
---
wesm closed pull request #1730: ARROW-2263: [Python] Prepend local pyarrow/
path to PYTHONPATH in test_cython.py
URL: https://github.com/apache/arrow/pull/1730
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/python/pyarrow/tests/test_cython.py
b/python/pyarrow/tests/test_cython.py
index df5e70ee7..57dbeb554 100644
--- a/python/pyarrow/tests/test_cython.py
+++ b/python/pyarrow/tests/test_cython.py
@@ -24,6 +24,7 @@
import pyarrow as pa
+import pyarrow.tests.util as test_util
here = os.path.dirname(os.path.abspath(__file__))
@@ -85,9 +86,14 @@ def test_cython_api(tmpdir):
with open('setup.py', 'w') as f:
f.write(setup_code)
+# ARROW-2263: Make environment with this pyarrow/ package first on the
+# PYTHONPATH, for local dev environments
+subprocess_env = test_util.get_modified_env_with_pythonpath()
+
# Compile extension module
subprocess.check_call([sys.executable, 'setup.py',
- 'build_ext', '--inplace'])
+ 'build_ext', '--inplace'],
+ env=subprocess_env)
# Check basic functionality
orig_path = sys.path[:]
diff --git a/python/pyarrow/tests/test_serialization.py
b/python/pyarrow/tests/test_serialization.py
index c17408457..64aab0671 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -28,6 +28,8 @@
import pyarrow as pa
import numpy as np
+import pyarrow.tests.util as test_util
+
try:
import torch
except ImportError:
@@ -624,18 +626,6 @@ def deserialize_regex(serialized, q):
p.join()
-def _get_modified_env_with_pythonpath():
-# Prepend pyarrow root directory to PYTHONPATH
-env = os.environ.copy()
-existing_pythonpath = env.get('PYTHONPATH', '')
-
-module_path = os.path.abspath(
-os.path.dirname(os.path.dirname(pa.__file__)))
-
-env['PYTHONPATH'] = os.pathsep.join((module_path, existing_pythonpath))
-return env
-
-
def test_deserialize_buffer_in_different_process():
import tempfile
import subprocess
@@ -645,7 +635,7 @@ def test_deserialize_buffer_in_different_process():
f.write(b.to_pybytes())
f.close()
-subprocess_env = _get_modified_env_with_pythonpath()
+subprocess_env = test_util.get_modified_env_with_pythonpath()
dir_path = os.path.dirname(os.path.realpath(__file__))
python_file = os.path.join(dir_path, 'deserialize_buffer.py')
diff --git a/python/pyarrow/tests/util.py b/python/pyarrow/tests/util.py
index a3ba9000c..8c8d23b0c 100644
--- a/python/pyarrow/tests/util.py
+++ b/python/pyarrow/tests/util.py
@@ -19,9 +19,12 @@
Utility functions for testing
"""
+import contextlib
import decimal
+import os
import random
-import contextlib
+
+import pyarrow as pa
def randsign():
@@ -91,3 +94,15 @@ def randdecimal(precision, scale):
return decimal.Decimal(
'{}.{}'.format(whole, str(fractional).rjust(scale, '0'))
)
+
+
+def get_modified_env_with_pythonpath():
+# Prepend pyarrow root directory to PYTHONPATH
+env = os.environ.copy()
+existing_pythonpath = env.get('PYTHONPATH', '')
+
+module_path = os.path.abspath(
+os.path.dirname(os.path.dirname(pa.__file__)))
+
+env['PYTHONPATH'] = os.pathsep.join((module_path, existing_pythonpath))
+return env
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> [Python] test_cython.py fails if pyarrow is not in import path (e.g. with
> inplace builds)
> -
>
> Key: ARROW-2263
> URL: https://issues.apache.org/jira/browse/ARROW-2263
> Project: Apache Arrow
> Issue Type: Bug
> Components: Python
>Reporter: Wes McKinney
>Assignee: Wes McKinney
>Priority: Major
> Labels: pull-request-available
> Fix For: 0.9.0
>
>
> see
> {code}
> $ py.test pyarrow/tests/test_cython.py
> = test session starts
> =
> platform linux -- Python 3.6.4, pytest-3.4.1, py-1.5.2, pluggy-0.6.0
> rootdir:
[jira] [Commented] (ARROW-2263) [Python] test_cython.py fails if pyarrow is not in import path (e.g. with inplace builds)
[
https://issues.apache.org/jira/browse/ARROW-2263?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16392332#comment-16392332
]
ASF GitHub Bot commented on ARROW-2263:
---
wesm commented on issue #1730: ARROW-2263: [Python] Prepend local pyarrow/ path
to PYTHONPATH in test_cython.py
URL: https://github.com/apache/arrow/pull/1730#issuecomment-371700652
This was bugging me -- turned out to be easy to fix.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> [Python] test_cython.py fails if pyarrow is not in import path (e.g. with
> inplace builds)
> -
>
> Key: ARROW-2263
> URL: https://issues.apache.org/jira/browse/ARROW-2263
> Project: Apache Arrow
> Issue Type: Bug
> Components: Python
>Reporter: Wes McKinney
>Assignee: Wes McKinney
>Priority: Major
> Labels: pull-request-available
> Fix For: 0.9.0
>
>
> see
> {code}
> $ py.test pyarrow/tests/test_cython.py
> = test session starts
> =
> platform linux -- Python 3.6.4, pytest-3.4.1, py-1.5.2, pluggy-0.6.0
> rootdir: /home/wesm/code/arrow/python, inifile: setup.cfg
> collected 1 item
>
> pyarrow/tests/test_cython.py F
> [100%]
> == FAILURES
> ===
> ___ test_cython_api
> ___
> tmpdir = local('/tmp/pytest-of-wesm/pytest-3/test_cython_api0')
> @pytest.mark.skipif(
> 'ARROW_HOME' not in os.environ,
> reason='ARROW_HOME environment variable not defined')
> def test_cython_api(tmpdir):
> """
> Basic test for the Cython API.
> """
> pytest.importorskip('Cython')
>
> ld_path_default = os.path.join(os.environ['ARROW_HOME'], 'lib')
>
> test_ld_path = os.environ.get('PYARROW_TEST_LD_PATH', ld_path_default)
>
> with tmpdir.as_cwd():
> # Set up temporary workspace
> pyx_file = 'pyarrow_cython_example.pyx'
> shutil.copyfile(os.path.join(here, pyx_file),
> os.path.join(str(tmpdir), pyx_file))
> # Create setup.py file
> if os.name == 'posix':
> compiler_opts = ['-std=c++11']
> else:
> compiler_opts = []
> setup_code = setup_template.format(pyx_file=pyx_file,
>compiler_opts=compiler_opts,
>test_ld_path=test_ld_path)
> with open('setup.py', 'w') as f:
> f.write(setup_code)
>
> # Compile extension module
> subprocess.check_call([sys.executable, 'setup.py',
> > 'build_ext', '--inplace'])
> pyarrow/tests/test_cython.py:90:
> _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> _ _ _ _ _ _ _ _ _
> popenargs = (['/home/wesm/miniconda/envs/arrow-dev/bin/python', 'setup.py',
> 'build_ext', '--inplace'],)
> kwargs = {}, retcode = 1
> cmd = ['/home/wesm/miniconda/envs/arrow-dev/bin/python', 'setup.py',
> 'build_ext', '--inplace']
> def check_call(*popenargs, **kwargs):
> """Run command with arguments. Wait for command to complete. If
> the exit code was zero then return, otherwise raise
> CalledProcessError. The CalledProcessError object will have the
> return code in the returncode attribute.
>
> The arguments are the same as for the call function. Example:
>
> check_call(["ls", "-l"])
> """
> retcode = call(*popenargs, **kwargs)
> if retcode:
> cmd = kwargs.get("args")
> if cmd is None:
> cmd = popenargs[0]
> > raise CalledProcessError(retcode, cmd)
> E subprocess.CalledProcessError: Command
> '['/home/wesm/miniconda/envs/arrow-dev/bin/python', 'setup.py', 'build_ext',
> '--inplace']' returned non-zero exit status 1.
> ../../../miniconda/envs/arrow-dev/lib/python3.6/subprocess.py:291:
> CalledProcessError
> Captured stderr call
> -
> Traceback (most recent call last):
> File "setup.py", line 7, in
>
[jira] [Commented] (ARROW-2263) [Python] test_cython.py fails if pyarrow is not in import path (e.g. with inplace builds)
[
https://issues.apache.org/jira/browse/ARROW-2263?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16392330#comment-16392330
]
ASF GitHub Bot commented on ARROW-2263:
---
wesm opened a new pull request #1730: ARROW-2263: [Python] Prepend local
pyarrow/ path to PYTHONPATH in test_cython.py
URL: https://github.com/apache/arrow/pull/1730
This was bugging me -- turned out to be easy to fix.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> [Python] test_cython.py fails if pyarrow is not in import path (e.g. with
> inplace builds)
> -
>
> Key: ARROW-2263
> URL: https://issues.apache.org/jira/browse/ARROW-2263
> Project: Apache Arrow
> Issue Type: Bug
> Components: Python
>Reporter: Wes McKinney
>Assignee: Wes McKinney
>Priority: Major
> Labels: pull-request-available
> Fix For: 0.9.0
>
>
> see
> {code}
> $ py.test pyarrow/tests/test_cython.py
> = test session starts
> =
> platform linux -- Python 3.6.4, pytest-3.4.1, py-1.5.2, pluggy-0.6.0
> rootdir: /home/wesm/code/arrow/python, inifile: setup.cfg
> collected 1 item
>
> pyarrow/tests/test_cython.py F
> [100%]
> == FAILURES
> ===
> ___ test_cython_api
> ___
> tmpdir = local('/tmp/pytest-of-wesm/pytest-3/test_cython_api0')
> @pytest.mark.skipif(
> 'ARROW_HOME' not in os.environ,
> reason='ARROW_HOME environment variable not defined')
> def test_cython_api(tmpdir):
> """
> Basic test for the Cython API.
> """
> pytest.importorskip('Cython')
>
> ld_path_default = os.path.join(os.environ['ARROW_HOME'], 'lib')
>
> test_ld_path = os.environ.get('PYARROW_TEST_LD_PATH', ld_path_default)
>
> with tmpdir.as_cwd():
> # Set up temporary workspace
> pyx_file = 'pyarrow_cython_example.pyx'
> shutil.copyfile(os.path.join(here, pyx_file),
> os.path.join(str(tmpdir), pyx_file))
> # Create setup.py file
> if os.name == 'posix':
> compiler_opts = ['-std=c++11']
> else:
> compiler_opts = []
> setup_code = setup_template.format(pyx_file=pyx_file,
>compiler_opts=compiler_opts,
>test_ld_path=test_ld_path)
> with open('setup.py', 'w') as f:
> f.write(setup_code)
>
> # Compile extension module
> subprocess.check_call([sys.executable, 'setup.py',
> > 'build_ext', '--inplace'])
> pyarrow/tests/test_cython.py:90:
> _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> _ _ _ _ _ _ _ _ _
> popenargs = (['/home/wesm/miniconda/envs/arrow-dev/bin/python', 'setup.py',
> 'build_ext', '--inplace'],)
> kwargs = {}, retcode = 1
> cmd = ['/home/wesm/miniconda/envs/arrow-dev/bin/python', 'setup.py',
> 'build_ext', '--inplace']
> def check_call(*popenargs, **kwargs):
> """Run command with arguments. Wait for command to complete. If
> the exit code was zero then return, otherwise raise
> CalledProcessError. The CalledProcessError object will have the
> return code in the returncode attribute.
>
> The arguments are the same as for the call function. Example:
>
> check_call(["ls", "-l"])
> """
> retcode = call(*popenargs, **kwargs)
> if retcode:
> cmd = kwargs.get("args")
> if cmd is None:
> cmd = popenargs[0]
> > raise CalledProcessError(retcode, cmd)
> E subprocess.CalledProcessError: Command
> '['/home/wesm/miniconda/envs/arrow-dev/bin/python', 'setup.py', 'build_ext',
> '--inplace']' returned non-zero exit status 1.
> ../../../miniconda/envs/arrow-dev/lib/python3.6/subprocess.py:291:
> CalledProcessError
> Captured stderr call
> -
> Traceback (most recent call last):
> File "setup.py", line 7, in
> import pyarrow as
