This is an automated email from the ASF dual-hosted git repository.
robertwb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new a424c64 [BEAM-3143] Type Inference Python 3 Compatibility (#4183)
a424c64 is described below
commit a424c64a3234a178a2a86161ee200c8be79d8f53
Author: Luke Zhu <[email protected]>
AuthorDate: Mon Dec 18 14:06:53 2017 -0500
[BEAM-3143] Type Inference Python 3 Compatibility (#4183)
---
sdks/python/apache_beam/typehints/opcodes.py | 34 ++++++++++++--
.../apache_beam/typehints/trivial_inference.py | 53 ++++++++++++++--------
.../typehints/trivial_inference_test.py | 18 +++++++-
3 files changed, 79 insertions(+), 26 deletions(-)
diff --git a/sdks/python/apache_beam/typehints/opcodes.py
b/sdks/python/apache_beam/typehints/opcodes.py
index dcca6d0..ccf0195 100644
--- a/sdks/python/apache_beam/typehints/opcodes.py
+++ b/sdks/python/apache_beam/typehints/opcodes.py
@@ -28,9 +28,13 @@ For internal use only; no backwards-compatibility guarantees.
"""
from __future__ import absolute_import
+import inspect
+import sys
import types
from functools import reduce
+import six
+
from . import typehints
from .trivial_inference import BoundMethod
from .trivial_inference import Const
@@ -147,7 +151,7 @@ binary_subtract = inplace_subtract = symmetric_binary_op
def binary_subscr(state, unused_arg):
tos = state.stack.pop()
- if tos in (str, unicode):
+ if tos in (str, six.text_type):
out = tos
else:
out = element_type(tos)
@@ -261,13 +265,22 @@ build_map = push_value(Dict[Any, Any])
def load_attr(state, arg):
+ """Replaces the top of the stack, TOS, with
+ getattr(TOS, co_names[arg])
+ """
o = state.stack.pop()
name = state.get_name(arg)
if isinstance(o, Const) and hasattr(o.value, name):
state.stack.append(Const(getattr(o.value, name)))
- elif (isinstance(o, type)
- and isinstance(getattr(o, name, None), types.MethodType)):
- state.stack.append(Const(BoundMethod(getattr(o, name))))
+ elif (inspect.isclass(o) and
+ isinstance(getattr(o, name, None),
+ (types.MethodType, types.FunctionType))):
+ # TODO(luke-zhu): Support other callable objects
+ if sys.version_info[0] == 2:
+ func = getattr(o, name).__func__
+ else:
+ func = getattr(o, name) # Python 3 has no unbound methods
+ state.stack.append(Const(BoundMethod(func, o)))
else:
state.stack.append(Any)
@@ -327,7 +340,18 @@ def call_function(state, arg, has_var=False, has_kw=False):
def make_function(state, arg):
- state.stack[-arg - 1:] = [Any] # a callable
+ """Creates a function with the arguments at the top of the stack.
+ """
+ # TODO(luke-zhu): Handle default argument types
+ globals = state.f.__globals__ # Inherits globals from the current frame
+ if sys.version_info[0] == 2:
+ func_code = state.stack[-1].value
+ func = types.FunctionType(func_code, globals)
+ else:
+ func_name = state.stack[-1].value
+ func_code = state.stack[-2].value
+ func = types.FunctionType(func_code, globals, name=func_name)
+ state.stack.append(Const(func))
def make_closure(state, arg):
diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py
b/sdks/python/apache_beam/typehints/trivial_inference.py
index a68bd18..28bf8f5 100644
--- a/sdks/python/apache_beam/typehints/trivial_inference.py
+++ b/sdks/python/apache_beam/typehints/trivial_inference.py
@@ -22,9 +22,9 @@ For internal use only; no backwards-compatibility guarantees.
from __future__ import absolute_import
from __future__ import print_function
-import __builtin__
import collections
import dis
+import inspect
import pprint
import sys
import types
@@ -32,6 +32,8 @@ from functools import reduce
from apache_beam.typehints import Any
from apache_beam.typehints import typehints
+from six.moves import builtins
+from six.moves import zip
class TypeInferenceError(ValueError):
@@ -46,9 +48,9 @@ def instance_to_type(o):
if o is None:
return type(None)
elif t not in typehints.DISALLOWED_PRIMITIVE_TYPES:
- if t == types.InstanceType:
+ if sys.version_info[0] == 2 and t == types.InstanceType:
return o.__class__
- elif t == BoundMethod:
+ if t == BoundMethod:
return types.MethodType
return t
elif t == tuple:
@@ -107,15 +109,12 @@ class FrameState(object):
def __init__(self, f, local_vars=None, stack=()):
self.f = f
- if sys.version_info[0] >= 3:
- self.co = f.__code__
- else:
- self.co = f.func_code
+ self.co = f.__code__
self.vars = list(local_vars)
self.stack = list(stack)
def __eq__(self, other):
- return self.__dict__ == other.__dict__
+ return isinstance(other, FrameState) and self.__dict__ == other.__dict__
def copy(self):
return FrameState(self.f, self.vars, self.stack)
@@ -133,8 +132,8 @@ class FrameState(object):
name = self.get_name(i)
if name in self.f.__globals__:
return Const(self.f.__globals__[name])
- if name in __builtin__.__dict__:
- return Const(__builtin__.__dict__[name])
+ if name in builtins.__dict__:
+ return Const(builtins.__dict__[name])
return Any
def get_name(self, i):
@@ -203,8 +202,15 @@ class BoundMethod(object):
"""Used to create a bound method when we only know the type of the instance.
"""
- def __init__(self, unbound):
- self.unbound = unbound
+ def __init__(self, func, type):
+ """Instantiates a bound method object.
+
+ Args:
+ func (types.FunctionType): The method's underlying function
+ type (type): The class of the method.
+ """
+ self.func = func
+ self.type = type
def hashable(c):
@@ -238,10 +244,9 @@ def infer_return_type(c, input_types, debug=False,
depth=5):
input_types = [Const(c.__self__)] + input_types
return infer_return_type_func(c.__func__, input_types, debug, depth)
elif isinstance(c, BoundMethod):
- input_types = [c.unbound.__self__.__class__] + input_types
- return infer_return_type_func(
- c.unbound.__func__, input_types, debug, depth)
- elif isinstance(c, type):
+ input_types = [c.type] + input_types
+ return infer_return_type_func(c.func, input_types, debug, depth)
+ elif inspect.isclass(c):
if c in typehints.DISALLOWED_PRIMITIVE_TYPES:
return {
list: typehints.List[Any],
@@ -303,15 +308,23 @@ def infer_return_type_func(f, input_types, debug=False,
depth=0):
last_pc = -1
while pc < end:
start = pc
- op = ord(code[pc])
-
+ if sys.version_info[0] == 2:
+ op = ord(code[pc])
+ else:
+ op = code[pc]
if debug:
print('-->' if pc == last_pc else ' ', end=' ')
print(repr(pc).rjust(4), end=' ')
print(dis.opname[op].ljust(20), end=' ')
+
pc += 1
if op >= dis.HAVE_ARGUMENT:
- arg = ord(code[pc]) + ord(code[pc + 1]) * 256 + extended_arg
+ if sys.version_info[0] == 2:
+ arg = ord(code[pc]) + ord(code[pc + 1]) * 256 + extended_arg
+ elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
+ arg = code[pc] + code[pc + 1] * 256 + extended_arg
+ else:
+ pass # TODO(luke-zhu): Python 3.6 bytecode to wordcode changes
extended_arg = 0
pc += 2
if op == dis.EXTENDED_ARG:
@@ -344,7 +357,7 @@ def infer_return_type_func(f, input_types, debug=False,
depth=0):
opname = dis.opname[op]
jmp = jmp_state = None
if opname.startswith('CALL_FUNCTION'):
- standard_args = (arg & 0xF) + (arg & 0xF0) / 8
+ standard_args = (arg & 0xF) + (arg & 0xF0) // 8
var_args = 'VAR' in opname
kw_args = 'KW' in opname
pop_count = standard_args + var_args + kw_args + 1
diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test.py
b/sdks/python/apache_beam/typehints/trivial_inference_test.py
index 37b2258..b017e8a 100644
--- a/sdks/python/apache_beam/typehints/trivial_inference_test.py
+++ b/sdks/python/apache_beam/typehints/trivial_inference_test.py
@@ -72,6 +72,17 @@ class TrivialInferenceTest(unittest.TestCase):
return None
self.assertReturnType(typehints.Union[int, type(None)], func, [int])
+ def testSimpleList(self):
+ self.assertReturnType(
+ typehints.List[int],
+ lambda xs: [1, 2],
+ [typehints.Tuple[int, ...]])
+
+ self.assertReturnType(
+ typehints.List[typehints.Any],
+ lambda xs: list(xs), # List is a disallowed builtin
+ [typehints.Tuple[int, ...]])
+
def testListComprehension(self):
self.assertReturnType(
typehints.List[int],
@@ -98,6 +109,12 @@ class TrivialInferenceTest(unittest.TestCase):
self.assertReturnType(
typehints.Iterable[typehints.Union[int, float]], foo, [int, float])
+ def testGeneratorComprehension(self):
+ self.assertReturnType(
+ typehints.Iterable[int],
+ lambda xs: (x for x in xs),
+ [typehints.Tuple[int, ...]])
+
def testBinOp(self):
self.assertReturnType(int, lambda a, b: a + b, [int, int])
self.assertReturnType(
@@ -130,7 +147,6 @@ class TrivialInferenceTest(unittest.TestCase):
def testMethod(self):
class A(object):
-
def m(self, x):
return x
--
To stop receiving notification emails like this one, please contact
['"[email protected]" <[email protected]>'].
