[
https://issues.apache.org/jira/browse/BEAM-3143?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16295485#comment-16295485
]
ASF GitHub Bot commented on BEAM-3143:
--------------------------------------
robertwb closed pull request #4183: [BEAM-3143] Type Inference Python 3
Compatibility
URL: https://github.com/apache/beam/pull/4183
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/sdks/python/apache_beam/typehints/opcodes.py
b/sdks/python/apache_beam/typehints/opcodes.py
index dcca6d02644..ccf0195a58f 100644
--- a/sdks/python/apache_beam/typehints/opcodes.py
+++ b/sdks/python/apache_beam/typehints/opcodes.py
@@ -28,9 +28,13 @@
"""
from __future__ import absolute_import
+import inspect
+import sys
import types
from functools import reduce
+import six
+
from . import typehints
from .trivial_inference import BoundMethod
from .trivial_inference import Const
@@ -147,7 +151,7 @@ def binary_true_divide(state, unused_arg):
def binary_subscr(state, unused_arg):
tos = state.stack.pop()
- if tos in (str, unicode):
+ if tos in (str, six.text_type):
out = tos
else:
out = element_type(tos)
@@ -261,13 +265,22 @@ def build_list(state, arg):
def load_attr(state, arg):
+ """Replaces the top of the stack, TOS, with
+ getattr(TOS, co_names[arg])
+ """
o = state.stack.pop()
name = state.get_name(arg)
if isinstance(o, Const) and hasattr(o.value, name):
state.stack.append(Const(getattr(o.value, name)))
- elif (isinstance(o, type)
- and isinstance(getattr(o, name, None), types.MethodType)):
- state.stack.append(Const(BoundMethod(getattr(o, name))))
+ elif (inspect.isclass(o) and
+ isinstance(getattr(o, name, None),
+ (types.MethodType, types.FunctionType))):
+ # TODO(luke-zhu): Support other callable objects
+ if sys.version_info[0] == 2:
+ func = getattr(o, name).__func__
+ else:
+ func = getattr(o, name) # Python 3 has no unbound methods
+ state.stack.append(Const(BoundMethod(func, o)))
else:
state.stack.append(Any)
@@ -327,7 +340,18 @@ def call_function(state, arg, has_var=False, has_kw=False):
def make_function(state, arg):
- state.stack[-arg - 1:] = [Any] # a callable
+ """Creates a function with the arguments at the top of the stack.
+ """
+ # TODO(luke-zhu): Handle default argument types
+ globals = state.f.__globals__ # Inherits globals from the current frame
+ if sys.version_info[0] == 2:
+ func_code = state.stack[-1].value
+ func = types.FunctionType(func_code, globals)
+ else:
+ func_name = state.stack[-1].value
+ func_code = state.stack[-2].value
+ func = types.FunctionType(func_code, globals, name=func_name)
+ state.stack.append(Const(func))
def make_closure(state, arg):
diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py
b/sdks/python/apache_beam/typehints/trivial_inference.py
index a68bd18b1c3..28bf8f5ba6f 100644
--- a/sdks/python/apache_beam/typehints/trivial_inference.py
+++ b/sdks/python/apache_beam/typehints/trivial_inference.py
@@ -22,9 +22,9 @@
from __future__ import absolute_import
from __future__ import print_function
-import __builtin__
import collections
import dis
+import inspect
import pprint
import sys
import types
@@ -32,6 +32,8 @@
from apache_beam.typehints import Any
from apache_beam.typehints import typehints
+from six.moves import builtins
+from six.moves import zip
class TypeInferenceError(ValueError):
@@ -46,9 +48,9 @@ def instance_to_type(o):
if o is None:
return type(None)
elif t not in typehints.DISALLOWED_PRIMITIVE_TYPES:
- if t == types.InstanceType:
+ if sys.version_info[0] == 2 and t == types.InstanceType:
return o.__class__
- elif t == BoundMethod:
+ if t == BoundMethod:
return types.MethodType
return t
elif t == tuple:
@@ -107,15 +109,12 @@ class FrameState(object):
def __init__(self, f, local_vars=None, stack=()):
self.f = f
- if sys.version_info[0] >= 3:
- self.co = f.__code__
- else:
- self.co = f.func_code
+ self.co = f.__code__
self.vars = list(local_vars)
self.stack = list(stack)
def __eq__(self, other):
- return self.__dict__ == other.__dict__
+ return isinstance(other, FrameState) and self.__dict__ == other.__dict__
def copy(self):
return FrameState(self.f, self.vars, self.stack)
@@ -133,8 +132,8 @@ def get_global(self, i):
name = self.get_name(i)
if name in self.f.__globals__:
return Const(self.f.__globals__[name])
- if name in __builtin__.__dict__:
- return Const(__builtin__.__dict__[name])
+ if name in builtins.__dict__:
+ return Const(builtins.__dict__[name])
return Any
def get_name(self, i):
@@ -203,8 +202,15 @@ class BoundMethod(object):
"""Used to create a bound method when we only know the type of the instance.
"""
- def __init__(self, unbound):
- self.unbound = unbound
+ def __init__(self, func, type):
+ """Instantiates a bound method object.
+
+ Args:
+ func (types.FunctionType): The method's underlying function
+ type (type): The class of the method.
+ """
+ self.func = func
+ self.type = type
def hashable(c):
@@ -238,10 +244,9 @@ def infer_return_type(c, input_types, debug=False,
depth=5):
input_types = [Const(c.__self__)] + input_types
return infer_return_type_func(c.__func__, input_types, debug, depth)
elif isinstance(c, BoundMethod):
- input_types = [c.unbound.__self__.__class__] + input_types
- return infer_return_type_func(
- c.unbound.__func__, input_types, debug, depth)
- elif isinstance(c, type):
+ input_types = [c.type] + input_types
+ return infer_return_type_func(c.func, input_types, debug, depth)
+ elif inspect.isclass(c):
if c in typehints.DISALLOWED_PRIMITIVE_TYPES:
return {
list: typehints.List[Any],
@@ -303,15 +308,23 @@ def infer_return_type_func(f, input_types, debug=False,
depth=0):
last_pc = -1
while pc < end:
start = pc
- op = ord(code[pc])
-
+ if sys.version_info[0] == 2:
+ op = ord(code[pc])
+ else:
+ op = code[pc]
if debug:
print('-->' if pc == last_pc else ' ', end=' ')
print(repr(pc).rjust(4), end=' ')
print(dis.opname[op].ljust(20), end=' ')
+
pc += 1
if op >= dis.HAVE_ARGUMENT:
- arg = ord(code[pc]) + ord(code[pc + 1]) * 256 + extended_arg
+ if sys.version_info[0] == 2:
+ arg = ord(code[pc]) + ord(code[pc + 1]) * 256 + extended_arg
+ elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
+ arg = code[pc] + code[pc + 1] * 256 + extended_arg
+ else:
+ pass # TODO(luke-zhu): Python 3.6 bytecode to wordcode changes
extended_arg = 0
pc += 2
if op == dis.EXTENDED_ARG:
@@ -344,7 +357,7 @@ def infer_return_type_func(f, input_types, debug=False,
depth=0):
opname = dis.opname[op]
jmp = jmp_state = None
if opname.startswith('CALL_FUNCTION'):
- standard_args = (arg & 0xF) + (arg & 0xF0) / 8
+ standard_args = (arg & 0xF) + (arg & 0xF0) // 8
var_args = 'VAR' in opname
kw_args = 'KW' in opname
pop_count = standard_args + var_args + kw_args + 1
diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test.py
b/sdks/python/apache_beam/typehints/trivial_inference_test.py
index 37b22584723..b017e8af497 100644
--- a/sdks/python/apache_beam/typehints/trivial_inference_test.py
+++ b/sdks/python/apache_beam/typehints/trivial_inference_test.py
@@ -72,6 +72,17 @@ def func(a):
return None
self.assertReturnType(typehints.Union[int, type(None)], func, [int])
+ def testSimpleList(self):
+ self.assertReturnType(
+ typehints.List[int],
+ lambda xs: [1, 2],
+ [typehints.Tuple[int, ...]])
+
+ self.assertReturnType(
+ typehints.List[typehints.Any],
+ lambda xs: list(xs), # List is a disallowed builtin
+ [typehints.Tuple[int, ...]])
+
def testListComprehension(self):
self.assertReturnType(
typehints.List[int],
@@ -98,6 +109,12 @@ def foo(x, y):
self.assertReturnType(
typehints.Iterable[typehints.Union[int, float]], foo, [int, float])
+ def testGeneratorComprehension(self):
+ self.assertReturnType(
+ typehints.Iterable[int],
+ lambda xs: (x for x in xs),
+ [typehints.Tuple[int, ...]])
+
def testBinOp(self):
self.assertReturnType(int, lambda a, b: a + b, [int, int])
self.assertReturnType(
@@ -130,7 +147,6 @@ def testGetAttr(self):
def testMethod(self):
class A(object):
-
def m(self, x):
return x
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> Fix type inference in Python 3 for generators
> ---------------------------------------------
>
> Key: BEAM-3143
> URL: https://issues.apache.org/jira/browse/BEAM-3143
> Project: Beam
> Issue Type: Sub-task
> Components: sdk-py-core
> Reporter: holdenk
>
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
