[
https://issues.apache.org/jira/browse/ARROW-2342?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16410300#comment-16410300
]
ASF GitHub Bot commented on ARROW-2342:
---------------------------------------
wesm closed pull request #1780: ARROW-2342: [Python] Allow pickling more types
URL: https://github.com/apache/arrow/pull/1780
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/python/pyarrow/tests/test_types.py
b/python/pyarrow/tests/test_types.py
index 6459496f6..b5170203c 100644
--- a/python/pyarrow/tests/test_types.py
+++ b/python/pyarrow/tests/test_types.py
@@ -15,12 +15,44 @@
# specific language governing permissions and limitations
# under the License.
+import pickle
+
import pytest
import pyarrow as pa
import pyarrow.types as types
+MANY_TYPES = [
+ pa.null(),
+ pa.bool_(),
+ pa.int32(),
+ pa.time32('s'),
+ pa.time64('us'),
+ pa.date32(),
+ pa.timestamp('us'),
+ pa.timestamp('us', tz='UTC'),
+ pa.timestamp('us', tz='Europe/Paris'),
+ pa.float16(),
+ pa.float32(),
+ pa.float64(),
+ pa.decimal128(19, 4),
+ pa.string(),
+ pa.binary(),
+ pa.binary(10),
+ pa.list_(pa.int32()),
+ pa.struct([pa.field('a', pa.int32()),
+ pa.field('b', pa.int8()),
+ pa.field('c', pa.string())]),
+ pa.union([pa.field('a', pa.binary(10)),
+ pa.field('b', pa.string())], mode=pa.lib.UnionMode_DENSE),
+ pa.union([pa.field('a', pa.binary(10)),
+ pa.field('b', pa.string())], mode=pa.lib.UnionMode_SPARSE),
+ # XXX Needs array pickling
+ # pa.dictionary(pa.int32(), pa.array(['a', 'b', 'c'])),
+]
+
+
def test_is_boolean():
assert types.is_boolean(pa.bool_())
assert not types.is_boolean(pa.int8())
@@ -163,27 +195,18 @@ def check_fields(ty, fields):
def test_types_hashable():
- types = [
- pa.null(),
- pa.int32(),
- pa.time32('s'),
- pa.time64('us'),
- pa.date32(),
- pa.timestamp('us'),
- pa.string(),
- pa.binary(),
- pa.binary(10),
- pa.list_(pa.int32()),
- pa.struct([pa.field('a', pa.int32()),
- pa.field('b', pa.int8()),
- pa.field('c', pa.string())])
- ]
-
in_dict = {}
- for i, type_ in enumerate(types):
+ for i, type_ in enumerate(MANY_TYPES):
assert hash(type_) == hash(type_)
in_dict[type_] = i
assert in_dict[type_] == i
+ assert len(in_dict) == len(MANY_TYPES)
+
+
+def test_types_picklable():
+ for ty in MANY_TYPES:
+ data = pickle.dumps(ty)
+ assert pickle.loads(data) == ty
@pytest.mark.parametrize('t,check_func', [
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index 129485008..a4391c7f9 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -247,6 +247,13 @@ cdef class TimestampType(DataType):
# Return DatetimeTZ
return pdcompat.make_datetimetz(self.tz)
+ def __getstate__(self):
+ return self.unit, self.tz
+
+ def __setstate__(self, state):
+ cdef DataType reconstituted = timestamp(*state)
+ self.init(reconstituted.sp_type)
+
cdef class Time32Type(DataType):
@@ -1192,6 +1199,7 @@ def union(children_fields, mode):
cdef dict _type_aliases = {
'null': null,
+ 'bool': bool_,
'i1': int8,
'int8': int8,
'i2': int16,
@@ -1208,9 +1216,14 @@ cdef dict _type_aliases = {
'uint32': uint32,
'u8': uint64,
'uint64': uint64,
+ 'f2': float16,
+ 'halffloat': float16,
+ 'float16': float16,
'f4': float32,
+ 'float': float32,
'float32': float32,
'f8': float64,
+ 'double': float64,
'float64': float64,
'string': string,
'str': string,
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> [Python] Aware timestamp type fails pickling
> --------------------------------------------
>
> Key: ARROW-2342
> URL: https://issues.apache.org/jira/browse/ARROW-2342
> Project: Apache Arrow
> Issue Type: Bug
> Components: Python
> Affects Versions: 0.9.0
> Reporter: Antoine Pitrou
> Assignee: Antoine Pitrou
> Priority: Major
> Labels: pull-request-available
> Fix For: 0.10.0
>
>
> {code:python}
> >>> ty = pa.timestamp('ms')
> >>> pickle.loads(pickle.dumps(ty))
> TimestampType(timestamp[ms])
> >>> ty = pa.timestamp('ms', tz='UTC')
> >>> pickle.loads(pickle.dumps(ty))
> Traceback (most recent call last):
> File "<ipython-input-15-889e5adab733>", line 1, in <module>
> pickle.loads(pickle.dumps(ty))
> File "types.pxi", line 82, in pyarrow.lib.DataType.__setstate__
> File "types.pxi", line 1246, in pyarrow.lib.type_for_alias
> ValueError: No type alias for timestamp[ms, tz=utc]
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
