[
https://issues.apache.org/jira/browse/ARROW-1951?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16305750#comment-16305750
]
ASF GitHub Bot commented on ARROW-1951:
---------------------------------------
pcmoritz closed pull request #1451: ARROW-1951: [Python] Add memcopy threads
argument to PlasmaClient put.
URL: https://github.com/apache/arrow/pull/1451
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx
index f2e8653d8..9b3e4093a 100644
--- a/python/pyarrow/plasma.pyx
+++ b/python/pyarrow/plasma.pyx
@@ -370,7 +370,7 @@ cdef class PlasmaClient:
object_buffers[i].metadata_size))
return result
- def put(self, object value, ObjectID object_id=None,
+ def put(self, object value, ObjectID object_id=None, int memcopy_threads=6,
serialization_context=None):
"""
Store a Python value into the object store.
@@ -382,6 +382,9 @@ cdef class PlasmaClient:
object_id : ObjectID, default None
If this is provided, the specified object ID will be used to refer
to the object.
+ memcopy_threads : int, default 6
+ The number of threads to use to write the serialized object into
+ the object store for large objects.
serialization_context : pyarrow.SerializationContext, default None
Custom serialization and deserialization context.
@@ -394,7 +397,7 @@ cdef class PlasmaClient:
serialized = pyarrow.serialize(value, serialization_context)
buffer = self.create(target_id, serialized.total_bytes)
stream = pyarrow.FixedSizeBufferWriter(buffer)
- stream.set_memcopy_threads(4)
+ stream.set_memcopy_threads(memcopy_threads)
serialized.write_to(stream)
self.seal(target_id)
return target_id
diff --git a/python/pyarrow/tests/test_plasma.py
b/python/pyarrow/tests/test_plasma.py
index b28bd60c4..decdc73d9 100644
--- a/python/pyarrow/tests/test_plasma.py
+++ b/python/pyarrow/tests/test_plasma.py
@@ -310,12 +310,14 @@ def __init__(self, val):
serialization_context = pa.SerializationContext()
serialization_context.register_type(CustomType, 20*b"\x00")
- object_id = self.plasma_client.put(val, None, serialization_context)
+ object_id = self.plasma_client.put(
+ val, None, serialization_context=serialization_context)
with pytest.raises(pa.ArrowSerializationError):
result = self.plasma_client.get(object_id)
- result = self.plasma_client.get(object_id, -1, serialization_context)
+ result = self.plasma_client.get(
+ object_id, -1, serialization_context=serialization_context)
assert result.val == val.val
def test_store_arrow_objects(self):
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> Add memcopy_threads to serialization context
> --------------------------------------------
>
> Key: ARROW-1951
> URL: https://issues.apache.org/jira/browse/ARROW-1951
> Project: Apache Arrow
> Issue Type: Improvement
> Reporter: Robert Nishihara
> Assignee: Robert Nishihara
> Priority: Minor
> Labels: pull-request-available
>
> Right now, when calling {{put}} with a plasma client, we set
> {{memcopy_threads}} to 4. We should expose this so it can be changed through
> Python.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
