Github user HyukjinKwon commented on a diff in the pull request:
https://github.com/apache/spark/pull/18734#discussion_r130765110
--- Diff: python/pyspark/cloudpickle.py ---
@@ -623,72 +816,82 @@ def save_file(self, obj):
return self.save_reduce(getattr, (sys,'stderr'), obj=obj)
if obj is sys.stdin:
raise pickle.PicklingError("Cannot pickle standard input")
- if hasattr(obj, 'isatty') and obj.isatty():
+ if obj.closed:
+ raise pickle.PicklingError("Cannot pickle closed files")
+ if hasattr(obj, 'isatty') and obj.isatty():
raise pickle.PicklingError("Cannot pickle files that map to
tty objects")
- if 'r' not in obj.mode:
- raise pickle.PicklingError("Cannot pickle files that are not
opened for reading")
+ if 'r' not in obj.mode and '+' not in obj.mode:
+ raise pickle.PicklingError("Cannot pickle files that are not
opened for reading: %s" % obj.mode)
+
name = obj.name
- try:
- fsize = os.stat(name).st_size
- except OSError:
- raise pickle.PicklingError("Cannot pickle file %s as it cannot
be stat" % name)
- if obj.closed:
- #create an empty closed string io
- retval = pystringIO.StringIO("")
- retval.close()
- elif not fsize: #empty file
- retval = pystringIO.StringIO("")
- try:
- tmpfile = file(name)
- tst = tmpfile.read(1)
- except IOError:
- raise pickle.PicklingError("Cannot pickle file %s as it
cannot be read" % name)
- tmpfile.close()
- if tst != '':
- raise pickle.PicklingError("Cannot pickle file %s as it
does not appear to map to a physical, real file" % name)
- else:
- try:
- tmpfile = file(name)
- contents = tmpfile.read()
- tmpfile.close()
- except IOError:
- raise pickle.PicklingError("Cannot pickle file %s as it
cannot be read" % name)
- retval = pystringIO.StringIO(contents)
+ retval = pystringIO.StringIO()
+
+ try:
+ # Read the whole file
curloc = obj.tell()
- retval.seek(curloc)
+ obj.seek(0)
+ contents = obj.read()
+ obj.seek(curloc)
+ except IOError:
+ raise pickle.PicklingError("Cannot pickle file %s as it cannot
be read" % name)
+ retval.write(contents)
+ retval.seek(curloc)
retval.name = name
self.save(retval)
self.memoize(obj)
+ def save_ellipsis(self, obj):
+ self.save_reduce(_gen_ellipsis, ())
+
+ def save_not_implemented(self, obj):
+ self.save_reduce(_gen_not_implemented, ())
+
if PY3:
dispatch[io.TextIOWrapper] = save_file
else:
dispatch[file] = save_file
- """Special functions for Add-on libraries"""
+ dispatch[type(Ellipsis)] = save_ellipsis
+ dispatch[type(NotImplemented)] = save_not_implemented
- def inject_numpy(self):
--- End diff --
I assume removing this is fine per
https://github.com/cloudpipe/cloudpickle/commit/1e91fa7c0f9b1e77604d83b3ba9aecde8603ece1.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
