[
https://issues.apache.org/jira/browse/ARROW-1756?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16236074#comment-16236074
]
ASF GitHub Bot commented on ARROW-1756:
---------------------------------------
wesm commented on a change in pull request #1276: ARROW-1756: [Python] Fix
large file read/write error on Mac
URL: https://github.com/apache/arrow/pull/1276#discussion_r148587221
##########
File path: cpp/src/arrow/io/file.cc
##########
@@ -246,31 +246,55 @@ static inline Status FileRead(int fd, uint8_t* buffer,
int64_t nbytes,
}
*bytes_read = static_cast<int64_t>(_read(fd, buffer,
static_cast<uint32_t>(nbytes)));
#else
- *bytes_read = static_cast<int64_t>(read(fd, buffer,
static_cast<size_t>(nbytes)));
+ int64_t n = 0;
+ int64_t N = nbytes;
+ int64_t ret = 0;
+
+ *bytes_read = 0;
+
+ while (ret != -1 && n < N) {
+ int64_t i = std::min(static_cast<int64_t>(INT32_MAX), N - n);
+ ret = static_cast<int64_t>(read(fd, buffer + n, static_cast<size_t>(i)));
+
+ if (ret != -1) {
+ *bytes_read += ret;
+ } else {
+ *bytes_read = ret;
+ }
+
+ n += i;
+ }
#endif
if (*bytes_read == -1) {
- // TODO(wesm): errno to string
- return Status::IOError("Error reading bytes from file");
+ return Status::IOError(std::string("Error reading bytes from file: ") +
+ std::string(strerror(errno)));
}
return Status::OK();
}
static inline Status FileWrite(int fd, const uint8_t* buffer, int64_t nbytes) {
- int ret;
+ int ret = 0;
#if defined(_MSC_VER)
if (nbytes > INT32_MAX) {
return Status::IOError("Unable to write > 2GB blocks to file yet");
}
ret = static_cast<int>(_write(fd, buffer, static_cast<uint32_t>(nbytes)));
#else
- ret = static_cast<int>(write(fd, buffer, static_cast<size_t>(nbytes)));
+ int64_t n = 0;
+ int64_t N = nbytes;
+
+ while (ret != -1 && n < N) {
+ int64_t i = std::min(static_cast<int64_t>(INT32_MAX), N - n);
+ ret = static_cast<int>(write(fd, buffer + n, static_cast<size_t>(i)));
+ n += i;
Review comment:
Similar comments as above.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> [Python] Observed int32 overflow in Feather write/read path
> -----------------------------------------------------------
>
> Key: ARROW-1756
> URL: https://issues.apache.org/jira/browse/ARROW-1756
> Project: Apache Arrow
> Issue Type: Bug
> Components: Python
> Reporter: Wes McKinney
> Assignee: Licht Takeuchi
> Priority: Major
> Labels: pull-request-available
> Fix For: 0.8.0
>
>
> See downstream report
> https://github.com/wesm/feather/issues/321
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
