[arrow] branch master updated: ARROW-1767: [C++] Support file reads and writes over 2GB on Windows

[wesm]

This is an automated email from the ASF dual-hosted git repository.

wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git


The following commit(s) were added to refs/heads/master by this push:
     new 4a33bad  ARROW-1767: [C++] Support file reads and writes over 2GB on 
Windows
4a33bad is described below

commit 4a33bad5f28830812fe4d47dcfdfb184d5ee43c0
Author: Licht-T <lich...@outlook.jp>
AuthorDate: Sun Nov 12 23:49:12 2017 -0500

    ARROW-1767: [C++] Support file reads and writes over 2GB on Windows
    
    This closes [ARROW-1767](https://issues.apache.org/jira/browse/ARROW-1767).
    
    Author: Licht-T <lich...@outlook.jp>
    
    Closes #1311 from Licht-T/feature-large-file-io-windows and squashes the 
following commits:
    
    690d2801 [Licht-T] ENH: Support large file io on Windows
---
 cpp/src/arrow/io/file.cc | 24 ++++++++++--------------
 1 file changed, 10 insertions(+), 14 deletions(-)

diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 057cad1..1ec5e23 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -255,19 +255,18 @@ static inline Status FileSeek(int fd, int64_t pos) {
 
 static inline Status FileRead(const int fd, uint8_t* buffer, const int64_t 
nbytes,
                               int64_t* bytes_read) {
-#if defined(_MSC_VER)
-  if (nbytes > ARROW_MAX_IO_CHUNKSIZE) {
-    return Status::IOError("Unable to read > 2GB blocks yet");
-  }
-  *bytes_read = static_cast<int64_t>(_read(fd, buffer, 
static_cast<uint32_t>(nbytes)));
-#else
   *bytes_read = 0;
 
   while (*bytes_read != -1 && *bytes_read < nbytes) {
     int64_t chunksize =
         std::min(static_cast<int64_t>(ARROW_MAX_IO_CHUNKSIZE), nbytes - 
*bytes_read);
+#if defined(_MSC_VER)
+    int64_t ret = static_cast<int64_t>(
+        _read(fd, buffer + *bytes_read, static_cast<uint32_t>(chunksize)));
+#else
     int64_t ret = static_cast<int64_t>(
         read(fd, buffer + *bytes_read, static_cast<size_t>(chunksize)));
+#endif
 
     if (ret != -1) {
       *bytes_read += ret;
@@ -279,7 +278,6 @@ static inline Status FileRead(const int fd, uint8_t* 
buffer, const int64_t nbyte
       *bytes_read = ret;
     }
   }
-#endif
 
   if (*bytes_read == -1) {
     return Status::IOError(std::string("Error reading bytes from file: ") +
@@ -292,25 +290,23 @@ static inline Status FileRead(const int fd, uint8_t* 
buffer, const int64_t nbyte
 static inline Status FileWrite(const int fd, const uint8_t* buffer,
                                const int64_t nbytes) {
   int ret = 0;
-#if defined(_MSC_VER)
-  if (nbytes > ARROW_MAX_IO_CHUNKSIZE) {
-    return Status::IOError("Unable to write > 2GB blocks to file yet");
-  }
-  ret = static_cast<int>(_write(fd, buffer, static_cast<uint32_t>(nbytes)));
-#else
   int64_t bytes_written = 0;
 
   while (ret != -1 && bytes_written < nbytes) {
     int64_t chunksize =
         std::min(static_cast<int64_t>(ARROW_MAX_IO_CHUNKSIZE), nbytes - 
bytes_written);
+#if defined(_MSC_VER)
+    ret = static_cast<int>(
+        _write(fd, buffer + bytes_written, static_cast<uint32_t>(chunksize)));
+#else
     ret = static_cast<int>(
         write(fd, buffer + bytes_written, static_cast<size_t>(chunksize)));
+#endif
 
     if (ret != -1) {
       bytes_written += ret;
     }
   }
-#endif
 
   if (ret == -1) {
     return Status::IOError(std::string("Error writing bytes from file: ") +

-- 
To stop receiving notification emails like this one, please contact
['"commits@arrow.apache.org" <commits@arrow.apache.org>'].