Hello,
the code for handling external file reads and writes currently does not
handle the case when the read() or write() operation returns a number
smaller than the requested amount.
Linux currently transfers at most 0x7ffff000 bytes per read() or
write(), meaning that if more than 2GB are read from an external file
only 2GB are read and the rest is filled up with zeros (because HDF5
thinks that it has reached EOF).
http://man7.org/linux/man-pages/man2/read.2.html#NOTES
https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=758170#10
(Note that the problem does not occur if the files datatype and the
memory datatype do not match, because in this case HDF5 will use smaller
reads into a buffer for converting the values.)
I've attached a patch which will restart the read() or write() operation
if a number smaller than the requested amount is returned (similar to
e.g. the code in H5FDcore.c). I've also added logic which will restart
the syscall after EINTR.
Best regards,
Steffen Kieß
Index: src/H5Defl.c
===================================================================
--- src/H5Defl.c (revision 28707)
+++ src/H5Defl.c (working copy)
@@ -298,14 +298,25 @@
#else /* NDEBUG */
to_read = MIN((size_t)(efl->slot[u].size - skip), size);
#endif /* NDEBUG */
- if((n = HDread(fd, buf, to_read)) < 0)
- HGOTO_ERROR(H5E_EFL, H5E_READERROR, FAIL, "read error in external raw data file")
- else if((size_t)n < to_read)
- HDmemset(buf + n, 0, to_read - (size_t)n);
+ while (to_read > 0) {
+ do {
+ n = HDread(fd, buf, to_read);
+ } while(-1 == n && EINTR == errno);
+ if(n < 0) {
+ HGOTO_ERROR(H5E_EFL, H5E_READERROR, FAIL, "read error in external raw data file")
+ } else if(n == 0) { // EOF
+ HDmemset(buf, 0, to_read);
+ size -= to_read;
+ buf += to_read;
+ to_read = 0;
+ } else {
+ size -= (size_t)n;
+ buf += n;
+ to_read -= (size_t)n;
+ }
+ }
HDclose(fd);
fd = -1;
- size -= to_read;
- buf += to_read;
skip = 0;
u++;
} /* end while */
@@ -346,6 +357,7 @@
#endif /* NDEBUG */
haddr_t cur;
hsize_t skip = 0;
+ ssize_t n;
size_t u; /* Local index variable */
herr_t ret_value = SUCCEED; /* Return value */
@@ -388,12 +400,20 @@
#else /* NDEBUG */
to_write = MIN((size_t)(efl->slot[u].size - skip), size);
#endif /* NDEBUG */
- if((size_t)HDwrite(fd, buf, to_write) != to_write)
- HGOTO_ERROR(H5E_EFL, H5E_READERROR, FAIL, "write error in external raw data file")
+ while (to_write > 0) {
+ do {
+ n = HDwrite(fd, buf, to_write);
+ } while(-1 == n && EINTR == errno);
+ if(n <= 0) { // write() should never return 0 if to_write is non-zero, treat as error
+ HGOTO_ERROR(H5E_EFL, H5E_WRITEERROR, FAIL, "write error in external raw data file")
+ } else {
+ size -= (size_t)n;
+ buf += n;
+ to_write -= (size_t)n;
+ }
+ }
HDclose (fd);
fd = -1;
- size -= to_write;
- buf += to_write;
skip = 0;
u++;
} /* end while */
_______________________________________________
Hdf-forum is for HDF software users discussion.
Hdf-forum@lists.hdfgroup.org
http://lists.hdfgroup.org/mailman/listinfo/hdf-forum_lists.hdfgroup.org
Twitter: https://twitter.com/hdf5
