Changeset: dbc4efe04eb2 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=dbc4efe04eb2
Modified Files:
common/stream/stream.c
Branch: Jul2017
Log Message:
Various small fixes.
- Use modern gzfread/gzfwrite functions (with implementation for
ancient systems we still support);
- Types and cast fixes;
- Implemented loops for some compression write functions.
- Use mode "r" for opening compressed text streams.
diffs (truncated from 654 to 300 lines):
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -142,19 +142,45 @@
#define normal_int_SWAP(s) ((int) _byteswap_ulong((unsigned long) (s)))
#define long_long_SWAP(l) ((lng) _byteswap_uint64((unsigned __int64) (s)))
#else
-#define short_int_SWAP(s) ((short)(((0x00ff&(s))<<8) | ((0xff00&(s))>>8)))
-
-#define normal_int_SWAP(i) (((0x000000ff&(i))<<24) | ((0x0000ff00&(i))<<8) | \
- ((0x00ff0000&(i))>>8) | ((0xff000000&(i))>>24))
-#define long_long_SWAP(l) \
- ((((lng)normal_int_SWAP(l))<<32) |\
- (0xffffffff&normal_int_SWAP(l>>32)))
+#define short_int_SWAP(s) \
+ ((short) (((0x00ff & (unsigned short) (s)) << 8) | \
+ ((0xff00 & (unsigned short) (s)) >> 8)))
+
+#define normal_int_SWAP(i) \
+ ((int) (((((unsigned) 0xff << 0) & (unsigned) (i)) << 24) | \
+ ((((unsigned) 0xff << 8) & (unsigned) (i)) << 8) | \
+ ((((unsigned) 0xff << 16) & (unsigned) (i)) >> 8) | \
+ ((((unsigned) 0xff << 24) & (unsigned) (i)) >> 24)))
+
+#define long_long_SWAP(l) \
+ ((lng) (((((ulng) 0xff << 0) & (ulng) (l)) << 56) | \
+ ((((ulng) 0xff << 8) & (ulng) (l)) << 40) | \
+ ((((ulng) 0xff << 16) & (ulng) (l)) << 24) | \
+ ((((ulng) 0xff << 24) & (ulng) (l)) << 8) | \
+ ((((ulng) 0xff << 32) & (ulng) (l)) >> 8) | \
+ ((((ulng) 0xff << 40) & (ulng) (l)) >> 24) | \
+ ((((ulng) 0xff << 48) & (ulng) (l)) >> 40) | \
+ ((((ulng) 0xff << 56) & (ulng) (l)) >> 56)))
#endif
#ifdef HAVE_HGE
-#define huge_int_SWAP(h) \
- ((((hge)long_long_SWAP(h))<<64) |\
- (0xffffffffffffffff&long_long_SWAP(h>>64)))
+#define huge_int_SWAP(h) \
+ ((hge) (((((uhge) 0xff << 0) & (uhge) (h)) << 120) | \
+ ((((uhge) 0xff << 8) & (uhge) (h)) << 104) | \
+ ((((uhge) 0xff << 16) & (uhge) (h)) << 88) | \
+ ((((uhge) 0xff << 24) & (uhge) (h)) << 72) | \
+ ((((uhge) 0xff << 32) & (uhge) (h)) << 56) | \
+ ((((uhge) 0xff << 40) & (uhge) (h)) << 40) | \
+ ((((uhge) 0xff << 48) & (uhge) (h)) << 24) | \
+ ((((uhge) 0xff << 56) & (uhge) (h)) << 8) | \
+ ((((uhge) 0xff << 64) & (uhge) (h)) >> 8) | \
+ ((((uhge) 0xff << 72) & (uhge) (h)) >> 24) | \
+ ((((uhge) 0xff << 80) & (uhge) (h)) >> 40) | \
+ ((((uhge) 0xff << 88) & (uhge) (h)) >> 56) | \
+ ((((uhge) 0xff << 96) & (uhge) (h)) >> 72) | \
+ ((((uhge) 0xff << 104) & (uhge) (h)) >> 88) | \
+ ((((uhge) 0xff << 112) & (uhge) (h)) >> 104) | \
+ ((((uhge) 0xff << 120) & (uhge) (h)) >> 120)))
#endif
@@ -965,70 +991,94 @@ open_stream(const char *filename, const
/* streams working on a gzip-compressed disk file */
#ifdef HAVE_LIBZ
+#if ZLIB_VERNUM < 0x1290
+/* simplistic version for ancient systems (CentOS 6, Ubuntu Trusty) */
+static z_size_t
+gzfread(void *buf, z_size_t size, z_size_t nitems, gzFile file)
+{
+ unsigned sz = nitems * size > (size_t) 1 << 30 ? 1 << 30 : (unsigned)
(nitems * size);
+ int len;
+
+ len = gzread(fp, buf, sz);
+ if (len == -1)
+ return 0;
+ return (z_size_t) len / size;
+}
+
+static z_size_t
+gzfwrite(const void *buf, z_size_t size, z_size_t nitems, gzFile file)
+{
+ z_size_t sz = nitems * size;
+
+ while (sz > 0) {
+ unsigned len = sz > ((z_size_t) 1 << 30) ? 1 << 30 : (unsigned)
sz;
+ int wlen;
+
+ wlen = gzwrite(file, buf, len);
+ if (wlen <= 0)
+ return 0;
+ buf = (const void *) ((const char *) buf + wlen);
+ sz -= (z_wize_t) wlen;
+ }
+ return nitems;
+}
+#endif
+
static ssize_t
stream_gzread(stream *s, void *buf, size_t elmsize, size_t cnt)
{
gzFile fp = (gzFile) s->stream_data.p;
- int size = (int) (elmsize * cnt);
- int err = 0;
+ z_size_t size;
if (fp == NULL) {
s->errnr = MNSTR_READ_ERROR;
return -1;
}
- if (size && !gzeof(fp)) {
- size = gzread(fp, buf, size);
- if (gzerror(fp, &err) != NULL && err < 0) {
- s->errnr = MNSTR_READ_ERROR;
- return -1;
- }
+ if (elmsize == 0 || cnt == 0)
+ return 0;
+
+ size = gzfread(buf, elmsize, cnt, fp);
#ifdef WIN32
- /* on Windows when in text mode, convert \r\n line
- * endings to \n */
- if (s->type == ST_ASCII) {
- char *p1, *p2, *pe;
-
- p1 = buf;
- pe = p1 + size;
- while (p1 < pe && *p1 != '\r')
- p1++;
- p2 = p1;
- while (p1 < pe) {
- if (*p1 == '\r' && p1[1] == '\n')
- size--;
- else
- *p2++ = *p1;
- p1++;
- }
+ /* on Windows when in text mode, convert \r\n line
+ * endings to \n */
+ if (s->type == ST_ASCII) {
+ char *p1, *p2, *pe;
+
+ p1 = buf;
+ pe = p1 + size;
+ while (p1 < pe && *p1 != '\r')
+ p1++;
+ p2 = p1;
+ while (p1 < pe) {
+ if (*p1 == '\r' && p1[1] == '\n')
+ size--;
+ else
+ *p2++ = *p1;
+ p1++;
}
+ }
#endif
- return (ssize_t) (size / elmsize);
- }
- return 0;
+
+ return size == 0 ? -1 : (ssize_t) size;
}
static ssize_t
stream_gzwrite(stream *s, const void *buf, size_t elmsize, size_t cnt)
{
gzFile fp = (gzFile) s->stream_data.p;
- int size = (int) (elmsize * cnt);
- int err = 0;
+ z_size_t size;
if (fp == NULL) {
s->errnr = MNSTR_WRITE_ERROR;
return -1;
}
- if (size) {
- size = gzwrite(fp, buf, size);
- if (gzerror(fp, &err) != NULL && err < 0) {
- s->errnr = MNSTR_WRITE_ERROR;
- return -1;
- }
- return (ssize_t) (size / elmsize);
- }
- return (ssize_t) cnt;
+ if (elmsize == 0 || cnt == 0)
+ return 0;
+
+ size = gzfwrite(buf, elmsize, cnt, fp);
+ return size == 0 ? -1 : (ssize_t) size;
}
static void
@@ -1138,7 +1188,7 @@ open_gzrastream(const char *filename)
{
stream *s;
- if ((s = open_gzstream(filename, "rb")) == NULL)
+ if ((s = open_gzstream(filename, "r")) == NULL)
return NULL;
s->type = ST_ASCII;
return s;
@@ -1190,7 +1240,7 @@ stream_bzclose(stream *s)
static ssize_t
stream_bzread(stream *s, void *buf, size_t elmsize, size_t cnt)
{
- int size = (int) (elmsize * cnt);
+ size_t size = elmsize * cnt;
int err;
void *punused;
int nunused;
@@ -1203,7 +1253,7 @@ stream_bzread(stream *s, void *buf, size
}
if (size == 0)
return 0;
- size = BZ2_bzRead(&err, bzp->b, buf, size);
+ size = (size_t) BZ2_bzRead(&err, bzp->b, buf, size > ((size_t) 1 << 30)
? 1 << 30 : (int) size);
if (err == BZ_STREAM_END) {
/* end of stream, but not necessarily end of file: get
* unused bits, close stream, and open again with the
@@ -1242,13 +1292,13 @@ stream_bzread(stream *s, void *buf, size
}
}
#endif
- return size / elmsize;
+ return (ssize_t) (size / elmsize);
}
static ssize_t
stream_bzwrite(stream *s, const void *buf, size_t elmsize, size_t cnt)
{
- int size = (int) (elmsize * cnt);
+ size_t size = elmsize * cnt;
int err;
struct bz *bzp = s->stream_data.p;
@@ -1256,15 +1306,19 @@ stream_bzwrite(stream *s, const void *bu
s->errnr = MNSTR_WRITE_ERROR;
return -1;
}
- if (size) {
- BZ2_bzWrite(&err, bzp->b, (void *) buf, size);
+ if (size == 0)
+ return 0;
+ while (size > 0) {
+ int sz = size > (1 << 30) ? 1 << 30 : (int) size;
+ BZ2_bzWrite(&err, bzp->b, (void *) buf, sz);
if (err != BZ_OK) {
+ stream_bzclose(s);
s->errnr = MNSTR_WRITE_ERROR;
return -1;
}
- return cnt;
- }
- return 0;
+ size -= (size_t) sz;
+ }
+ return (ssize_t) cnt;
}
static stream *
@@ -1387,7 +1441,7 @@ open_bzrastream(const char *filename)
{
stream *s;
- if ((s = open_bzstream(filename, "rb")) == NULL)
+ if ((s = open_bzstream(filename, "r")) == NULL)
return NULL;
s->type = ST_ASCII;
return s;
@@ -1419,7 +1473,7 @@ open_bzwastream(const char *filename, co
typedef struct xz_stream {
FILE *fp;
lzma_stream strm;
- int todo;
+ size_t todo;
uint8_t buf[XZBUFSIZ];
} xz_stream;
@@ -1676,7 +1730,7 @@ open_xzrastream(const char *filename)
{
stream *s;
- if ((s = open_xzstream(filename, "rb")) == NULL)
+ if ((s = open_xzstream(filename, "r")) == NULL)
return NULL;
s->type = ST_ASCII;
return s;
@@ -1876,7 +1930,7 @@ struct curl_data {
static struct curl_data *curl_handles;
#endif
-#define BLOCK_CURL (1 << 16)
+#define BLOCK_CURL ((size_t) 1 << 16)
/* this function is called by libcurl when there is data for us */
static size_t
@@ -2108,7 +2162,12 @@ open_urlstream(const char *url)
static ssize_t
socket_write(stream *s, const void *buf, size_t elmsize, size_t cnt)
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
