Changeset: a036c4aa15ec for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a036c4aa15ec
Added Files:
sql/server/rel_remote.c
sql/server/rel_remote.h
sql/test/miscellaneous/Tests/trace_test.stable.out
Modified Files:
common/stream/Makefile.ag
common/stream/stream.c
configure.ag
monetdb5/mal/Tests/tst611.stable.err
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_gencode.c
sql/backends/monet5/sql_gencode.h
sql/backends/monet5/sql_statement.c
sql/backends/monet5/sql_statement.h
sql/common/sql_mem.c
sql/common/sql_string.c
sql/common/sql_string.h
sql/include/sql_catalog.h
sql/include/sql_mem.h
sql/include/sql_relation.h
sql/server/Makefile.ag
sql/server/rel_distribute.h
sql/server/rel_dump.c
sql/server/rel_exp.c
sql/server/rel_exp.h
sql/server/rel_optimizer.c
sql/server/rel_planner.c
sql/server/rel_psm.c
sql/server/rel_psm.h
sql/server/rel_schema.c
sql/server/rel_schema.h
sql/server/rel_select.c
sql/server/rel_select.h
sql/server/rel_updates.c
sql/server/rel_xml.c
sql/server/sql_atom.c
sql/server/sql_atom.h
sql/server/sql_mvc.c
sql/server/sql_mvc.h
sql/server/sql_parser.y
sql/server/sql_semantic.c
sql/server/sql_semantic.h
sql/server/sql_symbol.c
sql/server/sql_symbol.h
sql/storage/sql_catalog.c
sql/test/BugTracker-2015/Tests/constant-random.Bug-3836.sql
sql/test/BugTracker-2015/Tests/constant-random.Bug-3836.stable.out
sql/test/Users/Tests/dropManyUsers.Bug-3764.stable.out
sql/test/miscellaneous/Tests/All
sql/test/miscellaneous/Tests/trace_test.sql
testing/Mfilter.py
Branch: default
Log Message:
Merge with default
diffs (truncated from 3253 to 300 lines):
diff --git a/common/stream/Makefile.ag b/common/stream/Makefile.ag
--- a/common/stream/Makefile.ag
+++ b/common/stream/Makefile.ag
@@ -10,6 +10,7 @@ MTSAFE
INCLUDES = $(zlib_CFLAGS) \
$(BZ_CFLAGS) \
+ $(liblzma_CFLAGS) \
$(openssl_CFLAGS) \
$(curl_CFLAGS)
@@ -19,6 +20,7 @@ lib_stream = {
LIBS = $(SOCKET_LIBS) \
$(zlib_LIBS) \
$(BZ_LIBS) \
+ $(liblzma_LIBS) \
$(openssl_LIBS) \
$(curl_LIBS) \
$(LTLIBICONV)
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -90,6 +90,9 @@
#ifdef HAVE_LIBBZ2
#include <bzlib.h>
#endif
+#ifdef HAVE_LIBLZMA
+#include <lzma.h>
+#endif
#ifdef HAVE_ICONV
#ifdef HAVE_ICONV_H
@@ -1389,6 +1392,295 @@ open_bzwastream(const char *filename, co
#endif
/* ------------------------------------------------------------------ */
+/* streams working on a lzma-compressed disk file */
+
+#ifdef HAVE_LIBLZMA
+#define XZBUFSIZ 64*1024
+typedef struct xz_stream {
+ FILE *fp;
+ lzma_stream strm;
+ int todo;
+ uint8_t buf[XZBUFSIZ];
+} xz_stream;
+
+static ssize_t
+stream_xzread(stream *s, void *buf, size_t elmsize, size_t cnt)
+{
+ xz_stream *xz = s->stream_data.p;
+ size_t size = elmsize * cnt, origsize = size, ressize = 0;
+ uint8_t *outbuf = buf;
+ lzma_action action = LZMA_RUN;
+
+ if (xz == NULL) {
+ s->errnr = MNSTR_READ_ERROR;
+ return -1;
+ }
+
+ xz->strm.next_in = xz->buf;
+ xz->strm.avail_in = xz->todo;
+ xz->strm.next_out = outbuf;
+ xz->strm.avail_out = size;
+ while (size && (xz->strm.avail_in || !feof(xz->fp))) {
+ lzma_ret ret;
+ size_t sz = (size>XZBUFSIZ) ? XZBUFSIZ : size;
+
+ if (!xz->strm.avail_in &&
+ (xz->strm.avail_in = fread(xz->buf, 1, sz, xz->fp)) == 0) {
+ s->errnr = MNSTR_READ_ERROR;
+ return -1;
+ }
+ xz->strm.next_in = xz->buf;
+ if (feof(xz->fp))
+ action = LZMA_FINISH;
+ ret = lzma_code(&xz->strm, action);
+ if (xz->strm.avail_out == 0 || ret == LZMA_STREAM_END) {
+ origsize -= xz->strm.avail_out; /* remaining space */
+ xz->todo = xz->strm.avail_in;
+ if (xz->todo > 0)
+ memmove(xz->buf, xz->strm.next_in, xz->todo);
+ outbuf[origsize] = 0; /* add EOS */
+ ressize = origsize;
+ break;
+ }
+ if (ret != LZMA_OK) {
+ s->errnr = MNSTR_READ_ERROR;
+ return -1;
+ }
+ }
+ if (ressize) {
+#ifdef WIN32
+ /* on Windows when in text mode, convert \r\n line
+ * endings to \n */
+ if (s->type == ST_ASCII) {
+ char *p1, *p2, *pe;
+
+ p1 = buf;
+ pe = p1 + ressize;
+ while (p1 < pe && *p1 != '\r')
+ p1++;
+ p2 = p1;
+ while (p1 < pe) {
+ if (*p1 == '\r' && p1[1] == '\n')
+ ressize--;
+ else
+ *p2++ = *p1;
+ p1++;
+ }
+ }
+#endif
+ return (ssize_t) (ressize / elmsize);
+ }
+ return 0;
+}
+
+static ssize_t
+stream_xzwrite(stream *s, const void *buf, size_t elmsize, size_t cnt)
+{
+ xz_stream *xz = s->stream_data.p;
+ size_t size = elmsize * cnt;
+ lzma_action action = LZMA_RUN;
+
+ if (xz == NULL) {
+ s->errnr = MNSTR_WRITE_ERROR;
+ return -1;
+ }
+
+ xz->strm.next_in = buf;
+ xz->strm.avail_in = size;
+ xz->strm.next_out = xz->buf;
+ xz->strm.avail_out = XZBUFSIZ;
+
+ size = 0;
+ while (xz->strm.avail_in) {
+ size_t sz = 0, isz = xz->strm.avail_in;
+
+ lzma_ret ret = lzma_code(&xz->strm, action);
+ if (xz->strm.avail_out == 0 || ret != LZMA_OK) {
+ s->errnr = MNSTR_WRITE_ERROR;
+ return -1;
+ }
+ sz = XZBUFSIZ - xz->strm.avail_out;
+ if (fwrite(xz->buf, 1, sz, xz->fp) != sz) {
+ s->errnr = MNSTR_WRITE_ERROR;
+ return -1;
+ }
+ assert(xz->strm.avail_in == 0);
+ size += isz;
+ xz->strm.next_out = xz->buf;
+ xz->strm.avail_out = XZBUFSIZ;
+ }
+ if (size)
+ return (ssize_t) (size / elmsize);
+ return (ssize_t) cnt;
+}
+
+static void
+stream_xzclose(stream *s)
+{
+ xz_stream *xz = s->stream_data.p;
+
+ if (xz) {
+ if (s->access == ST_WRITE) {
+ lzma_ret ret = lzma_code(&xz->strm, LZMA_FINISH);
+
+ if (xz->strm.avail_out && ret == LZMA_STREAM_END) {
+ size_t sz = XZBUFSIZ - xz->strm.avail_out;
+ if (fwrite(xz->buf, 1, sz, xz->fp) != sz)
+ s->errnr = MNSTR_WRITE_ERROR;
+ }
+ }
+ fflush(xz->fp);
+ fclose(xz->fp);
+ lzma_end(&xz->strm);
+ free(xz);
+ }
+ s->stream_data.p = NULL;
+}
+
+static int
+stream_xzflush(stream *s)
+{
+ xz_stream *xz = s->stream_data.p;
+
+ if (xz == NULL)
+ return -1;
+ if (s->access == ST_WRITE && fflush(xz->fp))
+ return -1;
+ return 0;
+}
+
+static stream *
+open_xzstream(const char *filename, const char *flags)
+{
+ stream *s;
+ xz_stream *xz;
+ uint32_t preset = 0;
+
+ if ((xz = malloc(sizeof(struct xz_stream))) == NULL)
+ return NULL;
+ if (xz)
+ memset(xz, 0, sizeof(xz_stream));
+ if (((flags[0] == 'r' &&
+ lzma_stream_decoder(&xz->strm, UINT64_MAX, LZMA_CONCATENATED) !=
LZMA_OK)) ||
+ (flags[0] == 'w' &&
+ lzma_easy_encoder(&xz->strm, preset, LZMA_CHECK_CRC64) !=
LZMA_OK)) {
+ free(xz);
+ return NULL;
+ }
+ if ((s = create_stream(filename)) == NULL) {
+ free(xz);
+ return NULL;
+ }
+#ifdef HAVE__WFOPEN
+ {_
+ wchar_t *wfname = utf8towchar(filename);
+ wchar_t *wflags = utf8towchar(flags);
+ if (wfname != NULL)
+ xz->fp = _wfopen(wfname, wflags);
+ } else
+ xz->fp = NULL;
+ if (wfname)
+ free(wfname);
+ if (wflags)
+ free(wflags);
+ }
+#else
+ {
+ char *fname = cvfilename(filename);
+ if (fname) {
+ xz->fp = fopen(fname, flags);
+ free(fname);
+ } else
+ xz->fp = NULL;
+ }
+#endif
+ if (xz->fp == NULL) {
+ destroy(s);
+ free(xz);
+ return NULL;
+ }
+ s->read = stream_xzread;
+ s->write = stream_xzwrite;
+ s->close = stream_xzclose;
+ s->flush = stream_xzflush;
+ s->stream_data.p = (void *) xz;
+ if (flags[0] == 'r' && flags[1] != 'b') {
+ char buf[UTF8BOMLENGTH];
+ if (stream_xzread(s, buf, 1, UTF8BOMLENGTH) == UTF8BOMLENGTH &&
+ strncmp(buf, UTF8BOM, UTF8BOMLENGTH) == 0) {
+ s->isutf8 = 1;
+ } else {
+ rewind(xz->fp);
+ }
+ }
+ return s;
+}
+
+static stream *
+open_xzrstream(const char *filename)
+{
+ stream *s;
+
+ if ((s = open_xzstream(filename, "rb")) == NULL)
+ return NULL;
+ s->type = ST_BIN;
+ if (s->errnr == MNSTR_NO__ERROR &&
+ stream_xzread(s, (void *) &s->byteorder, sizeof(s->byteorder), 1) <
1) {
+ stream_xzclose(s);
+ destroy(s);
+ return NULL;
+ }
+ return s;
+}
+
+static stream *
+open_xzwstream(const char *filename, const char *mode)
+{
+ stream *s;
+
+ if ((s = open_xzstream(filename, mode)) == NULL)
+ return NULL;
+ s->access = ST_WRITE;
+ s->type = ST_BIN;
+ if (s->errnr == MNSTR_NO__ERROR &&
+ stream_xzwrite(s, (void *) &s->byteorder, sizeof(s->byteorder), 1)
< 1) {
+ stream_xzclose(s);
+ destroy(s);
+ return NULL;
+ }
+ return s;
+}
+
+static stream *
+open_xzrastream(const char *filename)
+{
+ stream *s;
+
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
