Changeset: 4654c2290d0e for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4654c2290d0e
Added Files:
sql/test/BugTracker-2017/Tests/select.Bug-6259.sql
sql/test/BugTracker-2017/Tests/select.Bug-6259.stable.err
sql/test/BugTracker-2017/Tests/select.Bug-6259.stable.out
Modified Files:
ChangeLog
clients/Tests/exports.stable.out
clients/mapiclient/Tests/mclient--help.stable.err.Windows
common/stream/Makefile.ag
common/stream/stream.c
common/stream/stream.h
common/utils/mcrypt.c
configure.ag
gdk/ChangeLog.Dec2016
gdk/gdk.h
gdk/gdk_logger.c
gdk/gdk_posix.c
gdk/gdk_posix.h
gdk/gdk_select.c
gdk/gdk_storage.c
gdk/gdk_utils.c
monetdb5/ChangeLog.Dec2016
monetdb5/extras/rapi/rapi.c
monetdb5/mal/mal.h
monetdb5/mal/mal_client.c
monetdb5/mal/mal_client.h
monetdb5/mal/mal_profiler.c
monetdb5/mal/mal_profiler.h
monetdb5/mal/mal_scenario.c
monetdb5/mal/mal_session.c
monetdb5/mal/mal_session.h
monetdb5/modules/atoms/mtime.c
monetdb5/modules/atoms/mtime.h
monetdb5/modules/kernel/status.c
monetdb5/modules/mal/mal_mapi.c
monetdb5/scheduler/run_pipeline.h
sql/ChangeLog.Dec2016
sql/backends/monet5/UDF/pyapi/pyapi.h
sql/backends/monet5/sql.c
sql/backends/monet5/sql_execute.c
sql/backends/monet5/sql_result.c
sql/backends/monet5/sql_result.h
sql/backends/monet5/sql_scenario.c
sql/backends/monet5/sql_scenario.h
sql/test/BugTracker-2010/Tests/error-truncated.Bug-2615.stable.err.Darwin
sql/test/BugTracker-2017/Tests/All
sql/test/SQLite_regress/sqllogictest/Tests/select3.test.stable.out
sql/test/SQLite_regress/sqllogictest/Tests/select3.test.stable.out.int128
sql/test/pg_regress/Tests/reltime.sql
sql/test/pg_regress/Tests/reltime.stable.err
sql/test/pg_regress/Tests/reltime.stable.out
sql/test/sys-schema/Tests/check_ForeignKey_referential_integrity.sql
sql/test/sys-schema/Tests/check_ForeignKey_referential_integrity.stable.out
tools/mserver/mserver5.1.in
Branch: data-vaults
Log Message:
Merge with default
diffs (truncated from 3784 to 300 lines):
diff --git a/ChangeLog b/ChangeLog
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,9 @@
# ChangeLog file for devel
# This file is updated with Maddlog
+* Thu April 13 2017 Mark Raasveldt <m.raasve...@cwi.nl>
+- Added a new server-side protocol implementation. The new protocol is
backwards compatible with the old protocol. Clients can choose whether they
want to use the old or the new protocol during the initial handshake with the
server. The new protocol is a binary column-based protocol that is
significantly faster than the old protocol when transferring large result sets.
In addition, the new protocol supports compression using Snappy or LZ4.
+
* Thu Jan 12 2017 Martin van Dinther <martin.van.dint...@monetdbsolutions.com>
- Added 5 new sys schema tables: function_languages, function_types,
key_types, index_types and privilege_codes. They are pre-loaded with
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -298,7 +298,6 @@ size_t MT_getrss(void);
void MT_init(void);
int MT_join_thread(MT_Id t);
int MT_lockf(char *filename, int mode, off_t off, off_t len);
-struct Mallinfo MT_mallinfo(void);
void *MT_mmap(const char *path, int mode, size_t len);
int MT_munmap(void *p, size_t len);
int MT_path_absolute(const char *path);
@@ -1393,7 +1392,7 @@ char *MSP_locate_sqlscript(const char *m
void MSinitClientPrg(Client cntxt, str mod, str nme);
void MSresetInstructions(MalBlkPtr mb, int start);
void MSresetVariables(Client cntxt, MalBlkPtr mb, MalStkPtr glb, int start);
-void MSscheduleClient(str command, str challenge, bstream *fin, stream *fout);
+void MSscheduleClient(str command, str challenge, bstream *fin, stream *fout,
protocol_version protocol, size_t blocksize, int compute_column_widths);
void MSserveClient(void *dummy);
str MTIMEcompute_rule_foryear(date *ret, const rule *val, const int *year);
str MTIMEcurrent_date(date *d);
@@ -1444,6 +1443,8 @@ str MTIMEdaytime_extract_sql_seconds_bul
str MTIMEdaytime_fromstr(daytime *ret, const char *const *s);
str MTIMEepilogue(void *ret);
str MTIMEepoch2int(int *res, const timestamp *ts);
+str MTIMEepoch2lng(lng *res, const timestamp *ts);
+str MTIMEepoch_bulk(bat *ret, bat *bid);
str MTIMElocal_timezone(lng *res);
str MTIMEmonth_from_str(int *ret, const char *const *month);
str MTIMEmonth_to_str(str *ret, const int *month);
@@ -2568,6 +2569,15 @@ str zero_or_oneRef;
stream *append_wastream(const char *filename);
stream *append_wstream(const char *filename);
stream *block_stream(stream *s);
+stream *block_stream2(stream *s, size_t bufsiz, compression_method comp,
column_compression colcomp);
+buffer bs2_buffer(stream *s);
+column_compression bs2_colcomp(stream *ss);
+void bs2_resetbuf(stream *ss);
+int bs2_resizebuf(stream *ss, size_t bufsiz);
+void bs2_setpos(stream *ss, size_t pos);
+void *bs2_stealbuf(stream *ss);
+stream *bs_stealstream(stream *s);
+stream *bs_stream(stream *s);
bstream *bstream_create(stream *rs, size_t chunk_size);
void bstream_destroy(bstream *s);
ssize_t bstream_next(bstream *s);
@@ -2589,6 +2599,7 @@ size_t getFileSize(stream *s);
stream *iconv_rstream(stream *ss, const char *charset, const char *name);
stream *iconv_wstream(stream *ss, const char *charset, const char *name);
int isa_block_stream(stream *s);
+int isa_fixed_block_stream(stream *s);
int mnstr_byteorder(stream *s);
void mnstr_clearerr(stream *s);
void mnstr_close(stream *s);
@@ -2607,6 +2618,7 @@ int mnstr_printf(stream *s, _In_z_ _Prin
ssize_t mnstr_read(stream *s, void *buf, size_t elmsize, size_t cnt);
int mnstr_readBte(stream *s, signed char *val);
int mnstr_readBteArray(stream *s, signed char *val, size_t cnt);
+int mnstr_readChr(stream *s, char *val);
int mnstr_readHge(stream *s, hge *val);
int mnstr_readHgeArray(stream *s, hge *val, size_t cnt);
int mnstr_readInt(stream *s, int *val);
@@ -2615,6 +2627,7 @@ int mnstr_readLng(stream *s, lng *val);
int mnstr_readLngArray(stream *s, lng *val, size_t cnt);
int mnstr_readSht(stream *s, short *val);
int mnstr_readShtArray(stream *s, short *val, size_t cnt);
+int mnstr_readStr(stream *s, char *val);
ssize_t mnstr_read_block(stream *s, void *buf, size_t elmsize, size_t cnt);
ssize_t mnstr_readline(stream *s, void *buf, size_t maxcnt);
stream *mnstr_rstream(stream *s);
@@ -2624,6 +2637,9 @@ int mnstr_type(stream *s);
ssize_t mnstr_write(stream *s, const void *buf, size_t elmsize, size_t cnt);
int mnstr_writeBte(stream *s, signed char val);
int mnstr_writeBteArray(stream *s, const signed char *val, size_t cnt);
+int mnstr_writeChr(stream *s, char val);
+int mnstr_writeDbl(stream *s, double val);
+int mnstr_writeFlt(stream *s, float val);
int mnstr_writeHge(stream *s, hge val);
int mnstr_writeHgeArray(stream *s, const hge *val, size_t cnt);
int mnstr_writeInt(stream *s, int val);
@@ -2632,6 +2648,7 @@ int mnstr_writeLng(stream *s, lng val);
int mnstr_writeLngArray(stream *s, const lng *val, size_t cnt);
int mnstr_writeSht(stream *s, short val);
int mnstr_writeShtArray(stream *s, const short *val, size_t cnt);
+int mnstr_writeStr(stream *s, const char *val);
stream *mnstr_wstream(stream *s);
stream *open_rastream(const char *filename);
stream *open_rstream(const char *filename);
diff --git a/clients/mapiclient/Tests/mclient--help.stable.err.Windows
b/clients/mapiclient/Tests/mclient--help.stable.err.Windows
--- a/clients/mapiclient/Tests/mclient--help.stable.err.Windows
+++ b/clients/mapiclient/Tests/mclient--help.stable.err.Windows
@@ -20,7 +20,7 @@ Options are:
-e | --echo echo the query
-E charset | --encoding=charset specify encoding (character set) of the
terminal
-f kind | --format=kind specify output format {csv,tab,raw,sql,xml}
- -H | --history load/save cmdline history (default off)
+ -H | --history load/save cmdline history (default off)
-i | --interactive[=tm] interpret `\' commands on stdin, use time
formatting {ms,s,m}
-l language | --language=lang {sql,mal}
-L logfile | --log=logfile save client/server interaction
diff --git a/common/stream/Makefile.ag b/common/stream/Makefile.ag
--- a/common/stream/Makefile.ag
+++ b/common/stream/Makefile.ag
@@ -10,6 +10,8 @@ MTSAFE
INCLUDES = $(zlib_CFLAGS) \
$(BZ_CFLAGS) \
+ $(snappy_CFLAGS) \
+ $(lz4_CFLAGS) \
$(liblzma_CFLAGS) \
$(openssl_CFLAGS) \
$(curl_CFLAGS)
@@ -20,6 +22,8 @@ lib_stream = {
LIBS = $(SOCKET_LIBS) \
$(zlib_LIBS) \
$(BZ_LIBS) \
+ $(snappy_LIBS) \
+ $(lz4_LIBS) \
$(liblzma_LIBS) \
$(openssl_LIBS) \
$(curl_LIBS) \
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -93,6 +93,12 @@
#ifdef HAVE_LIBLZMA
#include <lzma.h>
#endif
+#ifdef HAVE_LIBSNAPPY
+#include <snappy-c.h>
+#endif
+#ifdef HAVE_LIBLZ4
+#include <lz4.h>
+#endif
#ifdef HAVE_ICONV
#ifdef HAVE_ICONV_H
@@ -3570,6 +3576,8 @@ buffer_wastream(buffer *b, const char *n
return s;
}
+
+
/* ------------------------------------------------------------------ */
/* A buffered stream consists of a sequence of blocks. Each block
@@ -3896,7 +3904,6 @@ bs_close(stream *ss)
assert(s);
if (s == NULL)
return;
- assert(s->s);
if (s->s)
s->s->close(s->s);
}
@@ -3909,7 +3916,6 @@ bs_destroy(stream *ss)
s = (bs *) ss->stream_data.p;
assert(s);
if (s) {
- assert(s->s);
if (s->s)
s->s->destroy(s->s);
free(s);
@@ -3924,6 +3930,19 @@ bs_clrerr(stream *s)
mnstr_clearerr(((bs *) s->stream_data.p)->s);
}
+stream* bs_stream(stream *s) {
+ assert(isa_block_stream(s));
+ return ((bs*)s->stream_data.p)->s;
+}
+
+stream* bs_stealstream(stream *s) {
+ stream *res;
+ assert(isa_block_stream(s));
+ res = ((bs*)s->stream_data.p)->s;
+ ((bs*)s->stream_data.p)->s = NULL;
+ return res;
+}
+
stream *
block_stream(stream *s)
{
@@ -3960,14 +3979,664 @@ block_stream(stream *s)
return ns;
}
+typedef struct bs2 {
+ stream *s; /* underlying stream */
+ size_t nr; /* how far we got in buf */
+ size_t itotal; /* amount available in current read block */
+ size_t bufsiz;
+ size_t readpos;
+ compression_method comp;
+ column_compression colcomp;
+ char *compbuf;
+ size_t compbufsiz;
+ char *buf;
+} bs2;
+
+
+static ssize_t
+compress_stream_data(bs2 *s) {
+ assert(s->comp != COMPRESSION_NONE);
+ if (s->comp == COMPRESSION_SNAPPY) {
+#ifdef HAVE_LIBSNAPPY
+ size_t compressed_length = s->compbufsiz;
+ snappy_status ret;
+ if ((ret = snappy_compress(s->buf, s->nr, s->compbuf,
&compressed_length)) != SNAPPY_OK) {
+ s->s->errnr = (int) ret;
+ return -1;
+ }
+ return compressed_length;
+#else
+ assert(0);
+ return -1;
+#endif
+ } else if (s->comp == COMPRESSION_LZ4) {
+#ifdef HAVE_LIBLZ4
+ int compressed_length = (int) s->compbufsiz;
+ if ((compressed_length = LZ4_compress_fast(s->buf, s->compbuf,
s->nr, compressed_length, 1)) == 0) {
+ s->s->errnr = -1;
+ return -1;
+ }
+ return compressed_length;
+#else
+ assert(0);
+ return -1;
+#endif
+ }
+ return -1;
+}
+
+
+static ssize_t
+decompress_stream_data(bs2 *s) {
+ assert(s->comp != COMPRESSION_NONE);
+ if (s->comp == COMPRESSION_SNAPPY) {
+#ifdef HAVE_LIBSNAPPY
+ snappy_status ret;
+ size_t uncompressed_length = s->bufsiz;
+ if ((ret = snappy_uncompress(s->compbuf, s->itotal, s->buf,
&uncompressed_length)) != SNAPPY_OK) {
+ s->s->errnr = (int) ret;
+ return -1;
+ }
+ return (ssize_t) uncompressed_length;
+#else
+ assert(0);
+ return -1;
+#endif
+ } else if (s->comp == COMPRESSION_LZ4) {
+#ifdef HAVE_LIBLZ4
+ int uncompressed_length = (int) s->bufsiz;
+ if ((uncompressed_length = LZ4_decompress_safe(s->compbuf,
s->buf, s->itotal, uncompressed_length)) <= 0) {
+ s->s->errnr = uncompressed_length;
+ return -1;
+ }
+ return uncompressed_length;
+#else
+ assert(0);
+ return -1;
+#endif
+ }
+ return -1;
+}
+
+static ssize_t
+compression_size_bound(bs2 *s) {
+ if (s->comp == COMPRESSION_NONE) {
+ return 0;
+ } else if (s->comp == COMPRESSION_SNAPPY) {
+#ifndef HAVE_LIBSNAPPY
+ return -1;
+#else
+ return snappy_max_compressed_length(s->bufsiz);
+#endif
+ } else if (s->comp == COMPRESSION_LZ4) {
+#ifndef HAVE_LIBLZ4
+ return -1;
+#else
+ return LZ4_compressBound(s->bufsiz);
+#endif
+ }
+ return -1;
+}
+
+static bs2 *
+bs2_create(stream *s, size_t bufsiz, compression_method comp)
+{
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
