Changeset: 04f4331c7f9b for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=04f4331c7f9b Added Files: sql/test/BugTracker-2017/Tests/select.Bug-6259.sql sql/test/BugTracker-2017/Tests/select.Bug-6259.stable.err sql/test/BugTracker-2017/Tests/select.Bug-6259.stable.out Modified Files: .hgtags ChangeLog MonetDB.spec clients/Tests/exports.stable.out clients/mapiclient/Tests/mclient--help.stable.err.Windows common/stream/Makefile.ag common/stream/stream.c common/stream/stream.h common/utils/mcrypt.c configure.ag debian/changelog gdk/ChangeLog.Dec2016 gdk/gdk.h gdk/gdk_bat.c gdk/gdk_batop.c gdk/gdk_logger.c gdk/gdk_orderidx.c gdk/gdk_posix.c gdk/gdk_posix.h gdk/gdk_select.c gdk/gdk_storage.c gdk/gdk_utils.c libversions monetdb5/ChangeLog.Dec2016 monetdb5/extras/rapi/rapi.c monetdb5/mal/mal.h monetdb5/mal/mal_client.c monetdb5/mal/mal_client.h monetdb5/mal/mal_profiler.c monetdb5/mal/mal_profiler.h monetdb5/mal/mal_scenario.c monetdb5/mal/mal_session.c monetdb5/mal/mal_session.h monetdb5/mal/mal_stack.c monetdb5/modules/atoms/mtime.c monetdb5/modules/atoms/mtime.h monetdb5/modules/kernel/status.c monetdb5/modules/mal/mal_mapi.c monetdb5/scheduler/run_pipeline.h sql/ChangeLog-Archive sql/ChangeLog.Dec2016 sql/backends/monet5/UDF/pyapi/pyapi.h sql/backends/monet5/sql.c sql/backends/monet5/sql_execute.c sql/backends/monet5/sql_result.c sql/backends/monet5/sql_result.h sql/backends/monet5/sql_scenario.c sql/backends/monet5/sql_scenario.h sql/backends/monet5/sql_upgrades.c sql/server/rel_optimizer.c sql/server/rel_updates.c sql/storage/store.c sql/test/BugTracker-2010/Tests/error-truncated.Bug-2615.stable.err.Darwin sql/test/BugTracker-2015/Tests/crash.Bug-3736.stable.out sql/test/BugTracker-2016/Tests/storagemodel.stable.out sql/test/BugTracker-2017/Tests/All sql/test/SQLite_regress/sqllogictest/Tests/select3.test.stable.out sql/test/SQLite_regress/sqllogictest/Tests/select3.test.stable.out.int128 sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128 sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128 sql/test/pg_regress/Tests/reltime.sql sql/test/pg_regress/Tests/reltime.stable.err sql/test/pg_regress/Tests/reltime.stable.out sql/test/sql_xml/Tests/xml.reqtests sql/test/sys-schema/Tests/check_ForeignKey_referential_integrity.sql sql/test/sys-schema/Tests/check_ForeignKey_referential_integrity.stable.out sql/test/testdb-upgrade-chain-hge/Tests/upgrade.stable.out.int128 sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out.int128 tools/mserver/mserver5.1.in Branch: wlcr Log Message:
Merge with default diffs (truncated from 4642 to 300 lines): diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -662,3 +662,5 @@ 94a35822a8d9dfc6458a2353168e37a21b16ea29 8dcbdc30cd49d1d03e3a21b37671d567d84943ae Dec2016_15 94a35822a8d9dfc6458a2353168e37a21b16ea29 Dec2016_SP3_release 8dcbdc30cd49d1d03e3a21b37671d567d84943ae Dec2016_SP3_release +07824a1c7beb75308b813378af33c243f1c2f511 Dec2016_17 +07824a1c7beb75308b813378af33c243f1c2f511 Dec2016_SP4_release diff --git a/ChangeLog b/ChangeLog --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,9 @@ # ChangeLog file for devel # This file is updated with Maddlog +* Thu April 13 2017 Mark Raasveldt <m.raasve...@cwi.nl> +- Added a new server-side protocol implementation. The new protocol is backwards compatible with the old protocol. Clients can choose whether they want to use the old or the new protocol during the initial handshake with the server. The new protocol is a binary column-based protocol that is significantly faster than the old protocol when transferring large result sets. In addition, the new protocol supports compression using Snappy or LZ4. + * Thu Jan 12 2017 Martin van Dinther <martin.van.dint...@monetdbsolutions.com> - Added 5 new sys schema tables: function_languages, function_types, key_types, index_types and privilege_codes. They are pre-loaded with diff --git a/MonetDB.spec b/MonetDB.spec --- a/MonetDB.spec +++ b/MonetDB.spec @@ -132,7 +132,7 @@ Vendor: MonetDB BV <i...@monetdb.org> Group: Applications/Databases License: MPLv2.0 URL: http://www.monetdb.org/ -Source: http://dev.monetdb.org/downloads/sources/Dec2016-SP3/%{name}-%{version}.tar.bz2 +Source: http://dev.monetdb.org/downloads/sources/Dec2016-SP4/%{name}-%{version}.tar.bz2 # we need systemd for the _unitdir macro to exist %if %{?rhel:0}%{!?rhel:1} || 0%{?rhel} >= 7 @@ -953,6 +953,22 @@ rm -f %{buildroot}%{_bindir}/Maddlog %postun -p /sbin/ldconfig %changelog +* Tue Apr 11 2017 Sjoerd Mullender <sjo...@acm.org> - 11.25.17-20170411 +- Rebuilt. +- BZ#6110: cast of a SQL boolean value to a string or clob or (var)char + is wrong +- BZ#6254: Crash (and assertion failure) after querying a view which + uses a correlated subquery in the select-list +- BZ#6256: Assertion Trigger on FULL OUTER JOIN with more than two + BETWEEN clauses +- BZ#6257: wrong count values (1 instead of 0) for correlated aggregation + queries +- BZ#6258: Vulnerability in FITS and NETCDF data vaults + +* Tue Apr 11 2017 Sjoerd Mullender <sjo...@acm.org> - 11.25.17-20170411 +- sql: Upgrade code was added for an old change in the sys.settimeout function. +- sql: A bug was fixed with the automatic "vacuum" operation on system tables. + * Thu Mar 30 2017 Sjoerd Mullender <sjo...@acm.org> - 11.25.15-20170330 - Rebuilt. - BZ#6250: Assertion failure when querying a Blob column with order diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -298,7 +298,6 @@ size_t MT_getrss(void); void MT_init(void); int MT_join_thread(MT_Id t); int MT_lockf(char *filename, int mode, off_t off, off_t len); -struct Mallinfo MT_mallinfo(void); void *MT_mmap(const char *path, int mode, size_t len); int MT_munmap(void *p, size_t len); int MT_path_absolute(const char *path); @@ -1393,7 +1392,7 @@ char *MSP_locate_sqlscript(const char *m void MSinitClientPrg(Client cntxt, str mod, str nme); void MSresetInstructions(MalBlkPtr mb, int start); void MSresetVariables(Client cntxt, MalBlkPtr mb, MalStkPtr glb, int start); -void MSscheduleClient(str command, str challenge, bstream *fin, stream *fout); +void MSscheduleClient(str command, str challenge, bstream *fin, stream *fout, protocol_version protocol, size_t blocksize, int compute_column_widths); void MSserveClient(void *dummy); str MTIMEcompute_rule_foryear(date *ret, const rule *val, const int *year); str MTIMEcurrent_date(date *d); @@ -1444,6 +1443,8 @@ str MTIMEdaytime_extract_sql_seconds_bul str MTIMEdaytime_fromstr(daytime *ret, const char *const *s); str MTIMEepilogue(void *ret); str MTIMEepoch2int(int *res, const timestamp *ts); +str MTIMEepoch2lng(lng *res, const timestamp *ts); +str MTIMEepoch_bulk(bat *ret, bat *bid); str MTIMElocal_timezone(lng *res); str MTIMEmonth_from_str(int *ret, const char *const *month); str MTIMEmonth_to_str(str *ret, const int *month); @@ -2604,6 +2605,15 @@ str zero_or_oneRef; stream *append_wastream(const char *filename); stream *append_wstream(const char *filename); stream *block_stream(stream *s); +stream *block_stream2(stream *s, size_t bufsiz, compression_method comp, column_compression colcomp); +buffer bs2_buffer(stream *s); +column_compression bs2_colcomp(stream *ss); +void bs2_resetbuf(stream *ss); +int bs2_resizebuf(stream *ss, size_t bufsiz); +void bs2_setpos(stream *ss, size_t pos); +void *bs2_stealbuf(stream *ss); +stream *bs_stealstream(stream *s); +stream *bs_stream(stream *s); bstream *bstream_create(stream *rs, size_t chunk_size); void bstream_destroy(bstream *s); ssize_t bstream_next(bstream *s); @@ -2625,6 +2635,7 @@ size_t getFileSize(stream *s); stream *iconv_rstream(stream *ss, const char *charset, const char *name); stream *iconv_wstream(stream *ss, const char *charset, const char *name); int isa_block_stream(stream *s); +int isa_fixed_block_stream(stream *s); int mnstr_byteorder(stream *s); void mnstr_clearerr(stream *s); void mnstr_close(stream *s); @@ -2643,6 +2654,7 @@ int mnstr_printf(stream *s, _In_z_ _Prin ssize_t mnstr_read(stream *s, void *buf, size_t elmsize, size_t cnt); int mnstr_readBte(stream *s, signed char *val); int mnstr_readBteArray(stream *s, signed char *val, size_t cnt); +int mnstr_readChr(stream *s, char *val); int mnstr_readHge(stream *s, hge *val); int mnstr_readHgeArray(stream *s, hge *val, size_t cnt); int mnstr_readInt(stream *s, int *val); @@ -2651,6 +2663,7 @@ int mnstr_readLng(stream *s, lng *val); int mnstr_readLngArray(stream *s, lng *val, size_t cnt); int mnstr_readSht(stream *s, short *val); int mnstr_readShtArray(stream *s, short *val, size_t cnt); +int mnstr_readStr(stream *s, char *val); ssize_t mnstr_read_block(stream *s, void *buf, size_t elmsize, size_t cnt); ssize_t mnstr_readline(stream *s, void *buf, size_t maxcnt); stream *mnstr_rstream(stream *s); @@ -2660,6 +2673,9 @@ int mnstr_type(stream *s); ssize_t mnstr_write(stream *s, const void *buf, size_t elmsize, size_t cnt); int mnstr_writeBte(stream *s, signed char val); int mnstr_writeBteArray(stream *s, const signed char *val, size_t cnt); +int mnstr_writeChr(stream *s, char val); +int mnstr_writeDbl(stream *s, double val); +int mnstr_writeFlt(stream *s, float val); int mnstr_writeHge(stream *s, hge val); int mnstr_writeHgeArray(stream *s, const hge *val, size_t cnt); int mnstr_writeInt(stream *s, int val); @@ -2668,6 +2684,7 @@ int mnstr_writeLng(stream *s, lng val); int mnstr_writeLngArray(stream *s, const lng *val, size_t cnt); int mnstr_writeSht(stream *s, short val); int mnstr_writeShtArray(stream *s, const short *val, size_t cnt); +int mnstr_writeStr(stream *s, const char *val); stream *mnstr_wstream(stream *s); stream *open_rastream(const char *filename); stream *open_rstream(const char *filename); diff --git a/clients/mapiclient/Tests/mclient--help.stable.err.Windows b/clients/mapiclient/Tests/mclient--help.stable.err.Windows --- a/clients/mapiclient/Tests/mclient--help.stable.err.Windows +++ b/clients/mapiclient/Tests/mclient--help.stable.err.Windows @@ -20,7 +20,7 @@ Options are: -e | --echo echo the query -E charset | --encoding=charset specify encoding (character set) of the terminal -f kind | --format=kind specify output format {csv,tab,raw,sql,xml} - -H | --history load/save cmdline history (default off) + -H | --history load/save cmdline history (default off) -i | --interactive[=tm] interpret `\' commands on stdin, use time formatting {ms,s,m} -l language | --language=lang {sql,mal} -L logfile | --log=logfile save client/server interaction diff --git a/common/stream/Makefile.ag b/common/stream/Makefile.ag --- a/common/stream/Makefile.ag +++ b/common/stream/Makefile.ag @@ -10,6 +10,8 @@ MTSAFE INCLUDES = $(zlib_CFLAGS) \ $(BZ_CFLAGS) \ + $(snappy_CFLAGS) \ + $(lz4_CFLAGS) \ $(liblzma_CFLAGS) \ $(openssl_CFLAGS) \ $(curl_CFLAGS) @@ -20,6 +22,8 @@ lib_stream = { LIBS = $(SOCKET_LIBS) \ $(zlib_LIBS) \ $(BZ_LIBS) \ + $(snappy_LIBS) \ + $(lz4_LIBS) \ $(liblzma_LIBS) \ $(openssl_LIBS) \ $(curl_LIBS) \ diff --git a/common/stream/stream.c b/common/stream/stream.c --- a/common/stream/stream.c +++ b/common/stream/stream.c @@ -93,6 +93,12 @@ #ifdef HAVE_LIBLZMA #include <lzma.h> #endif +#ifdef HAVE_LIBSNAPPY +#include <snappy-c.h> +#endif +#ifdef HAVE_LIBLZ4 +#include <lz4.h> +#endif #ifdef HAVE_ICONV #ifdef HAVE_ICONV_H @@ -3570,6 +3576,8 @@ buffer_wastream(buffer *b, const char *n return s; } + + /* ------------------------------------------------------------------ */ /* A buffered stream consists of a sequence of blocks. Each block @@ -3896,7 +3904,6 @@ bs_close(stream *ss) assert(s); if (s == NULL) return; - assert(s->s); if (s->s) s->s->close(s->s); } @@ -3909,7 +3916,6 @@ bs_destroy(stream *ss) s = (bs *) ss->stream_data.p; assert(s); if (s) { - assert(s->s); if (s->s) s->s->destroy(s->s); free(s); @@ -3924,6 +3930,19 @@ bs_clrerr(stream *s) mnstr_clearerr(((bs *) s->stream_data.p)->s); } +stream* bs_stream(stream *s) { + assert(isa_block_stream(s)); + return ((bs*)s->stream_data.p)->s; +} + +stream* bs_stealstream(stream *s) { + stream *res; + assert(isa_block_stream(s)); + res = ((bs*)s->stream_data.p)->s; + ((bs*)s->stream_data.p)->s = NULL; + return res; +} + stream * block_stream(stream *s) { @@ -3960,14 +3979,664 @@ block_stream(stream *s) return ns; } +typedef struct bs2 { + stream *s; /* underlying stream */ + size_t nr; /* how far we got in buf */ + size_t itotal; /* amount available in current read block */ + size_t bufsiz; + size_t readpos; + compression_method comp; + column_compression colcomp; + char *compbuf; + size_t compbufsiz; + char *buf; +} bs2; + + +static ssize_t +compress_stream_data(bs2 *s) { + assert(s->comp != COMPRESSION_NONE); + if (s->comp == COMPRESSION_SNAPPY) { +#ifdef HAVE_LIBSNAPPY + size_t compressed_length = s->compbufsiz; + snappy_status ret; + if ((ret = snappy_compress(s->buf, s->nr, s->compbuf, &compressed_length)) != SNAPPY_OK) { + s->s->errnr = (int) ret; + return -1; + } + return compressed_length; +#else + assert(0); + return -1; +#endif + } else if (s->comp == COMPRESSION_LZ4) { +#ifdef HAVE_LIBLZ4 + int compressed_length = (int) s->compbufsiz; + if ((compressed_length = LZ4_compress_fast(s->buf, s->compbuf, s->nr, compressed_length, 1)) == 0) { + s->s->errnr = -1; + return -1; + } + return compressed_length; +#else + assert(0); + return -1; +#endif + } + return -1; +} + + +static ssize_t +decompress_stream_data(bs2 *s) { + assert(s->comp != COMPRESSION_NONE); + if (s->comp == COMPRESSION_SNAPPY) { +#ifdef HAVE_LIBSNAPPY + snappy_status ret; + size_t uncompressed_length = s->bufsiz; + if ((ret = snappy_uncompress(s->compbuf, s->itotal, s->buf, &uncompressed_length)) != SNAPPY_OK) { + s->s->errnr = (int) ret; + return -1; + } _______________________________________________ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
