MonetDB: Mar2018 - Merge with Jul2017 branch.
Changeset: 8cbeb3388a48 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8cbeb3388a48 Modified Files: clients/mapiclient/mclient.c Branch: Mar2018 Log Message: Merge with Jul2017 branch. diffs (66 lines): diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c --- a/clients/mapiclient/mclient.c +++ b/clients/mapiclient/mclient.c @@ -1632,27 +1632,27 @@ SQLrenderer(MapiHdl hdl, char singleinst SQLseparator(len, printfields, '-'); rows = mapi_get_row_count(hdl); snprintf(buf, sizeof(buf), "%" PRId64 " rows", rows); - printf("%" PRId64 " tuple%s", rows, rows != 1 ? "s" : ""); + mnstr_printf(toConsole, "%" PRId64 " tuple%s", rows, rows != 1 ? "s" : ""); if (fields != printfields || croppedfields > 0) - printf(" !"); + mnstr_printf(toConsole, " !"); if (fields != printfields) { rows = fields - printfields; - printf("%" PRId64 " column%s dropped", rows, rows != 1 ? "s" : ""); + mnstr_printf(toConsole, "%" PRId64 " column%s dropped", rows, rows != 1 ? "s" : ""); } if (fields != printfields && croppedfields > 0) - printf(", "); + mnstr_printf(toConsole, ", "); if (croppedfields > 0) - printf("%d field%s truncated", + mnstr_printf(toConsole, "%d field%s truncated", croppedfields, croppedfields != 1 ? "s" : ""); if (fields != printfields || croppedfields > 0) { - printf("!"); + mnstr_printf(toConsole, "!"); if (firstcrop == 1) { firstcrop = 0; - printf("\nnote: to disable dropping columns and/or truncating fields use \\w-1"); + mnstr_printf(toConsole, "\nnote: to disable dropping columns and/or truncating fields use \\w-1"); } } - printf("\n"); + mnstr_printf(toConsole, "\n"); free(len); free(hdr); @@ -1837,7 +1837,7 @@ format_result(Mapi mid, MapiHdl hdl, cha formatter == TESTformatter) mnstr_printf(toConsole, "[ %" PRId64 "\t]\n", mapi_rows_affected(hdl)); else if (formatter == TRASHformatter) { - printf("%s\n", timerHuman(sqloptimizer, maloptimizer, querytime)); + mnstr_printf(toConsole, "%s\n", timerHuman(sqloptimizer, maloptimizer, querytime)); } else { aff = mapi_rows_affected(hdl); lid = mapi_get_last_id(hdl); @@ -1866,7 +1866,7 @@ format_result(Mapi mid, MapiHdl hdl, cha timerHuman(sqloptimizer, maloptimizer, querytime)); mnstr_printf(toConsole, "\n"); } else if (formatter == TRASHformatter) { - printf("%s\n", timerHuman(sqloptimizer, maloptimizer, querytime)); + mnstr_printf(toConsole, "%s\n", timerHuman(sqloptimizer, maloptimizer, querytime)); } continue; case Q_TRANS: @@ -1956,7 +1956,7 @@ format_result(Mapi mid, MapiHdl hdl, cha } s= timerHuman(sqloptimizer, maloptimizer, querytime); if (*s) - printf("%s\n", s); + mnstr_printf(toConsole, "%s\n", s); } } while (!mnstr_errnr(toConsole) && (rc = mapi_next_result(hdl)) == 1); if (mnstr_errnr(toConsole)) { ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Jul2017 - mclient: consistently use mnstr_printf(toCons...
Changeset: bf27f9cbba74 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bf27f9cbba74 Modified Files: clients/mapiclient/mclient.c Branch: Jul2017 Log Message: mclient: consistently use mnstr_printf(toConsole, ...) instead of occasional printf(...); in this way, all output goes through the pager and/of iconv, if used. diffs (69 lines): diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c --- a/clients/mapiclient/mclient.c +++ b/clients/mapiclient/mclient.c @@ -1306,7 +1306,7 @@ static void TIMERrenderer(MapiHdl hdl) { SQLqueryEcho(hdl); - printf("%s\n", timerHuman()); + mnstr_printf(toConsole, "%s\n", timerHuman()); } @@ -1673,30 +1673,30 @@ SQLrenderer(MapiHdl hdl, char singleinst #if 0 mark2 = strdup(buf);/* for the timer output */ #endif - printf(LLFMT " tuple%s%s%s%s", rows, rows != 1 ? "s" : "", + mnstr_printf(toConsole, LLFMT " tuple%s%s%s%s", rows, rows != 1 ? "s" : "", singleinstr ? " (" : "", singleinstr && formatter != TESTformatter ? timerHuman() : "", singleinstr ? ")" : ""); if (fields != printfields || croppedfields > 0) - printf(" !"); + mnstr_printf(toConsole, " !"); if (fields != printfields) { rows = fields - printfields; - printf(LLFMT " column%s dropped", rows, rows != 1 ? "s" : ""); + mnstr_printf(toConsole, LLFMT " column%s dropped", rows, rows != 1 ? "s" : ""); } if (fields != printfields && croppedfields > 0) - printf(", "); + mnstr_printf(toConsole, ", "); if (croppedfields > 0) - printf("%d field%s truncated", + mnstr_printf(toConsole, "%d field%s truncated", croppedfields, croppedfields != 1 ? "s" : ""); if (fields != printfields || croppedfields > 0) { - printf("!"); + mnstr_printf(toConsole, "!"); if (firstcrop == 1) { firstcrop = 0; - printf("\nnote: to disable dropping columns and/or truncating fields use \\w-1"); + mnstr_printf(toConsole, "\nnote: to disable dropping columns and/or truncating fields use \\w-1"); } } - printf("\n"); + mnstr_printf(toConsole, "\n"); free(len); free(hdr); @@ -1864,7 +1864,7 @@ format_result(Mapi mid, MapiHdl hdl, cha formatter == TESTformatter) mnstr_printf(toConsole, "[ " LLFMT "\t]\n", mapi_rows_affected(hdl)); else if (formatter == TIMERformatter) - printf("%s\n", timerHuman()); + mnstr_printf(toConsole, "%s\n", timerHuman()); else { aff = mapi_rows_affected(hdl); lid = mapi_get_last_id(hdl); @@ -1893,7 +1893,7 @@ format_result(Mapi mid, MapiHdl hdl, cha timerHuman()); mnstr_printf(toConsole, "\n"); } else if (formatter == TIMERformatter) - printf("%s\n", timerHuman()); + mnstr_printf(toConsole, "%s\n", timerHuman()); continue; case Q_TRANS: SQLqueryEcho(hdl); ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Mar2018 - mclient: avoid skipping resultsets twice with...
Changeset: 8e6b6b94b838 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8e6b6b94b838 Modified Files: clients/mapiclient/mclient.c Branch: Mar2018 Log Message: mclient: avoid skipping resultsets twice with (new) "trash" output format diffs (19 lines): diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c --- a/clients/mapiclient/mclient.c +++ b/clients/mapiclient/mclient.c @@ -1837,7 +1837,6 @@ format_result(Mapi mid, MapiHdl hdl, cha formatter == TESTformatter) mnstr_printf(toConsole, "[ %" PRId64 "\t]\n", mapi_rows_affected(hdl)); else if (formatter == TRASHformatter) { - mapi_next_result(hdl); printf("%s\n", timerHuman(sqloptimizer, maloptimizer, querytime)); } else { aff = mapi_rows_affected(hdl); @@ -1867,7 +1866,6 @@ format_result(Mapi mid, MapiHdl hdl, cha timerHuman(sqloptimizer, maloptimizer, querytime)); mnstr_printf(toConsole, "\n"); } else if (formatter == TRASHformatter) { - mapi_next_result(hdl); printf("%s\n", timerHuman(sqloptimizer, maloptimizer, querytime)); } continue; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Mar2018 - Merge with Jul2017 branch.
Changeset: a68070b785c0 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a68070b785c0 Modified Files: clients/mapiclient/mclient.c Branch: Mar2018 Log Message: Merge with Jul2017 branch. ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Jul2017 - mclient, TIMERrenderer(): do not skip results...
Changeset: 0b4a81e079d4 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0b4a81e079d4 Modified Files: clients/mapiclient/mclient.c Branch: Jul2017 Log Message: mclient, TIMERrenderer(): do not skip resultset twice no need to skip resultset in TIMERrenderer(), the loop over resultsets does that already. diffs (11 lines): diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c --- a/clients/mapiclient/mclient.c +++ b/clients/mapiclient/mclient.c @@ -1306,7 +1306,6 @@ static void TIMERrenderer(MapiHdl hdl) { SQLqueryEcho(hdl); - mapi_next_result(hdl); printf("%s\n", timerHuman()); } ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Tests for lz4 streams
Changeset: 9794b18f936b for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9794b18f936b Added Files: sql/test/Tests/copy-from-lz4.py sql/test/Tests/copy-from-lz4.stable.err sql/test/Tests/copy-from-lz4.stable.out sql/test/Tests/copy-into-lz4.sql.in sql/test/Tests/copy-into-lz4.stable.err sql/test/Tests/copy-into-lz4.stable.out sql/test/Tests/lz4-dump.sql sql/test/Tests/testlz4.lz4 Modified Files: ChangeLog README-Debian README-Fedora sql/test/Tests/All testing/Mtest.py.in Branch: default Log Message: Tests for lz4 streams diffs (truncated from 1476 to 300 lines): diff --git a/ChangeLog b/ChangeLog --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ # ChangeLog file for devel # This file is updated with Maddlog +* Tue Feb 13 2018 Pedro Ferreira+- Added support for lz4 compressed files on stream library diff --git a/README-Debian b/README-Debian --- a/README-Debian +++ b/README-Debian @@ -35,7 +35,7 @@ zlib1g-dev# optional: read and write . These packages are optional. libcfitsio-dev # optional: read FITS files -liblz4-dev # optional: compression in new (unused) MAPI protocol +liblz4-dev # optional: compression in new (unused) MAPI protocol, also used to read and write .lz4 compressed files libsnappy-dev # optional: compression in new (unused) MAPI protocol python3-dev# optional, needed for Python 3 integration python3-numpy # optional, needed for Python 3 integration diff --git a/README-Fedora b/README-Fedora --- a/README-Fedora +++ b/README-Fedora @@ -36,7 +36,7 @@ These packages are optional. cfitsio-devel # optional: read FITS files libasan# optional: --enable-sanitizer configuration (debug) liblas-devel # optional: read LIDAR files -lz4-devel # optional: compression in new (unused) MAPI protocol +lz4-devel # optional: compression in new (unused) MAPI protocol, also used to read and write .lz4 compressed files netcdf-devel # optional: read NetCDF files proj-devel # optional, only optionally used in geom module python3-devel # optional, needed for Python 3 integration diff --git a/sql/test/Tests/All b/sql/test/Tests/All --- a/sql/test/Tests/All +++ b/sql/test/Tests/All @@ -129,3 +129,6 @@ constant-not-in fsum THREADS=1?fsum1 + +NOT_WIN32_LIBLZ4?copy-into-lz4 +NOT_WIN32_LIBLZ4?copy-from-lz4 diff --git a/sql/test/Tests/copy-from-lz4.py b/sql/test/Tests/copy-from-lz4.py new file mode 100644 --- /dev/null +++ b/sql/test/Tests/copy-from-lz4.py @@ -0,0 +1,28 @@ +import os, sys + +try: +from MonetDBtesting import process +except ImportError: +import process + +def try_remove_file(): # maybe file locks would do better +try: +os.remove("/tmp/testing-dump.lz4") +except: +pass + +try_remove_file() + +s = process.server(args = [], stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) + +c = process.client('sql', stdin = open(os.path.join(os.getenv('TSTSRCDIR'), 'lz4-dump.sql')), + stdout = process.PIPE, stderr = process.PIPE, log = True) +out, err = c.communicate() +sys.stdout.write(out) +sys.stderr.write(err) + +out, err = s.communicate() +sys.stdout.write(out) +sys.stderr.write(err) + +try_remove_file() diff --git a/sql/test/Tests/copy-from-lz4.stable.err b/sql/test/Tests/copy-from-lz4.stable.err new file mode 100644 --- /dev/null +++ b/sql/test/Tests/copy-from-lz4.stable.err @@ -0,0 +1,34 @@ +stderr of test 'copy-from-lz4` in directory 'sql/test` itself: + + +# 18:01:50 > +# 18:01:50 > "/usr/bin/python2" "copy-from-lz4.py" "copy-from-lz4" +# 18:01:50 > + + +# 18:01:50 > +# 18:01:50 > Mtimeout -timeout 60 mclient -lsql -ftest -tnone -Eutf-8 -i -e --host=/var/tmp/mtest-16374 --port=35324 --database=mTests_sql_test < "/home/ferreira/repositories/MonetDB-default/src/sql/test/Tests/lz4-dump.sql" +# 18:01:50 > + +# builtin opt gdk_dbpath = /home/ferreira/repositories/MonetDB-default/BUILD/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 35324 +# cmdline opt mapi_usock = /var/tmp/mtest-16374/.s.monetdb.35324 +# cmdline opt monet_prompt = +# cmdline opt gdk_dbpath = /home/ferreira/repositories/MonetDB-default/BUILD/var/MonetDB/mTests_sql_test +# cmdline opt gdk_debug = 553648138 + +# 18:01:50 > +# 18:01:50 > "Done." +# 18:01:50 > + diff --git a/sql/test/Tests/copy-from-lz4.stable.out
MonetDB: default - Merge with Mar2018 branch.
Changeset: b8be1823eb29 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b8be1823eb29 Modified Files: buildtools/ChangeLog.Jul2017 clients/Tests/mclient-uri.SQL.sh clients/mapiclient/mclient.c gdk/gdk_batop.c gdk/gdk_project.c monetdb5/optimizer/opt_pushselect.c sql/backends/monet5/sql_cat.c sql/server/rel_psm.c sql/server/rel_schema.c sql/server/sql_mvc.c sql/server/sql_mvc.h sql/storage/sql_storage.h sql/storage/store.c Branch: default Log Message: Merge with Mar2018 branch. diffs (truncated from 1556 to 300 lines): diff --git a/buildtools/ChangeLog.Jul2017 b/buildtools/ChangeLog.Jul2017 --- a/buildtools/ChangeLog.Jul2017 +++ b/buildtools/ChangeLog.Jul2017 @@ -1,3 +1,7 @@ # ChangeLog file for buildtools # This file is updated with Maddlog +* Mon Feb 12 2018 Sjoerd Mullender+- Added the .pdb files needed for debug symbols to the Windows installer + for MonetDB/SQL. + diff --git a/clients/Tests/mclient-uri.SQL.sh b/clients/Tests/mclient-uri.SQL.sh --- a/clients/Tests/mclient-uri.SQL.sh +++ b/clients/Tests/mclient-uri.SQL.sh @@ -2,7 +2,7 @@ # test the URI parsing capabilities of the MAPI library -Mlog "mclient -d mapi:monetdb://$HOST:$MAPIPORT/$TSTDB?language=sql=monetdb -f test -t none-E utf-8 -s select 1" +Mlog "mclient -d mapi:monetdb://$HOST:$MAPIPORT/$TSTDB?language=sql=monetdb -f test -t none -E utf-8 -s select 1" mclient -d "mapi:monetdb://$HOST:$MAPIPORT/$TSTDB?language=sql=monetdb" -f test -E utf-8 -s 'select 1' Mlog "mclient -d mapi:monetdb://$MAPIHOST/.s.monetdb.$MAPIPORT?database=$TSTDB=sql=monetdb -f test -t none -E utf-8 -s select 1" diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c --- a/clients/mapiclient/mclient.c +++ b/clients/mapiclient/mclient.c @@ -88,9 +88,8 @@ enum formatters { TABLEformatter, // render as a bordered table CSVformatter, // render as a comma separate file XMLformatter, // render as a valid XML document - JSONformatter, // render as a valid JSON document TESTformatter, // for testing, escape characters - TRASHformatter, // remove the result set + TRASHformatter, // remove the result set SAMformatter, // render a SAM result set EXPANDEDformatter // render as multi-row single record }; @@ -127,8 +126,8 @@ static char *pager = 0; /* use external static int rowsperpage = 0;/* for SQL pagination */ static int pagewidth = 0; /* -1: take whatever is necessary, >0: limit */ static int pagewidthset = 0; /* whether the user set the width explicitly */ -static int croppedfields = 0; /* whatever got cropped/truncated */ -static char firstcrop = 1; /* first time we see cropping/truncation */ +static int croppedfields = 0; /* whatever got cropped/truncated */ +static char firstcrop = 1; /* first time we see cropping/truncation */ enum modifiers { NOmodifier, @@ -268,33 +267,31 @@ timerHuman(int64_t sqloptimizer, int64_t { timertype t = th - t0; - - (void) sqloptimizer; - if (timermode == T_CLOCK){ - if( t / 1000 < 950) { + if (timermode == T_CLOCK) { + if (t / 1000 < 950) { snprintf(htimbuf, sizeof(htimbuf), "clk: " TTFMT ".%03d ms" , t / 1000, (int) (t % 1000)); - return(htimbuf); + return htimbuf; } t /= 1000; if (t / 1000 < 60) { snprintf(htimbuf, sizeof(htimbuf), "clk: " TTFMT ".%02d sec", t / 1000, (int) ((t % 1000) / 100)); - return(htimbuf); + return htimbuf; } t /= 1000; snprintf(htimbuf, sizeof(htimbuf), "clk: " TTFMT ":%02d min", t / 60, (int) (t % 60)); - return(htimbuf); + return htimbuf; } /* for performance measures we use milliseconds as the base */ - if (timermode == T_PERF){ - snprintf(htimbuf, sizeof(htimbuf), "clk:%" PRId64 ".%03d sql:%" PRId64 ".%03d opt:%" PRId64 ".%03d run:%" PRId64 ".%03d ms", - t / 1000, (int)(t % 1000), - sqloptimizer/1000, (int)(sqloptimizer % 1000), - maloptimizer /1000, (int)(maloptimizer % 1000), - querytime /1000, (int)(querytime % 1000)); - return(htimbuf); + if (timermode == T_PERF) { + snprintf(htimbuf, sizeof(htimbuf), "clk:%" PRId64 ".%03d sql:%" PRId64 ".%03d opt:%" PRId64 ".%03d run:%" PRId64 ".%03d ms", +t / 1000, (int) (t % 1000), +sqloptimizer / 1000, (int) (sqloptimizer % 1000), +maloptimizer /
MonetDB: Mar2018 - Merge with Jul2017 branch.
Changeset: 045a9013d4dd for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=045a9013d4dd Modified Files: buildtools/ChangeLog.Jul2017 gdk/gdk_project.c Branch: Mar2018 Log Message: Merge with Jul2017 branch. diffs (26 lines): diff --git a/buildtools/ChangeLog.Jul2017 b/buildtools/ChangeLog.Jul2017 --- a/buildtools/ChangeLog.Jul2017 +++ b/buildtools/ChangeLog.Jul2017 @@ -1,3 +1,7 @@ # ChangeLog file for buildtools # This file is updated with Maddlog +* Mon Feb 12 2018 Sjoerd Mullender+- Added the .pdb files needed for debug symbols to the Windows installer + for MonetDB/SQL. + diff --git a/gdk/gdk_project.c b/gdk/gdk_project.c --- a/gdk/gdk_project.c +++ b/gdk/gdk_project.c @@ -226,9 +226,10 @@ BATproject(BAT *l, BAT *r) bn->tkey ? "-key" : ""); return bn; } + /* if l has type void, it is either empty or not dense (i.e. nil) */ if (l->ttype == TYPE_void || BATcount(l) == 0 || (r->ttype == TYPE_void && is_oid_nil(r->tseqbase))) { - /* trivial: all values are nil */ + /* trivial: all values are nil (includes no entries at all) */ const void *nil = ATOMnilptr(r->ttype); bn = BATconstant(l->hseqbase, r->ttype == TYPE_oid ? TYPE_void : r->ttype, ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Mar2018 - Layout.
Changeset: 13001411ac28 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=13001411ac28 Modified Files: clients/mapiclient/mclient.c Branch: Mar2018 Log Message: Layout. diffs (truncated from 308 to 300 lines): diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c --- a/clients/mapiclient/mclient.c +++ b/clients/mapiclient/mclient.c @@ -89,7 +89,7 @@ enum formatters { CSVformatter, // render as a comma separate file XMLformatter, // render as a valid XML document TESTformatter, // for testing, escape characters - TRASHformatter, // remove the result set + TRASHformatter, // remove the result set SAMformatter, // render a SAM result set EXPANDEDformatter // render as multi-row single record }; @@ -126,8 +126,8 @@ static char *pager = 0; /* use external static int rowsperpage = 0;/* for SQL pagination */ static int pagewidth = 0; /* -1: take whatever is necessary, >0: limit */ static int pagewidthset = 0; /* whether the user set the width explicitly */ -static int croppedfields = 0; /* whatever got cropped/truncated */ -static char firstcrop = 1; /* first time we see cropping/truncation */ +static int croppedfields = 0; /* whatever got cropped/truncated */ +static char firstcrop = 1; /* first time we see cropping/truncation */ enum modifiers { NOmodifier, @@ -267,33 +267,31 @@ timerHuman(int64_t sqloptimizer, int64_t { timertype t = th - t0; - - (void) sqloptimizer; - if (timermode == T_CLOCK){ - if( t / 1000 < 950) { + if (timermode == T_CLOCK) { + if (t / 1000 < 950) { snprintf(htimbuf, sizeof(htimbuf), "clk: " TTFMT ".%03d ms" , t / 1000, (int) (t % 1000)); - return(htimbuf); + return htimbuf; } t /= 1000; if (t / 1000 < 60) { snprintf(htimbuf, sizeof(htimbuf), "clk: " TTFMT ".%02d sec", t / 1000, (int) ((t % 1000) / 100)); - return(htimbuf); + return htimbuf; } t /= 1000; snprintf(htimbuf, sizeof(htimbuf), "clk: " TTFMT ":%02d min", t / 60, (int) (t % 60)); - return(htimbuf); + return htimbuf; } /* for performance measures we use milliseconds as the base */ - if (timermode == T_PERF){ - snprintf(htimbuf, sizeof(htimbuf), "clk:%" PRId64 ".%03d sql:%" PRId64 ".%03d opt:%" PRId64 ".%03d run:%" PRId64 ".%03d ms", -t / 1000, (int)(t % 1000), -sqloptimizer/1000, (int)(sqloptimizer % 1000), -maloptimizer /1000, (int)(maloptimizer % 1000), -querytime /1000, (int)(querytime % 1000)); - return(htimbuf); + if (timermode == T_PERF) { + snprintf(htimbuf, sizeof(htimbuf), "clk:%" PRId64 ".%03d sql:%" PRId64 ".%03d opt:%" PRId64 ".%03d run:%" PRId64 ".%03d ms", +t / 1000, (int) (t % 1000), +sqloptimizer / 1000, (int) (sqloptimizer % 1000), +maloptimizer / 1000, (int) (maloptimizer % 1000), +querytime / 1000, (int) (querytime % 1000)); + return htimbuf; } htimbuf[0] = 0; - return(htimbuf); + return htimbuf; } /* The Mapi library eats away the comment lines, which we need to @@ -554,7 +552,7 @@ SQLrow(int *len, int *numeric, char **re size_t ulen; int *cutafter = malloc(sizeof(int) * fields); - if (cutafter == NULL){ + if (cutafter == NULL) { fprintf(stderr,"Malloc for SQLrow failed"); exit(2); } @@ -1353,7 +1351,7 @@ SQLheader(MapiHdl hdl, int *len, int fie char **names = (char **) malloc(fields * sizeof(char *)); int *numeric = (int *) malloc(fields * sizeof(int)); - if (names == NULL || numeric == NULL){ + if (names == NULL || numeric == NULL) { free(names); free(numeric); fprintf(stderr,"Malloc for SQLheader failed"); @@ -1437,7 +1435,7 @@ SQLrenderer(MapiHdl hdl, char singleinst hdr = calloc(fields, sizeof(*hdr)); rest = calloc(fields, sizeof(*rest)); numeric = calloc(fields, sizeof(*numeric)); - if(len == NULL || hdr == NULL || rest == NULL || numeric == NULL) { + if (len == NULL || hdr == NULL || rest == NULL || numeric == NULL) { free(len); free(hdr); free(rest); @@ -1762,14 +1760,14 @@ start_pager(stream **saveFD) else { *saveFD = toConsole;
MonetDB: Mar2018 - cleaned-up "bad C code" that sneaked-in with ...
Changeset: ae856682eaae for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ae856682eaae Modified Files: clients/mapiclient/mclient.c Branch: Mar2018 Log Message: cleaned-up "bad C code" that sneaked-in with changeset 359ebf5841f7: no need for an extra branch in a switch statement diffs (25 lines): diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c --- a/clients/mapiclient/mclient.c +++ b/clients/mapiclient/mclient.c @@ -1927,6 +1927,8 @@ format_result(Mapi mid, MapiHdl hdl, cha else { char *s; switch (formatter) { + case TRASHformatter: + break; case XMLformatter: XMLrenderer(hdl); break; @@ -1952,9 +1954,9 @@ format_result(Mapi mid, MapiHdl hdl, cha case EXPANDEDformatter: EXPANDEDrenderer(hdl); break; - default: - if ( formatter != TRASHformatter) - RAWrenderer(hdl); + default: + RAWrenderer(hdl); + break; } s= timerHuman(sqloptimizer, maloptimizer, querytime); if (*s) ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Mar2018 - Add DISABLES CODE comment.
Changeset: 7cc764b8654a for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7cc764b8654a Modified Files: monetdb5/optimizer/opt_pushselect.c Branch: Mar2018 Log Message: Add DISABLES CODE comment. diffs (12 lines): diff --git a/monetdb5/optimizer/opt_pushselect.c b/monetdb5/optimizer/opt_pushselect.c --- a/monetdb5/optimizer/opt_pushselect.c +++ b/monetdb5/optimizer/opt_pushselect.c @@ -181,7 +181,7 @@ OPTpushselectImplementation(Client cntxt if (getModuleId(p) == sqlRef && getFunctionId(p) == deltaRef) push_down_delta++; - if (0 && getModuleId(p) == sqlRef && getFunctionId(p) == tidRef) { /* rewrite equal table ids */ + if (/* DISABLES CODE */ (0) && getModuleId(p) == sqlRef && getFunctionId(p) == tidRef) { /* rewrite equal table ids */ int sname = getArg(p, 2), tname = getArg(p, 3), s; for (s = 0; s < subselects.nr; s++) { ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Mar2018 - Use shift instead of multiply.
Changeset: 5665216e98e5 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5665216e98e5 Modified Files: gdk/gdk_batop.c Branch: Mar2018 Log Message: Use shift instead of multiply. diffs (22 lines): diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c --- a/gdk/gdk_batop.c +++ b/gdk/gdk_batop.c @@ -199,8 +199,7 @@ insert_string_bat(BAT *b, BAT *n, BAT *s if (toff == 0 && n->twidth == b->twidth && cand == NULL) { /* we don't need to do any translation of offset * values, so we can use fast memcpy */ - memcpy(Tloc(b, BUNlast(b)), Tloc(n, start), - cnt * n->twidth); + memcpy(Tloc(b, BUNlast(b)), Tloc(n, start), cnt << n->tshift); BATsetcount(b, BATcount(b) + cnt); } else if (toff != ~(size_t) 0) { /* we don't need to insert any actual strings since we @@ -444,7 +443,7 @@ append_varsized_bat(BAT *b, BAT *n, BAT * chunk of memory */ memcpy(Tloc(b, BUNlast(b)), Tloc(n, start), - cnt * b->twidth); + cnt << b->tshift); } else { var_t *restrict dst = (var_t *) Tloc(b, BUNlast(b)); oid hseq = n->hseqbase; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Mar2018 - Indent.
Changeset: 8ab35f1e4146 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8ab35f1e4146 Modified Files: clients/mapiclient/mclient.c Branch: Mar2018 Log Message: Indent. diffs (298 lines): diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c --- a/clients/mapiclient/mclient.c +++ b/clients/mapiclient/mclient.c @@ -285,11 +285,11 @@ timerHuman(int64_t sqloptimizer, int64_t } /* for performance measures we use milliseconds as the base */ if (timermode == T_PERF){ - snprintf(htimbuf, sizeof(htimbuf), "clk:%" PRId64 ".%03d sql:%" PRId64 ".%03d opt:%" PRId64 ".%03d run:%" PRId64 ".%03d ms", - t / 1000, (int)(t % 1000), - sqloptimizer/1000, (int)(sqloptimizer % 1000), - maloptimizer /1000, (int)(maloptimizer % 1000), - querytime /1000, (int)(querytime % 1000)); + snprintf(htimbuf, sizeof(htimbuf), "clk:%" PRId64 ".%03d sql:%" PRId64 ".%03d opt:%" PRId64 ".%03d run:%" PRId64 ".%03d ms", +t / 1000, (int)(t % 1000), +sqloptimizer/1000, (int)(sqloptimizer % 1000), +maloptimizer /1000, (int)(maloptimizer % 1000), +querytime /1000, (int)(querytime % 1000)); return(htimbuf); } htimbuf[0] = 0; @@ -579,8 +579,8 @@ SQLrow(int *len, int *numeric, char **re for (i = 0; i < fields; i++) { if (rest[i] == NULL || *rest[i] == 0) { mnstr_printf(toConsole, "%c %*s ", - first ? '|' : i > 0 && cutafter[i - 1] == 0 ? '>' : ':', - len[i], ""); +first ? '|' : i > 0 && cutafter[i - 1] == 0 ? '>' : ':', +len[i], ""); } else { ulen = utf8strlen(rest[i], NULL); @@ -677,7 +677,7 @@ SQLrow(int *len, int *numeric, char **re more = 1; } else { mnstr_printf(toConsole, "%c", - first ? '|' : i > 0 && cutafter[i - 1] == 0 ? '>' : ':'); +first ? '|' : i > 0 && cutafter[i - 1] == 0 ? '>' : ':'); if (numeric[i]) { mnstr_printf(toConsole, "%*s", (int) (len[i] - ulen), @@ -711,8 +711,8 @@ SQLrow(int *len, int *numeric, char **re } } mnstr_printf(toConsole, "%c%s\n", - first ? '|' : i > 0 && cutafter[i - 1] == 0 ? '>' : ':', - wm ? ">" : ""); +first ? '|' : i > 0 && cutafter[i - 1] == 0 ? '>' : ':', +wm ? ">" : ""); first = 0; rows++; } while (more); @@ -1543,7 +1543,7 @@ SQLrenderer(MapiHdl hdl, char singleinst for (i = 0; i < printfields; i++) { if (hdr[i] > len[i]) { if (max == -1 || - hdr[max] - len[max] < hdr[i] - len[i]) + hdr[max] - len[max] < hdr[i] - len[i]) max = i; } } @@ -1588,8 +1588,8 @@ SQLrenderer(MapiHdl hdl, char singleinst continue; if (rfields != fields) { mnstr_printf(stderr_stream, - "invalid tuple received from server, " - "got %d columns, expected %d, ignoring\n", rfields, fields); +"invalid tuple received from server, " +"got %d columns, expected %d, ignoring\n", rfields, fields); continue; } if (silent) @@ -1646,7 +1646,7 @@ SQLrenderer(MapiHdl hdl, char singleinst printf(", "); if (croppedfields > 0) printf("%d field%s truncated", - croppedfields, croppedfields != 1 ? "s" : ""); + croppedfields, croppedfields != 1 ? "s" : ""); if (fields != printfields || croppedfields > 0) { printf("!"); if (firstcrop == 1) { @@ -1730,11 +1730,13 @@ setWidth(void) pagewidth = ws.ws_col; else #endif + { #ifdef WIN32
MonetDB: default - Indent.
Changeset: 90aae7a9a53f for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=90aae7a9a53f Modified Files: common/stream/stream.c Branch: default Log Message: Indent. diffs (70 lines): diff --git a/common/stream/stream.c b/common/stream/stream.c --- a/common/stream/stream.c +++ b/common/stream/stream.c @@ -1744,7 +1744,7 @@ stream_lz4read(stream *restrict s, void remaining_to_decompress = size - total_read; total_decompressed = lz4->ring_buffer_size - lz4->total_processing; ret = LZ4F_decompress(lz4->context.dec_context, (char*)buf + total_read, _to_decompress, - (char*)lz4->ring_buffer + lz4->total_processing, _decompressed, NULL); + (char*)lz4->ring_buffer + lz4->total_processing, _decompressed, NULL); if(LZ4F_isError(ret)) { s->errnr = MNSTR_WRITE_ERROR; return -1; @@ -1755,23 +1755,23 @@ stream_lz4read(stream *restrict s, void } #ifdef WIN32 - /* on Windows when in text mode, convert \r\n line endings to \n */ - if (s->type == ST_ASCII) { - char *p1, *p2, *pe; - - p1 = buf; - pe = p1 + total_read; - while (p1 < pe && *p1 != '\r') - p1++; - p2 = p1; - while (p1 < pe) { - if (*p1 == '\r' && p1[1] == '\n') - total_read--; - else - *p2++ = *p1; - p1++; - } + /* on Windows when in text mode, convert \r\n line endings to \n */ + if (s->type == ST_ASCII) { + char *p1, *p2, *pe; + + p1 = buf; + pe = p1 + total_read; + while (p1 < pe && *p1 != '\r') + p1++; + p2 = p1; + while (p1 < pe) { + if (*p1 == '\r' && p1[1] == '\n') + total_read--; + else + *p2++ = *p1; + p1++; } + } #endif return (ssize_t) (total_read / elmsize); } @@ -1803,7 +1803,7 @@ stream_lz4write(stream *restrict s, cons assert(next_batch > 0); ret = LZ4F_compressUpdate(lz4->context.comp_context, ((char*)lz4->ring_buffer) + lz4->total_processing, - available, ((char*)buf) + total_written, next_batch, NULL); + available, ((char*)buf) + total_written, next_batch, NULL); if(LZ4F_isError(ret)) { s->errnr = MNSTR_WRITE_ERROR; return -1; @@ -2003,7 +2003,7 @@ open_lz4stream(const char *restrict file } else if (flags[0] == 'r' && flags[1] != 'b') { /* check for utf-8 encoding */ char buf[UTF8BOMLENGTH]; if (stream_lz4read(s, buf, 1, UTF8BOMLENGTH) == UTF8BOMLENGTH && - strncmp(buf, UTF8BOM, UTF8BOMLENGTH) == 0) { + strncmp(buf, UTF8BOM, UTF8BOMLENGTH) == 0) { s->isutf8 = 1; } else { rewind(lz4->fp); ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Added support for lz4 compressed files streams
Changeset: f0a793843277 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f0a793843277 Modified Files: common/stream/stream.c Branch: default Log Message: Added support for lz4 compressed files streams diffs (truncated from 432 to 300 lines): diff --git a/common/stream/stream.c b/common/stream/stream.c --- a/common/stream/stream.c +++ b/common/stream/stream.c @@ -93,6 +93,7 @@ #endif #ifdef HAVE_LIBLZ4 #include +#include #endif #ifdef HAVE_ICONV @@ -1700,6 +1701,384 @@ open_xzwastream(const char *restrict fil #endif /* -- */ +/* streams working on a lz4-compressed disk file */ + +#ifdef HAVE_LIBLZ4 +#define LZ4DECOMPBUFSIZ 128*1024 +typedef struct lz4_stream { + FILE *fp; + size_t total_processing; + size_t ring_buffer_size; + void* ring_buffer; + union { + LZ4F_compressionContext_t comp_context; + LZ4F_decompressionContext_t dec_context; + } context; +} lz4_stream; + +static ssize_t +stream_lz4read(stream *restrict s, void *restrict buf, size_t elmsize, size_t cnt) +{ + lz4_stream *lz4 = s->stream_data.p; + size_t size = elmsize * cnt, total_read = 0, total_decompressed, ret, remaining_to_decompress; + + if (lz4 == NULL || size <= 0) { + s->errnr = MNSTR_READ_ERROR; + return -1; + } + + while (total_read < size) { + if (lz4->total_processing == lz4->ring_buffer_size) { + if(feof(lz4->fp)) { + break; + } else { + lz4->ring_buffer_size = fread(lz4->ring_buffer, 1, LZ4_COMPRESSBOUND(LZ4DECOMPBUFSIZ), lz4->fp); + if (lz4->ring_buffer_size == 0 || ferror(lz4->fp)) { + s->errnr = MNSTR_READ_ERROR; + return -1; + } + lz4->total_processing = 0; + } + } + + remaining_to_decompress = size - total_read; + total_decompressed = lz4->ring_buffer_size - lz4->total_processing; + ret = LZ4F_decompress(lz4->context.dec_context, (char*)buf + total_read, _to_decompress, + (char*)lz4->ring_buffer + lz4->total_processing, _decompressed, NULL); + if(LZ4F_isError(ret)) { + s->errnr = MNSTR_WRITE_ERROR; + return -1; + } + + lz4->total_processing += total_decompressed; + total_read += remaining_to_decompress; + } + +#ifdef WIN32 + /* on Windows when in text mode, convert \r\n line endings to \n */ + if (s->type == ST_ASCII) { + char *p1, *p2, *pe; + + p1 = buf; + pe = p1 + total_read; + while (p1 < pe && *p1 != '\r') + p1++; + p2 = p1; + while (p1 < pe) { + if (*p1 == '\r' && p1[1] == '\n') + total_read--; + else + *p2++ = *p1; + p1++; + } + } +#endif + return (ssize_t) (total_read / elmsize); +} + +static ssize_t +stream_lz4write(stream *restrict s, const void *restrict buf, size_t elmsize, size_t cnt) +{ + lz4_stream *lz4 = s->stream_data.p; + size_t ret, size = elmsize * cnt, total_written = 0, next_batch, next_attempt, available, real_written; + + if (lz4 == NULL || size > LZ4_MAX_INPUT_SIZE || size <= 0) { + s->errnr = MNSTR_WRITE_ERROR; + return -1; + } + + while (total_written < size) { + next_batch = size - total_written; + available = lz4->ring_buffer_size - lz4->total_processing; + do { + next_attempt = LZ4F_compressBound(next_batch, NULL); /* lz4->ring_buffer must be at least 65548 bytes */ + if(next_attempt > available) { + next_batch >>= 1; + } else { + break; + } + if(next_batch == 0) + break; + } while(1); + assert(next_batch > 0); + + ret = LZ4F_compressUpdate(lz4->context.comp_context, ((char*)lz4->ring_buffer) + lz4->total_processing, + available, ((char*)buf) + total_written, next_batch, NULL); + if(LZ4F_isError(ret)) { + s->errnr =
MonetDB: Mar2018 - removing dead code that sneaked in with chang...
Changeset: a817784fb256 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a817784fb256 Modified Files: clients/mapiclient/mclient.c Branch: Mar2018 Log Message: removing dead code that sneaked in with changeset 359ebf5841f7: let's not pretend there is a JSON formatter; there is none. diffs (20 lines): diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c --- a/clients/mapiclient/mclient.c +++ b/clients/mapiclient/mclient.c @@ -88,7 +88,6 @@ enum formatters { TABLEformatter, // render as a bordered table CSVformatter, // render as a comma separate file XMLformatter, // render as a valid XML document - JSONformatter, // render as a valid JSON document TESTformatter, // for testing, escape characters TRASHformatter, // remove the result set SAMformatter, // render a SAM result set @@ -1703,8 +1702,6 @@ setFormatter(const char *s) formatter = RAWformatter; } else if (strcmp(s, "xml") == 0) { formatter = XMLformatter; - } else if (strcmp(s, "json") == 0) { - formatter = JSONformatter; } else if (strcmp(s, "test") == 0) { #ifdef _TWO_DIGIT_EXPONENT _set_output_format(_TWO_DIGIT_EXPONENT); ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Mar2018 - manually fixed "Manual update" (changeset ebc...
Changeset: 4baf3dbb32b3 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4baf3dbb32b3 Modified Files: clients/Tests/mclient-uri.SQL.sh Branch: Mar2018 Log Message: manually fixed "Manual update" (changeset ebcf8361248f) diffs (12 lines): diff --git a/clients/Tests/mclient-uri.SQL.sh b/clients/Tests/mclient-uri.SQL.sh --- a/clients/Tests/mclient-uri.SQL.sh +++ b/clients/Tests/mclient-uri.SQL.sh @@ -2,7 +2,7 @@ # test the URI parsing capabilities of the MAPI library -Mlog "mclient -d mapi:monetdb://$HOST:$MAPIPORT/$TSTDB?language=sql=monetdb -f test -t none-E utf-8 -s select 1" +Mlog "mclient -d mapi:monetdb://$HOST:$MAPIPORT/$TSTDB?language=sql=monetdb -f test -t none -E utf-8 -s select 1" mclient -d "mapi:monetdb://$HOST:$MAPIPORT/$TSTDB?language=sql=monetdb" -f test -E utf-8 -s 'select 1' Mlog "mclient -d mapi:monetdb://$MAPIHOST/.s.monetdb.$MAPIPORT?database=$TSTDB=sql=monetdb -f test -t none -E utf-8 -s select 1" ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list