Changeset: 9e241339490b for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9e241339490b Added Files: sql/test/BugTracker-2014/Tests/utf8bom.Bug-3436.SQL.py sql/test/BugTracker-2014/Tests/utf8bom.Bug-3436.stable.err sql/test/BugTracker-2014/Tests/utf8bom.Bug-3436.stable.out sql/test/BugTracker-2014/Tests/utf8bom.Bug-3436.zip Modified Files: MonetDB.spec common/stream/stream.c debian/changelog gdk/gdk_sample.c sql/test/BugTracker-2014/Tests/All tools/merovingian/utils/control.c Branch: default Log Message:
Merge with Jan2014 branch. diffs (286 lines): diff --git a/MonetDB.spec b/MonetDB.spec --- a/MonetDB.spec +++ b/MonetDB.spec @@ -800,8 +800,6 @@ rm -fr $RPM_BUILD_ROOT %changelog * Thu Feb 06 2014 Sjoerd Mullender <[email protected]> - 11.17.5-20140206 - Rebuilt. -- BZ#3390: Missing definition for pushSht in monetdb5/mal/mal_builder.h -- BZ#3402: We should have a C implementation of mal.multiplex. - BZ#3420: Database does not start after upgrade - BZ#3425: Temporal extraction glitches - BZ#3427: Consistent use of current_timestamp and now() diff --git a/common/stream/stream.c b/common/stream/stream.c --- a/common/stream/stream.c +++ b/common/stream/stream.c @@ -115,6 +115,9 @@ #define pclose _pclose #endif +#define UTF8BOM "\xEF\xBB\xBF" /* UTF-8 encoding of Unicode BOM */ +#define UTF8BOMLENGTH 3 /* length of above */ + #define short_int_SWAP(s) ((short)(((0x00ff&(s))<<8) | ((0xff00&(s))>>8))) #define normal_int_SWAP(i) (((0x000000ff&(i))<<24) | ((0x0000ff00&(i))<<8) | \ @@ -127,7 +130,8 @@ struct stream { short byteorder; - short access; /* read/write */ + char access; /* read/write */ + char isutf8; /* known to be UTF-8 due to BOM */ short type; /* ascii/binary */ char *name; unsigned int timeout; @@ -463,6 +467,7 @@ create_stream(const char *name) return NULL; s->byteorder = 1234; s->access = ST_READ; + s->isutf8 = 0; /* not known for sure */ s->type = ST_ASCII; s->name = strdup(name); s->stream_data.p = NULL; @@ -625,6 +630,8 @@ open_stream(const char *filename, const { stream *s; FILE *fp; + lng pos; + char buf[4]; if ((s = create_stream(filename)) == NULL) return NULL; @@ -639,6 +646,18 @@ open_stream(const char *filename, const s->fgetpos = file_fgetpos; s->fsetpos = file_fsetpos; s->stream_data.p = (void *) fp; + /* if file is opened for reading, and it starts with the UTF-8 + * encoding of the Unicode Byte Order Mark, skip the mark, and + * mark the stream as being a UTF-8 stream */ + if (fp != NULL && + flags[0] == 'r' && + file_fgetpos(s, &pos) == 0) { + if (file_read(s, buf, 1, UTF8BOMLENGTH) == 3 && + strncmp(buf, UTF8BOM, UTF8BOMLENGTH) == 0) + s->isutf8 = 1; + else + file_fsetpos(s, pos); + } return s; } @@ -2157,6 +2176,8 @@ ic_open(iconv_t cd, stream *ss, const ch stream *s; struct icstream *ic; + if (ss->isutf8) + return ss; if ((s = create_stream(name)) == NULL) return NULL; s->read = ic_read; @@ -2191,6 +2212,7 @@ iconv_rstream(stream *ss, const char *ch return NULL; s = ic_open(cd, ss, name); s->access = ST_READ; + s->isutf8 = 1; return s; } diff --git a/debian/changelog b/debian/changelog --- a/debian/changelog +++ b/debian/changelog @@ -7,8 +7,6 @@ monetdb (11.17.5-20140206) unstable; urg monetdb (11.17.5-20140206) unstable; urgency=low * Rebuilt. - * BZ#3390: Missing definition for pushSht in monetdb5/mal/mal_builder.h - * BZ#3402: We should have a C implementation of mal.multiplex. * BZ#3420: Database does not start after upgrade * BZ#3425: Temporal extraction glitches * BZ#3427: Consistent use of current_timestamp and now() diff --git a/gdk/gdk_sample.c b/gdk/gdk_sample.c --- a/gdk/gdk_sample.c +++ b/gdk/gdk_sample.c @@ -168,7 +168,7 @@ BATsample_(BAT *b, BUN n) } /* 1 left */ p += (BUN) rand() % cnt; - o[smp] = (oid) p; + o[smp] = (oid) p+1; /* property management */ BATsetcount(bn, n); diff --git a/sql/test/BugTracker-2014/Tests/All b/sql/test/BugTracker-2014/Tests/All --- a/sql/test/BugTracker-2014/Tests/All +++ b/sql/test/BugTracker-2014/Tests/All @@ -8,3 +8,4 @@ two-column-aggr-with-null.Bug-3428 sample-crash.Bug-3429 temporary.Bug-3430 indices.Bug-3435 +utf8bom.Bug-3436 diff --git a/sql/test/BugTracker-2014/Tests/utf8bom.Bug-3436.SQL.py b/sql/test/BugTracker-2014/Tests/utf8bom.Bug-3436.SQL.py new file mode 100644 --- /dev/null +++ b/sql/test/BugTracker-2014/Tests/utf8bom.Bug-3436.SQL.py @@ -0,0 +1,46 @@ +import os, sys, zipfile +try: + from MonetDBtesting import process +except ImportError: + import process + +TSTTRGDIR = os.environ['TSTTRGDIR'] + +archive = 'utf8bom.Bug-3436.zip' + +def mkpardir(path): + i = path.find('/') + d = '' + while i >= 0: + d = os.path.join(d, path[:i]) + if not os.path.exists(d): + os.mkdir(d) + path = path[i + 1:] + i = path.find('/') + +z = zipfile.ZipFile(archive) +print 'Archive: %s' % archive +for name in z.namelist(): + print ' inflating: %s' % name + mkpardir(name) + data = z.read(name) + f = open(name, 'wb') + f.write(data) + f.close() + +query = '''\ +start transaction; +create table utf8bom ( + city string, + id integer +); +copy into utf8bom from '%s' using delimiters ',','\\r\\n','"'; +select * from utf8bom order by id; +rollback; +''' + +c = process.client('sql', stdin = process.PIPE, stdout = process.PIPE, stderr = process.PIPE) +c.stdin.write(query % os.path.join(TSTTRGDIR, 'utf8bom.csv').replace('\\', r'\\')); +out, err = c.communicate() +sys.stdout.write(out) +sys.stderr.write(err) diff --git a/sql/test/BugTracker-2014/Tests/utf8bom.Bug-3436.stable.err b/sql/test/BugTracker-2014/Tests/utf8bom.Bug-3436.stable.err new file mode 100644 --- /dev/null +++ b/sql/test/BugTracker-2014/Tests/utf8bom.Bug-3436.stable.err @@ -0,0 +1,35 @@ +stderr of test 'utf8bom.Bug-3436` in directory 'sql/test/BugTracker-2014` itself: + + +# 16:38:43 > +# 16:38:43 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=31135" "--set" "mapi_usock=/var/tmp/mtest-23480/.s.monetdb.31135" "--set" "monet_prompt=" "--forcemito" "--set" "mal_listing=2" "--dbpath=/ufs/sjoerd/Monet-candidate/var/MonetDB/mTests_sql_test_BugTracker-2014" "--set" "mal_listing=0" +# 16:38:43 > + +# builtin opt gdk_dbpath = /ufs/sjoerd/Monet-candidate/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 50000 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 31135 +# cmdline opt mapi_usock = /var/tmp/mtest-23480/.s.monetdb.31135 +# cmdline opt monet_prompt = +# cmdline opt mal_listing = 2 +# cmdline opt gdk_dbpath = /ufs/sjoerd/Monet-candidate/var/MonetDB/mTests_sql_test_BugTracker-2014 +# cmdline opt mal_listing = 0 + +# 16:46:05 > +# 16:46:05 > "/usr/bin/python2" "utf8bom.Bug-3436.SQL.py" "utf8bom.Bug-3436" +# 16:46:05 > + + +# 16:46:05 > +# 16:46:05 > "Done." +# 16:46:05 > + diff --git a/sql/test/BugTracker-2014/Tests/utf8bom.Bug-3436.stable.out b/sql/test/BugTracker-2014/Tests/utf8bom.Bug-3436.stable.out new file mode 100644 --- /dev/null +++ b/sql/test/BugTracker-2014/Tests/utf8bom.Bug-3436.stable.out @@ -0,0 +1,49 @@ +stdout of test 'utf8bom.Bug-3436` in directory 'sql/test/BugTracker-2014` itself: + + +# 16:38:43 > +# 16:38:43 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=31135" "--set" "mapi_usock=/var/tmp/mtest-23480/.s.monetdb.31135" "--set" "monet_prompt=" "--forcemito" "--set" "mal_listing=2" "--dbpath=/ufs/sjoerd/Monet-candidate/var/MonetDB/mTests_sql_test_BugTracker-2014" "--set" "mal_listing=0" +# 16:38:43 > + +# MonetDB 5 server v11.17.6 (hg id: f65c0c865e47+) +# This is an unreleased version +# Serving database 'mTests_sql_test_BugTracker-2014', using 8 threads +# Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs dynamically linked +# Found 15.591 GiB available main-memory. +# Copyright (c) 1993-July 2008 CWI. +# Copyright (c) August 2008-2014 MonetDB B.V., all rights reserved +# Visit http://www.monetdb.org/ for further information +# Listening for connection requests on mapi:monetdb://madrid.ins.cwi.nl:31135/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-23480/.s.monetdb.31135 +# MonetDB/GIS module loaded +# MonetDB/JAQL module loaded +# MonetDB/SQL module loaded + +Ready. + +# 16:46:05 > +# 16:46:05 > "/usr/bin/python2" "utf8bom.Bug-3436.SQL.py" "utf8bom.Bug-3436" +# 16:46:05 > + +Archive: utf8bom.Bug-3436.zip + inflating: utf8bom.csv +#start transaction; +#create table utf8bom ( +# city string, +# id integer +#); +#copy into utf8bom from '/net/madrid/export/scratch2/sjoerd/bin/candidate/mTests/sql/test/BugTracker-2014/utf8bom.csv' using delimiters ',','\r\n','"'; +[ 2 ] +#select * from utf8bom order by id; +% sys.utf8bom, sys.utf8bom # table_name +% city, id # name +% clob, int # type +% 8, 4 # length +[ "Montréal", 1621 ] +[ "New York", 8392 ] +#rollback; + +# 16:46:05 > +# 16:46:05 > "Done." +# 16:46:05 > + diff --git a/sql/test/BugTracker-2014/Tests/utf8bom.Bug-3436.zip b/sql/test/BugTracker-2014/Tests/utf8bom.Bug-3436.zip new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..ea66c8163c4e91effba74655a8b3e569d0b4fdc7 GIT binary patch literal 210 zc$^FHW@h1H0D()DZZ2PP;>Fc~Y!K#VkYOk-NwY}G&(%vVE(;CeWMDos<8#nOATF)o zW?*Ew#KOS9Qu2y{;r;IYN`9&33X%Cm*-APV#+F9BTuQ$Ac_l@MS0?5t=@^<B8S-)k qcr!A|G2^mK0xZr5#0`xg7K+uZ5UbIw4e(}VgJ@!82n5pEU?Tt_DKN1B diff --git a/tools/merovingian/utils/control.c b/tools/merovingian/utils/control.c --- a/tools/merovingian/utils/control.c +++ b/tools/merovingian/utils/control.c @@ -113,7 +113,7 @@ char* control_send( if (len == 2) len += recv(sock, rbuf + len, sizeof(rbuf) - len, 0); /* perform login ritual */ - if (len <= 0) { + if (len <= 2) { snprintf(sbuf, sizeof(sbuf), "no response from monetdbd"); close(sock); return(strdup(sbuf)); _______________________________________________ checkin-list mailing list [email protected] https://www.monetdb.org/mailman/listinfo/checkin-list
