MonetDB: default - Merge with oscar branch.

2020-08-21 Thread Sjoerd Mullender 
Changeset: b7cb866974f5 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b7cb866974f5
Modified Files:
sql/test/BugTracker-2020/Tests/values-like-join.Bug-6954.stable.out
Branch: default
Log Message:

Merge with oscar branch.


diffs (49 lines):

diff --git 
a/sql/test/BugTracker-2020/Tests/values-like-join.Bug-6954.stable.out 
b/sql/test/BugTracker-2020/Tests/values-like-join.Bug-6954.stable.out
--- a/sql/test/BugTracker-2020/Tests/values-like-join.Bug-6954.stable.out
+++ b/sql/test/BugTracker-2020/Tests/values-like-join.Bug-6954.stable.out
@@ -21,27 +21,25 @@ project (
 % mal # name
 % clob # type
 % 197 # length
-function user.s2_0():void;
-X_6:void := querylog.define("explain select 1 from (values (\\'one\\'), 
(\\'two\\')) as l(s), (values (\\'three\\'), (\\'four\\')) as r(s) where l.s 
like r.s;":str, "default_pipe":str, 27:int);
-barrier X_99:bit := language.dataflow();
-X_10:bat[:str] := bat.new(nil:str);
-X_13:bat[:str] := bat.append(X_10:bat[:str], "one":str, true:bit);
-X_16:bat[:str] := bat.append(X_13:bat[:str], "two":str, true:bit);
+function user.main():void;
+X_1:void := querylog.define("explain select 1 from (values (\\'one\\'), 
(\\'two\\')) as l(s), (values (\\'three\\'), (\\'four\\')) as r(s) where l.s 
like r.s;":str, "default_pipe":str, 23:int);
+barrier X_92:bit := language.dataflow();
+X_33:bat[:str] := bat.pack(".%7":str);
+X_34:bat[:str] := bat.pack("%7":str);
+X_35:bat[:str] := bat.pack("tinyint":str);
+X_36:bat[:int] := bat.pack(1:int);
+X_37:bat[:int] := bat.pack(0:int);
+X_9:bat[:str] := bat.new(nil:str);
+X_11:bat[:str] := bat.appendBulk(X_9:bat[:str], true:bit, "one":str, 
"two":str);
 X_17:bat[:str] := bat.new(nil:str);
-X_19:bat[:str] := bat.append(X_17:bat[:str], "three":str, true:bit);
-X_27:bat[:str] := bat.append(X_19:bat[:str], "four":str, true:bit);
-X_40:bat[:str] := bat.pack(".%7":str);
-X_41:bat[:str] := bat.pack("%7":str);
-X_42:bat[:str] := bat.pack("tinyint":str);
-X_43:bat[:int] := bat.pack(1:int);
-X_44:bat[:int] := bat.pack(0:int);
-(X_30:bat[:oid], X_31:bat[:oid]) := algebra.likejoin(X_16:bat[:str], 
X_27:bat[:str], "":str, nil:BAT, nil:BAT, true:bit, nil:lng);
-X_35:bat[:str] := algebra.projection(X_30:bat[:oid], X_16:bat[:str]);
-X_38:bat[:bte] := algebra.project(X_35:bat[:str], 1:bte);
-language.pass(X_16:bat[:str]);
-exit X_99:bit;
-sql.resultSet(X_40:bat[:str], X_41:bat[:str], X_42:bat[:str], 
X_43:bat[:int], X_44:bat[:int], X_38:bat[:bte]);
-end user.s2_0;
+X_18:bat[:str] := bat.appendBulk(X_17:bat[:str], true:bit, "three":str, 
"four":str);
+X_21:bat[:oid] := algebra.likejoin(X_11:bat[:str], X_18:bat[:str], "":str, 
nil:BAT, nil:BAT, true:bit, nil:lng, false:bit);
+X_27:bat[:str] := algebra.projection(X_21:bat[:oid], X_11:bat[:str]);
+X_31:bat[:bte] := algebra.project(X_27:bat[:str], 1:bte);
+language.pass(X_11:bat[:str]);
+exit X_92:bit;
+sql.resultSet(X_33:bat[:str], X_34:bat[:str], X_35:bat[:str], 
X_36:bat[:int], X_37:bat[:int], X_31:bat[:bte]);
+end user.main;
 #inline   actions= 0 time=0 usec 
 #remapactions= 0 time=2 usec 
 #costmodelactions= 1 time=1 usec 
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - Merge with oscar branch.

2020-08-21 Thread Sjoerd Mullender 
Changeset: 4213d1d04e3d for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4213d1d04e3d
Modified Files:
sql/server/rel_select.c
sql/test/BugTracker-2020/Tests/All
Branch: default
Log Message:

Merge with oscar branch.


diffs (204 lines):

diff --git a/sql/test/BugTracker-2020/Tests/All 
b/sql/test/BugTracker-2020/Tests/All
--- a/sql/test/BugTracker-2020/Tests/All
+++ b/sql/test/BugTracker-2020/Tests/All
@@ -18,3 +18,4 @@ interval-math.Bug-6935
 tpch-cube.Bug-6938
 HAVE_PYMONETDB?remote-table-like.Bug-6841
 KNOWNFAIL?copy-empty-blob.Bug-6948
+values-like-join.Bug-6954
diff --git a/sql/test/BugTracker-2020/Tests/values-like-join.Bug-6954.sql 
b/sql/test/BugTracker-2020/Tests/values-like-join.Bug-6954.sql
new file mode 100644
--- /dev/null
+++ b/sql/test/BugTracker-2020/Tests/values-like-join.Bug-6954.sql
@@ -0,0 +1,3 @@
+plan select 1 from (values ('one'), ('two')) as l(s), (values ('three'), 
('four')) as r(s) where l.s like r.s;
+explain select 1 from (values ('one'), ('two')) as l(s), (values ('three'), 
('four')) as r(s) where l.s like r.s;
+select 1 from (values ('one'), ('two')) as l(s), (values ('three'), ('four')) 
as r(s) where l.s like r.s;
diff --git 
a/sql/test/BugTracker-2020/Tests/values-like-join.Bug-6954.stable.err 
b/sql/test/BugTracker-2020/Tests/values-like-join.Bug-6954.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/test/BugTracker-2020/Tests/values-like-join.Bug-6954.stable.err
@@ -0,0 +1,12 @@
+stderr of test 'values-like-join.Bug-6954` in directory 
'sql/test/BugTracker-2020` itself:
+
+
+# 10:15:34 >  
+# 10:15:34 >  "mclient" "-lsql" "-ftest" "-tnone" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-70857" "--port=32688"
+# 10:15:34 >  
+
+
+# 10:15:34 >  
+# 10:15:34 >  "Done."
+# 10:15:34 >  
+
diff --git 
a/sql/test/BugTracker-2020/Tests/values-like-join.Bug-6954.stable.out 
b/sql/test/BugTracker-2020/Tests/values-like-join.Bug-6954.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/test/BugTracker-2020/Tests/values-like-join.Bug-6954.stable.out
@@ -0,0 +1,82 @@
+stdout of test 'values-like-join.Bug-6954` in directory 
'sql/test/BugTracker-2020` itself:
+
+
+# 10:15:34 >  
+# 10:15:34 >  "mclient" "-lsql" "-ftest" "-tnone" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-70857" "--port=32688"
+# 10:15:34 >  
+
+#plan select 1 from (values ('one'), ('two')) as l(s), (values ('three'), 
('four')) as r(s) where l.s like r.s;
+% .plan # table_name
+% rel # name
+% clob # type
+% 94 # length
+project (
+| join (
+| |  [  [ char(3) "one", char(3) "two" ] as "l"."s" ],
+| |  [  [ char(5) "three", char(5) "four" ] as "r"."s" ]
+| ) [ (clob[char["l"."s"]] as "l"."s") FILTER like (clob[char["r"."s"]] as 
"r"."s", clob "") ]
+) [ tinyint "1" ]
+#explain select 1 from (values ('one'), ('two')) as l(s), (values ('three'), 
('four')) as r(s) where l.s like r.s;
+% .explain # table_name
+% mal # name
+% clob # type
+% 197 # length
+function user.s2_0():void;
+X_6:void := querylog.define("explain select 1 from (values (\\'one\\'), 
(\\'two\\')) as l(s), (values (\\'three\\'), (\\'four\\')) as r(s) where l.s 
like r.s;":str, "default_pipe":str, 27:int);
+barrier X_99:bit := language.dataflow();
+X_10:bat[:str] := bat.new(nil:str);
+X_13:bat[:str] := bat.append(X_10:bat[:str], "one":str, true:bit);
+X_16:bat[:str] := bat.append(X_13:bat[:str], "two":str, true:bit);
+X_17:bat[:str] := bat.new(nil:str);
+X_19:bat[:str] := bat.append(X_17:bat[:str], "three":str, true:bit);
+X_27:bat[:str] := bat.append(X_19:bat[:str], "four":str, true:bit);
+X_40:bat[:str] := bat.pack(".%7":str);
+X_41:bat[:str] := bat.pack("%7":str);
+X_42:bat[:str] := bat.pack("tinyint":str);
+X_43:bat[:int] := bat.pack(1:int);
+X_44:bat[:int] := bat.pack(0:int);
+(X_30:bat[:oid], X_31:bat[:oid]) := algebra.likejoin(X_16:bat[:str], 
X_27:bat[:str], "":str, nil:BAT, nil:BAT, true:bit, nil:lng);
+X_35:bat[:str] := algebra.projection(X_30:bat[:oid], X_16:bat[:str]);
+X_38:bat[:bte] := algebra.project(X_35:bat[:str], 1:bte);
+language.pass(X_16:bat[:str]);
+exit X_99:bit;
+sql.resultSet(X_40:bat[:str], X_41:bat[:str], X_42:bat[:str], 
X_43:bat[:int], X_44:bat[:int], X_38:bat[:bte]);
+end user.s2_0;
+#inline   actions= 0 time=0 usec 
+#remapactions= 0 time=2 usec 
+#costmodelactions= 1 time=1 usec 
+#coercion actions= 1 time=5 usec 
+#aliases  actions= 6 time=24 usec 
+#evaluate actions= 1 time=27 usec 
+#emptybindactions= 0 time=1 usec 
+#pushselect   actions= 0 time=24 usec 
+#aliases  actions= 1 time=7 usec 
+#mitosis  actions=1 time=1 usec 
+#mergetable   actions= 0 time=57 usec 
+#deadcode actions= 2 time=8 usec 
+#aliases  actions= 0 time=1 usec 
+#constantsactions= 3 time=6 usec 
+#commonTerms  actions= 0 time=4 usec 
+#projectionpath   actions= 0

MonetDB: default - Merge with oscar branch.

2020-08-19 Thread Sjoerd Mullender 
Changeset: d760325d2974 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d760325d2974
Modified Files:
gdk/gdk_hash.c
sql/storage/store.c
Branch: default
Log Message:

Merge with oscar branch.


diffs (61 lines):

diff --git a/gdk/gdk_hash.c b/gdk/gdk_hash.c
--- a/gdk/gdk_hash.c
+++ b/gdk/gdk_hash.c
@@ -682,8 +682,7 @@ BAThashsync(void *arg)
 #define finishhash(TYPE)   \
do {\
const TYPE *restrict v = (const TYPE *) BUNtloc(bi, 0); \
-   for (; p < ci->ncand; p++) {
\
-   c = hash_##TYPE(h, v + o - b->hseqbase);\
+   for (; p < ci->ncand; p++) {\
c = hash_##TYPE(h, v + o - b->hseqbase);\
hget = HASHget(h, c);   \
h->nheads += hget == hnil;  \
diff --git a/monetdb5/modules/mal/mal_mapi.c b/monetdb5/modules/mal/mal_mapi.c
--- a/monetdb5/modules/mal/mal_mapi.c
+++ b/monetdb5/modules/mal/mal_mapi.c
@@ -625,21 +625,23 @@ SERVERlisten(int port, const char *usock
if (setsockopt(sock, SOL_SOCKET, 
SO_REUSEADDR, (char *) , sizeof on) == SOCKET_ERROR) {
e = errno;
closesocket(sock);
+   sock = INVALID_SOCKET;
continue;
}
 
bind_check = bind(sock, (SOCKPTR) 
rp->ai_addr, (socklen_t) rp->ai_addrlen);
-   e = errno;
if (bind_check == SOCKET_ERROR) {
+   e = errno;
closesocket(sock);
+   sock = INVALID_SOCKET;
continue;
-   } else
-   break;
+   }
+   break;
}
if (result)
freeaddrinfo(result);
errno = e;
-   if (errno == 0 && sock != INVALID_SOCKET)
+   if (sock != INVALID_SOCKET)
break;
 
if (port > 65535) {
diff --git a/sql/storage/store.c b/sql/storage/store.c
--- a/sql/storage/store.c
+++ b/sql/storage/store.c
@@ -2369,10 +2369,12 @@ store_manager(void)
// In the main loop we always hold the lock except when sleeping
MT_lock_set(_lock);
 
-   while (!GDKexiting()) {
+   for (;;) {
int res;
 
if (!flusher_should_run()) {
+   if (GDKexiting())
+   break;
const int sleeptime = 100;
MT_lock_unset(_lock);
MT_sleep_ms(sleeptime);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - Merge with oscar branch.

2020-08-17 Thread Sjoerd Mullender 
Changeset: 13631c874360 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=13631c874360
Modified Files:
MonetDB.spec
gdk/gdk_utils.c
sql/test/BugTracker-2020/Tests/All
Branch: default
Log Message:

Merge with oscar branch.


diffs (truncated from 313 to 300 lines):

diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -97,7 +97,7 @@
 
 #define NAME(name,tpe,id) (name?name:"tpe id")
 
-#define LOG_DISABLED(lg) ((lg)->debug&128)
+#define LOG_DISABLED(lg) ((lg)->debug&128 || (lg)->inmemory)
 
 static const char *log_commands[] = {
NULL,
@@ -1069,9 +1069,9 @@ logger_open(logger *lg)
char id[BUFSIZ];
char *filename;
 
-   if (lg->inmemory || LOG_DISABLED(lg)) {
+   if (LOG_DISABLED(lg)) {
lg->end = 0;
-   if (lg->id) /* go back too last used id */
+   if (lg->id) /* go back to last used id */
lg->id--;
return GDK_SUCCEED;
}
@@ -1104,7 +1104,7 @@ logger_open(logger *lg)
 static inline void
 logger_close(logger *lg)
 {
-   if (!lg->inmemory)
+   if (!LOG_DISABLED(lg))
close_stream(lg->log);
lg->log = NULL;
 }
@@ -1691,7 +1691,7 @@ logger_load(int debug, const char *fn, c
bool needcommit = false;
int dbg = GDKdebug;
 
-   if (!lg->inmemory && !LOG_DISABLED(lg)) {
+   if (!LOG_DISABLED(lg)) {
if ((filenamestr = GDKfilepath(farmid, lg->dir, LOGFILE, NULL)) 
== NULL)
goto error;
len = snprintf(filename, FILENAME_MAX, "%s", filenamestr);
@@ -1721,7 +1721,7 @@ logger_load(int debug, const char *fn, c
lg->seqs_val = NULL;
lg->dseqs = NULL;
 
-   if (!lg->inmemory && !LOG_DISABLED(lg)) {
+   if (!LOG_DISABLED(lg)) {
/* try to open logfile backup, or failing that, the file
 * itself. we need to know whether this file exists when
 * checking the database consistency later on */
@@ -1808,7 +1808,7 @@ logger_load(int debug, const char *fn, c
goto error;
}
 
-   if (!lg->inmemory && !LOG_DISABLED(lg)) {
+   if (!LOG_DISABLED(lg)) {
if (GDKcreatedir(filename) != GDK_SUCCEED) {
GDKerror("cannot create directory for log file 
%s\n",
 filename);
@@ -2466,8 +2466,12 @@ logger_exit(logger *lg)
char filename[FILENAME_MAX];
int len, farmid;
 
-   if (lg->inmemory || LOG_DISABLED(lg)) {
+   if (LOG_DISABLED(lg)) {
logger_close(lg);
+   if (logger_commit(lg) != GDK_SUCCEED) {
+   TRC_CRITICAL(GDK, "logger_commit failed\n");
+   return GDK_FAIL;
+   }
lg->changes = 0;
return GDK_SUCCEED;
}
@@ -2567,7 +2571,7 @@ logger_cleanup(logger *lg)
char buf[BUFSIZ];
FILE *fp = NULL;
 
-   if (lg->inmemory || LOG_DISABLED(lg))
+   if (LOG_DISABLED(lg))
return GDK_SUCCEED;
 
farmid = BBPselectfarm(PERSISTENT, 0, offheap);
@@ -2682,7 +2686,7 @@ log_bat_persists(logger *lg, BAT *b, con
l.flag = (l.flag == LOG_USE)?LOG_USE_ID:LOG_CREATE_ID;
l.tid = lg->tid;
lg->changes++;
-   if (!lg->inmemory && !LOG_DISABLED(lg)) {
+   if (!LOG_DISABLED(lg)) {
if (log_write_format(lg, ) != GDK_SUCCEED ||
log_write_string(lg, name) != GDK_SUCCEED ||
(tpe && log_write_id(lg, tpe, id) != GDK_SUCCEED))
@@ -2715,7 +2719,7 @@ log_bat_persists(logger *lg, BAT *b, con
}
return GDK_SUCCEED;
}
-   if (lg->inmemory || LOG_DISABLED(lg))
+   if (LOG_DISABLED(lg))
return GDK_SUCCEED;
 
ha = "vid";
@@ -2774,7 +2778,7 @@ log_bat_transient(logger *lg, const char
//  assert(lg->tid == tid);
}
 
-   if (lg->inmemory || LOG_DISABLED(lg))
+   if (LOG_DISABLED(lg))
return GDK_SUCCEED;
 
if (log_write_format(lg, ) != GDK_SUCCEED ||
@@ -2801,7 +2805,7 @@ log_delta(logger *lg, BAT *uid, BAT *uva
l.nr = (BUNlast(uval));
lg->changes += l.nr;
 
-   if (LOG_DISABLED(lg) || lg->inmemory) {
+   if (LOG_DISABLED(lg)) {
/* logging is switched off */
return GDK_SUCCEED;
}
@@ -2870,7 +2874,7 @@ log_bat(logger *lg, BAT *b, const char *
l.nr = (BUNlast(b) - b->batInserted);
lg->changes += (b->batInserted)?l.nr:1; /* initial large inserts is 
counted as 1 change */
 
-   if (LOG_DISABLED(lg) || lg->inmemory) {
+   if (LOG_DISABLED(lg)) {
/* logging is switched off */
return GDK_SUCCEED;
}
@@ -2916,7 +2920,7 @@

MonetDB: default - Merge with oscar branch.

2020-08-04 Thread Sjoerd Mullender 
Changeset: cea8be21a803 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=cea8be21a803
Modified Files:
ctest/tools/monetdbe/test_helper.h
debian/libmonetdb-client12.install
debian/libmonetdb-stream13.install
debian/libmonetdb20.install
Branch: default
Log Message:

Merge with oscar branch.


diffs (truncated from 340 to 300 lines):

diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -547,7 +547,6 @@ used from the MAL level.
 
 %files -n MonetDB5-server-devel
 %defattr(-,root,root)
-%dir %{_includedir}/monetdb
 %{_includedir}/monetdb/mal*.h
 %{_libdir}/libmonetdb5.so
 %{_libdir}/pkgconfig/monetdb5.pc
@@ -632,6 +631,8 @@ library.  Also see %{name}-embedded-deve
 %package embedded-devel
 Summary: MonetDB as an embedded library development files
 Group: Applications/Databases
+Requires: %{name}-embedded%{?_isa} = %{version}-%{release}
+Requires: %{name}-devel%{?_isa} = %{version}-%{release}
 
 %description embedded-devel
 MonetDB is a database management system that is developed from a
@@ -639,12 +640,14 @@ main-memory perspective with use of a fu
 automatic index management, extensibility of data types and search
 accelerators.  It also has an SQL front end.
 
-This package contains the library and include files to turn MonetDB
-into an embeddable library.
+This package contains the library and include files to create a
+program that uses MonetDB as an embeddable library.
 
 %files embedded-devel
+%defattr(-,root,root)
 %{_libdir}/libmonetdbe.so
 %{_includedir}/monetdb/monetdbe.h
+%{_libdir}/pkgconfig/monetdbe.pc
 
 %package testing
 Summary: MonetDB - Monet Database Management System
@@ -821,7 +824,6 @@ rmdir %{buildroot}%{_sysconfdir}/tmpfile
 
 install -d -m 0750 %{buildroot}%{_localstatedir}/lib/monetdb
 install -d -m 0770 %{buildroot}%{_localstatedir}/monetdb5/dbfarm
-touch %{buildroot}%{_localstatedir}/monetdb5/dbfarm/.merovingian_lock
 install -d -m 0775 %{buildroot}%{_localstatedir}/log/monetdb
 install -d -m 0775 %{buildroot}%{_rundir}/monetdb
 
diff --git a/ctest/tools/monetdbe/test_helper.h 
b/ctest/tools/monetdbe/test_helper.h
--- a/ctest/tools/monetdbe/test_helper.h
+++ b/ctest/tools/monetdbe/test_helper.h
@@ -36,22 +36,22 @@ TYPE_DEFTEST_TPE(timestamp, monetdbe_dat
 
 #define Null  ._is_null = 1
 
-monetdbe_export bool check_column_bool  (monetdbe_result* result, size_t 
column_index, size_t expected_nr_column_entries, TEST_TPE_ID(int8_t) 
expected_column[]);
-monetdbe_export bool check_column_int8_t(monetdbe_result* result, size_t 
column_index, size_t expected_nr_column_entries, TEST_TPE_ID(int8_t)* 
expected_column);
-monetdbe_export bool check_column_int16_t   (monetdbe_result* result, size_t 
column_index, size_t expected_nr_column_entries, TEST_TPE_ID(int16_t)* 
expected_column);
-monetdbe_export bool check_column_int32_t   (monetdbe_result* result, size_t 
column_index, size_t expected_nr_column_entries, TEST_TPE_ID(int32_t)* 
expected_column);
-monetdbe_export bool check_column_int64_t   (monetdbe_result* result, size_t 
column_index, size_t expected_nr_column_entries, TEST_TPE_ID(int64_t)* 
expected_column);
+extern bool check_column_bool  (monetdbe_result* result, size_t 
column_index, size_t expected_nr_column_entries, TEST_TPE_ID(int8_t) 
expected_column[]);
+extern bool check_column_int8_t(monetdbe_result* result, size_t 
column_index, size_t expected_nr_column_entries, TEST_TPE_ID(int8_t)* 
expected_column);
+extern bool check_column_int16_t   (monetdbe_result* result, size_t 
column_index, size_t expected_nr_column_entries, TEST_TPE_ID(int16_t)* 
expected_column);
+extern bool check_column_int32_t   (monetdbe_result* result, size_t 
column_index, size_t expected_nr_column_entries, TEST_TPE_ID(int32_t)* 
expected_column);
+extern bool check_column_int64_t   (monetdbe_result* result, size_t 
column_index, size_t expected_nr_column_entries, TEST_TPE_ID(int64_t)* 
expected_column);
 #if HAVE_HGE
-monetdbe_export bool check_column_int128_t  (monetdbe_result* result, size_t 
column_index, size_t expected_nr_column_entries, TEST_TPE_ID(int128_t)* 
expected_column);
+extern bool check_column_int128_t  (monetdbe_result* result, size_t 
column_index, size_t expected_nr_column_entries, TEST_TPE_ID(int128_t)* 
expected_column);
 #endif
-monetdbe_export bool check_column_size_t(monetdbe_result* result, size_t 
column_index, size_t expected_nr_column_entries, TEST_TPE_ID(size_t)* 
expected_column);
-monetdbe_export bool check_column_float (monetdbe_result* result, size_t 
column_index, size_t expected_nr_column_entries, TEST_TPE_ID(float)* 
expected_column);
-monetdbe_export bool check_column_double(monetdbe_result* result, size_t 
column_index, size_t expected_nr_column_entries, TEST_TPE_ID(double)* 
expected_column);
-monetdbe_export bool check_column_str   (monetdbe_result* result, size_t 
column_index, size_t expected_nr_column_entries, TEST_TPE_ID(str)* 
expected_column);

MonetDB: default - Merge with oscar branch.

2020-07-31 Thread Sjoerd Mullender 
Changeset: 05a6c02139b5 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=05a6c02139b5
Modified Files:
sql/server/rel_updates.c
Branch: default
Log Message:

Merge with oscar branch.


diffs (59 lines):

diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -750,7 +750,7 @@ COLcopy(BAT *b, int tt, bool writable, r
 
/* make sure we use the correct capacity */
bn->batCapacity = (BUN) (bn->ttype ? bn->theap.size >> 
bn->tshift : 0);
-   } else if (BATatoms[tt].atomFix || tt != TYPE_void || 
ATOMextern(tt)) {
+   } else if (BATatoms[tt].atomFix || ATOMextern(tt)) {
/* case (4): one-by-one BUN insert (really slow) */
BUN p, q, r = 0;
BATiter bi = bat_iterator(b);
diff --git a/sql/server/rel_updates.c b/sql/server/rel_updates.c
--- a/sql/server/rel_updates.c
+++ b/sql/server/rel_updates.c
@@ -1665,7 +1665,7 @@ copyfrom(sql_query *query, dlist *qname,
size_t l = strlen(cs->type.type->sqlname);
char *fname = sa_alloc(sql->sa, l+8);
 
-   snprintf(fname, l+8, "str_to_%s", 
cs->type.type->sqlname);
+   snprintf(fname, l+8, "str_to_%s", 
strcmp(cs->type.type->sqlname, "timestamptz") == 0 ? "timestamp" : 
cs->type.type->sqlname);
sql_find_subtype(, "clob", 0, 0);
f = sql_bind_func_result(sql->sa, sys, fname, 
F_FUNC, >type, 2, , );
if (!f)
diff --git a/testing/Mfilter.py b/testing/Mfilter.py
--- a/testing/Mfilter.py
+++ b/testing/Mfilter.py
@@ -157,7 +157,8 @@ def mFilter (FILE, IGNORE) :
 
 
 ftmp = []
-ig = n = 0
+n = 0
+ig = False
 il = iw = ic = el = ew = ec = al = aw = ac = 0
 for iline in openutf8(FILE, 'rU'):
 iline = iline.replace('\033[?1034h','')
@@ -173,7 +174,7 @@ def mFilter (FILE, IGNORE) :
 iline = re.sub(r'(\d+(?:\.\d*)?e[-+]?)0*(\d+)', r'\1\2', iline)
 oline = xline = ""
 if iline == "#~BeginVariableOutput~#\n"  or  iline == 
"#~BeginProfilingOutput~#\n" or iline == "[ \"~BeginVariableOutput~\"\t]\n"  or 
 iline == "[ \"~BeginProfilingOutput~\"\t]\n":
-ig = 1
+ig = True
 n = 0
 if ig  and  ( len(iline) == 0  or  iline[0] != "!"  or  iline[:9] != 
"ERROR = !" ):
 # ignore differences in "VariableOutput" or "ProfilingOutput"
@@ -226,10 +227,10 @@ def mFilter (FILE, IGNORE) :
 else:
 oline = iline
 if iline == "#~EndVariableOutput~#\n" or iline == "[ 
\"~EndVariableOutput~\"\t]\n":
-ig = 0
+ig = False
 xline = "~ " + str(n) + " ~\n"
 if iline == "#~EndProfilingOutput~#\n" or iline == "[ 
\"~EndProfilingOutput~\"\t]\n":
-ig = 0
+ig = False
 for ln in oline, xline:
 if len(ln):
 w = len(ln.split())
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - Merge with oscar branch.

2020-07-30 Thread Sjoerd Mullender 
Changeset: 5900329ceaa4 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5900329ceaa4
Modified Files:
sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.ppc64.int128
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.ppc64
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.ppc64.int128
Branch: default
Log Message:

Merge with oscar branch.

___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - Merge with oscar branch.

2020-07-30 Thread Sjoerd Mullender 
Changeset: 91bbf711ed23 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=91bbf711ed23
Modified Files:
sql/backends/monet5/sql_upgrades.c
sql/server/rel_schema.c
sql/test/SQLancer/Tests/sqlancer04.stable.err
sql/test/SQLancer/Tests/sqlancer04.stable.out
sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.ppc64.int128
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.32bit
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.ppc64
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.ppc64.int128
sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade/Tests/upgrade.stable.out
sql/test/emptydb-upgrade/Tests/upgrade.stable.out.32bit
sql/test/emptydb-upgrade/Tests/upgrade.stable.out.int128
sql/test/emptydb/Tests/check.stable.out
sql/test/emptydb/Tests/check.stable.out.32bit
sql/test/emptydb/Tests/check.stable.out.int128
sql/test/testdb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out.32bit
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade/Tests/upgrade.stable.out
sql/test/testdb-upgrade/Tests/upgrade.stable.out.32bit
sql/test/testdb-upgrade/Tests/upgrade.stable.out.int128
Branch: default
Log Message:

Merge with oscar branch.


diffs (truncated from 153718 to 300 lines):

diff --git a/sql/backends/monet5/sql_upgrades.c 
b/sql/backends/monet5/sql_upgrades.c
--- a/sql/backends/monet5/sql_upgrades.c
+++ b/sql/backends/monet5/sql_upgrades.c
@@ -36,6 +36,7 @@ sql_fix_system_tables(Client c, mvc *sql
char *buf = GDKmalloc(bufsize), *err = NULL;
node *n;
sql_schema *s;
+   static const char *boolnames[2] = {"false", "true"};
 
if (buf == NULL)
throw(SQL, __func__, SQLSTATE(HY013) MAL_MALLOC_FAIL);
@@ -83,12 +84,12 @@ sql_fix_system_tables(Client c, mvc *sql
func->base.id, func->base.name,
func->imp, func->mod, (int) FUNC_LANG_INT,
(int) func->type,
-   func->side_effect ? "true" : "false",
-   func->varres ? "true" : "false",
-   func->vararg ? "true" : "false",
+   boolnames[func->side_effect],
+   boolnames[func->varres],
+   boolnames[func->vararg],
func->s ? func->s->base.id : s->base.id,
-   func->system ? "true" : "false",
-   func->semantics ? "true" : "false");
+   boolnames[func->system],
+   boolnames[func->semantics]);
if (func->res) {
for (m = func->res->h; m; m = m->next, number++) {
arg = m->data;
@@ -2315,132 +2316,6 @@ sql_update_oscar(Client c, mvc *sql, con
/* Move sys.degrees and sys.radians to sql_types.c 
definitions (I did this at the bat_logger) Remove the obsolete entries at 
privileges table */
pos += snprintf(buf + pos, bufsize - pos,
"delete from privileges where obj_id in 
(select obj_id from privileges left join functions on privileges.obj_id = 
functions.id where functions.id is null and privileges.obj_id not in ((SELECT 
tables.id from tables), 0));\n");
-   pos += snprintf(buf + pos, bufsize - pos,
-   "UPDATE sys.functions set semantics = false 
WHERE (name, func) IN (VALUES \n"
-   "('length', 'nitems'),\n"
-   "('octet_length', 'nitems'),\n"
-   "('>', '>'),\n"
-   "('>=', '>='),\n"
-   "('<', '<'),\n"
-   "('<=', '<='),\n"
-   "('min', 'min'),\n"
-   "('max', 'max'),\n"
-   "('sql_min', 'min'),\n"
-   "('sql_max', 'max'),\n"
-   "('least', 'min_no_nil'),\n"
-   "('greatest', 'max_no_nil'),\n"
-   "('sum',

MonetDB: default - Merge with oscar branch.

2020-07-29 Thread Sjoerd Mullender 
Changeset: 2be78b2193b5 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=2be78b2193b5
Modified Files:
sql/server/rel_dump.c
sql/server/rel_select.c
Branch: default
Log Message:

Merge with oscar branch.


diffs (212 lines):

diff --git a/sql/server/rel_dump.c b/sql/server/rel_dump.c
--- a/sql/server/rel_dump.c
+++ b/sql/server/rel_dump.c
@@ -1054,9 +1054,11 @@ exp_read(mvc *sql, sql_rel *lrel, sql_re
if (zero_if_empty)
set_zero_if_empty(exp);
} else {
+   int nops = list_length(exps);
list *ops = sa_list(sql->sa);
for( n = exps->h; n; n = n->next)
append(ops, exp_subtype(n->data));
+
f = sql_bind_func_(sql->sa, s, cname, ops, F_FUNC);
if (!f)
f = sql_bind_func_(sql->sa, s, cname, ops, 
F_ANALYTIC);
@@ -1095,7 +1097,7 @@ exp_read(mvc *sql, sql_rel *lrel, sql_re
if (f)
exp = exp_op( sql->sa, exps, f);
else
-   return sql_error(sql, -1, SQLSTATE(42000) 
"Function '%s%s%s %d' not found\n", tname ? tname : "", tname ? "." : "", 
cname, list_length(ops));
+   return sql_error(sql, -1, SQLSTATE(42000) 
"Function '%s%s%s %d' not found\n", tname ? tname : "", tname ? "." : "", 
cname, nops);
}
}
 
diff --git a/sql/server/rel_select.c b/sql/server/rel_select.c
--- a/sql/server/rel_select.c
+++ b/sql/server/rel_select.c
@@ -3635,6 +3635,7 @@ static sql_exp *
sql_exp *r = exps->h->next->data, *or = r;
sql_subtype *t2 = exp_subtype(r);
 
+   a = NULL; /* reset a */
if (rel_convert_types(sql, *rel, *rel, , , 1/*fix 
scale*/, type_equal) >= 0){
list *tps = sa_list(sql->sa);
 
@@ -4037,15 +4038,18 @@ rel_group_column(sql_query *query, sql_r
 
if (!e) {
char buf[ERRSIZE];
+   int status = sql->session->status;
+   strcpy(buf, sql->errstr);
/* reset error */
sql->session->status = 0;
-   strcpy(buf, sql->errstr);
sql->errstr[0] = '\0';
 
e = rel_selection_ref(query, rel, grp, selection);
if (!e) {
-   if (sql->errstr[0] == 0)
+   if (sql->errstr[0] == 0) {
+   sql->session->status = status;
strcpy(sql->errstr, buf);
+   }
return NULL;
}
}
diff --git a/sql/test/BugTracker-2020/Tests/All 
b/sql/test/BugTracker-2020/Tests/All
--- a/sql/test/BugTracker-2020/Tests/All
+++ b/sql/test/BugTracker-2020/Tests/All
@@ -15,4 +15,4 @@ select-and.Bug-6878
 copy-decimal-with-space.Bug-6917
 isauuid.Bug-6934
 tpch-cube.Bug-6938
-HAVE_PYMONETDB?remote-table-like.Bug-6641
+HAVE_PYMONETDB?remote-table-like.Bug-6841
diff --git a/sql/test/BugTracker-2020/Tests/remote-table-like.Bug-6641.py 
b/sql/test/BugTracker-2020/Tests/remote-table-like.Bug-6841.py
rename from sql/test/BugTracker-2020/Tests/remote-table-like.Bug-6641.py
rename to sql/test/BugTracker-2020/Tests/remote-table-like.Bug-6841.py
diff --git 
a/sql/test/BugTracker-2020/Tests/remote-table-like.Bug-6641.stable.err 
b/sql/test/BugTracker-2020/Tests/remote-table-like.Bug-6841.stable.err
rename from sql/test/BugTracker-2020/Tests/remote-table-like.Bug-6641.stable.err
rename to sql/test/BugTracker-2020/Tests/remote-table-like.Bug-6841.stable.err
--- a/sql/test/BugTracker-2020/Tests/remote-table-like.Bug-6641.stable.err
+++ b/sql/test/BugTracker-2020/Tests/remote-table-like.Bug-6841.stable.err
@@ -1,4 +1,4 @@
-stderr of test 'remote-table-like.Bug-6641` in directory 
'sql/test/BugTracker-2020` itself:
+stderr of test 'remote-table-like.Bug-6841` in directory 
'sql/test/BugTracker-2020` itself:
 
 
 # 10:29:34 >  
diff --git 
a/sql/test/BugTracker-2020/Tests/remote-table-like.Bug-6641.stable.out 
b/sql/test/BugTracker-2020/Tests/remote-table-like.Bug-6841.stable.out
rename from sql/test/BugTracker-2020/Tests/remote-table-like.Bug-6641.stable.out
rename to sql/test/BugTracker-2020/Tests/remote-table-like.Bug-6841.stable.out
--- a/sql/test/BugTracker-2020/Tests/remote-table-like.Bug-6641.stable.out
+++ b/sql/test/BugTracker-2020/Tests/remote-table-like.Bug-6841.stable.out
@@ -1,4 +1,4 @@
-stdout of test 'remote-table-like.Bug-6641` in directory 
'sql/test/BugTracker-2020` itself:
+stdout of test 'remote-table-like.Bug-6841` in directory 
'sql/test/BugTracker-2020` itself:
 
 
 # 10:29:34 >  
diff --git a/sql/test/SQLancer/Tests/sqlancer04.sql 
b/sql/test/SQLancer/Tests/sqlancer04.sql
---

MonetDB: default - Merge with oscar branch.

2020-07-22 Thread Sjoerd Mullender 
Changeset: b336df152e97 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b336df152e97
Modified Files:
sql/backends/monet5/sql_result.c
sql/backends/monet5/sql_upgrades.c
sql/test/testdb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade/Tests/upgrade.stable.out.int128
Branch: default
Log Message:

Merge with oscar branch.


diffs (truncated from 3135 to 300 lines):

diff --git a/sql/backends/monet5/sql_result.c b/sql/backends/monet5/sql_result.c
--- a/sql/backends/monet5/sql_result.c
+++ b/sql/backends/monet5/sql_result.c
@@ -613,8 +613,6 @@ bat_max_hgelength(BAT *b)
res += *s++ - '0';  
\
scale--;
\
}   
\
-   while(*s && isdigit((unsigned char) *s))
\
-   s++;
\
}   
\
while(*s && isspace((unsigned char) *s))
\
s++;
\
diff --git a/sql/backends/monet5/sql_upgrades.c 
b/sql/backends/monet5/sql_upgrades.c
--- a/sql/backends/monet5/sql_upgrades.c
+++ b/sql/backends/monet5/sql_upgrades.c
@@ -2142,7 +2142,7 @@ sql_update_jun2020_sp1_hugeint(Client c,
 #endif
 
 static str
-sql_update_default_lidar(Client c)
+sql_update_oscar_lidar(Client c)
 {
char *query =
"drop procedure sys.lidarattach(string);\n"
@@ -2153,7 +2153,7 @@ sql_update_default_lidar(Client c)
 }
 
 static str
-sql_update_default(Client c, mvc *sql, const char *prev_schema, bool 
*systabfixed)
+sql_update_oscar(Client c, mvc *sql, const char *prev_schema, bool 
*systabfixed)
 {
size_t bufsize = 8192, pos = 0;
char *err = NULL, *buf = GDKmalloc(bufsize);
@@ -2254,13 +2254,20 @@ sql_update_default(Client c, mvc *sql, c
"external name 
\"sql\".\"sql_variables\";\n"
"grant execute on function 
\"sys\".\"var\" to public;\n");
 
+   pos += snprintf(buf + pos, bufsize - pos,
+   "create procedure 
sys.hot_snapshot(tarfile string, onserver bool)\n"
+   "external name sql.hot_snapshot;\n"
+   "update sys.functions set system = true 
where system <> true and schema_id = (select id from sys.schemas where name = 
'sys')"
+   " and name in ('hot_snapshot') and type 
= %d;\n",
+   (int) F_PROC);
/* .snapshot user */
pos += snprintf(buf + pos, bufsize - pos,
-   "create user \".snapshot\""
-   " with encrypted password 
''"
-   " name 'Snapshot User'"
+   "create user \".snapshot\"\n"
+   " with encrypted password 
''\n"
+   " name 'Snapshot User'\n"
" schema sys;\n"
-   "grant execute on procedure sys.hot_snapshot to 
\".snapshot\";\n"
+   "grant execute on procedure 
sys.hot_snapshot(string) to \".snapshot\";\n"
+   "grant execute on procedure 
sys.hot_snapshot(string, bool) to \".snapshot\";\n"
);
 
/* additional snapshot function */
@@ -2292,13 +2299,6 @@ sql_update_default(Client c, mvc *sql, c
/* Move sys.degrees and sys.radians to sql_types.c 
definitions (I did this at the bat_logger) Remove the obsolete entries at 
privileges table */
pos += snprintf(buf + pos, bufsize - pos,
"delete from privileges where obj_id in 
(select obj_id from

MonetDB: default - Merge with oscar branch.

2020-07-20 Thread Sjoerd Mullender 
Changeset: c1f2267918fd for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c1f2267918fd
Modified Files:
debian/libmonetdb-client12.install
debian/libmonetdb-stream13.install
debian/libmonetdb20.install
sql/backends/monet5/sql_result.c
sql/backends/monet5/sql_upgrades.c
sql/storage/store.c
Branch: default
Log Message:

Merge with oscar branch.


diffs (truncated from 1380 to 300 lines):

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -775,3 +775,5 @@ 28480e096722b7f76ab021c0d16c68c6949f41b4
 80bd3e89c48032840f493a63d6c4c12ee6273b3d Jun2020_7
 28480e096722b7f76ab021c0d16c68c6949f41b4 Jun2020_release
 80bd3e89c48032840f493a63d6c4c12ee6273b3d Jun2020_release
+1e3bd8cd485a0880a7557aa5307e986a210a1e00 Jun2020_9
+1e3bd8cd485a0880a7557aa5307e986a210a1e00 Jun2020_SP1_release
diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -94,7 +94,7 @@ Group: Applications/Databases
 License: MPLv2.0
 URL: https://www.monetdb.org/
 BugURL: https://bugs.monetdb.org/
-Source: 
https://www.monetdb.org/downloads/sources/Jun2020/%{name}-%{version}.tar.bz2
+Source: 
https://www.monetdb.org/downloads/sources/Jun2020-SP1/%{name}-%{version}.tar.bz2
 
 # The Fedora packaging document says we need systemd-rpm-macros for
 # the _unitdir and _tmpfilesdir macros to exist; however on RHEL 7
@@ -937,6 +937,70 @@ fi
 %postun -p /sbin/ldconfig
 
 %changelog
+* Mon Jul 20 2020 Sjoerd Mullender  - 11.37.9-20200720
+- Rebuilt.
+- BZ#6844: sys.getUser('https://me:p...@www.monetdb.org/Doc') does not
+  return the user: me
+- BZ#6845: the url sys.get...(url) functions do not allow null as
+  a parameter
+- BZ#6858: json.keyarray(json '{ "":0 }') fails with error: Could not
+  allocate space
+- BZ#6859: only first character of the separator string in json.text(js
+  json, sep string) is used
+- BZ#6873: sys.hot_snapshot() creates incomplete snapshots if the
+  write-ahead log is very large
+- BZ#6876: tar files created by sys.hot_snapshot() produce warnings on
+  some implementations of tar
+- BZ#6877: MonetDB produces malformed LZ4 files
+- BZ#6878: SQL Connection Error when running SELECT queries containing
+  AND command
+- BZ#6880: Left fuzzy queries are much slower than other fuzzy queries.
+- BZ#6882: cgroups limits no longer respected?
+- BZ#6883: SQLancer crash on delete query
+- BZ#6884: SQLancer generates query with unclear error message
+- BZ#6885: SQLancer causes assertion error on UTF8_strlen
+- BZ#6886: SQLancer alter table add unique gives strange error message
+- BZ#6887: SQLancer crash on complex query
+- BZ#6888: SQLancer crash on cross join on view
+- BZ#6889: SQLancer crash on long query
+- BZ#6892: SQLancer crash on query with HAVING
+- BZ#6893: SQLancer inner join reporting GDK error
+- BZ#6894: SQLancer crash on rtrim function
+- BZ#6895: SQLancer causing 'algebra.select' undefined error
+- BZ#6896: SQLancer algebra.select' undefined 2
+- BZ#6897: SQLancer distinct aggregate with error on group by constant
+- BZ#6898: SQLancer crash on join query
+- BZ#6899: SQLancer TLP query with wrong results
+- BZ#6900: SQLancer generated SIGFPE
+- BZ#6901: SQLancer TLP query with wrong results 2
+- BZ#6902: SQLancer query: batcalc.between undefined
+- BZ#6903: SQLancer calc.abs undefined
+- BZ#6904: SQLancer aggr.subavg undefined
+- BZ#6905: SQLancer TLP query with wrong results 3
+- BZ#6906: SQLancer crash on complex join
+- BZ#6907: SQLancer algebra.select undefined
+- BZ#6908: SQLancer inputs not the same size
+- BZ#6909: SQLancer query with wrong results
+- BZ#6911: SQLancer query: 'calc.bit' undefined
+- BZ#6918: SQLancer query compilation error
+- BZ#6919: SQLancer insert function doesn't handle utf-8 strings
+- BZ#6920: SQLancer project_bte: does not match always
+- BZ#6922: Timestamp columns not migrated to new format
+- BZ#6923: Imprints data files for timestamp BAT not migrated to the
+  new format
+- BZ#6924: SQLancer query copy on unique pair of columns fails and
+  complex query with GDK error
+- BZ#6925: Count string rows in union of string tables leaks (RSS) memory
+- BZ#6926: SQLancer query with wrong results
+- BZ#6927: SQLancer inputs not the same size
+- BZ#6928: SQLancer crash on coalesce
+- BZ#6929: SQLancer calc.date undefined
+
+* Tue Jun  9 2020 Sjoerd Mullender  - 11.37.9-20200720
+- gdk: Hash buckets come in variable widths.  But if a BAT grows long enough so
+  that the BAT indexes that are stored in the buckets don't fit anymore,
+  the buckets need to be widened.  This is now fixed.
+
 * Fri May 29 2020 Sjoerd Mullender  - 11.37.7-20200529
 - Rebuilt.
 
diff --git a/cmake/monetdb-versions.cmake b/cmake/monetdb-versions.cmake
--- a/cmake/monetdb-versions.cmake
+++ b/cmake/monetdb-versions.cmake
@@ -58,7 +58,7 @@ set(MONETDB_VERSION "${MONETDB_VERSION_M
 # version of the GDK library (subdirectory gdk; also includes
 # common/options and common/utils)
 set(GDK_VERSION_CURRENT "20")

MonetDB: default - Merge with oscar branch.

2020-07-15 Thread Sjoerd Mullender 
Changeset: f77668f69f27 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f77668f69f27
Modified Files:
cmake/monetdb-toolchain.cmake
common/utils/matomic.h
gdk/gdk_join.c
monetdb5/modules/mal/batcalc.c
monetdb_config.h.in
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql_statement.c
sql/backends/monet5/sql_statement.h
sql/include/sql_catalog.h
sql/server/rel_dump.c
sql/server/rel_optimizer.c
sql/server/rel_rel.c
sql/server/rel_rel.h
sql/server/rel_unnest.c
sql/server/sql_mvc.c
sql/server/sql_mvc.h
sql/storage/bat/bat_storage.c
sql/storage/sql_storage.h
sql/storage/store.c
tools/monetdbe/monetdbe.c
tools/monetdbe/monetdbe.h
Branch: default
Log Message:

Merge with oscar branch.


diffs (truncated from 1060 to 300 lines):

diff --git a/cmake/monetdb-toolchain.cmake b/cmake/monetdb-toolchain.cmake
--- a/cmake/monetdb-toolchain.cmake
+++ b/cmake/monetdb-toolchain.cmake
@@ -67,8 +67,9 @@ function(monetdb_default_toolchain)
 elseif(${CMAKE_C_COMPILER_ID} STREQUAL "Intel")
   if(WIN32)
 set(COMPILER_OPTION "/")
-#MT_addCompilerFlag("${COMPILER_OPTION}DNO_ATOMIC_INSTRUCTIONS" 
"${COMPILER_OPTION}DNO_ATOMIC_INSTRUCTIONS=1" "${CMAKE_C_FLAGS}" "all" 
CMAKE_C_FLAGS)
 MT_addCompilerFlag("${COMPILER_OPTION}W3" "${COMPILER_OPTION}W3" 
"${CMAKE_C_FLAGS}" "all" CMAKE_C_FLAGS)
+MT_addCompilerFlag("${COMPILER_OPTION}Qdiag-disable:11074" 
"${COMPILER_OPTION}Qdiag-disable:11074" "${CMAKE_C_FLAGS}" "all" CMAKE_C_FLAGS)
+MT_addCompilerFlag("${COMPILER_OPTION}Qdiag-disable:11075" 
"${COMPILER_OPTION}Qdiag-disable:11075" "${CMAKE_C_FLAGS}" "all" CMAKE_C_FLAGS)
   else()
 set(COMPILER_OPTION "-")
 MT_addCompilerFlag("${COMPILER_OPTION}Wall" "${COMPILER_OPTION}Wall" 
"${CMAKE_C_FLAGS}" "all" CMAKE_C_FLAGS)
diff --git a/common/utils/matomic.h b/common/utils/matomic.h
--- a/common/utils/matomic.h
+++ b/common/utils/matomic.h
@@ -57,7 +57,9 @@
 /* define this if you don't want to use atomic instructions */
 /* #define NO_ATOMIC_INSTRUCTIONS */
 
-#if defined(HAVE_STDATOMIC_H) && !defined(__STDC_NO_ATOMICS__) && 
!defined(NO_ATOMIC_INSTRUCTIONS)
+/* ignore __STDC_NO_ATOMICS__ if compiling using Intel compiler on
+ * Windows since otherwise we can't compile this at all in C99 mode */
+#if defined(HAVE_STDATOMIC_H) && (!defined(__STDC_NO_ATOMICS__) || 
(defined(__INTEL_COMPILER) && defined(_WINDOWS))) && 
!defined(NO_ATOMIC_INSTRUCTIONS)
 
 #include 
 
diff --git a/gdk/gdk_join.c b/gdk/gdk_join.c
--- a/gdk/gdk_join.c
+++ b/gdk/gdk_join.c
@@ -3063,7 +3063,6 @@ bitmaskjoin(BAT *l, BAT *r,
const char *reason, lng t0)
 {
BAT *r1;
-   const oid *rp = BATtdense(r) ? NULL : Tloc(r, 0);
size_t nmsk = (lci->ncand + 31) / 32;
uint32_t *mask = GDKzalloc(nmsk * sizeof(uint32_t));
BUN cnt = 0;
@@ -3073,13 +3072,9 @@ bitmaskjoin(BAT *l, BAT *r,
 
for (BUN n = 0; n < rci->ncand; n++) {
oid o = canditer_next(rci) - r->hseqbase;
-   if (rp) {
-   o = rp[o];
-   if (is_oid_nil(o))
-   continue;
-   } else {
-   o = o - r->hseqbase + r->tseqbase;
-   }
+   o = BUNtoid(r, o);
+   if (is_oid_nil(o))
+   continue;
o += l->hseqbase;
if (o < lci->seq + l->tseqbase)
continue;
@@ -3362,39 +3357,58 @@ leftjoin(BAT **r1p, BAT **r2p, BAT *l, B
BBPunfix(sr->batCacheid);
if (rc != GDK_SUCCEED)
return rc;
+   if (r2p == NULL) {
+   BBPunfix(r2->batCacheid);
+   r2 = NULL;
+   }
if (semi)
r1->tkey = true;
-   BAT *ob;
-   rc = BATsort(, r2p ?  : NULL, NULL,
-r1, NULL, NULL, false, false, false);
-   BBPunfix(r1->batCacheid);
-   if (rc != GDK_SUCCEED) {
-   BBPunfix(r2->batCacheid);
-   return rc;
+   if (!VIEWtparent(r1) &&
+   r1->ttype == TYPE_oid &&
+   BBP_refs(r1->batCacheid) == 1 &&
+   (r2 == NULL ||
+(!VIEWtparent(r2) &&
+ BBP_refs(r2->batCacheid) == 1 &&
+ r2->ttype == TYPE_oid))) {
+   /* in-place sort if we can */
+   if (r2) {
+

MonetDB: default - Merge with oscar branch.

2020-07-14 Thread Sjoerd Mullender 
Changeset: c5f3e77f9a41 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c5f3e77f9a41
Modified Files:
.editorconfig
cmake/monetdb-defines.cmake
gdk/gdk_align.c
gdk/gdk_batop.c
gdk/gdk_join.c
gdk/gdk_private.h
gdk/gdk_utils.c
monetdb5/modules/atoms/blob.c
monetdb_config.h.in
sql/include/sql_catalog.h
sql/server/rel_optimizer.c
sql/server/sql_atom.c
sql/server/sql_mvc.c
sql/server/sql_mvc.h
sql/storage/sql_storage.h
sql/storage/store.c
Branch: default
Log Message:

Merge with oscar branch.


diffs (truncated from 511 to 300 lines):

diff --git a/.editorconfig b/.editorconfig
--- a/.editorconfig
+++ b/.editorconfig
@@ -19,4 +19,4 @@ charset = utf-8
 
 [gdk/*.{c,h}]
 tab_width = 8
-max_line_length = 80
+max_line_length = 72
diff --git a/cmake/monetdb-defines.cmake b/cmake/monetdb-defines.cmake
--- a/cmake/monetdb-defines.cmake
+++ b/cmake/monetdb-defines.cmake
@@ -290,6 +290,7 @@ macro(monetdb_configure_sizes)
   check_type_size(short SIZEOF_SHORT LANGUAGE C)
   check_type_size(int SIZEOF_INT LANGUAGE C)
   check_type_size(long SIZEOF_LONG LANGUAGE C)
+  check_type_size(double SIZEOF_DOUBLE LANGUAGE C)
   check_type_size(wchar_t SIZEOF_WCHAR_T LANGUAGE C)
   check_type_size(socklen_t HAVE_SOCKLEN_T LANGUAGE C)
   check_type_size(pid_t SIZEOF_PID_T LANGUAGE C)
diff --git a/gdk/gdk_align.c b/gdk/gdk_align.c
--- a/gdk/gdk_align.c
+++ b/gdk/gdk_align.c
@@ -303,7 +303,7 @@ VIEWreset(BAT *b)
goto bailout;
th->farmid = BBPselectfarm(b->batRole, b->ttype, 
varheap);
strconcat_len(th->filename, sizeof(th->filename),
- nme, ".tail", NULL);
+ nme, ".theap", NULL);
if (ATOMheap(b->ttype, th, cnt) != GDK_SUCCEED)
goto bailout;
}
@@ -366,7 +366,7 @@ VIEWreset(BAT *b)
b->batCapacity = cnt;
 
/* insert all of v in b, and quit */
-   if (BATappend(b, v, NULL, false) != GDK_SUCCEED)
+   if (BATappend2(b, v, NULL, false, false) != GDK_SUCCEED)
goto bailout;
BBPreclaim(v);
}
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -53,7 +53,7 @@ unshare_string_heap(BAT *b)
 #endif
 
 static gdk_return
-insert_string_bat(BAT *b, BAT *n, struct canditer *ci, bool force)
+insert_string_bat(BAT *b, BAT *n, struct canditer *ci, bool force, bool 
mayshare)
 {
BATiter ni; /* iterator */
size_t toff = ~(size_t) 0;  /* tail offset */
@@ -100,7 +100,8 @@ insert_string_bat(BAT *b, BAT *n, struct
 * wholesale copying of n's offset heap, but
 * we may still be able to share the string
 * heap */
-   if (oldcnt == 0 &&
+   if (mayshare &&
+   oldcnt == 0 &&
b->tvheap != n->tvheap &&
ci->tpe == cand_dense) {
if (b->tvheap->parentid != bid) {
@@ -393,7 +394,7 @@ insert_string_bat(BAT *b, BAT *n, struct
 }
 
 static gdk_return
-append_varsized_bat(BAT *b, BAT *n, struct canditer *ci)
+append_varsized_bat(BAT *b, BAT *n, struct canditer *ci, bool mayshare)
 {
BATiter ni;
BUN cnt = ci->ncand, r;
@@ -406,7 +407,8 @@ append_varsized_bat(BAT *b, BAT *n, stru
assert(b->twidth == SIZEOF_VAR_T);
if (cnt == 0)
return GDK_SUCCEED;
-   if (BATcount(b) == 0 &&
+   if (mayshare &&
+   BATcount(b) == 0 &&
b->batRole == TRANSIENT &&
n->batRestricted == BAT_READ &&
b->tvheap != n->tvheap) {
@@ -489,7 +491,7 @@ append_varsized_bat(BAT *b, BAT *n, stru
  * list s) to BAT b.  If b is empty, b will get the seqbase of s if it
  * was passed in, and else the seqbase of n. */
 gdk_return
-BATappend(BAT *b, BAT *n, BAT *s, bool force)
+BATappend2(BAT *b, BAT *n, BAT *s, bool force, bool mayshare)
 {
struct canditer ci;
BUN cnt;
@@ -675,11 +677,11 @@ BATappend(BAT *b, BAT *n, BAT *s, bool f
b->tnil |= n->tnil && cnt == BATcount(n);
}
if (b->ttype == TYPE_str) {
-   if (insert_string_bat(b, n, , force) != GDK_SUCCEED) {
+   if (insert_string_bat(b, n, , force, mayshare) != 
GDK_SUCCEED) {
return GDK_FAIL;
}
} else if (ATOMvarsized(b->ttype)) {
-   if (append_varsized_bat(b, n, ) != GDK_SUCCEED) {
+   if (append_varsized_bat(b, n, , mayshare) != GDK_SUCCEED) {
return GDK_FAIL;
}
} else {
@@ -725,6 +727,12 @@ BATappend(BAT *b, BAT

MonetDB: default - Merge with oscar branch.

2020-07-07 Thread Sjoerd Mullender 
Changeset: 02e8cd086fe3 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=02e8cd086fe3
Removed Files:
monetdb5/extras/rapi/converters.c.h
monetdb5/extras/rapi/rapi.h
monetdb5/modules/kernel/mmath.h
monetdb5/modules/mal/batcalc.h
monetdb5/modules/mal/calc.h
monetdb5/modules/mal/language.h
monetdb5/modules/mal/mdb.h
sql/server/cmp_exp_eval.h
sql/server/rel_outer_join_optimizer.h
Modified Files:
clients/Tests/exports.stable.out
gdk/gdk_aggr.c
gdk/gdk_atoms.c
gdk/gdk_calc.c
gdk/gdk_select.c
geom/lib/libgeom.c
geom/monetdb5/geom.h
monetdb5/extras/rapi/CMakeLists.txt
monetdb5/extras/rapi/rapi.c
monetdb5/mal/mal_linker.c
monetdb5/mal/mal_module.c
monetdb5/mal/mal_module.h
monetdb5/mal/mal_prelude.c
monetdb5/mal/mal_private.h
monetdb5/mal/mel.h
monetdb5/modules/atoms/color.c
monetdb5/modules/kernel/CMakeLists.txt
monetdb5/modules/kernel/algebra.c
monetdb5/modules/kernel/batmmath.c
monetdb5/modules/kernel/microbenchmark.c
monetdb5/modules/kernel/mmath.c
monetdb5/modules/kernel/mmath_private.h
monetdb5/modules/mal/CMakeLists.txt
monetdb5/modules/mal/batExtensions.c
monetdb5/modules/mal/batcalc.c
monetdb5/modules/mal/calc.c
monetdb5/modules/mal/language.c
monetdb5/modules/mal/manual.c
monetdb5/modules/mal/mdb.c
monetdb5/optimizer/opt_costModel.h
sql/backends/monet5/generator/generator.c
sql/backends/monet5/sql.h
sql/test/SQLancer/Tests/sqlancer02.stable.out
sql/test/orderidx/Tests/simpletable.stable.out
sql/test/orderidx/Tests/simpletable.stable.out.32bit
sql/test/orderidx/Tests/smalltable.stable.out
sql/test/orderidx/Tests/smalltable.stable.out.32bit
Branch: default
Log Message:

Merge with oscar branch.


diffs (truncated from 3942 to 300 lines):

diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -756,11 +756,6 @@ str BKCsetPersistent(void *r, const bat 
 str BKCsetTransient(void *r, const bat *bid);
 str BKCshrinkBAT(bat *ret, const bat *bid, const bat *did);
 ssize_t BLOBtostr(str *tostr, size_t *l, const void *pin, bool external);
-str CALCmax(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
-str CALCmax_no_nil(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
-str CALCmin(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
-str CALCmin_no_nil(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
-str CALCswitchbit(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
 str CLTInfo(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
 str CLTLogin(bat *ret, bat *nme);
 str CLTaddUser(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
@@ -797,91 +792,6 @@ str CLTstop(Client cntxt, MalBlkPtr mb, 
 str CLTstopSession(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
 str CLTsuspend(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
 str CLTwakeup(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDBATprod(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDBATstr_group_concat(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr 
pci);
-str CMDBATsum(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatABS(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatABS(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatADD(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatADD(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatADDenlarge(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatADDenlarge(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatADDsignal(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatADDsignal(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatAND(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatAND(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatBETWEEN(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatBETWEEN(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatCMP(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatCMP(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatDECR(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatDECR(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatDIV(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatDIV(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatDIVsignal(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str CMDbatDIVsignal(Client cntxt,

MonetDB: default - Merge with oscar branch.

2020-07-07 Thread Sjoerd Mullender 
Changeset: 9534661eaf9e for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9534661eaf9e
Added Files:
ctest/cmake/test_detect_bz2.c.in
ctest/cmake/test_detect_commoncrypto.c.in
ctest/cmake/test_detect_gdal.c.in
ctest/cmake/test_detect_geos.c.in
ctest/cmake/test_detect_iconv.c.in
ctest/cmake/test_detect_libpy3.c.in
ctest/cmake/test_detect_libr.c.in
ctest/cmake/test_detect_libz.c.in
ctest/cmake/test_detect_lz4.c.in
ctest/cmake/test_detect_lzma.c.in
ctest/cmake/test_detect_netcdf.c.in
ctest/cmake/test_detect_openssl.c.in
ctest/cmake/test_detect_pcre.c.in
ctest/cmake/test_detect_proj.c.in
ctest/cmake/test_detect_readline.c.in
ctest/cmake/test_detect_snappy.c.in
ctest/cmake/test_detect_uuid.c.in
ctest/cmake/test_detect_valgrind.c.in
ctest/cmake/test_detect_xml.c.in
documentation/old/README.rst
Modified Files:
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
gdk/gdk_cand.c
monetdb5/modules/atoms/batxml.c
monetdb5/modules/kernel/algebra.c
monetdb5/modules/kernel/bat5.c
monetdb5/modules/mal/batcalc.c
monetdb5/modules/mal/calc.c
monetdb5/modules/mal/pcre.c
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql_statement.c
sql/server/rel_optimizer.c
Branch: default
Log Message:

Merge with oscar branch.


diffs (truncated from 592 to 300 lines):

diff --git a/ctest/cmake/test_detect_bz2.c.in b/ctest/cmake/test_detect_bz2.c.in
new file mode 100644
--- /dev/null
+++ b/ctest/cmake/test_detect_bz2.c.in
@@ -0,0 +1,19 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2020 MonetDB B.V.
+ */
+
+#include "monetdb_config.h"
+
+int
+main(void)
+{
+#ifdef HAVE_LIBBZ2
+return 0;
+#else
+return 1;
+#endif
+}
diff --git a/ctest/cmake/test_detect_commoncrypto.c.in 
b/ctest/cmake/test_detect_commoncrypto.c.in
new file mode 100644
--- /dev/null
+++ b/ctest/cmake/test_detect_commoncrypto.c.in
@@ -0,0 +1,19 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2020 MonetDB B.V.
+ */
+
+#include "monetdb_config.h"
+
+int
+main(void)
+{
+#ifdef HAVE_COMMONCRYPTO
+return @DETECT@;
+#else
+return @UNDETECT@;
+#endif
+}
diff --git a/ctest/cmake/test_detect_gdal.c.in 
b/ctest/cmake/test_detect_gdal.c.in
new file mode 100644
--- /dev/null
+++ b/ctest/cmake/test_detect_gdal.c.in
@@ -0,0 +1,19 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2020 MonetDB B.V.
+ */
+
+#include "monetdb_config.h"
+
+int
+main(void)
+{
+#ifdef HAVE_SHP
+return @DETECT@;
+#else
+return @UNDETECT@;
+#endif
+}
diff --git a/ctest/cmake/test_detect_geos.c.in 
b/ctest/cmake/test_detect_geos.c.in
new file mode 100644
--- /dev/null
+++ b/ctest/cmake/test_detect_geos.c.in
@@ -0,0 +1,19 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2020 MonetDB B.V.
+ */
+
+#include "monetdb_config.h"
+
+int
+main(void)
+{
+#ifdef HAVE_GEOM
+return @DETECT@;
+#else
+return @UNDETECT@;
+#endif
+}
diff --git a/ctest/cmake/test_detect_iconv.c.in 
b/ctest/cmake/test_detect_iconv.c.in
new file mode 100644
--- /dev/null
+++ b/ctest/cmake/test_detect_iconv.c.in
@@ -0,0 +1,20 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2020 MonetDB B.V.
+ */
+
+#include "monetdb_config.h"
+
+int
+main(void)
+{
+#ifdef HAVE_ICONV
+return @DETECT@;
+#else
+return @UNDETECT@;
+
+#endif
+}
diff --git a/ctest/cmake/test_detect_libpy3.c.in 
b/ctest/cmake/test_detect_libpy3.c.in
new file mode 100644
--- /dev/null
+++ b/ctest/cmake/test_detect_libpy3.c.in
@@ -0,0 +1,19 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 -

MonetDB: default - Merge with oscar branch.

2020-07-06 Thread Sjoerd Mullender 
Changeset: ed3a8e6141a9 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ed3a8e6141a9
Modified Files:
monetdb5/modules/atoms/json.c
monetdb5/modules/atoms/str.c
sql/backends/monet5/sql_optimizer.c
sql/storage/bat/bat_storage.c
sql/test/SQLancer/Tests/sqlancer03.sql
sql/test/SQLancer/Tests/sqlancer03.stable.out
testing/Mtest.py.in
Branch: default
Log Message:

Merge with oscar branch.


diffs (truncated from 466 to 300 lines):

diff --git a/monetdb5/modules/atoms/json.c b/monetdb5/modules/atoms/json.c
--- a/monetdb5/modules/atoms/json.c
+++ b/monetdb5/modules/atoms/json.c
@@ -1223,7 +1223,7 @@ JSONplaintext(char **r, size_t *l, size_
break;
default:
if (*l < jt->elm[idx].valuelen + sep_len + 1) {
-   unsigned int offset = *ilen - *l;
+   size_t offset = *ilen - *l;
char *p = *r - offset;
*ilen *= 2;
*r = GDKrealloc(p, *ilen);
@@ -1249,39 +1249,21 @@ JSONplaintext(char **r, size_t *l, size_
 static str
 JSONjson2text(str *ret, json *js)
 {
-   JSON *jt;
-   size_t l, ilen;
-   str s;
-
-   jt = JSONparse(*js);
-
-   CHECK_JSON(jt);
-   ilen = l = strlen(*js) + 1;
-   s = GDKmalloc(l);
-   if(s == NULL) {
-   JSONfree(jt);
-   throw(MAL,"json2txt", SQLSTATE(HY013) MAL_MALLOC_FAIL);
-   }
-   s = JSONplaintext(, , , jt, 0, " ", 1);
-   s -= ilen - l;
-   l = strlen(s);
-   if (l)
-   s[l - 1] = 0;
-   *ret = s;
-   JSONfree(jt);
-   return MAL_SUCCEED;
+   char *sep = " ";
+   return JSONjson2textSeparator(ret, js, );
 }
 
 static str
 JSONjson2textSeparator(str *ret, json *js, str *sep)
 {
JSON *jt;
-   size_t l, ilen, sep_len = strlen(*sep);
+   size_t l, ilen, sep_len;
str s;
 
jt = JSONparse(*js);
 
CHECK_JSON(jt);
+   sep_len = strlen(*sep);
ilen = l = strlen(*js) + 1;
s = GDKmalloc(l);
if(s == NULL) {
diff --git a/monetdb5/modules/atoms/str.c b/monetdb5/modules/atoms/str.c
--- a/monetdb5/modules/atoms/str.c
+++ b/monetdb5/modules/atoms/str.c
@@ -3156,6 +3156,59 @@ UTF8_strtail(const char *s, int pos)
return (str) s;
 }
 
+static inline str
+UTF8_strncpy(char *restrict dst, const char *restrict s, int n)
+{
+   UTF8_assert(s);
+   while (*s && n) {
+   if ((*s & 0xF8) == 0xF0) {
+   /* 4 byte UTF-8 sequence */
+   *dst++ = *s++;
+   *dst++ = *s++;
+   *dst++ = *s++;
+   *dst++ = *s++;
+   } else if ((*s & 0xF0) == 0xE0) {
+   /* 3 byte UTF-8 sequence */
+   *dst++ = *s++;
+   *dst++ = *s++;
+   *dst++ = *s++;
+   } else if ((*s & 0xE0) == 0xC0) {
+   /* 2 byte UTF-8 sequence */
+   *dst++ = *s++;
+   *dst++ = *s++;
+   } else {
+   /* 1 byte UTF-8 "sequence" */
+   *dst++ = *s++;
+   }
+   n--;
+   }
+   *dst = '\0';
+   return dst;
+}
+
+static inline str
+UTF8_offset(char *restrict s, int n)
+{
+   UTF8_assert(s);
+   while (*s && n) {
+   if ((*s & 0xF8) == 0xF0) {
+   /* 4 byte UTF-8 sequence */
+   s += 4;
+   } else if ((*s & 0xF0) == 0xE0) {
+   /* 3 byte UTF-8 sequence */
+   s += 3;
+   } else if ((*s & 0xE0) == 0xC0) {
+   /* 2 byte UTF-8 sequence */
+   s += 2;
+   } else {
+   /* 1 byte UTF-8 "sequence" */
+   s++;
+   }
+   n--;
+   }
+   return s;
+}
+
 static str
 convertCase(BAT *from, BAT *to, str *res, const char *src, const char *malfunc)
 {
@@ -4120,20 +4173,18 @@ STRlocate(int *ret, const str *needle, c
 }
 
 str
-STRinsert(str *ret, const str *s, const int *start, const int *l, const str 
*s2)
+STRinsert(str *ret, const str *input, const int *start, const int *nchars, 
const str *input2)
 {
-   str v;
-   int strt = *start;
-   if (strNil(*s) || strNil(*s2) || is_int_nil(*start) || is_int_nil(*l)) {
+   str v, s = *input, s2 = *input2;
+   int strt = *start, l = *nchars;
+
+   if (strNil(s) || strNil(s2) || is_int_nil(strt) || is_int_nil(l)) {
if ((*ret = GDKstrdup(str_nil)) == NULL)
throw(MAL, "str.insert", SQLSTATE(HY013) 
MAL_MALLOC_FAIL);
} else {
-   size_t l1 = strlen(*s);
-   size_t l2 = strlen(*s2);
+   size_t l1 = UTF8_strlen(s);
 
-   if (l1

MonetDB: default - Merge with oscar branch.

2020-07-06 Thread Sjoerd Mullender 
Changeset: f8d6f96a7ac2 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f8d6f96a7ac2
Modified Files:
misc/python/fixlicense.py
sql/benchmarks/ATIS/Tests/select_group.stable.out
sql/benchmarks/ATIS/Tests/select_group.stable.out.int128
sql/benchmarks/tpcds/Tests/07.stable.out
sql/benchmarks/tpcds/Tests/09.stable.out
sql/benchmarks/tpcds/Tests/13.stable.out
sql/benchmarks/tpcds/Tests/13.stable.out.int128
sql/benchmarks/tpcds/Tests/18.stable.out
sql/benchmarks/tpcds/Tests/18.stable.out.int128
sql/benchmarks/tpcds/Tests/26.stable.out
sql/benchmarks/tpcds/Tests/27.stable.out
sql/benchmarks/tpcds/Tests/28.stable.out
sql/benchmarks/tpcds/Tests/85.stable.out
sql/benchmarks/tpch/LOCKED/Tests/01.stable.out
sql/benchmarks/tpch/LOCKED/Tests/01.stable.out.int128
sql/benchmarks/tpch/Tests/01.stable.out
sql/benchmarks/tpch/Tests/01.stable.out.int128
sql/test/mergetables/Tests/forex.stable.out
sql/test/mergetables/Tests/forex.stable.out.int128
sql/test/sys-schema/Tests/systemfunctions.stable.out
Branch: default
Log Message:

Merge with oscar branch.


diffs (truncated from 1542 to 300 lines):

diff --git a/misc/python/fixlicense.py b/misc/python/fixlicense.py
--- a/misc/python/fixlicense.py
+++ b/misc/python/fixlicense.py
@@ -85,50 +85,39 @@ def main():
 func(filename[:-1], pre=pre, post=post, start=start, end=end, 
verbose=verbose)
 
 suffixrules = {
-# suffix:(pre, post,  start,  end)
-'.ac':   ('',  '','dnl ', ''),
-'.ag':   ('',  '','# ',   ''),
-'.am':   ('',  '','# ',   ''),
-'.bash': ('',  '','# ',   ''),
-'.bat':  ('',  '','@REM ',''),
-'.brg':  ('/*',' */', ' * ',  ''),
-'.c':('/*',' */', ' * ',  ''),
-'.cc':   ('',  '','// ',  ''),
-'.cf':   ('',  '','# ',   ''),
-'.cpp':  ('',  '','// ',  ''),
-'.el':   ('',  '','; ',   ''),
-'.h':('/*',' */', ' * ',  ''),
-'.hs':   ('',  '','-- ',  ''),
-'.html': ('', '', ''),
-'.i':('',  '','// ',  ''),
-'.java': ('/*',' */', ' * ',  ''),
-'.l':('/*',' */', ' * ',  ''),
-'.m4':   ('',  '','dnl ', ''),
-'.mal':  ('',  '','# ',   ''),
-'.mil':  ('',  '','# ',   ''),
-'.mk':   ('',  '','# ',   ''),
-'.msc':  ('',  '','# ',   ''),
-'.php':  ('',  '# ',   ''),
-'.pc':   ('',  '','# ',   ''),
-'.pl':   ('',  '','# ',   ''),
-'.pm':   ('',  '','# ',   ''),
-'.py':   ('',  '','# ',   ''),
-'.R':('',  '','# ',   ''),
-'.rb':   ('',  '','# ',   ''),
-'.rc':   ('',  '','// ',  ''),
-'.rst':  ('',  '','.. ',  ''),
-'.sh':   ('',  '','# ',   ''),
-'.sql':  ('',  '','-- ',  ''),
-'.t':('',  '','# ',   ''),
-'.xml':  ('', '', ''),
-'.xq':   ('(:',':)',  '', ''),
-'.xs':   ('/*',' */', ' * ',  ''),
-'.y':('/*',' */', ' * ',  ''),
+# suffix: (pre, post,  start,  end)
+'.bash':  ('',  '','# ',   ''), # shell script
+'.bat':   ('',  '','@REM ',''), # Windows cmd batch script
+'.c': ('/*',' */', ' * ',  ''), # C source
+'.cc':('',  '','// ',  ''), # C++ source
+'.cmake': ('#[[',   '#]]', '# ',   ''), # CMake source
+'.cpp':   ('',  '','// ',  ''), # C++ source
+'.el':('',  '','; ',   ''), # Emacs Lisp
+'.fc':('',  '','# ',   ''), # SELinux file context
+'.h': ('/*',' */', ' * ',  ''), # C header file
+'.hs':('',  '','-- ',  ''), # Haskell source
+'.html':  ('', '', ''), # HTML source
+'.java':  ('/*',' */', ' * ',  ''), # Java source
+'.l': ('/*',' */', ' * ',  ''), # (f)lex source
+'.mal':   ('',  '','# ',   ''), # MonetDB Assembly Language
+'.php':   ('',  '# ',   ''), # PHP source
+'.pl':('',  '','# ',   ''), # Perl source
+'.pm':('',  '','# ',   ''), # Perl module source
+'.py':('',  '','# ',   ''), # Python source
+'.R': ('',  '','# ',   ''), # R source
+'.rb':('',  '','# ',   ''), # Ruby source
+'.rc':('',  '','// ',  ''), # Windows resource file
+'.rst':   ('',  '','.. ',  ''), # reStructured Text
+'.sh':('',  '','# ',   ''), # shell script
+'.sql':   ('',  '','-- ',  ''), # SQL source
+'.t': ('',  '','# ',   ''), # Perl test
+'.te':('',  '','# ',   ''), # SELinux
+'.xml':   ('', '', ''), # XML source
+'.y': ('/*',' */', ' * ',  ''), # yacc (bison) source
 # we also match some complete

MonetDB: default - Merge with oscar branch.

2020-07-06 Thread Sjoerd Mullender 
Changeset: 2667c0094e2e for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=2667c0094e2e
Modified Files:
monetdb5/mal/mal_authorize.c
Branch: default
Log Message:

Merge with oscar branch.


diffs (20 lines):

diff --git a/monetdb5/mal/mal_authorize.c b/monetdb5/mal/mal_authorize.c
--- a/monetdb5/mal/mal_authorize.c
+++ b/monetdb5/mal/mal_authorize.c
@@ -390,10 +390,12 @@ AUTHinitTables(const char *passwd) {
if (passwd == NULL)
passwd = "monetdb"; /* default password */
pw = mcrypt_BackendSum(passwd, strlen(passwd));
-   if(!pw && !GDKembedded())
-   throw(MAL, "initTables", SQLSTATE(42000) "Crypt backend 
hash not found");
-   else if (!pw && GDKembedded())
-   pw = strdup(passwd);
+   if(!pw) {
+   if (!GDKembedded())
+   throw(MAL, "initTables", SQLSTATE(42000) "Crypt 
backend hash not found");
+   else
+   pw = strdup(passwd);
+   }
msg = AUTHaddUser(, NULL, "monetdb", pw);
free(pw);
if (msg)
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - Merge with oscar branch.

2020-07-06 Thread Sjoerd Mullender 
Changeset: 4b4219a8c481 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4b4219a8c481
Added Files:
sql/test/json/Tests/jsontext.Bug-6859.stable.err
sql/test/json/Tests/jsontext.Bug-6859.stable.out
Modified Files:
NT/mkodbcwxs.py
NT/mksqlwxs.py
clients/mapilib/mapi.c
common/utils/msabaoth.c
ctest/tools/monetdbe/backup.c
ctest/tools/monetdbe/example_backup.c
ctest/tools/monetdbe/test_helper.c
ctest/tools/monetdbe/test_helper.h
ctest/tools/monetdbe/test_helper_template.h
geom/monetdb5/geom_funcs.mal
misc/python/fixlicense.py
monetdb5/mal/mal.h
monetdb5/mal/mal_prelude.h
monetdb5/mal/mel.h
monetdb5/modules/atoms/json.c
monetdb5/modules/mal/clients.c
monetdb5/modules/mal/tracer.c
monetdb5/modules/mal/tracer.h
sql/backends/monet5/UDF/pyapi3/pytypes.h
sql/backends/monet5/sql_import.h
sql/backends/monet5/sql_session.mal
sql/backends/monet5/vaults/fits/72_fits.sql
sql/common/sql_types.c
sql/server/cmp_exp_eval.h
sql/server/rel_distribute.c
sql/server/rel_outer_join_optimizer.h
sql/storage/store.c
sql/test/SQLancer/Tests/sqlancer03.sql
sql/test/SQLancer/Tests/sqlancer03.stable.out
testing/exportutils.py
testing/malcheck.py
tools/merovingian/utils/control.c
tools/monetdbe/monetdbe_mapi.c
tools/monetdbe/monetdbe_mapi.h
Branch: default
Log Message:

Merge with oscar branch.


diffs (truncated from 749 to 300 lines):

diff --git a/NT/mkodbcwxs.py b/NT/mkodbcwxs.py
--- a/NT/mkodbcwxs.py
+++ b/NT/mkodbcwxs.py
@@ -1,3 +1,9 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Copyright 1997 - July 2008 CWI, August 2008 - 2020 MonetDB B.V.
+
 # python mkodbcwxs.py VERSION makedefs.txt PREFIX > 
PREFIX/MonetDB-ODBC-Installer.wxs
 # "c:\Program Files (x86)\WiX Toolset v3.10\bin\candle.exe" -nologo -arch 
x64/x86 PREFIX/MonetDB-ODBC-Installer.wxs
 # "c:\Program Files (x86)\WiX Toolset v3.10\bin\light.exe" -nologo -sice:ICE03 
-sice:ICE60 -sice:ICE82 -ext WixUIExtension PREFIX/MonetDB-ODBC-Installer.wixobj
diff --git a/NT/mksqlwxs.py b/NT/mksqlwxs.py
--- a/NT/mksqlwxs.py
+++ b/NT/mksqlwxs.py
@@ -1,3 +1,9 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Copyright 1997 - July 2008 CWI, August 2008 - 2020 MonetDB B.V.
+
 # python mksqlwxs.py VERSION makedefs.txt PREFIX > 
PREFIX/MonetDB5-SQL-Installer.wxs
 # "c:\Program Files (x86)\WiX Toolset v3.10\bin\candle.exe" -nologo -arch 
x64/x86 PREFIX/MonetDB5-SQL-Installer.wxs
 # "c:\Program Files (x86)\WiX Toolset v3.10\bin\light.exe" -nologo -sice:ICE03 
-sice:ICE60 -sice:ICE82 -ext WixUIExtension PREFIX/MonetDB5-SQL-Installer.wixobj
diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -2716,12 +2716,14 @@ mapi_reconnect(Mapi mid)
} else
 #endif
{
+   (void)pwdhash;
snprintf(buf, sizeof(buf), "server requires 
unknown hash '%.100s'",
serverhash);
close_connection(mid);
return mapi_setError(mid, buf, __func__, 
MERROR);
}
 
+#if defined(HAVE_RIPEMD160_UPDATE) || defined(HAVE_SHA512_UPDATE) || 
defined(HAVE_SHA384_UPDATE) || defined(HAVE_SHA256_UPDATE) || 
defined(HAVE_SHA224_UPDATE) || defined(HAVE_SHA1_UPDATE)
if (pwdhash == NULL) {
snprintf(buf, sizeof(buf), "allocation failure 
or unknown hash '%.100s'",
serverhash);
@@ -2733,6 +2735,7 @@ mapi_reconnect(Mapi mid)
mid->password = malloc(1 + strlen(pwdhash) + 1);
sprintf(mid->password, "\1%s", pwdhash);
free(pwdhash);
+#endif
}
 
p = mid->password + 1;
diff --git a/common/utils/msabaoth.c b/common/utils/msabaoth.c
--- a/common/utils/msabaoth.c
+++ b/common/utils/msabaoth.c
@@ -579,8 +579,6 @@ msab_pickSecret(char **generated_secret)
char *secret;
char pathbuf[FILENAME_MAX];
char *e;
-   int fd;
-   FILE *f;
 
if ((e = getDBPath(pathbuf, sizeof(pathbuf), SECRETFILE)) != NULL)
return e;
@@ -615,7 +613,9 @@ msab_pickSecret(char **generated_secret)
return NULL;
 #endif
 #endif
-
+#if defined(HAVE_OPENSSL) || defined(HAVE_COMMONCRYPTO)
+