MonetDB: Apr2019 - Test for bug 6757.

2019-09-16 Thread Sjoerd Mullender 
Changeset: 302e9c2c813d for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=302e9c2c813d
Added Files:
sql/test/BugTracker-2019/Tests/double-free.Bug-6757.sql
sql/test/BugTracker-2019/Tests/double-free.Bug-6757.stable.err
sql/test/BugTracker-2019/Tests/double-free.Bug-6757.stable.out
Modified Files:
sql/test/BugTracker-2019/Tests/All
Branch: Apr2019
Log Message:

Test for bug 6757.


diffs (truncated from 1965 to 300 lines):

diff --git a/sql/test/BugTracker-2019/Tests/All 
b/sql/test/BugTracker-2019/Tests/All
--- a/sql/test/BugTracker-2019/Tests/All
+++ b/sql/test/BugTracker-2019/Tests/All
@@ -33,4 +33,5 @@ alter_table_drop_column.Bug-6749
 HAVE_PYMONETDB?remote-table-non-existent-column.Bug-6750
 cte-union.Bug-6755
 merge-table-limit.Bug-6756
+double-free.Bug-6757
 HAVE_LIBPY3?python-loader-string.Bug-6759
diff --git a/sql/test/BugTracker-2019/Tests/double-free.Bug-6757.sql 
b/sql/test/BugTracker-2019/Tests/double-free.Bug-6757.sql
new file mode 100644
--- /dev/null
+++ b/sql/test/BugTracker-2019/Tests/double-free.Bug-6757.sql
@@ -0,0 +1,976 @@
+start transaction;
+
+CREATE TABLE "sys"."params_str" (
+   "paramname" CHARACTER LARGE OBJECT,
+   "value" CHARACTER LARGE OBJECT,
+   "prob"  DOUBLE
+);
+CREATE TABLE "sys"."bm_0_obj_dict" (
+   "id"INTEGER   NOT NULL,
+   "idstr" CHARACTER LARGE OBJECT NOT NULL,
+   "prob"  FLOAT(51) NOT NULL,
+   CONSTRAINT "bm_0_obj_dict_id_pkey" PRIMARY KEY ("id"),
+   CONSTRAINT "bm_0_obj_dict_idstr_unique" UNIQUE ("idstr")
+);
+CREATE TABLE "sys"."bm_0_obj_type" (
+   "id"  INTEGER   NOT NULL,
+   "type"INTEGER   NOT NULL,
+   "typestr" CHARACTER LARGE OBJECT NOT NULL,
+   "prob"DOUBLENOT NULL,
+   CONSTRAINT "bm_0_obj_type_id_fkey" FOREIGN KEY ("id") REFERENCES 
"sys"."bm_0_obj_dict" ("id"),
+   CONSTRAINT "bm_0_obj_type_type_fkey" FOREIGN KEY ("type") REFERENCES 
"sys"."bm_0_obj_dict" ("id")
+);
+CREATE TABLE "sys"."tr_0_obj_dict" (
+   "id"INTEGER   NOT NULL,
+   "idstr" CHARACTER LARGE OBJECT NOT NULL,
+   "prob"  FLOAT(51) NOT NULL,
+   CONSTRAINT "tr_0_obj_dict_id_pkey" PRIMARY KEY ("id"),
+   CONSTRAINT "tr_0_obj_dict_idstr_unique" UNIQUE ("idstr")
+);
+CREATE TABLE "sys"."tr_0_obj_type" (
+   "id"  INTEGER   NOT NULL,
+   "type"INTEGER   NOT NULL,
+   "typestr" CHARACTER LARGE OBJECT NOT NULL,
+   "prob"DOUBLENOT NULL,
+   CONSTRAINT "tr_0_obj_type_id_fkey" FOREIGN KEY ("id") REFERENCES 
"sys"."tr_0_obj_dict" ("id"),
+   CONSTRAINT "tr_0_obj_type_type_fkey" FOREIGN KEY ("type") REFERENCES 
"sys"."tr_0_obj_dict" ("id")
+);
+
+CREATE TABLE "sys"."_cachedrel_4" (
+   "a1"   INTEGER,
+   "prob" FLOAT(51)
+);
+CREATE TABLE "sys"."_cachedrel_5" (
+   "a1"   INTEGER,
+   "prob" DOUBLE
+);
+CREATE TABLE "sys"."_cachedrel_6" (
+   "a1"   INTEGER,
+   "prob" DOUBLE
+);
+CREATE TABLE "sys"."_cachedrel_7" (
+   "a1"   INTEGER,
+   "prob" FLOAT(51)
+);
+CREATE TABLE "sys"."_cachedrel_8" (
+   "a1"   INTEGER,
+   "a2"   INTEGER,
+   "a3"   CHARACTER LARGE OBJECT,
+   "a4"   CHARACTER LARGE OBJECT,
+   "prob" DOUBLE
+);
+CREATE TABLE "sys"."_cachedrel_9" (
+   "a1"   INTEGER,
+   "a2"   INTEGER,
+   "a3"   CHARACTER LARGE OBJECT,
+   "a4"   CHARACTER LARGE OBJECT,
+   "prob" DOUBLE
+);
+CREATE TABLE "sys"."_cachedrel_10" (
+   "a1"   INTEGER,
+   "a2"   CHARACTER LARGE OBJECT,
+   "a3"   CHARACTER LARGE OBJECT,
+   "prob" DOUBLE
+);
+CREATE TABLE "sys"."_cachedrel_11" (
+   "a1"   INTEGER,
+   "a2"   CHARACTER LARGE OBJECT,
+   "a3"   CHARACTER LARGE OBJECT,
+   "prob" DOUBLE
+);
+CREATE TABLE "sys"."_cachedrel_12" (
+   "a1"   CHARACTER LARGE OBJECT,
+   "a2"   CHARACTER LARGE OBJECT,
+   "a3"   CHARACTER LARGE OBJECT,
+   "a4"   CHARACTER LARGE OBJECT,
+   "prob" DOUBLE
+);
+CREATE TABLE "sys"."_cachedrel_13" (
+   "a1"   INTEGER,
+   "prob" FLOAT(51)
+);
+CREATE TABLE "sys"."_cachedrel_14" (
+   "a1"   INTEGER,
+   "a2"   INTEGER,
+   "a3"   CHARACTER LARGE OBJECT,
+   "a4"   CHARACTER LARGE OBJECT,
+   "prob" DOUBLE
+);
+CREATE TABLE "sys"."_cachedrel_15" (
+   "a1"   INTEGER,
+   "a2"   INTEGER,
+   "a3"   CHARACTER LARGE OBJECT,
+   "a4"   CHARACTER LARGE OBJECT,
+   "prob" DOUBLE
+);
+CREATE TABLE "sys"."_cachedrel_16" (
+   "a1"   INTEGER,
+   "prob" FLOAT(51)
+);
+CREATE TABLE "sys"."_cachedrel_17" (
+   "a1"   INTEGER,
+   "a2"   INTEGER,
+   "a3"   CHARACTER LARGE OBJECT,
+   "a4"   CHARACTER LARGE OBJECT,
+   "prob" DOUBLE
+);
+CREATE TABLE "sys"."_cachedrel_18" (
+   "a1"   INTEGER,
+   "a2"   INTEGER,
+   "a3"   CHARACTER LARGE OBJECT,
+   "a4"   CHARACTER LARGE OBJECT,
+   "prob" DOUBLE
+);
+CREATE TABLE

MonetDB: default - Merge with Nov2019 branch.

2019-09-16 Thread Sjoerd Mullender 
Changeset: 1390f2503198 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1390f2503198
Modified Files:
gdk/gdk_bbp.c
gdk/gdk_heap.c
gdk/gdk_interprocess.c
gdk/gdk_storage.c
sql/backends/monet5/UDF/pyapi/pyapi.c
sql/server/rel_semantic.c
sql/storage/store.c
sql/test/subquery/Tests/subquery2.sql
tools/merovingian/daemon/controlrunner.c
Branch: default
Log Message:

Merge with Nov2019 branch.


diffs (truncated from 342 to 300 lines):

diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -1737,7 +1737,7 @@ BBPdir_subcommit(int cnt, bat *subcommit
assert(subcommit[n - 1] < subcommit[n]);
 #endif
 
-   if ((nbbpf = GDKfilelocate(0, "BBP", "w", "dir")) == NULL)
+   if ((nbbpf = GDKfilelocate(0, "BBP", "wx", "dir")) == NULL)
return GDK_FAIL;
 
n = (bat) ATOMIC_GET();
@@ -1849,7 +1849,7 @@ BBPdir(int cnt, bat *subcommit)
return BBPdir_subcommit(cnt, subcommit);
 
IODEBUG fprintf(stderr, "#BBPdir: writing BBP.dir (%d bats).\n", (int) 
(bat) ATOMIC_GET());
-   if ((fp = GDKfilelocate(0, "BBP", "w", "dir")) == NULL) {
+   if ((fp = GDKfilelocate(0, "BBP", "wx", "dir")) == NULL) {
goto bailout;
}
 
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -61,7 +61,7 @@ HEAPcreatefile(int farmid, size_t *maxsz
fn = path;
}
/* round up to mulitple of GDK_mmap_pagesize */
-   fd = GDKfdlocate(NOFARM, fn, "wb", NULL);
+   fd = GDKfdlocate(NOFARM, fn, "wxb", NULL);
if (fd >= 0) {
close(fd);
base = GDKload(NOFARM, fn, NULL, *maxsz, maxsz, STORE_MMAP);
@@ -119,36 +119,12 @@ HEAPalloc(Heap *h, size_t nitems, size_t
}
if (!GDKinmemory() && h->base == NULL) {
char *nme;
-   struct stat st;
 
-   if(!(nme = GDKfilepath(h->farmid, BATDIR, h->filename, NULL))) {
-   GDKerror("HEAPalloc: malloc failure");
+   nme = GDKfilepath(h->farmid, BATDIR, h->filename, NULL);
+   if (nme == NULL)
return GDK_FAIL;
-   }
-   if (stat(nme, ) < 0) {
-   h->storage = STORE_MMAP;
-   h->base = HEAPcreatefile(NOFARM, >size, nme);
-   } else {
-   int fd;
-
-   fd = GDKfdlocate(NOFARM, nme, "wb", NULL);
-   if (fd >= 0) {
-   char of[sizeof(h->filename)];
-   char *ext;
-   close(fd);
-   strncpy(of, h->filename, sizeof(of));
-#ifdef STATIC_CODE_ANALYSIS
-   /* help coverity */
-   of[sizeof(h->filename) - 1] = 0;
-#endif
-   ext = decompose_filename(of);
-   h->newstorage = STORE_MMAP;
-   if (HEAPload(h, of, ext, false) != GDK_SUCCEED)
-   h->base = NULL; /* superfluous */
-   /* success checked by looking at
-* h->base below */
-   }
-   }
+   h->storage = STORE_MMAP;
+   h->base = HEAPcreatefile(NOFARM, >size, nme);
GDKfree(nme);
}
if (h->base == NULL) {
@@ -270,6 +246,7 @@ HEAPextend(Heap *h, size_t size, bool ma
HEAPfree(, false);
return GDK_SUCCEED;
}
+   GDKclrerr();
}
fd = GDKfdlocate(h->farmid, nme, "wb", ext);
if (fd >= 0) {
@@ -445,7 +422,7 @@ GDKupgradevarheap(BAT *b, var_t v, bool 
const char *base = b->theap.base;
 
/* first save heap in file with extra .tmp extension */
-   if ((fd = GDKfdlocate(b->theap.farmid, b->theap.filename, "wb", 
"tmp")) < 0)
+   if ((fd = GDKfdlocate(b->theap.farmid, b->theap.filename, 
"wxb", "tmp")) < 0)
return GDK_FAIL;
while (size > 0) {
ret = write(fd, base, (unsigned) MIN(1 << 30, size));
diff --git a/gdk/gdk_interprocess.c b/gdk/gdk_interprocess.c
--- a/gdk/gdk_interprocess.c
+++ b/gdk/gdk_interprocess.c
@@ -28,7 +28,7 @@
 #include 
 #include 
 
-static size_t interprocess_unique_id = 1;
+static ATOMIC_TYPE interprocess_unique_id = ATOMIC_VAR_INIT(1);
 static key_t base_key = 8;
 
 // Regular ftok produces too many collisions
@@ -46,12 +46,7 @@ ftok_enhanced(int id, key_t * return_key
 size_t
 GDKuniqueid(size_t offset)
 {
-   // TODO: lock this

MonetDB: Nov2019 - Merge with Apr2019 branch.

2019-09-16 Thread Sjoerd Mullender 
Changeset: 059c7884a05b for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=059c7884a05b
Modified Files:
sql/server/rel_semantic.c
tools/merovingian/daemon/controlrunner.c
Branch: Nov2019
Log Message:

Merge with Apr2019 branch.


diffs (12 lines):

diff --git a/sql/server/rel_semantic.c b/sql/server/rel_semantic.c
--- a/sql/server/rel_semantic.c
+++ b/sql/server/rel_semantic.c
@@ -88,6 +88,8 @@ rel_parse(mvc *m, sql_schema *s, char *q
bstream_destroy(m->scanner.rs);
 
m->sym = NULL;
+   o.vars = m->vars;   /* may have been realloc'ed */
+   o.sizevars = m->sizevars;
if (m->session->status || m->errstr[0]) {
int status = m->session->status;
char errstr[ERRSIZE];
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Apr2019 - Don't overwrite m->vars and m->sizevars with ...

2019-09-16 Thread Sjoerd Mullender 
Changeset: b87bd0e2f8f8 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b87bd0e2f8f8
Modified Files:
sql/server/rel_semantic.c
Branch: Apr2019
Log Message:

Don't overwrite m->vars and m->sizevars with stale values.
This fixes bug 6757.


diffs (12 lines):

diff --git a/sql/server/rel_semantic.c b/sql/server/rel_semantic.c
--- a/sql/server/rel_semantic.c
+++ b/sql/server/rel_semantic.c
@@ -86,6 +86,8 @@ rel_parse(mvc *m, sql_schema *s, char *q
bstream_destroy(m->scanner.rs);
 
m->sym = NULL;
+   o.vars = m->vars;   /* may have been realloc'ed */
+   o.sizevars = m->sizevars;
if (m->session->status || m->errstr[0]) {
int status = m->session->status;
char errstr[ERRSIZE];
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Apr2019 - Fixed race condition.

2019-09-16 Thread Sjoerd Mullender 
Changeset: 8e49c3ad0b8e for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8e49c3ad0b8e
Modified Files:
tools/merovingian/daemon/controlrunner.c
Branch: Apr2019
Log Message:

Fixed race condition.
If connections are made fast enough and the first started thread
hasn't read it's argument yet before the next connection is made, it's
data pointer was overwritten.  We now allocate new memory for each
connection so that each thread is guaranteed to get the correct data.
(grafted from 8f7959c6064616b301951e81fd9c780e6294377a)


diffs (35 lines):

diff --git a/tools/merovingian/daemon/controlrunner.c 
b/tools/merovingian/daemon/controlrunner.c
--- a/tools/merovingian/daemon/controlrunner.c
+++ b/tools/merovingian/daemon/controlrunner.c
@@ -991,6 +991,7 @@ handle_client(void *p)
 {
int msgsock = * (int *) p;
 
+   free(p);
ctl_handle_client("(local)", msgsock, NULL, NULL);
shutdown(msgsock, SHUT_RDWR);
closesocket(msgsock);
@@ -1006,8 +1007,13 @@ controlRunner(void *d)
struct timeval tv;
int msgsock;
pthread_t tid;
+   int *p;
 
do {
+   if ((p = malloc(sizeof(int))) == NULL) {
+   Mfprintf(_mero_ctlerr, "malloc failed");
+   break;
+   }
FD_ZERO();
FD_SET(usock, );
 
@@ -1040,7 +1046,8 @@ controlRunner(void *d)
(void) fcntl(msgsock, F_SETFD, FD_CLOEXEC);
 #endif
 
-   if (pthread_create(, NULL, handle_client, ) != 0)
+   *p = msgsock;
+   if (pthread_create(, NULL, handle_client, p) != 0)
closesocket(msgsock);
else
pthread_detach(tid);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: grouping-analytics - Added more failing queries

2019-09-16 Thread Pedro Ferreira 
Changeset: 1fad3cad448b for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1fad3cad448b
Modified Files:
sql/test/analytics/Tests/analytics11.sql
Branch: grouping-analytics
Log Message:

Added more failing queries


diffs (55 lines):

diff --git a/sql/test/analytics/Tests/analytics11.sql 
b/sql/test/analytics/Tests/analytics11.sql
--- a/sql/test/analytics/Tests/analytics11.sql
+++ b/sql/test/analytics/Tests/analytics11.sql
@@ -187,6 +187,51 @@ FROM another_T
 GROUP BY ROLLUP(col1);
 
 SELECT
+col1 IN (SELECT SUM(ColID + col1) FROM tbl_ProductSales)
+FROM another_T
+GROUP BY ROLLUP(col1);
+
+SELECT
+col3 > ALL (SELECT 1 FROM tbl_ProductSales HAVING MIN(col4) > 30)
+FROM another_T
+GROUP BY ROLLUP(col3, col4);
+
+SELECT
+col1 = ALL (SELECT 1 FROM tbl_ProductSales HAVING MIN(col2) IS NULL)
+FROM another_T
+GROUP BY CUBE(col1, col2);
+
+SELECT
+SUM(col1) IN (SELECT DISTINCT col2 FROM another_T GROUP BY col2)
+FROM another_T
+GROUP BY CUBE(col4);
+
+SELECT
+GROUPING(col6, col7) IN (SELECT SUM(col2) FROM another_T GROUP BY col5),
+NOT GROUPING(col7, col6) IN (SELECT col3 FROM another_T)
+FROM another_T
+GROUP BY CUBE(col7, col6);
+
+SELECT
+NOT col1 * col5 = ALL (SELECT 1 FROM tbl_ProductSales HAVING MAX(col2) > 
2),
+NOT AVG(col2) * col1 <> ANY (SELECT 20 FROM tbl_ProductSales HAVING 
MAX(col1) IS NULL),
+NOT EXISTS (SELECT ColID - 12 FROM tbl_ProductSales GROUP BY ColID HAVING 
MAX(col1) IS NOT NULL OR MIN(col1) < MIN(col2)),
+CAST (NOT col1 IN (SELECT col2 FROM another_T GROUP BY col2) AS INTEGER) | 
CAST (col2 IN (SELECT col2 FROM another_T GROUP BY col2) AS INTEGER),
+CAST (EXISTS (SELECT MAX(col5) * MAX(col4) FROM another_T GROUP BY col5, 
col4) AS INTEGER) & CAST (GROUPING(col1, col5) IN (SELECT DISTINCT col2 FROM 
another_T GROUP BY col2) AS INTEGER)
+FROM another_T
+GROUP BY CUBE(col1, col2, col5);
+
+SELECT
+DISTINCT
+NOT col1 * col5 = ALL (SELECT 1 FROM tbl_ProductSales HAVING MAX(col2) > 
2),
+NOT AVG(col2) * col1 <> ANY (SELECT 20 FROM tbl_ProductSales HAVING 
MAX(col1) IS NULL),
+NOT EXISTS (SELECT ColID - 12 FROM tbl_ProductSales GROUP BY ColID HAVING 
MAX(col1) IS NOT NULL OR MIN(col1) < MIN(col2)),
+CAST (NOT col1 IN (SELECT col2 FROM another_T GROUP BY col2) AS INTEGER) | 
CAST (col2 IN (SELECT col2 FROM another_T GROUP BY col2) AS INTEGER),
+CAST (EXISTS (SELECT MAX(col5) * MAX(col4) FROM another_T GROUP BY col5, 
col4) AS INTEGER) & CAST (GROUPING(col1, col5) IN (SELECT DISTINCT col2 FROM 
another_T GROUP BY col2) AS INTEGER)
+FROM another_T
+GROUP BY CUBE(col1, col2, col5);
+
+SELECT
 GROUPING(col1, col2, col5, col8),
 col1 IN (SELECT ColID + col2 FROM tbl_ProductSales),
 col1 < ANY (SELECT MAX(ColID + col2) FROM tbl_ProductSales),
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Nov2019 - One more crashing query

2019-09-16 Thread Pedro Ferreira 
Changeset: 5e0f03221880 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5e0f03221880
Modified Files:
sql/test/subquery/Tests/subquery2.sql
Branch: Nov2019
Log Message:

One more crashing query


diffs (19 lines):

diff --git a/sql/test/subquery/Tests/subquery2.sql 
b/sql/test/subquery/Tests/subquery2.sql
--- a/sql/test/subquery/Tests/subquery2.sql
+++ b/sql/test/subquery/Tests/subquery2.sql
@@ -112,6 +112,15 @@ GROUP BY col1, col2, col5;
-- False False 1 0
-- True  False 1 0
 
+SELECT
+   SUM(col1) IN (SELECT DISTINCT col2 FROM another_T GROUP BY col2)
+FROM another_T
+GROUP BY col4;
+   -- False
+   -- False
+   -- False
+   -- False
+
 INSERT INTO tbl_ProductSales VALUES (0, 'a', 'b', 0);
 SELECT col1 IN (SELECT ColID + col1 FROM tbl_ProductSales) FROM another_T 
GROUP BY col1; 
-- True
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Nov2019 - Query with wrong results

2019-09-16 Thread Pedro Ferreira 
Changeset: fd964bb00308 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=fd964bb00308
Modified Files:
sql/test/subquery/Tests/subquery2.sql
Branch: Nov2019
Log Message:

Query with wrong results


diffs (21 lines):

diff --git a/sql/test/subquery/Tests/subquery2.sql 
b/sql/test/subquery/Tests/subquery2.sql
--- a/sql/test/subquery/Tests/subquery2.sql
+++ b/sql/test/subquery/Tests/subquery2.sql
@@ -101,6 +101,17 @@ GROUP BY col1, col2, col5, col8;
-- False True True True True
-- False True True True True
 
+SELECT
+   DISTINCT
+   NOT col1 * col5 = ALL (SELECT 1 FROM tbl_ProductSales HAVING MAX(col2) 
> 2),
+   NOT AVG(col2) * col1 <> ANY (SELECT 20 FROM tbl_ProductSales HAVING 
MAX(col1) IS NOT NULL OR MIN(col1) < MIN(col2)),
+   CAST (NOT col1 IN (SELECT col2 FROM another_T GROUP BY col2) AS 
INTEGER) | CAST (col2 IN (SELECT col2 FROM another_T GROUP BY col2) AS INTEGER),
+   CAST (EXISTS (SELECT MAX(col5) * MAX(col4) FROM another_T GROUP BY 
col5, col4) AS INTEGER) & CAST (AVG(col1) IN (SELECT DISTINCT col2 FROM 
another_T GROUP BY col2) AS INTEGER)
+FROM another_T
+GROUP BY col1, col2, col5;
+   -- False False 1 0
+   -- True  False 1 0
+
 INSERT INTO tbl_ProductSales VALUES (0, 'a', 'b', 0);
 SELECT col1 IN (SELECT ColID + col1 FROM tbl_ProductSales) FROM another_T 
GROUP BY col1; 
-- True
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Nov2019 - Remove code that is never executed.

2019-09-16 Thread Sjoerd Mullender 
Changeset: 5062fc422595 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5062fc422595
Modified Files:
gdk/gdk_heap.c
Branch: Nov2019
Log Message:

Remove code that is never executed.


diffs (43 lines):

diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -119,35 +119,12 @@ HEAPalloc(Heap *h, size_t nitems, size_t
}
if (!GDKinmemory() && h->base == NULL) {
char *nme;
-   struct stat st;
 
-   if ((nme = GDKfilepath(h->farmid, BATDIR, h->filename, NULL)) 
== NULL) {
+   nme = GDKfilepath(h->farmid, BATDIR, h->filename, NULL);
+   if (nme == NULL)
return GDK_FAIL;
-   }
-   if (stat(nme, ) < 0) {
-   h->storage = STORE_MMAP;
-   h->base = HEAPcreatefile(NOFARM, >size, nme);
-   } else {
-   int fd;
-
-   fd = GDKfdlocate(NOFARM, nme, "wb", NULL);
-   if (fd >= 0) {
-   char of[sizeof(h->filename)];
-   char *ext;
-   close(fd);
-   strncpy(of, h->filename, sizeof(of));
-#ifdef STATIC_CODE_ANALYSIS
-   /* help coverity */
-   of[sizeof(h->filename) - 1] = 0;
-#endif
-   ext = decompose_filename(of);
-   h->newstorage = STORE_MMAP;
-   if (HEAPload(h, of, ext, false) != GDK_SUCCEED)
-   h->base = NULL; /* superfluous */
-   /* success checked by looking at
-* h->base below */
-   }
-   }
+   h->storage = STORE_MMAP;
+   h->base = HEAPcreatefile(NOFARM, >size, nme);
GDKfree(nme);
}
if (h->base == NULL) {
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Nov2019 - Require that some files don't exist when crea...

2019-09-16 Thread Sjoerd Mullender 
Changeset: d4d8f52b5374 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d4d8f52b5374
Modified Files:
gdk/gdk_bbp.c
gdk/gdk_heap.c
gdk/gdk_storage.c
Branch: Nov2019
Log Message:

Require that some files don't exist when created.


diffs (62 lines):

diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -1737,7 +1737,7 @@ BBPdir_subcommit(int cnt, bat *subcommit
assert(subcommit[n - 1] < subcommit[n]);
 #endif
 
-   if ((nbbpf = GDKfilelocate(0, "BBP", "w", "dir")) == NULL)
+   if ((nbbpf = GDKfilelocate(0, "BBP", "wx", "dir")) == NULL)
return GDK_FAIL;
 
n = (bat) ATOMIC_GET();
@@ -1849,7 +1849,7 @@ BBPdir(int cnt, bat *subcommit)
return BBPdir_subcommit(cnt, subcommit);
 
IODEBUG fprintf(stderr, "#BBPdir: writing BBP.dir (%d bats).\n", (int) 
(bat) ATOMIC_GET());
-   if ((fp = GDKfilelocate(0, "BBP", "w", "dir")) == NULL) {
+   if ((fp = GDKfilelocate(0, "BBP", "wx", "dir")) == NULL) {
goto bailout;
}
 
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -61,7 +61,7 @@ HEAPcreatefile(int farmid, size_t *maxsz
fn = path;
}
/* round up to mulitple of GDK_mmap_pagesize */
-   fd = GDKfdlocate(NOFARM, fn, "wb", NULL);
+   fd = GDKfdlocate(NOFARM, fn, "wxb", NULL);
if (fd >= 0) {
close(fd);
base = GDKload(NOFARM, fn, NULL, *maxsz, maxsz, STORE_MMAP);
@@ -269,6 +269,7 @@ HEAPextend(Heap *h, size_t size, bool ma
HEAPfree(, false);
return GDK_SUCCEED;
}
+   GDKclrerr();
}
fd = GDKfdlocate(h->farmid, nme, "wb", ext);
if (fd >= 0) {
@@ -444,7 +445,7 @@ GDKupgradevarheap(BAT *b, var_t v, bool 
const char *base = b->theap.base;
 
/* first save heap in file with extra .tmp extension */
-   if ((fd = GDKfdlocate(b->theap.farmid, b->theap.filename, "wb", 
"tmp")) < 0)
+   if ((fd = GDKfdlocate(b->theap.farmid, b->theap.filename, 
"wxb", "tmp")) < 0)
return GDK_FAIL;
while (size > 0) {
ret = write(fd, base, (unsigned) MIN(1 << 30, size));
diff --git a/gdk/gdk_storage.c b/gdk/gdk_storage.c
--- a/gdk/gdk_storage.c
+++ b/gdk/gdk_storage.c
@@ -218,6 +218,8 @@ GDKfdlocate(int farmid, const char *nme,
 
if (strchr(mode, 'w')) {
flags |= O_WRONLY | O_CREAT;
+   if (strchr(mode, 'x'))
+   flags |= O_EXCL;
} else if (!strchr(mode, '+')) {
flags |= O_RDONLY;
} else {
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Nov2019 - Cleanup.

2019-09-16 Thread Sjoerd Mullender 
Changeset: 954f72db5e87 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=954f72db5e87
Modified Files:
gdk/gdk_storage.c
Branch: Nov2019
Log Message:

Cleanup.


diffs (35 lines):

diff --git a/gdk/gdk_storage.c b/gdk/gdk_storage.c
--- a/gdk/gdk_storage.c
+++ b/gdk/gdk_storage.c
@@ -194,7 +194,7 @@ int
 GDKfdlocate(int farmid, const char *nme, const char *mode, const char 
*extension)
 {
char *path = NULL;
-   int fd, flags = 0;
+   int fd, flags = O_CLOEXEC;
 
assert(!GDKinmemory());
if (nme == NULL || *nme == 0)
@@ -212,7 +212,7 @@ GDKfdlocate(int farmid, const char *nme,
mode++;
 #ifdef _CYGNUS_H_
} else {
-   flags = _FRDSEQ;/* WIN32 
CreateFile(FILE_FLAG_SEQUENTIAL_SCAN) */
+   flags |= _FRDSEQ;   /* WIN32 
CreateFile(FILE_FLAG_SEQUENTIAL_SCAN) */
 #endif
}
 
@@ -226,11 +226,11 @@ GDKfdlocate(int farmid, const char *nme,
 #ifdef WIN32
flags |= strchr(mode, 'b') ? O_BINARY : O_TEXT;
 #endif
-   fd = open(nme, flags | O_CLOEXEC, MONETDB_MODE);
+   fd = open(nme, flags, MONETDB_MODE);
if (fd < 0 && *mode == 'w') {
/* try to create the directory, in case that was the problem */
if (GDKcreatedir(nme) == GDK_SUCCEED) {
-   fd = open(nme, flags | O_CLOEXEC, MONETDB_MODE);
+   fd = open(nme, flags, MONETDB_MODE);
if (fd < 0)
GDKsyserror("GDKfdlocate: cannot open file 
%s\n", nme);
}
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Nov2019 - Use atomic instructions instead of locks.

2019-09-16 Thread Sjoerd Mullender 
Changeset: da119ef6923a for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=da119ef6923a
Modified Files:
gdk/gdk_heap.c
gdk/gdk_interprocess.c
sql/backends/monet5/UDF/pyapi/pyapi.c
Branch: Nov2019
Log Message:

Use atomic instructions instead of locks.


diffs (80 lines):

diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -121,8 +121,7 @@ HEAPalloc(Heap *h, size_t nitems, size_t
char *nme;
struct stat st;
 
-   if(!(nme = GDKfilepath(h->farmid, BATDIR, h->filename, NULL))) {
-   GDKerror("HEAPalloc: malloc failure");
+   if ((nme = GDKfilepath(h->farmid, BATDIR, h->filename, NULL)) 
== NULL) {
return GDK_FAIL;
}
if (stat(nme, ) < 0) {
diff --git a/gdk/gdk_interprocess.c b/gdk/gdk_interprocess.c
--- a/gdk/gdk_interprocess.c
+++ b/gdk/gdk_interprocess.c
@@ -28,7 +28,7 @@
 #include 
 #include 
 
-static size_t interprocess_unique_id = 1;
+static ATOMIC_TYPE interprocess_unique_id = ATOMIC_VAR_INIT(1);
 static key_t base_key = 8;
 
 // Regular ftok produces too many collisions
@@ -46,12 +46,7 @@ ftok_enhanced(int id, key_t * return_key
 size_t
 GDKuniqueid(size_t offset)
 {
-   // TODO: lock this here instead of in pyapi
-   size_t id;
-
-   id = interprocess_unique_id;
-   interprocess_unique_id += offset;
-   return id;
+   return (size_t) ATOMIC_ADD(_unique_id, (ATOMIC_BASE_TYPE) 
offset);
 }
 
 //! Create a memory mapped file if it does not exist and open it
@@ -75,19 +70,21 @@ GDKinitmmap(size_t id, size_t size, size
 size = (maxsize + GDK_mmap_pagesize - 1) & ~(GDK_mmap_pagesize - 1);
 if (size == 0)
 size = GDK_mmap_pagesize; */
-   fd = GDKfdlocate(0, address, "wb", "tmp");
-   if (fd < 0) {
-   return NULL;
-   }
path = GDKfilepath(0, BATDIR, address, "tmp");
if (path == NULL) {
return NULL;
}
-   close(fd);
-   if (GDKextend(path, size) != GDK_SUCCEED) {
+   fd = GDKfdlocate(NOFARM, path, "wb", NULL);
+   if (fd < 0) {
GDKfree(path);
return NULL;
}
+   if (GDKextendf(fd, size, path) != GDK_SUCCEED) {
+   close(fd);
+   GDKfree(path);
+   return NULL;
+   }
+   close(fd);
ptr = GDKmmap(path, mod, size);
GDKfree(path);
if (ptr == NULL) {
diff --git a/sql/backends/monet5/UDF/pyapi/pyapi.c 
b/sql/backends/monet5/UDF/pyapi/pyapi.c
--- a/sql/backends/monet5/UDF/pyapi/pyapi.c
+++ b/sql/backends/monet5/UDF/pyapi/pyapi.c
@@ -329,9 +329,7 @@ static str PyAPIeval(Client cntxt, MalBl
int mmap_count = 4 + pci->retc * 2;
 
// create initial shared memory
-   MT_lock_set();
mmap_id = GDKuniqueid(mmap_count);
-   MT_lock_unset();
 
mmap_ptrs = GDKzalloc(mmap_count * sizeof(void *));
mmap_sizes = GDKzalloc(mmap_count * sizeof(size_t));
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Nov2019 - More failing sql subqueries. Mal definition f...

2019-09-16 Thread Pedro Ferreira 
Changeset: 8ac5d434eb67 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8ac5d434eb67
Modified Files:
sql/test/subquery/Tests/subquery2.sql
Branch: Nov2019
Log Message:

More failing sql subqueries. Mal definition for aggr.not_exist is missing.


diffs (67 lines):

diff --git a/sql/test/subquery/Tests/subquery2.sql 
b/sql/test/subquery/Tests/subquery2.sql
--- a/sql/test/subquery/Tests/subquery2.sql
+++ b/sql/test/subquery/Tests/subquery2.sql
@@ -50,18 +50,18 @@ FROM another_T GROUP BY col1, col2, col5
-- False True
 
 SELECT
-col1 IN (SELECT ColID FROM tbl_ProductSales),
+   -col1 IN (SELECT ColID FROM tbl_ProductSales),
col5 = ALL (SELECT 1 FROM tbl_ProductSales HAVING MIN(col8) IS NULL)
 FROM another_T GROUP BY col1, col2, col5, col8;
-   -- True  True
+   -- False True
-- False True
-- False True
-- False True
 
 SELECT
-col1 + col5 = (SELECT MIN(ColID) FROM tbl_ProductSales),
-CAST(SUM(DISTINCT CASE WHEN col5 - col8 = (SELECT MIN(ColID / col2) FROM 
tbl_ProductSales) THEN col2 - 5 ELSE ABS(col1) END) AS BIGINT),
-(SELECT MAX(ColID + col2) FROM tbl_ProductSales) * DENSE_RANK() OVER 
(PARTITION BY AVG(DISTINCT col5))
+   col1 + col5 = (SELECT MIN(ColID) FROM tbl_ProductSales),
+   CAST(SUM(DISTINCT CASE WHEN col5 - col8 = (SELECT MIN(ColID / col2) 
FROM tbl_ProductSales) THEN col2 - 5 ELSE ABS(col1) END) AS BIGINT),
+   (SELECT MAX(ColID + col2) FROM tbl_ProductSales) * DENSE_RANK() OVER 
(PARTITION BY AVG(DISTINCT col5))
 FROM another_T
 GROUP BY col1, col2, col5, col8;
-- False 16
@@ -70,8 +70,8 @@ GROUP BY col1, col2, col5, col8;
-- False  2226
 
 SELECT
-col1 + col5 = (SELECT MIN(ColID) FROM tbl_ProductSales),
-MIN(col8) OVER (PARTITION BY col5 ORDER BY col1 ROWS UNBOUNDED PRECEDING)
+   col1 + col5 = (SELECT MIN(ColID) FROM tbl_ProductSales),
+   MIN(col8) OVER (PARTITION BY col5 ORDER BY col1 ROWS UNBOUNDED 
PRECEDING)
 FROM another_T
 GROUP BY col1, col2, col5, col8;
-- False 8
@@ -79,6 +79,28 @@ GROUP BY col1, col2, col5, col8;
-- False 888
-- False 
 
+SELECT
+   EXISTS (SELECT 1 FROM tbl_ProductSales),
+   NOT EXISTS (SELECT 1 FROM tbl_ProductSales)
+FROM another_T
+GROUP BY col1;
+   -- True False
+   -- True False
+   -- True False
+   -- True False
+
+SELECT
+   NOT -SUM(col2) NOT IN (SELECT ColID FROM tbl_ProductSales GROUP BY 
ColID HAVING SUM(ColID - col8) <> col5),
+   NOT col5 = ALL (SELECT 1 FROM tbl_ProductSales HAVING MAX(col8) > 2 AND 
MIN(col8) IS NOT NULL),
+   NOT col2 <> ANY (SELECT 20 FROM tbl_ProductSales GROUP BY ColID HAVING 
NOT MAX(col1) <> col1 * AVG(col1 + ColID) * ColID),
+   NOT EXISTS (SELECT ColID - 12 FROM tbl_ProductSales GROUP BY ColID 
HAVING MAX(col2) IS NULL OR NOT col8 <> 2 / col1)
+FROM another_T
+GROUP BY col1, col2, col5, col8;
+   -- False True True True True
+   -- False True True True True
+   -- False True True True True
+   -- False True True True True
+
 INSERT INTO tbl_ProductSales VALUES (0, 'a', 'b', 0);
 SELECT col1 IN (SELECT ColID + col1 FROM tbl_ProductSales) FROM another_T 
GROUP BY col1; 
-- True
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Nov2019 - More failing sql subqueries :(

2019-09-16 Thread Pedro Ferreira 
Changeset: 63117af9d7eb for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=63117af9d7eb
Modified Files:
sql/test/subquery/Tests/subquery2.sql
Branch: Nov2019
Log Message:

More failing sql subqueries :(


diffs (49 lines):

diff --git a/sql/test/subquery/Tests/subquery2.sql 
b/sql/test/subquery/Tests/subquery2.sql
--- a/sql/test/subquery/Tests/subquery2.sql
+++ b/sql/test/subquery/Tests/subquery2.sql
@@ -40,6 +40,45 @@ SELECT col1 IN (SELECT SUM(ColID + col1)
-- False
-- False
 
+SELECT
+   EXISTS (SELECT col2 FROM tbl_ProductSales WHERE tbl_ProductSales.ColID 
= another_T.col1),
+   (SELECT ColID FROM tbl_ProductSales) * DENSE_RANK() OVER (PARTITION BY 
AVG(DISTINCT col5))
+FROM another_T GROUP BY col1, col2, col5, col8;
+   -- True  True
+   -- False True
+   -- False True
+   -- False True
+
+SELECT
+col1 IN (SELECT ColID FROM tbl_ProductSales),
+   col5 = ALL (SELECT 1 FROM tbl_ProductSales HAVING MIN(col8) IS NULL)
+FROM another_T GROUP BY col1, col2, col5, col8;
+   -- True  True
+   -- False True
+   -- False True
+   -- False True
+
+SELECT
+col1 + col5 = (SELECT MIN(ColID) FROM tbl_ProductSales),
+CAST(SUM(DISTINCT CASE WHEN col5 - col8 = (SELECT MIN(ColID / col2) FROM 
tbl_ProductSales) THEN col2 - 5 ELSE ABS(col1) END) AS BIGINT),
+(SELECT MAX(ColID + col2) FROM tbl_ProductSales) * DENSE_RANK() OVER 
(PARTITION BY AVG(DISTINCT col5))
+FROM another_T
+GROUP BY col1, col2, col5, col8;
+   -- False 16
+   -- False 11   26
+   -- False 111  226
+   -- False  2226
+
+SELECT
+col1 + col5 = (SELECT MIN(ColID) FROM tbl_ProductSales),
+MIN(col8) OVER (PARTITION BY col5 ORDER BY col1 ROWS UNBOUNDED PRECEDING)
+FROM another_T
+GROUP BY col1, col2, col5, col8;
+   -- False 8
+   -- False 88
+   -- False 888
+   -- False 
+
 INSERT INTO tbl_ProductSales VALUES (0, 'a', 'b', 0);
 SELECT col1 IN (SELECT ColID + col1 FROM tbl_ProductSales) FROM another_T 
GROUP BY col1; 
-- True
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Nov2019 - Don't flush logger if there are no changes.

2019-09-16 Thread Sjoerd Mullender 
Changeset: da246ae1cd0b for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=da246ae1cd0b
Modified Files:
sql/storage/store.c
Branch: Nov2019
Log Message:

Don't flush logger if there are no changes.


diffs (21 lines):

diff --git a/sql/storage/store.c b/sql/storage/store.c
--- a/sql/storage/store.c
+++ b/sql/storage/store.c
@@ -2122,13 +2122,16 @@ flusher_should_run(void)
 {
// We will flush if we have a reason to and no reason not to.
char *reason_to = NULL, *reason_not_to = NULL;
+   int changes;
 
if (flusher.countdown_ms <= 0)
reason_to = "timer expired";
 
int many_changes = GDKdebug & FORCEMITOMASK ? 100 : 100;
-   if (logger_funcs.changes() >= many_changes)
+   if ((changes = logger_funcs.changes()) >= many_changes)
reason_to = "many changes";
+   else if (changes == 0)
+   reason_not_to = "no changes";
 
// Read and clear flush_now. If we decide not to flush
// we'll put it back.
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Nov2019 - Crashing sql subqueries :(

2019-09-16 Thread Pedro Ferreira 
Changeset: f8c4f48daa89 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f8c4f48daa89
Modified Files:
sql/test/subquery/Tests/subquery2.sql
Branch: Nov2019
Log Message:

Crashing sql subqueries :(


diffs (28 lines):

diff --git a/sql/test/subquery/Tests/subquery2.sql 
b/sql/test/subquery/Tests/subquery2.sql
--- a/sql/test/subquery/Tests/subquery2.sql
+++ b/sql/test/subquery/Tests/subquery2.sql
@@ -34,6 +34,12 @@ SELECT col1 IN (SELECT ColID + col1 FROM
-- False
-- False
 
+SELECT col1 IN (SELECT SUM(ColID + col1) FROM tbl_ProductSales) FROM another_T 
GROUP BY col1;
+   -- False
+   -- False
+   -- False
+   -- False
+
 INSERT INTO tbl_ProductSales VALUES (0, 'a', 'b', 0);
 SELECT col1 IN (SELECT ColID + col1 FROM tbl_ProductSales) FROM another_T 
GROUP BY col1; 
-- True
@@ -41,5 +47,11 @@ SELECT col1 IN (SELECT ColID + col1 FROM
-- True
-- True
 
+SELECT col1 IN (SELECT col1 * SUM(ColID + col1) FROM tbl_ProductSales) FROM 
another_T GROUP BY col1;
+   -- False
+   -- False
+   -- False
+   -- False
+
 DROP TABLE tbl_ProductSales;
 DROP TABLE another_T;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: grouping-analytics - Merge with default

2019-09-16 Thread Pedro Ferreira 
Changeset: c7d1b716b250 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c7d1b716b250
Added Files:
sql/test/BugTracker-2019/Tests/python-loader-string.Bug-6759.py
sql/test/BugTracker-2019/Tests/python-loader-string.Bug-6759.stable.err
sql/test/BugTracker-2019/Tests/python-loader-string.Bug-6759.stable.out
Modified Files:
NT/rules.msc
clients/mapiclient/mclient.c
configure.ag
gdk/gdk_logger.c
monetdb5/modules/mal/tablet.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_cat.c
sql/backends/monet5/sql_upgrades.c
sql/common/sql_types.c
sql/common/sql_types.h
sql/include/sql_catalog.h
sql/server/rel_psm.c
sql/server/rel_psm.h
sql/server/rel_schema.c
sql/server/rel_select.c
sql/server/rel_unnest.c
sql/server/rel_unnest.h
sql/server/sql_mvc.c
sql/server/sql_mvc.h
sql/server/sql_tokens.h
sql/storage/sql_catalog.c
sql/storage/sql_storage.h
sql/storage/store.c
sql/test/BugTracker-2009/Tests/All
sql/test/BugTracker-2018/Tests/All

sql/test/BugTracker-2018/Tests/sqlitelogictest-having-not-null-not-in.Bug-6557.sql

sql/test/BugTracker-2018/Tests/sqlitelogictest-having-not-null-not-in.Bug-6557.stable.out
sql/test/BugTracker-2019/Tests/All

sql/test/BugTracker-2019/Tests/duplicates-not-eliminated-long-CASE-stmt.Bug-6697.sql
sql/test/BugTracker/Tests/All
sql/test/SQLite_regress/sqllogictest/sqllogictest.py
testing/Mtest.py.in
tools/merovingian/daemon/controlrunner.c
Branch: grouping-analytics
Log Message:

Merge with default


diffs (truncated from 1788 to 300 lines):

diff --git a/NT/rules.msc b/NT/rules.msc
--- a/NT/rules.msc
+++ b/NT/rules.msc
@@ -42,6 +42,7 @@ BITSMCHN=X64
 # force W_CFLAGS off in debug builds: using icc, you get way more
 # warnings about casts which results in the code being uncompilable
 W_CFLAGS =
+HAVE_MAL_DEBUGGER = 1
 !ENDIF
 !IFDEF ENABLE_STRICT
 # strict mode: treat warnings as errors
@@ -386,6 +387,11 @@ create_winconfig_conds_new_py:
 !ELSE
$(ECHO) HAVE_SHP_FALSE='' >> "$(TOPDIR)\winconfig_conds_new.py"
 !ENDIF
+!IFDEF HAVE_MAL_DEBUGGER
+   $(ECHO) HAVE_MAL_DEBUGGER_FALSE='#' >> 
"$(TOPDIR)\winconfig_conds_new.py"
+!ELSE
+   $(ECHO) HAVE_MAL_DEBUGGER_FALSE='' >> "$(TOPDIR)\winconfig_conds_new.py"
+!ENDIF
 !IFDEF HAVE_SQL
$(ECHO) HAVE_SQL_FALSE='#' >> "$(TOPDIR)\winconfig_conds_new.py"
 !ELSE
diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c
--- a/clients/mapiclient/mclient.c
+++ b/clients/mapiclient/mclient.c
@@ -3664,6 +3664,9 @@ main(int argc, char **argv)
if (mode == SQL)
dump_version(mid, toConsole, "Database:");
 
+   mnstr_printf(toConsole, "FOLLOW US on 
https://twitter.com/MonetDB ");
+   mnstr_printf(toConsole, "or 
https://github.com/MonetDB/MonetDB\n;);
+
mnstr_printf(toConsole, "Type \\q to quit, \\? for a list of 
available commands\n");
if (mode == SQL)
mnstr_printf(toConsole, "auto commit mode: %s\n",
diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -838,6 +838,7 @@ AS_VAR_IF([enable_assert], [no], [
AC_MSG_RESULT([no])],
[AC_MSG_RESULT([yes])])
 
+AM_CONDITIONAL([HAVE_MAL_DEBUGGER], [test "x$enable_assert" != xno])
 
 AC_MSG_CHECKING([for --enable-optimize])
 AS_CASE([$enable_optimize], [yes], [
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -3034,8 +3034,10 @@ pre_allocate(logger *lg)
p = (lng) getfilepos(getFile(lg->log));
if (p == -1)
return GDK_FAIL;
-   if (p > LOG_LARGE)
+   if (p > LOG_LARGE) {
+   lg->id++;
return logger_open(lg);
+   }
if (p + DBLKSZ > lg->end) {
p &= ~(DBLKSZ - 1);
p += SEGSZ;
diff --git a/monetdb5/modules/mal/tablet.c b/monetdb5/modules/mal/tablet.c
--- a/monetdb5/modules/mal/tablet.c
+++ b/monetdb5/modules/mal/tablet.c
@@ -670,7 +670,7 @@ tablet_error(READERtask *task, lng row, 
BUNappend(task->cntxt->error_msg, msg, false) != 
GDK_SUCCEED ||
BUNappend(task->cntxt->error_input, fcn, false) != 
GDK_SUCCEED)
task->besteffort = 0;
-   if (!is_lng_nil(row) && task->rowerror)
+   if (!is_lng_nil(row) && task->rowerror && row < task->limit)
task->rowerror[row]++;
}
if (task->as->error == NULL) {
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -294,7 +294,7 @@ create_table_or_view(mvc *sql, char* sna
int check = 0;
 
if (STORE_READONLY)
-

MonetDB: default - Merge with Nov2019 branch

2019-09-16 Thread Pedro Ferreira 
Changeset: 218824eb44c5 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=218824eb44c5
Added Files:
sql/test/BugTracker-2019/Tests/python-loader-string.Bug-6759.py
sql/test/BugTracker-2019/Tests/python-loader-string.Bug-6759.stable.err
sql/test/BugTracker-2019/Tests/python-loader-string.Bug-6759.stable.out
Modified Files:
clients/mapiclient/mclient.c
gdk/gdk_logger.c
monetdb5/modules/mal/tablet.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_upgrades.c
sql/server/rel_psm.c
sql/test/BugTracker-2018/Tests/All

sql/test/BugTracker-2018/Tests/sqlitelogictest-having-not-null-not-in.Bug-6557.sql

sql/test/BugTracker-2018/Tests/sqlitelogictest-having-not-null-not-in.Bug-6557.stable.out
sql/test/BugTracker-2019/Tests/All

sql/test/BugTracker-2019/Tests/duplicates-not-eliminated-long-CASE-stmt.Bug-6697.sql
Branch: default
Log Message:

Merge with Nov2019 branch


diffs (truncated from 428 to 300 lines):

diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c
--- a/clients/mapiclient/mclient.c
+++ b/clients/mapiclient/mclient.c
@@ -3664,6 +3664,9 @@ main(int argc, char **argv)
if (mode == SQL)
dump_version(mid, toConsole, "Database:");
 
+   mnstr_printf(toConsole, "FOLLOW US on 
https://twitter.com/MonetDB ");
+   mnstr_printf(toConsole, "or 
https://github.com/MonetDB/MonetDB\n;);
+
mnstr_printf(toConsole, "Type \\q to quit, \\? for a list of 
available commands\n");
if (mode == SQL)
mnstr_printf(toConsole, "auto commit mode: %s\n",
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -3034,8 +3034,10 @@ pre_allocate(logger *lg)
p = (lng) getfilepos(getFile(lg->log));
if (p == -1)
return GDK_FAIL;
-   if (p > LOG_LARGE)
+   if (p > LOG_LARGE) {
+   lg->id++;
return logger_open(lg);
+   }
if (p + DBLKSZ > lg->end) {
p &= ~(DBLKSZ - 1);
p += SEGSZ;
diff --git a/monetdb5/modules/mal/tablet.c b/monetdb5/modules/mal/tablet.c
--- a/monetdb5/modules/mal/tablet.c
+++ b/monetdb5/modules/mal/tablet.c
@@ -670,7 +670,7 @@ tablet_error(READERtask *task, lng row, 
BUNappend(task->cntxt->error_msg, msg, false) != 
GDK_SUCCEED ||
BUNappend(task->cntxt->error_input, fcn, false) != 
GDK_SUCCEED)
task->besteffort = 0;
-   if (!is_lng_nil(row) && task->rowerror)
+   if (!is_lng_nil(row) && task->rowerror && row < task->limit)
task->rowerror[row]++;
}
if (task->as->error == NULL) {
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -464,8 +464,7 @@ create_table_from_emit(Client cntxt, cha
return msg;
 
/* for some reason we don't have an allocator here, so make one */
-   sql->sa = sa_create();
-   if (!sql->sa) {
+   if (!(sql->sa = sa_create())) {
msg = sql_error(sql, 02, SQLSTATE(HY001) "CREATE TABLE: %s", 
MAL_MALLOC_FAIL);
goto cleanup;
}
@@ -481,44 +480,44 @@ create_table_from_emit(Client cntxt, cha
goto cleanup;
}
 
-   for(i = 0; i < ncols; i++) {
+   for (i = 0; i < ncols; i++) {
BAT *b = columns[i].b;
-   sql_subtype *tpe = sql_bind_localtype(ATOMname(b->ttype));
+   str atoname = ATOMname(b->ttype);
+   sql_subtype tpe;
sql_column *col = NULL;
 
-   if (!tpe) {
-   msg = sql_error(sql, 02, SQLSTATE(3F000) "CREATE TABLE: 
could not find type for column");
-   goto cleanup;
+   if (!strcmp(atoname, "str"))
+   sql_find_subtype(, "clob", 0, 0);
+   else {
+   sql_subtype *t = sql_bind_localtype(atoname);
+   if (!t) {
+   msg = sql_error(sql, 02, SQLSTATE(3F000) 
"CREATE TABLE: could not find type for column");
+   goto cleanup;
+   }
+   tpe = *t;
}
 
-   col = mvc_create_column(sql, t, columns[i].name, tpe);
-   if (!col) {
+   if (!(col = mvc_create_column(sql, t, columns[i].name, ))) {
msg = sql_error(sql, 02, SQLSTATE(3F000) "CREATE TABLE: 
could not create column %s", columns[i].name);
goto cleanup;
}
}
-   msg = create_table_or_view(sql, sname, t->base.name, t, 0);
-   if (msg != MAL_SUCCEED) {
+   if ((msg = create_table_or_view(sql, sname,

MonetDB: Nov2019 - Merge with Apr2019 branch

2019-09-16 Thread Pedro Ferreira 
Changeset: b64b64deb697 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b64b64deb697
Added Files:
sql/test/BugTracker-2019/Tests/python-loader-string.Bug-6759.py
sql/test/BugTracker-2019/Tests/python-loader-string.Bug-6759.stable.err
sql/test/BugTracker-2019/Tests/python-loader-string.Bug-6759.stable.out
Modified Files:
monetdb5/modules/mal/tablet.c
sql/backends/monet5/sql.c
sql/test/BugTracker-2019/Tests/All

sql/test/BugTracker-2019/Tests/duplicates-not-eliminated-long-CASE-stmt.Bug-6697.sql
Branch: Nov2019
Log Message:

Merge with Apr2019 branch


diffs (truncated from 338 to 300 lines):

diff --git a/monetdb5/modules/mal/tablet.c b/monetdb5/modules/mal/tablet.c
--- a/monetdb5/modules/mal/tablet.c
+++ b/monetdb5/modules/mal/tablet.c
@@ -670,10 +670,8 @@ tablet_error(READERtask *task, lng row, 
BUNappend(task->cntxt->error_msg, msg, false) != 
GDK_SUCCEED ||
BUNappend(task->cntxt->error_input, fcn, false) != 
GDK_SUCCEED)
task->besteffort = 0;
-   if (!is_lng_nil(row) && task->rowerror) {
-   assert(row < task->limit);
+   if (!is_lng_nil(row) && task->rowerror && row < task->limit)
task->rowerror[row]++;
-   }
}
if (task->as->error == NULL) {
if (msg == NULL)
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -463,8 +463,7 @@ create_table_from_emit(Client cntxt, cha
return msg;
 
/* for some reason we don't have an allocator here, so make one */
-   sql->sa = sa_create();
-   if (!sql->sa) {
+   if (!(sql->sa = sa_create())) {
msg = sql_error(sql, 02, SQLSTATE(HY001) "CREATE TABLE: %s", 
MAL_MALLOC_FAIL);
goto cleanup;
}
@@ -480,44 +479,44 @@ create_table_from_emit(Client cntxt, cha
goto cleanup;
}
 
-   for(i = 0; i < ncols; i++) {
+   for (i = 0; i < ncols; i++) {
BAT *b = columns[i].b;
-   sql_subtype *tpe = sql_bind_localtype(ATOMname(b->ttype));
+   str atoname = ATOMname(b->ttype);
+   sql_subtype tpe;
sql_column *col = NULL;
 
-   if (!tpe) {
-   msg = sql_error(sql, 02, SQLSTATE(3F000) "CREATE TABLE: 
could not find type for column");
-   goto cleanup;
+   if (!strcmp(atoname, "str"))
+   sql_find_subtype(, "clob", 0, 0);
+   else {
+   sql_subtype *t = sql_bind_localtype(atoname);
+   if (!t) {
+   msg = sql_error(sql, 02, SQLSTATE(3F000) 
"CREATE TABLE: could not find type for column");
+   goto cleanup;
+   }
+   tpe = *t;
}
 
-   col = mvc_create_column(sql, t, columns[i].name, tpe);
-   if (!col) {
+   if (!(col = mvc_create_column(sql, t, columns[i].name, ))) {
msg = sql_error(sql, 02, SQLSTATE(3F000) "CREATE TABLE: 
could not create column %s", columns[i].name);
goto cleanup;
}
}
-   msg = create_table_or_view(sql, sname, t->base.name, t, 0);
-   if (msg != MAL_SUCCEED) {
+   if ((msg = create_table_or_view(sql, sname, t->base.name, t, 0)) != 
MAL_SUCCEED)
goto cleanup;
-   }
-   t = mvc_bind_table(sql, s, tname);
-   if (!t) {
+   if (!(t = mvc_bind_table(sql, s, tname))) {
msg = sql_error(sql, 02, SQLSTATE(3F000) "CREATE TABLE: could 
not bind table %s", tname);
goto cleanup;
}
-   for(i = 0; i < ncols; i++) {
+   for (i = 0; i < ncols; i++) {
BAT *b = columns[i].b;
sql_column *col = NULL;
 
-   col = mvc_bind_column(sql,t, columns[i].name);
-   if (!col) {
+   if (!(col = mvc_bind_column(sql, t, columns[i].name))) {
msg = sql_error(sql, 02, SQLSTATE(3F000) "CREATE TABLE: 
could not bind column %s", columns[i].name);
goto cleanup;
}
-   msg = mvc_append_column(sql->session->tr, col, b);
-   if (msg != MAL_SUCCEED) {
+   if ((msg = mvc_append_column(sql->session->tr, col, b)) != 
MAL_SUCCEED)
goto cleanup;
-   }
}
 
 cleanup:
@@ -543,8 +542,7 @@ append_to_table_from_emit(Client cntxt, 
return msg;
 
/* for some reason we don't have an allocator here, so make one */
-   sql->sa = sa_create();
-   if (!sql->sa) {
+   if (!(sql->sa = sa_create())) {
msg = sql_error(sql, 02,

MonetDB: Apr2019 - Added test and fix for bug 6759, (i.e. when c...

2019-09-16 Thread Pedro Ferreira 
Changeset: 1dd5b7a992e7 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1dd5b7a992e7
Added Files:
sql/test/BugTracker-2019/Tests/python-loader-string.Bug-6759.py
sql/test/BugTracker-2019/Tests/python-loader-string.Bug-6759.stable.err
sql/test/BugTracker-2019/Tests/python-loader-string.Bug-6759.stable.out
Modified Files:
sql/backends/monet5/sql.c
sql/test/BugTracker-2019/Tests/All
Branch: Apr2019
Log Message:

Added test and fix for bug 6759, (i.e. when creating a string column from the 
loader, use clob type instead of char)


diffs (truncated from 332 to 300 lines):

diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -458,8 +458,7 @@ create_table_from_emit(Client cntxt, cha
return msg;
 
/* for some reason we don't have an allocator here, so make one */
-   sql->sa = sa_create();
-   if (!sql->sa) {
+   if (!(sql->sa = sa_create())) {
msg = sql_error(sql, 02, SQLSTATE(HY001) "CREATE TABLE: %s", 
MAL_MALLOC_FAIL);
goto cleanup;
}
@@ -467,52 +466,52 @@ create_table_from_emit(Client cntxt, cha
if (!sname)
sname = "sys";
if (!(s = mvc_bind_schema(sql, sname))) {
-   msg = sql_error(sql, 02, "3F000!CREATE TABLE: no such schema 
'%s'", sname);
+   msg = sql_error(sql, 02, SQLSTATE(3F000) "CREATE TABLE: no such 
schema '%s'", sname);
goto cleanup;
}
if (!(t = mvc_create_table(sql, s, tname, tt_table, 0, 
SQL_DECLARED_TABLE, CA_COMMIT, -1, 0))) {
-   msg = sql_error(sql, 02, "3F000!CREATE TABLE: could not create 
table '%s'", tname);
+   msg = sql_error(sql, 02, SQLSTATE(3F000) "CREATE TABLE: could 
not create table '%s'", tname);
goto cleanup;
}
 
-   for(i = 0; i < ncols; i++) {
+   for (i = 0; i < ncols; i++) {
BAT *b = columns[i].b;
-   sql_subtype *tpe = sql_bind_localtype(ATOMname(b->ttype));
+   str atoname = ATOMname(b->ttype);
+   sql_subtype tpe;
sql_column *col = NULL;
 
-   if (!tpe) {
-   msg = sql_error(sql, 02, "3F000!CREATE TABLE: could not 
find type for column");
-   goto cleanup;
+   if (!strcmp(atoname, "str"))
+   sql_find_subtype(, "clob", 0, 0);
+   else {
+   sql_subtype *t = sql_bind_localtype(atoname);
+   if (!t) {
+   msg = sql_error(sql, 02, SQLSTATE(3F000) 
"CREATE TABLE: could not find type for column");
+   goto cleanup;
+   }
+   tpe = *t;
}
 
-   col = mvc_create_column(sql, t, columns[i].name, tpe);
-   if (!col) {
-   msg = sql_error(sql, 02, "3F000!CREATE TABLE: could not 
create column %s", columns[i].name);
+   if (!(col = mvc_create_column(sql, t, columns[i].name, ))) {
+   msg = sql_error(sql, 02, SQLSTATE(3F000) "CREATE TABLE: 
could not create column %s", columns[i].name);
goto cleanup;
}
}
-   msg = create_table_or_view(sql, sname, t->base.name, t, 0);
-   if (msg != MAL_SUCCEED) {
+   if ((msg = create_table_or_view(sql, sname, t->base.name, t, 0)) != 
MAL_SUCCEED)
goto cleanup;
-   }
-   t = mvc_bind_table(sql, s, tname);
-   if (!t) {
-   msg = sql_error(sql, 02, "3F000!CREATE TABLE: could not bind 
table %s", tname);
+   if (!(t = mvc_bind_table(sql, s, tname))) {
+   msg = sql_error(sql, 02, SQLSTATE(3F000) "CREATE TABLE: could 
not bind table %s", tname);
goto cleanup;
}
-   for(i = 0; i < ncols; i++) {
+   for (i = 0; i < ncols; i++) {
BAT *b = columns[i].b;
sql_column *col = NULL;
 
-   col = mvc_bind_column(sql,t, columns[i].name);
-   if (!col) {
-   msg = sql_error(sql, 02, "3F000!CREATE TABLE: could not 
bind column %s", columns[i].name);
+   if (!(col = mvc_bind_column(sql,t, columns[i].name))) {
+   msg = sql_error(sql, 02, SQLSTATE(3F000) "CREATE TABLE: 
could not bind column %s", columns[i].name);
goto cleanup;
}
-   msg = mvc_append_column(sql->session->tr, col, b);
-   if (msg != MAL_SUCCEED) {
+   if ((msg = mvc_append_column(sql->session->tr, col, b)) != 
MAL_SUCCEED)
goto cleanup;
-   }
}
 
 cleanup:
@@ -538,8 +537,7 @@ append_to_table_from_emit(Client cntxt, 
return msg;
 
/* for some reason we don't have an

MonetDB: Nov2019 - Removed debug statement from the test.

2019-09-16 Thread Pedro Ferreira 
Changeset: 1a412cecaaa4 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1a412cecaaa4
Modified Files:
sql/test/BugTracker-2018/Tests/All

sql/test/BugTracker-2018/Tests/sqlitelogictest-having-not-null-not-in.Bug-6557.sql

sql/test/BugTracker-2018/Tests/sqlitelogictest-having-not-null-not-in.Bug-6557.stable.out
Branch: Nov2019
Log Message:

Removed debug statement from the test.

It was still causing trouble on Windows testing machines.


diffs (32 lines):

diff --git a/sql/test/BugTracker-2018/Tests/All 
b/sql/test/BugTracker-2018/Tests/All
--- a/sql/test/BugTracker-2018/Tests/All
+++ b/sql/test/BugTracker-2018/Tests/All
@@ -37,7 +37,7 @@ sqlitelogictest-groupby-having-not-in.Bu
 sqlitelogictest-nested-case.Bug-6554
 sqlitelogictest-complex-case-extra-columns.Bug-6555
 sqlitelogictest-coalesce-division-by-zero.Bug-6556
-HAVE_MAL_DEBUGGER?sqlitelogictest-having-not-null-not-in.Bug-6557
+sqlitelogictest-having-not-null-not-in.Bug-6557
 sqlitelogictest-groupby-having-in.Bug-6560
 sqlitelogictest-groupby-having-in-cast.Bug-6561
 sqlitelogictest-groupby-having-not-null-not-in.Bug-6562
diff --git 
a/sql/test/BugTracker-2018/Tests/sqlitelogictest-having-not-null-not-in.Bug-6557.sql
 
b/sql/test/BugTracker-2018/Tests/sqlitelogictest-having-not-null-not-in.Bug-6557.sql
--- 
a/sql/test/BugTracker-2018/Tests/sqlitelogictest-having-not-null-not-in.Bug-6557.sql
+++ 
b/sql/test/BugTracker-2018/Tests/sqlitelogictest-having-not-null-not-in.Bug-6557.sql
@@ -8,4 +8,3 @@ SELECT CAST(SUM(col0) AS BIGINT) FROM ta
 SELECT DISTINCT COUNT(*) FROM tab0 WHERE NOT col2 NOT BETWEEN ( 35 ) AND ( 
NULL );
 SELECT CAST(- COUNT(*) * - - 61 + + + ( + COUNT(*) ) AS BIGINT) FROM tab0 
WHERE NOT col0 + + 10 BETWEEN NULL AND NULL;
 DROP TABLE tab0;
-debug SELECT CAST(SUM(col0) AS BIGINT) FROM (VALUES(83,0,38), (26,0,79), 
(43,81,24)) as tab0(col0,col1,col2) WHERE + + col0 NOT BETWEEN NULL AND + col2;
diff --git 
a/sql/test/BugTracker-2018/Tests/sqlitelogictest-having-not-null-not-in.Bug-6557.stable.out
 
b/sql/test/BugTracker-2018/Tests/sqlitelogictest-having-not-null-not-in.Bug-6557.stable.out
--- 
a/sql/test/BugTracker-2018/Tests/sqlitelogictest-having-not-null-not-in.Bug-6557.stable.out
+++ 
b/sql/test/BugTracker-2018/Tests/sqlitelogictest-having-not-null-not-in.Bug-6557.stable.out
@@ -64,8 +64,6 @@ stdout of test 'sqlitelogictest-having-n
 % 1 # length
 [ 0]
 #DROP TABLE tab0;
-#debug SELECT CAST(SUM(col0) AS BIGINT) FROM (VALUES(83,0,38), (26,0,79), 
(43,81,24)) as tab0(col0,col1,col2) WHERE + + col0 NOT BETWEEN NULL AND + col2;
-mdb>#X_11=0@0:void := querylog.define("debug select cast(sum(col0) as bigint) 
from (values(83,0,38), (26,0,79), (43,81,24)) as tab0(col0,col1,col2) where + + 
col0 not between null and + col2;", "default_pipe", 27:int);
 
 # 12:32:58 >  
 # 12:32:58 >  "Done."
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list