Changeset: 1390f2503198 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1390f2503198
Modified Files:
gdk/gdk_bbp.c
gdk/gdk_heap.c
gdk/gdk_interprocess.c
gdk/gdk_storage.c
sql/backends/monet5/UDF/pyapi/pyapi.c
sql/server/rel_semantic.c
sql/storage/store.c
sql/test/subquery/Tests/subquery2.sql
tools/merovingian/daemon/controlrunner.c
Branch: default
Log Message:
Merge with Nov2019 branch.
diffs (truncated from 342 to 300 lines):
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -1737,7 +1737,7 @@ BBPdir_subcommit(int cnt, bat *subcommit
assert(subcommit[n - 1] < subcommit[n]);
#endif
- if ((nbbpf = GDKfilelocate(0, "BBP", "w", "dir")) == NULL)
+ if ((nbbpf = GDKfilelocate(0, "BBP", "wx", "dir")) == NULL)
return GDK_FAIL;
n = (bat) ATOMIC_GET(&BBPsize);
@@ -1849,7 +1849,7 @@ BBPdir(int cnt, bat *subcommit)
return BBPdir_subcommit(cnt, subcommit);
IODEBUG fprintf(stderr, "#BBPdir: writing BBP.dir (%d bats).\n", (int)
(bat) ATOMIC_GET(&BBPsize));
- if ((fp = GDKfilelocate(0, "BBP", "w", "dir")) == NULL) {
+ if ((fp = GDKfilelocate(0, "BBP", "wx", "dir")) == NULL) {
goto bailout;
}
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -61,7 +61,7 @@ HEAPcreatefile(int farmid, size_t *maxsz
fn = path;
}
/* round up to mulitple of GDK_mmap_pagesize */
- fd = GDKfdlocate(NOFARM, fn, "wb", NULL);
+ fd = GDKfdlocate(NOFARM, fn, "wxb", NULL);
if (fd >= 0) {
close(fd);
base = GDKload(NOFARM, fn, NULL, *maxsz, maxsz, STORE_MMAP);
@@ -119,36 +119,12 @@ HEAPalloc(Heap *h, size_t nitems, size_t
}
if (!GDKinmemory() && h->base == NULL) {
char *nme;
- struct stat st;
- if(!(nme = GDKfilepath(h->farmid, BATDIR, h->filename, NULL))) {
- GDKerror("HEAPalloc: malloc failure");
+ nme = GDKfilepath(h->farmid, BATDIR, h->filename, NULL);
+ if (nme == NULL)
return GDK_FAIL;
- }
- if (stat(nme, &st) < 0) {
- h->storage = STORE_MMAP;
- h->base = HEAPcreatefile(NOFARM, &h->size, nme);
- } else {
- int fd;
-
- fd = GDKfdlocate(NOFARM, nme, "wb", NULL);
- if (fd >= 0) {
- char of[sizeof(h->filename)];
- char *ext;
- close(fd);
- strncpy(of, h->filename, sizeof(of));
-#ifdef STATIC_CODE_ANALYSIS
- /* help coverity */
- of[sizeof(h->filename) - 1] = 0;
-#endif
- ext = decompose_filename(of);
- h->newstorage = STORE_MMAP;
- if (HEAPload(h, of, ext, false) != GDK_SUCCEED)
- h->base = NULL; /* superfluous */
- /* success checked by looking at
- * h->base below */
- }
- }
+ h->storage = STORE_MMAP;
+ h->base = HEAPcreatefile(NOFARM, &h->size, nme);
GDKfree(nme);
}
if (h->base == NULL) {
@@ -270,6 +246,7 @@ HEAPextend(Heap *h, size_t size, bool ma
HEAPfree(&bak, false);
return GDK_SUCCEED;
}
+ GDKclrerr();
}
fd = GDKfdlocate(h->farmid, nme, "wb", ext);
if (fd >= 0) {
@@ -445,7 +422,7 @@ GDKupgradevarheap(BAT *b, var_t v, bool
const char *base = b->theap.base;
/* first save heap in file with extra .tmp extension */
- if ((fd = GDKfdlocate(b->theap.farmid, b->theap.filename, "wb",
"tmp")) < 0)
+ if ((fd = GDKfdlocate(b->theap.farmid, b->theap.filename,
"wxb", "tmp")) < 0)
return GDK_FAIL;
while (size > 0) {
ret = write(fd, base, (unsigned) MIN(1 << 30, size));
diff --git a/gdk/gdk_interprocess.c b/gdk/gdk_interprocess.c
--- a/gdk/gdk_interprocess.c
+++ b/gdk/gdk_interprocess.c
@@ -28,7 +28,7 @@
#include <sys/sem.h>
#include <time.h>
-static size_t interprocess_unique_id = 1;
+static ATOMIC_TYPE interprocess_unique_id = ATOMIC_VAR_INIT(1);
static key_t base_key = 800000000;
// Regular ftok produces too many collisions
@@ -46,12 +46,7 @@ ftok_enhanced(int id, key_t * return_key
size_t
GDKuniqueid(size_t offset)
{
- // TODO: lock this here instead of in pyapi
- size_t id;
-
- id = interprocess_unique_id;
- interprocess_unique_id += offset;
- return id;
+ return (size_t) ATOMIC_ADD(&interprocess_unique_id, (ATOMIC_BASE_TYPE)
offset);
}
//! Create a memory mapped file if it does not exist and open it
@@ -75,19 +70,21 @@ GDKinitmmap(size_t id, size_t size, size
size = (maxsize + GDK_mmap_pagesize - 1) & ~(GDK_mmap_pagesize - 1);
if (size == 0)
size = GDK_mmap_pagesize; */
- fd = GDKfdlocate(0, address, "wb", "tmp");
- if (fd < 0) {
- return NULL;
- }
path = GDKfilepath(0, BATDIR, address, "tmp");
if (path == NULL) {
return NULL;
}
- close(fd);
- if (GDKextend(path, size) != GDK_SUCCEED) {
+ fd = GDKfdlocate(NOFARM, path, "wb", NULL);
+ if (fd < 0) {
GDKfree(path);
return NULL;
}
+ if (GDKextendf(fd, size, path) != GDK_SUCCEED) {
+ close(fd);
+ GDKfree(path);
+ return NULL;
+ }
+ close(fd);
ptr = GDKmmap(path, mod, size);
GDKfree(path);
if (ptr == NULL) {
diff --git a/gdk/gdk_storage.c b/gdk/gdk_storage.c
--- a/gdk/gdk_storage.c
+++ b/gdk/gdk_storage.c
@@ -194,7 +194,7 @@ int
GDKfdlocate(int farmid, const char *nme, const char *mode, const char
*extension)
{
char *path = NULL;
- int fd, flags = 0;
+ int fd, flags = O_CLOEXEC;
assert(!GDKinmemory());
if (nme == NULL || *nme == 0)
@@ -212,12 +212,14 @@ GDKfdlocate(int farmid, const char *nme,
mode++;
#ifdef _CYGNUS_H_
} else {
- flags = _FRDSEQ; /* WIN32
CreateFile(FILE_FLAG_SEQUENTIAL_SCAN) */
+ flags |= _FRDSEQ; /* WIN32
CreateFile(FILE_FLAG_SEQUENTIAL_SCAN) */
#endif
}
if (strchr(mode, 'w')) {
flags |= O_WRONLY | O_CREAT;
+ if (strchr(mode, 'x'))
+ flags |= O_EXCL;
} else if (!strchr(mode, '+')) {
flags |= O_RDONLY;
} else {
@@ -226,11 +228,11 @@ GDKfdlocate(int farmid, const char *nme,
#ifdef WIN32
flags |= strchr(mode, 'b') ? O_BINARY : O_TEXT;
#endif
- fd = open(nme, flags | O_CLOEXEC, MONETDB_MODE);
+ fd = open(nme, flags, MONETDB_MODE);
if (fd < 0 && *mode == 'w') {
/* try to create the directory, in case that was the problem */
if (GDKcreatedir(nme) == GDK_SUCCEED) {
- fd = open(nme, flags | O_CLOEXEC, MONETDB_MODE);
+ fd = open(nme, flags, MONETDB_MODE);
if (fd < 0)
GDKsyserror("GDKfdlocate: cannot open file
%s\n", nme);
}
diff --git a/sql/backends/monet5/UDF/pyapi/pyapi.c
b/sql/backends/monet5/UDF/pyapi/pyapi.c
--- a/sql/backends/monet5/UDF/pyapi/pyapi.c
+++ b/sql/backends/monet5/UDF/pyapi/pyapi.c
@@ -329,9 +329,7 @@ static str PyAPIeval(Client cntxt, MalBl
int mmap_count = 4 + pci->retc * 2;
// create initial shared memory
- MT_lock_set(&pyapiLock);
mmap_id = GDKuniqueid(mmap_count);
- MT_lock_unset(&pyapiLock);
mmap_ptrs = GDKzalloc(mmap_count * sizeof(void *));
mmap_sizes = GDKzalloc(mmap_count * sizeof(size_t));
diff --git a/sql/server/rel_semantic.c b/sql/server/rel_semantic.c
--- a/sql/server/rel_semantic.c
+++ b/sql/server/rel_semantic.c
@@ -88,6 +88,8 @@ rel_parse(mvc *m, sql_schema *s, char *q
bstream_destroy(m->scanner.rs);
m->sym = NULL;
+ o.vars = m->vars; /* may have been realloc'ed */
+ o.sizevars = m->sizevars;
if (m->session->status || m->errstr[0]) {
int status = m->session->status;
char errstr[ERRSIZE];
diff --git a/sql/storage/store.c b/sql/storage/store.c
--- a/sql/storage/store.c
+++ b/sql/storage/store.c
@@ -2122,13 +2122,16 @@ flusher_should_run(void)
{
// We will flush if we have a reason to and no reason not to.
char *reason_to = NULL, *reason_not_to = NULL;
+ int changes;
if (flusher.countdown_ms <= 0)
reason_to = "timer expired";
int many_changes = GDKdebug & FORCEMITOMASK ? 100 : 1000000;
- if (logger_funcs.changes() >= many_changes)
+ if ((changes = logger_funcs.changes()) >= many_changes)
reason_to = "many changes";
+ else if (changes == 0)
+ reason_not_to = "no changes";
// Read and clear flush_now. If we decide not to flush
// we'll put it back.
diff --git a/sql/test/subquery/Tests/subquery2.sql
b/sql/test/subquery/Tests/subquery2.sql
--- a/sql/test/subquery/Tests/subquery2.sql
+++ b/sql/test/subquery/Tests/subquery2.sql
@@ -34,6 +34,93 @@ SELECT col1 IN (SELECT ColID + col1 FROM
-- False
-- False
+SELECT col1 IN (SELECT SUM(ColID + col1) FROM tbl_ProductSales) FROM another_T
GROUP BY col1;
+ -- False
+ -- False
+ -- False
+ -- False
+
+SELECT
+ EXISTS (SELECT col2 FROM tbl_ProductSales WHERE tbl_ProductSales.ColID
= another_T.col1),
+ (SELECT ColID FROM tbl_ProductSales) * DENSE_RANK() OVER (PARTITION BY
AVG(DISTINCT col5))
+FROM another_T GROUP BY col1, col2, col5, col8;
+ -- True True
+ -- False True
+ -- False True
+ -- False True
+
+SELECT
+ -col1 IN (SELECT ColID FROM tbl_ProductSales),
+ col5 = ALL (SELECT 1 FROM tbl_ProductSales HAVING MIN(col8) IS NULL)
+FROM another_T GROUP BY col1, col2, col5, col8;
+ -- False True
+ -- False True
+ -- False True
+ -- False True
+
+SELECT
+ col1 + col5 = (SELECT MIN(ColID) FROM tbl_ProductSales),
+ CAST(SUM(DISTINCT CASE WHEN col5 - col8 = (SELECT MIN(ColID / col2)
FROM tbl_ProductSales) THEN col2 - 5 ELSE ABS(col1) END) AS BIGINT),
+ (SELECT MAX(ColID + col2) FROM tbl_ProductSales) * DENSE_RANK() OVER
(PARTITION BY AVG(DISTINCT col5))
+FROM another_T
+GROUP BY col1, col2, col5, col8;
+ -- False 1 6
+ -- False 11 26
+ -- False 111 226
+ -- False 1111 2226
+
+SELECT
+ col1 + col5 = (SELECT MIN(ColID) FROM tbl_ProductSales),
+ MIN(col8) OVER (PARTITION BY col5 ORDER BY col1 ROWS UNBOUNDED
PRECEDING)
+FROM another_T
+GROUP BY col1, col2, col5, col8;
+ -- False 8
+ -- False 88
+ -- False 888
+ -- False 8888
+
+SELECT
+ EXISTS (SELECT 1 FROM tbl_ProductSales),
+ NOT EXISTS (SELECT 1 FROM tbl_ProductSales)
+FROM another_T
+GROUP BY col1;
+ -- True False
+ -- True False
+ -- True False
+ -- True False
+
+SELECT
+ NOT -SUM(col2) NOT IN (SELECT ColID FROM tbl_ProductSales GROUP BY
ColID HAVING SUM(ColID - col8) <> col5),
+ NOT col5 = ALL (SELECT 1 FROM tbl_ProductSales HAVING MAX(col8) > 2 AND
MIN(col8) IS NOT NULL),
+ NOT col2 <> ANY (SELECT 20 FROM tbl_ProductSales GROUP BY ColID HAVING
NOT MAX(col1) <> col1 * AVG(col1 + ColID) * ColID),
+ NOT EXISTS (SELECT ColID - 12 FROM tbl_ProductSales GROUP BY ColID
HAVING MAX(col2) IS NULL OR NOT col8 <> 2 / col1)
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
