Changeset: e59fe7ebf12f for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/e59fe7ebf12f
Modified Files:
gdk/gdk_align.c
gdk/gdk_batop.c
gdk/gdk_bbp.c
gdk/gdk_logger.c
gdk/gdk_storage.c
sql/storage/store.c
sql/test/BugTracker-2017/Tests/shutdown.Bug-6182.SQL.py
sql/test/BugTracker-2019/Tests/remote-table-non-existent-column.Bug-6750.py
sql/test/dict/Tests/dict03.py
sql/test/remote/Tests/creds.SQL.py
sql/test/remote/Tests/different_user.SQL.py
sql/test/remote/Tests/invalid_creds.SQL.py
sql/test/remote/Tests/ssbm.SQL.py
sql/test/rename/Tests/rename00.SQL.py
sql/test/strimps/Tests/persisted_strimp.py
sql/test/strimps/Tests/strimps_stable_counts.py
sql/test/wlcr/Tests/wlr01.py
sql/test/wlcr/Tests/wlr20.py
sql/test/wlcr/Tests/wlr30.py
sql/test/wlcr/Tests/wlr35.py
sql/test/wlcr/Tests/wlr40.py
sql/test/wlcr/Tests/wlr50.py
testing/Mtest.py.in
Branch: default
Log Message:
Merge with Jan2022 branch.
diffs (truncated from 320 to 300 lines):
diff --git a/gdk/gdk_align.c b/gdk/gdk_align.c
--- a/gdk/gdk_align.c
+++ b/gdk/gdk_align.c
@@ -147,6 +147,7 @@ VIEWcreate(oid seq, BAT *b)
}
if (bn->tvheap) {
BBPunshare(bn->tvheap->parentid);
+ BBPunfix(bn->tvheap->parentid);
HEAPdecref(bn->tvheap, false);
}
HEAPdecref(bn->theap, false);
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -85,8 +85,10 @@ insert_string_bat(BAT *b, BATiter *ni, s
/* we can share the vheaps, so we then only need to
* append the offsets */
MT_lock_set(&b->theaplock);
- if (b->tvheap->parentid != b->batCacheid)
+ if (b->tvheap->parentid != b->batCacheid) {
BBPunshare(b->tvheap->parentid);
+ BBPunfix(b->tvheap->parentid);
+ }
HEAPdecref(b->tvheap, b->tvheap->parentid == b->batCacheid);
HEAPincref(ni->vh);
b->tvheap = ni->vh;
@@ -144,6 +146,7 @@ insert_string_bat(BAT *b, BATiter *ni, s
}
memcpy(b->tvheap->base + toff, ni->vh->base,
ni->vhfree);
b->tvheap->free = toff + ni->vhfree;
+ b->tvheap->dirty = true;
MT_lock_unset(&b->theaplock);
}
}
@@ -350,8 +353,10 @@ append_varsized_bat(BAT *b, BATiter *ni,
* is read-only, we replace b's vheap with a reference
* to n's */
MT_lock_set(&b->theaplock);
- if (b->tvheap->parentid != b->batCacheid)
+ if (b->tvheap->parentid != b->batCacheid) {
BBPunshare(b->tvheap->parentid);
+ BBPunfix(b->tvheap->parentid);
+ }
BBPshare(ni->vh->parentid);
HEAPdecref(b->tvheap, true);
HEAPincref(ni->vh);
@@ -406,13 +411,15 @@ append_varsized_bat(BAT *b, BATiter *ni,
GDKfree(h);
return GDK_FAIL;
}
- BBPunshare(b->tvheap->parentid);
+ bat parid = b->tvheap->parentid;
+ BBPunshare(parid);
ATOMIC_INIT(&h->refs, 1);
MT_lock_set(&b->theaplock);
Heap *oh = b->tvheap;
b->tvheap = h;
MT_lock_unset(&b->theaplock);
HEAPdecref(oh, false);
+ BBPunfix(parid);
}
if (BATcount(b) == 0 && BATatoms[b->ttype].atomFix == NULL &&
ci->tpe == cand_dense && ci->ncand == ni->count) {
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -429,6 +429,8 @@ vheapinit(BAT *b, const char *buf, unsig
TRC_CRITICAL(GDK, "invalid format for BBP.dir on line %d",
lineno);
return -1;
}
+ if (b->batCount == 0)
+ free = 0;
if (b->ttype >= 0 &&
ATOMstorage(b->ttype) == TYPE_str &&
free < GDK_STRHASHTABLE * sizeof(stridx_t) + BATTINY * GDK_VARALIGN)
@@ -1933,7 +1935,7 @@ vheap_entry(FILE *fp, BATiter *bi, BUN s
(void) size;
if (bi->vh == NULL)
return 0;
- return fprintf(fp, " %zu", bi->vhfree);
+ return fprintf(fp, " %zu", size == 0 ? 0 : bi->vhfree);
}
static gdk_return
@@ -3944,7 +3946,11 @@ BBPsync(int cnt, bat *restrict subcommit
break;
}
bi = bat_iterator(BBP_desc(i));
- if (b) {
+ assert(sizes == NULL || size <= bi.count);
+ assert(sizes == NULL || bi.width == 0 ||
(bi.type == TYPE_msk ? ((size + 31) / 32) * 4 : size << bi.shift) <= bi.hfree);
+ if (size > bi.count) /* includes sizes==NULL */
+ size = bi.count;
+ if (b && size != 0) {
/* wait for BBPSAVING so that we
* can set it, wait for
* BBPUNLOADING before
@@ -3961,10 +3967,6 @@ BBPsync(int cnt, bat *restrict subcommit
BBP_status_on(i, BBPSAVING);
if (lock)
MT_lock_unset(&GDKswapLock(i));
- assert(sizes == NULL || size <=
bi.count);
- assert(sizes == NULL || bi.width == 0
|| (bi.type == TYPE_msk ? ((size + 31) / 32) * 4 : size << bi.shift) <=
bi.hfree);
- if (size > bi.count)
- size = bi.count;
MT_rwlock_rdlock(&b->thashlock);
ret = BATsave_locked(b, &bi, size);
MT_rwlock_rdunlock(&b->thashlock);
@@ -4398,7 +4400,8 @@ BBPdiskscan(const char *parent, size_t b
delete = true;
} else if (strncmp(p + 1, "tail", 4) == 0) {
BAT *b = getdesc(bid);
- delete = (b == NULL || !b->ttype ||
!b->batCopiedtodisk);
+ delete = (b == NULL || !b->ttype ||
!b->batCopiedtodisk || b->batCount == 0);
+ assert(b->batCount > 0 || b->theap->free == 0);
if (!delete) {
if (b->ttype == TYPE_str) {
switch (b->twidth) {
@@ -4423,7 +4426,7 @@ BBPdiskscan(const char *parent, size_t b
}
} else if (strncmp(p + 1, "theap", 5) == 0) {
BAT *b = getdesc(bid);
- delete = (b == NULL || !b->tvheap ||
!b->batCopiedtodisk);
+ delete = (b == NULL || !b->tvheap ||
!b->batCopiedtodisk || b->tvheap->free == 0);
} else if (strncmp(p + 1, "thashl", 6) == 0 ||
strncmp(p + 1, "thashb", 6) == 0) {
#ifdef PERSISTENTHASH
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -2255,7 +2255,7 @@ log_next_logfile(logger *lg, ulng ts)
{
if (!lg->pending || !lg->pending->next)
return 0;
- if (lg->pending->last_ts < ts)
+ if (lg->pending->last_ts <= ts)
return lg->pending->id;
return 0;
}
@@ -2373,6 +2373,8 @@ log_flush(logger *lg, ulng ts)
lng
log_changes(logger *lg)
{
+ if (LOG_DISABLED(lg))
+ return 0;
MT_lock_set(&lg->rotation_lock);
lng changes = lg->id - lg->saved_id - 1;
MT_lock_unset(&lg->rotation_lock);
diff --git a/gdk/gdk_storage.c b/gdk/gdk_storage.c
--- a/gdk/gdk_storage.c
+++ b/gdk/gdk_storage.c
@@ -800,7 +800,9 @@ BATload_intern(bat bid, bool lock)
/* LOAD bun heap */
if (b->ttype != TYPE_void) {
b->theap->storage = b->theap->newstorage = STORE_INVALID;
- if (HEAPload(b->theap, b->theap->filename, NULL,
b->batRestricted == BAT_READ) != GDK_SUCCEED) {
+ if ((b->batCount == 0 ?
+ HEAPalloc(b->theap, b->batCapacity, b->twidth,
ATOMsize(b->ttype)) :
+ HEAPload(b->theap, b->theap->filename, NULL,
b->batRestricted == BAT_READ)) != GDK_SUCCEED) {
HEAPfree(b->theap, false);
return NULL;
}
@@ -817,7 +819,9 @@ BATload_intern(bat bid, bool lock)
/* LOAD tail heap */
if (ATOMvarsized(b->ttype)) {
b->tvheap->storage = b->tvheap->newstorage = STORE_INVALID;
- if (HEAPload(b->tvheap, nme, "theap", b->batRestricted ==
BAT_READ) != GDK_SUCCEED) {
+ if ((b->tvheap->free == 0 ?
+ ATOMheap(b->ttype, b->tvheap, b->batCapacity) :
+ HEAPload(b->tvheap, nme, "theap", b->batRestricted ==
BAT_READ)) != GDK_SUCCEED) {
HEAPfree(b->theap, false);
HEAPfree(b->tvheap, false);
return NULL;
diff --git a/sql/storage/store.c b/sql/storage/store.c
--- a/sql/storage/store.c
+++ b/sql/storage/store.c
@@ -2208,7 +2208,7 @@ store_apply_deltas(sqlstore *store)
store_lock(store);
ulng oldest = store_oldest_pending(store);
store_unlock(store);
- TRC_DEBUG(SQL_STORE, "Store aplly deltas (" ULLFMT ")\n", oldest-1);
+ TRC_DEBUG(SQL_STORE, "Store apply deltas (" ULLFMT ")\n", oldest-1);
if (oldest)
res = store->logger_api.flush(store, oldest-1);
return res;
diff --git
a/sql/test/BugTracker-2019/Tests/remote-table-non-existent-column.Bug-6750.py
b/sql/test/BugTracker-2019/Tests/remote-table-non-existent-column.Bug-6750.py
---
a/sql/test/BugTracker-2019/Tests/remote-table-non-existent-column.Bug-6750.py
+++
b/sql/test/BugTracker-2019/Tests/remote-table-non-existent-column.Bug-6750.py
@@ -5,7 +5,6 @@ try:
except ImportError:
import process
-
with tempfile.TemporaryDirectory() as farm_dir:
os.makedirs(os.path.join(farm_dir, 'node1'))
with process.server(mapiport='0', dbname='node1',
dbfarm=os.path.join(farm_dir, 'node1'), stdin=process.PIPE,
stdout=process.PIPE, stderr=process.PIPE) as prc1:
diff --git a/sql/test/dict/Tests/dict03.py b/sql/test/dict/Tests/dict03.py
--- a/sql/test/dict/Tests/dict03.py
+++ b/sql/test/dict/Tests/dict03.py
@@ -9,7 +9,7 @@ from MonetDBtesting.sqltest import SQLTe
with tempfile.TemporaryDirectory() as farm_dir:
os.mkdir(os.path.join(farm_dir, 'db1'))
- with process.server(mapiport='0', dbname='db1',
dbfarm=os.path.join(farm_dir, 'db1'), stdin = process.PIPE, stdout =
process.PIPE, stderr = process.PIPE) as s:
+ with process.server(mapiport='0', dbname='db1',
dbfarm=os.path.join(farm_dir, 'db1'), stdin=process.PIPE, stdout=process.PIPE,
stderr=process.PIPE) as s:
with SQLTestCase() as mdb:
mdb.connect(database='db1', port=s.dbport, username="monetdb",
password="monetdb")
mdb.execute("""
@@ -33,14 +33,14 @@ with tempfile.TemporaryDirectory() as fa
mdb.execute("SELECT c0 FROM
t2;").assertSucceeded().assertDataResultMatch([(-1981639662,),])
s.communicate()
- with process.server(mapiport='0', dbname='db1',
dbfarm=os.path.join(farm_dir, 'db1'), stdin = process.PIPE, stdout =
process.PIPE, stderr = process.PIPE) as s:
+ with process.server(mapiport='0', dbname='db1',
dbfarm=os.path.join(farm_dir, 'db1'), stdin=process.PIPE, stdout=process.PIPE,
stderr=process.PIPE) as s:
with SQLTestCase() as mdb:
mdb.connect(database='db1', port=s.dbport, username="monetdb",
password="monetdb")
mdb.execute("SELECT c0 FROM t1 ORDER BY
c0;").assertSucceeded().assertDataResultMatch([('',),('3be汉字0',),('aa8877',)])
mdb.execute("SELECT c0 FROM
t2;").assertSucceeded().assertDataResultMatch([(-1981639662,),])
s.communicate()
- with process.server(mapiport='0', dbname='db1',
dbfarm=os.path.join(farm_dir, 'db1'), stdin = process.PIPE, stdout =
process.PIPE, stderr = process.PIPE) as s:
+ with process.server(mapiport='0', dbname='db1',
dbfarm=os.path.join(farm_dir, 'db1'), stdin=process.PIPE, stdout=process.PIPE,
stderr=process.PIPE) as s:
with SQLTestCase() as mdb:
mdb.connect(database='db1', port=s.dbport, username="monetdb",
password="monetdb")
mdb.execute("SELECT c0 FROM t1 ORDER BY
c0;").assertSucceeded().assertDataResultMatch([('',),('3be汉字0',),('aa8877',)])
diff --git a/sql/test/wlcr/Tests/wlr01.py b/sql/test/wlcr/Tests/wlr01.py
--- a/sql/test/wlcr/Tests/wlr01.py
+++ b/sql/test/wlcr/Tests/wlr01.py
@@ -17,7 +17,7 @@ dbnameclone = tstdb + 'clone'
with process.server(dbname=dbnameclone, mapiport='0', stdin=process.PIPE,
stdout=process.PIPE, stderr=process.PIPE) as slave, \
SQLTestCase() as tc:
- tc.connect(database=dbnameclone, port=tc.dbport)
+ tc.connect(database=dbnameclone, port=slave.dbport)
tc.execute("select 'hello';")\
.assertSucceeded()\
.assertRowCount(1)
diff --git a/sql/test/wlcr/Tests/wlr20.py b/sql/test/wlcr/Tests/wlr20.py
--- a/sql/test/wlcr/Tests/wlr20.py
+++ b/sql/test/wlcr/Tests/wlr20.py
@@ -18,7 +18,7 @@ dbnameclone = tstdb + 'clone'
with process.server(dbname=dbnameclone, mapiport='0', stdin=process.PIPE,
stdout=process.PIPE, stderr=process.PIPE) as slave, \
SQLTestCase() as tc:
- tc.connect(database=dbnameclone, port=tc.dbport)
+ tc.connect(database=dbnameclone, port=slave.dbport)
tc.execute("call wlr.master('%s');" % dbname).assertSucceeded()
tc.execute("call wlr.replicate(3);").assertSucceeded()
tc.execute("select * from tmp;")\
diff --git a/sql/test/wlcr/Tests/wlr30.py b/sql/test/wlcr/Tests/wlr30.py
--- a/sql/test/wlcr/Tests/wlr30.py
+++ b/sql/test/wlcr/Tests/wlr30.py
@@ -17,7 +17,7 @@ dbnameclone = tstdb + 'clone'
with process.server(dbname=dbnameclone, mapiport='0', stdin=process.PIPE,
stdout=process.PIPE, stderr=process.PIPE) as slave, \
SQLTestCase() as tc:
- tc.connect(database=dbnameclone, port=tc.dbport)
+ tc.connect(database=dbnameclone, port=slave.dbport)
tc.execute("select * from tmp;")\
.assertSucceeded()\
.assertDataResultMatch([(1, 'hello'), (2, 'world'), (3, 'blah'),
(4, 'bloh'), (5, 'red'), (6, 'fox')])
diff --git a/sql/test/wlcr/Tests/wlr35.py b/sql/test/wlcr/Tests/wlr35.py
--- a/sql/test/wlcr/Tests/wlr35.py
+++ b/sql/test/wlcr/Tests/wlr35.py
@@ -17,7 +17,7 @@ dbnameclone = tstdb + 'clone'
with process.server(dbname=dbnameclone, mapiport='0', stdin=process.PIPE,
stdout=process.PIPE, stderr=process.PIPE) as slave, \
SQLTestCase() as tc:
- tc.connect(database=dbnameclone, port=tc.dbport)
+ tc.connect(database=dbnameclone, port=slave.dbport)
tc.execute("call wlr.replicate(9);").assertSucceeded()
tc.execute("select * from tmp;")\
.assertSucceeded()\
diff --git a/sql/test/wlcr/Tests/wlr40.py b/sql/test/wlcr/Tests/wlr40.py
--- a/sql/test/wlcr/Tests/wlr40.py
+++ b/sql/test/wlcr/Tests/wlr40.py
@@ -17,7 +17,7 @@ dbnameclone = tstdb + 'clone'
with process.server(dbname=dbnameclone, mapiport='0', stdin=process.PIPE,
stdout=process.PIPE, stderr=process.PIPE) as slave, \
SQLTestCase() as tc:
- tc.connect(database=dbnameclone, port=tc.dbport)
+ tc.connect(database=dbnameclone, port=slave.dbport)
tc.execute("call wlr.master('%s');" % dbname).assertSucceeded()
tc.execute("call wlr.replicate(9);").assertSucceeded()
tc.execute("select * from tmp;")\
diff --git a/sql/test/wlcr/Tests/wlr50.py b/sql/test/wlcr/Tests/wlr50.py
--- a/sql/test/wlcr/Tests/wlr50.py
+++ b/sql/test/wlcr/Tests/wlr50.py
@@ -17,7 +17,7 @@ dbnameclone = tstdb + 'clone'
with process.server(dbname=dbnameclone, mapiport='0', stdin=process.PIPE,
stdout=process.PIPE, stderr=process.PIPE) as slave, \
SQLTestCase() as tc:
- tc.connect(database=dbnameclone, port=tc.dbport)
+ tc.connect(database=dbnameclone, port=slave.dbport)
tc.execute("call wlr.master('%s');" % dbname).assertSucceeded()
tc.execute("call wlr.replicate();").assertSucceeded()
tc.execute("select * from tmp;")\
diff --git a/testing/Mtest.py.in b/testing/Mtest.py.in
--- a/testing/Mtest.py.in
+++ b/testing/Mtest.py.in
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
