Changeset: 95b48fecbe2c for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/95b48fecbe2c
Modified Files:
MonetDB.spec
Branch: default
Log Message:
Merge with Jun2023 branch.
diffs (truncated from 762 to 300 lines):
diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -366,6 +366,7 @@ developer.
%{_bindir}/ODBCStmtAttr
%{_bindir}/ODBCgetInfo
%{_bindir}/ODBCmetadata
+%{_bindir}/ODBCtester
%{_bindir}/arraytest
%{_bindir}/bincopydata
%{_bindir}/odbcsample1
diff --git a/clients/odbc/driver/SQLGetInfo.c b/clients/odbc/driver/SQLGetInfo.c
--- a/clients/odbc/driver/SQLGetInfo.c
+++ b/clients/odbc/driver/SQLGetInfo.c
@@ -1038,7 +1038,8 @@ MNDBGetInfo(ODBCDbc *dbc,
case SQL_TIMEDATE_ADD_INTERVALS:
case SQL_TIMEDATE_DIFF_INTERVALS:
/* when server is 11.46 or higher */
- nValue = SQL_FN_TSI_SECOND |
+ nValue = SQL_FN_TSI_FRAC_SECOND |
+ SQL_FN_TSI_SECOND |
SQL_FN_TSI_MINUTE |
SQL_FN_TSI_HOUR |
SQL_FN_TSI_DAY |
@@ -1046,7 +1047,6 @@ MNDBGetInfo(ODBCDbc *dbc,
SQL_FN_TSI_MONTH |
SQL_FN_TSI_QUARTER |
SQL_FN_TSI_YEAR;
- /* SQL_FN_TSI_FRAC_SECOND | */
break;
case SQL_TIMEDATE_FUNCTIONS:
nValue = SQL_FN_TD_CURRENT_DATE |
diff --git a/clients/odbc/tests/ODBCgetInfo.c b/clients/odbc/tests/ODBCgetInfo.c
--- a/clients/odbc/tests/ODBCgetInfo.c
+++ b/clients/odbc/tests/ODBCgetInfo.c
@@ -1080,13 +1080,13 @@ const struct {
.info = SQL_TIMEDATE_ADD_INTERVALS,
.name = "SQL_TIMEDATE_ADD_INTERVALS",
.type = INTEGER,
- .i = SQL_FN_TSI_SECOND | SQL_FN_TSI_MINUTE | SQL_FN_TSI_HOUR |
SQL_FN_TSI_DAY | SQL_FN_TSI_WEEK | SQL_FN_TSI_MONTH | SQL_FN_TSI_QUARTER |
SQL_FN_TSI_YEAR,
+ .i = SQL_FN_TSI_FRAC_SECOND | SQL_FN_TSI_SECOND |
SQL_FN_TSI_MINUTE | SQL_FN_TSI_HOUR | SQL_FN_TSI_DAY | SQL_FN_TSI_WEEK |
SQL_FN_TSI_MONTH | SQL_FN_TSI_QUARTER | SQL_FN_TSI_YEAR,
},
{
.info = SQL_TIMEDATE_DIFF_INTERVALS,
.name = "SQL_TIMEDATE_DIFF_INTERVALS",
.type = INTEGER,
- .i = SQL_FN_TSI_SECOND | SQL_FN_TSI_MINUTE | SQL_FN_TSI_HOUR |
SQL_FN_TSI_DAY | SQL_FN_TSI_WEEK | SQL_FN_TSI_MONTH | SQL_FN_TSI_QUARTER |
SQL_FN_TSI_YEAR,
+ .i = SQL_FN_TSI_FRAC_SECOND | SQL_FN_TSI_SECOND |
SQL_FN_TSI_MINUTE | SQL_FN_TSI_HOUR | SQL_FN_TSI_DAY | SQL_FN_TSI_WEEK |
SQL_FN_TSI_MONTH | SQL_FN_TSI_QUARTER | SQL_FN_TSI_YEAR,
},
{
.info = SQL_TIMEDATE_FUNCTIONS,
diff --git a/debian/monetdb-client-testing.install
b/debian/monetdb-client-testing.install
--- a/debian/monetdb-client-testing.install
+++ b/debian/monetdb-client-testing.install
@@ -1,6 +1,7 @@
debian/tmp/usr/bin/ODBCStmtAttr usr/bin
debian/tmp/usr/bin/ODBCgetInfo usr/bin
debian/tmp/usr/bin/ODBCmetadata usr/bin
+debian/tmp/usr/bin/ODBCtester usr/bin
debian/tmp/usr/bin/arraytest usr/bin
debian/tmp/usr/bin/bincopydata usr/bin
debian/tmp/usr/bin/odbcsample1 usr/bin
diff --git a/gdk/ChangeLog.Sep2022 b/gdk/ChangeLog.Sep2022
--- a/gdk/ChangeLog.Sep2022
+++ b/gdk/ChangeLog.Sep2022
@@ -1,6 +1,13 @@
# ChangeLog file for GDK
# This file is updated with Maddlog
+* Thu Apr 20 2023 Sjoerd Mullender <sjo...@acm.org>
+- Fixed yet another occurrence of a missing .tailN file. This one could
+ happen if a string bat was appended to in stages so that between appends
+ the column was committed. If an append caused both a realloc of the
+ tail heap because it was getting longer and a realloc because it got
+ wider, the file might get removed before the GDK level commit happened.
+
* Fri Mar 24 2023 Sjoerd Mullender <sjo...@acm.org>
- When processing the WAL, if a to-be-destroyed object cannot be found,
don't stop, but keep processing the rest of the WAL.
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -2891,6 +2891,7 @@ PROPdestroy_nolock(BAT *b)
b->tprops = NULL;
while (p) {
n = p->next;
+ assert(p->id != (enum prop_t) 20);
VALclear(&p->v);
GDKfree(p);
p = n;
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -1913,6 +1913,12 @@ BBPexit(void)
HEAPdecref(b->tvheap, false);
b->tvheap = NULL;
}
+ if (b->oldtail) {
+ Heap *h = b->oldtail;
+ b->oldtail = NULL;
+ ATOMIC_AND(&h->refs,
~DELAYEDREMOVE);
+ HEAPdecref(h, false);
+ }
PROPdestroy_nolock(b);
MT_lock_unset(&b->theaplock);
BATfree(b);
@@ -3600,34 +3606,21 @@ BBPbackup(BAT *b, bool subcommit)
}
static inline void
-BBPcheckHeap(bool subcommit, Heap *h)
+BBPcheckHeap(Heap *h)
{
struct stat statb;
char *path;
- if (subcommit) {
- char *s = strrchr(h->filename, DIR_SEP);
- if (s)
- s++;
- else
- s = h->filename;
- path = GDKfilepath(0, BAKDIR, s, NULL);
- if (path == NULL)
- return;
- if (MT_stat(path, &statb) < 0) {
- GDKfree(path);
- path = GDKfilepath(0, BATDIR, h->filename, NULL);
- if (path == NULL)
- return;
- if (MT_stat(path, &statb) < 0) {
- GDKsyserror("cannot stat file %s (expected size
%zu)\n",
- path, h->free);
- assert(0);
- GDKfree(path);
- return;
- }
- }
- } else {
+ char *s = strrchr(h->filename, DIR_SEP);
+ if (s)
+ s++;
+ else
+ s = h->filename;
+ path = GDKfilepath(0, BAKDIR, s, NULL);
+ if (path == NULL)
+ return;
+ if (MT_stat(path, &statb) < 0) {
+ GDKfree(path);
path = GDKfilepath(0, BATDIR, h->filename, NULL);
if (path == NULL)
return;
@@ -3650,7 +3643,7 @@ BBPcheckHeap(bool subcommit, Heap *h)
}
static void
-BBPcheckBBPdir(bool subcommit)
+BBPcheckBBPdir(void)
{
FILE *fp;
int lineno = 0;
@@ -3658,10 +3651,14 @@ BBPcheckBBPdir(bool subcommit)
unsigned bbpversion;
lng logno, transid;
- fp = GDKfileopen(0, BATDIR, "BBP", "dir", "r");
+ fp = GDKfileopen(0, BAKDIR, "BBP", "dir", "r");
assert(fp != NULL);
- if (fp == NULL)
- return;
+ if (fp == NULL) {
+ fp = GDKfileopen(0, BATDIR, "BBP", "dir", "r");
+ assert(fp != NULL);
+ if (fp == NULL)
+ return;
+ }
bbpversion = BBPheader(fp, &lineno, &bbpsize, &logno, &transid);
if (bbpversion == 0) {
fclose(fp);
@@ -3716,9 +3713,9 @@ BBPcheckBBPdir(bool subcommit)
continue;
}
if (b.theap->free > 0)
- BBPcheckHeap(subcommit, b.theap);
+ BBPcheckHeap(b.theap);
if (b.tvheap != NULL && b.tvheap->free > 0)
- BBPcheckHeap(subcommit, b.tvheap);
+ BBPcheckHeap(b.tvheap);
}
}
@@ -3753,6 +3750,9 @@ BBPsync(int cnt, bat *restrict subcommit
TRC_DEBUG_IF(PERF) t0 = t1 = GDKusec();
+ if ((GDKdebug & TAILCHKMASK) && !GDKinmemory(0))
+ BBPcheckBBPdir();
+
ret = BBPprepare(subcommit != NULL);
/* PHASE 1: safeguard everything in a backup-dir */
@@ -3873,9 +3873,6 @@ BBPsync(int cnt, bat *restrict subcommit
* succeeded, so no changing of ret after this
* call anymore */
- if ((GDKdebug & TAILCHKMASK) && !GDKinmemory(0))
- BBPcheckBBPdir(subcommit != NULL);
-
if (MT_rename(bakdir, deldir) < 0 &&
/* maybe there was an old deldir, so remove and try again */
(GDKremovedir(0, DELDIR) != GDK_SUCCEED ||
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -601,9 +601,14 @@ GDKupgradevarheap(BAT *b, var_t v, BUN c
b->theap = new;
if (BBP_status(bid) & (BBPEXISTING|BBPDELETED) && b->oldtail == NULL) {
b->oldtail = old;
- ATOMIC_OR(&old->refs, DELAYEDREMOVE);
+ if ((ATOMIC_OR(&old->refs, DELAYEDREMOVE) & HEAPREFS) == 1) {
+ /* we have the only reference, we can free the
+ * memory */
+ HEAPfree(old, false);
+ }
} else {
- HEAPdecref(old, true);
+ ValPtr p = BATgetprop_nolock(b, (enum prop_t) 20);
+ HEAPdecref(old, p == NULL || strcmp(((Heap*)
p->val.pval)->filename, old->filename) != 0);
}
MT_lock_unset(&b->theaplock);
return GDK_SUCCEED;
diff --git a/monetdb5/modules/atoms/str.c b/monetdb5/modules/atoms/str.c
--- a/monetdb5/modules/atoms/str.c
+++ b/monetdb5/modules/atoms/str.c
@@ -4997,22 +4997,21 @@ STRasciify(str *r, const str *s)
iconv_t cd;
const str f = "UTF8", t = "ASCII//TRANSLIT";
str in = *s, out;
- /* Output string length LEN+1 when converting from UTF-8 TO ASCII
- should be enough. If for some reason LEN is needed is totality,
- +1 safeguards the \0.*/
- size_t in_len = strlen(in), out_len = in_len + 1;
+ size_t in_len = strlen(in), out_len = in_len * 4; /* oversized as a
single utf8 char could change into multiple
+
ascii char */
/* man iconv; /TRANSLIT */
if ((cd = iconv_open(t, f)) == (iconv_t)(-1))
throw(MAL, "str.asciify", "ICONV: cannot convert from (%s) to
(%s).", f, t);
if ((*r = out = GDKmalloc(out_len)) == NULL)
throw(MAL, "str.asciify", SQLSTATE(HY013) MAL_MALLOC_FAIL);
- if (iconv(cd, &in, &in_len, &out, &out_len) == (size_t) - 1) {
+ str o = out;
+ if (iconv(cd, &in, &in_len, &o, &out_len) == (size_t) - 1) {
GDKfree(out);
*r = NULL;
iconv_close(cd);
throw(MAL, "str.asciify", "ICONV: string conversion failed from
(%s) to (%s)", f, t);
}
- *out = '\0';
+ *o = '\0';
iconv_close(cd);
return MAL_SUCCEED;
#else
diff --git a/monetdb5/modules/kernel/batstr.c b/monetdb5/modules/kernel/batstr.c
--- a/monetdb5/modules/kernel/batstr.c
+++ b/monetdb5/modules/kernel/batstr.c
@@ -5063,7 +5063,7 @@ BATSTRasciify(bat *ret, bat *bid)
#ifdef HAVE_ICONV
BAT *b = NULL, *bn = NULL;
BATiter bi;
- BUN p, q;
+ BUN p, q = 0;
bool nils = false;
size_t prev_out_len = 0, in_len = 0, out_len = 0;
str s = NULL, out = NULL, in = NULL, msg = MAL_SUCCEED;
@@ -5080,6 +5080,11 @@ BATSTRasciify(bat *ret, bat *bid)
throw(MAL, "batstr.asciify", GDK_EXCEPTION);
}
bi = bat_iterator(b);
+ if ((s = out = GDKmalloc(64*1024)) == NULL) {
+ msg = createException(MAL,"batstr.asciify", MAL_MALLOC_FAIL);
+ goto exit;
+ }
+ prev_out_len = 64*1024;
BATloop(b, p, q) {
in = (str) BUNtail(bi, p);
if (strNil(in)) {
@@ -5090,25 +5095,17 @@ BATSTRasciify(bat *ret, bat *bid)
nils = true;
continue;
}
- in_len = strlen(in), out_len = in_len + 1;
- if (out == NULL) {
- if ((out = GDKmalloc(out_len)) == NULL) {
- msg = createException(MAL,"batstr.asciify",
MAL_MALLOC_FAIL);
- goto exit;
- }
- prev_out_len = out_len;
- }
- else if (out_len > prev_out_len) {
+ in_len = strlen(in), out_len = in_len*4; /* over sized as
single utf8 symbols change into multiple ascii characters */
_______________________________________________
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
