Changeset: 2a416951c2c3 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/2a416951c2c3
Modified Files:
gdk/gdk.h
gdk/gdk_bat.c
gdk/gdk_bbp.c
gdk/gdk_heap.c
gdk/gdk_select.c
gdk/gdk_string.c
sql/backends/monet5/sql.c
sql/storage/bat/bat_logger.c
sql/storage/bat/bat_storage.c
Branch: strheapvacuum
Log Message:
merge default
diffs (truncated from 2770 to 300 lines):
diff --git a/clients/examples/C/bincopydata.c b/clients/examples/C/bincopydata.c
--- a/clients/examples/C/bincopydata.c
+++ b/clients/examples/C/bincopydata.c
@@ -183,7 +183,7 @@ gen_newline_strings(FILE *f, bool bytesw
{
(void)byteswap;
for (long i = 0; i < nrecs; i++) {
- fprintf(f, "rn\r\nr\r%ld", i);
+ fprintf(f, "RN\r\nR\r%ld", i);
fputc(0, f);
}
}
diff --git a/common/utils/mcrypt.c b/common/utils/mcrypt.c
--- a/common/utils/mcrypt.c
+++ b/common/utils/mcrypt.c
@@ -514,7 +514,6 @@ mcrypt_hashPassword(
return NULL;
}
-#if (defined(HAVE_OPENSSL) || defined(HAVE_COMMONCRYPTO))
snprintf(ret, sizeof(ret),
"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
@@ -542,7 +541,6 @@ mcrypt_hashPassword(
ret[len] = '\0';
return strdup(ret);
-#endif
}
#ifndef HAVE_SHA512_UPDATE
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -559,7 +559,8 @@ typedef struct {
bte farmid; /* id of farm where heap is located */
bool cleanhash:1, /* string heaps must clean hash */
dirty:1, /* specific heap dirty marker */
- remove:1; /* remove storage file when freeing */
+ remove:1, /* remove storage file when freeing */
+ wasempty:1; /* heap was empty when last saved/created */
storage_t storage; /* storage mode (mmap/malloc). */
storage_t newstorage; /* new desired storage mode at re-allocation. */
bat parentid; /* cache id of VIEW parent bat */
@@ -1711,7 +1712,7 @@ Tputvalue(BAT *b, BUN p, const void *v,
if (rc != GDK_SUCCEED)
return rc;
if (b->twidth < SIZEOF_VAR_T &&
- (b->twidth <= 2 ? d - GDK_VAROFFSET : d) >= ((size_t) 1 <<
(8 * b->twidth))) {
+ (b->twidth <= 2 ? d - GDK_VAROFFSET : d) >= ((size_t) 1 <<
(8 << b->tshift))) {
/* doesn't fit in current heap, upgrade it */
rc = GDKupgradevarheap(b, d, 0, copyall);
if (rc != GDK_SUCCEED)
@@ -1733,9 +1734,9 @@ Tputvalue(BAT *b, BUN p, const void *v,
break;
#endif
}
- } else if (b->ttype == TYPE_msk) {
- mskSetVal(b, p, * (msk *) v);
} else {
+ /* msk is handled by tfastins_nocheck, our only caller */
+ assert(b->ttype != TYPE_msk);
return ATOMputFIX(b->ttype, Tloc(b, p), v);
}
return GDK_SUCCEED;
@@ -1750,8 +1751,10 @@ tfastins_nocheck(BAT *b, BUN p, const vo
((uint32_t *) b->theap->base)[b->theap->free / 4] = 0;
b->theap->free += 4;
}
- } else
- b->theap->free += s;
+ mskSetVal(b, p, * (msk *) v);
+ return GDK_SUCCEED;
+ }
+ b->theap->free += s;
return Tputvalue(b, p, v, false);
}
@@ -1803,7 +1806,7 @@ tfastins_nocheckVAR(BAT *b, BUN p, const
if ((rc = ATOMputVAR(b, &d, v)) != GDK_SUCCEED)
return rc;
if (b->twidth < SIZEOF_VAR_T &&
- (b->twidth <= 2 ? d - GDK_VAROFFSET : d) >= ((size_t) 1 << (8 *
b->twidth))) {
+ (b->twidth <= 2 ? d - GDK_VAROFFSET : d) >= ((size_t) 1 << (8 <<
b->tshift))) {
/* doesn't fit in current heap, upgrade it */
rc = GDKupgradevarheap(b, d, 0, false);
if (rc != GDK_SUCCEED)
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -526,7 +526,6 @@ gdk_return
BATextend(BAT *b, BUN newcap)
{
size_t theap_size;
- gdk_return rc = GDK_SUCCEED;
assert(newcap <= BUN_MAX);
BATcheck(b, GDK_FAIL);
@@ -547,28 +546,16 @@ BATextend(BAT *b, BUN newcap)
newcap = (newcap + 31) & ~(BUN)31; /* round up to multiple of
32 */
theap_size = (size_t) (newcap / 8); /* in bytes */
} else {
- theap_size = (size_t) newcap * Tsize(b);
+ theap_size = (size_t) newcap << b->tshift;
}
b->batCapacity = newcap;
if (b->theap->base) {
TRC_DEBUG(HEAP, "HEAPgrow in BATextend %s %zu %zu\n",
b->theap->filename, b->theap->size, theap_size);
- MT_lock_set(&b->theaplock);
- if (ATOMIC_GET(&b->theap->refs) == 1) {
- rc = HEAPextend(b->theap, theap_size, b->batRestricted
== BAT_READ);
- } else {
- MT_lock_unset(&b->theaplock);
- Heap *h = HEAPgrow(b->theap, theap_size);
- if (h == NULL)
- return GDK_FAIL;
- MT_lock_set(&b->theaplock);
- HEAPdecref(b->theap, false);
- b->theap = h;
- }
- MT_lock_unset(&b->theaplock);
+ return HEAPgrow(&b->theaplock, &b->theap, theap_size,
b->batRestricted == BAT_READ);
}
- return rc;
+ return GDK_SUCCEED;
}
@@ -938,7 +925,7 @@ COLcopy(BAT *b, int tt, bool writable, r
memcpy(Tloc(bn, 0), bi.base, bn->theap->free);
} else {
/* case (4): optimized for simple array copy */
- bn->theap->free = bunstocopy * Tsize(bn);
+ bn->theap->free = bunstocopy << bn->tshift;
bn->theap->dirty |= bunstocopy > 0;
memcpy(Tloc(bn, 0), bi.base, bn->theap->free);
}
@@ -1014,148 +1001,6 @@ COLcopy(BAT *b, int tt, bool writable, r
return NULL;
}
-static void
-setcolprops(BAT *b, const void *x)
-{
- bool isnil = b->ttype != TYPE_void &&
- ATOMnilptr(b->ttype) != NULL &&
- ATOMcmp(b->ttype, x, ATOMnilptr(b->ttype)) == 0;
- BATiter bi;
- BUN pos;
- const void *prv;
- int cmp;
-
- /* x may only be NULL if the column type is VOID */
- assert(x != NULL || b->ttype == TYPE_void);
- if (b->batCount == 0) {
- /* first value */
- b->tsorted = b->trevsorted = ATOMlinear(b->ttype);
- b->tnosorted = b->tnorevsorted = 0;
- b->tkey = true;
- b->tnokey[0] = b->tnokey[1] = 0;
- if (b->ttype == TYPE_void) {
- if (x) {
- b->tseqbase = * (const oid *) x;
- }
- b->tnil = is_oid_nil(b->tseqbase);
- b->tnonil = !b->tnil;
- } else {
- b->tnil = isnil;
- b->tnonil = !isnil;
- if (b->ttype == TYPE_oid) {
- b->tseqbase = * (const oid *) x;
- }
- if (!isnil && ATOMlinear(b->ttype)) {
- BATsetprop(b, GDK_MAX_VALUE, b->ttype, x);
- BATsetprop(b, GDK_MIN_VALUE, b->ttype, x);
- BATsetprop(b, GDK_MAX_POS, TYPE_oid, &(oid){0});
- BATsetprop(b, GDK_MIN_POS, TYPE_oid, &(oid){0});
- }
- }
- return;
- } else if (b->ttype == TYPE_void) {
- /* not the first value in a VOID column: we keep the
- * seqbase, and x is not used, so only some properties
- * are affected */
- if (!is_oid_nil(b->tseqbase)) {
- if (b->trevsorted) {
- b->tnorevsorted = BUNlast(b);
- b->trevsorted = false;
- }
- b->tnil = false;
- b->tnonil = true;
- } else {
- if (b->tkey) {
- b->tnokey[0] = 0;
- b->tnokey[1] = BUNlast(b);
- b->tkey = false;
- }
- b->tnil = true;
- b->tnonil = false;
- }
- return;
- } else if (ATOMlinear(b->ttype)) {
- const ValRecord *prop;
-
- bi = bat_iterator_nolock(b);
- pos = BUNlast(b);
- prv = BUNtail(bi, pos - 1);
- cmp = ATOMcmp(b->ttype, prv, x);
-
- if (b->tkey &&
- (cmp == 0 || /* definitely not KEY */
- (b->batCount > 1 && /* can't guarantee KEY if unordered */
- ((b->tsorted && cmp > 0) ||
- (b->trevsorted && cmp < 0) ||
- (!b->tsorted && !b->trevsorted))))) {
- b->tkey = false;
- if (cmp == 0) {
- b->tnokey[0] = pos - 1;
- b->tnokey[1] = pos;
- }
- }
- if (b->tsorted) {
- if (cmp > 0) {
- /* out of order */
- b->tsorted = false;
- b->tnosorted = pos;
- } else if (cmp < 0 && !isnil) {
- /* new largest value */
- BATsetprop(b, GDK_MAX_VALUE, b->ttype, x);
- BATsetprop(b, GDK_MAX_POS, TYPE_oid,
&(oid){BATcount(b)});
- }
- } else if (!isnil &&
- (prop = BATgetprop(b, GDK_MAX_VALUE)) != NULL &&
- ATOMcmp(b->ttype, VALptr(prop), x) < 0) {
- BATsetprop(b, GDK_MAX_VALUE, b->ttype, x);
- BATsetprop(b, GDK_MAX_POS, TYPE_oid,
&(oid){BATcount(b)});
- }
- if (b->trevsorted) {
- if (cmp < 0) {
- /* out of order */
- b->trevsorted = false;
- b->tnorevsorted = pos;
- /* if there is a nil in the BAT, it is
- * the smallest, but that doesn't
- * count for the property, so the new
- * value may still be smaller than the
- * smallest non-nil so far */
- if (!b->tnonil && !isnil &&
- (prop = BATgetprop(b, GDK_MIN_VALUE)) !=
NULL &&
- ATOMcmp(b->ttype, VALptr(prop), x) > 0) {
- BATsetprop(b, GDK_MIN_VALUE, b->ttype,
x);
- BATsetprop(b, GDK_MIN_POS, TYPE_oid,
&(oid){BATcount(b)});
- }
- } else if (cmp > 0 && !isnil) {
- /* new smallest value */
- BATsetprop(b, GDK_MIN_VALUE, b->ttype, x);
- BATsetprop(b, GDK_MIN_POS, TYPE_oid,
&(oid){BATcount(b)});
- }
- } else if (!isnil &&
- (prop = BATgetprop(b, GDK_MIN_VALUE)) != NULL &&
- ATOMcmp(b->ttype, VALptr(prop), x) > 0) {
- BATsetprop(b, GDK_MIN_VALUE, b->ttype, x);
- BATsetprop(b, GDK_MIN_POS, TYPE_oid,
&(oid){BATcount(b)});
- }
- if (BATtdense(b) && (cmp >= 0 || * (const oid *) prv + 1 != *
(const oid *) x)) {
- assert(b->ttype == TYPE_oid);
- b->tseqbase = oid_nil;
- }
- } else if (BATcount(b) == 1) {
- /* we'll only check keyness with a single other value */
- bi = bat_iterator_nolock(b);
- prv = BUNtail(bi, 0);
- b->tkey = ATOMcmp(b->ttype, prv, x) != 0;
- } else {
- /* no guarantees that we don't have duplicates */
- b->tkey = false;
- }
- if (isnil) {
- b->tnonil = false;
- b->tnil = true;
- }
-}
-
/* Append an array of values of length count to the bat. For
* fixed-sized values, `values' is an array of values, for
* variable-sized values, `values' is an array of pointers to values.
@@ -1220,24 +1065,128 @@ BUNappendmulti(BAT *b, const void *value
}
BATrmprop(b, GDK_UNIQUE_ESTIMATE);
- b->theap->dirty |= count > 0;
- MT_rwlock_wrlock(&b->thashlock);
+ b->theap->dirty = true;
const void *t = b->ttype == TYPE_msk ? &(msk){false} :
ATOMnilptr(b->ttype);
- for (BUN i = 0; i < count; i++) {
- if (values) {
- t = b->ttype && b->tvarsized ? ((void **) values)[i] :
- (void *) ((char *) values + i * Tsize(b));
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
