Changeset: 04690f561521 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/04690f561521
Modified Files:
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_bbp.c
gdk/gdk_heap.c
gdk/gdk_storage.c
sql/server/rel_statistics.c
Branch: persistent_sketches
Log Message:
small improvements, ie forcing sketches to disk
diffs (209 lines):
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -367,6 +367,7 @@ COLnew2(oid hseq, int tt, BUN cap, role_
goto bailout;
}
memset(bn->tsketch->base, 0, SKETCH_SIZE);
+ bn->tsketch->free = SKETCH_SIZE;
}
BBPcacheit(bn, true);
@@ -926,7 +927,7 @@ COLcopy2(BAT *b, int tt, bool writable,
bn->theap->dirty = true;
if (bn->tsketch && b->tsketch && bi.count) {
assert(b->tsketch->base);
- memcpy(bn->theap->base, b->tsketch->base, SKETCH_SIZE);
+ memcpy(bn->tsketch->base, b->tsketch->base,
SKETCH_SIZE);
bn->tsketch->dirty = true;
}
@@ -949,7 +950,7 @@ COLcopy2(BAT *b, int tt, bool writable,
if (bn->tsketch && bi.count) {
if (b->tsketch) {
assert(b->tsketch->base);
- memcpy(bn->theap->base, b->tsketch->base,
SKETCH_SIZE);
+ memcpy(bn->tsketch->base, b->tsketch->base,
SKETCH_SIZE);
bn->tsketch->dirty = true;
} else {
if (bat_sketch(&bi, NULL, (uint8_t
(*)[CLZ_BUCKETS]) bn->tsketch->base) != GDK_SUCCEED)
@@ -1292,7 +1293,7 @@ BUNappendmulti(BAT *b, const void *value
nunique = b->thash ? b->thash->nunique : 0;
}
if (b->tsketch) {
- printf("BUNappendmulti updating dense for bat
id %d\n", b->batCacheid);
+ //printf("BUNappendmulti updating dense for bat
id %d\n", b->batCacheid);
MT_lock_set(&b->theaplock);
if (sketch_update_many(&bi, (uint8_t
(*)[CLZ_BUCKETS]) b->tsketch->base,
values, count) !=
GDK_SUCCEED) {
@@ -1349,7 +1350,7 @@ BUNappendmulti(BAT *b, const void *value
}
nunique = b->thash ? b->thash->nunique : 0;
if (b->tsketch) {
- printf("BUNappendmulti updating dense for bat
id %d\n", b->batCacheid);
+ //printf("BUNappendmulti updating dense for bat
id %d\n", b->batCacheid);
MT_lock_set(&b->theaplock);
if(sketch_update_many(&bi, (uint8_t
(*)[CLZ_BUCKETS]) b->tsketch->base,
values, count) !=
GDK_SUCCEED) {
@@ -1518,12 +1519,14 @@ BUNappendmulti(BAT *b, const void *value
}
BATsetcount(b, p);
if (nunique != 0) {
+ /*
if (b->tsketch && b->tsketch->base)
printf("BUNappendmulti would be estimation %f for
unique bat id %d\n", sketch_estimate((uint8_t (*)[CLZ_BUCKETS])
b->tsketch->base), b->batCacheid);
+ */
b->tunique_est = (double) nunique;
} else if (b->tsketch) {
b->tunique_est = sketch_estimate((uint8_t (*)[CLZ_BUCKETS])
b->tsketch->base);
- printf("BUNappendmulti estimating %f for bat id %d\n",
b->tunique_est, b->batCacheid);
+ //printf("BUNappendmulti estimating %f for bat id %d\n",
b->tunique_est, b->batCacheid);
}
MT_lock_unset(&b->theaplock);
MT_rwlock_wrunlock(&b->thashlock);
@@ -1948,6 +1951,8 @@ BUNinplacemulti(BAT *b, const oid *posit
b->theap->dirty = true;
if (b->tvheap)
b->tvheap->dirty = true;
+ if (b->tsketch)
+ b->tsketch->dirty = true;
MT_lock_unset(&b->theaplock);
return GDK_SUCCEED;
@@ -2202,6 +2207,8 @@ BATsetcount(BAT *b, BUN cnt)
b->theap->dirty |= b->ttype != TYPE_void && cnt > 0;
b->theap->free = tailsize(b, cnt);
}
+ if (b->tsketch)
+ b->tsketch->dirty |= cnt > 0;
if (b->ttype == TYPE_void)
b->batCapacity = cnt;
if (cnt <= 1) {
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -1160,7 +1160,7 @@ BATappend2(BAT *b, BAT *n, BAT *s, bool
if (BATcount(b) == ci.ncand) { /* b was empty */
memcpy(b->tsketch->base, n->tsketch->base,
SKETCH_SIZE);
if (b->tunique_est == 0) {
- printf("BATappend2 (n has sketch)
estimating for bat id %d\n", b->batCacheid);
+ //printf("BATappend2 (n has sketch)
estimating for bat id %d\n", b->batCacheid);
b->tunique_est =
sketch_estimate((uint8_t (*)[CLZ_BUCKETS]) b->tsketch->base);
}
} else {
@@ -1176,7 +1176,7 @@ BATappend2(BAT *b, BAT *n, BAT *s, bool
if (BATcount(b) == ci.ncand) { /* b was empty */
memcpy(b->tsketch->base, tmp_sketch,
SKETCH_SIZE);
if (b->tunique_est == 0) {
- printf("BATappend2 (n does not have
sketch) estimating for bat id %d\n", b->batCacheid);
+ //printf("BATappend2 (n does not have
sketch) estimating for bat id %d\n", b->batCacheid);
b->tunique_est =
sketch_estimate((uint8_t (*)[CLZ_BUCKETS]) b->tsketch->base);
}
} else {
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -607,7 +607,7 @@ heapinit(BAT *b, const char *buf,
} else {
b->tvheap = NULL;
}
- if (ATOMsketchable(t) && b->theap->hasfile) {
+ if (ATOMsketchable(t) /*&& b->theap->hasfile*/) {
b->tsketch->free = SKETCH_SIZE;
b->tsketch->size = SKETCH_SIZE;
b->tsketch->base = NULL;
@@ -616,10 +616,9 @@ heapinit(BAT *b, const char *buf,
b->tsketch->farmid = BBPselectfarm(PERSISTENT, b->ttype,
sketchheap);
b->tsketch->role = PERSISTENT;
b->tsketch->dirty = false;
- b->tsketch->hasfile = true;
+ b->tsketch->hasfile = b->theap->hasfile;
b->tsketch->storage = STORE_INVALID;
b->tsketch->parentid = b->batCacheid;
-
} else {
b->tsketch = NULL;
}
@@ -4289,6 +4288,10 @@ BBPsync(int cnt, const bat *restrict sub
GDKmove(0, BAKDIR, fname, "theap", SUBDIR,
fname, "theap", false) == GDK_SUCCEED)
TRC_DEBUG(IO, "moved %s.theap from %s
to %s\n",
fname, BAKDIR, SUBDIR);
+ if (ATOMsketchable(b->ttype) /*&&
b->theap->hasfile */ &&
+ GDKmove(0, BAKDIR, fname, "tsketch",
SUBDIR, fname, "tsketch", false) == GDK_SUCCEED)
+ TRC_DEBUG(IO, "moved %s.tsketch from %s
to %s\n",
+ fname, BAKDIR, SUBDIR);
}
}
b = dirty_bat(&i, subcommit != NULL, lock);
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -578,6 +578,8 @@ GDKupgradevarheap(BAT *b, var_t v, BUN c
ValPtr p = BATgetprop_nolock(b, (enum prop_t) 20);
HEAPdecref(old, p == NULL || strcmp(((Heap*)
p->val.pval)->filename, old->filename) != 0);
}
+ if (b->tsketch)
+ b->tsketch->dirty = true;
return GDK_SUCCEED;
}
diff --git a/gdk/gdk_storage.c b/gdk/gdk_storage.c
--- a/gdk/gdk_storage.c
+++ b/gdk/gdk_storage.c
@@ -761,6 +761,13 @@ BATsave_iter(BAT *b, BATiter *bi, BUN si
}
} else {
const char *nme = BBP_physical(b->batCacheid);
+ if (bi->skh) {
+ if ((bi->hdirty || (bi->vh && bi->vhdirty)) &&
!bi->skhdirty) { /* TODO make sure we set tsketch->dirty correctly */
+ //assert(0);
+ bi->skhdirty = true;
+ }
+ assert(bi->skh->free == SKETCH_SIZE);
+ }
if ((!bi->copiedtodisk || bi->hdirty)
&& (err == GDK_SUCCEED && bi->type)) {
const char *tail = strchr(bi->h->filename, '.') + 1;
@@ -867,14 +874,25 @@ BATload_intern(bat bid, bool lock)
if (b->tsketch /* TODO */ && ATOMsketchable(b->ttype)) {
b->tsketch->storage = STORE_INVALID;
- if (HEAPload(b->tsketch, nme, "tsketch", false) !=
GDK_SUCCEED) {
- HEAPfree(b->theap, false);
- HEAPfree(b->tsketch, false);
- return NULL;
+ if (b->batCount > 0) {
+ if (HEAPload(b->tsketch, nme, "tsketch", false)
!= GDK_SUCCEED) {
+ assert(0);
+ HEAPfree(b->theap, false);
+ HEAPfree(b->tsketch, false);
+ return NULL;
+ }
} else {
+ if (HEAPalloc(b->tsketch, SKETCH_SIZE,
sizeof(uint8_t)) != GDK_SUCCEED) {
+ HEAPfree(b->theap, false);
+ HEAPfree(b->tsketch, false);
+ return NULL;
+ }
b->tunique_est = sketch_estimate((uint8_t
(*)[CLZ_BUCKETS]) b->tsketch->base);
- printf("BATload_intern estimating %f for bat id
%d\n", b->tunique_est, b->batCacheid);
+ //printf("BATload_intern estimating %f for bat
id %d\n", b->tunique_est, b->batCacheid);
}
+ b->tsketch->free = SKETCH_SIZE;
+ } else {
+ b->tsketch = NULL;
}
} else {
b->theap->base = NULL;
diff --git a/sql/server/rel_statistics.c b/sql/server/rel_statistics.c
--- a/sql/server/rel_statistics.c
+++ b/sql/server/rel_statistics.c
@@ -746,7 +746,7 @@ rel_calc_nuniques(mvc *sql, sql_rel *l,
if ((sub = atom_sub(sql->sa, max, min)) && (sub
= atom_inc(sql->sa, sub)) && (sub = atom_cast(sql->sa, sub,
sql_fetch_localtype(TYPE_oid)))) {
assert(0);
euniques = MIN(euniques, (BUN)
sub->data.val.oval);
- printf("euniques %lu\n", euniques);
+ //printf("euniques %lu\n", euniques);
}
}
if (euniques != BUN_NONE && (euniques > 0 && nuniques <
BUN_MAX/euniques))
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
