Changeset: 9810827ed4ad for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9810827ed4ad
Added Files:
gdk/gdk_commons.h
gdk/gdk_mosaic_templates.h
Modified Files:
gdk/Makefile.ag
gdk/gdk_mosaic.c
gdk/gdk_qsort.c
monetdb5/modules/mosaic/mosaic_utility.h
Branch: mosaic
Log Message:
Fix gdk_mosaic.c and refactor it a bit.
diffs (truncated from 457 to 300 lines):
diff --git a/gdk/Makefile.ag b/gdk/Makefile.ag
--- a/gdk/Makefile.ag
+++ b/gdk/Makefile.ag
@@ -16,7 +16,7 @@ lib_gdk = {
gdk_calc.c gdk_calc.h gdk_calc_compare.h gdk_calc_private.h \
gdk_ssort.c gdk_ssort_impl.h \
gdk_aggr.c \
- gdk.h gdk_batop.c \
+ gdk.h gdk_commons.h gdk_batop.c \
gdk_cand.h gdk_cand.c \
gdk_search.c gdk_hash.c gdk_hash.h gdk_tm.c \
gdk_orderidx.c \
@@ -36,7 +36,7 @@ lib_gdk = {
gdk_interprocess.c gdk_interprocess.h \
gdk_bitvector.c gdk_bitvector.h \
gdk_firstn.c \
- gdk_mosaic.c \
+ gdk_mosaic.c gdk_mosaic_templates.h \
gdk_analytic_bounds.c \
gdk_analytic_func.c gdk_analytic.h \
gdk_tracer.c gdk_tracer.h \
@@ -58,6 +58,7 @@ headers_h = {
HEADERS = h
SOURCES = \
gdk.h \
+ gdk_commons.h \
gdk_analytic.h \
gdk_atoms.h \
gdk_bbp.h \
diff --git a/gdk/gdk_commons.h b/gdk/gdk_commons.h
new file mode 100644
--- /dev/null
+++ b/gdk/gdk_commons.h
@@ -0,0 +1,11 @@
+
+#define ID(a) a
+#define glue2(a, b) a ## b
+#define glue(a, b, c) a ## b ## c
+#define glue4(a, b, c, d) a ## b ## c ## d
+#define CONCAT2(a, b) glue2(a, b)
+#define CONCAT3(a, b, c) glue(a, b, c)
+#define CONCAT4(a, b, c, d) glue4(a, b, c, d)
+#define CONCAT6(a, b, c, d, e, f) CONCAT2(glue(a, b, c), glue(d, e, f))
+#define _STRINGIFY(ARG) #ARG
+#define STRINGIFY(ARG) _STRINGIFY(ARG)
diff --git a/gdk/gdk_mosaic.c b/gdk/gdk_mosaic.c
--- a/gdk/gdk_mosaic.c
+++ b/gdk/gdk_mosaic.c
@@ -33,189 +33,36 @@ MOSsync(int fd) {
#endif
}
-static Heap *
-BATmosaic_heap(BAT *bn, BUN cap, const char *ext)
-{
- const char *nme;
- Heap *m = NULL;
- char *fname = 0;
-
- nme = BBP_physical(bn->batCacheid);
- if ( (m = (Heap*)GDKzalloc(sizeof(Heap))) == NULL ||
- (m->farmid = BBPselectfarm(bn->batRole, bn->ttype, mosaicheap))
< 0 ||
- (fname = GDKfilepath(NOFARM, NULL, nme, ext)) == NULL){
- if( fname)
- GDKfree(fname);
- GDKfree(m);
- return NULL;
- }
-
- if (strlen(fname) >= sizeof(m->filename)) {
- /* TODO: check if this can actually happen.*/
- GDKfree(fname);
- GDKfree(m);
- return NULL;
- }
-
- strcpy(m->filename, fname);
- GDKfree(fname);
-
- if( HEAPalloc(m, cap, Tsize(bn)) != GDK_SUCCEED)
- return NULL;
- return m;
-}
-
-static int
-MOS_sync(BAT *bn) {
- Heap *hp;
- int fd = -1, err = 0;
-
- if (!((BBP_status(bn->batCacheid) & BBPEXISTING) && bn->batInserted ==
bn->batCount)) {
- TRC_DEBUG(ALGO, "#BAT NOT persisting index %d\n",
bn->batCacheid);
- return err;
- }
-
- /*TODO: This part is normally - e.g. imprints & hash - done in a
different thread, look into this.*/
- /*only for large ones and when there is no concurrency:
MT_create_thread(&tid, <some-mosaic-specific-sync-function>, bn,
MT_THR_DETACHED);*/
- BBPfix(bn->batCacheid);
-
- hp = bn->tmosaic;
- if (HEAPsave(hp, hp->filename, NULL, true) != GDK_SUCCEED ||
- (fd = GDKfdlocate(hp->farmid, hp->filename, "rb+", NULL)) < 0) {
- GDKfree(hp);
- err = 1;
- bn->tmosaic = NULL;
- } else {
- ((oid *) hp->base)[0] |= (oid) 1 << 24;
- if (write(fd, hp->base, SIZEOF_SIZE_T) < 0)
- GDKerror("write mosaic heap failed");
- else
- MOSsync(fd);
- }
- if( fd >= 0){
- close(fd);
- fd = -1;
- }
-
- hp = bn->tvmosaic;
- if (HEAPsave(hp, hp->filename, NULL, true) != GDK_SUCCEED ||
- (fd = GDKfdlocate(hp->farmid, hp->filename, "rb+", NULL)) < 0) {
- GDKfree(hp);
- err = 1;
- } else {
- ((oid *) hp->base)[0] |= (oid) 1 << 24;
- if (write(fd, hp->base, SIZEOF_SIZE_T) < 0)
- GDKerror("write vmosaic heap failed");
- else
- MOSsync(fd);
- }
- if( fd >= 0)
- close(fd);
- BBPunfix(bn->batCacheid);
- return err;
-}
-
-static Heap *
-MOScheck_heap(BAT *b, const char *ext){
- Heap *hp;
- const char *nme = BBP_physical(b->batCacheid);
- int fd;
-
- if ((hp = GDKzalloc(sizeof(*hp))) != NULL &&
- (hp->farmid = BBPselectfarm(b->batRole, b->ttype, mosaicheap))
>= 0 ){
-
- sprintf(hp->filename, "%s.%s", nme, ext);
-
- /* check whether a persisted mosaic-specific heap can be found
*/
- if ((fd = GDKfdlocate(hp->farmid, nme, "rb+", ext)) >= 0) {
- struct stat st;
- int hdata;
-
- if (BATcount(b) > 0 && read(fd, &hdata, sizeof(hdata))
== sizeof(hdata) &&
- hdata == MOSAIC_VERSION &&
- fstat(fd, &st) == 0 &&
- st.st_size >= (off_t) (hp->size = hp->free =
(oid) BATcount(b) * SIZEOF_OID) &&
- HEAPload(hp, nme, ext, false) == GDK_SUCCEED) {
- close(fd);
- TRC_DEBUG(ALGO, "#BATcheckmosaic %s:
reusing persisted heap %d\n", ext, b->batCacheid);
- return hp;
- }
- close(fd);
- /* unlink unusable file */
- GDKunlink(hp->farmid, BATDIR, nme, ext);
- }
- GDKfree(hp->filename);
- }
- GDKfree(hp);
- GDKclrerr(); /* we're not currently interested in errors */
- return NULL;
-}
-
-static int
-MOScheck(BAT *b)
-{
- if (VIEWtparent(b)){
- b = BBPdescriptor(VIEWtparent(b));
- }
-
- if (b->tmosaic == (Heap *) 1)
- b->tmosaic = MOScheck_heap(b, "mosaic");
-
- /* a vmosaic can only exist if the mosaic exists as well */
- if (b->tvmosaic == (Heap *) 1)
- b->tvmosaic = MOScheck_heap(b, "vmosaic");
- return b->tmosaic != NULL;
-}
+#define HEAP mosaic
+#include "gdk_mosaic_templates.h"
+#undef HEAP
+#define HEAP vmosaic
+#include "gdk_mosaic_templates.h"
+#undef HEAP
void
MOSdestroy(BAT *bn) {
- Heap *h;
- /* If there is a view then don't drop the mosaic. However, not needed
because we don;t slice over the tmosaic BAT (yet) */
- if (!bn || VIEWtparent(bn))
- return;
- if (bn->tvmosaic){
- h= bn->tvmosaic;
- if( HEAPdelete(h, BBP_physical(bn->batCacheid), "vmosaic"))
- GDKerror("MOSdestroy vmosaic failed");
- bn->tvmosaic = NULL;
- GDKfree(h);
- }
- if (bn->tmosaic){
- h= bn->tmosaic;
- if( HEAPdelete(h, BBP_physical(bn->batCacheid), "mosaic"))
- GDKerror("MOSdestroy mosaic failed");
- bn->tmosaic = NULL;
- GDKfree(h);
- }
+ MOSdestroy_mosaic(bn);
+ MOSdestroy_vmosaic(bn);
}
+
int
BATcheckmosaic(BAT *bn) {
- /* A dictionary vmosaic is dependent on mosaic heap, no need to check */
- return MOScheck(bn);
+ return MOScheck_mosaic(bn) && MOScheck_vmosaic(bn);
}
gdk_return
BATmosaic(BAT *b, BUN cap) {
- Heap *m = NULL;
-
- if (b->tmosaic == NULL && (m = BATmosaic_heap(b, cap, "mosaic")) ) {
- m->parentid = b->batCacheid;
- b->tmosaic = m;
- } else
- return GDK_FAIL;
-
- if (b->tvmosaic == NULL && (m = BATmosaic_heap(b, 128 /*start with a
small dictionary*/, "vmosaic")) ) {
- m->parentid = b->batCacheid;
- b->tvmosaic = m;
- } else {
- MOSdestroy(b);
+ if (BATmosaic_mosaic(b, cap) != GDK_SUCCEED) {
return GDK_FAIL;
}
-#ifdef PERSISTENTMOSAIC
- MOS_sync(b);
-#endif
- b->batDirtydesc = TRUE;
+
+ if (BATmosaic_vmosaic(b, 128 /*start with a small dictionary*/) !=
GDK_SUCCEED) {
+ MOSdestroy_mosaic(b);
+ return GDK_FAIL;
+ }
+
return GDK_SUCCEED;
}
diff --git a/gdk/gdk_mosaic_templates.h b/gdk/gdk_mosaic_templates.h
new file mode 100644
--- /dev/null
+++ b/gdk/gdk_mosaic_templates.h
@@ -0,0 +1,153 @@
+
+
+#include "gdk_commons.h"
+
+#ifdef PERSISTENTMOSAIC
+#define PERSIST_MOSAIC(HEAP, BN) CONCAT3(MOS_, HEAP, _sync)(BN)
+#else
+#define PERSIST_MOSAIC(HEAP, BN)
+#endif
+
+static void
+CONCAT3(MOS_, HEAP, _sync)(void *arg) {
+ BAT *bn = arg;
+ if (!((BBP_status(bn->batCacheid) & BBPEXISTING) && bn->batInserted ==
bn->batCount)) {
+ TRC_DEBUG(ALGO, "#BAT NOT persisting index %d\n",
bn->batCacheid);
+ return;
+ }
+
+ /*TODO: This part is normally - e.g. imprints & hash - done in a
different thread, look into this.*/
+ /*only for large ones and when there is no concurrency:
MT_create_thread(&tid, <some-mosaic-specific-sync-function>, bn,
MT_THR_DETACHED);*/
+ BBPfix(bn->batCacheid);
+
+ Heap *hp = bn->CONCAT2(t, HEAP);
+ int fd;
+
+ if (HEAPsave(hp, hp->filename, NULL, true) != GDK_SUCCEED ||
+ (fd = GDKfdlocate(hp->farmid, hp->filename, "rb+", NULL)) < 0) {
+ BBPunfix(bn->batCacheid);
+ GDKfree(hp);
+ bn->CONCAT2(t, HEAP) = NULL;
+ GDKerror("Error while flushing heap " STRINGIFY(HEAP) "
failed");
+ return;
+ }
+ ((oid *) hp->base)[0] |= (oid) 1 << 24;
+ if (write(fd, hp->base, SIZEOF_SIZE_T) < 0)
+ GDKerror("Error while writing heap " STRINGIFY(HEAP) "
failed");
+ if (!(GDKdebug & FORCEMITOMASK)) {
+ MOSsync(fd);
+ }
+ close(fd);
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
