Changeset: 2a1facf8d040 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/2a1facf8d040
Modified Files:
gdk/gdk.h
gdk/gdk_private.h
gdk/gdk_strimps.c
gdk/gdk_strimps.h
monetdb5/modules/mal/pcre.c
monetdb5/modules/mal/strimps.c
sql/backends/monet5/sql_cat.c
Branch: Jan2022-strimps-integration
Log Message:
Integrate strimps with imprints in SQL level
diffs (298 lines):
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -1774,6 +1774,12 @@ gdk_export gdk_return BATimprints(BAT *b
gdk_export void IMPSdestroy(BAT *b);
gdk_export lng IMPSimprintsize(BAT *b);
+/* Strimps exported functions */
+gdk_export gdk_return STRMPcreate(BAT *b, BAT *s);
+gdk_export BAT *STRMPfilter(BAT *b, BAT *s, const char *q);
+gdk_export void STRMPdestroy(BAT *b);
+gdk_export bool BAThasstrimps(BAT *b);
+
/* The ordered index structure */
gdk_export gdk_return BATorderidx(BAT *b, bool stable);
@@ -2392,11 +2398,6 @@ gdk_export BAT *BATsample_with_seed(BAT
CALLBACK; \
} while (0)
-/*
- * String Imprints Development/Testing. TODO: remove the following.
- */
-
-#include "gdk_strimps.h"
typedef struct gdk_callback {
char *name;
int argc;
diff --git a/gdk/gdk_private.h b/gdk/gdk_private.h
--- a/gdk/gdk_private.h
+++ b/gdk/gdk_private.h
@@ -233,8 +233,6 @@ void STRMPincref(Strimps *strimps)
__attribute__((__visibility__("hidden")));
void STRMPdecref(Strimps *strimps, bool remove)
__attribute__((__visibility__("hidden")));
-void STRMPdestroy(BAT *b)
- __attribute__((__visibility__("hidden")));
void STRMPfree(BAT *b)
__attribute__((__visibility__("hidden")));
void MT_init_posix(void)
diff --git a/gdk/gdk_strimps.c b/gdk/gdk_strimps.c
--- a/gdk/gdk_strimps.c
+++ b/gdk/gdk_strimps.c
@@ -80,6 +80,8 @@
#include "gdk.h"
#include "gdk_private.h"
+#include "gdk_strimps.h"
+
/* Macros for accessing metadada of a strimp. These are recorded in the
* first 8 bytes of the heap.
@@ -698,6 +700,21 @@ STRMPcreateStrimpHeap(BAT *b, BAT *s)
return r;
}
+bool
+BAThasstrimps(BAT *b)
+{
+ BAT *pb;
+ if (VIEWtparent(b)) {
+ pb = BBP_cache(VIEWtparent(b));
+ assert(pb);
+ } else {
+ pb = b;
+ }
+
+ return BATcheckstrimps(pb);
+
+}
+
gdk_return
STRMPcreate(BAT *b, BAT *s)
{
@@ -762,58 +779,6 @@ STRMPcreate(BAT *b, BAT *s)
return GDK_SUCCEED;
}
-gdk_return
-STRMPappendBitstring(BAT *b, const char *s)
-{
- lng t0 = 0;
- BAT *pb;
- uint64_t *dh;
- Strimps *strmp;
- const float extend_factor = 1.5;
-
- TRC_DEBUG_IF(ACCELERATOR) t0 = GDKusec();
- if (ATOMstorage(b->ttype) != TYPE_str) {
- GDKerror("Cannot manipulate strimps index for non string
bats\n");
- return GDK_FAIL;
- }
-
- if (VIEWtparent(b)) {
- pb = BBP_cache(VIEWtparent(b));
- assert(pb);
- } else {
- pb = b;
- }
-
- if (!BATcheckstrimps(pb)) {
- GDKerror("Strimp missing, cannot append value\n");
- return GDK_FAIL;
- }
- MT_lock_set(&pb->batIdxLock);
- strmp = pb->tstrimps;
- /* Extend heap if there is not enough space */
- if (strmp->strimps.free >= strmp->strimps.size + sizeof(uint64_t)) {
- size_t sizes_offset = (char *)strmp->sizes_base -
strmp->strimps.base;
- size_t pairs_offset = (char *)strmp->pairs_base -
strmp->strimps.base;
- size_t bitstrings_offset = (char *)strmp->bitstrings_base -
strmp->strimps.base;
- if (HEAPextend(&(strmp->strimps),
(size_t)(extend_factor*BATcount(pb)*sizeof(uint64_t)), false) != GDK_SUCCEED) {
- MT_lock_unset(&pb->batIdxLock);
- GDKerror("Cannot extend heap\n");
- return GDK_FAIL;
- }
- strmp->sizes_base = (uint8_t *)strmp->strimps.base +
sizes_offset;
- strmp->pairs_base = (uint8_t *)strmp->strimps.base +
pairs_offset;
- strmp->bitstrings_base = strmp->strimps.base +
bitstrings_offset;
- }
- dh = (uint64_t *)strmp->strimps.base + pb->tstrimps->strimps.free;
- *dh = STRMPmakebitstring(s, strmp);
- strmp->strimps.free += sizeof(uint64_t);
-
- strmp->rec_cnt++;
- MT_lock_unset(&pb->batIdxLock);
-
- TRC_DEBUG(ACCELERATOR, "appending to strimp took " LLFMT " usec\n",
GDKusec()-t0);
- return GDK_SUCCEED;
-}
void
STRMPdecref(Strimps *strimps, bool remove)
@@ -842,7 +807,6 @@ void
STRMPdestroy(BAT *b)
{
if (b && b->tstrimps) {
- TRC_DEBUG(ACCELERATOR, "Destroying strimp %s\n",
b->tstrimps->strimps.filename);
MT_lock_set(&b->batIdxLock);
if (b->tstrimps == (Strimps *)1) {
b->tstrimps = NULL;
@@ -862,7 +826,6 @@ void
STRMPfree(BAT *b)
{
if (b && b->tstrimps) {
- TRC_DEBUG(ACCELERATOR, "Freeing strimp for BAT %s\n",
b->tstrimps->strimps.filename);
Strimps *s;
MT_lock_set(&b->batIdxLock);
if ((s = b->tstrimps) != NULL && s != (Strimps *)1) {
@@ -1024,4 +987,60 @@ STRMPcreate(BAT *b, BAT *s)
TRC_DEBUG(ACCELERATOR, "strimp creation took " LLFMT " usec\n",
GDKusec()-t0);
return GDK_SUCCEED;
}
+
+/* Update the strimp by computing a bitstring and adding it to the heap.
+ This will probably be useful later when strimps take updates into
+ account. */
+gdk_return
+STRMPappendBitstring(BAT *b, const char *s)
+{
+ lng t0 = 0;
+ BAT *pb;
+ uint64_t *dh;
+ Strimps *strmp;
+ const float extend_factor = 1.5;
+
+ TRC_DEBUG_IF(ACCELERATOR) t0 = GDKusec();
+ if (ATOMstorage(b->ttype) != TYPE_str) {
+ GDKerror("Cannot manipulate strimps index for non string
bats\n");
+ return GDK_FAIL;
+ }
+
+ if (VIEWtparent(b)) {
+ pb = BBP_cache(VIEWtparent(b));
+ assert(pb);
+ } else {
+ pb = b;
+ }
+
+ if (!BATcheckstrimps(pb)) {
+ GDKerror("Strimp missing, cannot append value\n");
+ return GDK_FAIL;
+ }
+ MT_lock_set(&pb->batIdxLock);
+ strmp = pb->tstrimps;
+ /* Extend heap if there is not enough space */
+ if (strmp->strimps.free >= strmp->strimps.size + sizeof(uint64_t)) {
+ size_t sizes_offset = (char *)strmp->sizes_base -
strmp->strimps.base;
+ size_t pairs_offset = (char *)strmp->pairs_base -
strmp->strimps.base;
+ size_t bitstrings_offset = (char *)strmp->bitstrings_base -
strmp->strimps.base;
+ if (HEAPextend(&(strmp->strimps),
(size_t)(extend_factor*BATcount(pb)*sizeof(uint64_t)), false) != GDK_SUCCEED) {
+ MT_lock_unset(&pb->batIdxLock);
+ GDKerror("Cannot extend heap\n");
+ return GDK_FAIL;
+ }
+ strmp->sizes_base = (uint8_t *)strmp->strimps.base +
sizes_offset;
+ strmp->pairs_base = (uint8_t *)strmp->strimps.base +
pairs_offset;
+ strmp->bitstrings_base = strmp->strimps.base +
bitstrings_offset;
+ }
+ dh = (uint64_t *)strmp->strimps.base + pb->tstrimps->strimps.free;
+ *dh = STRMPmakebitstring(s, strmp);
+ strmp->strimps.free += sizeof(uint64_t);
+
+ strmp->rec_cnt++;
+ MT_lock_unset(&pb->batIdxLock);
+
+ TRC_DEBUG(ACCELERATOR, "appending to strimp took " LLFMT " usec\n",
GDKusec()-t0);
+ return GDK_SUCCEED;
+}
#endif
diff --git a/gdk/gdk_strimps.h b/gdk/gdk_strimps.h
--- a/gdk/gdk_strimps.h
+++ b/gdk/gdk_strimps.h
@@ -34,7 +34,4 @@ typedef struct {
} PairHistogramElem;
-gdk_export gdk_return STRMPcreate(BAT *b, BAT *s);
-gdk_export BAT *STRMPfilter(BAT *b, BAT *s, const char *q);
-gdk_export gdk_return STRMPappendBitstring(BAT *b, const char *s);
#endif /* _GDK_STRIMPS_H_ */
diff --git a/monetdb5/modules/mal/pcre.c b/monetdb5/modules/mal/pcre.c
--- a/monetdb5/modules/mal/pcre.c
+++ b/monetdb5/modules/mal/pcre.c
@@ -29,6 +29,8 @@
#include <wchar.h>
#include <wctype.h>
+#include "gdk_strimps.h"
+
#ifdef HAVE_LIBPCRE
#include <pcre.h>
#ifndef PCRE_STUDY_JIT_COMPILE
@@ -1871,8 +1873,7 @@ PCRElikeselect(bat *ret, const bat *bid,
str msg = MAL_SUCCEED;
char *ppat = NULL;
bool use_re = false, use_strcmp = false, empty = false;
- bool use_strimps = GDKgetenv_isyes("gdk_use_strimps"), with_strimps =
false;
- BUN strimp_creation_threshold = GDKgetenv_int("gdk_strimps_threshold",
1000000);
+ bool with_strimps = false;
if ((b = BATdescriptor(*bid)) == NULL) {
msg = createException(MAL, "algebra.likeselect",
SQLSTATE(HY002) RUNTIME_OBJECT_MISSING);
@@ -1895,9 +1896,9 @@ PCRElikeselect(bat *ret, const bat *bid,
* A better solution is to run the PCRElikeselect as a LIKE query with
* strimps and return the complement of the result.
*/
- if (!empty && use_strimps && BATcount(b) >= strimp_creation_threshold
&& !*anti) {
- BAT *tmp_s = NULL;
- if (STRMPcreate(b, NULL) == GDK_SUCCEED && (tmp_s =
STRMPfilter(b, s, *pat))) {
+ if (!empty && BAThasstrimps(b) && !*anti) {
+ BAT *tmp_s = STRMPfilter(b, s, *pat);
+ if (tmp_s) {
if (s)
BBPunfix(s->batCacheid);
s = tmp_s;
diff --git a/monetdb5/modules/mal/strimps.c b/monetdb5/modules/mal/strimps.c
--- a/monetdb5/modules/mal/strimps.c
+++ b/monetdb5/modules/mal/strimps.c
@@ -6,6 +6,7 @@
* Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
*/
#include "monetdb_config.h"
+#include "mal.h"
#include "mal_client.h"
#include "mal_interpreter.h"
#include "mal_exception.h"
diff --git a/sql/backends/monet5/sql_cat.c b/sql/backends/monet5/sql_cat.c
--- a/sql/backends/monet5/sql_cat.c
+++ b/sql/backends/monet5/sql_cat.c
@@ -745,6 +745,10 @@ IDXdrop(mvc *sql, const char *sname, con
if (!(b = BATdescriptor(nb->batCacheid)))
throw(SQL,"sql.drop_index", SQLSTATE(HY005) "Column can
not be accessed");
}
+
+ /* TODO: This should be removed when strimps are better integrated with
imprints */
+ if (func == IMPSdestroy && b->ttype == TYPE_str)
+ func = STRMPdestroy;
func(b);
BBPunfix(b->batCacheid);
return MAL_SUCCEED;
@@ -1278,7 +1282,15 @@ alter_table(Client cntxt, mvc *sql, char
if (!(b =
BATdescriptor(nb->batCacheid)))
throw(SQL,"sql.alter_table",SQLSTATE(HY005) "Cannot access imprints index
%s_%s_%s", s->base.name, t->base.name, i->base.name);
}
- r = BATimprints(b);
+ if(b->ttype == TYPE_str) {
+ if (t->access != TABLE_READONLY)
+ throw(SQL, "sql.alter_TABLE",
SQLSTATE(HY005) "Cannot create string imprint index %s on non read only table
%s.%s", i->base.name, s->base.name, t->base.name);
+ r = STRMPcreate(b, NULL);
+ }
+ else {
+ r = BATimprints(b);
+ }
+
BBPunfix(b->batCacheid);
if (r != GDK_SUCCEED)
throw(SQL, "sql.alter_table",
GDK_EXCEPTION);
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
