Changeset: be2254a721bb for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/be2254a721bb Modified Files: monetdb5/modules/atoms/str.c monetdb5/modules/mal/txtsim.c Branch: txtsim-imp Log Message:
maxlevenshteinjoin finished and cleaned up
diffs (truncated from 668 to 300 lines):
diff --git a/monetdb5/modules/atoms/str.c b/monetdb5/modules/atoms/str.c
--- a/monetdb5/modules/atoms/str.c
+++ b/monetdb5/modules/atoms/str.c
@@ -5221,71 +5221,71 @@ STRcontainsselect(bat *ret, const bat *b
#define VALUE(s, x) (s##vars + VarHeapVal(s##vals, (x), s##i.width))
/* nested loop implementation for batstr joins */
-#define batstr_join_loop(STRCMP, STR_LEN) \
- do { \
- for (BUN ridx = 0; ridx < rci.ncand; ridx++) { \
- GDK_CHECK_TIMEOUT(timeoffset, counter, \
- GOTO_LABEL_TIMEOUT_HANDLER(bailout)); \
- ro = canditer_next(&rci); \
- vr = VALUE(r, ro - rbase); \
- rlen = STR_LEN; \
- nl = 0; \
- canditer_reset(&lci); \
- for (BUN lidx = 0; lidx < lci.ncand; lidx++) { \
- lo = canditer_next(&lci); \
- vl = VALUE(l, lo - lbase); \
- if (strNil(vl)) { \
- continue; \
- } else if (!(STRCMP)) { \
- continue; \
- } \
- if (BATcount(r1) == BATcapacity(r1)) { \
- newcap = BATgrows(r1); \
- BATsetcount(r1, BATcount(r1)); \
- if (r2) \
- BATsetcount(r2, BATcount(r2)); \
+#define batstr_join_loop(STRCMP, STR_LEN)
\
+ do {
\
+ for (BUN ridx = 0; ridx < rci.ncand; ridx++) {
\
+ GDK_CHECK_TIMEOUT(timeoffset, counter,
\
+ GOTO_LABEL_TIMEOUT_HANDLER(bailout));
\
+ ro = canditer_next(&rci);
\
+ vr = VALUE(r, ro - rbase);
\
+ rlen = STR_LEN;
\
+ nl = 0;
\
+ canditer_reset(&lci);
\
+ for (BUN lidx = 0; lidx < lci.ncand; lidx++) {
\
+ lo = canditer_next(&lci);
\
+ vl = VALUE(l, lo - lbase);
\
+ if (strNil(vl)) {
\
+ continue;
\
+ } else if (!(STRCMP)) {
\
+ continue;
\
+ }
\
+ if (BATcount(r1) == BATcapacity(r1)) {
\
+ newcap = BATgrows(r1);
\
+ BATsetcount(r1, BATcount(r1));
\
+ if (r2)
\
+ BATsetcount(r2, BATcount(r2));
\
if (BATextend(r1, newcap) !=
GDK_SUCCEED || (r2 && BATextend(r2, newcap) != GDK_SUCCEED)) { \
msg = createException(MAL,
"pcre.join", SQLSTATE(HY013) MAL_MALLOC_FAIL); \
- goto bailout; \
- } \
- assert(!r2 || BATcapacity(r1) ==
BATcapacity(r2)); \
- } \
- if (BATcount(r1) > 0) { \
- if (lastl + 1 != lo) \
- r1->tseqbase = oid_nil; \
- if (nl == 0) { \
- if (r2) \
- r2->trevsorted = false;
\
- if (lastl > lo) { \
- r1->tsorted = false; \
- r1->tkey = false; \
- } else if (lastl < lo) { \
- r1->trevsorted = false;
\
- } else { \
- r1->tkey = false; \
- } \
- } \
- } \
- APPEND(r1, lo); \
- if (r2) \
- APPEND(r2, ro); \
- lastl = lo; \
- nl++; \
- } \
- if (r2) { \
- if (nl > 1) { \
- r2->tkey = false; \
- r2->tseqbase = oid_nil; \
- r1->trevsorted = false; \
- } else if (nl == 0) { \
- rskipped = BATcount(r2) > 0; \
- } else if (rskipped) { \
- r2->tseqbase = oid_nil; \
- } \
- } else if (nl > 1) { \
- r1->trevsorted = false; \
- } \
- } \
+ goto bailout;
\
+ }
\
+ assert(!r2 || BATcapacity(r1) ==
BATcapacity(r2)); \
+ }
\
+ if (BATcount(r1) > 0) {
\
+ if (lastl + 1 != lo)
\
+ r1->tseqbase = oid_nil;
\
+ if (nl == 0) {
\
+ if (r2)
\
+ r2->trevsorted = false;
\
+ if (lastl > lo) {
\
+ r1->tsorted = false;
\
+ r1->tkey = false;
\
+ } else if (lastl < lo) {
\
+ r1->trevsorted = false;
\
+ } else {
\
+ r1->tkey = false;
\
+ }
\
+ }
\
+ }
\
+ APPEND(r1, lo);
\
+ if (r2)
\
+ APPEND(r2, ro);
\
+ lastl = lo;
\
+ nl++;
\
+ }
\
+ if (r2) {
\
+ if (nl > 1) {
\
+ r2->tkey = false;
\
+ r2->tseqbase = oid_nil;
\
+ r1->trevsorted = false;
\
+ } else if (nl == 0) {
\
+ rskipped = BATcount(r2) > 0;
\
+ } else if (rskipped) {
\
+ r2->tseqbase = oid_nil;
\
+ }
\
+ } else if (nl > 1) {
\
+ r1->trevsorted = false;
\
+ }
\
+ }
\
} while (0)
static str
@@ -5457,9 +5457,9 @@ STRjoin(bat *r1, bat *r2, const bat lid,
BBPreclaim(result1);
BBPreclaim(result2);
}
- fail:
- BBPunfix(left->batCacheid);
- BBPunfix(right->batCacheid);
+ fail:
+ BBPreclaim(left);
+ BBPreclaim(right);
BBPreclaim(candleft);
BBPreclaim(candright);
return msg;
diff --git a/monetdb5/modules/mal/txtsim.c b/monetdb5/modules/mal/txtsim.c
--- a/monetdb5/modules/mal/txtsim.c
+++ b/monetdb5/modules/mal/txtsim.c
@@ -224,6 +224,7 @@ levenshtein(int *res, const str *X, cons
throw(MAL, "txtsim.levenshtein", "Illegal unicode code point");
}
+/* Levenshtein OP but with column externaly allocated */
static inline int
levenshtein2(const str X, const str Y, const size_t xlen, const size_t ylen,
unsigned int *column,
const int insdel_cost, const int replace_cost, const
int max)
@@ -320,7 +321,6 @@ TXTSIMmaxlevenshtein(Client cntxt, MalBl
}
return levenshtein(res, X, Y, insdel_cost, replace_cost, *k);
- return MAL_SUCCEED;
}
static str
@@ -332,96 +332,96 @@ BATTXTSIMmaxlevenshtein(Client cntxt, Ma
bat *lid = getArgReference_bat(stk, pci, 1);
bat *rid = getArgReference_bat(stk, pci, 2);
int *k = getArgReference_int(stk, pci, 3);
- int insdel_cost = 1, replace_cost = 1;
- if (pci->argc == 6) {
- insdel_cost = *getArgReference_int(stk, pci, 4);
- replace_cost = *getArgReference_int(stk, pci, 5);
- }
- BAT *left = NULL, *right = NULL;
- BAT *bn = NULL;
+ int insdel_cost = pci->argc == 6? *getArgReference_int(stk, pci, 4) : 1,
+ replace_cost = pci->argc == 6? *getArgReference_int(stk, pci,
5) : 1;
+ BAT *left = NULL, *right = NULL, *bn = NULL;
BUN p,q;
BATiter li, ri;
unsigned int *buffer = NULL;
- str lv,rv;
+ str lv, rv, msg = MAL_SUCCEED;
size_t llen=0, rlen=0, maxlen=0;
int d;
bit v;
- str msg = MAL_SUCCEED;
- if ((left = BATdescriptor(*lid)) == NULL)
- goto failed;
- if ((right = BATdescriptor(*rid)) == NULL)
- goto failed;
-
- if (BATcount(left) != BATcount(right))
- goto failed;
-
- if ((bn = COLnew(0, TYPE_bit, BATcount(left), TRANSIENT)) == NULL)
- goto failed;
+ if ((left = BATdescriptor(*lid)) == NULL) {
+ msg = createException(MAL, "battxtsim.maxlevenshtein",
SQLSTATE(HY002) RUNTIME_OBJECT_MISSING);
+ goto exit;
+ }
+ if ((right = BATdescriptor(*rid)) == NULL) {
+ msg = createException(MAL, "battxtsim.maxlevenshtein",
SQLSTATE(HY002) RUNTIME_OBJECT_MISSING);
+ goto exit;
+ }
+ if (BATcount(left) != BATcount(right)) {
+ msg = createException(MAL, "battxtsim.maxlevenshtein", "Columns
must be aligned");
+ goto exit;
+ }
+ if ((bn = COLnew(0, TYPE_bit, BATcount(left), TRANSIENT)) == NULL) {
+ msg = createException(MAL, "battxtsim.maxlevenshtein",
SQLSTATE(HY013) MAL_MALLOC_FAIL);
+ goto exit;
+ }
li = bat_iterator(left);
ri = bat_iterator(right);
-
BATloop(left, p, q) {
lv = (str) BUNtail(li, p);
rv = (str) BUNtail(ri, p);
llen = UTF8_strlen(lv);
rlen = UTF8_strlen(rv);
-
if (abs((int)llen - (int)rlen) > (int)*k)
v = false;
else {
if (llen > maxlen) {
maxlen = llen;
unsigned int *tmp = GDKrealloc(buffer, (maxlen
+ 1) * sizeof(unsigned int));
+ if (tmp == NULL) {
+ bat_iterator_end(&li);
+ bat_iterator_end(&ri);
+ msg = createException(MAL,
"battxtsim.maxlevenshtein", SQLSTATE(HY013) MAL_MALLOC_FAIL);
+ goto exit;
+ }
buffer = tmp;
}
- d = levenshtein2(lv, rv, llen, rlen, buffer, (int)*k,
insdel_cost, replace_cost);
+ d = levenshtein2(lv, rv, llen, rlen, buffer,
insdel_cost, replace_cost, (int)*k);
v = (bit)(d <= (int)*k);
}
- if (BUNappend(bn, (const void *)&v, false) != GDK_SUCCEED)
- goto bunins_failed;
+ if (BUNappend(bn, (const void *)&v, false) != GDK_SUCCEED) {
+ bat_iterator_end(&li);
+ bat_iterator_end(&ri);
+ msg = createException(MAL, "battxtsim.maxlevenshtein",
"BUNappend failed");
+ goto exit;
+ }
}
bat_iterator_end(&li);
bat_iterator_end(&ri);
- GDKfree(buffer);
-
- BBPreclaim(left);
- BBPreclaim(right);
*res = bn->batCacheid;
BBPkeepref(bn);
-
- return MAL_SUCCEED;
-
-bunins_failed:
-failed:
- if (buffer) GDKfree(buffer);
- if (left) BBPreclaim(left);
- if (right) BBPreclaim(right);
- if (bn) BBPreclaim(bn);
+ exit:
+ GDKfree(buffer);
+ BBPreclaim(left);
+ BBPreclaim(right);
if (msg != MAL_SUCCEED)
- return msg;
- throw(MAL, "battxtsim.maxlevenshtein", OPERATION_FAILED);
- return MAL_SUCCEED;
+ BBPreclaim(bn);
+ return msg;
}
#define JARO_WINKLER_SCALING_FACTOR 0.1
#define JARO_WINKLER_PREFIX_LEN 4
typedef struct {
- size_t matches; /* accumulator for number of matches for this item
*/
- BUN o; /* position in the BAT */
- str val; /* string value */
- int *cp_sequence; /* string as array of Unicode codepoints */
+ size_t matches; /* accumulator for number of matches for this item */
+ BUN o; /* position in the BAT */
+ str val; /* string value */
+ int *cp_sequence; /* string as array of Unicode codepoints */
int len; /* string length in characters (multi-byte characters
count as 1)*/
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
