Changeset: ac76f6e21ca7 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/ac76f6e21ca7
Removed Files:
monetdb5/modules/mal/ngrams_old.c
Branch: strimps_v3
Log Message:
remove old imp
diffs (truncated from 861 to 300 lines):
diff --git a/monetdb5/modules/mal/ngrams_old.c
b/monetdb5/modules/mal/ngrams_old.c
deleted file mode 100644
--- a/monetdb5/modules/mal/ngrams_old.c
+++ /dev/null
@@ -1,856 +0,0 @@
-#include <monetdb_config.h>
-#include <mal_exception.h>
-#include <gdk_cand.h>
-#include <string.h>
-
-#if 0
-#define GZ 128
-#define CHAR_MAP(s) (s&127)
-#else
-#define GZ 64
-#define CHAR_MAP(s) (s&63)
-#endif
-#define SZ_1GRAM GZ
-#define SZ_2GRAM (GZ*GZ)
-#define SZ_3GRAM (GZ*GZ*GZ)
-
-static str
-NGcx(bit *r, str *h, str *needle)
-{
- *r = strstr(*h, *needle) != NULL;
- return MAL_SUCCEED;
-}
-
-static str
-NGcxselect(bat *R, bat *H, bat *C, str *Needle, bit *anti)
-{
- (void)R;
- (void)H;
- (void)C;
- (void)Needle;
- (void)anti;
- return MAL_SUCCEED;
-}
-
-#if 0
-#define NGRAM_TYPE hge
-#define NGRAM_TYPEID TYPE_hge
-#define NGRAM_TYPENIL hge_nil
-#define NGRAM_CST(v) ((hge)LL_CONSTANT(v))
-#define NGRAM_BITS 127
-#else
-#define NGRAM_TYPE lng
-#define NGRAM_TYPEID TYPE_lng
-#define NGRAM_TYPENIL lng_nil
-#define NGRAM_CST(v) LL_CONSTANT(v)
-#define NGRAM_BITS 63
-#endif
-
-#define NGRAM_MULTIPLE 16
-
-typedef struct {
- NGRAM_TYPE max, small;
- unsigned int *h;
- unsigned int *pos;
- unsigned int *rid;
- NGRAM_TYPE *idx;
- NGRAM_TYPE *sigs;
-} Ngrams;
-
-static void
-ngrams_destroy(Ngrams *n)
-{
- if (n) {
- GDKfree(n->h);
- GDKfree(n->idx);
- GDKfree(n->pos);
- GDKfree(n->rid);
- GDKfree(n->sigs);
- }
- GDKfree(n);
-}
-
-static Ngrams *
-ngrams_create(BAT *b, size_t ngramsize)
-{
- Ngrams *n = NULL;
- size_t sz = BATcount(b);
-
- n = (Ngrams*)GDKmalloc(sizeof(Ngrams));
- if (n) {
- n->h = (unsigned int*)GDKmalloc(ngramsize*sizeof(int));
- n->pos=(unsigned int*)GDKzalloc(ngramsize*sizeof(int));
- n->rid=(unsigned int*)GDKmalloc(NGRAM_MULTIPLE* sz *
sizeof(int));
-
- n->idx = (NGRAM_TYPE*)GDKmalloc(ngramsize*sizeof(NGRAM_TYPE));
- n->sigs=(NGRAM_TYPE*)GDKmalloc(sz * sizeof(NGRAM_TYPE));
- }
- if (!n || !n->h || !n->idx || !n->pos || !n->rid || !n->sigs) {
- ngrams_destroy(n);
- return NULL;
- }
- return n;
-}
-
-static int
-ngrams_init_1gram(Ngrams *n, BAT *b)
-{
- BUN cnt = BATcount(b);
- NGRAM_TYPE *h = (NGRAM_TYPE *)GDKzalloc(SZ_1GRAM*sizeof(NGRAM_TYPE)),
*hist = (NGRAM_TYPE*)h, sum = 0;
- int *id = (int*)GDKmalloc(SZ_1GRAM*sizeof(int)), i;
- NGRAM_TYPE *idx = n->idx;
-
- if (!h || !id) {
- GDKfree(h);
- GDKfree(id);
- return -1;
- }
-
- BATiter bi = bat_iterator(b);
- for(BUN i=0; i<cnt; i++) {
- const char *s = BUNtail(bi,i);
- if (!strNil(s) && *s) { /* skipped */
- for(; *s; s++) {
- h[CHAR_MAP(*s)]++;
- }
- }
- }
- bat_iterator_end(&bi);
-
- int bc = 0;
-
- for(int i=0; i<SZ_1GRAM; i++) {
- id[i] = i;
- idx[i] = 0;
- n->h[i] = (unsigned int)hist[i];
- }
- GDKqsort(h, id, NULL, SZ_1GRAM, sizeof(NGRAM_TYPE), sizeof(int),
NGRAM_TYPEID, true, false);
- for(i=SZ_1GRAM-1; i>=0; i--) {
- if ((BUN)(sum + hist[i]) >= (NGRAM_MULTIPLE*cnt)-1)
- break;
- sum += hist[i];
- }
- NGRAM_TYPE larger_cnt = hist[i];
- for(; hist[i] == larger_cnt; i++)
- ;
- NGRAM_TYPE max = hist[0], small = hist[i];
- printf("max %d, first smaller %d nr of larger %d sum %ld, cnt %ld\n",
(int)hist[0], (int)small, i, sum, cnt);
- n->max = max;
- n->small = small;
- for(int i=0; i<SZ_1GRAM && hist[i] > 0; i++) {
- unsigned int x=id[i];
- idx[x] = NGRAM_CST(1)<<bc;
- assert(idx[x] > 0);
- bc++;
- bc %= NGRAM_BITS;
- }
-
- bi = bat_iterator(b);
- NGRAM_TYPE *sp = n->sigs;
- unsigned int pos = 1;
- for(BUN i=0; i<cnt; i++) {
- const char *s = BUNtail(bi,i);
- NGRAM_TYPE sig = 0;
- if (!strNil(s) && s[0]) { /* too short skipped */
- for(; *s; s++) {
- int k = CHAR_MAP(*s);
- sig |= idx[k];
- if (n->h[k] <= n->small) {
- if (n->pos[k] == 0) {
- n->pos[k] = pos;
- pos += n->h[k];
- n->h[k] = 0;
- }
- /* deduplicate */
- int done = (n->h[k] > 0 &&
n->rid[n->pos[k] + n->h[k]-1] == i);
- if (!done) {
- n->rid[n->pos[k] + n->h[k]] = i;
- n->h[k]++;
- }
- }
- }
- *sp = sig;
- } else {
- *sp = NGRAM_TYPENIL;
- }
- sp++;
- }
- bat_iterator_end(&bi);
-
- GDKfree(h);
- GDKfree(id);
- return 0;
-}
-
-static str
-NGc1join_intern(bat *L, bat *R, bat *H, bat *N, bat *lc, bat *rc, bit
*nil_matches, lng *estimate, bit *anti)
-{
- (void)nil_matches;
- (void)estimate;
- BAT *h = BATdescriptor(*H);
- BAT *n = BATdescriptor(*N);
- BAT *r = NULL;
-
- if (lc && !is_bat_nil(*lc))
- assert(0);
- if (rc && !is_bat_nil(*rc))
- assert(0);
-
- if (*anti)
- throw(MAL, "gram.c1", "No anti contains yet\n");
- if (!h || !n) {
- BBPreclaim(h);
- BBPreclaim(n);
- throw(MAL, "gram.c1", RUNTIME_OBJECT_MISSING);
- }
-
- if (BATcount(n) < 10) {
- printf("todo fall back to select \n");
- }
-
- Ngrams *ngi = ngrams_create(h, SZ_1GRAM);
- if (ngi && ngrams_init_1gram(ngi, h) == 0) { /* TODO add locks and only
create ngram once for full (parent bat) */
- BUN cnt = BATcount(h);
- /* create L/R */
- BAT *l = COLnew(0, TYPE_oid, 10*cnt, TRANSIENT);
- if (R)
- r = COLnew(0, TYPE_oid, 10*cnt, TRANSIENT);
-
- int ncnt = 0, ncnt1 = 0, ncnt2 = 0, ncnt3 = 0, ncnt4 = 0, ncnt5
= 0;
- BATiter ni = bat_iterator(n);
- BATiter hi = bat_iterator(h);
- NGRAM_TYPE nmax = 0;
- oid *ol = Tloc(l, 0), *el = ol + 10*cnt;
- oid *or = NULL;
- if (r)
- or = Tloc(r, 0);
- cnt = BATcount(n);
- /* if needed grow */
- for(BUN i = 0; i<cnt; i++) {
- const char *s = BUNtail(ni,i), *os = s;
- NGRAM_TYPE sig = 0;
-
- if ((ol+1000) > el)
- break;
- if (!strNil(s) && s[0]) {
- NGRAM_TYPE min = ngi->max;
- unsigned int min_pos = 0;
- for(; *s; s++) {
- unsigned int k = CHAR_MAP(*s);
- sig |= ngi->idx[k];
- if (ngi->h[k] < min) {
- min = ngi->h[k];
- min_pos = k; /* encoded min
ngram */
- }
- }
- ncnt++;
- if (min <= ngi->small) {
- unsigned int rr = ngi->pos[min_pos];
- int hcnt = ngi->h[min_pos];
- ncnt1++;
- //printf("list used %d pos %d,%d,
%s\n", hcnt, rr, min_pos, os);
- for(int k = 0; k<hcnt; k++, rr++) {
- unsigned int hr = ngi->rid[rr];
- if (((ngi->sigs[hr] & sig) ==
sig)) {
- char *hs = BUNtail(hi,
hr);
- ncnt3++;
- if (strstr(hs, os) !=
NULL) {
- *ol++ = hr;
- if (R)
- *or++ =
(oid)i;
- }
- }
- }
- } else {
- unsigned int hcnt = BATcount(h);
- ncnt2++;
- for(size_t k = 0; k<hcnt; k++) {
- if (((ngi->sigs[k] & sig) ==
sig)) {
- char *hs = BUNtail(hi,
k);
- ncnt4++;
- if (strstr(hs, os) !=
NULL) {
- *ol++ = k;
- if (R)
- *or++ =
(oid)i;
- }
- }
- }
- }
- //printf("%d %d\n", min_pos, (int)min);
- if (min > nmax)
- nmax = min;
- } else if (!strNil(s)) { /* skipped */
- unsigned int hcnt = BATcount(h);
- ncnt++;
- for(size_t k = 0; k<hcnt; k++) {
- char *hs = BUNtail(hi, k);
- ncnt5++;
- if (strstr(hs, os) != NULL) {
- *ol++ = k;
- if (R)
- *or++ = (oid)i;
- }
- }
- }
- }
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
