Changeset: e1e6ae03fd98 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e1e6ae03fd98
Modified Files:
monetdb5/modules/mal/Tests/qgram.mal
monetdb5/modules/mal/Tests/qgram.stable.out
monetdb5/modules/mal/txtsim.c
Branch: default
Log Message:
Updated txtsim.str2qgrams to properly deal with UTF-8.
Also added test using UTF-8.
diffs (111 lines):
diff --git a/monetdb5/modules/mal/Tests/qgram.mal
b/monetdb5/modules/mal/Tests/qgram.mal
--- a/monetdb5/modules/mal/Tests/qgram.mal
+++ b/monetdb5/modules/mal/Tests/qgram.mal
@@ -1,2 +1,4 @@
b := txtsim.str2qgrams("hello world");
io.print(b);
+b := txtsim.str2qgrams("hällö wørłð");
+io.print(b);
diff --git a/monetdb5/modules/mal/Tests/qgram.stable.out
b/monetdb5/modules/mal/Tests/qgram.stable.out
--- a/monetdb5/modules/mal/Tests/qgram.stable.out
+++ b/monetdb5/modules/mal/Tests/qgram.stable.out
@@ -21,6 +21,8 @@ stdout of test 'qgram` in directory 'mod
function user.main():void;
b := txtsim.str2qgrams("hello world");
io.print(b);
+ b := txtsim.str2qgrams("hällö wørłð");
+ io.print(b);
end main;
#-------------------------#
# h t # name
@@ -38,6 +40,22 @@ end main;
[ 9@0, "orld" ]
[ 10@0, "rld$" ]
[ 11@0, "ld$$" ]
+#-------------------------#
+# h t # name
+# void str # type
+#-------------------------#
+[ 0@0, "##hä" ]
+[ 1@0, "#häl" ]
+[ 2@0, "häll" ]
+[ 3@0, "ällö" ]
+[ 4@0, "llö " ]
+[ 5@0, "lö w" ]
+[ 6@0, "ö wø" ]
+[ 7@0, " wør" ]
+[ 8@0, "wørł" ]
+[ 9@0, "ørłð" ]
+[ 10@0, "rłð$" ]
+[ 11@0, "łð$$" ]
# 23:21:31 >
# 23:21:31 > "Done."
diff --git a/monetdb5/modules/mal/txtsim.c b/monetdb5/modules/mal/txtsim.c
--- a/monetdb5/modules/mal/txtsim.c
+++ b/monetdb5/modules/mal/txtsim.c
@@ -924,20 +924,44 @@ CMDqgramselfjoin(BAT **res, BAT *qgram,
throw(MAL, "txtsim.qgramselfjoin", MAL_MALLOC_FAIL);
}
+/* copy up to utf8len UTF-8 encoded characters from src to buf
+ * stop early if buf (size given by bufsize) is too small, or if src runs out
+ * return number of UTF-8 characters copied (excluding NUL)
+ * close with NUL if enough space */
+static size_t
+utf8strncpy(char *buf, size_t bufsize, const char *src, size_t utf8len)
+{
+ size_t cnt = 0;
+
+ while (utf8len != 0 && *src != 0 && bufsize != 0) {
+ bufsize--;
+ utf8len--;
+ cnt++;
+ if (((*buf++ = *src++) & 0x80) != 0) {
+ while ((*src & 0xC0) == 0x80 && bufsize != 0) {
+ *buf++ = *src++;
+ bufsize--;
+ }
+ }
+ }
+ if (bufsize != 0)
+ *buf = 0;
+ return cnt;
+}
+
str
CMDstr2qgrams(int *ret, str *val)
{
BAT *bn;
size_t i, len = strlen(*val) + 5;
str s = GDKmalloc(len);
- char qgram[5];
+ char qgram[4 * 6 + 1]; /* 4 UTF-8 code points plus NULL byte */
if (s == NULL)
throw(MAL, "txtsim.str2qgram", MAL_MALLOC_FAIL);
strcpy(s, "##");
strcpy(s + 2, *val);
strcpy(s + len - 3, "$$");
- qgram[4] = 0; /* we're going to deal with 4
char strings */
bn = BATnew(TYPE_void, TYPE_str, (BUN) strlen(*val));
if (bn == NULL) {
GDKfree(s);
@@ -945,9 +969,15 @@ CMDstr2qgrams(int *ret, str *val)
}
BATseqbase(bn, 0);
- for (i = 0; i < len - 4; i++){
- strncpy(qgram, s + i, 4);
+ i = 0;
+ while (s[i]) {
+ if (utf8strncpy(qgram, sizeof(qgram), s + i, 4) < 4)
+ break;
BUNappend(bn, qgram, FALSE);
+ if ((s[i++] & 0xC0) == 0xC0) {
+ while ((s[i] & 0xC0) == 0x80)
+ i++;
+ }
}
BBPkeepref(*ret = bn->batCacheid);
GDKfree(s);
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
