Changeset: 1c1b4a5f8e20 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1c1b4a5f8e20
Modified Files:
clients/Tests/exports.stable.out
monetdb5/modules/atoms/str.c
monetdb5/modules/atoms/str.h
monetdb5/modules/kernel/batstr.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_cast.c
sql/backends/monet5/sql_result.c
Branch: default
Log Message:
Some cleanup of the str module.
diffs (truncated from 1667 to 300 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -2057,6 +2057,7 @@ str STRLtrim(str *res, str *arg1);
str STRPrefix(bit *res, str *arg1, str *arg2);
str STRReverseStrSearch(int *res, str *arg1, str *arg2);
str STRRtrim(str *res, str *arg1);
+str STRSQLLength(int *res, str *s);
str STRStrip(str *res, str *arg1);
str STRSubString(str *res, str *arg1, int *offset, int *length);
str STRSubstitute(str *res, str *arg1, str *arg2, str *arg3, bit *g);
@@ -2096,7 +2097,6 @@ str STRcstSuffixbat(bat *ret, str *cst,
str STRcststrSearchbat(bat *ret, str *cst, bat *r);
str STRfindUnescapedOccurrence(str b, str c, str esc);
str STRinsert(str *ret, str *s, int *start, int *l, str *s2);
-int STRlike(str s, str pat, str esc);
str STRlikewrap(bit *ret, str *s, str *pat, str *esc);
str STRlikewrap2(bit *ret, str *s, str *pat);
str STRlocate(int *ret, str *s1, str *s2);
@@ -2307,7 +2307,6 @@ void clrFilter(Module cntxt, str mod, st
void clrFilterVariable(MalBlkPtr mb, int i);
void clrFunction(InstrPtr p);
void clrInstruction(InstrPtr p);
-int codeset(str *res);
int color_fromstr(char *colorStr, int *len, color **c);
int color_tostr(char **colorStr, int *len, color *c);
str columnBindRef;
@@ -2861,28 +2860,9 @@ str stopProfiling(void);
str stopRef;
void strAfterCall(ValPtr v, ValPtr bak);
void strBeforeCall(ValPtr v, ValPtr bak);
-int strBytes(int *res, str s);
-int strConcat(str *res, str s, ptr val, int t);
str strEpilogue(void);
-int strFromWChr(str *res, int *c);
-int strIconv(str *res, str org, str f, str t);
-int strLength(int *res, str s);
-int strLower(str *res, str s);
-int strLtrim(str *res, str s);
-int strPrefix(bit *res, str s, str prefix);
bat *strPrelude(void);
str strRef;
-int strReverseStrSearch(int *res, str s, str s2);
-int strRtrim(str *res, str s);
-int strSQLLength(int *res, str s);
-int strStrSearch(int *res, str s, str s2);
-int strStrip(str *res, str s);
-int strSubString(str *res, str s, int *offset, int *length);
-int strSubstitute(str *res, str s, str src, str dst, bit *g);
-int strSuffix(bit *res, str s, str suffix);
-int strTail(str *res, str s, int *offset);
-int strUpper(str *res, str s);
-int strWChrAt(int *res, str val, int *at);
str streamsRef;
int stringLength(Client cntxt);
str stringdiff_impl(int *res, str *s1, str *s2);
diff --git a/monetdb5/modules/atoms/str.c b/monetdb5/modules/atoms/str.c
--- a/monetdb5/modules/atoms/str.c
+++ b/monetdb5/modules/atoms/str.c
@@ -20,50 +20,54 @@
/*
* N.J. Nes, M.L. Kersten
* The String Module
- * Strings can be created in many ways. Already in the built-in operations
- * each atom can be cast to a string using the str(atom) mil command.
- * The string module gives the possibility of construction string as a
- * substring of the a given string (s). There are two such construction
functions.
- * The first is the substring from some position (offset) until the end of
- * the string. The second start again on the given offset position but only
- * copies count number of bytes. The functions fail when the position and
- * count fall out of bounds. A negative position indicates that the position is
- * computed from the end of the source string.
+ * Strings can be created in many ways. Already in the built-in
+ * operations each atom can be cast to a string using the str(atom)
+ * mil command. The string module gives the possibility of
+ * construction string as a substring of the a given string (s). There
+ * are two such construction functions. The first is the substring
+ * from some position (offset) until the end of the string. The second
+ * start again on the given offset position but only copies count
+ * number of bytes. The functions fail when the position and count
+ * fall out of bounds. A negative position indicates that the position
+ * is computed from the end of the source string.
*
* The strings can be compared using the "=" and "!=" operators.
*
- * The operator "+" concatenates a string and an atom. The atom will be
- * converted to a string using the atom to string c function. The
- * string and the result of the conversion are concatenated to form a new
- * string. This string is returned.
+ * The operator "+" concatenates a string and an atom. The atom will
+ * be converted to a string using the atom to string c function. The
+ * string and the result of the conversion are concatenated to form a
+ * new string. This string is returned.
*
* The length function returns the length of the string. The length is
* the number of characters in the string. The maximum string length
* handled by the kernel is 32-bits long.
*
- * chrAt() returns the character at position index in the string s. The
- * function will fail when the index is out of range. The range is
- * from 0 to length(s)-1.
+ * chrAt() returns the character at position index in the string
+ * s. The function will fail when the index is out of range. The range
+ * is from 0 to length(s)-1.
*
- * The startsWith and endsWith functions test if the string s starts with or
- * ends with the given prefix or suffix.
+ * The startsWith and endsWith functions test if the string s starts
+ * with or ends with the given prefix or suffix.
*
- * The toLower and toUpper functions cast the string to lower or upper case
- * characters.
+ * The toLower and toUpper functions cast the string to lower or upper
+ * case characters.
*
* The search(str,chr) function searches for the first occurrence of a
- * character from the begining of the string. The search(chr,str) searches
- * for the last occurrence (or first from the end of the string). The last
- * search function locates the position of first occurrence of the string s2
- * in string s. All search functions return -1 when the search failed.
- * Otherwise the position is returned.
+ * character from the begining of the string. The search(chr,str)
+ * searches for the last occurrence (or first from the end of the
+ * string). The last search function locates the position of first
+ * occurrence of the string s2 in string s. All search functions
+ * return -1 when the search failed. Otherwise the position is
+ * returned.
*
- * All string functions fail when an incorrect string (NULL pointer) is given.
- * In the current implementation, a fail is signaled by returning nil, since
- * this facilitates the use of the string module in bulk operations.
+ * All string functions fail when an incorrect string (NULL pointer)
+ * is given. In the current implementation, a fail is signaled by
+ * returning nil, since this facilitates the use of the string module
+ * in bulk operations.
*
- * All functions in the module have now been converted to Unicode. Internally,
- * we use UTF-8 to store strings as Unicode in zero-terminated byte-sequences.
+ * All functions in the module have now been converted to
+ * Unicode. Internally, we use UTF-8 to store strings as Unicode in
+ * zero-terminated byte-sequences.
*/
#include "monetdb_config.h"
#include "str.h"
@@ -78,74 +82,79 @@
/*
* UTF-8 Handling
- * UTF-8 is a way to store Unicode strings in zero-terminated byte sequences,
which you can e.g.
- * strcmp() with old 8-bit Latin-1 strcmp() functions and which then gives the
same results as doing
- * the strcmp() on equivalent Latin-1 and ASCII character strings stored in
simple one-byte sequences.
- * These characteristics make UTF-8 an attractive format for upgrading an
ASCII-oriented computer
- * program towards one that supports Unicode. That is why we use UTF-8 in
Monet.
+ * UTF-8 is a way to store Unicode strings in zero-terminated byte
+ * sequences, which you can e.g. strcmp() with old 8-bit Latin-1
+ * strcmp() functions and which then gives the same results as doing
+ * the strcmp() on equivalent Latin-1 and ASCII character strings
+ * stored in simple one-byte sequences. These characteristics make
+ * UTF-8 an attractive format for upgrading an ASCII-oriented computer
+ * program towards one that supports Unicode. That is why we use UTF-8
+ * in MonetDB.
*
- * For Monet, UTF-8 mostly has no consequences, as strings stored in BATs are
regarded as data,
- * and it does not matter for the database kernel whether the zero-terminated
byte sequence it is
- * processing has UTF-8 or Latin-1 semantics. This module is the only place
where explicit string
- * functionality is located. We {\bf do} have to adapt the behavior of the MIL
length(), search(),
- * substring() and the like commands to the fact that one (Unicode) character
is now stored in
- * a variable number of bytes (possibly > 1).
+ * For MonetDB, UTF-8 mostly has no consequences, as strings stored in
+ * BATs are regarded as data, and it does not matter for the database
+ * kernel whether the zero-terminated byte sequence it is processing
+ * has UTF-8 or Latin-1 semantics. This module is the only place where
+ * explicit string functionality is located. We {\bf do} have to adapt
+ * the behavior of the MIL length(), search(), substring() and the
+ * like commands to the fact that one (Unicode) character is now
+ * stored in a variable number of bytes (possibly > 1).
*
- * One of the things that become more complex in Unicode are
uppercase/lowercase conversions. The
- * below tables are the simple one-to-one Unicode case mappings. We do not
support the special casing mappings
- * (e.g. from one to two letters).
+ * One of the things that become more complex in Unicode are
+ * uppercase/lowercase conversions. The below tables are the simple
+ * one-to-one Unicode case mappings. We do not support the special
+ * casing mappings (e.g. from one to two letters).
*
* References:
- * \begin{verbatim}
* simple casing: http://www.unicode.org/Public/UNIDATA/UnicodeData.txt
* complex casing: http://www.unicode.org/Public/UNIDATA/SpecialCasing.txt
- * \end{verbatim}
*
- * The Unicode case conversion implementation in Monet fills a mapping BAT of
int,int combinations,
- * in which we perform high-performance hash-lookup (all code inlined).
+ * The Unicode case conversion implementation in MonetDB fills a
+ * mapping BAT of int,int combinations, in which we perform
+ * high-performance hash-lookup (all code inlined).
*/
-/* This table was generated from the Unicode 5.0.0 spec.
- The table is generated by using the codes for conversion to lower
- case and for conversion to title case and upper case.
- A few code points have been moved in order to get reasonable
- conversions (if two code points are converted to the same value,
- the first one in this table wins). The code points that have
- been interchanged are:
- U+0345 (COMBINING GREEK YPOGEGRAMMENI) / U+03B9 (GREEK SMALL LETTER IOTA)
<-> U+0399 (GREEK CAPITAL LETTER IOTA)
- U+00B5 (MICRO SIGN) / U+03BC (GREEK SMALL LETTER MU) <-> U+039C (GREEK
CAPITAL LETTER MU)
- U+03C2 (GREEK SMALL LETTER FINAL SIGMA) / U+03C3 (GREEK SMALL LETTER SIGMA)
<-> U+3A3 (GREEK CAPITAL LETTER SIGMA)
- In addition, there are a few code points where there are different
- versions for upper case and title case. These had to be switched
- around a little so that the mappings are done sensibly.
-
- The following combinations are included in this order:
- lower case <-> title case
- lower case <- upper case
- upper case -> title case
- The conversion title case -> upper case was removed
-
- The relevant code points are:
- U+01C4 (LATIN CAPITAL LETTER DZ WITH CARON)
- U+01C5 (LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON)
- U+01C6 (LATIN SMALL LETTER DZ WITH CARON)
- U+01C7 (LATIN CAPITAL LETTER LJ)
- U+01C8 (LATIN CAPITAL LETTER L WITH SMALL LETTER J)
- U+01C9 (LATIN SMALL LETTER LJ)
- U+01CA (LATIN CAPITAL LETTER NJ)
- U+01CB (LATIN CAPITAL LETTER N WITH SMALL LETTER J)
- U+01CC (LATIN SMALL LETTER NJ)
- U+01F1 (LATIN CAPITAL LETTER DZ)
- U+01F2 (LATIN CAPITAL LETTER D WITH SMALL LETTER Z)
- U+01F3 (LATIN SMALL LETTER DZ)
-
- The script used was basically:
+/* This table was generated from the Unicode 5.0.0 spec. The table is
+ * generated by using the codes for conversion to lower case and for
+ * conversion to title case and upper case. A few code points have
+ * been moved in order to get reasonable conversions (if two code
+ * points are converted to the same value, the first one in this table
+ * wins). The code points that have been interchanged are:
+ * U+0345 (COMBINING GREEK YPOGEGRAMMENI) / U+03B9 (GREEK SMALL LETTER IOTA)
<-> U+0399 (GREEK CAPITAL LETTER IOTA)
+ * U+00B5 (MICRO SIGN) / U+03BC (GREEK SMALL LETTER MU) <-> U+039C (GREEK
CAPITAL LETTER MU)
+ * U+03C2 (GREEK SMALL LETTER FINAL SIGMA) / U+03C3 (GREEK SMALL LETTER SIGMA)
<-> U+3A3 (GREEK CAPITAL LETTER SIGMA)
+ *
+ * In addition, there are a few code points where there are different
+ * versions for upper case and title case. These had to be switched
+ * around a little so that the mappings are done sensibly.
+ *
+ * The following combinations are included in this order:
+ * lower case <-> title case
+ * lower case <- upper case
+ * upper case -> title case
+ * The conversion title case -> upper case was removed
+ *
+ * The relevant code points are:
+ * U+01C4 (LATIN CAPITAL LETTER DZ WITH CARON)
+ * U+01C5 (LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON)
+ * U+01C6 (LATIN SMALL LETTER DZ WITH CARON)
+ * U+01C7 (LATIN CAPITAL LETTER LJ)
+ * U+01C8 (LATIN CAPITAL LETTER L WITH SMALL LETTER J)
+ * U+01C9 (LATIN SMALL LETTER LJ)
+ * U+01CA (LATIN CAPITAL LETTER NJ)
+ * U+01CB (LATIN CAPITAL LETTER N WITH SMALL LETTER J)
+ * U+01CC (LATIN SMALL LETTER NJ)
+ * U+01F1 (LATIN CAPITAL LETTER DZ)
+ * U+01F2 (LATIN CAPITAL LETTER D WITH SMALL LETTER Z)
+ * U+01F3 (LATIN SMALL LETTER DZ)
+ *
+ * The script used was basically:
(cut -d\; -f1,14 UnicodeData.txt | sed -n 's/\(.*\);\(..*\)/\2;\1/p'
cut -d\; -f1,15 UnicodeData.txt | grep -v ';$'
cut -d\; -f1,13 UnicodeData.txt | grep -v ';$'
) | grep -v '^\([^ ]*\);\1$' | sort -t\; -u | sed
's/\(.*\);\(.*\)/{0x\1,0x\2,},/'
- with some hand munging afterward. The data file is UnicodeData.txt
- from http://www.unicode.org/.
+ * with some hand munging afterward. The data file is UnicodeData.txt
+ * from http://www.unicode.org/.
*/
struct UTF8_lower_upper {
unsigned int lower, upper;
@@ -1134,74 +1143,78 @@ strEpilogue(void)
return MAL_SUCCEED;
}
-#define UTF8_GETCHAR(X1,X2)\
- if (*X2 < 0x80) {\
- (X1) = *(X2)++;\
- } else if (*(X2) < 0xE0) {\
- (X1) = (*(X2)++ & 0x1F) << 6;\
- (X1) |= (*(X2)++ & 0x3F);\
- } else if (*(X2) < 0xF0) {\
- (X1) = (*(X2)++ & 0x0F) << 12;\
- (X1) |= (*(X2)++ & 0x3F) << 6;\
- (X1) |= (*(X2)++ & 0x3F);\
- } else if (*X2 < 0xF8) {\
- (X1) = (*(X2)++ & 0x07) << 18;\
- (X1) |= (*(X2)++ & 0x3F) << 12;\
- (X1) |= (*(X2)++ & 0x3F) << 6;\
- (X1) |= (*(X2)++ & 0x3F);\
- } else if (*X2 < 0xFC) {\
- (X1) = (*(X2)++ & 0x03) << 24;\
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
