Changeset: 32993207234e for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=32993207234e
Modified Files:
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/exports.stable.out
monetdb5/ChangeLog
monetdb5/modules/mal/pcre.c
monetdb5/modules/mal/pcre.mal
Branch: default
Log Message:
Implemented pcre.replace_first.
Many thanks to Roberto Cornacchia who actually did the implementation work.
diffs (truncated from 672 to 300 lines):
diff --git a/clients/Tests/MAL-signatures.stable.out
b/clients/Tests/MAL-signatures.stable.out
--- a/clients/Tests/MAL-signatures.stable.out
+++ b/clients/Tests/MAL-signatures.stable.out
@@ -8238,6 +8238,7 @@ Ready.
[ "batmtime", "sql_seconds", "command
batmtime.sql_seconds(d:bat[:daytime]):bat[:int] ",
"MTIMEdaytime_extract_sql_seconds_bulk;", "" ]
[ "batmtime", "year", "command batmtime.year(d:bat[:date]):bat[:int] ",
"MTIMEdate_extract_year_bulk;", "" ]
[ "batpcre", "replace", "command batpcre.replace(orig:bat[:str],
pat:str, repl:str, flag:str):bat[:str] ", "PCREreplace_bat_wrap;",
"" ]
+[ "batpcre", "replace_first", "command
batpcre.replace_first(orig:bat[:str], pat:str, repl:str, flag:str):bat[:str] ",
"PCREreplacefirst_bat_wrap;", "" ]
[ "batpyapi", "eval", "pattern batpyapi.eval(fptr:ptr, expr:str,
arg:any...):any... ", "PYAPI2PyAPIevalStd;", "Execute a simple Python
script value" ]
[ "batpyapi", "eval_aggr", "pattern batpyapi.eval_aggr(fptr:ptr, expr:str,
arg:any...):any... ", "PYAPI2PyAPIevalAggr;", "grouped aggregates through
Python" ]
[ "batpyapi", "eval_loader", "pattern batpyapi.eval_loader(fptr:ptr,
expr:str):any... ", "PYAPI2PyAPIevalLoader;", "loader functions
through Python" ]
@@ -10704,6 +10705,7 @@ Ready.
[ "pcre", "pcre_quote", "command pcre.pcre_quote(s:str):str ",
"PCREquote;", "Return a PCRE pattern string that matches the argument
exactly." ]
[ "pcre", "prelude", "command pcre.prelude():void ", "pcre_init;",
"Initialize pcre" ]
[ "pcre", "replace", "command pcre.replace(origin:str, pat:str,
repl:str, flags:str):str ", "PCREreplace_wrap;", "Replace _all_ matches of
\"pattern\" in \"origin_str\" with \"replacement\".\n\t Parameter \"flags\"
accept these flags: 'i', 'm', 's', and 'x'.\n\t 'e': if present, an empty
string is considered to be a valid match\n\t 'i': if present, the match
operates in case-insensitive mode.\n\t\tOtherwise, in case-sensitive mode.\n\t
'm': if present, the match operates in multi-line mode.\n\t 's': if present,
the match operates in \"dot-all\"\n\t The specifications of the flags can be
found in \"man pcreapi\"\n\t The flag letters may be repeated.\n\t No other
letters than 'e', 'i', 'm', 's' and 'x' are allowed in \"flags\".\n\t Returns
the replaced string, or if no matches found, the original string." ]
+[ "pcre", "replace_first", "command pcre.replace_first(origin:str,
pat:str, repl:str, flags:str):str ", "PCREreplace_wrap;", "Replace _the
first_ match of \"pattern\" in \"origin_str\" with \"replacement\".\n\t
Parameter \"flags\" accept these flags: 'i', 'm', 's', and 'x'.\n\t 'e': if
present, an empty string is considered to be a valid match\n\t 'i': if
present, the match operates in case-insensitive mode.\n\t\tOtherwise, in
case-sensitive mode.\n\t 'm': if present, the match operates in multi-line
mode.\n\t 's': if present, the match operates in \"dot-all\"\n\t The
specifications of the flags can be found in \"man pcreapi\"\n\t The flag
letters may be repeated.\n\t No other letters than 'e', 'i', 'm', 's' and 'x'
are allowed in \"flags\".\n\t Returns the replaced string, or if no matches
found, the original string." ]
[ "pcre", "sql2pcre", "command pcre.sql2pcre(pat:str, esc:str):str ",
"PCREsql2pcre;", "Convert a SQL like pattern with the given escape
character into a PCRE pattern." ]
[ "profiler", "cleanup", "command profiler.cleanup():void ",
"CMDcleanupTraces;", "Remove the temporary tables for profiling" ]
[ "profiler", "closestream", "command profiler.closestream():void ",
"CMDcloseProfilerStream;", "Stop offline proviling" ]
diff --git a/clients/Tests/MAL-signatures.stable.out.int128
b/clients/Tests/MAL-signatures.stable.out.int128
--- a/clients/Tests/MAL-signatures.stable.out.int128
+++ b/clients/Tests/MAL-signatures.stable.out.int128
@@ -11806,6 +11806,7 @@ Ready.
[ "batmtime", "sql_seconds", "command
batmtime.sql_seconds(d:bat[:daytime]):bat[:int] ",
"MTIMEdaytime_extract_sql_seconds_bulk;", "" ]
[ "batmtime", "year", "command batmtime.year(d:bat[:date]):bat[:int] ",
"MTIMEdate_extract_year_bulk;", "" ]
[ "batpcre", "replace", "command batpcre.replace(orig:bat[:str],
pat:str, repl:str, flag:str):bat[:str] ", "PCREreplace_bat_wrap;",
"" ]
+[ "batpcre", "replace_first", "command
batpcre.replace_first(orig:bat[:str], pat:str, repl:str, flag:str):bat[:str] ",
"PCREreplacefirst_bat_wrap;", "" ]
[ "batpyapi", "eval", "pattern batpyapi.eval(fptr:ptr, expr:str,
arg:any...):any... ", "PYAPI2PyAPIevalStd;", "Execute a simple Python
script value" ]
[ "batpyapi", "eval_aggr", "pattern batpyapi.eval_aggr(fptr:ptr, expr:str,
arg:any...):any... ", "PYAPI2PyAPIevalAggr;", "grouped aggregates through
Python" ]
[ "batpyapi", "eval_loader", "pattern batpyapi.eval_loader(fptr:ptr,
expr:str):any... ", "PYAPI2PyAPIevalLoader;", "loader functions
through Python" ]
@@ -14915,6 +14916,7 @@ Ready.
[ "pcre", "pcre_quote", "command pcre.pcre_quote(s:str):str ",
"PCREquote;", "Return a PCRE pattern string that matches the argument
exactly." ]
[ "pcre", "prelude", "command pcre.prelude():void ", "pcre_init;",
"Initialize pcre" ]
[ "pcre", "replace", "command pcre.replace(origin:str, pat:str,
repl:str, flags:str):str ", "PCREreplace_wrap;", "Replace _all_ matches of
\"pattern\" in \"origin_str\" with \"replacement\".\n\t Parameter \"flags\"
accept these flags: 'i', 'm', 's', and 'x'.\n\t 'e': if present, an empty
string is considered to be a valid match\n\t 'i': if present, the match
operates in case-insensitive mode.\n\t\tOtherwise, in case-sensitive mode.\n\t
'm': if present, the match operates in multi-line mode.\n\t 's': if present,
the match operates in \"dot-all\"\n\t The specifications of the flags can be
found in \"man pcreapi\"\n\t The flag letters may be repeated.\n\t No other
letters than 'e', 'i', 'm', 's' and 'x' are allowed in \"flags\".\n\t Returns
the replaced string, or if no matches found, the original string." ]
+[ "pcre", "replace_first", "command pcre.replace_first(origin:str,
pat:str, repl:str, flags:str):str ", "PCREreplace_wrap;", "Replace _the
first_ match of \"pattern\" in \"origin_str\" with \"replacement\".\n\t
Parameter \"flags\" accept these flags: 'i', 'm', 's', and 'x'.\n\t 'e': if
present, an empty string is considered to be a valid match\n\t 'i': if
present, the match operates in case-insensitive mode.\n\t\tOtherwise, in
case-sensitive mode.\n\t 'm': if present, the match operates in multi-line
mode.\n\t 's': if present, the match operates in \"dot-all\"\n\t The
specifications of the flags can be found in \"man pcreapi\"\n\t The flag
letters may be repeated.\n\t No other letters than 'e', 'i', 'm', 's' and 'x'
are allowed in \"flags\".\n\t Returns the replaced string, or if no matches
found, the original string." ]
[ "pcre", "sql2pcre", "command pcre.sql2pcre(pat:str, esc:str):str ",
"PCREsql2pcre;", "Convert a SQL like pattern with the given escape
character into a PCRE pattern." ]
[ "profiler", "cleanup", "command profiler.cleanup():void ",
"CMDcleanupTraces;", "Remove the temporary tables for profiling" ]
[ "profiler", "closestream", "command profiler.closestream():void ",
"CMDcloseProfilerStream;", "Stop offline proviling" ]
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -1614,6 +1614,8 @@ str PCREpatindex(int *ret, const str *pa
str PCREquote(str *r, const str *v);
str PCREreplace_bat_wrap(bat *res, const bat *or, const str *pat, const str
*repl, const str *flags);
str PCREreplace_wrap(str *res, const str *or, const str *pat, const str *repl,
const str *flags);
+str PCREreplacefirst_bat_wrap(bat *res, const bat *or, const str *pat, const
str *repl, const str *flags);
+str PCREreplacefirst_wrap(str *res, const str *or, const str *pat, const str
*repl, const str *flags);
str PCREsql2pcre(str *ret, const str *pat, const str *esc);
str PROFexitClient(Client c);
str PROFinitClient(Client c);
diff --git a/monetdb5/ChangeLog b/monetdb5/ChangeLog
--- a/monetdb5/ChangeLog
+++ b/monetdb5/ChangeLog
@@ -1,6 +1,10 @@
# ChangeLog file for MonetDB5
# This file is updated with Maddlog
+* Wed Jan 24 2018 Sjoerd Mullender <[email protected]>
+- Implemented function pcre.replace_first which is like pcre.replace,
+ except it only replaces the first match.
+
* Wed Aug 23 2017 Martin Kersten <[email protected]>
- The EXPLAIN command now shows all the MAL type resolutions, because in
general users may not be aware of their signatures. It also simplifies
diff --git a/monetdb5/modules/mal/pcre.c b/monetdb5/modules/mal/pcre.c
--- a/monetdb5/modules/mal/pcre.c
+++ b/monetdb5/modules/mal/pcre.c
@@ -1,3 +1,4 @@
+#pragma GCC optimize ("-fno-stack-protector")
/*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -43,6 +44,8 @@ mal_export str PCREindex(int *ret, const
mal_export str PCREpatindex(int *ret, const str *pat, const str *val);
mal_export str PCREreplace_wrap(str *res, const str *or, const str *pat, const
str *repl, const str *flags);
mal_export str PCREreplace_bat_wrap(bat *res, const bat *or, const str *pat,
const str *repl, const str *flags);
+mal_export str PCREreplacefirst_wrap(str *res, const str *or, const str *pat,
const str *repl, const str *flags);
+mal_export str PCREreplacefirst_bat_wrap(bat *res, const bat *or, const str
*pat, const str *repl, const str *flags);
mal_export str PCREsql2pcre(str *ret, const str *pat, const str *esc);
mal_export str PCRElike3(bit *ret, const str *s, const str *pat, const str
*esc);
@@ -549,22 +552,28 @@ re_likeselect(BAT **bnp, BAT *b, BAT *s,
throw(MAL, "pcre.likeselect", OPERATION_FAILED);
}
-#define MAX_NR_CAPTURES 1024 /* Maximal number of captured substrings in one
original string */
+#define MAX_NR_MATCHES 1024 /* Maximal number of matches in one original
string */
+#define MAX_NR_CAPTURES 10 /* Maximal number of captured substrings per match
in one original string */
static str
-pcre_replace(str *res, const char *origin_str, const char *pattern, const char
*replacement, const char *flags)
+pcre_replace(str *res, const char *origin_str, const char *pattern,
+ const char *replacement, const char *flags, bool
global)
{
#ifdef HAVE_LIBPCRE
const char *err_p = NULL;
pcre *pcre_code = NULL;
pcre_extra *extra;
char *tmpres;
- int i, j, k, len, errpos = 0, offset = 0;
+ char tmpbackref[4]; /* enough for 0 to 999 */
+ int i, j, k, m, len, errpos = 0, offset = 0;
int compile_options = PCRE_UTF8, exec_options = PCRE_NOTEMPTY;
int *ovector, ovecsize;
int len_origin_str = (int) strlen(origin_str);
int len_replacement = (int) strlen(replacement);
- int capture_offsets[MAX_NR_CAPTURES * 2], ncaptures = 0, len_del = 0;
+ int len_replacement_with_backrefs = 0;
+ int nmatches = 0;
+ int capture_offsets[MAX_NR_MATCHES][MAX_NR_CAPTURES * 2], ncaptures =
0, len_del = 0;
+ int backrefs[MAX_NR_CAPTURES] , backref_offsets[MAX_NR_CAPTURES * 2],
nbackrefs = 0;
while (*flags) {
switch (*flags) {
@@ -584,13 +593,17 @@ pcre_replace(str *res, const char *origi
compile_options |= PCRE_EXTENDED;
break;
default:
- throw(MAL, "pcre.replace", ILLEGAL_ARGUMENT ":
unsupported flag character '%c'\n", *flags);
+ throw(MAL, global ? "pcre.replace" :
"pcre.replace_first",
+ ILLEGAL_ARGUMENT ": unsupported flag
character '%c'\n",
+ *flags);
}
flags++;
}
if ((pcre_code = pcre_compile(pattern, compile_options, &err_p,
&errpos, NULL)) == NULL) {
- throw(MAL, "pcre.replace", OPERATION_FAILED ": pcre compile of
pattern (%s) failed at %d with\n'%s'.\n", pattern, errpos, err_p);
+ throw(MAL, global ? "pcre.replace" : "pcre.replace_first",
+ OPERATION_FAILED ": pcre compile of pattern (%s)
failed at %d with\n'%s'.\n",
+ pattern, errpos, err_p);
}
/* Since the compiled pattern is going to be used several times, it is
@@ -600,59 +613,110 @@ pcre_replace(str *res, const char *origi
extra = pcre_study(pcre_code, 0, &err_p);
if (err_p != NULL) {
pcre_free(pcre_code);
- throw(MAL, "pcre.replace", OPERATION_FAILED ": pcre study of
pattern (%s) failed with '%s'.\n", pattern, err_p);
+ throw(MAL, global ? "pcre.replace" : "pcre.replace_first",
+ OPERATION_FAILED ": pcre study of pattern (%s) failed
with '%s'.\n",
+ pattern, err_p);
}
pcre_fullinfo(pcre_code, extra, PCRE_INFO_CAPTURECOUNT, &i);
ovecsize = (i + 1) * 3;
if ((ovector = (int *) GDKmalloc(sizeof(int) * ovecsize)) == NULL) {
pcre_free_study(extra);
pcre_free(pcre_code);
- throw(MAL, "pcre_replace", SQLSTATE(HY001) MAL_MALLOC_FAIL);
+ throw(MAL, global ? "pcre.replace" : "pcre.replace_first",
+ SQLSTATE(HY001) MAL_MALLOC_FAIL);
}
+ /* The pattern can match multiple substrings,
+ * and each substring can contain multiple captures.
+ * The first capture is always the entire matching substring.
+ */
i = 0;
do {
j = pcre_exec(pcre_code, extra, origin_str, len_origin_str,
offset, exec_options, ovector,
ovecsize);
if (j > 0) {
- capture_offsets[i] = ovector[0];
- capture_offsets[i+1] = ovector[1];
- ncaptures++;
- i += 2;
- len_del += (ovector[1] - ovector[0]);
+ ncaptures = 0;
+ for (i=0; (i < j) && (ncaptures < MAX_NR_CAPTURES);
i++) {
+ capture_offsets[nmatches][i*2] = ovector[i*2];
+ capture_offsets[nmatches][i*2+1] =
ovector[i*2+1];
+ ncaptures++;
+ }
offset = ovector[1];
+ len_del += ovector[1] - ovector[0];
+ nmatches++;
}
- } while (j > 0 && offset < len_origin_str && ncaptures <
MAX_NR_CAPTURES);
+ } while (j > 0 && global && offset < len_origin_str && ncaptures <
MAX_NR_CAPTURES);
pcre_free_study(extra);
+ GDKfree(ovector);
- if (ncaptures > 0) {
- tmpres = GDKmalloc(len_origin_str - len_del + (len_replacement
* ncaptures) + 1);
- if (tmpres) {
- j = k = 0;
+ if (nmatches > 0) {
+ /* identify back references in the replacement string */
+ for (i=0; i<len_replacement; i++) {
+ if (replacement[i] == '$' || replacement[i] == '\\') {
+ if (i>0 && replacement[i-1] != '\\') {
+ for (k=0; i+k+1<len_replacement; k++) {
+ if (!isdigit((unsigned
char)replacement[i+k+1]))
+ break;
+ }
+ if (k>0) {
+ strncpy(tmpbackref,
replacement+i+1, k);
+ tmpbackref[i+1+k] = '\0';
+ backrefs[nbackrefs] =
atoi(tmpbackref);
+ backref_offsets[nbackrefs*2] =
i;
+ backref_offsets[nbackrefs*2+1]
= i+k+1;
+ nbackrefs++;
+ }
+ }
+ }
+ }
- /* possibly copy the substring before the first captured
- * substring */
- strncpy(tmpres, origin_str, capture_offsets[j]);
- k = capture_offsets[j];
- j++;
+ /* length of all replacements including backreferences */
+ len_replacement_with_backrefs = len_replacement * nmatches;
+ for (m=0; m<nmatches; m++) {
+ for(i=0; i<nbackrefs; i++) {
+ len_replacement_with_backrefs -=
backref_offsets[i*2+1] - backref_offsets[i*2];
+ if (backrefs[i] < ncaptures) {
+ len_replacement_with_backrefs +=
capture_offsets[m][backrefs[i]*2+1] - capture_offsets[m][backrefs[i]*2];
+ }
+ }
+ }
+ tmpres = GDKmalloc(len_origin_str - len_del +
len_replacement_with_backrefs + 1);
+ if (tmpres) {
+ k=0;
+ for (m=0; m<nmatches; m++) {
+ /* copy the substring before the matching
substring */
+ len = capture_offsets[m][0] -
(m>0)*capture_offsets[m-1][1];
+ strncpy(tmpres+k,
origin_str+(m>0)*capture_offsets[m-1][1], len);
+ k += len;
- for (i = 0; i < ncaptures - 1; i++) {
- strncpy(tmpres+k, replacement, len_replacement);
- k += len_replacement;
- /* copy the substring between two captured
substrings */
- len = capture_offsets[j+1] - capture_offsets[j];
- strncpy(tmpres+k,
origin_str+capture_offsets[j], len);
- k += len;
- j += 2;
+ /* replace match */
+ if (nbackrefs > 0) {
+ for(i=0; i<nbackrefs; i++) {
+ /* copy the replacement
substring before the backreference */
+ len = backref_offsets[i*2] -
(i>0)*backref_offsets[(i-1)*2+1];
+ strncpy(tmpres+k,
replacement+(i>0)*backref_offsets[(i-1)*2+1], len);
+ k += len;
+ /* copy backreference */
+ if (backrefs[i] < ncaptures) {
+ len =
capture_offsets[m][backrefs[i]*2+1] - capture_offsets[m][backrefs[i]*2];
+ strncpy(tmpres+k,
origin_str+capture_offsets[m][backrefs[i]*2], len);
+ k += len;
+ }
+ }
+ /* copy the replacement string after
last backreference */
+ len = len_replacement -
backref_offsets[(i-1)*2+1];
+ strncpy(tmpres+k,
replacement+backref_offsets[(i-1)*2+1], len);
+ k += len;
+ } else {
+ /* no backreferences, copy the entire
replacement */
+ strncpy(tmpres+k, replacement,
len_replacement);
+ k += len_replacement;
+ }
}
- /* replace the last captured substring */
- strncpy(tmpres+k, replacement, len_replacement);
- k += len_replacement;
- /* possibly copy the substring after the last captured
substring */
- len = len_origin_str - capture_offsets[j];
- strncpy(tmpres+k, origin_str+capture_offsets[j], len);
- k += len;
+ /* copy the substring after the last matching substring
*/
+ strncpy(tmpres+k, origin_str+capture_offsets[m-1][1],
len_origin_str - capture_offsets[m-1][1]);
+ k += len_origin_str - capture_offsets[m-1][1];
tmpres[k] = '\0';
}
} else {
@@ -661,9 +725,9 @@ pcre_replace(str *res, const char *origi
}
pcre_free(pcre_code);
- GDKfree(ovector);
if (tmpres == NULL)
- throw(MAL, "pcre.replace", SQLSTATE(HY001) MAL_MALLOC_FAIL);
+ throw(MAL, global ? "pcre.replace" : "pcre.replace_first",
+ SQLSTATE(HY001) MAL_MALLOC_FAIL);
*res = tmpres;
return MAL_SUCCEED;
#else
@@ -672,17 +736,22 @@ pcre_replace(str *res, const char *origi
(void) pattern;
(void) replacement;
(void) flags;
- throw(MAL, "pcre.replace", "Database was compiled without PCRE
support.");
+ (void) global;
+ throw(MAL, global ? "pcre.replace" : "pcre.replace_first",
+ "Database was compiled without PCRE support.");
#endif
}
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
