Changeset: 33cfedf29e90 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=33cfedf29e90
Added Files:
sql/test/copy/Tests/incorrect_columns.stable.err
sql/test/copy/Tests/incorrect_columns.stable.out
Modified Files:
monetdb5/modules/mal/tablet.c
sql/test/copy/Tests/incorrect_columns.sql
sql/test/copy/Tests/key_copy.sql
sql/test/copy/Tests/key_copy.stable.err
Branch: Jul2015
Log Message:
Fixing errors in Best Effort loading
- properly handle the case of less/more fields in a line
The line is added to the reject list
- properly construct the rejects message
- continue parsing lines in best-effort mode
diffs (truncated from 397 to 300 lines):
diff --git a/monetdb5/modules/mal/tablet.c b/monetdb5/modules/mal/tablet.c
--- a/monetdb5/modules/mal/tablet.c
+++ b/monetdb5/modules/mal/tablet.c
@@ -838,14 +838,14 @@ mycpstr(char *t, const char *s)
}
static str
-SQLload_error(READERtask *task, lng idx)
+SQLload_error(READERtask *task, lng idx, unsigned int attrs)
{
str line;
char *s;
size_t sz = 0;
unsigned int i;
- for (i = 0; i < task->as->nr_attrs; i++) {
+ for (i = 0; i < attrs; i++) {
if (task->fields[i][idx])
sz += mystrlen(task->fields[i][idx]);
sz += task->seplen;
@@ -856,10 +856,10 @@ SQLload_error(READERtask *task, lng idx)
tablet_error(task, idx, int_nil, "SQLload malloc error",
"SQLload_error");
return 0;
}
- for (i = 0; i < task->as->nr_attrs; i++) {
+ for (i = 0; i < attrs; i++) {
if (task->fields[i][idx])
s = mycpstr(s, task->fields[i][idx]);
- if (i < task->as->nr_attrs - 1)
+ if (i < attrs - 1)
s = mycpstr(s, task->csep);
}
strcat(line, task->rsep);
@@ -893,7 +893,7 @@ SQLinsert_val(READERtask *task, int col,
if (adt == NULL) {
lng row = task->cnt + idx + 1;
snprintf(buf, BUFSIZ, "'%s' expected", fmt->type);
- err = SQLload_error(task, idx);
+ err = SQLload_error(task, idx, task->as->nr_attrs);
if (task->rowerror) {
MT_lock_set(&errorlock, "insert_val");
col++;
@@ -925,7 +925,7 @@ SQLinsert_val(READERtask *task, int col,
BUNappend(task->cntxt->error_row, &row, FALSE);
BUNappend(task->cntxt->error_fld, &col, FALSE);
BUNappend(task->cntxt->error_msg, "insert failed", FALSE);
- err = SQLload_error(task, idx);
+ err = SQLload_error(task, idx,task->as->nr_attrs);
BUNappend(task->cntxt->error_input, err, FALSE);
GDKfree(err);
task->rowerror[row - 1]++;
@@ -953,8 +953,10 @@ SQLworker_column(READERtask *task, int c
MT_lock_unset(&mal_copyLock, "tablet insert value");
for (i = 0; i < task->top[task->cur]; i++) {
- if (!fmt[col].skip && SQLinsert_val(task, col, i) < 0)
- return -1;
+ if (!fmt[col].skip && SQLinsert_val(task, col, i) < 0){
+ if(task->besteffort == 0)
+ return -1;
+ }
}
return 0;
@@ -984,10 +986,11 @@ SQLload_parse_line(READERtask *task, int
//mnstr_printf(GDKout, "#SQL break line id %d state %d\n%s", task->id,
idx, line);
#endif
assert(idx < task->top[task->cur]);
+ assert(line);
errmsg[0] = 0;
if (task->quote || task->seplen != 1) {
- for (i = 0; i < as->nr_attrs; i++) {
+ for (i = 0; *line && i < as->nr_attrs; i++) {
task->fields[i][idx] = line;
/* recognize fields starting with a quote, keep them */
if (*line == task->quote) {
@@ -1000,7 +1003,7 @@ SQLload_parse_line(READERtask *task, int
mnstr_printf(GDKout, "after #1 %s\n", s);
#endif
if (!line) {
- errline = SQLload_error(task,
task->top[task->cur]);
+ errline = SQLload_error(task, idx, i+1);
snprintf(errmsg, BUFSIZ, "Quote (%c)
missing", task->quote);
tablet_error(task, idx, (int) i,
errmsg, errline);
GDKfree(errline);
@@ -1023,8 +1026,8 @@ SQLload_parse_line(READERtask *task, int
/* not enough fields */
if (i < as->nr_attrs - 1) {
- errline = SQLload_error(task,
task->top[task->cur]);
- snprintf(errmsg, BUFSIZ, "Separator missing
'%s' ", fmt->sep);
+ errline = SQLload_error(task, idx, i+1);
+ snprintf(errmsg, BUFSIZ, "Column value "BUNFMT"
missing ", i+1);
tablet_error(task, idx, (int) i, errmsg,
errline);
GDKfree(errline);
error++;
@@ -1033,18 +1036,14 @@ SQLload_parse_line(READERtask *task, int
for (; i < as->nr_attrs; i++)
task->fields[i][idx] = NULL;
i--;
- }
+ }
endoffieldcheck:
;
/* check for user defined NULL string */
if (!fmt->skip && fmt->nullstr && task->fields[i][idx]
&& strncasecmp(task->fields[i][idx], fmt->nullstr, fmt->null_length + 1) == 0)
task->fields[i][idx] = 0;
}
-#ifdef _DEBUG_TABLET_
- if (error)
- mnstr_printf(GDKout, "#line break failed %d:%s\n", idx,
line);
-#endif
- return error ? -1 : 0;
+ goto endofline;
}
assert(!task->quote);
assert(task->seplen == 1);
@@ -1068,8 +1067,8 @@ SQLload_parse_line(READERtask *task, int
#endif
/* not enough fields */
if (i < as->nr_attrs - 1) {
- errline = SQLload_error(task, task->top[task->cur]);
- snprintf(errmsg, BUFSIZ, "Separator missing '%s' ",
fmt->sep);
+ errline = SQLload_error(task, idx,i+1);
+ snprintf(errmsg, BUFSIZ, "Column value "BUNFMT"
missing",i+1);
tablet_error(task, idx, (int) i, errmsg, errline);
GDKfree(errline);
error++;
@@ -1085,6 +1084,15 @@ SQLload_parse_line(READERtask *task, int
task->fields[i][idx] = 0;
}
}
+endofline:
+ /* check for too many values as well*/
+ if (*line && i == as->nr_attrs) {
+ errline = SQLload_error(task, idx, task->as->nr_attrs);
+ snprintf(errmsg, BUFSIZ, "Leftover data '%s'",line);
+ tablet_error(task, idx, (int) i, errmsg, errline);
+ GDKfree(errline);
+ error++;
+ }
#ifdef _DEBUG_TABLET_
if (error)
mnstr_printf(GDKout, "#line break failed %d:%s\n", idx, line);
@@ -1125,8 +1133,10 @@ SQLworker(void *arg)
for (j = piece * task->id; j < task->top[task->cur] &&
j < piece * (task->id +1); j++)
if (task->lines[task->cur][j]) {
if (SQLload_parse_line(task, j) < 0) {
- task->error++;
- break;
+ task->errorcnt++;
+ // early break unless best
effort
+ if(task->besteffort == 0)
+ break;
}
}
task->wtime = GDKusec() - t0;
@@ -1550,7 +1560,7 @@ SQLproducer(void *p)
BUN
SQLload_file(Client cntxt, Tablet *as, bstream *b, stream *out, char *csep,
char *rsep, char quote, lng skip, lng maxrow, int best)
{
- BUN cnt = 0, cntstart = 0;
+ BUN cnt = 0, cntstart = 0, leftover = 0;
int res = 0; /* < 0: error, > 0: success, == 0: continue
processing */
int j;
BUN i, attr;
@@ -1794,7 +1804,7 @@ SQLload_file(Client cntxt, Tablet *as, b
#define trimerrors(TYPE)
\
do {
\
TYPE *src, *dst;
\
- BUN leftover= BATcount(task->as->format[attr].c);
\
+ leftover= BATcount(task->as->format[attr].c);
\
limit = leftover - cntstart;
\
dst =src= (TYPE *)
BUNtloc(task->as->format[attr].ci,cntstart); \
for(j = 0; j < (int) limit; j++, src++){
\
@@ -1813,8 +1823,14 @@ SQLload_file(Client cntxt, Tablet *as, b
#endif
if (best && BATcount(as->format[0].c)) {
BUN limit;
+ int width;
+
for (attr = 0; attr < as->nr_attrs; attr++) {
- switch (ATOMsize(as->format[attr].c->ttype)) {
+ if( as->format[attr].c->ttype == TYPE_str)
+ width = as->format[attr].c->T->width;
+ else
+ width =
ATOMsize(as->format[attr].c->ttype);
+ switch (width){
case 1:
trimerrors(unsigned char);
break;
diff --git a/sql/test/copy/Tests/incorrect_columns.sql
b/sql/test/copy/Tests/incorrect_columns.sql
--- a/sql/test/copy/Tests/incorrect_columns.sql
+++ b/sql/test/copy/Tests/incorrect_columns.sql
@@ -1,12 +1,14 @@
-- test the situation where the number of column values are incorrect
create table t_columns(i int, t string);
-copy 5 records into t_columns from stdin delimiters ',','\n' best effort;
+copy 7 records into t_columns from stdin delimiters ',','\n' best effort;
1,hello
2
no tag
-3,too much, data
+3,too much,xyz
4,world
+5,wereld
+6,maan
select * from t_columns;
diff --git a/sql/test/copy/Tests/incorrect_columns.stable.err
b/sql/test/copy/Tests/incorrect_columns.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/test/copy/Tests/incorrect_columns.stable.err
@@ -0,0 +1,37 @@
+stderr of test 'incorrect_columns` in directory 'sql/test/copy` itself:
+
+
+# 08:11:47 >
+# 08:11:47 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"mapi_open=true" "--set" "mapi_port=37579" "--set"
"mapi_usock=/var/tmp/mtest-26664/.s.monetdb.37579" "--set" "monet_prompt="
"--forcemito" "--set" "mal_listing=2"
"--dbpath=/export/scratch1/mk/Jul2015//Linux/var/MonetDB/mTests_sql_test_copy"
"--set" "mal_listing=0" "--set" "embedded_r=yes"
+# 08:11:47 >
+
+# builtin opt gdk_dbpath =
/export/scratch1/mk/Jul2015//Linux/var/monetdb5/dbfarm/demo
+# builtin opt gdk_debug = 0
+# builtin opt gdk_vmtrim = no
+# builtin opt monet_prompt = >
+# builtin opt monet_daemon = no
+# builtin opt mapi_port = 50000
+# builtin opt mapi_open = false
+# builtin opt mapi_autosense = false
+# builtin opt sql_optimizer = default_pipe
+# builtin opt sql_debug = 0
+# cmdline opt gdk_nr_threads = 0
+# cmdline opt mapi_open = true
+# cmdline opt mapi_port = 37579
+# cmdline opt mapi_usock = /var/tmp/mtest-26664/.s.monetdb.37579
+# cmdline opt monet_prompt =
+# cmdline opt mal_listing = 2
+# cmdline opt gdk_dbpath =
/export/scratch1/mk/Jul2015//Linux/var/MonetDB/mTests_sql_test_copy
+# cmdline opt mal_listing = 0
+# cmdline opt embedded_r = yes
+# cmdline opt gdk_debug = 536870922
+
+# 08:11:47 >
+# 08:11:47 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e"
"--host=/var/tmp/mtest-26664" "--port=37579"
+# 08:11:47 >
+
+
+# 16:14:08 >
+# 16:14:08 > "Done."
+# 16:14:08 >
+
diff --git a/sql/test/copy/Tests/incorrect_columns.stable.out
b/sql/test/copy/Tests/incorrect_columns.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/test/copy/Tests/incorrect_columns.stable.out
@@ -0,0 +1,99 @@
+stdout of test 'incorrect_columns` in directory 'sql/test/copy` itself:
+
+
+# 08:11:47 >
+# 08:11:47 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"mapi_open=true" "--set" "mapi_port=37579" "--set"
"mapi_usock=/var/tmp/mtest-26664/.s.monetdb.37579" "--set" "monet_prompt="
"--forcemito" "--set" "mal_listing=2"
"--dbpath=/export/scratch1/mk/Jul2015//Linux/var/MonetDB/mTests_sql_test_copy"
"--set" "mal_listing=0" "--set" "embedded_r=yes"
+# 08:11:47 >
+
+# MonetDB 5 server v11.21.2
+# This is an unreleased version
+# Serving database 'mTests_sql_test_copy', using 8 threads
+# Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs and 128bit
integers dynamically linked
+# Found 15.590 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2015 MonetDB B.V., all rights reserved
+# Visit http://www.monetdb.org/ for further information
+# Listening for connection requests on mapi:monetdb://vienna.ins.cwi.nl:37579/
+# Listening for UNIX domain connection requests on
mapi:monetdb:///var/tmp/mtest-26664/.s.monetdb.37579
+# MonetDB/GIS module loaded
+# Start processing logs sql/sql_logs version 52200
+# Finished processing logs sql/sql_logs
+# MonetDB/SQL module loaded
+# MonetDB/R module loaded
+
+Ready.
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+# loading sql script: 15_querylog.sql
+# loading sql script: 16_tracelog.sql
+# loading sql script: 17_temporal.sql
+# loading sql script: 20_vacuum.sql
+# loading sql script: 21_dependency_functions.sql
+# loading sql script: 22_clients.sql
+# loading sql script: 23_skyserver.sql
+# loading sql script: 24_zorder.sql
+# loading sql script: 25_debug.sql
+# loading sql script: 26_sysmon.sql
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
