Changeset: c510f63ad670 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c510f63ad670
Modified Files:
monetdb5/modules/mal/tablet.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_result.c
sql/test/BugTracker-2008/Tests/copy-format.SF-2468513.stable.err
sql/test/BugTracker-2008/Tests/varchar.SF-2471371.stable.err
sql/test/BugTracker-2009/Tests/copy_into_error_off_by_one.SF-2833696.stable.err
sql/test/BugTracker-2009/Tests/copy_multiple_files.SF-2902320.stable.err
sql/test/BugTracker-2009/Tests/copy_multiple_files.SF-2902320.stable.out
sql/test/BugTracker-2010/Tests/copy-into-too-long-string.Bug-2358.stable.err
sql/test/BugTracker-2010/Tests/error-truncated.Bug-2615.stable.err
sql/test/BugTracker-2010/Tests/incomplete-utf8-sequence.Bug-2575.stable.err
sql/test/BugTracker/Tests/copy_decimal.SF-1582957.stable.err
sql/test/BugTracker/Tests/copy_overflow_null_bug.SF-1842550.stable.err
sql/test/leaks/Tests/check1_bam.stable.out.int128
sql/test/leaks/Tests/check2_bam.stable.out.int128
sql/test/leaks/Tests/check3_bam.stable.out.int128
sql/test/leaks/Tests/check4_bam.stable.out.int128
sql/test/leaks/Tests/check5_bam.stable.out.int128
Branch: resultset
Log Message:
Backward compatibility
Not using BEST EFFORT should lead to old behavior,
stop at first error.
diffs (truncated from 1048 to 300 lines):
diff --git a/monetdb5/modules/mal/tablet.c b/monetdb5/modules/mal/tablet.c
--- a/monetdb5/modules/mal/tablet.c
+++ b/monetdb5/modules/mal/tablet.c
@@ -748,7 +748,7 @@ SQLload_error(READERtask *task, lng idx)
* If the string starts with the quote identified from SQL, we locate the tail
* and interpret the body.
*/
-static inline void
+static inline int
SQLinsert_val(READERtask *task, int col, lng idx)
{
Column *fmt = task->as->format+col;
@@ -759,6 +759,7 @@ SQLinsert_val(READERtask *task, int col,
char quote = task->quote;
ptr key = 0;
char *err;
+ int ret =0;
/* include testing on the terminating null byte !! */
if (fmt->nullstr && strncasecmp(s, fmt->nullstr, fmt->null_length + 1)
== 0) {
@@ -786,18 +787,24 @@ SQLinsert_val(READERtask *task, int col,
if (adt == NULL) {
BUN row = BATcount(fmt->c);
- snprintf(buf, BUFSIZ, "'%s' expected ", fmt->type);
+ snprintf(buf, BUFSIZ, "'%s' expected", fmt->type);
err = SQLload_error(task,idx);
if( task->rowerror){
MT_lock_set(&errorlock, "insert_val");
+ row++;
+ col++;
BUNappend(task->cntxt->error_row, &row, FALSE);
BUNappend(task->cntxt->error_fld, &col, FALSE);
BUNappend(task->cntxt->error_msg, buf, FALSE);
BUNappend(task->cntxt->error_input, err, FALSE);
+ snprintf(buf, BUFSIZ, "line "BUNFMT" field %d '%s'
expected in '%s'",row, col, fmt->type, s);
+ if (task->as->error == NULL && (task->as->error =
GDKstrdup(buf)) == NULL)
+ task->as->error = M5OutOfMemory;
task->rowerror[(int)row]++;
task->errorcnt++;
MT_lock_unset(&errorlock, "insert_val");
}
+ ret = -1 * (task->besteffort ==0);
GDKfree(err);
/* replace it with a nil */
adt = fmt->nildata;
@@ -805,7 +812,7 @@ SQLinsert_val(READERtask *task, int col,
}
/* key may be NULL but that's not a problem, as long as we have void */
bunfastins(fmt->c, key, adt);
- return ;
+ return ret;
bunins_failed:
if( task->rowerror){
BUN row = BATcount(fmt->c);
@@ -818,6 +825,7 @@ SQLinsert_val(READERtask *task, int col,
task->errorcnt++;
MT_lock_unset(&errorlock, "insert_val");
}
+ return -1;
}
static int
@@ -839,9 +847,8 @@ SQLworker_column(READERtask *task, int c
for (i = 0; i < task->next; i++){
if (task->fields[col][i])
- SQLinsert_val(task, col, i);
- if( task->errorcnt && task->besteffort == 0)
- break;
+ if( SQLinsert_val(task, col, i) < 0)
+ return -1;
}
return 0;
@@ -1065,7 +1072,7 @@ BUN
SQLload_file(Client cntxt, Tablet *as, bstream *b, stream *out, char *csep,
char *rsep, char quote, lng skip, lng maxrow, int best)
{
char *s, *e, *end;
- BUN cnt = 0;
+ BUN cnt = 0, cntstart=0;
int res = 0; /* < 0: error, > 0: success, ==
0: continue processing */
int j;
BUN i;
@@ -1089,7 +1096,7 @@ SQLload_file(Client cntxt, Tablet *as, b
/* create the reject tables */
MT_lock_set(&mal_contextLock, "copy.initialization");
if( task->cntxt->error_row == NULL){
- task->cntxt->error_row = BATnew(TYPE_void, TYPE_oid,0,TRANSIENT);
+ task->cntxt->error_row = BATnew(TYPE_void, TYPE_lng,0,TRANSIENT);
BATseqbase(task->cntxt->error_row,0);
task->cntxt->error_fld = BATnew(TYPE_void, TYPE_int,0,TRANSIENT);
BATseqbase(task->cntxt->error_fld,0);
@@ -1231,6 +1238,8 @@ SQLload_file(Client cntxt, Tablet *as, b
}
while ((task->b->pos < task->b->len || !task->b->eof) && cnt < (BUN)
maxrow && res == 0) {
+ // track how many elements are in the aggregated BATs
+ cntstart = BATcount(task->as->format[0].c);
if (task->errbuf && task->errbuf[0]) {
msg = catchKernelException(cntxt, msg);
if (msg) {
@@ -1434,6 +1443,11 @@ SQLload_file(Client cntxt, Tablet *as, b
}
/* trim the BATs discarding error tuples */
+ if ( task->errorcnt && task->as->error == NULL && best == 0){
+ task->as->error = M5OutOfMemory;
+ res = -1;
+ goto bailout;
+ }
/*
for( attr=0; attr < task->nr_attrs; attr++)
switch(){
@@ -1477,7 +1491,8 @@ SQLload_file(Client cntxt, Tablet *as, b
res = -1;
}
// check for a possible vacuum of error rows
- if( task->errorcnt ){
+ if( res >= 0 && task->errorcnt ){
+ (void) cntstart;
// compress all columns
}
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -2342,7 +2342,7 @@ mvc_result_set_wrap( Client cntxt, MalBl
bid = *getArgReference_bat(stk,pci,6);
b = BATdescriptor(bid);
if ( b == NULL)
- throw(MAL,"sql.resultset","failed to access order column");
+ throw(MAL,"sql.resultset","Failed to access order column");
res = *res_id = mvc_result_table(m, pci->argc - (pci->retc + 5), 1, b);
if (res < 0)
msg = createException(SQL, "sql.resultSet", "failed");
@@ -2369,7 +2369,7 @@ mvc_result_set_wrap( Client cntxt, MalBl
tpename = BUNtail(itertpe,o);
b = BATdescriptor(bid);
if ( b == NULL)
- msg= createException(MAL,"sql.resultset","failed to
access result column");
+ msg= createException(MAL,"sql.resultset","Failed to
access result column");
else
if (mvc_result_column(m, tblname, colname, tpename, *digits++,
*scaledigits++, b))
msg = createException(SQL, "sql.resultset",
"mvc_result_column failed");
@@ -2428,7 +2428,7 @@ mvc_export_table_wrap( Client cntxt, Mal
bid = *getArgReference_bat(stk,pci,12);
order = BATdescriptor(bid);
if ( order == NULL)
- throw(MAL,"sql.resultset","failed to access order column");
+ throw(MAL,"sql.resultset","Failed to access order column");
res = *res_id = mvc_result_table(m, pci->argc - (pci->retc + 11), 1,
order);
t = m->results;
if (res < 0){
@@ -2473,7 +2473,7 @@ mvc_export_table_wrap( Client cntxt, Mal
tpename = BUNtail(itertpe,o);
b = BATdescriptor(bid);
if ( b == NULL)
- msg= createException(MAL,"sql.resultset","failed to
access result column");
+ msg= createException(MAL,"sql.resultset","Failed to
access result column");
else
if (mvc_result_column(m, tblname, colname, tpename, *digits++,
*scaledigits++, b))
msg = createException(SQL, "sql.resultset",
"mvc_result_column failed");
@@ -3137,9 +3137,9 @@ mvc_import_table_wrap(Client cntxt, MalB
GDKfree(ssep);
GDKfree(ns);
if (s == NULL)
- throw(IO, "bstreams.create", "failed to create block stream");
+ throw(IO, "bstreams.create", "Failed to create block stream");
if (b == NULL)
- throw(SQL, "importTable", "%sfailed to import table",
be->mvc->errstr);
+ throw(SQL, "importTable", "%s Failed to import table",
be->mvc->errstr);
bat2return(stk, pci, b);
GDKfree(b);
return MAL_SUCCEED;
@@ -3207,7 +3207,7 @@ mvc_import_table_stdin(Client cntxt, Mal
GDKfree(ssep);
GDKfree(ns);
if (!b)
- throw(SQL, "importTable", "%sfailed to import table",
m->errstr);
+ throw(SQL, "importTable", "%s. Failed to import table",
m->errstr);
bat2return(stk, pci, b);
GDKfree(b);
return MAL_SUCCEED;
@@ -3252,10 +3252,10 @@ mvc_bin_import_table_wrap(Client cntxt,
sql_column *col = n->data;
if (ATOMvarsized(col->type.type->localtype) &&
col->type.type->localtype != TYPE_str)
- throw(SQL, "sql", "failed to attach file %s",
*getArgReference_str(stk, pci, i));
+ throw(SQL, "sql", "Failed to attach file %s",
*getArgReference_str(stk, pci, i));
f = fopen(*getArgReference_str(stk, pci, i), "r");
if (f == NULL)
- throw(SQL, "sql", "failed to open file %s",
*getArgReference_str(stk, pci, i));
+ throw(SQL, "sql", "Failed to open file %s",
*getArgReference_str(stk, pci, i));
fclose(f);
}
@@ -3268,7 +3268,7 @@ mvc_bin_import_table_wrap(Client cntxt,
if (tpe < TYPE_str || tpe == TYPE_date || tpe == TYPE_daytime
|| tpe == TYPE_timestamp) {
c = BATattach(col->type.type->localtype,
*getArgReference_str(stk, pci, i), TRANSIENT);
if (c == NULL)
- throw(SQL, "sql", "failed to attach file %s",
*getArgReference_str(stk, pci, i));
+ throw(SQL, "sql", "Failed to attach file %s",
*getArgReference_str(stk, pci, i));
BATsetaccess(c, BAT_READ);
BATderiveProps(c, 0);
} else if (tpe == TYPE_str) {
@@ -3280,12 +3280,12 @@ mvc_bin_import_table_wrap(Client cntxt,
/* this code should be extended to deal with larger
text strings. */
f = fopen(*getArgReference_str(stk, pci, i), "r");
if (f == NULL)
- throw(SQL, "sql", "failed to re-open file %s",
*getArgReference_str(stk, pci, i));
+ throw(SQL, "sql", "Failed to re-open file %s",
*getArgReference_str(stk, pci, i));
buf = GDKmalloc(bufsiz);
if (!buf) {
fclose(f);
- throw(SQL, "sql", "failed to create buffer");
+ throw(SQL, "sql", "Failed to create buffer");
}
while (fgets(buf, bufsiz, f) != NULL) {
char *t = strrchr(buf, '\n');
@@ -3296,7 +3296,7 @@ mvc_bin_import_table_wrap(Client cntxt,
fclose(f);
GDKfree(buf);
} else {
- throw(SQL, "sql", "failed to attach file %s",
*getArgReference_str(stk, pci, i));
+ throw(SQL, "sql", "Failed to attach file %s",
*getArgReference_str(stk, pci, i));
}
if (i != (pci->retc + 2) && cnt != BATcount(c))
throw(SQL, "sql", "binary files for table '%s' have
inconsistent counts", tname);
diff --git a/sql/backends/monet5/sql_result.c b/sql/backends/monet5/sql_result.c
--- a/sql/backends/monet5/sql_result.c
+++ b/sql/backends/monet5/sql_result.c
@@ -794,9 +794,13 @@ mvc_import_table(Client cntxt, mvc *m, b
fmt[i].ci = bat_iterator(fmt[i].c);
}
}
- if ( (locked || (msg = TABLETcreate_bats(&as, (BUN) (sz < 0 ?
1000 : sz))) == MAL_SUCCEED) &&
- (bats = (BAT**) GDKmalloc(sizeof(BAT *) *
as.nr_attrs)) ){
+ if ( (locked || (msg = TABLETcreate_bats(&as, (BUN) (sz < 0 ?
1000 : sz))) == MAL_SUCCEED) ){
if (SQLload_file(cntxt, &as, bs, out, sep, rsep, ssep ?
ssep[0] : 0, offset, sz, best) != BUN_NONE && !as.error) {
+ bats = (BAT**) GDKzalloc(sizeof(BAT *) *
as.nr_attrs);
+ if ( bats == NULL){
+ TABLETdestroy_format(&as);
+ return NULL;
+ }
if (locked)
msg = TABLETcollect_parts(bats,&as,
cnt);
else
diff --git a/sql/test/BugTracker-2008/Tests/copy-format.SF-2468513.stable.err
b/sql/test/BugTracker-2008/Tests/copy-format.SF-2468513.stable.err
--- a/sql/test/BugTracker-2008/Tests/copy-format.SF-2468513.stable.err
+++ b/sql/test/BugTracker-2008/Tests/copy-format.SF-2468513.stable.err
@@ -87,36 +87,31 @@ stderr of test 'copy-format.SF-2468513`
# 11:46:56 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e"
"--host=/var/tmp/mtest-23209" "--port=33225"
# 11:46:56 >
-MAPI = (monetdb) /var/tmp/mtest-23209/.s.monetdb.33225
+MAPI = (monetdb) /var/tmp/mtest-12407/.s.monetdb.36976
QUERY = COPY 1 RECORDS INTO my_copytest FROM stdin USING DELIMITERS '|','\n'
NULL as '';
123|1.01||a|b
-ERROR = !value '1.01' from line 1 field 2 not inserted, expecting type int
- !failed to import table
-MAPI = (monetdb) /var/tmp/mtest-23209/.s.monetdb.33225
+ERROR = !line 1 field 2 'int' expected in '1.01'. Failed to import table
+MAPI = (monetdb) /var/tmp/mtest-12407/.s.monetdb.36976
QUERY = COPY 1 RECORDS INTO my_copytest FROM stdin USING DELIMITERS '|','\n'
NULL as '';
553|.02||a|b
-ERROR = !value '.02' from line 1 field 2 not inserted, expecting type int
- !failed to import table
-MAPI = (monetdb) /var/tmp/mtest-23209/.s.monetdb.33225
+ERROR = !line 1 field 2 'int' expected in '.02'. Failed to import table
+MAPI = (monetdb) /var/tmp/mtest-12407/.s.monetdb.36976
QUERY = COPY 1 RECORDS INTO my_copytest FROM stdin USING DELIMITERS '|','\n'
NULL as '';
223|2.03||a|b
-ERROR = !value '2.03' from line 1 field 2 not inserted, expecting type int
- !failed to import table
-MAPI = (monetdb) /var/tmp/mtest-23209/.s.monetdb.33225
+ERROR = !line 1 field 2 'int' expected in '2.03'. Failed to import table
+MAPI = (monetdb) /var/tmp/mtest-12407/.s.monetdb.36976
QUERY = COPY 1 RECORDS INTO my_copytest FROM stdin USING DELIMITERS '|','\n'
NULL as '';
223|2||aaa|b
-ERROR = !value 'aaa' from line 1 field 4 not inserted, expecting type
varchar(1)
- !failed to import table
-MAPI = (monetdb) /var/tmp/mtest-23209/.s.monetdb.33225
+ERROR = !line 1 field 4 'varchar(1)' expected in 'aaa'. Failed to import table
+MAPI = (monetdb) /var/tmp/mtest-12407/.s.monetdb.36976
QUERY = COPY 1 RECORDS INTO my_copytest FROM stdin USING DELIMITERS '|','\n'
NULL as '';
223|3||a|bbb
-ERROR = !value 'bbb' from line 1 field 5 not inserted, expecting type
varchar(1)
- !failed to import table
+ERROR = !line 1 field 5 'varchar(1)' expected in 'bbb'. Failed to import table
# 09:34:56 >
# 09:34:56 > Done.
diff --git a/sql/test/BugTracker-2008/Tests/varchar.SF-2471371.stable.err
b/sql/test/BugTracker-2008/Tests/varchar.SF-2471371.stable.err
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
