Changeset: bfc20317cd61 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bfc20317cd61
Added Files:
sql/test/copy/Tests/int_parse_best.sql
sql/test/copy/Tests/int_parse_best.stable.err
sql/test/copy/Tests/int_parse_best.stable.out
Modified Files:
monetdb5/modules/mal/tablet.c
sql/backends/monet5/sql_result.c
sql/test/copy/Tests/All
sql/test/copy/Tests/int_parse.sql
sql/test/copy/Tests/int_parse.stable.err
sql/test/copy/Tests/int_parse.stable.out
Branch: resultset
Log Message:
Best effor test.
Reject tables are filled, but the trimming action still does
not lead to any insert in the sql table.
diffs (truncated from 535 to 300 lines):
diff --git a/monetdb5/modules/mal/tablet.c b/monetdb5/modules/mal/tablet.c
--- a/monetdb5/modules/mal/tablet.c
+++ b/monetdb5/modules/mal/tablet.c
@@ -699,10 +699,9 @@ tablet_error(READERtask *task, lng row,
BUNappend(task->cntxt->error_input, fcn, FALSE);
if (task->as->error == NULL && (msg == NULL || (task->as->error
= GDKstrdup(msg)) == NULL))
task->as->error = M5OutOfMemory;
- if ( row != lng_nil){
- task->rowerror[row]++;
- task->errorcnt++;
- }
+ if ( row != lng_nil)
+ task->rowerror[row-1]++;
+ task->errorcnt++;
MT_lock_unset(&errorlock, "tablet_error");
}
}
@@ -800,7 +799,7 @@ SQLinsert_val(READERtask *task, int col,
snprintf(buf, BUFSIZ, "line "BUNFMT" field %d '%s'
expected in '%s'",row, col, fmt->type, s);
if (task->as->error == NULL && (task->as->error =
GDKstrdup(buf)) == NULL)
task->as->error = M5OutOfMemory;
- task->rowerror[(int)row]++;
+ task->rowerror[(int)row-1]++;
task->errorcnt++;
MT_lock_unset(&errorlock, "insert_val");
}
@@ -821,7 +820,7 @@ SQLinsert_val(READERtask *task, int col,
BUNappend(task->cntxt->error_fld, &col, FALSE);
BUNappend(task->cntxt->error_msg, "insert failed", FALSE);
BUNappend(task->cntxt->error_input, err, FALSE);
- task->rowerror[(int)row]++;
+ task->rowerror[(int)row -1]++;
task->errorcnt++;
MT_lock_unset(&errorlock, "insert_val");
}
@@ -1075,7 +1074,7 @@ SQLload_file(Client cntxt, Tablet *as, b
BUN cnt = 0, cntstart=0;
int res = 0; /* < 0: error, > 0: success, ==
0: continue processing */
int j;
- BUN i;
+ BUN i, attr;
size_t rseplen;
READERtask *task = (READERtask *) GDKzalloc(sizeof(READERtask));
READERtask ptask[MAXWORKERS];
@@ -1441,26 +1440,6 @@ SQLload_file(Client cntxt, Tablet *as, b
}
}
}
-
- /* trim the BATs discarding error tuples */
- if ( task->errorcnt && task->as->error == NULL && best == 0){
- task->as->error = M5OutOfMemory;
- res = -1;
- goto bailout;
- }
-/*
- for( attr=0; attr < task->nr_attrs; attr++)
- switch(){
- case 4:
- {
- for(j = 0; j < task->limit; j++){
- if ( task->rowerror[j]){
- continue;
- }
- }
- }
- }
-*/
/* shuffle remainder and continue reading */
#ifdef _DEBUG_TABLET_
mnstr_printf(GDKout, "shuffle %d:%s\n", (int) strlen(s), s);
@@ -1468,11 +1447,57 @@ SQLload_file(Client cntxt, Tablet *as, b
tio = GDKusec();
tio = t1 - tio;
- if (res == 0 && task->next) {
- /* await completion of the BAT updates */
+ /* await completion of the BAT updates */
+ if (res == 0 && task->next)
for (j = 0; j < threads; j++)
MT_sema_down(&ptask[j].reply, "SQLload_file");
+
+ /* trim the BATs discarding error tuples */
+#define trimerrors(TYPE) \
+{ unsigned long *src, *dst;\
+ BUN leftover= BATcount(task->as->format[attr].c);\
+ limit = leftover - cntstart;\
+ dst =src= (unsigned long *)
BUNtloc(task->as->format[attr].ci,cntstart);\
+ for(j = 0; j < (int) limit; j++, src++){\
+ if ( task->rowerror[j]){\
+ leftover--;\
+ cnt--;\
+ continue;\
+ }\
+ *dst++ = *src;\
+ }\
+ BATsetcount(task->as->format[attr].c, leftover );\
+}
+ if( best && BATcount(as->format[0].c)) {
+ BUN limit;
+ for( attr=0; attr < as->nr_attrs; attr++){
+ switch(ATOMsize(as->format[attr].c->ttype)){
+ case 1: trimerrors(unsigned bte); break;
+ case 2: trimerrors(unsigned short); break;
+ case 4:
+ { unsigned int *src, *dst;
+ BUN leftover=
BATcount(task->as->format[attr].c);
+ limit = leftover - cntstart;
+ dst =src= (unsigned int *)
BUNtloc(task->as->format[attr].ci,cntstart);
+ mnstr_printf(GDKout,"#trim
"BUNFMT" to "BUNFMT" ",cntstart, leftover);
+ for(j = 0; j < (int) limit;
j++) mnstr_printf(GDKout,"%d",task->rowerror[j]);
+ for(j = 0; j < (int) limit;
j++, src++){
+ if ( task->rowerror[j]){
+ leftover--;
+ cnt--;
+ continue;
+ }
+ *dst++ = *src;
+ }
+ mnstr_printf(GDKout,"
"BUNFMT"\n", leftover);
+
BATsetcount(task->as->format[attr].c, leftover );
+ }
+ break;
+ case 8: trimerrors(unsigned long);
+ }
+ }
}
+
if ((e == NULL || s >= end || e >= end) && cnt < (BUN) maxrow) {
#ifdef SQLLOADTHREAD
MT_sema_down(&task->consumer, "SQLload_file");
@@ -1488,12 +1513,8 @@ SQLload_file(Client cntxt, Tablet *as, b
tablet_error(task, lng_nil, int_nil, "Incomplete record at end
of file","SQLload_file");
/* indicate that we did read everything (even if we couldn't
deal with it */
task->b->pos = task->b->len;
- res = -1;
- }
- // check for a possible vacuum of error rows
- if( res >= 0 && task->errorcnt ){
- (void) cntstart;
- // compress all columns
+ if( best == 0)
+ res = -1;
}
if (GDKdebug & GRPalgorithms)
diff --git a/sql/backends/monet5/sql_result.c b/sql/backends/monet5/sql_result.c
--- a/sql/backends/monet5/sql_result.c
+++ b/sql/backends/monet5/sql_result.c
@@ -795,7 +795,7 @@ mvc_import_table(Client cntxt, mvc *m, b
}
}
if ( (locked || (msg = TABLETcreate_bats(&as, (BUN) (sz < 0 ?
1000 : sz))) == MAL_SUCCEED) ){
- if (SQLload_file(cntxt, &as, bs, out, sep, rsep, ssep ?
ssep[0] : 0, offset, sz, best) != BUN_NONE && !as.error) {
+ if (SQLload_file(cntxt, &as, bs, out, sep, rsep, ssep ?
ssep[0] : 0, offset, sz, best) != BUN_NONE && (best || !as.error)) {
bats = (BAT**) GDKzalloc(sizeof(BAT *) *
as.nr_attrs);
if ( bats == NULL){
TABLETdestroy_format(&as);
diff --git a/sql/test/copy/Tests/All b/sql/test/copy/Tests/All
--- a/sql/test/copy/Tests/All
+++ b/sql/test/copy/Tests/All
@@ -3,6 +3,7 @@ escaped_char
null_as_string
null_as_string_errors
null_as_string_output
+int_parse_best
int_parse
load_stdin_incorrect_line_nr
overflow_error
diff --git a/sql/test/copy/Tests/int_parse.sql
b/sql/test/copy/Tests/int_parse.sql
--- a/sql/test/copy/Tests/int_parse.sql
+++ b/sql/test/copy/Tests/int_parse.sql
@@ -7,6 +7,7 @@ copy 1 records into t_int from stdin USI
""
select * from t_int;
+delete from t_int;
-- A decimal value entered at a place where we expect an int
-- should be recognized. It is the residu of dumping an int
@@ -16,15 +17,15 @@ 3.00
-4.0000
select * from t_int;
-select * from sys.rejects;
-call sys.clearrejects();
+delete from t_int;
-copy 1 records into t_int from stdin USING DELIMITERS ',','\n','\"' NULL AS '';
+copy 3 records into t_int from stdin USING DELIMITERS ',','\n','\"' NULL AS '';
+0
5.1
+9
select * from t_int;
-select * from sys.rejects;
-call sys.clearrejects();
+delete from t_int;
copy 5 records into t_int from stdin delimiters ',','\n' NULL as '';
1
@@ -34,7 +35,6 @@ null
abc
select * from t_int;
-select * from sys.rejects;
-call sys.clearrejects();
+delete from t_int;
drop table t_int;
diff --git a/sql/test/copy/Tests/int_parse.stable.err
b/sql/test/copy/Tests/int_parse.stable.err
--- a/sql/test/copy/Tests/int_parse.stable.err
+++ b/sql/test/copy/Tests/int_parse.stable.err
@@ -79,11 +79,13 @@ stderr of test 'int_parse` in directory
# 19:40:24 > mclient -lsql -umonetdb -Pmonetdb --host=eir --port=37659
# 19:40:24 >
-MAPI = (monetdb) /var/tmp/mtest-17547/.s.monetdb.38582
-QUERY = copy 1 records into t_int from stdin USING DELIMITERS ',','\n','\"'
NULL AS '';
+MAPI = (monetdb) /var/tmp/mtest-4363/.s.monetdb.38666
+QUERY = copy 3 records into t_int from stdin USING DELIMITERS ',','\n','\"'
NULL AS '';
+ 0
5.1
-ERROR = !line 1 field 1 'int' expected in '5.1'. Failed to import table
-MAPI = (monetdb) /var/tmp/mtest-17547/.s.monetdb.38582
+ 9
+
+MAPI = (monetdb) /var/tmp/mtest-4363/.s.monetdb.38666
QUERY = copy 5 records into t_int from stdin delimiters ',','\n' NULL as '';
1
nil
diff --git a/sql/test/copy/Tests/int_parse.stable.out
b/sql/test/copy/Tests/int_parse.stable.out
--- a/sql/test/copy/Tests/int_parse.stable.out
+++ b/sql/test/copy/Tests/int_parse.stable.out
@@ -30,48 +30,34 @@ Ready.
% int # type
% 1 # length
[ NULL ]
+#delete from t_int;
+[ 1 ]
+#copy 2 records into t_int from stdin USING DELIMITERS ',','\n','\"' NULL AS
'';
+#3.00
+#-4.0000
[ 2 ]
% sys.t_int # table_name
% i # name
% int # type
% 2 # length
-[ NULL ]
[ 3 ]
[ -4 ]
-#select * from sys.rejects;
-% .rejects, .rejects, .rejects, .rejects # table_name
-% rowid, fldid, message, input # name
-% bigint, int, clob, clob # type
-% 1, 1, 0, 0 # length
+#delete from t_int;
+[ 2 ]
#select * from t_int;
% sys.t_int # table_name
% i # name
% int # type
-% 2 # length
-[ NULL ]
-[ 3 ]
-[ -4 ]
-#select * from sys.rejects;
-% .rejects, .rejects, .rejects, .rejects # table_name
-% rowid, fldid, message, input # name
-% bigint, int, clob, clob # type
-% 1, 1, 14, 4 # length
-[ 1, 1, "'int' expected", "5.1\n" ]
+% 1 # length
+#delete from t_int;
+[ 0 ]
#select * from t_int;
% sys.t_int # table_name
% i # name
% int # type
-% 2 # length
-[ NULL ]
-[ 3 ]
-[ -4 ]
-#select * from sys.rejects;
-% .rejects, .rejects, .rejects, .rejects # table_name
-% rowid, fldid, message, input # name
-% bigint, int, clob, clob # type
-% 1, 1, 14, 5 # length
-[ 4, 1, "'int' expected", "null\n" ]
-[ 5, 1, "'int' expected", "abc\n" ]
+% 1 # length
+#delete from t_int;
+[ 0 ]
#drop table t_int;
# 08:29:21 >
diff --git a/sql/test/copy/Tests/int_parse_best.sql
b/sql/test/copy/Tests/int_parse_best.sql
new file mode 100644
--- /dev/null
+++ b/sql/test/copy/Tests/int_parse_best.sql
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
