Changeset: d633b25eb73e for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d633b25eb73e
Modified Files:
geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err
monetdb5/modules/mal/tablet.c
sql/test/BugTracker-2008/Tests/copy-format.SF-2468513.stable.err
sql/test/BugTracker-2008/Tests/varchar.SF-2471371.stable.err
sql/test/BugTracker-2009/Tests/copy_into_error_off_by_one.SF-2833696.stable.err
sql/test/BugTracker-2010/Tests/copy-into-too-long-string.Bug-2358.stable.err
sql/test/BugTracker-2010/Tests/incomplete-utf8-sequence.Bug-2575.stable.err
sql/test/BugTracker/Tests/copy_decimal.SF-1582957.stable.err
sql/test/BugTracker/Tests/copy_overflow_null_bug.SF-1842550.stable.err
sql/test/copy/Tests/int_parse.stable.err
sql/test/copy/Tests/load_stdin_incorrect_line_nr.stable.err
sql/test/copy/Tests/null_as_string_errors.stable.err
sql/test/testdb/Tests/testdb-load.stable.out
Branch: Jan2014
Log Message:
Don't read more than necessary during COPY INTO.
This fixes bug 3481.
diffs (truncated from 311 to 300 lines):
diff --git a/geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err
b/geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err
--- a/geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err
+++ b/geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err
@@ -13,7 +13,6 @@ stderr of test 'copy_into_crash.SF-19754
MAPI = (monetdb) /var/tmp/mtest-17297/.s.monetdb.35706
QUERY = COPY 1 RECORDS INTO nodes from STDIN USING DELIMITERS ',', '\n';
45111956, 'POINT(52.0697 4.3723)'
-
ERROR = !value ' 'POINT(52.0697 4.3723)'' from line 1 field 2 not inserted,
expecting type point
!failed to import table
diff --git a/monetdb5/modules/mal/tablet.c b/monetdb5/modules/mal/tablet.c
--- a/monetdb5/modules/mal/tablet.c
+++ b/monetdb5/modules/mal/tablet.c
@@ -1067,7 +1067,7 @@ SQLworkdivider(READERtask *task, READERt
/*
* Reading is handled by a separate task as a preparation for
- * mode parallelism
+ * more parallelism
*/
static void
SQLloader(void *p)
@@ -1126,7 +1126,6 @@ SQLload_file(Client cntxt, Tablet *as, b
assert(rsep);
assert(csep);
assert(maxrow < 0 || maxrow <= (lng) BUN_MAX);
- rseplen = strlen(rsep);
task->fields = (char ***) GDKzalloc(as->nr_attrs * sizeof(char **));
task->cols = (int *) GDKzalloc(as->nr_attrs * sizeof(int));
task->time = (lng *) GDKzalloc(as->nr_attrs * sizeof(lng));
@@ -1146,7 +1145,7 @@ SQLload_file(Client cntxt, Tablet *as, b
task->csep = csep;
task->seplen = strlen(csep);
task->rsep = rsep;
- task->rseplen = strlen(rsep);
+ task->rseplen = rseplen = strlen(rsep);
task->errbuf = cntxt->errbuf;
task->input = task->base + 1; /* wrap the buffer with null bytes */
task->base[b->size + 1] = 0;
@@ -1163,9 +1162,7 @@ SQLload_file(Client cntxt, Tablet *as, b
mlock(task->time, as->nr_attrs * sizeof(lng));
mlock(task->base, b->size + 2);
#endif
- MT_lock_set(&errorlock, "SQLload_file");
as->error = NULL;
- MT_lock_unset(&errorlock, "SQLload_file");
/* there is no point in creating more threads than we have columns */
if (as->nr_attrs < (BUN) threads)
@@ -1177,10 +1174,8 @@ SQLload_file(Client cntxt, Tablet *as, b
for (i = 0; i < as->nr_attrs; i++) {
task->fields[i] = GDKzalloc(sizeof(char *) * task->limit);
if (task->fields[i] == 0) {
- MT_lock_set(&errorlock, "SQLload_file");
if (task->as->error == NULL)
as->error = M5OutOfMemory;
- MT_lock_unset(&errorlock, "SQLload_file");
goto bailout;
}
#ifdef MLOCK_TST
@@ -1216,10 +1211,10 @@ SQLload_file(Client cntxt, Tablet *as, b
#ifdef _DEBUG_TABLET_
mnstr_printf(GDKout, "parallel bulk load " LLFMT " - " LLFMT "\n",
skip, maxrow);
#endif
+ if (maxrow < 0)
+ maxrow = (lng) BUN_MAX;
tio = GDKusec();
- MT_sema_up(&task->producer, "SQLload_file");
- MT_sema_down(&task->consumer, "SQLload_file");
tio = GDKusec() - tio;
t1 = GDKusec();
#ifdef MLOCK_TST
@@ -1283,7 +1278,7 @@ SQLload_file(Client cntxt, Tablet *as, b
* the middle of the record separator). If this is too
* costly, we have to rethink the matter. */
e = s;
- while (s < end && task->next < task->limit && (maxrow < 0 ||
cnt < (BUN) maxrow)) {
+ while (s < end && task->next < task->limit && cnt < (BUN)
maxrow) {
char q = 0;
/* tokenize the record completely the format of the
input
* should comply to the following grammar rule [
@@ -1388,7 +1383,8 @@ SQLload_file(Client cntxt, Tablet *as, b
}
}
/* start feeding new data */
- MT_sema_up(&task->producer, "SQLload_file");
+ if ((e == NULL || s >= end || e >= end) && cnt < (BUN) maxrow)
+ MT_sema_up(&task->producer, "SQLload_file");
t1 = GDKusec() - t1;
total += t1;
iototal += tio;
@@ -1444,7 +1440,8 @@ SQLload_file(Client cntxt, Tablet *as, b
}
if (task->ateof)
break;
- MT_sema_down(&task->consumer, "SQLload_file");
+ if ((e == NULL || s >= end || e >= end) && cnt < (BUN) maxrow)
+ MT_sema_down(&task->consumer, "SQLload_file");
}
if (task->b->pos < task->b->len && cnt < (BUN) maxrow && task->ateof) {
@@ -1461,7 +1458,7 @@ SQLload_file(Client cntxt, Tablet *as, b
}
if (GDKdebug & GRPalgorithms) {
- if (cnt < (BUN) maxrow && maxrow > 0)
+ if (cnt < (BUN) maxrow)
/* providing a precise count is not always easy, instead
* consider maxrow as an upper bound */
mnstr_printf(GDKout, "#SQLload_file: read error, tuples
missing (after loading " BUNFMT " records)\n", BATcount(as->format[0].c));
diff --git a/sql/test/BugTracker-2008/Tests/copy-format.SF-2468513.stable.err
b/sql/test/BugTracker-2008/Tests/copy-format.SF-2468513.stable.err
--- a/sql/test/BugTracker-2008/Tests/copy-format.SF-2468513.stable.err
+++ b/sql/test/BugTracker-2008/Tests/copy-format.SF-2468513.stable.err
@@ -91,35 +91,30 @@ MAPI = (monetdb) /var/tmp/mtest-23209/.
QUERY = COPY 1 RECORDS INTO my_copytest FROM stdin USING DELIMITERS '|','\n'
NULL as '';
123|1.01||a|b
-
ERROR = !value '1.01' from line 1 field 2 not inserted, expecting type int
!failed to import table
MAPI = (monetdb) /var/tmp/mtest-23209/.s.monetdb.33225
QUERY = COPY 1 RECORDS INTO my_copytest FROM stdin USING DELIMITERS '|','\n'
NULL as '';
553|.02||a|b
-
ERROR = !value '.02' from line 1 field 2 not inserted, expecting type int
!failed to import table
MAPI = (monetdb) /var/tmp/mtest-23209/.s.monetdb.33225
QUERY = COPY 1 RECORDS INTO my_copytest FROM stdin USING DELIMITERS '|','\n'
NULL as '';
223|2.03||a|b
-
ERROR = !value '2.03' from line 1 field 2 not inserted, expecting type int
!failed to import table
MAPI = (monetdb) /var/tmp/mtest-23209/.s.monetdb.33225
QUERY = COPY 1 RECORDS INTO my_copytest FROM stdin USING DELIMITERS '|','\n'
NULL as '';
223|2||aaa|b
-
ERROR = !value 'aaa' from line 1 field 4 not inserted, expecting type
varchar(1)
!failed to import table
MAPI = (monetdb) /var/tmp/mtest-23209/.s.monetdb.33225
QUERY = COPY 1 RECORDS INTO my_copytest FROM stdin USING DELIMITERS '|','\n'
NULL as '';
223|3||a|bbb
-
ERROR = !value 'bbb' from line 1 field 5 not inserted, expecting type
varchar(1)
!failed to import table
diff --git a/sql/test/BugTracker-2008/Tests/varchar.SF-2471371.stable.err
b/sql/test/BugTracker-2008/Tests/varchar.SF-2471371.stable.err
--- a/sql/test/BugTracker-2008/Tests/varchar.SF-2471371.stable.err
+++ b/sql/test/BugTracker-2008/Tests/varchar.SF-2471371.stable.err
@@ -90,13 +90,11 @@ stderr of test 'varchar.SF-2471371` in d
MAPI = (monetdb) /var/tmp/mtest-23209/.s.monetdb.33225
QUERY = COPY 1 RECORDS INTO myvar_test FROM stdin USING DELIMITERS '|','\n'
NULL as '';
a|b2b
-
ERROR = !value 'b2b' from line 1 field 2 not inserted, expecting type
varchar(2)
!failed to import table
MAPI = (monetdb) /var/tmp/mtest-23209/.s.monetdb.33225
QUERY = COPY 1 RECORDS INTO myvar_test FROM stdin USING DELIMITERS '|','\n'
NULL as '';
aa|bb
-
ERROR = !value 'aa' from line 1 field 1 not inserted, expecting type varchar(1)
!failed to import table
diff --git
a/sql/test/BugTracker-2009/Tests/copy_into_error_off_by_one.SF-2833696.stable.err
b/sql/test/BugTracker-2009/Tests/copy_into_error_off_by_one.SF-2833696.stable.err
---
a/sql/test/BugTracker-2009/Tests/copy_into_error_off_by_one.SF-2833696.stable.err
+++
b/sql/test/BugTracker-2009/Tests/copy_into_error_off_by_one.SF-2833696.stable.err
@@ -73,7 +73,6 @@ MAPI = (monetdb) /var/tmp/mtest-23209/.
QUERY = COPY 2 RECORDS INTO number FROM stdin USING DELIMITERS '\n';
1
bla
- drop table number;
ERROR = !value 'bla' from line 2 field 1 not inserted, expecting type int
!failed to import table
diff --git
a/sql/test/BugTracker-2010/Tests/copy-into-too-long-string.Bug-2358.stable.err
b/sql/test/BugTracker-2010/Tests/copy-into-too-long-string.Bug-2358.stable.err
---
a/sql/test/BugTracker-2010/Tests/copy-into-too-long-string.Bug-2358.stable.err
+++
b/sql/test/BugTracker-2010/Tests/copy-into-too-long-string.Bug-2358.stable.err
@@ -75,7 +75,6 @@ MAPI = (monetdb) /var/tmp/mtest-23209/.
QUERY = COPY 1 RECORDS INTO t FROM STDIN USING DELIMITERS '|','
','"' NULL AS '';
"abcd"
-
ERROR = !value 'abcd' from line 1 field 1 not inserted, expecting type
varchar(3)
!failed to import table
diff --git
a/sql/test/BugTracker-2010/Tests/incomplete-utf8-sequence.Bug-2575.stable.err
b/sql/test/BugTracker-2010/Tests/incomplete-utf8-sequence.Bug-2575.stable.err
---
a/sql/test/BugTracker-2010/Tests/incomplete-utf8-sequence.Bug-2575.stable.err
+++
b/sql/test/BugTracker-2010/Tests/incomplete-utf8-sequence.Bug-2575.stable.err
@@ -71,7 +71,6 @@ MAPI = (monetdb) /var/tmp/mtest-23209/.
QUERY = copy 2 records into tbl_bug2575 from stdin using delimiters
'\t','\n','';
10001160000 29 .v.
10001690001 0 coÃ
-
ERROR = !value 'coÃ' from line 2 field 3 not inserted, expecting type char(3)
!failed to import table
diff --git a/sql/test/BugTracker/Tests/copy_decimal.SF-1582957.stable.err
b/sql/test/BugTracker/Tests/copy_decimal.SF-1582957.stable.err
--- a/sql/test/BugTracker/Tests/copy_decimal.SF-1582957.stable.err
+++ b/sql/test/BugTracker/Tests/copy_decimal.SF-1582957.stable.err
@@ -33,7 +33,6 @@ stderr of test 'copy_decimal.SF-1582957`
MAPI = (monetdb) /var/tmp/mtest-23209/.s.monetdb.33225
QUERY = COPY 1 RECORDS INTO LINEITEM FROM stdin USING DELIMITERS '|','|\n';
1|156|4|1|17.000|17954.55|0.04|0.02|N|O|1996-03-13|1996-02-12|1996-03-22|DELIVER
IN PERSON|TRUCK|blithely regular ideas caj|
-
ERROR = !value '17.000' from line 1 field 5 not inserted, expecting type
decimal(15,2)
!failed to import table
diff --git
a/sql/test/BugTracker/Tests/copy_overflow_null_bug.SF-1842550.stable.err
b/sql/test/BugTracker/Tests/copy_overflow_null_bug.SF-1842550.stable.err
--- a/sql/test/BugTracker/Tests/copy_overflow_null_bug.SF-1842550.stable.err
+++ b/sql/test/BugTracker/Tests/copy_overflow_null_bug.SF-1842550.stable.err
@@ -34,7 +34,6 @@ MAPI = (monetdb) /var/tmp/mtest-23209/.
QUERY = COPY 2 RECORDS INTO bulk FROM stdin USING DELIMITERS '|', '\n', '''';
5536|'5536'
53605|'53605'
-
ERROR = !value '53605' from line 2 field 1 not inserted, expecting type
smallint
!failed to import table
diff --git a/sql/test/copy/Tests/int_parse.stable.err
b/sql/test/copy/Tests/int_parse.stable.err
--- a/sql/test/copy/Tests/int_parse.stable.err
+++ b/sql/test/copy/Tests/int_parse.stable.err
@@ -86,7 +86,6 @@ stderr of test 'int_parse` in directory
MAPI = monetdb@sofia:38256
QUERY = copy 1 records into t_int from stdin USING DELIMITERS ',','\n','\"'
NULL AS '';
5.1
-
ERROR = !value '5.1' from line 1 field 1 not inserted, expecting type int
!failed to import table
MAPI = monetdb@sofia:38256
@@ -95,7 +94,6 @@ QUERY = copy 4 records into t_int from s
nil
null
abc
-
ERROR = !value 'null' from line 3 field 1 not inserted, expecting type int
!failed to import table
diff --git a/sql/test/copy/Tests/load_stdin_incorrect_line_nr.stable.err
b/sql/test/copy/Tests/load_stdin_incorrect_line_nr.stable.err
--- a/sql/test/copy/Tests/load_stdin_incorrect_line_nr.stable.err
+++ b/sql/test/copy/Tests/load_stdin_incorrect_line_nr.stable.err
@@ -73,7 +73,6 @@ MAPI = monetdb@sofia:38256
QUERY = COPY 2 RECORDS INTO "number" FROM stdin USING DELIMITERS ';', '\n';
1
bla
-
ERROR = !value 'bla' from line 2 field 1 not inserted, expecting type int
!failed to import table
diff --git a/sql/test/copy/Tests/null_as_string_errors.stable.err
b/sql/test/copy/Tests/null_as_string_errors.stable.err
--- a/sql/test/copy/Tests/null_as_string_errors.stable.err
+++ b/sql/test/copy/Tests/null_as_string_errors.stable.err
@@ -87,37 +87,31 @@ stderr of test 'null_as_string_errors` i
MAPI = monetdb@sofia:38256
QUERY = copy 1 records into null_as_string from stdin delimiters ',','\n' NULL
as '';
NULL,NULL,NULL
-
ERROR = !value 'NULL' from line 1 field 1 not inserted, expecting type int
!failed to import table
MAPI = monetdb@sofia:38256
QUERY = copy 1 records into null_as_string from stdin delimiters ',','\n' NULL
as '';
NULL,zero,0
-
ERROR = !value 'NULL' from line 1 field 1 not inserted, expecting type int
!failed to import table
MAPI = monetdb@sofia:38256
QUERY = copy 1 records into null_as_string from stdin delimiters ',','\n' NULL
as '';
2,two,NULL
-
ERROR = !value 'NULL' from line 1 field 3 not inserted, expecting type
decimal(5,2)
!failed to import table
MAPI = monetdb@sofia:38256
QUERY = copy 1 records into null_as_string from stdin delimiters ',','\n';
,,
-
ERROR = !value '' from line 1 field 1 not inserted, expecting type int
!failed to import table
MAPI = monetdb@sofia:38256
QUERY = copy 1 records into null_as_string from stdin delimiters ',','\n';
,zero,0
-
ERROR = !value '' from line 1 field 1 not inserted, expecting type int
!failed to import table
MAPI = monetdb@sofia:38256
QUERY = copy 1 records into null_as_string from stdin delimiters ',','\n';
2,two,
-
ERROR = !value '' from line 1 field 3 not inserted, expecting type decimal(5,2)
!failed to import table
diff --git a/sql/test/testdb/Tests/testdb-load.stable.out
b/sql/test/testdb/Tests/testdb-load.stable.out
--- a/sql/test/testdb/Tests/testdb-load.stable.out
+++ b/sql/test/testdb/Tests/testdb-load.stable.out
@@ -203,12 +203,12 @@ Ready.
# "key1" INTEGER,
# "key2" INTEGER
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
