Changeset: b9f2f0816a7d for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/b9f2f0816a7d
Modified Files:
sql/backends/monet5/copy.c
sql/backends/monet5/copy_convert.c
sql/backends/monet5/rel_copy.c
Branch: default
Log Message:
small csv loading improvements
diffs (117 lines):
diff --git a/sql/backends/monet5/copy.c b/sql/backends/monet5/copy.c
--- a/sql/backends/monet5/copy.c
+++ b/sql/backends/monet5/copy.c
@@ -370,6 +370,54 @@ find_end_of_lines1(struct scan_state *st
}
static const unsigned char *
+find_end_of_lines2(struct scan_state *st, BUN *newlines_count, BUN maxcount )
+{
+ bool found = true;
+ int quote_char = st->quote_char;
+ int line_sep0 = st->line_sep_str[0];
+ int line_sep1 = st->line_sep_str[1];
+ bool escape_enabled = st->escape_enabled;
+ unsigned char *end = st->end;
+ // these are updated
+ unsigned char *pos = st->pos;
+ bool quoted = st->quoted;
+ bool escape_pending = st->escape_pending;
+
+ BUN newline_count = 0;
+ const unsigned char *latest_pos = NULL;
+ while (found && pos < end && newline_count < maxcount) {
+ found = false;
+ for (; pos < end; pos++) {
+ if (escape_pending) {
+ escape_pending = false;
+ continue;
+ }
+ if (escape_enabled && *pos == '\\') {
+ escape_pending = true;
+ continue;
+ }
+ bool is_quote = (quote_char != 0 && *pos == quote_char);
+ quoted ^= is_quote;
+ if (!quoted && *pos == line_sep0 && pos[1] ==
line_sep1) {
+ found = true;
+ break;
+ }
+ }
+ if (found) {
+ *pos = 0;
+ pos+=2;
+ newline_count++;
+ latest_pos = pos;
+ }
+ }
+ st->pos = pos;
+ st->quoted = quoted;
+ st->escape_pending = escape_pending;
+ *newlines_count = newline_count;
+ return latest_pos;
+}
+
+static const unsigned char *
find_end_of_linesN(struct scan_state *st, BUN *newlines_count, BUN maxcount )
{
bool found = true;
@@ -455,6 +503,8 @@ COPYfixlines(lng *ret_linecount, BUN *e,
const unsigned char *latest_pos = NULL;
if (state.line_sep_len == 1 && !state.quote_char &&
!state.escape_enabled)
latest_pos = find_end_of_lines1nn(&state, &newline_count,
r->maxcount);
+ else if (state.line_sep_len == 2)
+ latest_pos = find_end_of_lines2(&state, &newline_count,
r->maxcount);
else if (state.line_sep_len == 1)
latest_pos = find_end_of_lines1(&state, &newline_count,
r->maxcount);
else
diff --git a/sql/backends/monet5/copy_convert.c
b/sql/backends/monet5/copy_convert.c
--- a/sql/backends/monet5/copy_convert.c
+++ b/sql/backends/monet5/copy_convert.c
@@ -48,6 +48,7 @@ COPYparse_generic(Client cntxt, MalBlkPt
size_t buffer_len;
const void *nil_ptr;
struct error_handling errors;
+ bool has_nil = false;
errors.init = 0;
@@ -76,6 +77,7 @@ COPYparse_generic(Client cntxt, MalBlkPt
if (is_int_nil(offset)) {
to_insert = nil_ptr;
+ has_nil = true;
} else {
ssize_t len = BATatoms[tpe].atomFromStr(ma, src,
&buffer_len, &buffer, false);
if (len >= 0) {
@@ -84,6 +86,7 @@ COPYparse_generic(Client cntxt, MalBlkPt
ok = copy_report_error(&errors, (lng) i, -1,
"invalid %s: %s", ATOMname(tpe), src);
GDKclrerr();
to_insert = nil_ptr;
+ has_nil = true;
}
}
if (ok != GDK_SUCCEED) {
@@ -99,8 +102,8 @@ COPYparse_generic(Client cntxt, MalBlkPt
BATsetcount(ret, n);
// we don't know anything about the data we just parsed
ret->tkey = false;
- ret->tnil = false;
- ret->tnonil = false;
+ ret->tnil = has_nil?true:false;
+ ret->tnonil = has_nil?false:true;
ret->tsorted = false;
ret->trevsorted = false;
end:
diff --git a/sql/backends/monet5/rel_copy.c b/sql/backends/monet5/rel_copy.c
--- a/sql/backends/monet5/rel_copy.c
+++ b/sql/backends/monet5/rel_copy.c
@@ -111,7 +111,7 @@ emit_pipelined_loop(
q = pushArgument(mb, q, var_line_sep);
q = pushArgument(mb, q, var_quote_char);
q = pushStr(mb, q, null_representation);
- q = pushBit(mb, q, escape);
+ q = pushBit(mb, q, escape || onclient );
if (fixed_width == NULL)
fixed_width = (str)str_nil;
q = pushStr(mb, q, fixed_width);
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
