Changeset: a17e8aa5bef1 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/a17e8aa5bef1
Modified Files:
sql/backends/monet5/sql_copyinto.c
sql/backends/monet5/sql_copyinto.h
sql/backends/monet5/sql_result.c
sql/backends/monet5/sql_result.h
Branch: directappend
Log Message:
Move code from sql_result.{c,h} to sql_copyinto.{c,h}
diffs (truncated from 1695 to 300 lines):
diff --git a/sql/backends/monet5/sql_copyinto.c
b/sql/backends/monet5/sql_copyinto.c
--- a/sql/backends/monet5/sql_copyinto.c
+++ b/sql/backends/monet5/sql_copyinto.c
@@ -36,6 +36,8 @@
*/
#include "monetdb_config.h"
+#include "sql.h"
+
#include "sql_copyinto.h"
#include "str.h"
#include "mapi_prompt.h"
@@ -43,6 +45,70 @@
#include <string.h>
#include <ctype.h>
+
+typedef struct Column_t {
+ const char *name; /* column title */
+ const char *sep;
+ const char *rsep;
+ int seplen;
+ char *type;
+ int adt; /* type index */
+ BAT *c; /* set to NULL when
scalar is meant */
+ BATiter ci;
+ BUN p;
+ unsigned int tabs; /* field size in tab positions
*/
+ const char *nullstr; /* null representation */
+ size_t null_length; /* its length */
+ unsigned int width; /* actual column width */
+ unsigned int maxwidth; /* permissible width */
+ int fieldstart; /* Fixed character field load
positions */
+ int fieldwidth;
+ int scale, precision;
+ void *(*frstr)(struct Column_t *fmt, int type, void **dst, size_t
*dst_len, const char *s);
+ void *extra;
+ void *data;
+ int skip; /* only skip to the
next field */
+ size_t len;
+ bit ws; /* if set we need to
skip white space */
+ char quote; /* if set use this
character for string quotes */
+ const void *nildata;
+ size_t nil_len;
+ int size;
+ void *appendcol; /* temporary, can probably use
Columnt_t.extra in the future */
+} Column;
+
+/*
+ * All table printing is based on building a report structure first.
+ * This table structure is private to a client, which made us to
+ * keep it in an ADT.
+ */
+
+typedef struct Table_t {
+ BUN offset;
+ BUN nr; /* allocated space for
table loads */
+ BUN nr_attrs; /* attributes found sofar */
+ Column *format; /* remove later */
+ str error; /* last error */
+ int tryall; /* skip erroneous lines
*/
+ str filename; /* source */
+ BAT *complaints; /* lines that did not match the
required input */
+} Tablet;
+
+// Callback interface to append the data directly instead of storing it in
intermediate BATs.
+// SQLload_file doesn't know how to manipulate the sql transaction
bookkeeping, caller does.
+typedef str (*loadfile_claim_fptr)(void *state, size_t nrows, size_t ncols,
Column *cols[]);
+typedef str (*loadfile_append_one_fptr)(void *state, size_t idx, const void
*data, void *col);
+typedef str (*loadfile_append_batch_fptr)(void *state, const void *data, BUN
count, int width, void *col);
+typedef BAT *(*loadfile_get_offsets_bat_fptr)(void *state);
+typedef struct LoadOps {
+ void *state;
+ loadfile_claim_fptr claim;
+ loadfile_append_one_fptr append_one;
+ loadfile_append_batch_fptr append_batch;
+ loadfile_get_offsets_bat_fptr get_offsets;
+} LoadOps;
+
+
#define MAXWORKERS 64
#define MAXBUFFERS 2
/* We restrict the row length to be 32MB for the time being */
@@ -74,7 +140,7 @@ void_bat_create(int adt, BUN nr)
return b;
}
-void
+static void
TABLETdestroy_format(Tablet *as)
{
BUN p;
@@ -89,7 +155,7 @@ TABLETdestroy_format(Tablet *as)
GDKfree(fmt);
}
-str
+static str
TABLETcreate_bats(Tablet *as, BUN est)
{
Column *fmt = as->format;
@@ -114,7 +180,7 @@ TABLETcreate_bats(Tablet *as, BUN est)
return MAL_SUCCEED;
}
-str
+static str
TABLETcollect(BAT **bats, Tablet *as)
{
Column *fmt = as->format;
@@ -144,54 +210,8 @@ TABLETcollect(BAT **bats, Tablet *as)
return MAL_SUCCEED;
}
-str
-TABLETcollect_parts(BAT **bats, Tablet *as, BUN offset)
-{
- Column *fmt = as->format;
- BUN i, j;
- BUN cnt = 0;
-
- for (i = 0; i < as->nr_attrs && !cnt; i++)
- if (!fmt[i].skip)
- cnt = BATcount(fmt[i].c);
- for (i = 0, j = 0; i < as->nr_attrs; i++) {
- BAT *b, *bv = NULL;
- if (fmt[i].skip)
- continue;
- b = fmt[i].c;
- b->tsorted = b->trevsorted = false;
- b->tkey = false;
- BATsettrivprop(b);
- if ((b = BATsetaccess(b, BAT_READ)) == NULL) {
- fmt[i].c = NULL;
- throw(SQL, "copy", "Failed to set access at tablet part
" BUNFMT "\n", cnt);
- }
- bv = BATslice(b, (offset > 0) ? offset - 1 : 0, BATcount(b));
- bats[j] = bv;
-
- b->tkey = (offset > 0) ? FALSE : bv->tkey;
- b->tnonil &= bv->tnonil;
- if (b->tsorted != bv->tsorted)
- b->tsorted = false;
- if (b->trevsorted != bv->trevsorted)
- b->trevsorted = false;
- if (BATtdense(b))
- b->tkey = true;
- b->batDirtydesc = true;
-
- if (offset > 0) {
- BBPunfix(bv->batCacheid);
- bats[j] = BATslice(b, offset, BATcount(b));
- }
- if (cnt != BATcount(b))
- throw(SQL, "copy", "Count " BUNFMT " differs from "
BUNFMT "\n", BATcount(b), cnt);
- j++;
- }
- return MAL_SUCCEED;
-}
// the starting quote character has already been skipped
-
static char *
tablet_skip_string(char *s, char quote, bool escape)
{
@@ -213,143 +233,6 @@ tablet_skip_string(char *s, char quote,
return s + i;
}
-static int
-TABLET_error(stream *s)
-{
- char *err = mnstr_error(s);
- /* use free as stream allocates outside GDK */
- if (err)
- free(err);
- return -1;
-}
-
-/* The output line is first built before being sent. It solves a problem
- with UDP, where you may loose most of the information using short writes
-*/
-static inline int
-output_line(char **buf, size_t *len, char **localbuf, size_t *locallen, Column
*fmt, stream *fd, BUN nr_attrs, oid id)
-{
- BUN i;
- ssize_t fill = 0;
-
- for (i = 0; i < nr_attrs; i++) {
- if (fmt[i].c == NULL)
- continue;
- if (id < fmt[i].c->hseqbase || id >= fmt[i].c->hseqbase +
BATcount(fmt[i].c))
- break;
- fmt[i].p = id - fmt[i].c->hseqbase;
- }
- if (i == nr_attrs) {
- for (i = 0; i < nr_attrs; i++) {
- Column *f = fmt + i;
- const char *p;
- ssize_t l;
-
- if (f->c) {
- p = BUNtail(f->ci, f->p);
-
- if (!p || ATOMcmp(f->adt, ATOMnilptr(f->adt),
p) == 0) {
- p = f->nullstr;
- l = (ssize_t) strlen(f->nullstr);
- } else {
- l = f->tostr(f->extra, localbuf,
locallen, f->adt, p);
- if (l < 0)
- return -1;
- p = *localbuf;
- }
- if (fill + l + f->seplen >= (ssize_t) *len) {
- /* extend the buffer */
- char *nbuf;
- nbuf = GDKrealloc(*buf, fill + l +
f->seplen + BUFSIZ);
- if( nbuf == NULL)
- return -1; /* *buf freed by
caller */
- *buf = nbuf;
- *len = fill + l + f->seplen + BUFSIZ;
- }
- strncpy(*buf + fill, p, l);
- fill += l;
- }
- strncpy(*buf + fill, f->sep, f->seplen);
- fill += f->seplen;
- }
- }
- if (fd && mnstr_write(fd, *buf, 1, fill) != fill)
- return TABLET_error(fd);
- return 0;
-}
-
-static inline int
-output_line_dense(char **buf, size_t *len, char **localbuf, size_t *locallen,
Column *fmt, stream *fd, BUN nr_attrs)
-{
- BUN i;
- ssize_t fill = 0;
-
- for (i = 0; i < nr_attrs; i++) {
- Column *f = fmt + i;
- const char *p;
- ssize_t l;
-
- if (f->c) {
- p = BUNtail(f->ci, f->p);
-
- if (!p || ATOMcmp(f->adt, ATOMnilptr(f->adt), p) == 0) {
- p = f->nullstr;
- l = (ssize_t) strlen(p);
- } else {
- l = f->tostr(f->extra, localbuf, locallen,
f->adt, p);
- if (l < 0)
- return -1;
- p = *localbuf;
- }
- if (fill + l + f->seplen >= (ssize_t) *len) {
- /* extend the buffer */
- char *nbuf;
- nbuf = GDKrealloc(*buf, fill + l + f->seplen +
BUFSIZ);
- if( nbuf == NULL)
- return -1; /* *buf freed by caller
*/
- *buf = nbuf;
- *len = fill + l + f->seplen + BUFSIZ;
- }
- strncpy(*buf + fill, p, l);
- fill += l;
- f->p++;
- }
- strncpy(*buf + fill, f->sep, f->seplen);
- fill += f->seplen;
- }
- if (fd && mnstr_write(fd, *buf, 1, fill) != fill)
- return TABLET_error(fd);
- return 0;
-}
-
-static inline int
-output_line_lookup(char **buf, size_t *len, Column *fmt, stream *fd, BUN
nr_attrs, oid id)
-{
- BUN i;
-
- for (i = 0; i < nr_attrs; i++) {
- Column *f = fmt + i;
-
- if (f->c) {
- const void *p = BUNtail(f->ci, id - f->c->hseqbase);
-
- if (!p || ATOMcmp(f->adt, ATOMnilptr(f->adt), p) == 0) {
- size_t l = strlen(f->nullstr);
- if (mnstr_write(fd, f->nullstr, 1, l) !=
(ssize_t) l)
- return TABLET_error(fd);
- } else {
- ssize_t l = f->tostr(f->extra, buf, len,
f->adt, p);
-
- if (l < 0 || mnstr_write(fd, *buf, 1, l) != l)
- return TABLET_error(fd);
- }
- }
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
