Changeset: 3dbc36f15c4b for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/3dbc36f15c4b
Modified Files:
sql/backends/monet5/sql.c
sql/backends/monet5/sql.h
sql/backends/monet5/sql_bincopyconvert.c
sql/backends/monet5/sql_bincopyconvert.h
sql/backends/monet5/sql_bincopyfrom.c
Branch: copyintobinary
Log Message:
Implement COPY INTO BINARY for the trivial types
diffs (truncated from 378 to 300 lines):
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -3084,18 +3084,6 @@ bat2return(MalStkPtr stk, InstrPtr pci,
static const char fwftsep[2] = {STREAM_FWF_FIELD_SEP, '\0'};
static const char fwfrsep[2] = {STREAM_FWF_RECORD_SEP, '\0'};
-static str
-mvc_export_bin_column_wrap(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci)
-{
- (void)cntxt;
- (void)mb;
- (void)stk;
- (void)pci;
-
- throw(SQL, "sql.export_bin_column", SQLSTATE(42000) "not implemented");
-}
-
-
/* str mvc_import_table_wrap(int *res, sql_table **t, unsigned char* *T,
unsigned char* *R, unsigned char* *S, unsigned char* *N, str *fname, lng *sz,
lng *offset, int *besteffort, str *fixed_width, int *onclient, int *escape); */
str
mvc_import_table_wrap(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
@@ -5201,8 +5189,8 @@ static mel_func sql_init_funcs[] = {
pattern("sql", "exportChunk", mvc_export_chunk_wrap, true, "Export a chunk of
the result set (in order) to stream s", args(1,3,
arg("",void),arg("s",streams),arg("res_id",int))),
pattern("sql", "exportChunk", mvc_export_chunk_wrap, true, "Export a chunk of
the result set (in order) to stream s", args(1,5,
arg("",void),arg("s",streams),arg("res_id",int),arg("offset",int),arg("nr",int))),
pattern("sql", "exportOperation", mvc_export_operation_wrap, true, "Export
result of schema/transaction queries", args(1,1, arg("",void))),
- pattern("sql", "export_bin_column", mvc_export_bin_column_wrap, true, "export
column as binary", args(1, 5, arg("", lng), batargany("col", 1),
arg("byteswap", bit), arg("filename", str), arg("onclient", int))),
- pattern("sql", "export_bin_column", mvc_export_bin_column_wrap, true, "export
column as binary", args(1, 5, arg("", lng), argany("val", 1), arg("byteswap",
bit), arg("filename", str), arg("onclient", int))),
+ pattern("sql", "export_bin_column", mvc_bin_export_column_wrap, true, "export
column as binary", args(1, 5, arg("", lng), batargany("col", 1),
arg("byteswap", bit), arg("filename", str), arg("onclient", int))),
+ pattern("sql", "export_bin_column", mvc_bin_export_column_wrap, true, "export
column as binary", args(1, 5, arg("", lng), argany("val", 1), arg("byteswap",
bit), arg("filename", str), arg("onclient", int))),
pattern("sql", "affectedRows", mvc_affected_rows_wrap, true, "export the
number of affected rows by the current query", args(1,3,
arg("",int),arg("mvc",int),arg("nr",lng))),
pattern("sql", "copy_from", mvc_import_table_wrap, true, "Import a table from
bstream s with the \ngiven tuple and seperators (sep/rsep)", args(1,13,
batvarargany("",0),arg("t",ptr),arg("sep",str),arg("rsep",str),arg("ssep",str),arg("ns",str),arg("fname",str),arg("nr",lng),arg("offset",lng),arg("best",int),arg("fwf",str),arg("onclient",int),arg("escape",int))),
//we use bat.single now
diff --git a/sql/backends/monet5/sql.h b/sql/backends/monet5/sql.h
--- a/sql/backends/monet5/sql.h
+++ b/sql/backends/monet5/sql.h
@@ -93,6 +93,7 @@ extern str mvc_export_row_wrap(Client cn
extern str mvc_import_table_wrap(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci);
extern str mvc_bin_import_table_wrap(Client cntxt, MalBlkPtr mb, MalStkPtr
stk, InstrPtr pci);
extern str mvc_bin_import_column_wrap(Client cntxt, MalBlkPtr mb, MalStkPtr
stk, InstrPtr pci);
+extern str mvc_bin_export_column_wrap(Client cntxt, MalBlkPtr mb, MalStkPtr
stk, InstrPtr pci);
extern str setVariable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
extern str getVariable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
extern str sql_variables(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
diff --git a/sql/backends/monet5/sql_bincopyconvert.c
b/sql/backends/monet5/sql_bincopyconvert.c
--- a/sql/backends/monet5/sql_bincopyconvert.c
+++ b/sql/backends/monet5/sql_bincopyconvert.c
@@ -233,6 +233,7 @@ append_text(BAT *bat, char *start)
static str
load_zero_terminated_text(BAT *bat, stream *s, int *eof_reached)
{
+ const char *mal_operator = "sql.importColumn";
str msg = MAL_SUCCEED;
bstream *bs = NULL;
@@ -287,7 +288,7 @@ end:
-static struct type_rec type_recs[] = {
+static struct type_record_t type_recs[] = {
// no conversion, no byteswapping
{ "bte", "bte", .decoder=NULL, },
{ "uuid", "uuid", .decoder=NULL, },
@@ -313,11 +314,11 @@ static struct type_rec type_recs[] = {
};
-struct type_rec*
-find_type_rec(str name)
+type_record_t*
+find_type_rec(const char *name)
{
- struct type_rec *end = (struct type_rec*)((char *)type_recs +
sizeof(type_recs));
- for (struct type_rec *t = &type_recs[0]; t < end; t++)
+ struct type_record_t *end = (struct type_record_t*)((char *)type_recs +
sizeof(type_recs));
+ for (struct type_record_t *t = &type_recs[0]; t < end; t++)
if (strcmp(t->method, name) == 0)
return t;
return NULL;
diff --git a/sql/backends/monet5/sql_bincopyconvert.h
b/sql/backends/monet5/sql_bincopyconvert.h
--- a/sql/backends/monet5/sql_bincopyconvert.h
+++ b/sql/backends/monet5/sql_bincopyconvert.h
@@ -38,23 +38,30 @@
typedef str (*bincopy_decoder_t)(void *dst,void *src, size_t count, bool
byteswap);
typedef str (*bincopy_loader_t)(BAT *bat, stream *s, int *eof_reached);
-struct type_rec {
+typedef str (*bincopy_encoder_t)(void *dst, void *src, size_t count, bool
byteswap);
+typedef str (*bincopy_dumper_t)(BAT *bat, stream *s);
+
+struct type_record_t {
char *method;
char *gdk_type;
size_t record_size;
bool trivial_if_no_byteswap;
+
bincopy_decoder_t decoder;
bincopy_loader_t loader;
-};
-extern struct type_rec *find_type_rec(str name);
+ bincopy_encoder_t encoder;
+ bincopy_dumper_t dumper;
+};
+typedef const struct type_record_t type_record_t;
+
+extern type_record_t *find_type_rec(const char *name);
#define bailout(...) do { \
- msg = createException(MAL, "sql.importColumn", SQLSTATE(42000)
__VA_ARGS__); \
+ msg = createException(MAL, mal_operator, SQLSTATE(42000)
__VA_ARGS__); \
goto end; \
} while (0)
-
#endif
diff --git a/sql/backends/monet5/sql_bincopyfrom.c
b/sql/backends/monet5/sql_bincopyfrom.c
--- a/sql/backends/monet5/sql_bincopyfrom.c
+++ b/sql/backends/monet5/sql_bincopyfrom.c
@@ -12,6 +12,7 @@
#include "monetdb_config.h"
#include "mapi_prompt.h"
+#include "gdk.h"
#include "sql.h"
#include "mal_backend.h"
#include "mal_interpreter.h"
@@ -23,6 +24,7 @@
static str
load_trivial(BAT *bat, stream *s, BUN rows_estimate, int *eof_seen)
{
+ const char *mal_operator = "sql.importColumn";
str msg = MAL_SUCCEED;
int tt = BATttype(bat);
const size_t asz = (size_t) ATOMsize(tt);
@@ -94,6 +96,7 @@ end:
static str
load_fixed_width(BAT *bat, stream *s, bool byteswap, bincopy_decoder_t
convert, size_t record_size, int *eof_reached)
{
+ const char *mal_operator = "sql.importColumn";
str msg = MAL_SUCCEED;
bstream *bs = NULL;
@@ -158,10 +161,11 @@ end:
static str
-load_column(struct type_rec *rec, const char *name, BAT *bat, stream *s, bool
byteswap, BUN rows_estimate, int *eof_reached)
+load_column(type_record_t *rec, const char *name, BAT *bat, stream *s, bool
byteswap, BUN rows_estimate, int *eof_reached)
{
+ const char *mal_operator = "sql.importColumn";
+ BUN orig_count, new_count;
str msg = MAL_SUCCEED;
- BUN orig_count, new_count;
BUN rows_added;
orig_count = BATcount(bat);
@@ -199,16 +203,17 @@ load_column(struct type_rec *rec, const
/* Import a single file into a new BAT.
*/
static str
-importColumn(backend *be, bat *ret, BUN *retcnt, str method, bool byteswap,
str path, int onclient, BUN nrows)
+import_column(backend *be, bat *ret, BUN *retcnt, str method, bool byteswap,
str path, int onclient, BUN nrows)
{
// In this function we create the BAT and open the file, and tidy
// up when things go wrong. The actual work happens in load_column().
+ const str mal_operator = "sql.importColumn";
+
// These are managed by the end: block.
str msg = MAL_SUCCEED;
int gdk_type;
BAT *bat = NULL;
- stream *stream_to_close = NULL;
int eof_reached = -1; // 1 = read to the end; 0 = stopped reading
early; -1 = unset, a bug.
// This one is not managed by the end: block
@@ -219,7 +224,7 @@ importColumn(backend *be, bat *ret, BUN
*retcnt = 0;
// Figure out what kind of data we have
- struct type_rec *rec = find_type_rec(method);
+ type_record_t *rec = find_type_rec(method);
if (rec == NULL)
bailout("COPY BINARY FROM not implemented for '%s'", method);
@@ -233,13 +238,12 @@ importColumn(backend *be, bat *ret, BUN
// Open the input stream
if (onclient) {
- s = stream_to_close = mapi_request_upload(path, true,
be->mvc->scanner.rs, be->mvc->scanner.ws);
+ s = mapi_request_upload(path, true, be->mvc->scanner.rs,
be->mvc->scanner.ws);
} else {
- s = stream_to_close = open_rstream(path);
+ s = open_rstream(path);
}
if (!s) {
- msg = mnstr_error(NULL);
- goto end;
+ bailout("%s", mnstr_peek_error(NULL));
}
// Do the work
@@ -253,8 +257,8 @@ importColumn(backend *be, bat *ret, BUN
// Fall through into the end block which will clean things up
end:
- if (stream_to_close)
- close_stream(stream_to_close);
+ if (s)
+ close_stream(s);
// Manage the return values and `bat`.
if (msg == MAL_SUCCEED) {
@@ -294,7 +298,159 @@ mvc_bin_import_column_wrap(Client cntxt,
backend *be = cntxt->sqlcontext;
- return importColumn(be, ret, retcnt, method, byteswap, path, onclient,
nrows);
+ return import_column(be, ret, retcnt, method, byteswap, path, onclient,
nrows);
+}
+
+
+
+
+
+static str
+dump_trivial(BAT *b, stream *s)
+{
+ const char *mal_operator = "sql.export_bin_column";
+ str msg = MAL_SUCCEED;
+
+ int tpe = BATttype(b);
+ assert(!ATOMvarsized(tpe));
+
+ char *start = Tloc(b, 0);
+ char *end = Tloc(b, BATcount(b));
+
+ char *p = start;
+ while (p < end) {
+ ssize_t nwritten = mnstr_write(s, p, 1, end - p);
+ if (nwritten < 0)
+ bailout("%s", mnstr_peek_error(s));
+ if (nwritten == 0)
+ bailout("Unexpected EOF on %s", mnstr_name(s));
+ p += nwritten;
+ }
+
+end:
+ return msg;
+}
+
+static str
+dump_fixed_width(BAT *b, stream *s, bool byteswap, bincopy_encoder_t encoder,
size_t record_size)
+{
+ const char *mal_operator = "sql.export_bin_column";
+ (void)b;
+ (void)s;
+ (void)byteswap;
+ (void)encoder;
+ (void)record_size;
+ throw(SQL, mal_operator, "dump_fixed_width not implemented");
+}
+
+static str
+dump_column(const struct type_record_t *rec, BAT *b, bool byteswap, stream *s)
+{
+ str msg = MAL_SUCCEED;
+
+ // cannot have loader AND converter
+ assert(rec->decoder == NULL || rec->loader == NULL);
+
+ // loaders cannot be trivial
+ assert( rec->loader == NULL || !rec->trivial_if_no_byteswap);
+
+ // Temporary measure while not all dumpers have been implemented
+ assert(rec->dumper || rec->encoder || rec->trivial_if_no_byteswap ||
BATttype(b) == TYPE_bit);
+
+ if (rec->dumper) {
+ msg = rec->dumper(b, s);
+ } else if (rec->encoder == NULL || (rec->trivial_if_no_byteswap &&
!byteswap)) {
+ msg = dump_trivial(b, s);
+ } else {
+ msg = dump_fixed_width(b, s, byteswap, rec->encoder,
rec->record_size);
+ }
+
+ return msg;
+}
+
+
+static str
+export_column(backend *be, BAT *b, bool byteswap, str filename, bool onclient)
+{
+ const char *mal_operator = "sql.export_bin_column";
+ str msg = MAL_SUCCEED;
+ stream *s = NULL;
+
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
