Changeset: f8a318294820 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/f8a318294820
Modified Files:
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql_bincopyconvert.c
sql/backends/monet5/sql_bincopyfrom.c
Branch: copyintobinary
Log Message:
Comments and variable names
diffs (234 lines):
diff --git a/sql/backends/monet5/rel_bin.c b/sql/backends/monet5/rel_bin.c
--- a/sql/backends/monet5/rel_bin.c
+++ b/sql/backends/monet5/rel_bin.c
@@ -1109,8 +1109,9 @@ exp2bin_coalesce(backend *be, sql_exp *f
return res;
}
+// This is the per-column portion of exp2bin_copyfrombinary
static stmt *
-emit_loadcolumn(backend *be, stmt *importTable_args[], int *count_var, node
*file_node, node *type_node)
+emit_loadcolumn(backend *be, stmt *onclient_stmt, stmt *bswap_stmt, int
*count_var, node *file_node, node *type_node)
{
MalBlkPtr mb = be->mb;
@@ -1126,8 +1127,8 @@ emit_loadcolumn(backend *be, stmt *impor
// For the time being we just use the name of the storage type as the
method
// name.
const char *method = ATOMname(data_type);
+
int width;
-
switch (subtype->type->eclass) {
case EC_DEC:
case EC_STRING:
@@ -1138,12 +1139,6 @@ emit_loadcolumn(backend *be, stmt *impor
break;
}
-
- //
arg("sname",str),arg("tname",str),arg("onclient",int),arg("bswap",bit)
- stmt *onclient_arg = importTable_args[2];
- stmt *bswap_arg = importTable_args[3];
-
-
int new_count_var = newTmpVariable(mb, TYPE_oid);
InstrPtr p = newStmt(mb, sqlRef, importColumnRef);
@@ -1152,9 +1147,9 @@ emit_loadcolumn(backend *be, stmt *impor
//
p = pushStr(mb, p, method);
p = pushInt(mb, p, width);
- p = pushArgument(mb, p, bswap_arg->nr);
+ p = pushArgument(mb, p, bswap_stmt->nr);
p = pushArgument(mb, p, file_stmt->nr);
- p = pushArgument(mb, p, onclient_arg->nr);
+ p = pushArgument(mb, p, onclient_stmt->nr);
if (*count_var < 0)
p = pushOid(mb, p, 0);
else
@@ -1166,6 +1161,7 @@ emit_loadcolumn(backend *be, stmt *impor
return s;
}
+// Try to predict which column will be quickest to load first
static int
node_type_score(node *n)
{
@@ -1177,49 +1173,39 @@ node_type_score(node *n)
}
static stmt*
-exp2bin_copyfrombinary(backend *be, sql_exp *fe, stmt *left, stmt *right, stmt
*isel, int depth)
+exp2bin_copyfrombinary(backend *be, sql_exp *fe, stmt *left, stmt *right, stmt
*isel)
{
- (void)depth;
mvc *sql = be->mvc;
assert(left == NULL); (void)left;
assert(right == NULL); (void)right;
assert(isel == NULL); (void)isel;
- (void)be;
- (void)fe;
sql_subfunc *f = fe->f;
list *arg_list = fe->l;
list *type_list = f->res;
-
- // There are four arguments preceding the list of files.
- // Translate them and remember the variable number of the result.
assert(4 + list_length(type_list) == list_length(arg_list));
- node *argnode = arg_list->h;
- stmt *arg_stmts[4] = { 0 };
- for (int i = 0; i < 4; i++) {
- sql_exp *arg_exp = argnode->data;
- arg_stmts[i] = exp_bin(be, arg_exp, NULL, NULL, NULL, NULL,
NULL, NULL, 0, 0, 0);
- argnode = argnode->next;
- }
-
- // If it's on server we can optimize a little
+
+ sql_exp * onclient_exp = arg_list->h->next->next->data;
+ stmt *onclient_stmt = exp_bin(be, onclient_exp, NULL, NULL, NULL, NULL,
NULL, NULL, 0, 0, 0);
+ sql_exp *bswap_exp = arg_list->h->next->next->next->data;
+ stmt *bswap_stmt = exp_bin(be, bswap_exp, NULL, NULL, NULL, NULL, NULL,
NULL, 0, 0, 0);
+
+ // If it's ON SERVER we can optimize by running the imports in parallel
bool onserver = false;
- node *onclient_arg = arg_list->h->next->next;
- sql_exp *onclient_exp = onclient_arg->data;
if (onclient_exp->type == e_atom) {
atom *onclient_atom = onclient_exp->l;
int onclient = onclient_atom->data.val.ival;
onserver = (onclient == 0);
}
- node *const first_file = argnode;
+ node *const first_file = arg_list->h->next->next->next->next;
node *const first_type = type_list->h;
node *file, *type;
// The first column we load determines the number of rows.
// We pass it on to the other columns.
// The first column to load should therefore be an 'easy' one.
- // We identify columns by their type node.
+ // We identify the columns by the address of their type node.
node *prototype_file = first_file;
node *prototype_type = first_type;
int score = node_type_score(prototype_type);
@@ -1235,7 +1221,7 @@ exp2bin_copyfrombinary(backend *be, sql_
// Emit the columns
int count_var = -1;
list *columns = sa_list(sql->sa);
- stmt *prototype_stmt = emit_loadcolumn(be, arg_stmts, &count_var,
prototype_file, prototype_type);
+ stmt *prototype_stmt = emit_loadcolumn(be, onclient_stmt, bswap_stmt,
&count_var, prototype_file, prototype_type);
if (!prototype_stmt)
return NULL;
int orig_count_var = count_var;
@@ -1244,13 +1230,16 @@ exp2bin_copyfrombinary(backend *be, sql_
if (type == prototype_type) {
s = prototype_stmt;
} else {
- s = emit_loadcolumn(be, arg_stmts, &count_var, file,
type);
+ s = emit_loadcolumn(be, onclient_stmt, bswap_stmt,
&count_var, file, type);
if (!s)
return NULL;
}
list_append(columns, s);
- if (onserver)
+ if (onserver) {
+ // Not threading the count variable from one
importColumn to the next
+ // makes it possible to run them in parallel in a
dataflow region.
count_var = orig_count_var;
+ }
}
return stmt_list(be, columns);
@@ -1421,7 +1410,7 @@ exp_bin(backend *be, sql_exp *e, stmt *l
if (strcmp(fname, "coalesce") == 0)
return exp2bin_coalesce(be, e, left, right,
sel, depth);
if (strcmp(fname, "copyfrombinary") == 0)
- return exp2bin_copyfrombinary(be, e, left,
right, sel, depth);
+ return exp2bin_copyfrombinary(be, e, left,
right, sel);
}
if (!list_empty(exps)) {
unsigned nrcols = 0;
diff --git a/sql/backends/monet5/sql_bincopyconvert.c
b/sql/backends/monet5/sql_bincopyconvert.c
--- a/sql/backends/monet5/sql_bincopyconvert.c
+++ b/sql/backends/monet5/sql_bincopyconvert.c
@@ -21,17 +21,18 @@ validate_bit(void *dst_, void *src_, siz
{
(void)width;
(void)byteswap;
- unsigned char *dst = dst_;
+ bit *dst = dst_;
const unsigned char *src = src_;
for (size_t i = 0; i < count; i++) {
if (*src > 1)
throw(SQL, "convert_bit", SQLSTATE(22003) "invalid
boolean byte value: %d", *src);
- *dst++ = *src++;
+ *dst++ = (bit)*src++;
}
return MAL_SUCCEED;
}
+// width is only nonzero for DECIMAL types. For plain integer types it is 0.
#define VALIDATE_DECIMAL(TYP) do { \
if (width) { \
TYP m = 1; \
@@ -109,7 +110,8 @@ byteswap_flt(void *dst_, void *src_, siz
{
(void)width;
- // Verify that size and alignment requirements of flt do not exceed int
+ // Verify that size and alignment requirements of flt do not exceed int.
+ // This is important because we use the int32 byteswap to byteswap the
floats.
assert(sizeof(uint32_t) == sizeof(flt));
assert(sizeof(struct { char dummy; uint32_t ui; }) >= sizeof(struct {
char dummy; flt f; }));
@@ -127,6 +129,7 @@ byteswap_dbl(void *dst_, void *src_, siz
(void)width;
// Verify that size and alignment requirements of dbl do not exceed lng
+ // This is important because we use the int64 byteswap to byteswap the
doubles.
assert(sizeof(uint64_t) == sizeof(dbl));
assert(sizeof(struct { char dummy; uint64_t ui; }) >= sizeof(struct {
char dummy; dbl f; }));
@@ -416,7 +419,6 @@ dump_zero_terminated_text(BAT *bat, stre
assert(ATOMstorage(tpe) == TYPE_str); (void)tpe;
assert(mnstr_isbinary(s));
-
BUN end = BATcount(bat);
BATiter bi = bat_iterator(bat);
for (BUN p = 0; p < end; p++) {
diff --git a/sql/backends/monet5/sql_bincopyfrom.c
b/sql/backends/monet5/sql_bincopyfrom.c
--- a/sql/backends/monet5/sql_bincopyfrom.c
+++ b/sql/backends/monet5/sql_bincopyfrom.c
@@ -105,6 +105,7 @@ load_fixed_width(BAT *bat, stream *s, in
record_size = (size_t) ATOMsize(tt);
}
+ // Read whole number of records
size_t chunk_size = 1<<20;
assert(record_size > 0);
chunk_size -= chunk_size % record_size;
@@ -184,8 +185,8 @@ load_column(type_record_t *rec, const ch
msg = loader(bat, s, eof_reached, width, byteswap);
} else if (decoder) {
msg = load_fixed_width(bat, s, width, byteswap, rec->decoder,
rec->record_size, eof_reached);
+ } else {
// load the bytes directly into the bat, as-is
- } else {
msg = load_trivial(bat, s, rows_estimate, eof_reached);
}
@@ -217,8 +218,6 @@ import_column(backend *be, bat *ret, BUN
int gdk_type;
BAT *bat = NULL;
int eof_reached = -1; // 1 = read to the end; 0 = stopped reading
early; -1 = unset, a bug.
-
- // This one is not managed by the end: block
stream *s = NULL;
// Set safe values
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
