Changeset: bab274540221 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/bab274540221
Modified Files:
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_bincopyconvert.c
sql/backends/monet5/sql_bincopyconvert.h
sql/backends/monet5/sql_bincopyfrom.c
Branch: copyintobinary
Log Message:
Validate DECIMAL range in COPY BINARY INTO
diffs (truncated from 348 to 300 lines):
diff --git a/sql/backends/monet5/rel_bin.c b/sql/backends/monet5/rel_bin.c
--- a/sql/backends/monet5/rel_bin.c
+++ b/sql/backends/monet5/rel_bin.c
@@ -1126,6 +1126,18 @@ emit_loadcolumn(backend *be, stmt *impor
// For the time being we just use the name of the storage type as the
method
// name.
const char *method = ATOMname(data_type);
+ int width;
+
+ switch (subtype->type->eclass) {
+ case EC_DEC:
+ case EC_STRING:
+ width = subtype->digits;
+ break;
+ default:
+ width = 0;
+ break;
+ }
+
//
arg("sname",str),arg("tname",str),arg("onclient",int),arg("bswap",bit)
stmt *onclient_arg = importTable_args[2];
@@ -1139,6 +1151,7 @@ emit_loadcolumn(backend *be, stmt *impor
p = pushReturn(mb, p, new_count_var);
//
p = pushStr(mb, p, method);
+ p = pushInt(mb, p, width);
p = pushArgument(mb, p, bswap_arg->nr);
p = pushArgument(mb, p, file_stmt->nr);
p = pushArgument(mb, p, onclient_arg->nr);
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -5195,7 +5195,7 @@ static mel_func sql_init_funcs[] = {
pattern("sql", "copy_from", mvc_import_table_wrap, true, "Import a table from
bstream s with the \ngiven tuple and seperators (sep/rsep)", args(1,13,
batvarargany("",0),arg("t",ptr),arg("sep",str),arg("rsep",str),arg("ssep",str),arg("ns",str),arg("fname",str),arg("nr",lng),arg("offset",lng),arg("best",int),arg("fwf",str),arg("onclient",int),arg("escape",int))),
//we use bat.single now
//pattern("sql", "single", CMDBATsingle, false, "", args(1,2,
batargany("",2),argany("x",2))),
- pattern("sql", "importColumn", mvc_bin_import_column_wrap, false, "Import a
column from the given file", args(2, 7, batargany("", 0),arg("", oid),
arg("method",str),arg("bswap",bit),arg("path",str),arg("onclient",int),arg("nrows",oid))),
+ pattern("sql", "importColumn", mvc_bin_import_column_wrap, false, "Import a
column from the given file", args(2, 8, batargany("", 0),arg("", oid),
arg("method",str),arg("width",int),arg("bswap",bit),arg("path",str),arg("onclient",int),arg("nrows",oid))),
command("aggr", "not_unique", not_unique, false, "check if the tail sorted
bat b doesn't have unique tail values", args(1,2, arg("",bit),batarg("b",oid))),
command("sql", "optimizers", getPipeCatalog, false, "", args(3,3,
batarg("",str),batarg("",str),batarg("",str))),
pattern("sql", "optimizer_updates", SQLoptimizersUpdate, false, "", noargs),
diff --git a/sql/backends/monet5/sql_bincopyconvert.c
b/sql/backends/monet5/sql_bincopyconvert.c
--- a/sql/backends/monet5/sql_bincopyconvert.c
+++ b/sql/backends/monet5/sql_bincopyconvert.c
@@ -17,8 +17,9 @@
#include "mal_interpreter.h"
static str
-validate_bit(void *dst_, void *src_, size_t count, bool byteswap)
+validate_bit(void *dst_, void *src_, size_t count, int width, bool byteswap)
{
+ (void)width;
(void)byteswap;
unsigned char *dst = dst_;
const unsigned char *src = src_;
@@ -31,55 +32,83 @@ validate_bit(void *dst_, void *src_, siz
return MAL_SUCCEED;
}
+#define VALIDATE_DECIMAL(TYP) do { \
+ if (width) { \
+ TYP m = 1; \
+ for (int i = 0; i < width; i++) \
+ m *= 10; \
+ dst = dst_; \
+ for (size_t i = 0; i < count; i++) { \
+ if (dst[i] >= m || dst[i] <= -m) \
+ throw(SQL, "convert", SQLSTATE(22003)
"decimal out of range"); \
+ } \
+ } \
+ } while (0)
+
+
static str
-byteswap_sht(void *dst_, void *src_, size_t count, bool byteswap)
+byteswap_sht(void *dst_, void *src_, size_t count, int width, bool byteswap)
{
assert(byteswap); (void)byteswap; // otherwise, why call us?
sht *dst = dst_;
const sht *src = src_;
for (size_t i = 0; i < count; i++)
*dst++ = copy_binary_byteswap16(*src++);
+
+ VALIDATE_DECIMAL(sht);
+
return MAL_SUCCEED;
}
static str
-byteswap_int(void *dst_, void *src_, size_t count, bool byteswap)
+byteswap_int(void *dst_, void *src_, size_t count, int width, bool byteswap)
{
assert(byteswap); (void)byteswap; // otherwise, why call us?
int *dst = dst_;
const int *src = src_;
for (size_t i = 0; i < count; i++)
*dst++ = copy_binary_byteswap32(*src++);
+
+ VALIDATE_DECIMAL(int);
+
return MAL_SUCCEED;
}
static str
-byteswap_lng(void *dst_, void *src_, size_t count, bool byteswap)
+byteswap_lng(void *dst_, void *src_, size_t count, int width, bool byteswap)
{
assert(byteswap); (void)byteswap; // otherwise, why call us?
lng *dst = dst_;
const lng *src = src_;
for (size_t i = 0; i < count; i++)
*dst++ = copy_binary_byteswap64(*src++);
+
+ VALIDATE_DECIMAL(lng);
+
return MAL_SUCCEED;
}
#ifdef HAVE_HGE
static str
-byteswap_hge(void *dst_, void *src_, size_t count, bool byteswap)
+byteswap_hge(void *dst_, void *src_, size_t count, int width, bool byteswap)
{
assert(byteswap); (void)byteswap; // otherwise, why call us?
hge *dst = dst_;
const hge *src = src_;
for (size_t i = 0; i < count; i++)
*dst++ = copy_binary_byteswap128(*src++);
+
+ VALIDATE_DECIMAL(hge);
+
return MAL_SUCCEED;
}
#endif
static str
-byteswap_flt(void *dst_, void *src_, size_t count, bool byteswap)
+byteswap_flt(void *dst_, void *src_, size_t count, int width, bool byteswap)
{
+ (void)width;
+
// Verify that size and alignment requirements of flt do not exceed int
assert(sizeof(uint32_t) == sizeof(flt));
assert(sizeof(struct { char dummy; uint32_t ui; }) >= sizeof(struct {
char dummy; flt f; }));
@@ -93,8 +122,10 @@ byteswap_flt(void *dst_, void *src_, siz
}
static str
-byteswap_dbl(void *dst_, void *src_, size_t count, bool byteswap)
+byteswap_dbl(void *dst_, void *src_, size_t count, int width, bool byteswap)
{
+ (void)width;
+
// Verify that size and alignment requirements of dbl do not exceed lng
assert(sizeof(uint64_t) == sizeof(dbl));
assert(sizeof(struct { char dummy; uint64_t ui; }) >= sizeof(struct {
char dummy; dbl f; }));
@@ -109,8 +140,10 @@ byteswap_dbl(void *dst_, void *src_, siz
static str
-decode_date(void *dst_, void *src_, size_t count, bool byteswap)
+decode_date(void *dst_, void *src_, size_t count, int width, bool byteswap)
{
+ (void)width;
+
date *dst = dst_;
copy_binary_date *src = src_;
@@ -127,8 +160,9 @@ decode_date(void *dst_, void *src_, size
}
static str
-encode_date(void *dst_, void *src_, size_t count, bool byteswap)
+encode_date(void *dst_, void *src_, size_t count, int width, bool byteswap)
{
+ (void)width;
copy_binary_date *dst = dst_;
date *src = src_;
for (size_t i = 0; i < count; i++) {
@@ -146,8 +180,10 @@ encode_date(void *dst_, void *src_, size
}
static str
-decode_time(void *dst_, void *src_, size_t count, bool byteswap)
+decode_time(void *dst_, void *src_, size_t count, int width, bool byteswap)
{
+ (void)width;
+
daytime *dst = dst_;
copy_binary_time *src = src_;
@@ -164,8 +200,9 @@ decode_time(void *dst_, void *src_, size
}
static str
-encode_time(void *dst_, void *src_, size_t count, bool byteswap)
+encode_time(void *dst_, void *src_, size_t count, int width, bool byteswap)
{
+ (void)width;
copy_binary_time *dst = dst_;
daytime *src = src_;
for (size_t i = 0; i < count; i++) {
@@ -184,8 +221,10 @@ encode_time(void *dst_, void *src_, size
}
static str
-decode_timestamp(void *dst_, void *src_, size_t count, bool byteswap)
+decode_timestamp(void *dst_, void *src_, size_t count, int width, bool
byteswap)
{
+ (void)width;
+
timestamp *dst = dst_;
copy_binary_timestamp *src = src_;
@@ -205,8 +244,10 @@ decode_timestamp(void *dst_, void *src_,
static str
-encode_timestamp(void *dst_, void *src_, size_t count, bool byteswap)
+encode_timestamp(void *dst_, void *src_, size_t count, int width, bool
byteswap)
{
+ (void)width;
+
copy_binary_timestamp *dst = dst_;
timestamp *src = src_;
for (size_t i = 0; i < count; i++) {
@@ -289,8 +330,9 @@ bad_utf8:
// Load items from the stream and put them in the BAT.
// Because it's text read from a binary stream, we replace \r\n with \n.
static str
-load_zero_terminated_text(BAT *bat, stream *s, int *eof_reached, bool byteswap)
+load_zero_terminated_text(BAT *bat, stream *s, int *eof_reached, int width,
bool byteswap)
{
+ (void)width;
(void)byteswap;
const char *mal_operator = "sql.importColumn";
str msg = MAL_SUCCEED;
diff --git a/sql/backends/monet5/sql_bincopyconvert.h
b/sql/backends/monet5/sql_bincopyconvert.h
--- a/sql/backends/monet5/sql_bincopyconvert.h
+++ b/sql/backends/monet5/sql_bincopyconvert.h
@@ -35,10 +35,10 @@
// the stream and the BAT and does everything necessary to load the data
into
// the BAT.
-typedef str (*bincopy_decoder_t)(void *dst,void *src, size_t count, bool
byteswap);
-typedef str (*bincopy_loader_t)(BAT *bat, stream *s, int *eof_reached, bool
byteswap);
+typedef str (*bincopy_decoder_t)(void *dst,void *src, size_t count, int width,
bool byteswap);
+typedef str (*bincopy_loader_t)(BAT *bat, stream *s, int *eof_reached, int
width, bool byteswap);
-typedef str (*bincopy_encoder_t)(void *dst, void *src, size_t count, bool
byteswap);
+typedef str (*bincopy_encoder_t)(void *dst, void *src, size_t count, int
width, bool byteswap);
typedef str (*bincopy_dumper_t)(BAT *bat, stream *s, bool byteswap);
struct type_record_t {
diff --git a/sql/backends/monet5/sql_bincopyfrom.c
b/sql/backends/monet5/sql_bincopyfrom.c
--- a/sql/backends/monet5/sql_bincopyfrom.c
+++ b/sql/backends/monet5/sql_bincopyfrom.c
@@ -94,7 +94,7 @@ end:
}
static str
-load_fixed_width(BAT *bat, stream *s, bool byteswap, bincopy_decoder_t
convert, size_t record_size, int *eof_reached)
+load_fixed_width(BAT *bat, stream *s, int width, bool byteswap,
bincopy_decoder_t convert, size_t record_size, int *eof_reached)
{
const char *mal_operator = "sql.importColumn";
str msg = MAL_SUCCEED;
@@ -129,7 +129,7 @@ load_fixed_width(BAT *bat, stream *s, bo
if (BATextend(bat, newCount) != GDK_SUCCEED)
bailout("%s", GDK_EXCEPTION);
- msg = convert(Tloc(bat, count), &bs->buf[bs->pos], n, byteswap);
+ msg = convert(Tloc(bat, count), &bs->buf[bs->pos], n, width,
byteswap);
if (msg != MAL_SUCCEED)
goto end;
BATsetcount(bat, newCount);
@@ -161,7 +161,7 @@ end:
static str
-load_column(type_record_t *rec, const char *name, BAT *bat, stream *s, bool
byteswap, BUN rows_estimate, int *eof_reached)
+load_column(type_record_t *rec, const char *name, BAT *bat, stream *s, int
width, bool byteswap, BUN rows_estimate, int *eof_reached)
{
const char *mal_operator = "sql.importColumn";
BUN orig_count, new_count;
@@ -181,9 +181,9 @@ load_column(type_record_t *rec, const ch
orig_count = BATcount(bat);
if (loader) {
- msg = loader(bat, s, eof_reached, byteswap);
+ msg = loader(bat, s, eof_reached, width, byteswap);
} else if (decoder) {
- msg = load_fixed_width(bat, s, byteswap, rec->decoder,
rec->record_size, eof_reached);
+ msg = load_fixed_width(bat, s, width, byteswap, rec->decoder,
rec->record_size, eof_reached);
// load the bytes directly into the bat, as-is
} else {
msg = load_trivial(bat, s, rows_estimate, eof_reached);
@@ -205,7 +205,7 @@ load_column(type_record_t *rec, const ch
/* Import a single file into a new BAT.
*/
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
