Changeset: e2a6fb40ad46 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/e2a6fb40ad46
Modified Files:
sql/backends/monet5/sql_bincopy.c
sql/backends/monet5/sql_bincopyconvert.c
sql/backends/monet5/sql_bincopyconvert.h
Branch: Jun2023
Log Message:
separate byteswapping and validation
This should fix the decimal errors on Power.
diffs (truncated from 350 to 300 lines):
diff --git a/sql/backends/monet5/sql_bincopy.c
b/sql/backends/monet5/sql_bincopy.c
--- a/sql/backends/monet5/sql_bincopy.c
+++ b/sql/backends/monet5/sql_bincopy.c
@@ -30,7 +30,7 @@
static str
-load_trivial(BAT *bat, stream *s, BUN rows_estimate, int *eof_seen)
+load_trivial(BAT *bat, stream *s, const char *filename, bincopy_validate_t
validate, int width, BUN rows_estimate, int *eof_seen)
{
const char *mal_operator = "sql.importColumn";
str msg = MAL_SUCCEED;
@@ -62,6 +62,7 @@ load_trivial(BAT *bat, stream *s, BUN ro
char *start = Tloc(bat, validCount);
char *cur = start;
char *end = Tloc(bat, newCount);
+ char *validated = start;
while (cur < end) {
ssize_t nread = mnstr_read(s, cur, 1, end - cur);
if (nread < 0)
@@ -75,6 +76,13 @@ load_trivial(BAT *bat, stream *s, BUN ro
end = cur;
}
cur += (size_t) nread;
+ if (validate) {
+ size_t to_validate = (cur - validated) / asz;
+ msg = validate(validated, to_validate, width,
filename);
+ if (msg != MAL_SUCCEED)
+ break;
+ validated += to_validate * asz;
+ }
}
if (msg != NULL)
goto end;
@@ -102,7 +110,7 @@ end:
}
static str
-load_fixed_width(BAT *bat, stream *s, int width, bool byteswap,
bincopy_decoder_t convert, size_t record_size, int *eof_reached)
+load_fixed_width(BAT *bat, stream *s, const char *filename, int width, bool
byteswap, bincopy_decoder_t convert, bincopy_validate_t validate, size_t
record_size, int *eof_reached)
{
const char *mal_operator = "sql.importColumn";
str msg = MAL_SUCCEED;
@@ -138,7 +146,9 @@ load_fixed_width(BAT *bat, stream *s, in
if (BATextend(bat, newCount) != GDK_SUCCEED)
bailout("%s", GDK_EXCEPTION);
- msg = convert(Tloc(bat, count), &bs->buf[bs->pos], n, width,
byteswap);
+ msg = convert(Tloc(bat, count), &bs->buf[bs->pos], n, byteswap);
+ if (validate != NULL && msg == MAL_SUCCEED)
+ msg = validate(Tloc(bat, count), n, width, filename);
if (msg != MAL_SUCCEED)
goto end;
BATsetcount(bat, newCount);
@@ -192,10 +202,10 @@ load_column(type_record_t *rec, const ch
if (loader) {
msg = loader(bat, s, eof_reached, width, byteswap);
} else if (decoder) {
- msg = load_fixed_width(bat, s, width, byteswap, rec->decoder,
rec->record_size, eof_reached);
+ msg = load_fixed_width(bat, s, name, width, byteswap,
rec->decoder, rec->validate, rec->record_size, eof_reached);
} else {
// load the bytes directly into the bat, as-is
- msg = load_trivial(bat, s, rows_estimate, eof_reached);
+ msg = load_trivial(bat, s, name, rec->validate, width,
rows_estimate, eof_reached);
}
new_count = BATcount(bat);
@@ -369,7 +379,7 @@ dump_fixed_width(BAT *b, stream *s, BUN
n = end - pos;
if (n > batch_size)
n = batch_size;
- msg = encoder(buffer, Tloc(b, pos), n, 0, byteswap);
+ msg = encoder(buffer, Tloc(b, pos), n, byteswap);
if (msg != MAL_SUCCEED)
goto end;
msg = write_out(buffer, buffer + n * record_size, s);
diff --git a/sql/backends/monet5/sql_bincopyconvert.c
b/sql/backends/monet5/sql_bincopyconvert.c
--- a/sql/backends/monet5/sql_bincopyconvert.c
+++ b/sql/backends/monet5/sql_bincopyconvert.c
@@ -27,99 +27,112 @@
static str
-validate_bit(void *dst_, void *src_, size_t count, int width, bool byteswap)
+validate_bit(void *dst_, size_t count, int width, const char *filename)
{
(void)width;
- (void)byteswap;
- bit *dst = dst_;
- const unsigned char *src = src_;
+ const unsigned char *data = dst_;
for (size_t i = 0; i < count; i++) {
- if (*src > 1 && *src != 0x80)
- throw(SQL, "convert_bit", SQLSTATE(22003) "invalid
boolean byte value: %d", *src);
- *dst++ = (bit)*src++;
+ if (data[i] > 1 && data[i] != 0x80)
+ throw(SQL, "convert_bit", SQLSTATE(22003) "invalid
boolean byte value %d in %s", data[i], filename);
}
return MAL_SUCCEED;
}
// width is only nonzero for DECIMAL types. For plain integer types it is 0.
-#define VALIDATE_DECIMAL(TYP) do { \
+#define VALIDATE_DECIMAL(TYP,NIL_VALUE) do { \
if (width) { \
TYP m = 1; \
for (int i = 0; i < width; i++) \
m *= 10; \
- dst = dst_; \
+ TYP *dst = dst_; \
for (size_t i = 0; i < count; i++) { \
+ if (dst[i] == NIL_VALUE) \
+ continue; \
if (dst[i] >= m || dst[i] <= -m) \
- throw(SQL, "convert", SQLSTATE(22003)
"decimal out of range"); \
+ throw(SQL, "convert", SQLSTATE(22003)
"decimal out of range in %s", filename); \
} \
} \
} while (0)
static str
-byteswap_sht(void *dst_, void *src_, size_t count, int width, bool byteswap)
+byteswap_sht(void *dst_, void *src_, size_t count, bool byteswap)
{
assert(byteswap); (void)byteswap; // otherwise, why call us?
sht *dst = dst_;
const sht *src = src_;
for (size_t i = 0; i < count; i++)
*dst++ = copy_binary_byteswap16(*src++);
-
- VALIDATE_DECIMAL(sht);
-
return MAL_SUCCEED;
}
static str
-byteswap_int(void *dst_, void *src_, size_t count, int width, bool byteswap)
+validate_sht(void *dst_, size_t count, int width, const char *filename)
+{
+ VALIDATE_DECIMAL(sht, sht_nil);
+ return MAL_SUCCEED;
+}
+
+static str
+byteswap_int(void *dst_, void *src_, size_t count, bool byteswap)
{
assert(byteswap); (void)byteswap; // otherwise, why call us?
int *dst = dst_;
const int *src = src_;
for (size_t i = 0; i < count; i++)
*dst++ = copy_binary_byteswap32(*src++);
-
- VALIDATE_DECIMAL(int);
-
return MAL_SUCCEED;
}
static str
-byteswap_lng(void *dst_, void *src_, size_t count, int width, bool byteswap)
+validate_int(void *dst_, size_t count, int width, const char *filename)
+{
+ VALIDATE_DECIMAL(int, int_nil);
+ return MAL_SUCCEED;
+}
+
+static str
+byteswap_lng(void *dst_, void *src_, size_t count, bool byteswap)
{
assert(byteswap); (void)byteswap; // otherwise, why call us?
lng *dst = dst_;
const lng *src = src_;
for (size_t i = 0; i < count; i++)
*dst++ = copy_binary_byteswap64(*src++);
+ return MAL_SUCCEED;
+}
- VALIDATE_DECIMAL(lng);
-
+static str
+validate_lng(void *dst_, size_t count, int width, const char *filename)
+{
+ VALIDATE_DECIMAL(lng, lng_nil);
return MAL_SUCCEED;
}
#ifdef HAVE_HGE
static str
-byteswap_hge(void *dst_, void *src_, size_t count, int width, bool byteswap)
+byteswap_hge(void *dst_, void *src_, size_t count, bool byteswap)
{
assert(byteswap); (void)byteswap; // otherwise, why call us?
hge *dst = dst_;
const hge *src = src_;
for (size_t i = 0; i < count; i++)
*dst++ = copy_binary_byteswap128(*src++);
+ return MAL_SUCCEED;
+}
- VALIDATE_DECIMAL(hge);
-
+static str
+validate_hge(void *dst_, size_t count, int width, const char *filename)
+{
+ VALIDATE_DECIMAL(hge, hge_nil);
return MAL_SUCCEED;
}
#endif
static str
-byteswap_flt(void *dst_, void *src_, size_t count, int width, bool byteswap)
+byteswap_flt(void *dst_, void *src_, size_t count, bool byteswap)
{
- (void)width;
-
// Verify that size and alignment requirements of flt do not exceed int.
// This is important because we use the int32 byteswap to byteswap the
floats.
assert(sizeof(uint32_t) == sizeof(flt));
@@ -134,10 +147,8 @@ byteswap_flt(void *dst_, void *src_, siz
}
static str
-byteswap_dbl(void *dst_, void *src_, size_t count, int width, bool byteswap)
+byteswap_dbl(void *dst_, void *src_, size_t count, bool byteswap)
{
- (void)width;
-
// Verify that size and alignment requirements of dbl do not exceed lng
// This is important because we use the int64 byteswap to byteswap the
doubles.
assert(sizeof(uint64_t) == sizeof(dbl));
@@ -153,10 +164,8 @@ byteswap_dbl(void *dst_, void *src_, siz
static str
-decode_date(void *dst_, void *src_, size_t count, int width, bool byteswap)
+decode_date(void *dst_, void *src_, size_t count, bool byteswap)
{
- (void)width;
-
date *dst = dst_;
copy_binary_date *src = src_;
@@ -173,9 +182,8 @@ decode_date(void *dst_, void *src_, size
}
static str
-encode_date(void *dst_, void *src_, size_t count, int width, bool byteswap)
+encode_date(void *dst_, void *src_, size_t count, bool byteswap)
{
- (void)width;
copy_binary_date *dst = dst_;
date *src = src_;
for (size_t i = 0; i < count; i++) {
@@ -201,10 +209,8 @@ encode_date(void *dst_, void *src_, size
}
static str
-decode_time(void *dst_, void *src_, size_t count, int width, bool byteswap)
+decode_time(void *dst_, void *src_, size_t count, bool byteswap)
{
- (void)width;
-
daytime *dst = dst_;
copy_binary_time *src = src_;
@@ -221,9 +227,8 @@ decode_time(void *dst_, void *src_, size
}
static str
-encode_time(void *dst_, void *src_, size_t count, int width, bool byteswap)
+encode_time(void *dst_, void *src_, size_t count, bool byteswap)
{
- (void)width;
copy_binary_time *dst = dst_;
daytime *src = src_;
for (size_t i = 0; i < count; i++) {
@@ -252,10 +257,8 @@ encode_time(void *dst_, void *src_, size
}
static str
-decode_timestamp(void *dst_, void *src_, size_t count, int width, bool
byteswap)
+decode_timestamp(void *dst_, void *src_, size_t count, bool byteswap)
{
- (void)width;
-
timestamp *dst = dst_;
copy_binary_timestamp *src = src_;
@@ -275,10 +278,8 @@ decode_timestamp(void *dst_, void *src_,
static str
-encode_timestamp(void *dst_, void *src_, size_t count, int width, bool
byteswap)
+encode_timestamp(void *dst_, void *src_, size_t count, bool byteswap)
{
- (void)width;
-
copy_binary_timestamp *dst = dst_;
timestamp *src = src_;
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
