Changeset: 759aab4cdb4c for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/759aab4cdb4c
Added Files:
sql/backends/monet5/vaults/parquet/Tests/test_parquet_unsigned.test.in
Modified Files:
sql/backends/monet5/vaults/parquet/Tests/All
sql/backends/monet5/vaults/parquet/parquet.c
sql/backends/monet5/vaults/parquet/pqc_reader.c
Branch: default
Log Message:
as monetdb doesn't have unsigned ints, parquet reader up casts unsigneds into
signed integers with enough space.
diffs (160 lines):
diff --git a/sql/backends/monet5/vaults/parquet/Tests/All
b/sql/backends/monet5/vaults/parquet/Tests/All
--- a/sql/backends/monet5/vaults/parquet/Tests/All
+++ b/sql/backends/monet5/vaults/parquet/Tests/All
@@ -1,3 +1,4 @@
#invalid_parquet
HAVE_SNAPPY?test_parquet_reader
HAVE_DATA_PATH&HAVE_ZSTD&HAVE_SNAPPY?parquet_testing
+test_parquet_unsigned
diff --git
a/sql/backends/monet5/vaults/parquet/Tests/test_parquet_unsigned.test.in
b/sql/backends/monet5/vaults/parquet/Tests/test_parquet_unsigned.test.in
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/vaults/parquet/Tests/test_parquet_unsigned.test.in
@@ -0,0 +1,27 @@
+query IIII nosort
+SELECT * FROM '$QTSTDATAPATH/parquet/parquet_testing/unsigned.parquet'
+----
+1
+1
+1
+1
+2
+2
+2
+2
+3
+3
+3
+3
+4
+4
+4
+4
+5
+5
+5
+5
+255
+65535
+4294967295
+18446744073709551615
diff --git a/sql/backends/monet5/vaults/parquet/parquet.c
b/sql/backends/monet5/vaults/parquet/parquet.c
--- a/sql/backends/monet5/vaults/parquet/parquet.c
+++ b/sql/backends/monet5/vaults/parquet/parquet.c
@@ -244,6 +244,28 @@ pqc_find_subtype(mvc *sql, const pqc_sch
return tpe;
break;
case inttype:
+ if (!pse->isSigned) {
+ switch(pse->precision) {
+ case 8:
+ if (sql_find_subtype(tpe,
"smallint", pse->precision + 1, 0))
+ return tpe;
+ break;
+ case 16:
+ if (sql_find_subtype(tpe,
"int", pse->precision + 1, 0))
+ return tpe;
+ break;
+ case 32:
+ if (sql_find_subtype(tpe,
"bigint", pse->precision + 1, 0))
+ return tpe;
+ break;
+ case 64:
+ if (sql_find_subtype(tpe,
"hugeint", pse->precision + 1, 0))
+ return tpe;
+ break;
+ }
+ }
+ if (pse->precision == 8 && sql_find_subtype(tpe,
"tinyint", pse->precision, 0))
+ return tpe;
if (pse->precision == 8 && sql_find_subtype(tpe,
"tinyint", pse->precision, 0))
return tpe;
if (pse->precision == 16 && sql_find_subtype(tpe,
"smallint", pse->precision, 0))
@@ -252,7 +274,6 @@ pqc_find_subtype(mvc *sql, const pqc_sch
return tpe;
if (pse->precision == 64 && sql_find_subtype(tpe,
"bigint", pse->precision, 0))
return tpe;
- //if (pse->size == 96 && sql_find_subtype(tpe,
"hugeint", pse->precision, 0))
if (pse->precision == 96 && sql_find_subtype(tpe,
"timestamp", 6, 0))
return tpe;
break;
@@ -321,18 +342,27 @@ pqc_find_localtype(const pqc_schema_elem
return TYPE_bte;
break;
case inttype:
- //if (pse->size == 8)
- if (pse->precision == 8)
- return TYPE_bte;
- //if (pse->size == 16)
- if (pse->precision == 16)
- return TYPE_sht;
- //if (pse->size == 32)
- if (pse->precision == 32)
- return TYPE_int;
- //if (pse->size == 64)
- if (pse->precision == 64)
- return TYPE_lng;
+ if (!pse->isSigned) {
+ if (pse->precision == 8)
+ return TYPE_sht;
+ if (pse->precision == 16)
+ return TYPE_int;
+ if (pse->precision == 32)
+ return TYPE_lng;
+#ifdef HAVE_HGE
+ if (pse->precision == 64)
+ return TYPE_hge;
+#endif
+ } else {
+ if (pse->precision == 8)
+ return TYPE_bte;
+ if (pse->precision == 16)
+ return TYPE_sht;
+ if (pse->precision == 32)
+ return TYPE_int;
+ if (pse->precision == 64)
+ return TYPE_lng;
+ }
if (pse->precision == 96)
return TYPE_timestamp;
#ifdef HAVE_HGE
diff --git a/sql/backends/monet5/vaults/parquet/pqc_reader.c
b/sql/backends/monet5/vaults/parquet/pqc_reader.c
--- a/sql/backends/monet5/vaults/parquet/pqc_reader.c
+++ b/sql/backends/monet5/vaults/parquet/pqc_reader.c
@@ -2100,7 +2100,35 @@ pqc_read_page_chunk( pqc_reader_t *r, pq
}
}
/* convert data */
- if (r->pse->type == datetype) {
+ if (r->pse->precision == 64 && !r->pse->isSigned) {
+ uint64_t *l = output;
+ l += nrows-1;
+ int128_t *r = output;
+ r += nrows-1;
+ for(uint64_t i=0; i< nrows; i++)
+ r[-i] = l[-i];
+ } else if (r->pse->precision == 32 && !r->pse->isSigned) {
+ uint32_t *l = output;
+ l += nrows-1;
+ int64_t *r = output;
+ r += nrows-1;
+ for(uint64_t i=0; i< nrows; i++)
+ r[-i] = l[-i];
+ } else if (r->pse->precision == 16 && !r->pse->isSigned) {
+ uint16_t *l = output;
+ l += nrows-1;
+ int32_t *r = output;
+ r += nrows-1;
+ for(uint64_t i=0; i< nrows; i++)
+ r[-i] = l[-i];
+ } else if (r->pse->precision == 8 && !r->pse->isSigned) {
+ uint8_t *l = output;
+ l += nrows-1;
+ int16_t *r = output;
+ r += nrows-1;
+ for(uint64_t i=0; i< nrows; i++)
+ r[-i] = l[-i];
+ } else if (r->pse->type == datetype) {
/* days since epoch ie 1-1-1970 */
uint32_t epoch_date = mkdate(1970, 1, 1);
if (r->pse->precision == 32) {
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
