Changeset: ede3daea8eb0 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/ede3daea8eb0
Modified Files:
sql/backends/monet5/vaults/parquet/pqc_dict_lookup.h
sql/backends/monet5/vaults/parquet/pqc_dict_lookup_var.h
sql/backends/monet5/vaults/parquet/pqc_filemetadata.c
sql/backends/monet5/vaults/parquet/pqc_reader.c
sql/backends/monet5/vaults/parquet/pqc_thrift.c
sql/backends/monet5/vaults/parquet/pqc_thrift.h
Branch: default
Log Message:
Fixes for big-endian.
The get_uint* functions now always read little-endian, so separate
conversions that happened all over the place for big-endian are no
longer needed. Also, reading decimals now does do the necessary
conversion.
diffs (truncated from 460 to 300 lines):
diff --git a/sql/backends/monet5/vaults/parquet/pqc_dict_lookup.h
b/sql/backends/monet5/vaults/parquet/pqc_dict_lookup.h
--- a/sql/backends/monet5/vaults/parquet/pqc_dict_lookup.h
+++ b/sql/backends/monet5/vaults/parquet/pqc_dict_lookup.h
@@ -59,13 +59,13 @@ pqc_dict_lookup( pqc_creader_t *cr, void
pos += (sh)/8;
} else if (nr_bits < 16) {
for(; i < nrows && j < cr->remaining; j++, i++)
{
- uint16_t v =
pqc_sht(get_uint16(data+pos));
+ uint16_t v = get_uint16(data+pos);
uint32_t idx = (v >> sh)&mask;
sh += nr_bits;
if (sh >= 16) {
pos+=2;
sh -= 16;
- uint16_t v =
pqc_sht(get_uint16(data+pos));
+ uint16_t v =
get_uint16(data+pos);
idx |= (v << (nr_bits-sh))&mask;
}
assert(idx < cr->dict_num_values);
@@ -75,13 +75,13 @@ pqc_dict_lookup( pqc_creader_t *cr, void
pos += (sh)/8;
} else if (nr_bits < 32) {
for(; i < nrows && j < cr->remaining; j++, i++)
{
- uint32_t v =
pqc_int(get_uint32(data+pos));
+ uint32_t v = get_uint32(data+pos);
uint32_t idx = (v >> sh)&mask;
sh += nr_bits;
if (sh >= 32) {
pos+=4;
sh -= 32;
- uint32_t v =
pqc_int(get_uint32(data+pos));
+ uint32_t v =
get_uint32(data+pos);
idx |= (v << (nr_bits-sh))&mask;
}
assert(idx < cr->dict_num_values);
@@ -145,13 +145,13 @@ pqc_dict_lookup( pqc_creader_t *cr, void
}
} else if (nr_bits < 16) {
for (; i < nrows && j < m; j++, i++) {
- uint16_t v =
pqc_sht(get_uint16(data+pos));
+ uint16_t v = get_uint16(data+pos);
uint32_t idx = (v >> sh)&mask;
sh += nr_bits;
if (sh >= 16) {
pos+=2;
sh -= 16;
- uint16_t v =
pqc_sht(get_uint16(data+pos));
+ uint16_t v =
get_uint16(data+pos);
idx |= (v << (nr_bits-sh))&mask;
}
assert(idx < cr->dict_num_values);
@@ -166,13 +166,13 @@ pqc_dict_lookup( pqc_creader_t *cr, void
}
} else if (nr_bits < 32) {
for (; i < nrows && j < m; j++, i++) {
- uint32_t v =
pqc_int(get_uint32(data+pos));
+ uint32_t v = get_uint32(data+pos);
uint32_t idx = (v >> sh)&mask;
sh += nr_bits;
if (sh >= 32) {
pos+=4;
sh -= 32;
- uint32_t v =
pqc_int(get_uint32(data+pos));
+ uint32_t v =
get_uint32(data+pos);
idx |= (v << (nr_bits-sh))&mask;
}
assert(idx < cr->dict_num_values);
@@ -210,7 +210,7 @@ pqc_dict_lookup( pqc_creader_t *cr, void
}
} else if (nr_bits <= 16) { /* rle */
len>>=1;
- uint16_t idx = pqc_sht(get_uint16(data+pos));
+ uint16_t idx = get_uint16(data+pos);
idx &= mask;
uint32_t j = 0;
pos += 2;
@@ -223,7 +223,7 @@ pqc_dict_lookup( pqc_creader_t *cr, void
}
} else if (nr_bits <= 24) { /* rle */
len>>=1;
- uint32_t idx = pqc_int(get_uint32(data+pos));
+ uint32_t idx = get_uint32(data+pos);
idx &= mask;
uint32_t j = 0;
pos += 3;
diff --git a/sql/backends/monet5/vaults/parquet/pqc_dict_lookup_var.h
b/sql/backends/monet5/vaults/parquet/pqc_dict_lookup_var.h
--- a/sql/backends/monet5/vaults/parquet/pqc_dict_lookup_var.h
+++ b/sql/backends/monet5/vaults/parquet/pqc_dict_lookup_var.h
@@ -60,13 +60,13 @@ pqc_dict_lookup( pqc_creader_t *cr, void
pos += (sh)/8;
} else if (nr_bits < 16) {
for(; i < nrows && j < cr->remaining; j++, i++)
{
- uint16_t v =
pqc_sht(get_uint16(data+pos));
+ uint16_t v = get_uint16(data+pos);
uint32_t idx = (v >> sh)&mask;
sh += nr_bits;
if (sh >= 16) {
pos+=2;
sh -= 16;
- uint16_t v =
pqc_sht(get_uint16(data+pos));
+ uint16_t v =
get_uint16(data+pos);
idx |= (v << (nr_bits-sh))&mask;
}
assert(idx < cr->dict_num_values);
@@ -76,13 +76,13 @@ pqc_dict_lookup( pqc_creader_t *cr, void
pos += (sh)/8;
} else if (nr_bits < 32) {
for(; i < nrows && j < cr->remaining; j++, i++)
{
- uint32_t v =
pqc_int(get_uint32(data+pos));
+ uint32_t v = get_uint32(data+pos);
uint32_t idx = (v >> sh)&mask;
sh += nr_bits;
if (sh >= 32) {
pos+=4;
sh -= 32;
- uint32_t v =
pqc_int(get_uint32(data+pos));
+ uint32_t v =
get_uint32(data+pos);
idx |= (v << (nr_bits-sh))&mask;
}
assert(idx < cr->dict_num_values);
@@ -156,13 +156,13 @@ pqc_dict_lookup( pqc_creader_t *cr, void
}
} else if (nr_bits < 16) {
for (; i < nrows && j < m; j++, i++) {
- uint16_t v =
pqc_sht(get_uint16(data+pos));
+ uint16_t v = get_uint16(data+pos);
uint32_t idx = (v >> sh)&mask;
sh += nr_bits;
if (sh >= 16) {
pos+=2;
sh -= 16;
- uint16_t v =
pqc_sht(get_uint16(data+pos));
+ uint16_t v =
get_uint16(data+pos);
idx |= (v << (nr_bits-sh))&mask;
}
assert(idx < cr->dict_num_values);
@@ -177,13 +177,13 @@ pqc_dict_lookup( pqc_creader_t *cr, void
}
} else if (nr_bits < 32) {
for (; i < nrows && j < m; j++, i++) {
- uint32_t v =
pqc_int(get_uint32(data+pos));
+ uint32_t v = get_uint32(data+pos);
uint32_t idx = (v >> sh)&mask;
sh += nr_bits;
if (sh >= 32) {
pos+=4;
sh -= 32;
- uint32_t v =
pqc_int(get_uint32(data+pos));
+ uint32_t v =
get_uint32(data+pos);
idx |= (v << (nr_bits-sh))&mask;
}
assert(idx < cr->dict_num_values);
@@ -219,7 +219,7 @@ pqc_dict_lookup( pqc_creader_t *cr, void
}
} else if (nr_bits <= 16) { /* rle */
len>>=1;
- uint16_t idx = pqc_sht(get_uint16(data+pos));
+ uint16_t idx = get_uint16(data+pos);
pos += 2;
for(; i < nrows && j < len; j++, i++)
dst[i] = offset+offsets[idx];
@@ -230,7 +230,7 @@ pqc_dict_lookup( pqc_creader_t *cr, void
}
} else if (nr_bits <= 32) { /* rle */
len>>=1;
- uint32_t idx = pqc_int(get_uint32(data+pos));
+ uint32_t idx = get_uint32(data+pos);
pos += 4;
for(; i < nrows && j < len; j++, i++)
dst[i] = offset+offsets[idx];
diff --git a/sql/backends/monet5/vaults/parquet/pqc_filemetadata.c
b/sql/backends/monet5/vaults/parquet/pqc_filemetadata.c
--- a/sql/backends/monet5/vaults/parquet/pqc_filemetadata.c
+++ b/sql/backends/monet5/vaults/parquet/pqc_filemetadata.c
@@ -851,8 +851,8 @@ pqc_binary2string(pqc_schema_element *ps
if (pse->size == 64)
return snprintf(buf, sz, LLFMT,
*(int64_t*)bindata);
if (pse->size == 96) {
- uint64_t nanoseconds =
pqc_lng(get_uint64((unsigned char*)bindata));
- uint32_t julian_day =
pqc_int(get_uint32((unsigned char*)bindata+8));
+ uint64_t nanoseconds = get_uint64((unsigned
char*)bindata);
+ uint32_t julian_day = get_uint32((unsigned
char*)bindata+8);
nanoseconds /= LL_CONSTANT(1000);
julian_day -= 2440588;
diff --git a/sql/backends/monet5/vaults/parquet/pqc_reader.c
b/sql/backends/monet5/vaults/parquet/pqc_reader.c
--- a/sql/backends/monet5/vaults/parquet/pqc_reader.c
+++ b/sql/backends/monet5/vaults/parquet/pqc_reader.c
@@ -783,21 +783,21 @@ pqc_page_header( pqc_reader_t *r, pqc_cr
unsigned char *s = (unsigned char*)pr->dict;
if (r->pse->size == 16)
for(uint32_t i = 0; i < num_values; i++,
s+=sizeof(uint16_t))
- *(uint16_t*)s = pqc_sht(get_uint16(s));
+ *(uint16_t*)s = get_uint16(s);
if (r->pse->size == 32)
for(uint32_t i = 0; i < num_values; i++,
s+=sizeof(uint32_t))
- *(uint32_t*)s = pqc_int(get_uint32(s));
+ *(uint32_t*)s = get_uint32(s);
if (r->pse->size == 64)
for(uint32_t i = 0; i < num_values; i++,
s+=sizeof(uint64_t))
- *(uint64_t*)s = pqc_lng(get_uint64(s));
+ *(uint64_t*)s = get_uint64(s);
}
#endif
if (num_values && r->pse->type == inttype && r->pse->precision
== 96) { /* remove the 32 useless bits */
ulng *d = (ulng*)pr->dict;
unsigned char *s = (unsigned char*)pr->dict;
for(uint32_t i = 0; i < num_values; i++, s += 12) {
- uint64_t nanoseconds = pqc_lng(get_uint64(s));
- uint32_t julian_day = pqc_int(get_uint32(s+8));
+ uint64_t nanoseconds = get_uint64(s);
+ uint32_t julian_day = get_uint32(s+8);
nanoseconds /= LL_CONSTANT(1000);
julian_day -= 2440588;
@@ -1298,13 +1298,13 @@ pqc_dict_lookup( pqc_reader_t *r, pqc_cr
} else if (nr_bits < 16) {
int m = len*8;
for (int64_t j = 0; i < nrows && j < m;
j++, i++) {
- uint16_t v =
pqc_sht(get_uint16(data+pos));
+ uint16_t v =
get_uint16(data+pos);
uint32_t idx = (v >> sh)&mask;
sh += nr_bits;
if (sh >= 16) {
pos+=2;
sh -= 16;
- uint16_t v =
pqc_sht(get_uint16(data+pos));
+ uint16_t v =
get_uint16(data+pos);
idx |= (v <<
(nr_bits-sh))&mask;
if (j==(m-1) && sh >= 8)
pos++;
@@ -1317,13 +1317,13 @@ pqc_dict_lookup( pqc_reader_t *r, pqc_cr
} else if (nr_bits < 32) {
int m = len*8;
for (int64_t j = 0; i < nrows && j < m;
j++, i++) {
- uint32_t v =
pqc_int(get_uint32(data+pos));
+ uint32_t v =
get_uint32(data+pos);
uint32_t idx = (v >> sh)&mask;
sh += nr_bits;
if (sh >= 32) {
pos+=2;
sh -= 32;
- uint32_t v =
pqc_int(get_uint32(data+pos));
+ uint32_t v =
get_uint32(data+pos);
idx |= (v <<
(nr_bits-sh))&mask;
if (j==(m-1) && sh >= 8)
pos++;
@@ -1357,7 +1357,7 @@ pqc_dict_lookup( pqc_reader_t *r, pqc_cr
}
} else if (nr_bits <= 16) { /* rle */
len>>=1;
- uint16_t idx = pqc_sht(get_uint16(data+pos));
+ uint16_t idx = get_uint16(data+pos);
uint32_t j = 0;
pos += 2;
for(; i < nrows && j < len; j++, i++)
@@ -1369,7 +1369,7 @@ pqc_dict_lookup( pqc_reader_t *r, pqc_cr
}
} else if (nr_bits <= 32) { /* rle */
len>>=1;
- uint32_t idx = pqc_int(get_uint32(data+pos));
+ uint32_t idx = get_uint32(data+pos);
uint32_t j = 0;
pos += 4;
for(; i < nrows && j < len; j++, i++)
@@ -1995,7 +1995,7 @@ pqc_read_page_chunk( pqc_reader_t *r, pq
if (!voutput) {
size_t
blobsizes = 0;
for (uint32_t i
= 0; i < nrows; i++) {
- int len
= pqc_int(get_uint32(src));
+ int len
= get_uint32(src);
src +=
sizeof(int) + len;
blobsizes += (sizeof(size_t) + len +15)&(~15);
}
@@ -2007,7 +2007,7 @@ pqc_read_page_chunk( pqc_reader_t *r, pq
char *dst =
voutput, *fdst = voutput;
for (uint32_t i
= 0; i < nrows; i++) {
p[i] =
offset + (dst - fdst);
- int len
= pqc_int(get_uint32(src));
+ int len
= get_uint32(src);
src +=
sizeof(int);
size_t
*n = (size_t*)dst;
*n =
len;
@@ -2023,7 +2023,7 @@ pqc_read_page_chunk( pqc_reader_t *r, pq
int width =
pse_get_width(r->pse);
/* FIXED ARRAY little
endian len followed by big endian data, what where they smoking */
for (uint32_t i = 0; i
< nrows; i++, dst += width) {
- int len =
pqc_int(get_uint32(((unsigned char*)cr->data)+pos));
+ int len =
get_uint32(((unsigned char*)cr->data)+pos);
pos += 4;
*(uint16_t*)dst
= 0;
int offset =
width - len;
@@ -2058,8 +2058,8 @@ pqc_read_page_chunk( pqc_reader_t *r, pq
ulng *d = output;
unsigned char *s = ((unsigned
char*)cr->data)+pos;
for(uint64_t i = 0; i < nrows;
i++, s += 12) {
- uint64_t nanoseconds =
pqc_lng(get_uint64(s));
- uint32_t julian_day =
pqc_int(get_uint32(s+8));
+ uint64_t nanoseconds =
get_uint64(s);
+ uint32_t julian_day =
get_uint32(s+8);
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
