Changeset: 751d48508542 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/751d48508542
Added Files:
sql/test/bincopy/Tests/bincopy_blobs_on_client.SQL.py
sql/test/bincopy/Tests/bincopy_blobs_on_server.SQL.py
Modified Files:
clients/examples/C/bincopydata.c
sql/backends/monet5/sql_bincopyconvert.c
sql/test/bincopy/Tests/All
sql/test/bincopy/Tests/bincopy_support.py
Branch: binresultset
Log Message:
Add COPY BINARY support for BLOBs
Format: one 64 bit integer indicating the length of the blob in
bytes, followed by the blob bytes. No padding.
The byte order of the header word is governed by the byte order
specified in the COPY command, e.g., COPY BIG ENDIAN BINARY
uses big endian integers.
NULLs are indicated by header 0xFFFF_FFFF_FFFF_FFFF.
diffs (280 lines):
diff --git a/clients/examples/C/bincopydata.c b/clients/examples/C/bincopydata.c
--- a/clients/examples/C/bincopydata.c
+++ b/clients/examples/C/bincopydata.c
@@ -210,6 +210,34 @@ gen_null_strings(FILE *f, bool byteswap,
}
static void
+gen_null_blobs(FILE *f, bool byteswap, long nrecs)
+{
+ uint8_t *buffer = malloc(nrecs);
+ for (long i = 0; i < nrecs; i++) {
+ buffer[i] = 0xD0 + 3 - (i % 3);
+ }
+
+ for (long i = 0; i < nrecs; i++) {
+ uint64_t header;
+ size_t n;
+ if (i % 3 == 2) {
+ // null
+ n = 0;
+ header = (uint64_t)-1;
+ } else {
+ n = (i % 1000);
+ header = n;
+ }
+ if (byteswap)
+ copy_binary_convert64(&header);
+ assert(sizeof(header) == 8);
+ fwrite(&header, sizeof(header), 1, f);
+ if (n > 0)
+ fwrite(buffer, 1, n, f);
+ }
+}
+
+static void
gen_json(FILE *f, bool byteswap, long nrecs)
{
(void)byteswap;
@@ -245,6 +273,7 @@ static struct gen {
{ "broken_strings", gen_broken_strings },
{ "newline_strings", gen_newline_strings },
{ "null_strings", gen_null_strings },
+ { "null_blobs", gen_null_blobs },
//
{ "timestamps", gen_timestamps },
{ "timestamp_times", gen_timestamp_times },
diff --git a/sql/backends/monet5/sql_bincopyconvert.c
b/sql/backends/monet5/sql_bincopyconvert.c
--- a/sql/backends/monet5/sql_bincopyconvert.c
+++ b/sql/backends/monet5/sql_bincopyconvert.c
@@ -394,6 +394,126 @@ end:
}
+// Read BLOBs. Every blob is preceded by a 64bit header word indicating its
length.
+// NULLs are indicated by length==-1
+static str
+load_blob(BAT *bat, stream *s, int *eof_reached, int width, bool byteswap)
+{
+ (void)width;
+ const char *mal_operator = "sql.importColumn";
+ str msg = MAL_SUCCEED;
+ const blob *nil_value = ATOMnilptr(TYPE_blob);
+ blob *buffer = NULL;
+ size_t buffer_size = 0;
+ union {
+ uint64_t length;
+ char bytes[8];
+ } header;
+
+ *eof_reached = 0;
+
+ while (1) {
+ const blob *value;
+ // Read the header
+ ssize_t nread = mnstr_read(s, header.bytes, 1, 8);
+ if (nread < 0) {
+ bailout("%s", mnstr_peek_error(s));
+ } else if (nread == 0) {
+ *eof_reached = 1;
+ break;
+ } else if (nread < 8) {
+ bailout("incomplete blob at end of file");
+ }
+ if (byteswap) {
+ copy_binary_convert64(&header.length);
+ }
+
+ if (header.length == ~(uint64_t)0) {
+ value = nil_value;
+ } else {
+ size_t length;
+ size_t needed;
+
+ if (header.length >= VAR_MAX) {
+ bailout("blob too long");
+ }
+ length = (size_t) header.length;
+
+ // Reallocate the buffer
+ needed = sizeof(blob) + length;
+ if (buffer_size < needed) {
+ // do not use GDKrealloc, no need to copy the
old contents
+ GDKfree(buffer);
+ size_t allocate = needed;
+ allocate += allocate / 16; // add a little
margin
+ allocate += ((-allocate) % 0x100000); //
round up to nearest MiB
+ assert(allocate >= needed);
+ buffer = GDKmalloc(allocate);
+ if (!buffer) {
+ msg = createException(SQL, "sql",
SQLSTATE(HY013) MAL_MALLOC_FAIL);
+ goto end;
+ }
+ buffer_size = allocate;
+ }
+
+ // Fill the buffer
+ buffer->nitems = length;
+ if (length > 0) {
+ nread = mnstr_read(s, buffer->data, length, 1);
+ if (nread < 0) {
+ bailout("%s", mnstr_peek_error(s));
+ } else if (nread != 1) {
+ bailout("Incomplete blob at end of
file");
+ }
+ }
+
+ value = buffer;
+ }
+
+ if (BUNappend(bat, value, false) != GDK_SUCCEED) {
+ msg = createException(SQL, mal_operator,
GDK_EXCEPTION);
+ goto end;
+ }
+ }
+
+end:
+ GDKfree(buffer);
+ return msg;
+}
+
+static str
+dump_blob(BAT *bat, stream *s, BUN start, BUN length, bool byteswap)
+{
+ const char *mal_operator = "sql.export_bin_column";
+ str msg = MAL_SUCCEED;
+ int tpe = BATttype(bat);
+ assert(ATOMstorage(tpe) == TYPE_blob); (void)tpe;
+ assert(mnstr_isbinary(s));
+
+ BUN end = start + length;
+ assert(end <= BATcount(bat));
+ BATiter bi = bat_iterator(bat);
+ uint64_t nil_header = ~(uint64_t)0;
+ for (BUN p = start; p < end; p++) {
+ const blob *b = BUNtvar(bi, p);
+ uint64_t header = is_blob_nil(b) ? nil_header :
(uint64_t)b->nitems;
+ if (byteswap)
+ copy_binary_convert64(&header);
+ if (mnstr_write(s, &header, 8, 1) != 1) {
+ bailout("%s", mnstr_peek_error(s));
+ }
+ if (!is_blob_nil(b) && mnstr_write(s, b->data,b->nitems, 1) !=
1) {
+ bailout("%s", mnstr_peek_error(s));
+ }
+ }
+
+end:
+ bat_iterator_end(&bi);
+ return msg;
+
+}
+
+
static struct type_record_t type_recs[] = {
// no conversion, no byteswapping
@@ -413,6 +533,8 @@ static struct type_record_t type_recs[]
{ "hge", "hge", .trivial_if_no_byteswap=true, .decoder=byteswap_hge,
.encoder=byteswap_hge},
#endif
+ { "blob", "blob", .loader=load_blob, .dumper=dump_blob },
+
// \0-terminated text records
{ "str", "str", .loader=load_zero_terminated_text,
.dumper=dump_zero_terminated_text },
{ "url", "url", .loader=load_zero_terminated_text,
.dumper=dump_zero_terminated_text },
diff --git a/sql/test/bincopy/Tests/All b/sql/test/bincopy/Tests/All
--- a/sql/test/bincopy/Tests/All
+++ b/sql/test/bincopy/Tests/All
@@ -46,6 +46,8 @@ bincopy_big_endians_on_client
bincopy_big_endians_on_server
bincopy_native_endians_on_client
bincopy_native_endians_on_server
+bincopy_blobs_on_client
+bincopy_blobs_on_server
bincopy_invalid_json
bincopy_default_values
diff --git a/sql/test/bincopy/Tests/bincopy_blobs_on_client.SQL.py
b/sql/test/bincopy/Tests/bincopy_blobs_on_client.SQL.py
new file mode 100644
--- /dev/null
+++ b/sql/test/bincopy/Tests/bincopy_blobs_on_client.SQL.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+sys.path.append(os.getenv('TSTSRCDIR'))
+from bincopy_support import run_test
+from bincopy_support import NULL_BLOBS, NULL_BLOBS_LE, NULL_BLOBS_BE
+
+run_test('client', NULL_BLOBS)
+
+run_test('client', NULL_BLOBS_LE)
+
+run_test('client', NULL_BLOBS_BE)
diff --git a/sql/test/bincopy/Tests/bincopy_blobs_on_server.SQL.py
b/sql/test/bincopy/Tests/bincopy_blobs_on_server.SQL.py
new file mode 100644
--- /dev/null
+++ b/sql/test/bincopy/Tests/bincopy_blobs_on_server.SQL.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+
+import sys
+
+import os
+sys.path.append(os.getenv('TSTSRCDIR'))
+from bincopy_support import run_test
+from bincopy_support import NULL_BLOBS, NULL_BLOBS_LE, NULL_BLOBS_BE
+
+run_test('server', NULL_BLOBS)
+
+run_test('server', NULL_BLOBS_LE)
+
+run_test('server', NULL_BLOBS_BE)
diff --git a/sql/test/bincopy/Tests/bincopy_support.py
b/sql/test/bincopy/Tests/bincopy_support.py
--- a/sql/test/bincopy/Tests/bincopy_support.py
+++ b/sql/test/bincopy/Tests/bincopy_support.py
@@ -188,6 +188,42 @@ WHERE (id % 2 = 0 AND s IS NULL)
OR (id % 2 = 1 AND s = 'banana');
""", [f"{NRECS}"])
+NULL_BLOBS = ("""
+CREATE TABLE foo(id INT NOT NULL, b BLOB);
+COPY BINARY INTO foo(id, b) FROM @ints@, @null_blobs@ @ON@;
+COPY SELECT id, b FROM foo INTO BINARY @>ints@, @>null_blobs@ @ON@;
+SELECT
+ (SELECT COUNT(*) FROM foo WHERE (b IS NULL) <> (id % 3 = 2)) AS
nulls_wrong,
+ (SELECT COUNT(*) FROM foo WHERE b IS NOT NULL AND id % 1000 <> length(b))
AS lengths_wrong,
+ (SELECT b FROM foo WHERE id = 6) AS blob5
+ ;
+""", ["0,0,D3D2D1D3D2D1"])
+
+NULL_BLOBS_LE = ("""
+CREATE TABLE foo(id INT NOT NULL, b BLOB);
+COPY LITTLE ENDIAN BINARY INTO foo(id, b) FROM @le_ints@, @le_null_blobs@ @ON@;
+COPY SELECT id, b FROM foo INTO LITTLE ENDIAN BINARY @>le_ints@,
@>le_null_blobs@ @ON@;
+SELECT
+ (SELECT COUNT(*) FROM foo WHERE (b IS NULL) <> (id % 3 = 2)) AS
nulls_wrong,
+ (SELECT COUNT(*) FROM foo WHERE b IS NOT NULL AND id % 1000 <> length(b))
AS lengths_wrong,
+ (SELECT b FROM foo WHERE id = 6) AS blob5
+ ;
+""", ["0,0,D3D2D1D3D2D1"])
+
+
+NULL_BLOBS_BE = ("""
+CREATE TABLE foo(id INT NOT NULL, b BLOB);
+COPY BIG ENDIAN BINARY INTO foo(id, b) FROM @be_ints@, @be_null_blobs@ @ON@;
+COPY SELECT id, b FROM foo INTO BIG ENDIAN BINARY @>be_ints@, @>be_null_blobs@
@ON@;
+SELECT
+ (SELECT COUNT(*) FROM foo WHERE (b IS NULL) <> (id % 3 = 2)) AS
nulls_wrong,
+ (SELECT COUNT(*) FROM foo WHERE b IS NOT NULL AND id % 1000 <> length(b))
AS lengths_wrong,
+ (SELECT b FROM foo WHERE id = 6) AS blob5
+ ;
+""", ["0,0,D3D2D1D3D2D1"])
+
+
+
TIMESTAMPS = ("""
CREATE TABLE foo(
id INT NOT NULL,
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
