Changeset: e659665a8b8e for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/e659665a8b8e
Modified Files:
sql/storage/bat/bat_storage.c
Branch: default
Log Message:
Merged with Jan2022
diffs (truncated from 628 to 300 lines):
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -2413,7 +2413,7 @@ string_writer(logger *lg, BAT *b, lng of
}
char *dst = buf;
for(; p < end && sz < bufsz; p++) {
- char *s = BUNtail(bi, p);
+ char *s = BUNtvar(bi, p);
size_t len = strlen(s)+1;
if ((sz+len) > bufsz) {
if (len > bufsz)
@@ -2641,6 +2641,10 @@ log_delta(logger *lg, BAT *uid, BAT *uva
if (uval->ttype == TYPE_msk) {
if (!mnstr_writeIntArray(lg->output_log, vi.base,
(BUNlast(uval) + 31) / 32))
ok = GDK_FAIL;
+ } else if (uval->ttype < TYPE_str && !isVIEW(uval)) {
+ const void *t = BUNtail(vi, 0);
+
+ ok = wt(t, lg->output_log, (size_t)nr);
} else if (uval->ttype == TYPE_str) {
/* efficient string writes */
ok = string_writer(lg, uval, 0, nr);
diff --git a/sql/backends/monet5/dict.c b/sql/backends/monet5/dict.c
--- a/sql/backends/monet5/dict.c
+++ b/sql/backends/monet5/dict.c
@@ -33,8 +33,8 @@ static void
BATmaxminpos_bte(BAT *o, bte m)
{
BUN minpos = BUN_NONE, maxpos = BUN_NONE, p, q;
- bte minval = m<0?-128:0;
- bte maxval = m<0?127:m;
+ bte minval = m<0?GDK_bte_min:0; /* Later once nils use a bitmask we can
include -128 in the range */
+ bte maxval = m<0?GDK_bte_max:m;
assert(o->ttype == TYPE_bte);
o->tnil = m<0?true:false;
@@ -60,8 +60,8 @@ static void
BATmaxminpos_sht(BAT *o, sht m)
{
BUN minpos = BUN_NONE, maxpos = BUN_NONE, p, q;
- sht minval = m<0?-32768:0;
- sht maxval = m<0?32767:m;
+ sht minval = m<0?GDK_sht_min:0; /* Later once nils use a bitmask we can
include -32768 in the range */
+ sht maxval = m<0?GDK_sht_max:m;
assert(o->ttype == TYPE_sht);
o->tnil = m<0?true:false;
@@ -279,7 +279,21 @@ DICTcompress_col(Client cntxt, MalBlkPtr
return msg;
}
-/* improve decompress of hge types */
+#define decompress_loop(TPE) \
+ do { \
+ TPE *up = Tloc(u, 0); \
+ TPE *restrict bp = Tloc(b, 0); \
+ BATloop(o, p, q) { \
+ TPE v = up[op[p]]; \
+ nils |= is_##TPE##_nil(v); \
+ bp[p] = v; \
+ } \
+ BATsetcount(b, BATcount(o)); \
+ BATnegateprops(b); \
+ b->tnil = nils; \
+ b->tnonil = !nils; \
+ } while (0)
+
BAT *
DICTdecompress_(BAT *o, BAT *u, role_t role)
{
@@ -294,33 +308,19 @@ DICTdecompress_(BAT *o, BAT *u, role_t r
if (o->ttype == TYPE_bte) {
unsigned char *op = Tloc(o, 0);
- if (ATOMbasetype(u->ttype) == TYPE_int) {
- int *up = Tloc(u, 0);
- int *bp = Tloc(b, 0);
-
- BATloop(o, p, q) {
- int v = up[op[p]];
- nils |= is_int_nil(v);
- bp[p] = v;
- }
- BATsetcount(b, BATcount(o));
- BATnegateprops(b);
- b->tnil = nils;
- b->tnonil = !nils;
- } else if (ATOMbasetype(u->ttype) == TYPE_lng) {
- lng *up = Tloc(u, 0);
- lng *bp = Tloc(b, 0);
-
- BATloop(o, p, q) {
- lng v = up[op[p]];
- nils |= is_lng_nil(v);
- bp[p] = v;
- }
- BATsetcount(b, BATcount(o));
- BATnegateprops(b);
- b->tnil = nils;
- b->tnonil = !nils;
- } else {
+ switch (ATOMbasetype(u->ttype)) {
+ case TYPE_int:
+ decompress_loop(int);
+ break;
+ case TYPE_lng:
+ decompress_loop(lng);
+ break;
+#ifdef HAVE_HGE
+ case TYPE_hge:
+ decompress_loop(hge);
+ break;
+#endif
+ default:
BATloop(o, p, q) {
BUN up = op[p];
if (BUNappend(b, BUNtail(ui, up), false) !=
GDK_SUCCEED) {
@@ -334,33 +334,19 @@ DICTdecompress_(BAT *o, BAT *u, role_t r
assert(o->ttype == TYPE_sht);
unsigned short *op = Tloc(o, 0);
- if (ATOMbasetype(u->ttype) == TYPE_int) {
- int *up = Tloc(u, 0);
- int *bp = Tloc(b, 0);
-
- BATloop(o, p, q) {
- int v = up[op[p]];
- nils |= is_int_nil(v);
- bp[p] = v;
- }
- BATsetcount(b, BATcount(o));
- BATnegateprops(b);
- b->tnil = nils;
- b->tnonil = !nils;
- } else if (ATOMbasetype(u->ttype) == TYPE_lng) {
- lng *up = Tloc(u, 0);
- lng *bp = Tloc(b, 0);
-
- BATloop(o, p, q) {
- lng v = up[op[p]];
- nils |= is_lng_nil(v);
- bp[p] = v;
- }
- BATsetcount(b, BATcount(o));
- BATnegateprops(b);
- b->tnil = nils;
- b->tnonil = !nils;
- } else {
+ switch (ATOMbasetype(u->ttype)) {
+ case TYPE_int:
+ decompress_loop(int);
+ break;
+ case TYPE_lng:
+ decompress_loop(lng);
+ break;
+#ifdef HAVE_HGE
+ case TYPE_hge:
+ decompress_loop(hge);
+ break;
+#endif
+ default:
BATloop(o, p, q) {
BUN up = op[p];
if (BUNappend(b, BUNtail(ui, up), false) !=
GDK_SUCCEED) {
@@ -998,7 +984,7 @@ DICTprepare4append(BAT **noffsets, BAT *
}
if (!f) {
if (BATcount(dict) >= (64*1024)-1) {
- assert(0);
+ assert(0);
bat_destroy(n);
bat_iterator_end(&bi);
return -2;
diff --git a/sql/jdbc/tests/Tests/OnClientDownloadData.sql
b/sql/jdbc/tests/Tests/OnClientDownloadData.sql
new file mode 100644
--- /dev/null
+++ b/sql/jdbc/tests/Tests/OnClientDownloadData.sql
@@ -0,0 +1,36 @@
+-- test ON CLIENT data exports with different delimiters
+COPY SELECT * FROM sys.tables ORDER BY id INTO 'sys_tables_by_id.dsv' ON
CLIENT;
+COPY SELECT * FROM sys.tables ORDER BY id INTO 'sys_tables_by_id.csv' ON
CLIENT USING DELIMITERS ',' , E'\n' , '"' NULL AS '';
+COPY SELECT * FROM sys.tables ORDER BY id INTO 'sys_tables_by_id.tsv' ON
CLIENT USING DELIMITERS E'\t' , E'\n' , '"' NULL AS '';
+COPY SELECT * FROM sys.tables ORDER BY id INTO 'sys_tables_by_id.psv' ON
CLIENT USING DELIMITERS '|' , E'\n' , '"' NULL AS '';
+
+-- and compressed exports (only .gz is supported)
+COPY SELECT * FROM sys.tables ORDER BY id INTO 'sys_tables_by_id.csv.gz' ON
CLIENT USING DELIMITERS ',' , E'\n' , '"' NULL AS '';
+
+
+-- test error handling
+COPY SELECT * FROM sys.tables ORDER BY id INTO '' ON CLIENT USING DELIMITERS
',' , E'\n' , '"' NULL AS '';
+-- Error [22000] Missing file name
+COPY SELECT * FROM sys.tables ORDER BY id INTO '.' ON CLIENT USING DELIMITERS
',' , E'\n' , '"' NULL AS '';
+-- Error [22000] File already exists: /home/dinther/csvfiles
+COPY SELECT * FROM sys.tables ORDER BY id INTO '..' ON CLIENT USING DELIMITERS
',' , E'\n' , '"' NULL AS '';
+-- Error [22000] File is not in download directory: /home/dinther/csvfiles
+COPY SELECT * FROM sys.tables ORDER BY id INTO '../b' ON CLIENT USING
DELIMITERS ',' , E'\n' , '"' NULL AS '';
+-- Error [22000] File is not in download directory: /home/dinther/csvfiles
+COPY SELECT * FROM sys.tables ORDER BY id INTO
'/home/Doesnotexist/sys_tables_by_id.dsv' ON CLIENT;
+-- Error [22000] File is not in download directory: /home/dinther/csvfiles
+
+COPY SELECT * FROM sys.tables ORDER BY id INTO 'sys_tables_by_id.dsv' ON
CLIENT;
+-- Error [22000] File already exists:
/home/dinther/csvfiles/sys_tables_by_id.dsv
+COPY SELECT * FROM sys.tables ORDER BY id INTO 'sys_tables_by_id.csv.gz' ON
CLIENT USING DELIMITERS ',' , E'\n' , '"' NULL AS '';
+-- Error [22000] File already exists:
/home/dinther/csvfiles/sys_tables_by_id.csv.gz
+
+COPY SELECT * FROM sys.tables ORDER BY id INTO 'sys_tables_by_id.csv.bz2' ON
CLIENT USING DELIMITERS ',' , E'\n' , '"' NULL AS '';
+-- Error [22000] Requested compression .bz2 is not supported. Use .gz instead.
+COPY SELECT * FROM sys.tables ORDER BY id INTO 'sys_tables_by_id.csv.lz4' ON
CLIENT USING DELIMITERS ',' , E'\n' , '"' NULL AS '';
+-- Error [22000] Requested compression .lz4 is not supported. Use .gz instead.
+COPY SELECT * FROM sys.tables ORDER BY id INTO 'sys_tables_by_id.csv.xz' ON
CLIENT USING DELIMITERS ',' , E'\n' , '"' NULL AS '';
+-- Error [22000] Requested compression .xz is not supported. Use .gz instead.
+COPY SELECT * FROM sys.tables ORDER BY id INTO 'sys_tables_by_id.csv.zip' ON
CLIENT USING DELIMITERS ',' , E'\n' , '"' NULL AS '';
+-- Error [22000] Requested compression .zip is not supported. Use .gz instead.
+
diff --git a/sql/jdbc/tests/Tests/OnClientUploadData.sql
b/sql/jdbc/tests/Tests/OnClientUploadData.sql
new file mode 100644
--- /dev/null
+++ b/sql/jdbc/tests/Tests/OnClientUploadData.sql
@@ -0,0 +1,41 @@
+DROP TABLE IF EXISTS importedFileData;
+CREATE TABLE importedFileData AS SELECT * FROM sys.tables WITH NO DATA;
+
+COPY INTO importedFileData FROM 'sys_tables_by_id.dsv' ON CLIENT USING
DELIMITERS '|' , E'\n' , '"' NULL AS '';
+SELECT COUNT(*) > 0 FROM importedFileData;
+TRUNCATE importedFileData;
+
+COPY INTO importedFileData FROM 'sys_tables_by_id.csv' ON CLIENT USING
DELIMITERS ',' , E'\n' , '"' NULL AS '';
+SELECT COUNT(*) > 0 FROM importedFileData;
+TRUNCATE importedFileData;
+
+COPY 20 OFFSET 5 RECORDS INTO importedFileData FROM 'sys_tables_by_id.csv' ON
CLIENT USING DELIMITERS ',' , E'\n' , '"' NULL AS '';
+SELECT COUNT(*) > 0 FROM importedFileData;
+TRUNCATE importedFileData;
+
+COPY INTO importedFileData FROM 'sys_tables_by_id.tsv' ON CLIENT USING
DELIMITERS E'\t' , E'\n' , '"' NULL AS '';
+SELECT COUNT(*) > 0 FROM importedFileData;
+TRUNCATE importedFileData;
+
+COPY INTO importedFileData FROM 'sys_tables_by_id.psv' ON CLIENT USING
DELIMITERS '|' , E'\n' , '"' NULL AS '';
+SELECT COUNT(*) > 0 FROM importedFileData;
+TRUNCATE importedFileData;
+
+-- compressed .gz imports
+COPY INTO importedFileData FROM 'sys_tables_by_id.csv.gz' ON CLIENT USING
DELIMITERS ',' , E'\n' , '"' NULL AS '';
+SELECT COUNT(*) > 0 FROM importedFileData;
+TRUNCATE importedFileData;
+
+COPY 80 OFFSET 5 RECORDS INTO importedFileData FROM 'sys_tables_by_id.csv.gz'
ON CLIENT USING DELIMITERS ',' , E'\n' , '"' NULL AS '';
+SELECT COUNT(*) > 0 FROM importedFileData;
+TRUNCATE importedFileData;
+
+
+-- other compression formats are NOT supported
+COPY INTO importedFileData FROM 'sys_tables_by_id.csv.bz2' ON CLIENT USING
DELIMITERS ',' , E'\n' , '"' NULL AS '';
+COPY INTO importedFileData FROM 'sys_tables_by_id.csv.lz4' ON CLIENT USING
DELIMITERS ',' , E'\n' , '"' NULL AS '';
+COPY INTO importedFileData FROM 'sys_tables_by_id.csv.xz' ON CLIENT USING
DELIMITERS ',' , E'\n' , '"' NULL AS '';
+COPY INTO importedFileData FROM 'sys_tables_by_id.csv.zip' ON CLIENT USING
DELIMITERS ',' , E'\n' , '"' NULL AS '';
+
+DROP TABLE IF EXISTS importedFileData;
+
diff --git a/sql/jdbc/tests/Tests/Test_JdbcClient.SQL.py
b/sql/jdbc/tests/Tests/Test_JdbcClient.SQL.py
--- a/sql/jdbc/tests/Tests/Test_JdbcClient.SQL.py
+++ b/sql/jdbc/tests/Tests/Test_JdbcClient.SQL.py
@@ -1,5 +1,4 @@
-import os
-import sys
+import os, sys
from subprocess import run, PIPE, CalledProcessError
HOST=os.getenv('HOST')
@@ -15,34 +14,60 @@ PASSWORD='monetdb'
if __name__ == '__main__':
with open(os.path.join('.monetdb'), 'w') as f:
f.write('\n'.join(['user=monetdb', 'password=monetdb']))
+
cmd = ['java', CLIENT, '-h', HOST, '-p', MAPIPORT, '-d', TSTDB, '--help']
try:
p = run(cmd, stdout=PIPE, stderr=PIPE, check=True, encoding='utf-8')
except CalledProcessError as e:
raise SystemExit(e.stderr)
+
cmd = ['java', CLIENT, '-h', HOST, '-p', MAPIPORT, '-d', TSTDB, '-f',
os.path.join(TSTSRCBASE, TSTDIR, 'Tests/JdbcClient_create_tables.sql')]
try:
p = run(cmd, stdout=PIPE, stderr=PIPE, check=True, encoding='utf-8')
except CalledProcessError as e:
print(e.stderr, file=sys.stderr)
raise SystemExit('ERROR: failed to create tables!')
+
cmd = ['java', CLIENT, '-h', HOST, '-p', MAPIPORT, '-d', TSTDB, '-f',
os.path.join(TSTSRCBASE, TSTDIR, 'Tests/JdbcClient_inserts_selects.sql')]
try:
p = run(cmd, stdout=PIPE, stderr=PIPE, check=True, encoding='utf-8')
except CalledProcessError as e:
print(e.stderr, file=sys.stderr)
raise SystemExit('ERROR: failed to insert!')
+
cmd = ['java', CLIENT, '-h', HOST, '-p', MAPIPORT, '-d', TSTDB, '-D']
try:
p = run(cmd, stdout=PIPE, stderr=PIPE, check=True, encoding='utf-8')
except CalledProcessError as e:
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
