Changeset: 9794b18f936b for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9794b18f936b
Added Files:
sql/test/Tests/copy-from-lz4.py
sql/test/Tests/copy-from-lz4.stable.err
sql/test/Tests/copy-from-lz4.stable.out
sql/test/Tests/copy-into-lz4.sql.in
sql/test/Tests/copy-into-lz4.stable.err
sql/test/Tests/copy-into-lz4.stable.out
sql/test/Tests/lz4-dump.sql
sql/test/Tests/testlz4.lz4
Modified Files:
ChangeLog
README-Debian
README-Fedora
sql/test/Tests/All
testing/Mtest.py.in
Branch: default
Log Message:
Tests for lz4 streams
diffs (truncated from 1476 to 300 lines):
diff --git a/ChangeLog b/ChangeLog
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,5 @@
# ChangeLog file for devel
# This file is updated with Maddlog
+* Tue Feb 13 2018 Pedro Ferreira <pedro.ferre...@monetdbsolutions.com>
+- Added support for lz4 compressed files on stream library
diff --git a/README-Debian b/README-Debian
--- a/README-Debian
+++ b/README-Debian
@@ -35,7 +35,7 @@ zlib1g-dev # optional: read and write .
These packages are optional.
libcfitsio-dev # optional: read FITS files
-liblz4-dev # optional: compression in new (unused) MAPI protocol
+liblz4-dev # optional: compression in new (unused) MAPI protocol,
also used to read and write .lz4 compressed files
libsnappy-dev # optional: compression in new (unused) MAPI protocol
python3-dev # optional, needed for Python 3 integration
python3-numpy # optional, needed for Python 3 integration
diff --git a/README-Fedora b/README-Fedora
--- a/README-Fedora
+++ b/README-Fedora
@@ -36,7 +36,7 @@ These packages are optional.
cfitsio-devel # optional: read FITS files
libasan # optional: --enable-sanitizer configuration
(debug)
liblas-devel # optional: read LIDAR files
-lz4-devel # optional: compression in new (unused) MAPI protocol
+lz4-devel # optional: compression in new (unused) MAPI protocol,
also used to read and write .lz4 compressed files
netcdf-devel # optional: read NetCDF files
proj-devel # optional, only optionally used in geom module
python3-devel # optional, needed for Python 3 integration
diff --git a/sql/test/Tests/All b/sql/test/Tests/All
--- a/sql/test/Tests/All
+++ b/sql/test/Tests/All
@@ -129,3 +129,6 @@ constant-not-in
fsum
THREADS=1?fsum1
+
+NOT_WIN32&HAVE_LIBLZ4?copy-into-lz4
+NOT_WIN32&HAVE_LIBLZ4?copy-from-lz4
diff --git a/sql/test/Tests/copy-from-lz4.py b/sql/test/Tests/copy-from-lz4.py
new file mode 100644
--- /dev/null
+++ b/sql/test/Tests/copy-from-lz4.py
@@ -0,0 +1,28 @@
+import os, sys
+
+try:
+ from MonetDBtesting import process
+except ImportError:
+ import process
+
+def try_remove_file(): # maybe file locks would do better
+ try:
+ os.remove("/tmp/testing-dump.lz4")
+ except:
+ pass
+
+try_remove_file()
+
+s = process.server(args = [], stdin = process.PIPE, stdout = process.PIPE,
stderr = process.PIPE)
+
+c = process.client('sql', stdin = open(os.path.join(os.getenv('TSTSRCDIR'),
'lz4-dump.sql')),
+ stdout = process.PIPE, stderr = process.PIPE, log = True)
+out, err = c.communicate()
+sys.stdout.write(out)
+sys.stderr.write(err)
+
+out, err = s.communicate()
+sys.stdout.write(out)
+sys.stderr.write(err)
+
+try_remove_file()
diff --git a/sql/test/Tests/copy-from-lz4.stable.err
b/sql/test/Tests/copy-from-lz4.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/test/Tests/copy-from-lz4.stable.err
@@ -0,0 +1,34 @@
+stderr of test 'copy-from-lz4` in directory 'sql/test` itself:
+
+
+# 18:01:50 >
+# 18:01:50 > "/usr/bin/python2" "copy-from-lz4.py" "copy-from-lz4"
+# 18:01:50 >
+
+
+# 18:01:50 >
+# 18:01:50 > Mtimeout -timeout 60 mclient -lsql -ftest -tnone -Eutf-8 -i -e
--host=/var/tmp/mtest-16374 --port=35324 --database=mTests_sql_test <
"/home/ferreira/repositories/MonetDB-default/src/sql/test/Tests/lz4-dump.sql"
+# 18:01:50 >
+
+# builtin opt gdk_dbpath =
/home/ferreira/repositories/MonetDB-default/BUILD/var/monetdb5/dbfarm/demo
+# builtin opt gdk_debug = 0
+# builtin opt gdk_vmtrim = no
+# builtin opt monet_prompt = >
+# builtin opt monet_daemon = no
+# builtin opt mapi_port = 50000
+# builtin opt mapi_open = false
+# builtin opt mapi_autosense = false
+# builtin opt sql_optimizer = default_pipe
+# builtin opt sql_debug = 0
+# cmdline opt gdk_nr_threads = 0
+# cmdline opt mapi_open = true
+# cmdline opt mapi_port = 35324
+# cmdline opt mapi_usock = /var/tmp/mtest-16374/.s.monetdb.35324
+# cmdline opt monet_prompt =
+# cmdline opt gdk_dbpath =
/home/ferreira/repositories/MonetDB-default/BUILD/var/MonetDB/mTests_sql_test
+# cmdline opt gdk_debug = 553648138
+
+# 18:01:50 >
+# 18:01:50 > "Done."
+# 18:01:50 >
+
diff --git a/sql/test/Tests/copy-from-lz4.stable.out
b/sql/test/Tests/copy-from-lz4.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/test/Tests/copy-from-lz4.stable.out
@@ -0,0 +1,103 @@
+stdout of test 'copy-from-lz4` in directory 'sql/test` itself:
+
+
+# 18:01:50 >
+# 18:01:50 > "/usr/bin/python2" "copy-from-lz4.py" "copy-from-lz4"
+# 18:01:50 >
+
+
+# 18:01:50 >
+# 18:01:50 > Mtimeout -timeout 60 mclient -lsql -ftest -tnone -Eutf-8 -i -e
--host=/var/tmp/mtest-16374 --port=35324 --database=mTests_sql_test <
"/home/ferreira/repositories/MonetDB-default/src/sql/test/Tests/lz4-dump.sql"
+# 18:01:50 >
+
+#CREATE TABLE outputlz4 (a bigint, b real, c clob);
+#CREATE TABLE readlz4 (a bigint, b real, c clob);
+#COPY 4 RECORDS INTO outputlz4 (a, b, c) FROM STDIN USING DELIMITERS
',','\n','"' NULL AS '';
+#1,2.0,"another"
+#2,2.1,"test"
+#3,2.2,"to perform"
+#,1.0,
+[ 4 ]
+#SELECT a, b, c FROM outputlz4;
+% sys.outputlz4, sys.outputlz4, sys.outputlz4 # table_name
+% a, b, c # name
+% bigint, real, clob # type
+% 1, 15, 10 # length
+[ 1, 2, "another" ]
+[ 2, 2.1, "test" ]
+[ 3, 2.2, "to perform" ]
+[ NULL, 1, NULL ]
+#COPY (SELECT a, b, c FROM outputlz4) INTO '/tmp/testing-dump.lz4' USING
DELIMITERS ',','\n','"' NULL AS '';
+[ 4 ]
+#COPY 4 RECORDS INTO readlz4 (a, b, c) FROM '/tmp/testing-dump.lz4' USING
DELIMITERS ',','\n','"' NULL AS '';
+[ 4 ]
+#SELECT a, b, c FROM readlz4;
+% sys.readlz4, sys.readlz4, sys.readlz4 # table_name
+% a, b, c # name
+% bigint, real, clob # type
+% 1, 15, 10 # length
+[ 1, 2, "another" ]
+[ 2, 2.1, "test" ]
+[ 3, 2.2, "to perform" ]
+[ NULL, 1, NULL ]
+#DROP TABLE outputlz4;
+#DROP TABLE readlz4;
+# MonetDB 5 server v11.30.0
+# This is an unreleased version
+# Serving database 'mTests_sql_test', using 8 threads
+# Compiled for x86_64-pc-linux-gnu/64bit with 128bit integers
+# Found 15.492 GiB available main-memory.
+# Copyright (c) 1993 - July 2008 CWI.
+# Copyright (c) August 2008 - 2018 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on
mapi:monetdb://dhcp-23.eduroam.cwi.nl:35324/
+# Listening for UNIX domain connection requests on
mapi:monetdb:///var/tmp/mtest-16374/.s.monetdb.35324
+# MonetDB/GIS module loaded
+# MonetDB/SQL module loaded
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+# loading sql script: 15_querylog.sql
+# loading sql script: 16_tracelog.sql
+# loading sql script: 17_temporal.sql
+# loading sql script: 18_index.sql
+# loading sql script: 20_vacuum.sql
+# loading sql script: 21_dependency_functions.sql
+# loading sql script: 21_dependency_views.sql
+# loading sql script: 22_clients.sql
+# loading sql script: 23_skyserver.sql
+# loading sql script: 25_debug.sql
+# loading sql script: 26_sysmon.sql
+# loading sql script: 27_rejects.sql
+# loading sql script: 39_analytics.sql
+# loading sql script: 39_analytics_hge.sql
+# loading sql script: 40_geom.sql
+# loading sql script: 40_json.sql
+# loading sql script: 40_json_hge.sql
+# loading sql script: 41_md5sum.sql
+# loading sql script: 45_uuid.sql
+# loading sql script: 46_profiler.sql
+# loading sql script: 51_sys_schema_extension.sql
+# loading sql script: 60_wlcr.sql
+# loading sql script: 72_fits.sql
+# loading sql script: 74_netcdf.sql
+# loading sql script: 75_lidar.sql
+# loading sql script: 75_shp.sql
+# loading sql script: 75_storagemodel.sql
+# loading sql script: 80_statistics.sql
+# loading sql script: 80_udf.sql
+# loading sql script: 80_udf_hge.sql
+# loading sql script: 85_bam.sql
+# loading sql script: 90_generator.sql
+# loading sql script: 90_generator_hge.sql
+# loading sql script: 97_comments.sql
+# loading sql script: 99_system.sql
+
+# 18:01:50 >
+# 18:01:50 > "Done."
+# 18:01:50 >
+
diff --git a/sql/test/Tests/copy-into-lz4.sql.in
b/sql/test/Tests/copy-into-lz4.sql.in
new file mode 100644
--- /dev/null
+++ b/sql/test/Tests/copy-into-lz4.sql.in
@@ -0,0 +1,4 @@
+CREATE TABLE testlz4 (a bigint, b real, c clob, d varchar(64));
+COPY INTO testlz4 FROM '$TSTSRCDIR/testlz4.lz4' USING DELIMITERS ',','\n','"'
NULL AS '';
+SELECT a, b, c, d FROM testlz4 limit 1024;
+DROP TABLE testlz4;
diff --git a/sql/test/Tests/copy-into-lz4.stable.err
b/sql/test/Tests/copy-into-lz4.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/test/Tests/copy-into-lz4.stable.err
@@ -0,0 +1,36 @@
+stderr of test 'copy-into-lz4` in directory 'sql/test` itself:
+
+
+# 15:41:39 >
+# 15:41:39 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"mapi_open=true" "--set" "mapi_port=31899" "--set"
"mapi_usock=/var/tmp/mtest-8247/.s.monetdb.31899" "--set" "monet_prompt="
"--forcemito"
"--dbpath=/home/ferreira/repositories/MonetDB-default/BUILD/var/MonetDB/mTests_sql_test"
"--set" "embedded_r=yes" "--set" "embedded_py=true"
+# 15:41:39 >
+
+# builtin opt gdk_dbpath =
/home/ferreira/repositories/MonetDB-default/BUILD/var/monetdb5/dbfarm/demo
+# builtin opt gdk_debug = 0
+# builtin opt gdk_vmtrim = no
+# builtin opt monet_prompt = >
+# builtin opt monet_daemon = no
+# builtin opt mapi_port = 50000
+# builtin opt mapi_open = false
+# builtin opt mapi_autosense = false
+# builtin opt sql_optimizer = default_pipe
+# builtin opt sql_debug = 0
+# cmdline opt gdk_nr_threads = 0
+# cmdline opt mapi_open = true
+# cmdline opt mapi_port = 31899
+# cmdline opt mapi_usock = /var/tmp/mtest-8247/.s.monetdb.31899
+# cmdline opt monet_prompt =
+# cmdline opt gdk_dbpath =
/home/ferreira/repositories/MonetDB-default/BUILD/var/MonetDB/mTests_sql_test
+# cmdline opt embedded_r = yes
+# cmdline opt embedded_py = true
+# cmdline opt gdk_debug = 553648138
+
+# 15:41:40 >
+# 15:41:40 > "mclient" "-lsql" "-ftest" "-tnone" "-Eutf-8" "-i" "-e"
"--host=/var/tmp/mtest-8247" "--port=31899"
+# 15:41:40 >
+
+
+# 15:41:43 >
+# 15:41:43 > "Done."
+# 15:41:43 >
+
diff --git a/sql/test/Tests/copy-into-lz4.stable.out
b/sql/test/Tests/copy-into-lz4.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/test/Tests/copy-into-lz4.stable.out
@@ -0,0 +1,1108 @@
+stdout of test 'copy-into-lz4` in directory 'sql/test` itself:
+
+
+# 15:41:39 >
+# 15:41:39 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"mapi_open=true" "--set" "mapi_port=31899" "--set"
"mapi_usock=/var/tmp/mtest-8247/.s.monetdb.31899" "--set" "monet_prompt="
"--forcemito"
"--dbpath=/home/ferreira/repositories/MonetDB-default/BUILD/var/MonetDB/mTests_sql_test"
"--set" "embedded_r=yes" "--set" "embedded_py=true"
+# 15:41:39 >
+
+# MonetDB 5 server v11.28.0
+# This is an unreleased version
+# Serving database 'mTests_sql_test', using 8 threads
+# Compiled for x86_64-pc-linux-gnu/64bit with 128bit integers
+# Found 15.492 GiB available main-memory.
+# Copyright (c) 1993 - July 2008 CWI.
+# Copyright (c) August 2008 - 2018 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on
mapi:monetdb://dhcp-23.eduroam.cwi.nl:31899/
+# Listening for UNIX domain connection requests on
mapi:monetdb:///var/tmp/mtest-8247/.s.monetdb.31899
+# MonetDB/GIS module loaded
+# MonetDB/SQL module loaded
+# MonetDB/Python2 module loaded
+# MonetDB/R module loaded
+
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
