Changeset: af355d8e398f for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=af355d8e398f Modified Files: .hgtags ChangeLog HowToStart.rst Makefile.ag MonetDB.spec NT/monetdb_config.h.in NT/rules.msc bootstrap buildtools/autogen/autogen/msc.py clients/ChangeLog clients/R/Tests/deps-test.R clients/Tests/All clients/Tests/MAL-signatures.stable.out clients/Tests/MAL-signatures.stable.out.int128 clients/Tests/exports.stable.out clients/examples/C/Makefile.ag clients/mapiclient/Makefile.ag clients/mapiclient/Tests/mclient--help.stable.err clients/mapiclient/Tests/mclient--help.stable.err.Windows clients/mapiclient/dump.c clients/mapiclient/mclient.c clients/mapiclient/msqldump.h clients/mapiclient/tomograph.c clients/mapilib/Makefile.ag clients/mapilib/mapi.c clients/mapilib/mapi.h clients/mapilib/mapi.rc clients/odbc/driver/ODBCConvert.c clients/odbc/driver/driver.rc clients/odbc/winsetup/setup.rc common/Makefile.ag common/stream/Makefile.ag common/stream/stream.c common/stream/stream.h common/utils/Makefile.ag common/utils/mcrypt.c configure.ag debian/changelog debian/control debian/rules gdk/ChangeLog gdk/ChangeLog-Archive gdk/Makefile.ag gdk/gdk.h gdk/gdk_aggr.c gdk/gdk_align.c gdk/gdk_atoms.c gdk/gdk_atoms.h gdk/gdk_bat.c gdk/gdk_batop.c gdk/gdk_bbp.c gdk/gdk_bbp.h gdk/gdk_calc.c gdk/gdk_cross.c gdk/gdk_delta.c gdk/gdk_group.c gdk/gdk_hash.c gdk/gdk_heap.c gdk/gdk_imprints.c gdk/gdk_interprocess.c gdk/gdk_interprocess.h gdk/gdk_join.c gdk/gdk_logger.c gdk/gdk_logger.h gdk/gdk_orderidx.c gdk/gdk_private.h gdk/gdk_project.c gdk/gdk_search.c gdk/gdk_select.c gdk/gdk_ssort_impl.h gdk/gdk_storage.c gdk/gdk_system.c gdk/gdk_tm.c gdk/gdk_unique.c gdk/gdk_utils.c gdk/gdk_utils.h gdk/gdk_value.c gdk/libbat.rc geom/monetdb5/geom.c geom/monetdb5/geomBulk.c geom/monetdb5/geom_upgrade.c libversions monetdb5/ChangeLog monetdb5/extras/Makefile.ag monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out.Windows monetdb5/extras/mal_optimizer_template/opt_sql_append.c monetdb5/extras/rapi/rapi.c monetdb5/mal/Makefile.ag monetdb5/mal/Tests/performanceTests/performanceLog monetdb5/mal/mal.c monetdb5/mal/mal.h monetdb5/mal/mal_authorize.c monetdb5/mal/mal_authorize.h monetdb5/mal/mal_builder.c monetdb5/mal/mal_builder.h monetdb5/mal/mal_client.c monetdb5/mal/mal_client.h monetdb5/mal/mal_dataflow.c monetdb5/mal/mal_debugger.c monetdb5/mal/mal_exception.c monetdb5/mal/mal_factory.c monetdb5/mal/mal_function.c monetdb5/mal/mal_function.h monetdb5/mal/mal_import.c monetdb5/mal/mal_instruction.c monetdb5/mal/mal_instruction.h monetdb5/mal/mal_interpreter.c monetdb5/mal/mal_linker.c monetdb5/mal/mal_linker.h monetdb5/mal/mal_listing.c monetdb5/mal/mal_module.c monetdb5/mal/mal_module.h monetdb5/mal/mal_parser.c monetdb5/mal/mal_parser.h monetdb5/mal/mal_profiler.c monetdb5/mal/mal_profiler.h monetdb5/mal/mal_resolve.c monetdb5/mal/mal_resource.h monetdb5/mal/mal_runtime.c monetdb5/mal/mal_runtime.h monetdb5/mal/mal_sabaoth.c monetdb5/mal/mal_sabaoth.h monetdb5/mal/mal_session.c monetdb5/mal/mal_session.h monetdb5/mal/mal_stack.c monetdb5/modules/atoms/Makefile.ag monetdb5/modules/atoms/batxml.c monetdb5/modules/atoms/blob.c monetdb5/modules/atoms/inet.c monetdb5/modules/atoms/mtime.c monetdb5/modules/atoms/mtime.h monetdb5/modules/atoms/str.c monetdb5/modules/atoms/uuid.c monetdb5/modules/atoms/xml.c monetdb5/modules/kernel/Tests/All monetdb5/modules/kernel/aggr.c monetdb5/modules/kernel/algebra.c monetdb5/modules/kernel/algebra.h monetdb5/modules/kernel/algebra.mal monetdb5/modules/kernel/bat5.c monetdb5/modules/kernel/bat5.h monetdb5/modules/kernel/bat5.mal monetdb5/modules/kernel/batcolor.c monetdb5/modules/kernel/batmmath.c monetdb5/modules/kernel/batmmath.h monetdb5/modules/kernel/batmmath.mal monetdb5/modules/kernel/batstr.c monetdb5/modules/kernel/group.c monetdb5/modules/kernel/group.h monetdb5/modules/kernel/group.mal monetdb5/modules/kernel/mmath.c monetdb5/modules/kernel/mmath.h monetdb5/modules/kernel/mmath.mal monetdb5/modules/mal/01_calc.mal monetdb5/modules/mal/01_calc.mal.sh monetdb5/modules/mal/Makefile.ag monetdb5/modules/mal/Tests/orderidx00.malC monetdb5/modules/mal/Tests/orderidx01.malC monetdb5/modules/mal/Tests/orderidx02.malC monetdb5/modules/mal/Tests/orderidx04.malC monetdb5/modules/mal/calc.c monetdb5/modules/mal/clients.c monetdb5/modules/mal/groupby.c monetdb5/modules/mal/inspect.c monetdb5/modules/mal/language.mal monetdb5/modules/mal/mal_init.mal monetdb5/modules/mal/mal_io.c monetdb5/modules/mal/mal_io.h monetdb5/modules/mal/mal_io.mal monetdb5/modules/mal/mal_mapi.c monetdb5/modules/mal/mal_mapi.h monetdb5/modules/mal/manifold.c monetdb5/modules/mal/manual.c monetdb5/modules/mal/mat.c monetdb5/modules/mal/mdb.c monetdb5/modules/mal/mdb.h monetdb5/modules/mal/mdb.mal monetdb5/modules/mal/mkey.c monetdb5/modules/mal/orderidx.c monetdb5/modules/mal/pcre.c monetdb5/modules/mal/pcre.mal monetdb5/modules/mal/profiler.c monetdb5/modules/mal/profiler.h monetdb5/modules/mal/profiler.mal monetdb5/modules/mal/querylog.c monetdb5/modules/mal/remote.c monetdb5/modules/mal/sabaoth.c monetdb5/modules/mal/sysmon.c monetdb5/modules/mal/sysmon.mal monetdb5/modules/mal/tablet.c monetdb5/modules/mal/tokenizer.c monetdb5/modules/mal/txtsim.c monetdb5/optimizer/Makefile.ag monetdb5/optimizer/Tests/All monetdb5/optimizer/Tests/GCexample01.stable.out monetdb5/optimizer/Tests/dataflow.stable.out monetdb5/optimizer/Tests/dataflow3.stable.out monetdb5/optimizer/Tests/dataflow4.stable.out monetdb5/optimizer/opt_aliases.c monetdb5/optimizer/opt_candidates.c monetdb5/optimizer/opt_commonTerms.c monetdb5/optimizer/opt_costModel.c monetdb5/optimizer/opt_dataflow.c monetdb5/optimizer/opt_deadcode.c monetdb5/optimizer/opt_emptybind.c monetdb5/optimizer/opt_evaluate.c monetdb5/optimizer/opt_garbageCollector.c monetdb5/optimizer/opt_generator.c monetdb5/optimizer/opt_jit.c monetdb5/optimizer/opt_json.c monetdb5/optimizer/opt_macro.c monetdb5/optimizer/opt_matpack.c monetdb5/optimizer/opt_mergetable.c monetdb5/optimizer/opt_mitosis.c monetdb5/optimizer/opt_pipes.c monetdb5/optimizer/opt_prelude.c monetdb5/optimizer/opt_prelude.h monetdb5/optimizer/opt_profiler.c monetdb5/optimizer/opt_pushselect.c monetdb5/optimizer/opt_querylog.c monetdb5/optimizer/opt_remap.c monetdb5/optimizer/opt_remoteQueries.c monetdb5/optimizer/opt_reorder.c monetdb5/optimizer/opt_support.c monetdb5/optimizer/opt_support.h monetdb5/optimizer/opt_volcano.c monetdb5/optimizer/opt_wrapper.c monetdb5/optimizer/optimizer.c monetdb5/optimizer/optimizer.h monetdb5/optimizer/optimizer.mal monetdb5/scheduler/run_adder.c monetdb5/scheduler/run_memo.c monetdb5/tools/libmonetdb5.rc sql/ChangeLog sql/backends/monet5/40_sql.mal sql/backends/monet5/LSST/lsst.c sql/backends/monet5/LSST/lsst.h sql/backends/monet5/LSST/lsst.mal sql/backends/monet5/Makefile.ag sql/backends/monet5/Tests/All sql/backends/monet5/UDF/cudf/Tests/udf-fuse.stable.out sql/backends/monet5/UDF/cudf/Tests/udf-reverse.stable.out sql/backends/monet5/UDF/pyapi/50_pyapi.mal sql/backends/monet5/UDF/pyapi/Makefile.ag sql/backends/monet5/UDF/pyapi/connection.c sql/backends/monet5/UDF/pyapi/connection.h sql/backends/monet5/UDF/pyapi/conversion.c sql/backends/monet5/UDF/pyapi/convert_loops.h sql/backends/monet5/UDF/pyapi/emit.c sql/backends/monet5/UDF/pyapi/emit.h sql/backends/monet5/UDF/pyapi/formatinput.c sql/backends/monet5/UDF/pyapi/formatinput.h sql/backends/monet5/UDF/pyapi/pyapi.c sql/backends/monet5/UDF/pyapi/pyapi.h sql/backends/monet5/UDF/pyapi/pyapi.mal sql/backends/monet5/UDF/pyapi/pyapi_locatepython.bat sql/backends/monet5/UDF/pyapi/pyheader.h sql/backends/monet5/UDF/pyapi/pyloader.c sql/backends/monet5/UDF/pyapi/pytypes.c sql/backends/monet5/UDF/pyapi/pytypes.h sql/backends/monet5/UDF/pyapi/type_conversion.c sql/backends/monet5/UDF/pyapi/type_conversion.h sql/backends/monet5/UDF/pyapi/undef.h sql/backends/monet5/UDF/pyapi/unicode.c sql/backends/monet5/UDF/pyapi/unicode.h sql/backends/monet5/generator/generator.c sql/backends/monet5/mal_backend.c sql/backends/monet5/mal_backend.h sql/backends/monet5/rel_bin.c sql/backends/monet5/rel_bin.h sql/backends/monet5/sql.c sql/backends/monet5/sql.h sql/backends/monet5/sql.mal sql/backends/monet5/sql_bat2time.c sql/backends/monet5/sql_cast.c sql/backends/monet5/sql_cast_impl_down_from_flt.h sql/backends/monet5/sql_cast_impl_up_to_flt.h sql/backends/monet5/sql_execute.c sql/backends/monet5/sql_fround.c sql/backends/monet5/sql_gencode.c sql/backends/monet5/sql_gencode.h sql/backends/monet5/sql_optimizer.c sql/backends/monet5/sql_result.c sql/backends/monet5/sql_result.h sql/backends/monet5/sql_round.c sql/backends/monet5/sql_round_impl.h sql/backends/monet5/sql_scenario.c sql/backends/monet5/sql_statement.c sql/backends/monet5/sql_statement.h sql/backends/monet5/sql_statistics.c sql/backends/monet5/sql_upgrades.c sql/backends/monet5/sql_user.c sql/backends/monet5/vaults/bam/bam_wrapper.c sql/backends/monet5/vaults/fits/fits.c sql/backends/monet5/vaults/lidar/lidar.c sql/backends/monet5/vaults/netcdf/netcdf.c sql/backends/monet5/vaults/shp/75_shp.sql sql/backends/monet5/vaults/shp/shp.c sql/benchmarks/tpch/load.sh sql/common/sql_hash.c sql/common/sql_keyword.c sql/common/sql_list.c sql/common/sql_mem.c sql/common/sql_string.c sql/common/sql_types.c sql/include/sql_catalog.h sql/include/sql_query.h sql/include/sql_relation.h sql/jdbc/tests/Tests/All sql/jdbc/tests/Tests/Test_Dobjects.stable.out sql/scripts/46_profiler.sql sql/scripts/51_sys_schema_extension.sql sql/server/rel_dump.c sql/server/rel_exp.c sql/server/rel_exp.h sql/server/rel_optimizer.c sql/server/rel_psm.c sql/server/rel_rel.c sql/server/rel_schema.c sql/server/rel_schema.h sql/server/rel_select.c sql/server/rel_semantic.c sql/server/rel_sequence.c sql/server/rel_updates.c sql/server/sql_atom.c sql/server/sql_mvc.c sql/server/sql_mvc.h sql/server/sql_parser.y sql/server/sql_privileges.c sql/server/sql_qc.c sql/server/sql_scan.c sql/server/sql_scan.h sql/storage/Makefile.ag sql/storage/bat/bat_logger.c sql/storage/bat/bat_storage.c sql/storage/bat/bat_table.c sql/storage/bat/bat_utils.c sql/storage/bat/res_table.c sql/storage/sql_catalog.c sql/storage/sql_storage.h sql/storage/store.c sql/storage/store_dependency.c sql/storage/store_sequence.c sql/test/BugTracker-2009/Tests/All sql/test/BugTracker-2009/Tests/explain_gives_crash.SF-2741829.stable.out sql/test/BugTracker-2009/Tests/segfault_when_quitting_debugger.SF-2538837.stable.out sql/test/BugTracker-2010/Tests/LIMIT_OFFSET_big-endian.Bug-2622.stable.out sql/test/BugTracker-2010/Tests/error-truncated.Bug-2615.stable.err.Darwin sql/test/BugTracker-2010/Tests/error-truncated.Bug-2615.stable.err.Windows sql/test/BugTracker-2011/Tests/crash_on_alias.Bug-2798.stable.out sql/test/BugTracker-2012/Tests/aggregate_incorrect_precision_scale.Bug-3182.stable.out sql/test/BugTracker-2012/Tests/aggregate_incorrect_precision_scale.Bug-3182.stable.out.int128 sql/test/BugTracker-2012/Tests/large-number-operation-strange-results.Bug-2929.stable.out.int128 sql/test/BugTracker-2012/Tests/power_priority.Bug-2291.stable.out sql/test/BugTracker-2012/Tests/rewrite_like_into_likesubselect.Bug-3179.stable.out sql/test/BugTracker-2012/Tests/url_script_test.Bug-2972.stable.out sql/test/BugTracker-2013/Tests/between.Bug-3259.sql sql/test/BugTracker-2013/Tests/between.Bug-3259.stable.out sql/test/BugTracker-2014/Tests/manifold.Bug-3556.stable.out sql/test/BugTracker-2014/Tests/nil_2dec_lng.Bug-3592.sql sql/test/BugTracker-2014/Tests/querylog.Bug-3607.stable.out sql/test/BugTracker-2015/Tests/crash_timestamp_convert.Bug-3816.stable.out sql/test/BugTracker-2015/Tests/schemadiff.Bug-3778.stable.out sql/test/BugTracker-2016/Tests/All sql/test/BugTracker-2016/Tests/column_alias_in_where_clause.Bug-3947.stable.out.int128 sql/test/BugTracker-2016/Tests/convert-function-test.Bug-3460.stable.out sql/test/BugTracker-2016/Tests/convert-function-test.Bug-3460.stable.out.int128 sql/test/BugTracker-2016/Tests/decimal_vs_integer.Bug-3941.stable.out sql/test/BugTracker-2016/Tests/ifthenelse-void.Bug-6075.stable.out sql/test/BugTracker-2016/Tests/storagemodel.stable.out sql/test/BugTracker/Tests/explain.SF-1739353.stable.out sql/test/BugTracker/Tests/groupby_orderby_nonselected.SF-1723863.stable.err sql/test/BugTracker/Tests/jdbc_no_debug.SF-1739356.stable.out sql/test/BugTracker/Tests/rand_not_executed_for_every_row.SF-1723791.stable.out sql/test/BugTracker/Tests/subquery_orderby.SF-1688915.stable.err sql/test/BugTracker/Tests/subquery_orderby.SF-1688915.stable.out sql/test/Tests/All sql/test/Tests/setoptimizer.stable.out sql/test/Tests/systemfunctions.stable.out sql/test/Tests/systemfunctions.stable.out.int128 sql/test/bugs/Tests/object_name-bug-sf-985242.stable.out sql/test/emptydb-upgrade-chain-hge/Tests/All sql/test/emptydb-upgrade-chain-hge/Tests/check.stable.out.int128 sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128 sql/test/emptydb-upgrade-chain/Tests/All sql/test/emptydb-upgrade-chain/Tests/check.stable.out sql/test/emptydb-upgrade-chain/Tests/check.stable.out.32bit sql/test/emptydb-upgrade-chain/Tests/check.stable.out.int128 sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128 sql/test/emptydb-upgrade-hge/Tests/All sql/test/emptydb-upgrade-hge/Tests/check.stable.out.int128 sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128 sql/test/emptydb-upgrade/Tests/All sql/test/emptydb-upgrade/Tests/check.stable.out sql/test/emptydb-upgrade/Tests/check.stable.out.32bit sql/test/emptydb-upgrade/Tests/check.stable.out.int128 sql/test/emptydb-upgrade/Tests/upgrade.stable.out sql/test/emptydb-upgrade/Tests/upgrade.stable.out.int128 sql/test/emptydb/Tests/All sql/test/emptydb/Tests/check.stable.out sql/test/emptydb/Tests/check.stable.out.32bit sql/test/emptydb/Tests/check.stable.out.int128 sql/test/leaks/Tests/check0.stable.out sql/test/leaks/Tests/check1.stable.out sql/test/leaks/Tests/check1.stable.out.int128 sql/test/leaks/Tests/check2.stable.out sql/test/leaks/Tests/check2.stable.out.int128 sql/test/leaks/Tests/check3.stable.out sql/test/leaks/Tests/check3.stable.out.int128 sql/test/leaks/Tests/check4.stable.out sql/test/leaks/Tests/check4.stable.out.int128 sql/test/leaks/Tests/check5.stable.out sql/test/leaks/Tests/check5.stable.out.int128 sql/test/leaks/Tests/drop3.stable.out sql/test/leaks/Tests/drop3.stable.out.int128 sql/test/leaks/Tests/select1.stable.out sql/test/leaks/Tests/select1.stable.out.int128 sql/test/leaks/Tests/select2.stable.out sql/test/leaks/Tests/select2.stable.out.int128 sql/test/leaks/Tests/temp1.stable.out sql/test/leaks/Tests/temp1.stable.out.int128 sql/test/leaks/Tests/temp2.stable.out sql/test/leaks/Tests/temp2.stable.out.int128 sql/test/leaks/Tests/temp3.stable.out sql/test/leaks/Tests/temp3.stable.out.int128 sql/test/malloc_fail/Tests/All sql/test/mapi/Tests/utf8test.SQL.py sql/test/mapi/Tests/utf8test.stable.out sql/test/mergetables/Tests/forex.stable.out sql/test/mergetables/Tests/forex.stable.out.int128 sql/test/mergetables/Tests/mergequery.stable.out sql/test/pg_regress/Tests/comments.stable.err sql/test/pg_regress/Tests/float8.stable.err sql/test/pg_regress/Tests/float8.stable.err.Windows sql/test/pg_regress/Tests/float8.stable.err.int128 sql/test/pg_regress/Tests/int4.stable.out sql/test/pg_regress/Tests/interval.stable.out.int128 sql/test/pg_regress/Tests/numeric.sql sql/test/pg_regress/Tests/numeric.stable.err sql/test/pg_regress/Tests/strings.stable.out sql/test/quantiles/Tests/quantiles.stable.out sql/test/remote/Tests/partition_elim.stable.out sql/test/sys-schema/Tests/check_ForeignKey_referential_integrity.stable.out sql/test/sys-schema/Tests/check_PrimaryKey_uniqueness.stable.out sql/test/testdb-upgrade-chain-hge/Tests/upgrade.stable.out.int128 sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out.int128 sql/test/testdb-upgrade-hge/Tests/upgrade.stable.out.int128 sql/test/testdb-upgrade/Tests/upgrade.stable.out sql/test/testdb-upgrade/Tests/upgrade.stable.out.int128 testing/Mfilter.py testing/Mtest.py.in testing/process.py tools/merovingian/daemon/client.c tools/merovingian/daemon/connections.c tools/merovingian/daemon/controlrunner.c tools/merovingian/daemon/forkmserver.c tools/mserver/Makefile.ag tools/mserver/monet_version.c.in tools/mserver/mserver5.c tools/mserver/shutdowntest.c Branch: default Log Message:
Merge with protocol branch. diffs (truncated from 2328 to 300 lines): diff --git a/ChangeLog b/ChangeLog --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,9 @@ # ChangeLog file for devel # This file is updated with Maddlog +* Thu April 13 2017 Mark Raasveldt <[email protected]> +- Added a new server-side protocol implementation. The new protocol is backwards compatible with the old protocol. Clients can choose whether they want to use the old or the new protocol during the initial handshake with the server. The new protocol is a binary column-based protocol that is significantly faster than the old protocol when transferring large result sets. In addition, the new protocol supports compression using Snappy or LZ4. + * Thu Jan 12 2017 Martin van Dinther <[email protected]> - Added 5 new sys schema tables: function_languages, function_types, key_types, index_types and privilege_codes. They are pre-loaded with diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -1393,7 +1393,7 @@ char *MSP_locate_sqlscript(const char *m void MSinitClientPrg(Client cntxt, str mod, str nme); void MSresetInstructions(MalBlkPtr mb, int start); void MSresetVariables(Client cntxt, MalBlkPtr mb, MalStkPtr glb, int start); -void MSscheduleClient(str command, str challenge, bstream *fin, stream *fout); +void MSscheduleClient(str command, str challenge, bstream *fin, stream *fout, protocol_version protocol, size_t blocksize, int compute_column_widths); void MSserveClient(void *dummy); str MTIMEcompute_rule_foryear(date *ret, const rule *val, const int *year); str MTIMEcurrent_date(date *d); @@ -1444,6 +1444,8 @@ str MTIMEdaytime_extract_sql_seconds_bul str MTIMEdaytime_fromstr(daytime *ret, const char *const *s); str MTIMEepilogue(void *ret); str MTIMEepoch2int(int *res, const timestamp *ts); +str MTIMEepoch2lng(lng *res, const timestamp *ts); +str MTIMEepoch_bulk(bat *ret, bat *bid); str MTIMElocal_timezone(lng *res); str MTIMEmonth_from_str(int *ret, const char *const *month); str MTIMEmonth_to_str(str *ret, const int *month); @@ -2564,6 +2566,15 @@ str zero_or_oneRef; stream *append_wastream(const char *filename); stream *append_wstream(const char *filename); stream *block_stream(stream *s); +stream *block_stream2(stream *s, size_t bufsiz, compression_method comp, column_compression colcomp); +buffer bs2_buffer(stream *s); +column_compression bs2_colcomp(stream *ss); +void bs2_resetbuf(stream *ss); +int bs2_resizebuf(stream *ss, size_t bufsiz); +void bs2_setpos(stream *ss, size_t pos); +void *bs2_stealbuf(stream *ss); +stream *bs_stealstream(stream *s); +stream *bs_stream(stream *s); bstream *bstream_create(stream *rs, size_t chunk_size); void bstream_destroy(bstream *s); ssize_t bstream_next(bstream *s); @@ -2585,6 +2596,7 @@ size_t getFileSize(stream *s); stream *iconv_rstream(stream *ss, const char *charset, const char *name); stream *iconv_wstream(stream *ss, const char *charset, const char *name); int isa_block_stream(stream *s); +int isa_fixed_block_stream(stream *s); int mnstr_byteorder(stream *s); void mnstr_clearerr(stream *s); void mnstr_close(stream *s); @@ -2603,6 +2615,7 @@ int mnstr_printf(stream *s, _In_z_ _Prin ssize_t mnstr_read(stream *s, void *buf, size_t elmsize, size_t cnt); int mnstr_readBte(stream *s, signed char *val); int mnstr_readBteArray(stream *s, signed char *val, size_t cnt); +int mnstr_readChr(stream *s, char *val); int mnstr_readHge(stream *s, hge *val); int mnstr_readHgeArray(stream *s, hge *val, size_t cnt); int mnstr_readInt(stream *s, int *val); @@ -2611,6 +2624,7 @@ int mnstr_readLng(stream *s, lng *val); int mnstr_readLngArray(stream *s, lng *val, size_t cnt); int mnstr_readSht(stream *s, short *val); int mnstr_readShtArray(stream *s, short *val, size_t cnt); +int mnstr_readStr(stream *s, char *val); ssize_t mnstr_read_block(stream *s, void *buf, size_t elmsize, size_t cnt); ssize_t mnstr_readline(stream *s, void *buf, size_t maxcnt); stream *mnstr_rstream(stream *s); @@ -2620,6 +2634,9 @@ int mnstr_type(stream *s); ssize_t mnstr_write(stream *s, const void *buf, size_t elmsize, size_t cnt); int mnstr_writeBte(stream *s, signed char val); int mnstr_writeBteArray(stream *s, const signed char *val, size_t cnt); +int mnstr_writeChr(stream *s, char val); +int mnstr_writeDbl(stream *s, double val); +int mnstr_writeFlt(stream *s, float val); int mnstr_writeHge(stream *s, hge val); int mnstr_writeHgeArray(stream *s, const hge *val, size_t cnt); int mnstr_writeInt(stream *s, int val); @@ -2628,6 +2645,7 @@ int mnstr_writeLng(stream *s, lng val); int mnstr_writeLngArray(stream *s, const lng *val, size_t cnt); int mnstr_writeSht(stream *s, short val); int mnstr_writeShtArray(stream *s, const short *val, size_t cnt); +int mnstr_writeStr(stream *s, const char *val); stream *mnstr_wstream(stream *s); stream *open_rastream(const char *filename); stream *open_rstream(const char *filename); diff --git a/clients/mapiclient/Tests/mclient--help.stable.err.Windows b/clients/mapiclient/Tests/mclient--help.stable.err.Windows --- a/clients/mapiclient/Tests/mclient--help.stable.err.Windows +++ b/clients/mapiclient/Tests/mclient--help.stable.err.Windows @@ -20,7 +20,7 @@ Options are: -e | --echo echo the query -E charset | --encoding=charset specify encoding (character set) of the terminal -f kind | --format=kind specify output format {csv,tab,raw,sql,xml} - -H | --history load/save cmdline history (default off) + -H | --history load/save cmdline history (default off) -i | --interactive[=tm] interpret `\' commands on stdin, use time formatting {ms,s,m} -l language | --language=lang {sql,mal} -L logfile | --log=logfile save client/server interaction diff --git a/common/stream/Makefile.ag b/common/stream/Makefile.ag --- a/common/stream/Makefile.ag +++ b/common/stream/Makefile.ag @@ -10,6 +10,8 @@ MTSAFE INCLUDES = $(zlib_CFLAGS) \ $(BZ_CFLAGS) \ + $(snappy_CFLAGS) \ + $(lz4_CFLAGS) \ $(liblzma_CFLAGS) \ $(openssl_CFLAGS) \ $(curl_CFLAGS) @@ -20,6 +22,8 @@ lib_stream = { LIBS = $(SOCKET_LIBS) \ $(zlib_LIBS) \ $(BZ_LIBS) \ + $(snappy_LIBS) \ + $(lz4_LIBS) \ $(liblzma_LIBS) \ $(openssl_LIBS) \ $(curl_LIBS) \ diff --git a/common/stream/stream.c b/common/stream/stream.c --- a/common/stream/stream.c +++ b/common/stream/stream.c @@ -93,6 +93,12 @@ #ifdef HAVE_LIBLZMA #include <lzma.h> #endif +#ifdef HAVE_LIBSNAPPY +#include <snappy-c.h> +#endif +#ifdef HAVE_LIBLZ4 +#include <lz4.h> +#endif #ifdef HAVE_ICONV #ifdef HAVE_ICONV_H @@ -3570,6 +3576,8 @@ buffer_wastream(buffer *b, const char *n return s; } + + /* ------------------------------------------------------------------ */ /* A buffered stream consists of a sequence of blocks. Each block @@ -3924,6 +3932,19 @@ bs_clrerr(stream *s) mnstr_clearerr(((bs *) s->stream_data.p)->s); } +stream* bs_stream(stream *s) { + assert(isa_block_stream(s)); + return ((bs*)s->stream_data.p)->s; +} + +stream* bs_stealstream(stream *s) { + stream *res; + assert(isa_block_stream(s)); + res = ((bs*)s->stream_data.p)->s; + ((bs*)s->stream_data.p)->s = NULL; + return res; +} + stream * block_stream(stream *s) { @@ -3960,14 +3981,664 @@ block_stream(stream *s) return ns; } +typedef struct bs2 { + stream *s; /* underlying stream */ + size_t nr; /* how far we got in buf */ + size_t itotal; /* amount available in current read block */ + size_t bufsiz; + size_t readpos; + compression_method comp; + column_compression colcomp; + char *compbuf; + size_t compbufsiz; + char *buf; +} bs2; + + +static ssize_t +compress_stream_data(bs2 *s) { + assert(s->comp != COMPRESSION_NONE); + if (s->comp == COMPRESSION_SNAPPY) { +#ifdef HAVE_LIBSNAPPY + size_t compressed_length = s->compbufsiz; + snappy_status ret; + if ((ret = snappy_compress(s->buf, s->nr, s->compbuf, &compressed_length)) != SNAPPY_OK) { + s->s->errnr = (int) ret; + return -1; + } + return compressed_length; +#else + assert(0); + return -1; +#endif + } else if (s->comp == COMPRESSION_LZ4) { +#ifdef HAVE_LIBLZ4 + int compressed_length = (int) s->compbufsiz; + if ((compressed_length = LZ4_compress_fast(s->buf, s->compbuf, s->nr, compressed_length, 1)) == 0) { + s->s->errnr = -1; + return -1; + } + return compressed_length; +#else + assert(0); + return -1; +#endif + } + return -1; +} + + +static ssize_t +decompress_stream_data(bs2 *s) { + assert(s->comp != COMPRESSION_NONE); + if (s->comp == COMPRESSION_SNAPPY) { +#ifdef HAVE_LIBSNAPPY + snappy_status ret; + size_t uncompressed_length = s->bufsiz; + if ((ret = snappy_uncompress(s->compbuf, s->itotal, s->buf, &uncompressed_length)) != SNAPPY_OK) { + s->s->errnr = (int) ret; + return -1; + } + return (ssize_t) uncompressed_length; +#else + assert(0); + return -1; +#endif + } else if (s->comp == COMPRESSION_LZ4) { +#ifdef HAVE_LIBLZ4 + int uncompressed_length = (int) s->bufsiz; + if ((uncompressed_length = LZ4_decompress_safe(s->compbuf, s->buf, s->itotal, uncompressed_length)) <= 0) { + s->s->errnr = uncompressed_length; + return -1; + } + return uncompressed_length; +#else + assert(0); + return -1; +#endif + } + return -1; +} + +static ssize_t +compression_size_bound(bs2 *s) { + if (s->comp == COMPRESSION_NONE) { + return 0; + } else if (s->comp == COMPRESSION_SNAPPY) { +#ifndef HAVE_LIBSNAPPY + return -1; +#else + return snappy_max_compressed_length(s->bufsiz); +#endif + } else if (s->comp == COMPRESSION_LZ4) { +#ifndef HAVE_LIBLZ4 + return -1; +#else + return LZ4_compressBound(s->bufsiz); +#endif + } + return -1; +} + +static bs2 * +bs2_create(stream *s, size_t bufsiz, compression_method comp) +{ + /* should be a binary stream */ + bs2 *ns; + ssize_t compress_bound = 0; + + if ((ns = malloc(sizeof(*ns))) == NULL) + return NULL; + if ((ns->buf = malloc(bufsiz)) == NULL) { + free(ns); + return NULL; + } + + ns->s = s; + ns->nr = 0; + ns->itotal = 0; + ns->bufsiz = bufsiz; + ns->comp = comp; + ns->compbuf = NULL; + + compress_bound = compression_size_bound(ns); + if (compress_bound > 0) { + ns->compbufsiz = compress_bound; + ns->compbuf = malloc(ns->compbufsiz); + if (!ns->compbuf) { + free(ns); _______________________________________________ checkin-list mailing list [email protected] https://www.monetdb.org/mailman/listinfo/checkin-list
