Changeset: 22999318fbaf for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=22999318fbaf
Modified Files:
clients/mapilib/Makefile.ag
clients/mapilib/mapi.c
configure.ag
sql/backends/monet5/Makefile.ag
sql/backends/monet5/sql_result.c
Branch: protocol
Log Message:
Add optional SIMD PFOR compression to integer columns in prot10.
diffs (206 lines):
diff --git a/clients/mapilib/Makefile.ag b/clients/mapilib/Makefile.ag
--- a/clients/mapilib/Makefile.ag
+++ b/clients/mapilib/Makefile.ag
@@ -7,14 +7,14 @@
MTSAFE
INCLUDES = ../../common/options ../../common/stream ../../common/utils \
- $(MSGCONTROL_FLAGS)
+ $(MSGCONTROL_FLAGS) $(pfor_CFLAGS)
lib_mapi = {
VERSION = $(MAPI_VERSION)
SOURCES = mapi.c mapi.rc
LIBS = $(SOCKET_LIBS) ../../common/stream/libstream \
../../common/options/libmoptions \
- ../../common/utils/libmcrypt $(openssl_LIBS)
+ ../../common/utils/libmcrypt $(openssl_LIBS) $(pfor_LIBS)
}
headers_mapi = {
diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -812,6 +812,10 @@
# endif
#endif
+#ifdef HAVE_PFOR
+#include <simdcomp.h>
+#endif
+
#ifndef INVALID_SOCKET
#define INVALID_SOCKET (-1)
#endif
@@ -5588,13 +5592,31 @@ mapi_fetch_row(MapiHdl hdl)
fprintf(stderr, "Read strings from
position %zu\n", result->fields[i].buffer_ptr - initbuf);
#endif
} else {
+#ifdef HAVE_PFOR
+ if
(strcasecmp(result->fields[i].columntype, "int") == 0) {
+ lng b = *((lng*) buf);
+ buf += sizeof(lng);
+ lng length = *((lng*)(buf));
+ buf += sizeof(lng);
+
+ uint8_t *resbuffer =
malloc(nrows * sizeof(int));
+ simdunpack_length((const
__m128i *)buf, nrows, (uint32_t*) resbuffer, b);
+ result->fields[i].buffer_ptr =
resbuffer;
+#ifdef PROT10_DEBUG
+ fprintf(stderr, "Read PFOR compressed
elements (b=%lld,length=%lld) from position %zu\n", b, length, (buf - 2 *
sizeof(lng)) - initbuf);
+#endif
+ buf += length;
+ } else {
+#endif
buf += nrows *
result->fields[i].columnlength;
#ifdef PROT10_DEBUG
fprintf(stderr, "Read elements from
position %zu\n", result->fields[i].buffer_ptr - initbuf);
#endif
+#ifdef HAVE_PFOR
+ }
+#endif
}
}
- assert(result->fields[result->fieldcnt - 1].buffer_ptr
- result->fields[0].buffer_ptr < (long) hdl->mid->blocksize);
result->tuple_count += nrows;
} else {
for (i = 0; i < (size_t) result->fieldcnt; i++) {
diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -1683,6 +1683,50 @@ AC_SUBST([lz4_LIBS])
AM_CONDITIONAL([HAVE_LIBLZ4], [test x$have_lz4 != xno])
+dnl check for pfor (de)compression library
+org_have_pfor=auto
+have_pfor=$org_have_pfor
+pfor_CFLAGS=""
+pfor_LIBS=""
+AC_ARG_WITH([pfor],
+ [AS_HELP_STRING([--with-pfor=DIR],
+ [pfor library is installed in DIR])],
+ [have_pfor="$withval"])
+
+AS_CASE(["$have_pfor"],
+ [yes|no|auto], [],
+ [
+ pfor_CFLAGS="-I$withval/include"
+ pfor_LIBS="-L$withval"])
+
+AC_MSG_CHECKING([for pfor])
+AS_VAR_IF([have_pfor], [no], [], [
+ save_CPPFLAGS="$CPPFLAGS"
+ CPPFLAGS="$CPPFLAGS $pfor_CFLAGS"
+ save_LIBS="$LIBS"
+ LIBS="$LIBS $pfor_LIBS -lsimdcomp"
+ AC_LINK_IFELSE([AC_LANG_PROGRAM([[@%:@include <stdio.h>
+@%:@include <simdcomputil.h>]], [[(void)bits(42);]])],
+ pfor_LIBS="$pfor_LIBS -lsimdcomp",
+ [ AS_VAR_IF([have_pfor], [auto], [], [
+ AC_MSG_ERROR([pfor library not found])])
+ have_pfor=no
+ why_have_pfor="(pfor library not found)" ])
+ LIBS="$save_LIBS"
+ CPPFLAGS="$save_CPPFLAGS"])
+
+AS_VAR_IF([have_pfor], [no], [], [
+ AC_DEFINE([HAVE_PFOR], 1, [Define if you have the pfor library])
+ AC_MSG_RESULT([yes: $pfor_LIBS])], [
+ pfor_CFLAGS=""
+ pfor_LIBS=""
+ AC_MSG_RESULT([no])])
+
+AC_SUBST([pfor_CFLAGS])
+AC_SUBST([pfor_LIBS])
+AM_CONDITIONAL([HAVE_PFOR], [test x$have_pfor != xno])
+
+
DL_LIBS=""
AC_CHECK_LIB([dl], [dlopen], [DL_LIBS="-ldl" ] )
AC_SUBST([DL_LIBS])
diff --git a/sql/backends/monet5/Makefile.ag b/sql/backends/monet5/Makefile.ag
--- a/sql/backends/monet5/Makefile.ag
+++ b/sql/backends/monet5/Makefile.ag
@@ -17,7 +17,8 @@ INCLUDES = ../../include ../../common ..
../../../common/options \
../../../common/stream \
../../../common/utils \
- ../../../gdk
+ ../../../gdk \
+ $(pfor_CFLAGS)
lib__sql = {
MODULE
@@ -54,7 +55,7 @@ lib__sql = {
../../../common/stream/libstream \
../../../common/utils/libmcrypt \
$(PTHREAD_LIBS) \
- $(openssl_LIBS) $(MATH_LIBS)
+ $(openssl_LIBS) $(MATH_LIBS) $(pfor_LIBS)
}
headers_mal = {
diff --git a/sql/backends/monet5/sql_result.c b/sql/backends/monet5/sql_result.c
--- a/sql/backends/monet5/sql_result.c
+++ b/sql/backends/monet5/sql_result.c
@@ -20,6 +20,10 @@
#include <bat/bat_storage.h>
#include <rel_exp.h>
+#ifdef HAVE_PFOR
+#include <simdcomp.h>
+#endif
+
#ifndef HAVE_LLABS
#define llabs(x) ((x) < 0 ? -(x) : (x))
#endif
@@ -2128,6 +2132,38 @@ static int mvc_export_resultset_prot10(r
if (strcasecmp(c->type.type->sqlname,
"decimal") == 0) {
atom_size = sizeof(dbl);
}
+#ifdef HAVE_PFOR
+ if (strcasecmp(c->type.type->sqlname, "int") ==
0) {
+ // use PFOR for integer columns
+ size_t N = row - srow;
+ char *datain = Tloc(iterators[i].b,
srow);
+ __m128i * endofbuf;
+ uint8_t * buffer;
+ uint32_t b;
+ lng length;
+
+ b = maxbits_length(datain, N);
+ buffer =
malloc(simdpack_compressedbytes(N,b)); // allocate just enough memory
+ endofbuf = simdpack_length(datain, N,
(__m128i *)buffer, b);
+ if (!buffer || !endofbuf || endofbuf <=
buffer) {
+ printf("Compression failed!\n");
+ fres = -1;
+ goto cleanup;
+ }
+ length = (lng)((char*) endofbuf -
(char*) buffer);
+ if (!mnstr_writeLng(s, b) ||
!mnstr_writeLng(s, length) || mnstr_write(s, buffer, length, 1) != 1) {
+ fres = -1;
+ printf("Compression failed.\n");
+ goto cleanup;
+ }
+#ifdef PROT10_DEBUG
+ fprintf(stderr, "Write PFOR compressed
elements (b=%lld, length=%lld) to position %lld\n", b, length, bufpos);
+ bufpos += sizeof(lng) * 2 + length;
+#endif
+ free(buffer);
+
+ } else {
+#endif
#ifdef PROT10_DEBUG
fprintf(stderr, "Write elements of size %zu to
position %lld\n", atom_size * (row - srow), bufpos);
bufpos += (atom_size * (row - srow));
@@ -2136,6 +2172,9 @@ static int mvc_export_resultset_prot10(r
fres = -1;
goto cleanup;
}
+#ifdef HAVE_PFOR
+ }
+#endif
}
}
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
