Changeset: a6c7dd424a5d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a6c7dd424a5d
Added Files:
gdk/shared_memory.c
gdk/shared_memory.h
monetdb5/extras/pyapi/50_pyapi.mal
monetdb5/extras/pyapi/Makefile.ag
monetdb5/extras/pyapi/Tests/All
monetdb5/extras/pyapi/Tests/pyapi_numpy_boolean.malC
monetdb5/extras/pyapi/Tests/pyapi_numpy_boolean.stable.err
monetdb5/extras/pyapi/Tests/pyapi_numpy_boolean.stable.out
monetdb5/extras/pyapi/Tests/pyapi_numpy_numeric_nested.malC
monetdb5/extras/pyapi/Tests/pyapi_numpy_numeric_nested.stable.err
monetdb5/extras/pyapi/Tests/pyapi_numpy_numeric_nested.stable.out
monetdb5/extras/pyapi/Tests/pyapi_pandas.malC
monetdb5/extras/pyapi/Tests/pyapi_pandas.stable.err
monetdb5/extras/pyapi/Tests/pyapi_pandas.stable.out
monetdb5/extras/pyapi/Tests/pyapi_returntypes.malC
monetdb5/extras/pyapi/Tests/pyapi_returntypes.stable.err
monetdb5/extras/pyapi/Tests/pyapi_returntypes.stable.out
monetdb5/extras/pyapi/Tests/pyapi_types_boolean.malC
monetdb5/extras/pyapi/Tests/pyapi_types_boolean.stable.err
monetdb5/extras/pyapi/Tests/pyapi_types_boolean.stable.out
monetdb5/extras/pyapi/Tests/pyapi_types_huge.malC
monetdb5/extras/pyapi/Tests/pyapi_types_huge.stable.err
monetdb5/extras/pyapi/Tests/pyapi_types_huge.stable.out
monetdb5/extras/pyapi/Tests/pyapi_types_numeric.malC
monetdb5/extras/pyapi/Tests/pyapi_types_numeric.stable.err
monetdb5/extras/pyapi/Tests/pyapi_types_numeric.stable.out
monetdb5/extras/pyapi/Tests/pyapi_types_string.malC
monetdb5/extras/pyapi/Tests/pyapi_types_string.stable.err
monetdb5/extras/pyapi/Tests/pyapi_types_string.stable.out
monetdb5/extras/pyapi/connection.c
monetdb5/extras/pyapi/connection.h
monetdb5/extras/pyapi/formatinput.c
monetdb5/extras/pyapi/formatinput.h
monetdb5/extras/pyapi/pyapi.c
monetdb5/extras/pyapi/pyapi.h
monetdb5/extras/pyapi/pyapi.mal
monetdb5/extras/pyapi/pytypes.c
monetdb5/extras/pyapi/pytypes.h
monetdb5/extras/pyapi/type_conversion.c
monetdb5/extras/pyapi/type_conversion.h
monetdb5/extras/pyapi/unicode.c
monetdb5/extras/pyapi/unicode.h
sql/backends/monet5/Tests/pyapi00.sql
sql/backends/monet5/Tests/pyapi00.stable.err
sql/backends/monet5/Tests/pyapi00.stable.out
sql/backends/monet5/Tests/pyapi01.sql
sql/backends/monet5/Tests/pyapi01.stable.err
sql/backends/monet5/Tests/pyapi01.stable.out
sql/backends/monet5/Tests/pyapi02.sql
sql/backends/monet5/Tests/pyapi02.sql.orig
sql/backends/monet5/Tests/pyapi02.stable.err
sql/backends/monet5/Tests/pyapi02.stable.out
sql/backends/monet5/Tests/pyapi03.sql
sql/backends/monet5/Tests/pyapi03.stable.err
sql/backends/monet5/Tests/pyapi03.stable.out
sql/backends/monet5/Tests/pyapi04.sql
sql/backends/monet5/Tests/pyapi04.stable.err
sql/backends/monet5/Tests/pyapi04.stable.out
sql/backends/monet5/Tests/pyapi05.sql
sql/backends/monet5/Tests/pyapi05.stable.err
sql/backends/monet5/Tests/pyapi05.stable.out
sql/backends/monet5/Tests/pyapi06.sql
sql/backends/monet5/Tests/pyapi06.stable.err
sql/backends/monet5/Tests/pyapi06.stable.out
sql/backends/monet5/Tests/pyapi07.sql
sql/backends/monet5/Tests/pyapi07.stable.err
sql/backends/monet5/Tests/pyapi07.stable.out
sql/backends/monet5/Tests/pyapi08.sql
sql/backends/monet5/Tests/pyapi08.sql.orig
sql/backends/monet5/Tests/pyapi08.stable.err
sql/backends/monet5/Tests/pyapi08.stable.out
sql/backends/monet5/Tests/pyapi09.sql
sql/backends/monet5/Tests/pyapi09.stable.err
sql/backends/monet5/Tests/pyapi09.stable.out
sql/backends/monet5/Tests/pyapi10.sql
sql/backends/monet5/Tests/pyapi10.stable.err
sql/backends/monet5/Tests/pyapi10.stable.out
sql/backends/monet5/Tests/pyapi11.sql
sql/backends/monet5/Tests/pyapi11.stable.err
sql/backends/monet5/Tests/pyapi11.stable.out
sql/backends/monet5/Tests/pyapi12.sql
sql/backends/monet5/Tests/pyapi12.stable.err
sql/backends/monet5/Tests/pyapi12.stable.out
sql/backends/monet5/Tests/pyapi13.sql
sql/backends/monet5/Tests/pyapi13.stable.err
sql/backends/monet5/Tests/pyapi13.stable.out
sql/backends/monet5/Tests/pyapi14.sql
sql/backends/monet5/Tests/pyapi14.stable.err
sql/backends/monet5/Tests/pyapi14.stable.out
sql/backends/monet5/Tests/pyapi15.sql
sql/backends/monet5/Tests/pyapi16.sql
sql/backends/monet5/Tests/pyapi16.stable.err
sql/backends/monet5/Tests/pyapi16.stable.out
sql/backends/monet5/Tests/pyapi17.sql
sql/backends/monet5/Tests/pyapi17.stable.err
sql/backends/monet5/Tests/pyapi17.stable.out
sql/backends/monet5/Tests/pyapi18.sql
sql/backends/monet5/Tests/pyapi18.stable.err
sql/backends/monet5/Tests/pyapi18.stable.out
sql/backends/monet5/Tests/pyapi19.sql
sql/backends/monet5/Tests/pyapi19.stable.err
sql/backends/monet5/Tests/pyapi19.stable.out
sql/backends/monet5/Tests/pyapi20.sql
sql/backends/monet5/Tests/pyapi20.stable.err
sql/backends/monet5/Tests/pyapi20.stable.out
sql/backends/monet5/Tests/pyapi21.sql
sql/backends/monet5/Tests/pyapi21.stable.err
sql/backends/monet5/Tests/pyapi21.stable.out
sql/backends/monet5/Tests/pyapi23.sql
sql/backends/monet5/Tests/pyapi23.stable.err
sql/backends/monet5/Tests/pyapi23.stable.out
sql/backends/monet5/Tests/pyapi24.sql
sql/backends/monet5/Tests/pyapi24.stable.err
sql/backends/monet5/Tests/pyapi24.stable.out
sql/backends/monet5/Tests/pyapi25.sql
sql/backends/monet5/Tests/pyapi25.stable.err
sql/backends/monet5/Tests/pyapi25.stable.out
sql/backends/monet5/Tests/pyapi26.sql
sql/backends/monet5/Tests/pyapi26.stable.err
sql/backends/monet5/Tests/pyapi26.stable.out
sql/backends/monet5/Tests/pyapi27.sql
sql/backends/monet5/Tests/pyapi27.stable.err
sql/backends/monet5/Tests/pyapi27.stable.out
Modified Files:
clients/mapiclient/mclient.c
configure.ag
gdk/Makefile.ag
gdk/gdk.h
gdk/gdk_heap.c
monetdb5/extras/Makefile.ag
monetdb5/optimizer/opt_prelude.c
monetdb5/optimizer/opt_prelude.h
monetdb5/optimizer/opt_support.c
sql/backends/monet5/Tests/All
sql/backends/monet5/mal_backend.h
sql/backends/monet5/sql.c
sql/backends/monet5/sql_execute.c
sql/backends/monet5/sql_gencode.c
sql/backends/monet5/sql_result.c
sql/include/sql_catalog.h
sql/server/rel_psm.c
sql/server/sql_parser.y
testing/Mtest.py.in
Branch: pythonudf
Log Message:
Initial branch commit; Python UDFs.
This branch is a clean version of the pyapi branch. It does not have any of the
unnecessary benchmarking code, and does not have any of the monetdblite code.
Only Python UDFs and associated test cases.
diffs (truncated from 15543 to 300 lines):
diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c
--- a/clients/mapiclient/mclient.c
+++ b/clients/mapiclient/mclient.c
@@ -1220,8 +1220,7 @@ static void
TIMERrenderer(MapiHdl hdl)
{
SQLqueryEcho(hdl);
- while (fetch_line(hdl) != 0)
- ;
+ mapi_next_result(hdl);
printf("%s\n", timerHuman());
}
@@ -3270,6 +3269,9 @@ main(int argc, char **argv)
setFormatter("raw");
}
}
+ if (formatter == TIMERformatter) {
+ mapi_cache_limit(mid, 1);
+ }
/* give the user a welcome message with some general info */
if (!has_fileargs && command == NULL && isatty(fileno(stdin))) {
char *lang;
diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -223,6 +223,13 @@ AC_ARG_ENABLE(rintegration,
enable_rintegration=$enableval,
enable_rintegration=$dft_rintegration)
+dft_pyintegration=auto
+AC_ARG_ENABLE(pyintegration,
+ AS_HELP_STRING([--enable-pyintegration],
+ [enable support for Python integration into MonetDB
(default=auto)]),
+ enable_pyintegration=$enableval,
+ enable_pyintegration=$dft_pyintegration)
+
dft_odbc=auto
AC_ARG_ENABLE(odbc,
AS_HELP_STRING([--enable-odbc],
@@ -2245,6 +2252,7 @@ AC_SUBST(SPHINXCLIENT_CFLAGS, $SPHINXCLI
AC_SUBST(SPHINXCLIENT_LIBS, $SPHINXCLIENT_LIBS)
AM_CONDITIONAL(HAVE_SPHINXCLIENT, test x"$have_sphinxclient" != xno)
+# R API (R UDFs)
have_libr=no
if test "x$enable_rintegration" != xno; then
case "$enable_rintegration" in
@@ -2280,6 +2288,85 @@ if test "x$enable_rintegration" != xno;
fi
AM_CONDITIONAL(HAVE_LIBR, test x"$have_libr" != xno)
+# Python API (Python UDFs)
+have_libpy=no
+have_libpandas=no
+have_libscipy=no
+if test "x$enable_pyintegration" != xno; then
+ case "$enable_pyintegration" in
+ yes|auto)
+ XPATH="$PATH"
+ ;;
+ /*)
+ XPATH="$enable_pyintegration"
+ enable_pyintegration=yes
+ ;;
+ *)
+ AC_MSG_ERROR([--enable-pyintegration value must be
yes|no|auto|absolute path of python-config])
+ ;;
+ esac
+ AC_PATH_PROG(PYCMD,python-config,,$XPATH)
+ if test "x$PYCMD" = x; then
+ if test "x$enable_pyintegration" = xyes; then
+ AC_MSG_ERROR([python-config library required for Python
integration support])
+ else
+ have_libpy="no"
+ why_have_libpy="(python-config command not found)"
+ enable_pyintegration=no
+ disable_pyintegration="(python-config command not
found)"
+ fi
+ fi
+ PYTHON_CMD=`$PYCMD --exec-prefix`/bin/python
+ NUMPYVER=`$PYTHON_CMD -c "import numpy; print
numpy.__version__.split('.').__getitem__(1) >= 7"`
+ #check numpyconfig.h because autoconf tests includes by compiling a
small C program, and other numpy headers do not compile without Python.h
+ AC_CHECK_HEADER(
+ [numpy/numpyconfig.h],
+ [NUMPYHEADERS=True],
+ [NUMPYHEADERS=`$PYTHON_CMD -c "import numpy, os; print
os.path.isfile(os.path.join(numpy.get_include(), 'numpy/arrayobject.h'))"`]
+ )
+ if [test "x$NUMPYVER" = x] || [test "x$NUMPYVER" = xFalse]; then
+ if test "x$enable_pyintegration" = xyes; then
+ AC_MSG_ERROR([numpy version >= 1.7.0 required for
Python integration support])
+ else
+ have_libpy="no"
+ why_have_libpy="(numpy version >= 1.7.0 not found)"
+ enable_pyintegration=no
+ disable_pyintegration="(numpy version >= 1.7.0 not
found)"
+ fi
+ elif [test "x$NUMPYHEADERS" = xFalse ]; then
+ if test "x$enable_pyintegration" = xyes; then
+ AC_MSG_ERROR([numpy/arrayobject.h not found])
+ else
+ have_libpy="no"
+ why_have_libpy="(numpy/arrayobject.h not found)"
+ enable_pyintegration=no
+ disable_pyintegration="(numpy/arrayobject.h not found)"
+ fi
+ elif libpy_CFLAGS=`\`$PYCMD --exec-prefix\`/bin/python -c "from
distutils.sysconfig import get_python_inc; import numpy; print ' -I' +
get_python_inc() + ' -I' + numpy.get_include();"` && libpy_LIBS=`$PYCMD
--ldflags`; then
+ have_libpy=yes
+ AC_DEFINE(HAVE_LIBPY, 1, [Define if we can link to python])
+ AC_SUBST(libpy_CFLAGS, $libpy_CFLAGS)
+ AC_SUBST(libpy_LIBS, $libpy_LIBS)
+ PANDAS=`$PYTHON_CMD -c "import pandas; print 1"`
+ if [test "x$PANDAS" = x1]; then
+ have_libpandas=yes
+ fi
+ SCIPY=`$PYTHON_CMD -c "import scipy; print 1"`
+ if [test "x$SCIPY" = x1]; then
+ have_libscipy=yes
+ fi
+ fi
+fi
+AM_CONDITIONAL(HAVE_LIBPY, test x"$have_libpy" != xno)
+AM_CONDITIONAL(HAVE_LIBPANDAS, test x"$have_libpandas" != xno)
+AM_CONDITIONAL(HAVE_LIBSCIPY, test x"$have_libscipy" != xno)
+
+AC_CHECK_HEADER([sys/sem.h], [SEMOP=True], [SEMOP=False])
+if [test "x$SEMOP" = "xTrue"]; then
+ AC_CHECK_FUNC(semtimedop, [AC_DEFINE(HAVE_SEMTIMEDOP, 1, [Define if
semtimedop exists])],)
+ AC_CHECK_FUNC(fork, [AC_DEFINE(HAVE_FORK, 1, [Define if fork exists])],
)
+fi
+
# ODBC, only used by ODBC driver
if test "x$enable_odbc" != xno; then
have_unixodbc=auto
@@ -3426,6 +3513,7 @@ for comp in \
'geom ' \
'console ' \
'microhttpd ' \
+ 'pyintegration' \
'rintegration' \
'odbc ' \
'jdbc ' \
diff --git a/gdk/Makefile.ag b/gdk/Makefile.ag
--- a/gdk/Makefile.ag
+++ b/gdk/Makefile.ag
@@ -31,7 +31,7 @@ lib_gdk = {
gdk_unique.c \
gdk_firstn.c \
bat.feps bat1.feps bat2.feps \
- libbat.rc
+ libbat.rc shared_memory.c shared_memory.h
LIBS = ../common/options/libmoptions \
../common/stream/libstream \
../common/utils/libmutils \
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -641,9 +641,12 @@ typedef enum { GDK_FAIL, GDK_SUCCEED } g
/* Heap storage modes */
typedef enum {
- STORE_MEM = 0, /* load into GDKmalloced memory */
- STORE_MMAP = 1, /* mmap() into virtual memory */
- STORE_PRIV = 2, /* BAT copy of copy-on-write mmap */
+ STORE_MEM = 0, /* load into GDKmalloced memory */
+ STORE_MMAP = 1, /* mmap() into virtual memory */
+ STORE_PRIV = 2, /* BAT copy of copy-on-write mmap */
+ STORE_CMEM = 3, /* Indicates the value is stored in regular C
memory rather than GDK memory.*/
+ STORE_NOWN = 4, /* Indicates that the bat does not own the chunk
of memory and is not in charge of freeing it.*/
+ STORE_MMAPABS = 5, /* mmap() into virtual memory from an absolute
path (not part of dbfarm) */
STORE_INVALID /* invalid value, used to indicate error */
} storage_t;
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -562,6 +562,9 @@ HEAPfree(Heap *h, int remove)
" " PTRFMT "\n",
h->size, PTRFMTCAST h->base);
GDKfree(h->base);
+ } else if (h->storage == STORE_CMEM) {
+ //heap is stored in regular C memory rather than GDK
memory,so we call free()
+ free(h->base);
} else { /* mapped file, or STORE_PRIV */
gdk_return ret = GDKmunmap(h->base, h->size);
@@ -576,6 +579,16 @@ HEAPfree(Heap *h, int remove)
h->size, (int) ret);
}
}
+#ifdef HAVE_FORK
+ if (h->storage == STORE_MMAPABS) {
+ // heap is stored in a mmap() file, but h->filename points to
the absolute path
+ if (h->filename && unlink(h->filename) < 0 && errno != ENOENT) {
+ perror(h->filename);
+ }
+ GDKfree(h->filename);
+ h->filename = NULL;
+ }
+#endif
h->base = NULL;
if (h->filename) {
if (remove) {
diff --git a/gdk/shared_memory.c b/gdk/shared_memory.c
new file mode 100644
--- /dev/null
+++ b/gdk/shared_memory.c
@@ -0,0 +1,207 @@
+
+#include "shared_memory.h"
+
+#ifdef HAVE_FORK
+
+#include "gdk.h"
+#include "gdk_private.h"
+#include "../monetdb5/mal/mal_exception.h"
+
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+
+#include <sys/types.h>
+#include <sys/ipc.h>
+#include <sys/shm.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sched.h>
+#include <errno.h>
+#include <sys/sem.h>
+#include <time.h>
+
+static size_t shm_unique_id = 1;
+static key_t base_key = 800000000;
+
+str ftok_enhanced(int id, key_t *return_key);
+str init_process_semaphore(int id, int count, int flags, int *semid);
+
+size_t get_unique_id(size_t offset) {
+ size_t id;
+
+ id = shm_unique_id;
+ shm_unique_id += offset;
+ return id;
+}
+
+str snprintf_mmap_file(str file, size_t max, size_t id) {
+ snprintf(file, max, "pymmap%zu", id);
+ return MAL_SUCCEED;
+}
+
+str init_mmap_memory(size_t base_id, size_t id_offset, size_t maxsize, void
***return_ptr, size_t **return_size, char **single_ptr) {
+ char address[100];
+ void *ptr;
+ int fd;
+ size_t size = maxsize;
+ int mod = MMAP_READ | MMAP_WRITE | MMAP_SEQUENTIAL | MMAP_SYNC |
MAP_SHARED;
+ char *path = NULL;
+ snprintf_mmap_file(address, 100, base_id + id_offset);
+
+ /* round up to multiple of GDK_mmap_pagesize with a
+ * minimum of one
+ size = (maxsize + GDK_mmap_pagesize - 1) & ~(GDK_mmap_pagesize - 1);
+ if (size == 0)
+ size = GDK_mmap_pagesize; */
+ fd = GDKfdlocate(NOFARM, address, "wb", "tmp");
+ if (fd < 0) {
+ return createException(MAL, "shared_memory.get", "Failure in
GDKfdlocate(NOFARM, %s, \"wb\", NULL)", address);
+ }
+ close(fd);
+ path = GDKfilepath(NOFARM, BATDIR, address, "tmp");
+ if (path == NULL) {
+ return createException(MAL, "shared_memory.get", "Failure in
GDKfilepath(NOFARM, "BATDIR",%s,\"tmp\")", address);
+ }
+ if (GDKextend(path, size) != GDK_SUCCEED) {
+ return createException(MAL, "shared_memory.get", "Failure in
GDKextend(%s,%zu)", path, size);
+ }
+ ptr = GDKmmap(path, mod, size);
+ if (ptr == NULL) {
+ return createException(MAL, "shared_memory.get", "Failure in
GDKmmap(%s, %d, %zu)", path, mod, size);
+ }
+ GDKfree(path);
+ if (return_ptr != NULL) (*return_ptr)[id_offset] = ptr;
+ if (return_size != NULL) (*return_size)[id_offset] = size;
+ if (single_ptr != NULL) *single_ptr = ptr;
+ return MAL_SUCCEED;
+}
+
+str release_mmap_memory(void *ptr, size_t size, size_t id) {
+ char address[100];
+ char *path;
+ int ret;
+ snprintf_mmap_file(address, 100, id);
+ if (GDKmunmap(ptr, size) != GDK_SUCCEED) {
+ return createException(MAL, "shared_memory.get", "Failure in
GDKmunmap(%p, %zu)", ptr, size);
+ }
+ path = GDKfilepath(NOFARM, BATDIR, address, "tmp");
+ if (path == NULL) {
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
