Changeset: 34be56701bf5 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=34be56701bf5
Modified Files:
configure.ag
monetdb5/extras/Makefile.ag
monetdb5/extras/pyapi/Makefile.ag
monetdb5/extras/pyapi/connection.c
monetdb5/extras/pyapi/connection.h
monetdb5/extras/pyapi/formatinput.c
monetdb5/extras/pyapi/formatinput.h
monetdb5/extras/pyapi/pyapi.c
monetdb5/extras/pyapi/pyapi.h
monetdb5/extras/pyapi/pyapi.mal
monetdb5/extras/pyapi/pytypes.c
monetdb5/extras/pyapi/pytypes.h
monetdb5/extras/pyapi/type_conversion.c
monetdb5/extras/pyapi/type_conversion.h
monetdb5/extras/pyapi/unicode.c
monetdb5/extras/pyapi/unicode.h
monetdb5/optimizer/opt_prelude.c
monetdb5/optimizer/opt_prelude.h
monetdb5/optimizer/opt_support.c
sql/backends/monet5/sql_gencode.c
sql/include/sql_catalog.h
sql/server/rel_psm.c
sql/server/sql_parser.y
tools/merovingian/daemon/forkmserver.c
tools/merovingian/daemon/merovingian.c
tools/merovingian/utils/properties.c
tools/merovingian/utils/utils.c
Branch: python3udf
Log Message:
Initial commit for new branch testing Python3 UDFs using LANGUAGE
PYTHON3/PYTHON3_MAP.
This should be a relatively short lived branch.
Python3 UDFs can be enabled during configure using --enable-py3integration and
the mserver should be started with --set embedded_py=true to enable them at
runtime.
One caveat is that you can't currently compile with both PYTHON2 and PYTHON3
UDFs enabled because of runtime linking issues. Both PYTHON2 and PYTHON3 use
the same functions, and if MonetDB is linked against both py2 and py3 libraries
then py2 code will be called when py3 code is meant to be called, causing
strange issues. Thus currently you have to specify if you want py2 or py3 udfs
at compile time.
diffs (truncated from 862 to 300 lines):
diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -279,6 +279,13 @@ AC_ARG_ENABLE(pyintegration,
enable_pyintegration=$enableval,
enable_pyintegration=$dft_pyintegration)
+dft_py3integration=auto
+AC_ARG_ENABLE(py3integration,
+ AS_HELP_STRING([--enable-py3integration],
+ [enable support for Python 3 integration into MonetDB
(default=auto)]),
+ enable_py3integration=$enableval,
+ enable_py3integration=$dft_py3integration)
+
dft_odbc=auto
AC_ARG_ENABLE(odbc,
AS_HELP_STRING([--enable-odbc],
@@ -1284,13 +1291,13 @@ case "$have_python3" in
;;
esac
-AC_ARG_WITH(pyconfig,
- AS_HELP_STRING([--with-pyconfig=FILE], [python-config is installed as
FILE]),
- have_pyconfig="$withval")
-
-AC_ARG_WITH(pyversion,
- AS_HELP_STRING([--with-pyversion=FILE], [python is installed as FILE]),
- have_pyversion="$withval")
+AC_ARG_WITH(py2config,
+ AS_HELP_STRING([--with-py2config=FILE], [python2-config is installed as
FILE]),
+ have_py2config="$withval")
+
+AC_ARG_WITH(py3config,
+ AS_HELP_STRING([--with-py3config=FILE], [python3-config is installed as
FILE]),
+ have_py3config="$withval")
dnl Figure out a default for PYTHON2 or PYTHON3
AC_PATH_PROG(PYTHON,python,no,$PATH)
@@ -2331,27 +2338,24 @@ if test "x$enable_pyintegration" != xno;
AC_MSG_ERROR([--enable-pyintegration value must be
yes|no|auto|absolute path of python-config])
;;
esac
- if test "x$have_pyconfig" = x; then
- AC_PATH_PROG(PYCMD,python-config,,$XPATH)
- if test "x$PYCMD" = x; then
+ if test "x$have_py2config" = x; then
+ AC_PATH_PROG(PY2CONFIG,python2-config,,$XPATH)
+ if test "x$PY2CONFIG" = x; then
if test "x$enable_pyintegration" = xyes; then
- AC_MSG_ERROR([python-config library required
for Python integration support])
+ AC_MSG_ERROR([python2-config library required
for Python integration support])
else
have_libpy="no"
- why_have_libpy="(python-config command not
found)"
+ why_have_libpy="(python2-config command not
found)"
enable_pyintegration=no
- disable_pyintegration="(python-config command
not found)"
+ disable_pyintegration="(python2-config command
not found)"
fi
fi
else
- PYCMD="$have_pyconfig"
+ PY2CONFIG="$have_py2config"
fi
- if test "x$have_pyversion" = x; then
- PYTHON_CMD=$PYTHON2
- else
- PYTHON_CMD=$have_pyversion
- fi
+ PYTHON_CMD=$PYTHON2
+
NUMPYVER=`$PYTHON_CMD -c "import numpy;
print(int(numpy.__version__.split('.').__getitem__(1)) >= 7)"`
#check numpyconfig.h because autoconf tests includes by compiling a
small C program, and other numpy headers do not compile without Python.h
AC_CHECK_HEADER(
@@ -2379,7 +2383,7 @@ if test "x$enable_pyintegration" != xno;
fi
else
libpy_CFLAGS=`$PYTHON_CMD -c "from distutils.sysconfig import
get_python_inc; import numpy; print(' -I' + get_python_inc() + ' -I' +
numpy.get_include());"`
- libpy_LIBS=`$PYCMD --ldflags`
+ libpy_LIBS=`$PY2CONFIG --ldflags`
HAVEPYTHONHEADER=`$PYTHON_CMD -c "import distutils.sysconfig,
os; print(os.path.isfile(os.path.join(distutils.sysconfig.get_python_inc(),
'Python.h')))"`
if [test "x$HAVEPYTHONHEADER" = xTrue]; then
have_libpy=yes
@@ -2410,6 +2414,88 @@ AM_CONDITIONAL(HAVE_LIBPY, test x"$have_
AM_CONDITIONAL(HAVE_LIBPANDAS, test x"$have_libpandas" != xno)
AM_CONDITIONAL(HAVE_LIBSCIPY, test x"$have_libscipy" != xno)
+# Python 3 API
+have_libpy3=no
+if test "x$enable_py3integration" != xno; then
+ case "$enable_py3integration" in
+ yes|auto)
+ XPATH="$PATH"
+ ;;
+ /*)
+ XPATH="$enable_py3integration"
+ enable_py3integration=yes
+ ;;
+ *)
+ AC_MSG_ERROR([--enable-py3integration value must be
yes|no|auto|absolute path of python-config])
+ ;;
+ esac
+ if test "x$have_py3config" = x; then
+ # for some god forsaken reason autoconf thinks python2-config
and python3-config are the same program, so we have to explicitly tell it not
to select python2-config when we want python3-config
+ AC_PATH_PROG(PY3CONFIG,python3-config,,$XPATH,$PY2CONFIG)
+ if test "x$PY3CONFIG" = x; then
+ if test "x$enable_py3integration" = xyes; then
+ AC_MSG_ERROR([python3-config required for
Python3 integration support])
+ else
+ have_libpy3="no"
+ why_have_libpy3="(python3-config command not
found)"
+ enable_py3integration=no
+ disable_py3integration="(python3-config command
not found)"
+ fi
+ fi
+ else
+ PY3CONFIG="$have_py3config"
+ fi
+
+ PYTHON_CMD=$PYTHON3
+
+ NUMPYVER=`$PYTHON_CMD -c "import numpy;
print(int(numpy.__version__.split('.').__getitem__(1)) >= 7)"`
+ # check numpyconfig.h because autoconf tests includes by compiling a
small C program, and other numpy headers do not compile without Python.h
+ AC_CHECK_HEADER(
+ [numpy/numpyconfig.h],
+ [NUMPYHEADERS=True],
+ [NUMPYHEADERS=`$PYTHON_CMD -c "import numpy, os;
print(os.path.isfile(os.path.join(numpy.get_include(),
'numpy/arrayobject.h')))"`]
+ )
+ if [test "x$NUMPYVER" = x] || [test "x$NUMPYVER" = xFalse]; then
+ if test "x$enable_py3integration" = xyes; then
+ AC_MSG_ERROR([numpy version >= 1.7.0 required for
Python3 integration support])
+ else
+ have_libpy3="no"
+ why_have_libpy3="(numpy version >= 1.7.0 not found)"
+ enable_py3integration=no
+ disable_py3integration="(numpy version >= 1.7.0 not
found)"
+ fi
+ elif [test "x$NUMPYHEADERS" = xFalse ]; then
+ if test "x$enable_py3integration" = xyes; then
+ AC_MSG_ERROR([numpy/arrayobject.h not found])
+ else
+ have_libpy3="no"
+ why_have_libpy3="(numpy/arrayobject.h not found)"
+ enable_py3integration=no
+ disable_py3integration="(numpy/arrayobject.h not found)"
+ fi
+ else
+ libpy3_CFLAGS=`$PYTHON_CMD -c "from distutils.sysconfig import
get_python_inc; import numpy; print(' -I' + get_python_inc() + ' -I' +
numpy.get_include());"`
+ libpy3_LIBS=`$PY3CONFIG --ldflags`
+ HAVEPYTHONHEADER=`$PYTHON_CMD -c "import distutils.sysconfig,
os; print(os.path.isfile(os.path.join(distutils.sysconfig.get_python_inc(),
'Python.h')))"`
+ if [test "x$HAVEPYTHONHEADER" = xTrue]; then
+ have_libpy3=yes
+ AC_DEFINE(HAVE_LIBPY3, 1, [Define if we can link to
python])
+ AC_SUBST(libpy3_CFLAGS, $libpy3_CFLAGS)
+ AC_SUBST(libpy3_LIBS, $libpy3_LIBS)
+ else
+ if test "x$enable_py3integration" = xyes; then
+ AC_MSG_ERROR([Python.h not found])
+ else
+ have_libpy3="no"
+ why_have_libpy3="Python.h not found"
+ enable_py3integration=no
+ disable_py3integration="Python.h not found"
+ fi
+ fi
+ fi
+fi
+AM_CONDITIONAL(HAVE_LIBPY3, test x"$have_libpy3" != xno)
+
AC_CHECK_HEADER([sys/sem.h], [SEMOP=True], [SEMOP=False])
if [test "x$SEMOP" = "xTrue"]; then
AC_CHECK_FUNC(semtimedop, [AC_DEFINE(HAVE_SEMTIMEDOP, 1, [Define if
semtimedop exists])],)
@@ -3698,6 +3784,7 @@ for comp in \
'console ' \
'microhttpd ' \
'pyintegration' \
+ 'py3integration' \
'rintegration' \
'odbc ' \
'jdbc ' \
diff --git a/monetdb5/extras/Makefile.ag b/monetdb5/extras/Makefile.ag
--- a/monetdb5/extras/Makefile.ag
+++ b/monetdb5/extras/Makefile.ag
@@ -4,5 +4,5 @@
#
# Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V.
-SUBDIRS = HAVE_SPHINXCLIENT?sphinx mal_optimizer_template HAVE_LIBR?rapi
HAVE_LIBPY?pyapi
+SUBDIRS = HAVE_SPHINXCLIENT?sphinx mal_optimizer_template HAVE_LIBR?rapi
HAVE_LIBPY?pyapi HAVE_LIBPY3?pyapi3
diff --git a/monetdb5/extras/pyapi/Makefile.ag
b/monetdb5/extras/pyapi/Makefile.ag
--- a/monetdb5/extras/pyapi/Makefile.ag
+++ b/monetdb5/extras/pyapi/Makefile.ag
@@ -18,6 +18,7 @@ INCLUDES = \
../../../sql/server \
../../../sql/storage \
$(libpy_CFLAGS)
+
MTSAFE
diff --git a/monetdb5/extras/pyapi/connection.c
b/monetdb5/extras/pyapi/connection.c
--- a/monetdb5/extras/pyapi/connection.c
+++ b/monetdb5/extras/pyapi/connection.c
@@ -178,9 +178,22 @@ static PyMethodDef _connectionObject_met
{NULL,NULL,0,NULL} /* Sentinel */
};
-PyTypeObject Py_ConnectionType = {
- PyObject_HEAD_INIT(NULL)
- 0,
+PyTypeObject Py_ConnectionType = {
+ _PyObject_EXTRA_INIT
+// in python3 they use structs within structs to represent this information,
and many compilers throw warnings if you don't use separate braces
+// to initialize these separate structs. However, in Python2, they use
#defines to put this information in, so we have these nice #ifdefs
+#ifdef IS_PY3K
+ { {
+#endif
+ 1, NULL
+#ifdef IS_PY3K
+ }
+#endif
+ , 0
+#ifdef IS_PY3K
+ }
+#endif
+ ,
"monetdb._connection",
sizeof(Py_ConnectionObject),
0,
@@ -262,7 +275,7 @@ PyObject *Py_Connection_Create(Client cn
return (PyObject*) op;
}
-void _connection_init(void)
+NUMPY_IMPORT_ARRAY_RETTYPE _connection_init(void)
{
import_array();
@@ -270,5 +283,6 @@ void _connection_init(void)
LOAD_SQL_FUNCTION_PTR(SQLstatementIntern, "lib_sql.dll");
if (PyType_Ready(&Py_ConnectionType) < 0)
- return;
+ return NUMPY_IMPORT_ARRAY_RETVAL;
+ return NUMPY_IMPORT_ARRAY_RETVAL;
}
diff --git a/monetdb5/extras/pyapi/connection.h
b/monetdb5/extras/pyapi/connection.h
--- a/monetdb5/extras/pyapi/connection.h
+++ b/monetdb5/extras/pyapi/connection.h
@@ -24,14 +24,14 @@ typedef struct {
int query_sem;
} Py_ConnectionObject;
-extern PyTypeObject Py_ConnectionType;
+PyAPI_DATA(PyTypeObject) Py_ConnectionType;
#define Py_Connection_Check(op) (Py_TYPE(op) == &Py_ConnectionType)
#define Py_Connection_CheckExact(op) (Py_TYPE(op) == &Py_ConnectionType)
PyObject *Py_Connection_Create(Client cntxt, bit mapped, QueryStruct
*query_ptr, int query_sem);
-void _connection_init(void);
+NUMPY_IMPORT_ARRAY_RETTYPE _connection_init(void);
char* _connection_query(Client cntxt, char* query, res_table** result);
void _connection_cleanup_result(void* output);
diff --git a/monetdb5/extras/pyapi/formatinput.c
b/monetdb5/extras/pyapi/formatinput.c
--- a/monetdb5/extras/pyapi/formatinput.c
+++ b/monetdb5/extras/pyapi/formatinput.c
@@ -375,6 +375,7 @@ finally:
return newcode;
}
-void _formatinput_init(void) {
+NUMPY_IMPORT_ARRAY_RETTYPE _formatinput_init(void) {
import_array();
+ return NUMPY_IMPORT_ARRAY_RETVAL;
}
diff --git a/monetdb5/extras/pyapi/formatinput.h
b/monetdb5/extras/pyapi/formatinput.h
--- a/monetdb5/extras/pyapi/formatinput.h
+++ b/monetdb5/extras/pyapi/formatinput.h
@@ -22,6 +22,6 @@ extern PyObject *marshal_loads;
char* FormatCode(char* code, char **args, size_t argcount, size_t tabwidth,
PyObject **code_object, char **return_message);
-void _formatinput_init(void);
+NUMPY_IMPORT_ARRAY_RETTYPE _formatinput_init(void);
#endif /* _PY_FORMAT_INPUT_LIB_ */
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -33,10 +33,13 @@
#define PythonUnicodeType char
#else
#define PythonUnicodeType Py_UNICODE
-
#endif
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
