FWIW I bet it would be helpful to have a general-purpose minimal executor library.
> On Mar 11, 2016, at 6:55 PM, Benjamin Mahler <[email protected]> wrote: > > +vinod > > This breaks the build for me on OS X, it appears this line is the culprit: > > EXTRA_LINK_ARGS = ['-Wl,--as-needed'] > > This leads to the following: > > clang++ -bundle -undefined dynamic_lookup -arch x86_64 -arch i386 -Wl,-F. > -L/usr/local/opt/subversion/lib -O2 -O2 -Wno-unused-local-typedef -std=c++11 > -stdlib=libc++ -DGTEST_USE_OWN_TR1_TUPLE=1 -DGTEST_LANG_CXX11 > -Qunused-arguments -I/usr/local/opt/subversion/include/subversion-1 > -I/usr/include/apr-1 -I/usr/include/apr-1.0 -Qunused-arguments > build/temp.macosx-10.11-intel-2.7/src/mesos/executor/mesos_executor_driver_impl.o > build/temp.macosx-10.11-intel-2.7/src/mesos/executor/module.o > build/temp.macosx-10.11-intel-2.7/src/mesos/executor/proxy_executor.o > /Users/bmahler/git/mesos/build/src/.libs/libmesos_no_3rdparty.a > /Users/bmahler/git/mesos/build/3rdparty/libprocess/.libs/libprocess.a > /Users/bmahler/git/mesos/build/3rdparty/leveldb-1.4/libleveldb.a > /Users/bmahler/git/mesos/build/3rdparty/zookeeper-3.4.5/src/c/.libs/libzookeeper_mt.a > > /Users/bmahler/git/mesos/build/3rdparty/libprocess/3rdparty/glog-0.3.3/.libs/libglog.a > > /Users/bmahler/git/mesos/build/3rdparty/libprocess/3rdparty/protobuf-2.5.0/src/.libs/libprotobuf.a > -o build/lib.macosx-10.11-intel-2.7/mesos/executor/_executor.so > -Wl,--as-needed -L/usr/local/opt/subversion/lib -levent_openssl -lcrypto > -lssl -levent_pthreads -levent -lsasl2 -lsvn_delta-1 -lsvn_subr-1 -lapr-1 > -lcurl -lz > ld: unknown option: --as-needed > clang: error: linker command failed with exit code 1 (use -v to see > invocation) > > On Fri, Mar 11, 2016 at 1:56 PM, <[email protected]> wrote: > New python lib with only the executor driver. > > This patch produces a new python egg, mesos.executor, which contains only the > code needed to create a MesosExecutorDriver. By doing so, the linker can > remove > unused code in libmesos_no_3rdparty.a, and therefor not include any external > dependencies in the resulting _mesos.so. > > Review: https://reviews.apache.org/r/41049/ > > > Project: http://git-wip-us.apache.org/repos/asf/mesos/repo > Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/c81a52ec > Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/c81a52ec > Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/c81a52ec > > Branch: refs/heads/master > Commit: c81a52ec22266e1f2beb61b224c0f0d9be82521f > Parents: 482dc14 > Author: Steve Niemitz <[email protected]> > Authored: Fri Mar 11 16:56:13 2016 -0500 > Committer: Vinod Kone <[email protected]> > Committed: Fri Mar 11 16:56:13 2016 -0500 > > ---------------------------------------------------------------------- > configure.ac | 7 +- > src/Makefile.am | 33 +- > src/python/executor/setup.py.in | 39 + > src/python/executor/src/mesos/__init__.py | 10 + > .../executor/src/mesos/executor/__init__.py | 17 + > .../executor/mesos_executor_driver_impl.cpp | 347 ++++++++ > .../executor/mesos_executor_driver_impl.hpp | 103 +++ > .../executor/src/mesos/executor/module.cpp | 91 +++ > .../src/mesos/executor/proxy_executor.cpp | 273 +++++++ > .../src/mesos/executor/proxy_executor.hpp | 64 ++ > src/python/native/ext_modules.py.in | 151 ---- > src/python/native/setup.py.in | 9 +- > src/python/native/src/mesos/native/__init__.py | 7 +- > .../mesos/native/mesos_executor_driver_impl.cpp | 347 -------- > .../mesos/native/mesos_executor_driver_impl.hpp | 103 --- > .../native/mesos_scheduler_driver_impl.cpp | 782 ------------------- > .../native/mesos_scheduler_driver_impl.hpp | 134 ---- > src/python/native/src/mesos/native/module.cpp | 100 --- > src/python/native/src/mesos/native/module.hpp | 136 ---- > .../native/src/mesos/native/proxy_executor.cpp | 273 ------- > .../native/src/mesos/native/proxy_executor.hpp | 64 -- > .../native/src/mesos/native/proxy_scheduler.cpp | 384 --------- > .../native/src/mesos/native/proxy_scheduler.hpp | 72 -- > src/python/native_common/common.hpp | 136 ++++ > src/python/native_common/ext_modules.py.in | 154 ++++ > src/python/scheduler/setup.py.in | 39 + > src/python/scheduler/src/mesos/__init__.py | 10 + > .../scheduler/src/mesos/scheduler/__init__.py | 17 + > .../scheduler/mesos_scheduler_driver_impl.cpp | 782 +++++++++++++++++++ > .../scheduler/mesos_scheduler_driver_impl.hpp | 134 ++++ > .../scheduler/src/mesos/scheduler/module.cpp | 91 +++ > .../src/mesos/scheduler/proxy_scheduler.cpp | 384 +++++++++ > .../src/mesos/scheduler/proxy_scheduler.hpp | 72 ++ > 33 files changed, 2795 insertions(+), 2570 deletions(-) > ---------------------------------------------------------------------- > > > http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/configure.ac > ---------------------------------------------------------------------- > diff --git a/configure.ac b/configure.ac > index a20382e..8e4f035 100644 > --- a/configure.ac > +++ b/configure.ac > @@ -1315,8 +1315,13 @@ There are two possible workarounds for this issue: > AC_CONFIG_FILES([src/python/setup.py]) > AC_CONFIG_FILES([src/python/cli/setup.py]) > AC_CONFIG_FILES([src/python/interface/setup.py]) > - AC_CONFIG_FILES([src/python/native/ext_modules.py]) > + AC_CONFIG_FILES([src/python/native_common/ext_modules.py]) > + AC_CONFIG_FILES([src/python/executor/setup.py]) > AC_CONFIG_FILES([src/python/native/setup.py]) > + AC_CONFIG_FILES([src/python/scheduler/setup.py]) > + > + > AC_CONFIG_LINKS([src/python/executor/ext_modules.py:src/python/native_common/ext_modules.py]) > + > AC_CONFIG_LINKS([src/python/scheduler/ext_modules.py:src/python/native_common/ext_modules.py]) > > # When clang is being used, make sure that the distutils python- > # config cflags extraction does not cause build errors (MESOS-1079). > > http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/Makefile.am > ---------------------------------------------------------------------- > diff --git a/src/Makefile.am b/src/Makefile.am > index f59ae12..8abef3b 100644 > --- a/src/Makefile.am > +++ b/src/Makefile.am > @@ -1467,18 +1467,23 @@ PYTHON_SOURCE = > \ > python/interface/src/mesos/interface/__init__.py \ > python/interface/src/mesos/v1/__init__.py \ > python/interface/src/mesos/v1/interface/__init__.py \ > + python/native_common/common.hpp \ > python/native/src/mesos/__init__.py \ > python/native/src/mesos/native/__init__.py \ > - python/native/src/mesos/native/mesos_executor_driver_impl.cpp > \ > - python/native/src/mesos/native/mesos_executor_driver_impl.hpp > \ > - python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp \ > - python/native/src/mesos/native/mesos_scheduler_driver_impl.hpp \ > - python/native/src/mesos/native/module.cpp \ > - python/native/src/mesos/native/module.hpp \ > - python/native/src/mesos/native/proxy_executor.cpp \ > - python/native/src/mesos/native/proxy_executor.hpp \ > - python/native/src/mesos/native/proxy_scheduler.cpp \ > - python/native/src/mesos/native/proxy_scheduler.hpp \ > + python/executor/src/mesos/__init__.py > \ > + python/executor/src/mesos/executor/__init__.py \ > + python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp \ > + python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp \ > + python/executor/src/mesos/executor/module.cpp > \ > + python/executor/src/mesos/executor/proxy_executor.cpp > \ > + python/executor/src/mesos/executor/proxy_executor.hpp > \ > + python/scheduler/src/mesos/__init__.py \ > + python/scheduler/src/mesos/scheduler/__init__.py \ > + python/scheduler/src/mesos/scheduler/mesos_scheduler_driver_impl.cpp \ > + python/scheduler/src/mesos/scheduler/mesos_scheduler_driver_impl.hpp \ > + python/scheduler/src/mesos/scheduler/module.cpp \ > + python/scheduler/src/mesos/scheduler/proxy_scheduler.cpp \ > + python/scheduler/src/mesos/scheduler/proxy_scheduler.hpp \ > python/src/mesos/__init__.py > > EXTRA_DIST += $(PYTHON_SOURCE) > @@ -1530,14 +1535,18 @@ $(PYTHON_SOURCE): > MESOS_EGGS = > \ > python/dist/mesos-$(PACKAGE_VERSION)$(PYTHON_EGG_PUREPY_POSTFIX).egg > \ > python/dist/mesos.cli-$(PACKAGE_VERSION)$(PYTHON_EGG_PUREPY_POSTFIX).egg > \ > + python/dist/mesos.executor-$(PACKAGE_VERSION)$(PYTHON_EGG_POSTFIX).egg > \ > > python/dist/mesos.interface-$(PACKAGE_VERSION)$(PYTHON_EGG_PUREPY_POSTFIX).egg > \ > - python/dist/mesos.native-$(PACKAGE_VERSION)$(PYTHON_EGG_POSTFIX).egg > + python/dist/mesos.native-$(PACKAGE_VERSION)$(PYTHON_EGG_POSTFIX).egg > \ > + python/dist/mesos.scheduler-$(PACKAGE_VERSION)$(PYTHON_EGG_POSTFIX).egg > > MESOS_WHLS = > \ > python/dist/mesos-$(PACKAGE_VERSION)$(PYTHON_WHL_PUREPY_POSTFIX).whl > \ > python/dist/mesos.cli-$(PACKAGE_VERSION)$(PYTHON_WHL_PUREPY_POSTFIX).whl > \ > + python/dist/mesos.executor-$(PACKAGE_VERSION)$(PYTHON_WHL_POSTFIX).whl > \ > > python/dist/mesos.interface-$(PACKAGE_VERSION)$(PYTHON_WHL_PUREPY_POSTFIX).whl > \ > - python/dist/mesos.native-$(PACKAGE_VERSION)$(PYTHON_WHL_POSTFIX).whl > + python/dist/mesos.native-$(PACKAGE_VERSION)$(PYTHON_WHL_POSTFIX).whl > \ > + python/dist/mesos.scheduler-$(PACKAGE_VERSION)$(PYTHON_WHL_POSTFIX).whl > > # The python source is in directories of the form: python/interface. The make > # target is of the form: python/dist/mesos.interface-0.20.0-py2.7.egg. To > build > > http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/setup.py.in > ---------------------------------------------------------------------- > diff --git a/src/python/executor/setup.py.in b/src/python/executor/setup.py.in > new file mode 100644 > index 0000000..76db97f > --- /dev/null > +++ b/src/python/executor/setup.py.in > @@ -0,0 +1,39 @@ > +#!/usr/bin/env python > + > +# Licensed to the Apache Software Foundation (ASF) under one > +# or more contributor license agreements. See the NOTICE file > +# distributed with this work for additional information > +# regarding copyright ownership. The ASF licenses this file > +# to you under the Apache License, Version 2.0 (the > +# "License"); you may not use this file except in compliance > +# with the License. You may obtain a copy of the License at > +# > +# http://www.apache.org/licenses/LICENSE-2.0 > +# > +# Unless required by applicable law or agreed to in writing, software > +# distributed under the License is distributed on an "AS IS" BASIS, > +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > +# See the License for the specific language governing permissions and > +# limitations under the License. > + > +import ext_modules > + > +config = { > + 'name': 'mesos.executor', > + 'version': '@PACKAGE_VERSION@', > + 'description': 'Mesos native executor driver implementation', > + 'author': 'Apache Mesos', > + 'author_email': '[email protected]', > + 'url': 'http://pypi.python.org/pypi/mesos.executor', > + 'namespace_packages': [ 'mesos' ], > + 'packages': [ 'mesos', 'mesos.executor' ], > + 'package_dir': { '': 'src' }, > + 'install_requires': [ 'mesos.interface == @PACKAGE_VERSION@' ], > + 'license': 'Apache 2.0', > + 'keywords': 'mesos', > + 'classifiers': [ ], > + 'ext_modules': [ ext_modules.executor_module ] > +} > + > +from setuptools import setup > +setup(**config) > > http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/__init__.py > ---------------------------------------------------------------------- > diff --git a/src/python/executor/src/mesos/__init__.py > b/src/python/executor/src/mesos/__init__.py > new file mode 100644 > index 0000000..3fcba01 > --- /dev/null > +++ b/src/python/executor/src/mesos/__init__.py > @@ -0,0 +1,10 @@ > +# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages > +# Because python does not normally allow the contents of a package to be > +# retrieved from more than one location, this code snippet ensures that the > +# namespace package machinery is operating and that the current package is > +# registered as a namespace package. > +try: > + __import__('pkg_resources').declare_namespace(__name__) > +except ImportError: > + from pkgutil import extend_path > + __path__ = extend_path(__path__, __name__) > > http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/__init__.py > ---------------------------------------------------------------------- > diff --git a/src/python/executor/src/mesos/executor/__init__.py > b/src/python/executor/src/mesos/executor/__init__.py > new file mode 100644 > index 0000000..6808019 > --- /dev/null > +++ b/src/python/executor/src/mesos/executor/__init__.py > @@ -0,0 +1,17 @@ > +# Licensed to the Apache Software Foundation (ASF) under one > +# or more contributor license agreements. See the NOTICE file > +# distributed with this work for additional information > +# regarding copyright ownership. The ASF licenses this file > +# to you under the Apache License, Version 2.0 (the > +# "License"); you may not use this file except in compliance > +# with the License. You may obtain a copy of the License at > +# > +# http://www.apache.org/licenses/LICENSE-2.0 > +# > +# Unless required by applicable law or agreed to in writing, software > +# distributed under the License is distributed on an "AS IS" BASIS, > +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > +# See the License for the specific language governing permissions and > +# limitations under the License. > + > +from ._executor import MesosExecutorDriverImpl as MesosExecutorDriver > > http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp > ---------------------------------------------------------------------- > diff --git > a/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp > b/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp > new file mode 100644 > index 0000000..4189411 > --- /dev/null > +++ b/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp > @@ -0,0 +1,347 @@ > +// Licensed to the Apache Software Foundation (ASF) under one > +// or more contributor license agreements. See the NOTICE file > +// distributed with this work for additional information > +// regarding copyright ownership. The ASF licenses this file > +// to you under the Apache License, Version 2.0 (the > +// "License"); you may not use this file except in compliance > +// with the License. You may obtain a copy of the License at > +// > +// http://www.apache.org/licenses/LICENSE-2.0 > +// > +// Unless required by applicable law or agreed to in writing, software > +// distributed under the License is distributed on an "AS IS" BASIS, > +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > +// See the License for the specific language governing permissions and > +// limitations under the License. > + > +// Python.h must be included before standard headers. > +// See: http://docs.python.org/2/c-api/intro.html#include-files > +#include <Python.h> > + > +#include <string> > + > +#include "common.hpp" > +#include "mesos_executor_driver_impl.hpp" > +#include "proxy_executor.hpp" > + > +using namespace mesos; > +using namespace mesos::python; > + > +using std::cerr; > +using std::endl; > +using std::string; > +using std::vector; > +using std::map; > + > + > +namespace mesos { namespace python { > + > +/** > + * Python type object for MesosExecutorDriverImpl. > + */ > +PyTypeObject MesosExecutorDriverImplType = { > + PyObject_HEAD_INIT(NULL) > + 0, /* ob_size */ > + "_mesos.MesosExecutorDriverImpl", /* tp_name */ > + sizeof(MesosExecutorDriverImpl), /* tp_basicsize */ > + 0, /* tp_itemsize */ > + (destructor) MesosExecutorDriverImpl_dealloc, /* tp_dealloc */ > + 0, /* tp_print */ > + 0, /* tp_getattr */ > + 0, /* tp_setattr */ > + 0, /* tp_compare */ > + 0, /* tp_repr */ > + 0, /* tp_as_number */ > + 0, /* tp_as_sequence */ > + 0, /* tp_as_mapping */ > + 0, /* tp_hash */ > + 0, /* tp_call */ > + 0, /* tp_str */ > + 0, /* tp_getattro */ > + 0, /* tp_setattro */ > + 0, /* tp_as_buffer */ > + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ > + "Private MesosExecutorDriver implementation", /* tp_doc */ > + (traverseproc) MesosExecutorDriverImpl_traverse, /* tp_traverse */ > + (inquiry) MesosExecutorDriverImpl_clear, /* tp_clear */ > + 0, /* tp_richcompare */ > + 0, /* tp_weaklistoffset */ > + 0, /* tp_iter */ > + 0, /* tp_iternext */ > + MesosExecutorDriverImpl_methods, /* tp_methods */ > + 0, /* tp_members */ > + 0, /* tp_getset */ > + 0, /* tp_base */ > + 0, /* tp_dict */ > + 0, /* tp_descr_get */ > + 0, /* tp_descr_set */ > + 0, /* tp_dictoffset */ > + (initproc) MesosExecutorDriverImpl_init, /* tp_init */ > + 0, /* tp_alloc */ > + MesosExecutorDriverImpl_new, /* tp_new */ > +}; > + > + > +/** > + * List of Python methods in MesosExecutorDriverImpl. > + */ > +PyMethodDef MesosExecutorDriverImpl_methods[] = { > + { "start", > + (PyCFunction) MesosExecutorDriverImpl_start, > + METH_NOARGS, > + "Start the driver to connect to Mesos" > + }, > + { "stop", > + (PyCFunction) MesosExecutorDriverImpl_stop, > + METH_NOARGS, > + "Stop the driver, disconnecting from Mesos" > + }, > + { "abort", > + (PyCFunction) MesosExecutorDriverImpl_abort, > + METH_NOARGS, > + "Abort the driver, disallowing calls from and to the driver" > + }, > + { "join", > + (PyCFunction) MesosExecutorDriverImpl_join, > + METH_NOARGS, > + "Wait for a running driver to disconnect from Mesos" > + }, > + { "run", > + (PyCFunction) MesosExecutorDriverImpl_run, > + METH_NOARGS, > + "Start a driver and run it, returning when it disconnects from Mesos" > + }, > + { "sendStatusUpdate", > + (PyCFunction) MesosExecutorDriverImpl_sendStatusUpdate, > + METH_VARARGS, > + "Send a status update for a task" > + }, > + { "sendFrameworkMessage", > + (PyCFunction) MesosExecutorDriverImpl_sendFrameworkMessage, > + METH_VARARGS, > + "Send a FrameworkMessage to a slave" > + }, > + { NULL } /* Sentinel */ > +}; > + > + > +/** > + * Create, but don't initialize, a new MesosExecutorDriverImpl > + * (called by Python before init method). > + */ > +PyObject* MesosExecutorDriverImpl_new(PyTypeObject *type, > + PyObject *args, > + PyObject *kwds) > +{ > + MesosExecutorDriverImpl *self; > + self = (MesosExecutorDriverImpl *) type->tp_alloc(type, 0); > + if (self != NULL) { > + self->driver = NULL; > + self->proxyExecutor = NULL; > + self->pythonExecutor = NULL; > + } > + return (PyObject*) self; > +} > + > + > +/** > + * Initialize a MesosExecutorDriverImpl with constructor arguments. > + */ > +int MesosExecutorDriverImpl_init(MesosExecutorDriverImpl *self, > + PyObject *args, > + PyObject *kwds) > +{ > + PyObject *pythonExecutor = NULL; > + > + if (!PyArg_ParseTuple(args, "O", &pythonExecutor)) { > + return -1; > + } > + > + if (pythonExecutor != NULL) { > + PyObject* tmp = self->pythonExecutor; > + Py_INCREF(pythonExecutor); > + self->pythonExecutor = pythonExecutor; > + Py_XDECREF(tmp); > + } > + > + if (self->driver != NULL) { > + delete self->driver; > + self->driver = NULL; > + } > + > + if (self->proxyExecutor != NULL) { > + delete self->proxyExecutor; > + self->proxyExecutor = NULL; > + } > + > + self->proxyExecutor = new ProxyExecutor(self); > + self->driver = new MesosExecutorDriver(self->proxyExecutor); > + > + return 0; > +} > + > + > +/** > + * Free a MesosExecutorDriverImpl. > + */ > +void MesosExecutorDriverImpl_dealloc(MesosExecutorDriverImpl* self) > +{ > + if (self->driver != NULL) { > + // We need to wrap the driver destructor in an "allow threads" > + // macro since the MesosExecutorDriver destructor waits for the > + // ExecutorProcess to terminate and there might be a thread that > + // is trying to acquire the GIL to call through the > + // ProxyExecutor. It will only be after this thread executes that > + // the ExecutorProcess might actually get a terminate. > + Py_BEGIN_ALLOW_THREADS > + delete self->driver; > + Py_END_ALLOW_THREADS > + self->driver = NULL; > + } > + > + if (self->proxyExecutor != NULL) { > + delete self->proxyExecutor; > + self->proxyExecutor = NULL; > + } > + > + MesosExecutorDriverImpl_clear(self); > + self->ob_type->tp_free((PyObject*) self); > +} > + > + > +/** > + * Traverse fields of a MesosExecutorDriverImpl on a cyclic GC search. > + * See http://docs.python.org/extending/newtypes.html. > + */ > +int MesosExecutorDriverImpl_traverse(MesosExecutorDriverImpl* self, > + visitproc visit, > + void* arg) > +{ > + Py_VISIT(self->pythonExecutor); > + return 0; > +} > + > + > +/** > + * Clear fields of a MesosExecutorDriverImpl that can participate in > + * GC cycles. See http://docs.python.org/extending/newtypes.html. > + */ > +int MesosExecutorDriverImpl_clear(MesosExecutorDriverImpl* self) > +{ > + Py_CLEAR(self->pythonExecutor); > + return 0; > +} > + > + > +PyObject* MesosExecutorDriverImpl_start(MesosExecutorDriverImpl* self) > +{ > + if (self->driver == NULL) { > + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL"); > + return NULL; > + } > + > + Status status = self->driver->start(); > + return PyInt_FromLong(status); // Sets an exception if creating the int > fails. > +} > + > + > +PyObject* MesosExecutorDriverImpl_stop(MesosExecutorDriverImpl* self) > +{ > + if (self->driver == NULL) { > + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL"); > + return NULL; > + } > + > + Status status = self->driver->stop(); > + return PyInt_FromLong(status); // Sets an exception if creating the int > fails. > +} > + > + > +PyObject* MesosExecutorDriverImpl_abort(MesosExecutorDriverImpl* self) > +{ > + if (self->driver == NULL) { > + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL"); > + return NULL; > + } > + > + Status status = self->driver->abort(); > + return PyInt_FromLong(status); // Sets an exception if creating the int > fails. > +} > + > + > +PyObject* MesosExecutorDriverImpl_join(MesosExecutorDriverImpl* self) > +{ > + if (self->driver == NULL) { > + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL"); > + return NULL; > + } > + > + Status status; > + Py_BEGIN_ALLOW_THREADS > + status = self->driver->join(); > + Py_END_ALLOW_THREADS > + return PyInt_FromLong(status); // Sets an exception if creating the int > fails. > +} > + > + > +PyObject* MesosExecutorDriverImpl_run(MesosExecutorDriverImpl* self) > +{ > + if (self->driver == NULL) { > + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL"); > + return NULL; > + } > + > + Status status; > + Py_BEGIN_ALLOW_THREADS > + status = self->driver->run(); > + Py_END_ALLOW_THREADS > + return PyInt_FromLong(status); // Sets an exception if creating the int > fails. > +} > + > + > +PyObject* MesosExecutorDriverImpl_sendStatusUpdate( > + MesosExecutorDriverImpl* self, > + PyObject* args) > +{ > + if (self->driver == NULL) { > + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL"); > + return NULL; > + } > + > + PyObject* statusObj = NULL; > + TaskStatus taskStatus; > + if (!PyArg_ParseTuple(args, "O", &statusObj)) { > + return NULL; > + } > + if (!readPythonProtobuf(statusObj, &taskStatus)) { > + PyErr_Format(PyExc_Exception, > + "Could not deserialize Python TaskStatus"); > + return NULL; > + } > + > + Status status = self->driver->sendStatusUpdate(taskStatus); > + return PyInt_FromLong(status); // Sets an exception if creating the int > fails. > +} > + > + > +PyObject* MesosExecutorDriverImpl_sendFrameworkMessage( > + MesosExecutorDriverImpl* self, > + PyObject* args) > +{ > + if (self->driver == NULL) { > + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL"); > + return NULL; > + } > + > + const char* data; > + int length; > + if (!PyArg_ParseTuple(args, "s#", &data, &length)) { > + return NULL; > + } > + > + Status status = self->driver->sendFrameworkMessage(string(data, length)); > + return PyInt_FromLong(status); // Sets an exception if creating the int > fails. > +} > + > +} // namespace python { > +} // namespace mesos { > > http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp > ---------------------------------------------------------------------- > diff --git > a/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp > b/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp > new file mode 100644 > index 0000000..6e672f8 > --- /dev/null > +++ b/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp > @@ -0,0 +1,103 @@ > +// Licensed to the Apache Software Foundation (ASF) under one > +// or more contributor license agreements. See the NOTICE file > +// distributed with this work for additional information > +// regarding copyright ownership. The ASF licenses this file > +// to you under the Apache License, Version 2.0 (the > +// "License"); you may not use this file except in compliance > +// with the License. You may obtain a copy of the License at > +// > +// http://www.apache.org/licenses/LICENSE-2.0 > +// > +// Unless required by applicable law or agreed to in writing, software > +// distributed under the License is distributed on an "AS IS" BASIS, > +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > +// See the License for the specific language governing permissions and > +// limitations under the License. > + > +#ifndef MESOS_EXECUTOR_DRIVER_IMPL_HPP > +#define MESOS_EXECUTOR_DRIVER_IMPL_HPP > + > +#include <mesos/executor.hpp> > + > + > +namespace mesos { namespace python { > + > +class ProxyExecutor; > + > +/** > + * Python object structure for MesosExecutorDriverImpl objects. > + */ > +struct MesosExecutorDriverImpl { > + PyObject_HEAD > + /* Type-specific fields go here. */ > + MesosExecutorDriver* driver; > + ProxyExecutor* proxyExecutor; > + PyObject* pythonExecutor; > +}; > + > +/** > + * Python type object for MesosExecutorDriverImpl. > + */ > +extern PyTypeObject MesosExecutorDriverImplType; > + > +/** > + * List of Python methods in MesosExecutorDriverImpl. > + */ > +extern PyMethodDef MesosExecutorDriverImpl_methods[]; > + > +/** > + * Create, but don't initialize, a new MesosExecutorDriverImpl > + * (called by Python before init method). > + */ > +PyObject* MesosExecutorDriverImpl_new(PyTypeObject *type, > + PyObject *args, > + PyObject *kwds); > + > +/** > + * Initialize a MesosExecutorDriverImpl with constructor arguments. > + */ > +int MesosExecutorDriverImpl_init(MesosExecutorDriverImpl *self, > + PyObject *args, > + PyObject *kwds); > + > +/** > + * Free a MesosExecutorDriverImpl. > + */ > +void MesosExecutorDriverImpl_dealloc(MesosExecutorDriverImpl* self); > + > +/** > + * Traverse fields of a MesosExecutorDriverImpl on a cyclic GC search. > + * See http://docs.python.org/extending/newtypes.html. > + */ > +int MesosExecutorDriverImpl_traverse(MesosExecutorDriverImpl* self, > + visitproc visit, > + void* arg); > +/** > + * Clear fields of a MesosExecutorDriverImpl that can participate in > + * GC cycles. See http://docs.python.org/extending/newtypes.html. > + */ > +int MesosExecutorDriverImpl_clear(MesosExecutorDriverImpl* self); > + > +// MesosExecutorDriverImpl methods. > +PyObject* MesosExecutorDriverImpl_start(MesosExecutorDriverImpl* self); > + > +PyObject* MesosExecutorDriverImpl_stop(MesosExecutorDriverImpl* self); > + > +PyObject* MesosExecutorDriverImpl_abort(MesosExecutorDriverImpl* self); > + > +PyObject* MesosExecutorDriverImpl_join(MesosExecutorDriverImpl* self); > + > +PyObject* MesosExecutorDriverImpl_run(MesosExecutorDriverImpl* self); > + > +PyObject* MesosExecutorDriverImpl_sendStatusUpdate( > + MesosExecutorDriverImpl* self, > + PyObject* args); > + > +PyObject* MesosExecutorDriverImpl_sendFrameworkMessage( > + MesosExecutorDriverImpl* self, > + PyObject* args); > + > +} // namespace python { > +} // namespace mesos { > + > +#endif /* MESOS_EXECUTOR_DRIVER_IMPL_HPP */ > > http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/module.cpp > ---------------------------------------------------------------------- > diff --git a/src/python/executor/src/mesos/executor/module.cpp > b/src/python/executor/src/mesos/executor/module.cpp > new file mode 100644 > index 0000000..f8c6382 > --- /dev/null > +++ b/src/python/executor/src/mesos/executor/module.cpp > @@ -0,0 +1,91 @@ > +// Licensed to the Apache Software Foundation (ASF) under one > +// or more contributor license agreements. See the NOTICE file > +// distributed with this work for additional information > +// regarding copyright ownership. The ASF licenses this file > +// to you under the Apache License, Version 2.0 (the > +// "License"); you may not use this file except in compliance > +// with the License. You may obtain a copy of the License at > +// > +// http://www.apache.org/licenses/LICENSE-2.0 > +// > +// Unless required by applicable law or agreed to in writing, software > +// distributed under the License is distributed on an "AS IS" BASIS, > +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > +// See the License for the specific language governing permissions and > +// limitations under the License. > + > +/** > + * This file defines the _mesos.so binary module used by the Mesos Python > API. > + * This module contains private implementations of MesosSchedulerDriver and > + * MesosExecutorDriver as Python types that get called from the public module > + * called mesos (in <root>/src/python/src/mesos.py). This design was chosen > + * so that most of the API (e.g. the Scheduler and Executor interfaces) can > + * be written in Python, and only the parts that need to call into C++ are > + * in C++. Note that the mesos module also contains public classes called > + * MesosSchedulerDriver and MesosExecutorDriver. These call into the private > + * _mesos.MesosSchedulerDriverImpl and _mesos.MesosExecutorDriverImpl. > + */ > + > +// Python.h must be included before standard headers. > +// See: http://docs.python.org/2/c-api/intro.html#include-files > +#include <Python.h> > + > +#include <iostream> > + > +#include <mesos/executor.hpp> > + > +#include "common.hpp" > +#include "mesos_executor_driver_impl.hpp" > +#include "proxy_executor.hpp" > + > +using namespace mesos; > +using namespace mesos::python; > + > +using std::map; > +using std::string; > +using std::vector; > + > + > +/** > + * The Python module object for mesos_pb2 (which contains the protobuf > + * classes generated for Python). > + */ > +PyObject* mesos::python::mesos_pb2 = NULL; > + > + > +namespace { > + > +/** > + * Method list for our Python module. > + */ > +PyMethodDef MODULE_METHODS[] = { > + {NULL, NULL, 0, NULL} /* Sentinel */ > +}; > + > +} // namespace { > + > + > +/** > + * Entry point called by Python to initialize our module. > + */ > +PyMODINIT_FUNC init_executor() > +{ > + // Ensure that the interpreter's threading support is enabled. > + PyEval_InitThreads(); > + > + // Import the mesos_pb2 module (on which we depend for protobuf classes) > + mesos_pb2 = PyImport_ImportModule("mesos.interface.mesos_pb2"); > + if (mesos_pb2 == NULL) > + return; > + > + // Initialize our Python types. > + if (PyType_Ready(&MesosExecutorDriverImplType) < 0) > + return; > + > + // Create the _mesos module and add our types to it. > + PyObject* module = Py_InitModule("_executor", MODULE_METHODS); > + Py_INCREF(&MesosExecutorDriverImplType); > + PyModule_AddObject(module, > + "MesosExecutorDriverImpl", > + (PyObject*) &MesosExecutorDriverImplType); > +} > > http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/proxy_executor.cpp > ---------------------------------------------------------------------- > diff --git a/src/python/executor/src/mesos/executor/proxy_executor.cpp > b/src/python/executor/src/mesos/executor/proxy_executor.cpp > new file mode 100644 > index 0000000..da1a49b > --- /dev/null > +++ b/src/python/executor/src/mesos/executor/proxy_executor.cpp > @@ -0,0 +1,273 @@ > +// Licensed to the Apache Software Foundation (ASF) under one > +// or more contributor license agreements. See the NOTICE file > +// distributed with this work for additional information > +// regarding copyright ownership. The ASF licenses this file > +// to you under the Apache License, Version 2.0 (the > +// "License"); you may not use this file except in compliance > +// with the License. You may obtain a copy of the License at > +// > +// http://www.apache.org/licenses/LICENSE-2.0 > +// > +// Unless required by applicable law or agreed to in writing, software > +// distributed under the License is distributed on an "AS IS" BASIS, > +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > +// See the License for the specific language governing permissions and > +// limitations under the License. > + > +// Python.h must be included before standard headers. > +// See: http://docs.python.org/2/c-api/intro.html#include-files > +#include <Python.h> > + > +#include <iostream> > + > +#include "common.hpp" > +#include "mesos_executor_driver_impl.hpp" > +#include "proxy_executor.hpp" > + > +using namespace mesos; > + > +using std::cerr; > +using std::endl; > +using std::map; > +using std::string; > +using std::vector; > + > +namespace mesos { > +namespace python { > + > +void ProxyExecutor::registered(ExecutorDriver* driver, > + const ExecutorInfo& executorInfo, > + const FrameworkInfo& frameworkInfo, > + const SlaveInfo& slaveInfo) > +{ > + InterpreterLock lock; > + > + PyObject* executorInfoObj = NULL; > + PyObject* frameworkInfoObj = NULL; > + PyObject* slaveInfoObj = NULL; > + PyObject* res = NULL; > + > + executorInfoObj = createPythonProtobuf(executorInfo, "ExecutorInfo"); > + frameworkInfoObj = createPythonProtobuf(frameworkInfo, "FrameworkInfo"); > + slaveInfoObj = createPythonProtobuf(slaveInfo, "SlaveInfo"); > + > + if (executorInfoObj == NULL || > + frameworkInfoObj == NULL || > + slaveInfoObj == NULL) { > + goto cleanup; // createPythonProtobuf will have set an exception. > + } > + > + res = PyObject_CallMethod(impl->pythonExecutor, > + (char*) "registered", > + (char*) "OOOO", > + impl, > + executorInfoObj, > + frameworkInfoObj, > + slaveInfoObj); > + if (res == NULL) { > + cerr << "Failed to call executor registered" << endl; > + goto cleanup; > + } > + > +cleanup: > + if (PyErr_Occurred()) { > + PyErr_Print(); > + driver->abort(); > + } > + Py_XDECREF(executorInfoObj); > + Py_XDECREF(frameworkInfoObj); > + Py_XDECREF(slaveInfoObj); > + Py_XDECREF(res); > +} > + > + > +void ProxyExecutor::reregistered(ExecutorDriver* driver, > + const SlaveInfo& slaveInfo) > +{ > + InterpreterLock lock; > + > + PyObject* slaveInfoObj = NULL; > + PyObject* res = NULL; > + > + slaveInfoObj = createPythonProtobuf(slaveInfo, "SlaveInfo"); > + > + if (slaveInfoObj == NULL) { > + goto cleanup; // createPythonProtobuf will have set an exception. > + } > + > + res = PyObject_CallMethod(impl->pythonExecutor, > + (char*) "reregistered", > + (char*) "OO", > + impl, > + slaveInfoObj); > + if (res == NULL) { > + cerr << "Failed to call executor re-registered" << endl; > + goto cleanup; > + } > + > +cleanup: > + if (PyErr_Occurred()) { > + PyErr_Print(); > + driver->abort(); > + } > + Py_XDECREF(slaveInfoObj); > + Py_XDECREF(res); > +} > + > + > +void ProxyExecutor::disconnected(ExecutorDriver* driver) > +{ > + InterpreterLock lock; > + PyObject* res = PyObject_CallMethod(impl->pythonExecutor, > + (char*) "disconnected", > + (char*) "O", > + impl); > + if (res == NULL) { > + cerr << "Failed to call executor's disconnected" << endl; > + goto cleanup; > + } > +cleanup: > + if (PyErr_Occurred()) { > + PyErr_Print(); > + driver->abort(); > + } > + Py_XDECREF(res); > +} > + > + > +void ProxyExecutor::launchTask(ExecutorDriver* driver, > + const TaskInfo& task) > +{ > + InterpreterLock lock; > + > + PyObject* taskObj = NULL; > + PyObject* res = NULL; > + > + taskObj = createPythonProtobuf(task, "TaskInfo"); > + if (taskObj == NULL) { > + goto cleanup; // createPythonProtobuf will have set an exception. > + } > + > + res = PyObject_CallMethod(impl->pythonExecutor, > + (char*) "launchTask", > + (char*) "OO", > + impl, > + taskObj); > + if (res == NULL) { > + cerr << "Failed to call executor's launchTask" << endl; > + goto cleanup; > + } > + > +cleanup: > + if (PyErr_Occurred()) { > + PyErr_Print(); > + driver->abort(); > + } > + Py_XDECREF(taskObj); > + Py_XDECREF(res); > +} > + > + > +void ProxyExecutor::killTask(ExecutorDriver* driver, > + const TaskID& taskId) > +{ > + InterpreterLock lock; > + > + PyObject* taskIdObj = NULL; > + PyObject* res = NULL; > + > + taskIdObj = createPythonProtobuf(taskId, "TaskID"); > + if (taskIdObj == NULL) { > + goto cleanup; // createPythonProtobuf will have set an exception. > + } > + > + res = PyObject_CallMethod(impl->pythonExecutor, > + (char*) "killTask", > + (char*) "OO", > + impl, > + taskIdObj); > + if (res == NULL) { > + cerr << "Failed to call executor's killTask" << endl; > + goto cleanup; > + } > + > +cleanup: > + if (PyErr_Occurred()) { > + PyErr_Print(); > + driver->abort(); > + } > + Py_XDECREF(taskIdObj); > + Py_XDECREF(res); > +} > + > + > +void ProxyExecutor::frameworkMessage(ExecutorDriver* driver, > + const string& data) > +{ > + InterpreterLock lock; > + > + PyObject* res = NULL; > + > + res = PyObject_CallMethod(impl->pythonExecutor, > + (char*) "frameworkMessage", > + (char*) "Os#", > + impl, > + data.data(), > + data.length()); > + if (res == NULL) { > + cerr << "Failed to call executor's frameworkMessage" << endl; > + goto cleanup; > + } > + > +cleanup: > + if (PyErr_Occurred()) { > + PyErr_Print(); > + driver->abort(); > + } > + Py_XDECREF(res); > +} > + > + > +void ProxyExecutor::shutdown(ExecutorDriver* driver) > +{ > + InterpreterLock lock; > + PyObject* res = PyObject_CallMethod(impl->pythonExecutor, > + (char*) "shutdown", > + (char*) "O", > + impl); > + if (res == NULL) { > + cerr << "Failed to call executor's shutdown" << endl; > + goto cleanup; > + } > +cleanup: > + if (PyErr_Occurred()) { > + PyErr_Print(); > + driver->abort(); > + } > + Py_XDECREF(res); > +} > + > + > +void ProxyExecutor::error(ExecutorDriver* driver, const string& message) > +{ > + InterpreterLock lock; > + PyObject* res = PyObject_CallMethod(impl->pythonExecutor, > + (char*) "error", > + (char*) "Os#", > + impl, > + message.data(), > + message.length()); > + if (res == NULL) { > + cerr << "Failed to call executor's error" << endl; > + goto cleanup; > + } > +cleanup: > + if (PyErr_Occurred()) { > + PyErr_Print(); > + // No need for driver.stop(); it should stop itself. > + } > + Py_XDECREF(res); > +} > + > +} // namespace python { > +} // namespace mesos { > > http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/proxy_executor.hpp > ---------------------------------------------------------------------- > diff --git a/src/python/executor/src/mesos/executor/proxy_executor.hpp > b/src/python/executor/src/mesos/executor/proxy_executor.hpp > new file mode 100644 > index 0000000..23d64fd > --- /dev/null > +++ b/src/python/executor/src/mesos/executor/proxy_executor.hpp > @@ -0,0 +1,64 @@ > +// Licensed to the Apache Software Foundation (ASF) under one > +// or more contributor license agreements. See the NOTICE file > +// distributed with this work for additional information > +// regarding copyright ownership. The ASF licenses this file > +// to you under the Apache License, Version 2.0 (the > +// "License"); you may not use this file except in compliance > +// with the License. You may obtain a copy of the License at > +// > +// http://www.apache.org/licenses/LICENSE-2.0 > +// > +// Unless required by applicable law or agreed to in writing, software > +// distributed under the License is distributed on an "AS IS" BASIS, > +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > +// See the License for the specific language governing permissions and > +// limitations under the License. > + > +#ifndef PROXY_EXECUTOR_HPP > +#define PROXY_EXECUTOR_HPP > + > +// Python.h must be included before standard headers. > +// See: http://docs.python.org/2/c-api/intro.html#include-files > +#include <Python.h> > + > +#include <string> > +#include <vector> > + > +#include <mesos/executor.hpp> > + > +namespace mesos { > +namespace python { > + > +struct MesosExecutorDriverImpl; > + > +/** > + * Proxy Executor implementation that will call into Python. > + */ > +class ProxyExecutor : public Executor > +{ > +public: > + explicit ProxyExecutor(MesosExecutorDriverImpl *_impl) : impl(_impl) {} > + > + virtual ~ProxyExecutor() {} > + > + virtual void registered(ExecutorDriver* driver, > + const ExecutorInfo& executorInfo, > + const FrameworkInfo& frameworkInfo, > + const SlaveInfo& slaveInfo); > + virtual void reregistered(ExecutorDriver* driver, const SlaveInfo& > slaveInfo); > + virtual void disconnected(ExecutorDriver* driver); > + virtual void launchTask(ExecutorDriver* driver, const TaskInfo& task); > + virtual void killTask(ExecutorDriver* driver, const TaskID& taskId); > + virtual void frameworkMessage(ExecutorDriver* driver, > + const std::string& data); > + virtual void shutdown(ExecutorDriver* driver); > + virtual void error(ExecutorDriver* driver, const std::string& message); > + > +private: > + MesosExecutorDriverImpl *impl; > +}; > + > +} // namespace python { > +} // namespace mesos { > + > +#endif // PROXY_EXECUTOR_HPP > > http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/ext_modules.py.in > ---------------------------------------------------------------------- > diff --git a/src/python/native/ext_modules.py.in > b/src/python/native/ext_modules.py.in > deleted file mode 100644 > index eb93864..0000000 > --- a/src/python/native/ext_modules.py.in > +++ /dev/null > @@ -1,151 +0,0 @@ > -# Licensed to the Apache Software Foundation (ASF) under one > -# or more contributor license agreements. See the NOTICE file > -# distributed with this work for additional information > -# regarding copyright ownership. The ASF licenses this file > -# to you under the Apache License, Version 2.0 (the > -# "License"); you may not use this file except in compliance > -# with the License. You may obtain a copy of the License at > -# > -# http://www.apache.org/licenses/LICENSE-2.0 > -# > -# Unless required by applicable law or agreed to in writing, software > -# distributed under the License is distributed on an "AS IS" BASIS, > -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > -# See the License for the specific language governing permissions and > -# limitations under the License. > - > -import errno > -import glob > -import os > -import shutil > - > -from setuptools import Extension > - > -abs_top_srcdir = '@abs_top_srcdir@' > -abs_top_builddir = '@abs_top_builddir@' > - > -src_python_native = os.path.join( > - 'src', 'python', 'native', 'src', 'mesos', 'native') > - > -leveldb = os.path.join('3rdparty', 'leveldb-1.4') > -zookeeper = os.path.join('3rdparty', 'zookeeper-3.4.5', 'src', 'c') > -libprocess = os.path.join('3rdparty', 'libprocess') > - > -# Even though a statically compiled libprocess should include glog, > -# libev, gperftools, and protobuf before installation this isn't the > -# case, so while a libtool managed build will correctly pull in these > -# libraries when building the final result, we need to explicitly > -# include them here (or more precisely, down where we actually include > -# libev.a and libprofiler.a). > -glog = os.path.join(libprocess, '3rdparty', 'glog-0.3.3') > -gperftools = os.path.join(libprocess, '3rdparty', 'gperftools-2.0') > -protobuf = os.path.join(libprocess, '3rdparty', 'protobuf-2.5.0') > - > -# Build the list of source files. Note that each source must be > -# relative to our current directory (where this script lives). > -SOURCES = [ > - os.path.join('src', 'mesos', 'native', file) > - for file in os.listdir(os.path.join(abs_top_srcdir, > src_python_native)) > - if file.endswith('.cpp') > -] > - > -INCLUDE_DIRS = [ > - os.path.join(abs_top_srcdir, 'include'), > - os.path.join(abs_top_builddir, 'include'), > - # Needed for the *.pb.h protobuf includes. > - os.path.join(abs_top_builddir, 'include', 'mesos'), > - os.path.join(abs_top_builddir, 'src'), > - os.path.join(abs_top_builddir, src_python_native), > - os.path.join(abs_top_builddir, protobuf, 'src'), > -] > - > -LIBRARY_DIRS = [] > - > -EXTRA_OBJECTS = [ > - os.path.join(abs_top_builddir, 'src', '.libs', 'libmesos_no_3rdparty.a'), > - os.path.join(abs_top_builddir, libprocess, '.libs', 'libprocess.a') > -] > - > -# For leveldb, we need to check for the presence of libleveldb.a, since > -# it is possible to disable leveldb inside mesos. > -libglog = os.path.join(abs_top_builddir, glog, '.libs', 'libglog.a') > -libleveldb = os.path.join(abs_top_builddir, leveldb, 'libleveldb.a') > -libzookeeper = os.path.join( > - abs_top_builddir, zookeeper, '.libs', 'libzookeeper_mt.a') > -libprotobuf = os.path.join( > - abs_top_builddir, protobuf, 'src', '.libs', 'libprotobuf.a') > - > -if os.path.exists(libleveldb): > - EXTRA_OBJECTS.append(libleveldb) > -else: > - EXTRA_OBJECTS.append('-lleveldb') > - > -if os.path.exists(libzookeeper): > - EXTRA_OBJECTS.append(libzookeeper) > -else: > - EXTRA_OBJECTS.append('-lzookeeper_mt') > - > -if os.path.exists(libglog): > - EXTRA_OBJECTS.append(libglog) > -else: > - EXTRA_OBJECTS.append('-lglog') > - > -if os.path.exists(libprotobuf): > - EXTRA_OBJECTS.append(libprotobuf) > -else: > - EXTRA_OBJECTS.append('-lprotobuf') > - > - > -# libev is a special case because it needs to be enabled only when > -# libevent *is not* enabled through the top level ./configure. > -# > -# TODO(hartem): this entire block MUST be removed once libev is deprecated > -# in favor of libevent. > -if '@ENABLE_LIBEVENT_TRUE@' == '#': > - libev = os.path.join(libprocess, '3rdparty', 'libev-4.15') > - libev = os.path.join(abs_top_builddir, libev, '.libs', 'libev.a') > - > - if os.path.exists(libev): > - EXTRA_OBJECTS.append(libev) > - else: > - EXTRA_OBJECTS.append('-lev') > - > - > -# For gperftools, we need to check for the presence of libprofiler.a, since > -# it is possible to disable perftools inside libprocess. > -libprofiler = os.path.join( > - abs_top_builddir, gperftools, '.libs', 'libprofiler.a') > - > -if os.path.exists(libprofiler): > - EXTRA_OBJECTS.append(libprofiler) > - > -EXTRA_LINK_ARGS = [] > - > -# Add any flags from LDFLAGS. > -if 'LDFLAGS' in os.environ: > - for flag in os.environ['LDFLAGS'].split(): > - EXTRA_LINK_ARGS.append(flag) > - > -# Add any libraries from LIBS. > -if 'LIBS' in os.environ: > - for library in os.environ['LIBS'].split(): > - EXTRA_LINK_ARGS.append(library) > - > -DEPENDS = [ > - os.path.join(abs_top_srcdir, 'src', 'python', source) > - for source in SOURCES > -] > - > -# Note that we add EXTRA_OBJECTS to our dependency list to make sure > -# that we rebuild this module when one of them changes (e.g., > -# libprocess). > -mesos_module = \ > - Extension('mesos.native._mesos', > - sources = SOURCES, > - include_dirs = INCLUDE_DIRS, > - library_dirs = LIBRARY_DIRS, > - extra_objects = EXTRA_OBJECTS, > - extra_link_args = EXTRA_LINK_ARGS, > - depends = EXTRA_OBJECTS, > - language = 'c++', > - ) > > http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/setup.py.in > ---------------------------------------------------------------------- > diff --git a/src/python/native/setup.py.in b/src/python/native/setup.py.in > index 49ed612..10a5002 100644 > --- a/src/python/native/setup.py.in > +++ b/src/python/native/setup.py.in > @@ -16,8 +16,6 @@ > # See the License for the specific language governing permissions and > # limitations under the License. > > -import ext_modules > - > config = { > 'name': 'mesos.native', > 'version': '@PACKAGE_VERSION@', > @@ -28,13 +26,12 @@ config = { > 'namespace_packages': [ 'mesos' ], > 'packages': [ 'mesos', 'mesos.native' ], > 'package_dir': { '': 'src' }, > - 'install_requires': [ 'mesos.interface == @PACKAGE_VERSION@' ], > + 'install_requires': [ 'mesos.executor == @PACKAGE_VERSION@', > + 'mesos.scheduler == @PACKAGE_VERSION@'], > 'license': 'Apache 2.0', > 'keywords': 'mesos', > - 'classifiers': [ ], > - 'ext_modules': [ ext_modules.mesos_module ] > + 'classifiers': [ ] > } > > from setuptools import setup > - > setup(**config) > > http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/src/mesos/native/__init__.py > ---------------------------------------------------------------------- > diff --git a/src/python/native/src/mesos/native/__init__.py > b/src/python/native/src/mesos/native/__init__.py > index 226f943..b581537 100644 > --- a/src/python/native/src/mesos/native/__init__.py > +++ b/src/python/native/src/mesos/native/__init__.py > @@ -14,8 +14,5 @@ > # See the License for the specific language governing permissions and > # limitations under the License. > > -from ._mesos import MesosExecutorDriverImpl > -from ._mesos import MesosSchedulerDriverImpl > - > -MesosExecutorDriver = MesosExecutorDriverImpl > -MesosSchedulerDriver = MesosSchedulerDriverImpl > +from mesos.executor import MesosExecutorDriver > +from mesos.scheduler import MesosSchedulerDriver > \ No newline at end of file > > http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/src/mesos/native/mesos_executor_driver_impl.cpp > ---------------------------------------------------------------------- > diff --git > a/src/python/native/src/mesos/native/mesos_executor_driver_impl.cpp > b/src/python/native/src/mesos/native/mesos_executor_driver_impl.cpp > deleted file mode 100644 > index 7838a07..0000000 > --- a/src/python/native/src/mesos/native/mesos_executor_driver_impl.cpp > +++ /dev/null > @@ -1,347 +0,0 @@ > -// Licensed to the Apache Software Foundation (ASF) under one > -// or more contributor license agreements. See the NOTICE file > -// distributed with this work for additional information > -// regarding copyright ownership. The ASF licenses this file > -// to you under the Apache License, Version 2.0 (the > -// "License"); you may not use this file except in compliance > -// with the License. You may obtain a copy of the License at > -// > -// http://www.apache.org/licenses/LICENSE-2.0 > -// > -// Unless required by applicable law or agreed to in writing, software > -// distributed under the License is distributed on an "AS IS" BASIS, > -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > -// See the License for the specific language governing permissions and > -// limitations under the License. > - > -// Python.h must be included before standard headers. > -// See: http://docs.python.org/2/c-api/intro.html#include-files > -#include <Python.h> > - > -#include <string> > - > -#include "mesos_executor_driver_impl.hpp" > -#include "module.hpp" > -#include "proxy_executor.hpp" > - > -using namespace mesos; > -using namespace mesos::python; > - > -using std::cerr; > -using std::endl; > -using std::string; > -using std::vector; > -using std::map; > - > - > -namespace mesos { namespace python { > - > -/** > - * Python type object for MesosExecutorDriverImpl. > - */ > -PyTypeObject MesosExecutorDriverImplType = { > - PyObject_HEAD_INIT(NULL) > - 0, /* ob_size */ > - "_mesos.MesosExecutorDriverImpl", /* tp_name */ > - sizeof(MesosExecutorDriverImpl), /* tp_basicsize */ > - 0, /* tp_itemsize */ > - (destructor) MesosExecutorDriverImpl_dealloc, /* tp_dealloc */ > - 0, /* tp_print */ > - 0, /* tp_getattr */ > - 0, /* tp_setattr */ > - 0, /* tp_compare */ > - 0, /* tp_repr */ > - 0, /* tp_as_number */ > - 0, /* tp_as_sequence */ > - 0, /* tp_as_mapping */ > - 0, /* tp_hash */ > - 0, /* tp_call */ > - 0, /* tp_str */ > - 0, /* tp_getattro */ > - 0, /* tp_setattro */ > - 0, /* tp_as_buffer */ > - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ > - "Private MesosExecutorDriver implementation", /* tp_doc */ > - (traverseproc) MesosExecutorDriverImpl_traverse, /* tp_traverse */ > - (inquiry) MesosExecutorDriverImpl_clear, /* tp_clear */ > - 0, /* tp_richcompare */ > - 0, /* tp_weaklistoffset */ > - 0, /* tp_iter */ > - 0, /* tp_iternext */ > - MesosExecutorDriverImpl_methods, /* tp_methods */ > - 0, /* tp_members */ > - 0, /* tp_getset */ > - 0, /* tp_base */ > - 0, /* tp_dict */ > - 0, /* tp_descr_get */ > - 0, /* tp_descr_set */ > - 0, /* tp_dictoffset */ > - (initproc) MesosExecutorDriverImpl_init, /* tp_init */ > - 0, /* tp_alloc */ > - MesosExecutorDriverImpl_new, /* tp_new */ > -}; > - > - > -/** > - * List of Python methods in MesosExecutorDriverImpl. > - */ > -PyMethodDef MesosExecutorDriverImpl_methods[] = { > - { "start", > - (PyCFunction) MesosExecutorDriverImpl_start, > - METH_NOARGS, > - "Start the driver to connect to Mesos" > - }, > - { "stop", > - (PyCFunction) MesosExecutorDriverImpl_stop, > - METH_NOARGS, > - "Stop the driver, disconnecting from Mesos" > - }, > - { "abort", > - (PyCFunction) MesosExecutorDriverImpl_abort, > - METH_NOARGS, > - "Abort the driver, disallowing calls from and to the driver" > - }, > - { "join", > - (PyCFunction) MesosExecutorDriverImpl_join, > - METH_NOARGS, > - "Wait for a running driver to disconnect from Mesos" > - }, > - { "run", > - (PyCFunction) MesosExecutorDriverImpl_run, > - METH_NOARGS, > - "Start a driver and run it, returning when it disconnects from Mesos" > - }, > - { "sendStatusUpdate", > - (PyCFunction) MesosExecutorDriverImpl_sendStatusUpdate, > - METH_VARARGS, > - "Send a status update for a task" > - }, > - { "sendFrameworkMessage", > - (PyCFunction) MesosExecutorDriverImpl_sendFrameworkMessage, > - METH_VARARGS, > - "Send a FrameworkMessage to a slave" > - }, > - { NULL } /* Sentinel */ > -}; > - > - > -/** > - * Create, but don't initialize, a new MesosExecutorDriverImpl > - * (called by Python before init method). > - */ > -PyObject* MesosExecutorDriverImpl_new(PyTypeObject *type, > - PyObject *args, > - PyObject *kwds) > -{ > - MesosExecutorDriverImpl *self; > - self = (MesosExecutorDriverImpl *) type->tp_alloc(type, 0); > - if (self != NULL) { > - self->driver = NULL; > - self->proxyExecutor = NULL; > - self->pythonExecutor = NULL; > - } > - return (PyObject*) self; > -} > - > - > -/** > - * Initialize a MesosExecutorDriverImpl with constructor arguments. > - */ > -int MesosExecutorDriverImpl_init(MesosExecutorDriverImpl *self, > - PyObject *args, > - PyObject *kwds) > -{ > - PyObject *pythonExecutor = NULL; > - > - if (!PyArg_ParseTuple(args, "O", &pythonExecutor)) { > - return -1; > - } > - > - if (pythonExecutor != NULL) { > - PyObject* tmp = self->pythonExecutor; > - Py_INCREF(pythonExecutor); > - self->pythonExecutor = pythonExecutor; > - Py_XDECREF(tmp); > - } > - > - if (self->driver != NULL) { > - delete self->driver; > - self->driver = NULL; > - } > - > - if (self->proxyExecutor != NULL) { > - delete self->proxyExecutor; > - self->proxyExecutor = NULL; > - } > - > - self->proxyExecutor = new ProxyExecutor(self); > - self->driver = new MesosExecutorDriver(self->proxyExecutor); > - > - return 0; > -} > - > - > -/** > - * Free a MesosExecutorDriverImpl. > - */ > -void MesosExecutorDriverImpl_dealloc(MesosExecutorDriverImpl* self) > -{ > - if (self->driver != NULL) { > - // We need to wrap the driver destructor in an "allow threads" > - // macro since the MesosExecutorDriver destructor waits for the > - // ExecutorProcess to terminate and there might be a thread that > - // is trying to acquire the GIL to call through the > - // ProxyExecutor. It will only be after this thread executes that > - // the ExecutorProcess might actually get a terminate. > - Py_BEGIN_ALLOW_THREADS > - delete self->driver; > - Py_END_ALLOW_THREADS > - self->driver = NULL; > - } > - > - if (self->proxyExecutor != NULL) { > - delete self->proxyExecutor; > - self->proxyExecutor = NULL; > - } > - > - MesosExecutorDriverImpl_clear(self); > - self->ob_type->tp_free((PyObject*) self); > -} > - > - > -/** > - * Traverse fields of a MesosExecutorDriverImpl on a cyclic GC search. > - * See http://docs.python.org/extending/newtypes.html. > - */ > -int MesosExecutorDriverImpl_traverse(MesosExecutorDriverImpl* self, > - visitproc visit, > - void* arg) > -{ > - Py_VISIT(self->pythonExecutor); > - return 0; > -} > - > - > -/** > - * Clear fields of a MesosExecutorDriverImpl that can participate in > - * GC cycles. See http://docs.python.org/extending/newtypes.html. > - */ > -int MesosExecutorDriverImpl_clear(MesosExecutorDriverImpl* self) > -{ > - Py_CLEAR(self->pythonExecutor); > - return 0; > -} > - > - > -PyObject* MesosExecutorDriverImpl_start(MesosExecutorDriverImpl* self) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL"); > - return NULL; > - } > - > - Status status = self->driver->start(); > - return PyInt_FromLong(status); // Sets an exception if creating the int > fails. > -} > - > - > -PyObject* MesosExecutorDriverImpl_stop(MesosExecutorDriverImpl* self) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL"); > - return NULL; > - } > - > - Status status = self->driver->stop(); > - return PyInt_FromLong(status); // Sets an exception if creating the int > fails. > -} > - > - > -PyObject* MesosExecutorDriverImpl_abort(MesosExecutorDriverImpl* self) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL"); > - return NULL; > - } > - > - Status status = self->driver->abort(); > - return PyInt_FromLong(status); // Sets an exception if creating the int > fails. > -} > - > - > -PyObject* MesosExecutorDriverImpl_join(MesosExecutorDriverImpl* self) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL"); > - return NULL; > - } > - > - Status status; > - Py_BEGIN_ALLOW_THREADS > - status = self->driver->join(); > - Py_END_ALLOW_THREADS > - return PyInt_FromLong(status); // Sets an exception if creating the int > fails. > -} > - > - > -PyObject* MesosExecutorDriverImpl_run(MesosExecutorDriverImpl* self) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL"); > - return NULL; > - } > - > - Status status; > - Py_BEGIN_ALLOW_THREADS > - status = self->driver->run(); > - Py_END_ALLOW_THREADS > - return PyInt_FromLong(status); // Sets an exception if creating the int > fails. > -} > - > - > -PyObject* MesosExecutorDriverImpl_sendStatusUpdate( > - MesosExecutorDriverImpl* self, > - PyObject* args) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL"); > - return NULL; > - } > - > - PyObject* statusObj = NULL; > - TaskStatus taskStatus; > - if (!PyArg_ParseTuple(args, "O", &statusObj)) { > - return NULL; > - } > - if (!readPythonProtobuf(statusObj, &taskStatus)) { > - PyErr_Format(PyExc_Exception, > - "Could not deserialize Python TaskStatus"); > - return NULL; > - } > - > - Status status = self->driver->sendStatusUpdate(taskStatus); > - return PyInt_FromLong(status); // Sets an exception if creating the int > fails. > -} > - > - > -PyObject* MesosExecutorDriverImpl_sendFrameworkMessage( > - MesosExecutorDriverImpl* self, > - PyObject* args) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL"); > - return NULL; > - } > - > - const char* data; > - int length; > - if (!PyArg_ParseTuple(args, "s#", &data, &length)) { > - return NULL; > - } > - > - Status status = self->driver->sendFrameworkMessage(string(data, length)); > - return PyInt_FromLong(status); // Sets an exception if creating the int > fails. > -} > - > -} // namespace python { > -} // namespace mesos { > > http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/src/mesos/native/mesos_executor_driver_impl.hpp > ---------------------------------------------------------------------- > diff --git > a/src/python/native/src/mesos/native/mesos_executor_driver_impl.hpp > b/src/python/native/src/mesos/native/mesos_executor_driver_impl.hpp > deleted file mode 100644 > index 6e672f8..0000000 > --- a/src/python/native/src/mesos/native/mesos_executor_driver_impl.hpp > +++ /dev/null > @@ -1,103 +0,0 @@ > -// Licensed to the Apache Software Foundation (ASF) under one > -// or more contributor license agreements. See the NOTICE file > -// distributed with this work for additional information > -// regarding copyright ownership. The ASF licenses this file > -// to you under the Apache License, Version 2.0 (the > -// "License"); you may not use this file except in compliance > -// with the License. You may obtain a copy of the License at > -// > -// http://www.apache.org/licenses/LICENSE-2.0 > -// > -// Unless required by applicable law or agreed to in writing, software > -// distributed under the License is distributed on an "AS IS" BASIS, > -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > -// See the License for the specific language governing permissions and > -// limitations under the License. > - > -#ifndef MESOS_EXECUTOR_DRIVER_IMPL_HPP > -#define MESOS_EXECUTOR_DRIVER_IMPL_HPP > - > -#include <mesos/executor.hpp> > - > - > -namespace mesos { namespace python { > - > -class ProxyExecutor; > - > -/** > - * Python object structure for MesosExecutorDriverImpl objects. > - */ > -struct MesosExecutorDriverImpl { > - PyObject_HEAD > - /* Type-specific fields go here. */ > - MesosExecutorDriver* driver; > - ProxyExecutor* proxyExecutor; > - PyObject* pythonExecutor; > -}; > - > -/** > - * Python type object for MesosExecutorDriverImpl. > - */ > -extern PyTypeObject MesosExecutorDriverImplType; > - > -/** > - * List of Python methods in MesosExecutorDriverImpl. > - */ > -extern PyMethodDef MesosExecutorDriverImpl_methods[]; > - > -/** > - * Create, but don't initialize, a new MesosExecutorDriverImpl > - * (called by Python before init method). > - */ > -PyObject* MesosExecutorDriverImpl_new(PyTypeObject *type, > - PyObject *args, > - PyObject *kwds); > - > -/** > - * Initialize a MesosExecutorDriverImpl with constructor arguments. > - */ > -int MesosExecutorDriverImpl_init(MesosExecutorDriverImpl *self, > - PyObject *args, > - PyObject *kwds); > - > -/** > - * Free a MesosExecutorDriverImpl. > - */ > -void MesosExecutorDriverImpl_dealloc(MesosExecutorDriverImpl* self); > - > -/** > - * Traverse fields of a MesosExecutorDriverImpl on a cyclic GC search. > - * See http://docs.python.org/extending/newtypes.html. > - */ > -int MesosExecutorDriverImpl_traverse(MesosExecutorDriverImpl* self, > - visitproc visit, > - void* arg); > -/** > - * Clear fields of a MesosExecutorDriverImpl that can participate in > - * GC cycles. See http://docs.python.org/extending/newtypes.html. > - */ > -int MesosExecutorDriverImpl_clear(MesosExecutorDriverImpl* self); > - > -// MesosExecutorDriverImpl methods. > -PyObject* MesosExecutorDriverImpl_start(MesosExecutorDriverImpl* self); > - > -PyObject* MesosExecutorDriverImpl_stop(MesosExecutorDriverImpl* self); > - > -PyObject* MesosExecutorDriverImpl_abort(MesosExecutorDriverImpl* self); > - > -PyObject* MesosExecutorDriverImpl_join(MesosExecutorDriverImpl* self); > - > -PyObject* MesosExecutorDriverImpl_run(MesosExecutorDriverImpl* self); > - > -PyObject* MesosExecutorDriverImpl_sendStatusUpdate( > - MesosExecutorDriverImpl* self, > - PyObject* args); > - > -PyObject* MesosExecutorDriverImpl_sendFrameworkMessage( > - MesosExecutorDriverImpl* self, > - PyObject* args); > - > -} // namespace python { > -} // namespace mesos { > - > -#endif /* MESOS_EXECUTOR_DRIVER_IMPL_HPP */ > > http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp > ---------------------------------------------------------------------- > diff --git > a/src/python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp > b/src/python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp > deleted file mode 100644 > index f8be49b..0000000 > --- a/src/python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp > +++ /dev/null > @@ -1,782 +0,0 @@ > -// Licensed to the Apache Software Foundation (ASF) under one > -// or more contributor license agreements. See the NOTICE file > -// distributed with this work for additional information > -// regarding copyright ownership. The ASF licenses this file > -// to you under the Apache License, Version 2.0 (the > -// "License"); you may not use this file except in compliance > -// with the License. You may obtain a copy of the License at > -// > -// http://www.apache.org/licenses/LICENSE-2.0 > -// > -// Unless required by applicable law or agreed to in writing, software > -// distributed under the License is distributed on an "AS IS" BASIS, > -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > -// See the License for the specific language governing permissions and > -// limitations under the License. > - > -// Python.h must be included before standard headers. > -// See: http://docs.python.org/2/c-api/intro.html#include-files > -#include <Python.h> > - > -#include <string> > - > -#include "mesos_scheduler_driver_impl.hpp" > -#include "module.hpp" > -#include "proxy_scheduler.hpp" > - > -using namespace mesos; > -using namespace mesos::python; > - > -using std::cerr; > -using std::endl; > -using std::string; > -using std::vector; > -using std::map; > - > -namespace mesos { > -namespace python { > - > -/** > - * Python type object for MesosSchedulerDriverImpl. > - */ > -PyTypeObject MesosSchedulerDriverImplType = { > - PyObject_HEAD_INIT(NULL) > - 0, /* ob_size */ > - "_mesos.MesosSchedulerDriverImpl", /* tp_name */ > - sizeof(MesosSchedulerDriverImpl), /* tp_basicsize */ > - 0, /* tp_itemsize */ > - (destructor) MesosSchedulerDriverImpl_dealloc, /* tp_dealloc */ > - 0, /* tp_print */ > - 0, /* tp_getattr */ > - 0, /* tp_setattr */ > - 0, /* tp_compare */ > - 0, /* tp_repr */ > - 0, /* tp_as_number */ > - 0, /* tp_as_sequence */ > - 0, /* tp_as_mapping */ > - 0, /* tp_hash */ > - 0, /* tp_call */ > - 0, /* tp_str */ > - 0, /* tp_getattro */ > - 0, /* tp_setattro */ > - 0, /* tp_as_buffer */ > - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ > - "Private MesosSchedulerDriver implementation", /* tp_doc */ > - (traverseproc) MesosSchedulerDriverImpl_traverse, /* tp_traverse */ > - (inquiry) MesosSchedulerDriverImpl_clear, /* tp_clear */ > - 0, /* tp_richcompare */ > - 0, /* tp_weaklistoffset */ > - 0, /* tp_iter */ > - 0, /* tp_iternext */ > - MesosSchedulerDriverImpl_methods, /* tp_methods */ > - 0, /* tp_members */ > - 0, /* tp_getset */ > - 0, /* tp_base */ > - 0, /* tp_dict */ > - 0, /* tp_descr_get */ > - 0, /* tp_descr_set */ > - 0, /* tp_dictoffset */ > - (initproc) MesosSchedulerDriverImpl_init, /* tp_init */ > - 0, /* tp_alloc */ > - MesosSchedulerDriverImpl_new, /* tp_new */ > -}; > - > - > -/** > - * List of Python methods in MesosSchedulerDriverImpl. > - */ > -PyMethodDef MesosSchedulerDriverImpl_methods[] = { > - { "start", > - (PyCFunction) MesosSchedulerDriverImpl_start, > - METH_NOARGS, > - "Start the driver to connect to Mesos" > - }, > - { "stop", > - (PyCFunction) MesosSchedulerDriverImpl_stop, > - METH_VARARGS, > - "Stop the driver, disconnecting from Mesos" > - }, > - { "abort", > - (PyCFunction) MesosSchedulerDriverImpl_abort, > - METH_NOARGS, > - "Abort the driver, disabling calls from and to the driver" > - }, > - { "join", > - (PyCFunction) MesosSchedulerDriverImpl_join, > - METH_NOARGS, > - "Wait for a running driver to disconnect from Mesos" > - }, > - { "run", > - (PyCFunction) MesosSchedulerDriverImpl_run, > - METH_NOARGS, > - "Start a driver and run it, returning when it disconnects from Mesos" > - }, > - { "requestResources", > - (PyCFunction) MesosSchedulerDriverImpl_requestResources, > - METH_VARARGS, > - "Request resources from the Mesos allocator" > - }, > - { "launchTasks", > - (PyCFunction) MesosSchedulerDriverImpl_launchTasks, > - METH_VARARGS, > - "Reply to a Mesos offer with a list of tasks" > - }, > - { "killTask", > - (PyCFunction) MesosSchedulerDriverImpl_killTask, > - METH_VARARGS, > - "Kill the task with the given ID" > - }, > - { "acceptOffers", > - (PyCFunction) MesosSchedulerDriverImpl_acceptOffers, > - METH_VARARGS, > - "Reply to a Mesos offer with a list of offer operations" > - }, > - { "declineOffer", > - (PyCFunction) MesosSchedulerDriverImpl_declineOffer, > - METH_VARARGS, > - "Decline a Mesos offer" > - }, > - { "reviveOffers", > - (PyCFunction) MesosSchedulerDriverImpl_reviveOffers, > - METH_NOARGS, > - "Remove all filters and ask Mesos for new offers" > - }, > - { "suppressOffers", > - (PyCFunction) MesosSchedulerDriverImpl_suppressOffers, > - METH_NOARGS, > - "Set suppressed attribute as true for the Framework" > - }, > - { "acknowledgeStatusUpdate", > - (PyCFunction) MesosSchedulerDriverImpl_acknowledgeStatusUpdate, > - METH_VARARGS, > - "Acknowledge a status update" > - }, > - { "sendFrameworkMessage", > - (PyCFunction) MesosSchedulerDriverImpl_sendFrameworkMessage, > - METH_VARARGS, > - "Send a FrameworkMessage to a slave" > - }, > - { "reconcileTasks", > - (PyCFunction) MesosSchedulerDriverImpl_reconcileTasks, > - METH_VARARGS, > - "Master sends status updates if task status is different from expected" > - }, > - { NULL } /* Sentinel */ > -}; > - > - > -/** > - * Create, but don't initialize, a new MesosSchedulerDriverImpl > - * (called by Python before init method). > - */ > -PyObject* MesosSchedulerDriverImpl_new(PyTypeObject* type, > - PyObject* args, > - PyObject* kwds) > -{ > - MesosSchedulerDriverImpl* self; > - self = (MesosSchedulerDriverImpl*) type->tp_alloc(type, 0); > - if (self != NULL) { > - self->driver = NULL; > - self->proxyScheduler = NULL; > - self->pythonScheduler = NULL; > - } > - return (PyObject*) self; > -} > - > - > -/** > - * Initialize a MesosSchedulerDriverImpl with constructor arguments. > - */ > -int MesosSchedulerDriverImpl_init(MesosSchedulerDriverImpl* self, > - PyObject* args, > - PyObject* kwds) > -{ > - // Note: We use an integer for 'implicitAcknoweldgements' because > - // it is the recommended way to pass booleans through CPython. > - PyObject* schedulerObj = NULL; > - PyObject* frameworkObj = NULL; > - const char* master; > - int implicitAcknowledgements = 1; // Enabled by default. > - PyObject* credentialObj = NULL; > - > - if (!PyArg_ParseTuple( > - args, > - "OOs|iO", > - &schedulerObj, > - &frameworkObj, > - &master, > - &implicitAcknowledgements, > - &credentialObj)) { > - return -1; > - } > - > - if (schedulerObj != NULL) { > - PyObject* tmp = self->pythonScheduler; > - Py_INCREF(schedulerObj); > - self->pythonScheduler = schedulerObj; > - Py_XDECREF(tmp); > - } > - > - FrameworkInfo framework; > - if (frameworkObj != NULL) { > - if (!readPythonProtobuf(frameworkObj, &framework)) { > - PyErr_Format(PyExc_Exception, > - "Could not deserialize Python FrameworkInfo"); > - return -1; > - } > - } > - > - Credential credential; > - if (credentialObj != NULL) { > - if (!readPythonProtobuf(credentialObj, &credential)) { > - PyErr_Format(PyExc_Exception, "Could not deserialize Python > Credential"); > - return -1; > - } > - } > - > - > - if (self->driver != NULL) { > - delete self->driver; > - self->driver = NULL; > - } > - > - if (self->proxyScheduler != NULL) { > - delete self->proxyScheduler; > - self->proxyScheduler = NULL; > - } > - > - self->proxyScheduler = new ProxyScheduler(self); > - > - if (credentialObj != NULL) { > - self->driver = new MesosSchedulerDriver( > - self->proxyScheduler, > - framework, > - master, > - implicitAcknowledgements != 0, > - credential); > - } else { > - self->driver = new MesosSchedulerDriver( > - self->proxyScheduler, > - framework, > - master, > - implicitAcknowledgements != 0); > - } > - > - return 0; > -} > - > - > -/** > - * Free a MesosSchedulerDriverImpl. > - */ > -void MesosSchedulerDriverImpl_dealloc(MesosSchedulerDriverImpl* self) > -{ > - if (self->driver != NULL) { > - // We need to wrap the driver destructor in an "allow threads" > - // macro since the MesosSchedulerDriver destructor waits for the > - // SchedulerProcess to terminate and there might be a thread that > - // is trying to acquire the GIL to call through the > - // ProxyScheduler. It will only be after this thread executes that > - // the SchedulerProcess might actually get a terminate. > - Py_BEGIN_ALLOW_THREADS > - delete self->driver; > - Py_END_ALLOW_THREADS > - self->driver = NULL; > - } > - > - if (self->proxyScheduler != NULL) { > - delete self->proxyScheduler; > - self->proxyScheduler = NULL; > - } > - > - MesosSchedulerDriverImpl_clear(self); > - self->ob_type->tp_free((PyObject*) self); > -} > - > - > -/** > - * Traverse fields of a MesosSchedulerDriverImpl on a cyclic GC search. > - * See http://docs.python.org/extending/newtypes.html. > - */ > -int MesosSchedulerDriverImpl_traverse(MesosSchedulerDriverImpl* self, > - visitproc visit, > - void* arg) > -{ > - Py_VISIT(self->pythonScheduler); > - return 0; > -} > - > - > -/** > - * Clear fields of a MesosSchedulerDriverImpl that can participate in > - * GC cycles. See http://docs.python.org/extending/newtypes.html. > - */ > -int MesosSchedulerDriverImpl_clear(MesosSchedulerDriverImpl* self) > -{ > - Py_CLEAR(self->pythonScheduler); > - return 0; > -} > - > - > -PyObject* MesosSchedulerDriverImpl_start(MesosSchedulerDriverImpl* self) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL"); > - return NULL; > - } > - > - Status status = self->driver->start(); > - return PyInt_FromLong(status); // Sets exception if creating long fails. > -} > - > - > -PyObject* MesosSchedulerDriverImpl_stop(MesosSchedulerDriverImpl* self, > - PyObject* args) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL"); > - return NULL; > - } > - > - bool failover = false; // Should match default in mesos.py. > - > - if (!PyArg_ParseTuple(args, "|b", &failover)) { > - return NULL; > - } > - > - Status status = self->driver->stop(failover); > - return PyInt_FromLong(status); // Sets exception if creating long fails. > -} > - > - > -PyObject* MesosSchedulerDriverImpl_abort(MesosSchedulerDriverImpl* self) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL"); > - return NULL; > - } > - > - Status status = self->driver->abort(); > - return PyInt_FromLong(status); // Sets exception if creating long fails. > -} > - > - > -PyObject* MesosSchedulerDriverImpl_join(MesosSchedulerDriverImpl* self) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL"); > - return NULL; > - } > - > - Status status; > - Py_BEGIN_ALLOW_THREADS > - status = self->driver->join(); > - Py_END_ALLOW_THREADS > - return PyInt_FromLong(status); // Sets exception if creating long fails. > -} > - > - > -PyObject* MesosSchedulerDriverImpl_run(MesosSchedulerDriverImpl* self) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL"); > - return NULL; > - } > - > - Status status; > - Py_BEGIN_ALLOW_THREADS > - status = self->driver->run(); > - Py_END_ALLOW_THREADS > - return PyInt_FromLong(status); // Sets exception if creating long fails. > -} > - > - > -PyObject* MesosSchedulerDriverImpl_requestResources( > - MesosSchedulerDriverImpl* self, > - PyObject* args) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL"); > - return NULL; > - } > - > - PyObject* requestsObj = NULL; > - vector<Request> requests; > - > - if (!PyArg_ParseTuple(args, "O", &requestsObj)) { > - return NULL; > - } > - > - if (!PyList_Check(requestsObj)) { > - PyErr_Format(PyExc_Exception, > - "Parameter 2 to requestsResources is not a list"); > - return NULL; > - } > - Py_ssize_t len = PyList_Size(requestsObj); > - for (int i = 0; i < len; i++) { > - PyObject* requestObj = PyList_GetItem(requestsObj, i); > - if (requestObj == NULL) { > - return NULL; // Exception will have been set by PyList_GetItem. > - } > - Request request; > - if (!readPythonProtobuf(requestObj, &request)) { > - PyErr_Format(PyExc_Exception, "Could not deserialize Python Request"); > - return NULL; > - } > - requests.push_back(request); > - } > - > - Status status = self->driver->requestResources(requests); > - return PyInt_FromLong(status); // Sets exception if creating long fails. > -} > - > - > -PyObject* MesosSchedulerDriverImpl_launchTasks(MesosSchedulerDriverImpl* > self, > - PyObject* args) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL"); > - return NULL; > - } > - > - PyObject* offerIdsObj = NULL; > - PyObject* tasksObj = NULL; > - PyObject* filtersObj = NULL; > - vector<OfferID> offerIds; > - vector<TaskInfo> tasks; > - Filters filters; > - > - if (!PyArg_ParseTuple(args, "OO|O", &offerIdsObj, &tasksObj, &filtersObj)) > { > - return NULL; > - } > - > - // Offer argument can be a list of offer ids or a single offer id (for > - // backward compatibility). > - if (!PyList_Check(offerIdsObj)) { > - OfferID offerId; > - if (!readPythonProtobuf(offerIdsObj, &offerId)) { > - PyErr_Format(PyExc_Exception, "Could not deserialize Python OfferID"); > - return NULL; > - } > - offerIds.push_back(offerId); > - } else { > - Py_ssize_t len = PyList_Size(offerIdsObj); > - for (int i = 0; i < len; i++) { > - PyObject* offerObj = PyList_GetItem(offerIdsObj, i); > - if (offerObj == NULL) { > - return NULL; > - } > - OfferID offerId; > - if (!readPythonProtobuf(offerObj, &offerId)) { > - PyErr_Format(PyExc_Exception, > - "Could not deserialize Python OfferID"); > - return NULL; > - } > - offerIds.push_back(offerId); > - } > - } > - > - if (!PyList_Check(tasksObj)) { > - PyErr_Format(PyExc_Exception, "Parameter 2 to launchTasks is not a > list"); > - return NULL; > - } > - Py_ssize_t len = PyList_Size(tasksObj); > - for (int i = 0; i < len; i++) { > - PyObject* taskObj = PyList_GetItem(tasksObj, i); > - if (taskObj == NULL) { > - return NULL; // Exception will have been set by PyList_GetItem. > - } > - TaskInfo task; > - if (!readPythonProtobuf(taskObj, &task)) { > - PyErr_Format(PyExc_Exception, > - "Could not deserialize Python TaskInfo"); > - return NULL; > - } > - tasks.push_back(task); > - } > - > - if (filtersObj != NULL) { > - if (!readPythonProtobuf(filtersObj, &filters)) { > - PyErr_Format(PyExc_Exception, > - "Could not deserialize Python Filters"); > - return NULL; > - } > - } > - > - Status status = self->driver->launchTasks(offerIds, tasks, filters); > - return PyInt_FromLong(status); // Sets exception if creating long fails. > -} > - > - > -PyObject* MesosSchedulerDriverImpl_killTask(MesosSchedulerDriverImpl* self, > - PyObject* args) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL"); > - return NULL; > - } > - > - PyObject* tidObj = NULL; > - TaskID tid; > - if (!PyArg_ParseTuple(args, "O", &tidObj)) { > - return NULL; > - } > - if (!readPythonProtobuf(tidObj, &tid)) { > - PyErr_Format(PyExc_Exception, "Could not deserialize Python TaskID"); > - return NULL; > - } > - > - Status status = self->driver->killTask(tid); > - return PyInt_FromLong(status); // Sets exception if creating long fails. > -} > - > - > -PyObject* MesosSchedulerDriverImpl_acceptOffers(MesosSchedulerDriverImpl* > self, > - PyObject* args) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL"); > - return NULL; > - } > - > - PyObject* offerIdsObj = NULL; > - PyObject* operationsObj = NULL; > - PyObject* filtersObj = NULL; > - Py_ssize_t len = 0; > - vector<OfferID> offerIds; > - vector<Offer::Operation> operations; > - Filters filters; > - > - if (!PyArg_ParseTuple(args, > - "OO|O", > - &offerIdsObj, > - &operationsObj, > - &filtersObj)) { > - return NULL; > - } > - > - if (!PyList_Check(offerIdsObj)) { > - PyErr_Format(PyExc_Exception, "Parameter 1 to acceptOffers is not a > list"); > - return NULL; > - } > - > - len = PyList_Size(offerIdsObj); > - for (int i = 0; i < len; i++) { > - PyObject* offerObj = PyList_GetItem(offerIdsObj, i); > - if (offerObj == NULL) { > - return NULL; > - } > - > - OfferID offerId; > - if (!readPythonProtobuf(offerObj, &offerId)) { > - PyErr_Format(PyExc_Exception, > - "Could not deserialize Python OfferID"); > - return NULL; > - } > - offerIds.push_back(offerId); > - } > - > - if (!PyList_Check(operationsObj)) { > - PyErr_Format(PyExc_Exception, "Parameter 2 to acceptOffers is not a > list"); > - return NULL; > - } > - > - len = PyList_Size(operationsObj); > - for (int i = 0; i < len; i++) { > - PyObject* operationObj = PyList_GetItem(operationsObj, i); > - if (operationObj == NULL) { > - return NULL; // Exception will have been set by PyList_GetItem. > - } > - > - Offer::Operation operation; > - if (!readPythonProtobuf(operationObj, &operation)) { > - PyErr_Format(PyExc_Exception, > - "Could not deserialize Python Offer.Operation"); > - return NULL; > - } > - operations.push_back(operation); > - } > - > - if (filtersObj != NULL) { > - if (!readPythonProtobuf(filtersObj, &filters)) { > - PyErr_Format(PyExc_Exception, > - "Could not deserialize Python Filters"); > - return NULL; > - } > - } > - > - Status status = self->driver->acceptOffers(offerIds, operations, filters); > - return PyInt_FromLong(status); // Sets exception if creating long fails. > -} > - > - > -PyObject* MesosSchedulerDriverImpl_declineOffer(MesosSchedulerDriverImpl* > self, > - PyObject* args) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL"); > - return NULL; > - } > - > - PyObject* offerIdObj = NULL; > - PyObject* filtersObj = NULL; > - OfferID offerId; > - Filters filters; > - > - if (!PyArg_ParseTuple(args, "O|O", &offerIdObj, &filtersObj)) { > - return NULL; > - } > - > - if (!readPythonProtobuf(offerIdObj, &offerId)) { > - PyErr_Format(PyExc_Exception, "Could not deserialize Python OfferID"); > - return NULL; > - } > - > - if (filtersObj != NULL) { > - if (!readPythonProtobuf(filtersObj, &filters)) { > - PyErr_Format(PyExc_Exception, > - "Could not deserialize Python Filters"); > - return NULL; > - } > - } > - > - Status status = self->driver->declineOffer(offerId, filters); > - return PyInt_FromLong(status); // Sets exception if creating long fails. > -} > - > - > -PyObject* MesosSchedulerDriverImpl_reviveOffers(MesosSchedulerDriverImpl* > self) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL"); > - return NULL; > - } > - > - Status status = self->driver->reviveOffers(); > - return PyInt_FromLong(status); // Sets exception if creating long fails. > -} > - > - > -PyObject* MesosSchedulerDriverImpl_suppressOffers( > - MesosSchedulerDriverImpl* self) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL"); > - return NULL; > - } > - > - Status status = self->driver->suppressOffers(); > - return PyInt_FromLong(status); // Sets exception if creating long fails. > -} > - > - > -PyObject* MesosSchedulerDriverImpl_acknowledgeStatusUpdate( > - MesosSchedulerDriverImpl* self, > - PyObject* args) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL"); > - return NULL; > - } > - > - PyObject* taskStatusObj = NULL; > - TaskStatus taskStatus; > - > - if (!PyArg_ParseTuple(args, "O", &taskStatusObj)) { > - return NULL; > - } > - > - if (!readPythonProtobuf(taskStatusObj, &taskStatus)) { > - PyErr_Format(PyExc_Exception, "Could not deserialize Python TaskStatus"); > - return NULL; > - } > - > - Status status = self->driver->acknowledgeStatusUpdate(taskStatus); > - > - return PyInt_FromLong(status); // Sets exception if creating long fails. > -} > - > - > -PyObject* MesosSchedulerDriverImpl_sendFrameworkMessage( > - MesosSchedulerDriverImpl* self, > - PyObject* args) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL"); > - return NULL; > - } > - > - PyObject* slaveIdObj = NULL; > - PyObject* executorIdObj = NULL; > - SlaveID slaveId; > - ExecutorID executorId; > - const char* data; > - int length; > - > - if (!PyArg_ParseTuple( > - args, "OOs#", &executorIdObj, &slaveIdObj, &data, &length)) { > - return NULL; > - } > - > - if (!readPythonProtobuf(executorIdObj, &executorId)) { > - PyErr_Format(PyExc_Exception, "Could not deserialize Python ExecutorID"); > - return NULL; > - } > - > - if (!readPythonProtobuf(slaveIdObj, &slaveId)) { > - PyErr_Format(PyExc_Exception, "Could not deserialize Python SlaveID"); > - return NULL; > - } > - > - Status status = self->driver->sendFrameworkMessage( > - executorId, slaveId, string(data, length)); > - > - return PyInt_FromLong(status); // Sets exception if creating long fails. > -} > - > - > -PyObject* MesosSchedulerDriverImpl_reconcileTasks( > - MesosSchedulerDriverImpl* self, > - PyObject* args) > -{ > - if (self->driver == NULL) { > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL"); > - return NULL; > - } > - > - PyObject* statusesObj = NULL; > - vector<TaskStatus> statuses; > - > - if (!PyArg_ParseTuple(args, "O", &statusesObj)) { > - return NULL; > - } > - > - if (!PyList_Check(statusesObj)) { > - PyErr_Format(PyExc_Exception, > - "Parameter 1 to reconcileTasks is not a list"); > - > - return NULL; > - } > - > - Py_ssize_t len = PyList_Size(statusesObj); > - for (int i = 0; i < len; i++) { > - PyObject* statusObj = PyList_GetItem(statusesObj, i); > - if (statusObj == NULL) { > - return NULL; > - } > - > - TaskStatus status; > - if (!readPythonProtobuf(statusObj, &status)) { > - PyErr_Format(PyExc_Exception, > - "Could not deserialize Python TaskStatus"); > - return NULL; > - } > - statuses.push_back(status); > - } > - > - Status status = self->driver->reconcileTasks(statuses); > - return PyInt_FromLong(status); > -} > - > -} // namespace python { > -} // namespace mesos { > >
