Repository: mesos Updated Branches: refs/heads/master 3047bbe41 -> c5a68be12
http://git-wip-us.apache.org/repos/asf/mesos/blob/c5a68be1/src/python/native/src/mesos/native/proxy_executor.cpp ---------------------------------------------------------------------- diff --git a/src/python/native/src/mesos/native/proxy_executor.cpp b/src/python/native/src/mesos/native/proxy_executor.cpp new file mode 100644 index 0000000..5e8637e --- /dev/null +++ b/src/python/native/src/mesos/native/proxy_executor.cpp @@ -0,0 +1,275 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Python.h must be included before standard headers. +// See: http://docs.python.org/2/c-api/intro.html#include-files +#include <Python.h> + +#include <iostream> + +#include "proxy_executor.hpp" +#include "module.hpp" +#include "mesos_executor_driver_impl.hpp" + +using namespace mesos; + +using std::cerr; +using std::endl; +using std::string; +using std::vector; +using std::map; + +namespace mesos { +namespace python { + +void ProxyExecutor::registered(ExecutorDriver* driver, + const ExecutorInfo& executorInfo, + const FrameworkInfo& frameworkInfo, + const SlaveInfo& slaveInfo) +{ + InterpreterLock lock; + + PyObject* executorInfoObj = NULL; + PyObject* frameworkInfoObj = NULL; + PyObject* slaveInfoObj = NULL; + PyObject* res = NULL; + + executorInfoObj = createPythonProtobuf(executorInfo, "ExecutorInfo"); + frameworkInfoObj = createPythonProtobuf(frameworkInfo, "FrameworkInfo"); + slaveInfoObj = createPythonProtobuf(slaveInfo, "SlaveInfo"); + + if (executorInfoObj == NULL || + frameworkInfoObj == NULL || + slaveInfoObj == NULL) { + goto cleanup; // createPythonProtobuf will have set an exception + } + + res = PyObject_CallMethod(impl->pythonExecutor, + (char*) "registered", + (char*) "OOOO", + impl, + executorInfoObj, + frameworkInfoObj, + slaveInfoObj); + if (res == NULL) { + cerr << "Failed to call executor registered" << endl; + goto cleanup; + } + +cleanup: + if (PyErr_Occurred()) { + PyErr_Print(); + driver->abort(); + } + Py_XDECREF(executorInfoObj); + Py_XDECREF(frameworkInfoObj); + Py_XDECREF(slaveInfoObj); + Py_XDECREF(res); +} + + +void ProxyExecutor::reregistered(ExecutorDriver* driver, + const SlaveInfo& slaveInfo) +{ + InterpreterLock lock; + + PyObject* slaveInfoObj = NULL; + PyObject* res = NULL; + + slaveInfoObj = createPythonProtobuf(slaveInfo, "SlaveInfo"); + + if (slaveInfoObj == NULL) { + goto cleanup; // createPythonProtobuf will have set an exception + } + + res = PyObject_CallMethod(impl->pythonExecutor, + (char*) "reregistered", + (char*) "OO", + impl, + slaveInfoObj); + if (res == NULL) { + cerr << "Failed to call executor re-registered" << endl; + goto cleanup; + } + +cleanup: + if (PyErr_Occurred()) { + PyErr_Print(); + driver->abort(); + } + Py_XDECREF(slaveInfoObj); + Py_XDECREF(res); +} + + +void ProxyExecutor::disconnected(ExecutorDriver* driver) +{ + InterpreterLock lock; + PyObject* res = PyObject_CallMethod(impl->pythonExecutor, + (char*) "disconnected", + (char*) "O", + impl); + if (res == NULL) { + cerr << "Failed to call executor's disconnected" << endl; + goto cleanup; + } +cleanup: + if (PyErr_Occurred()) { + PyErr_Print(); + driver->abort(); + } + Py_XDECREF(res); +} + + +void ProxyExecutor::launchTask(ExecutorDriver* driver, + const TaskInfo& task) +{ + InterpreterLock lock; + + PyObject* taskObj = NULL; + PyObject* res = NULL; + + taskObj = createPythonProtobuf(task, "TaskInfo"); + if (taskObj == NULL) { + goto cleanup; // createPythonProtobuf will have set an exception + } + + res = PyObject_CallMethod(impl->pythonExecutor, + (char*) "launchTask", + (char*) "OO", + impl, + taskObj); + if (res == NULL) { + cerr << "Failed to call executor's launchTask" << endl; + goto cleanup; + } + +cleanup: + if (PyErr_Occurred()) { + PyErr_Print(); + driver->abort(); + } + Py_XDECREF(taskObj); + Py_XDECREF(res); +} + + +void ProxyExecutor::killTask(ExecutorDriver* driver, + const TaskID& taskId) +{ + InterpreterLock lock; + + PyObject* taskIdObj = NULL; + PyObject* res = NULL; + + taskIdObj = createPythonProtobuf(taskId, "TaskID"); + if (taskIdObj == NULL) { + goto cleanup; // createPythonProtobuf will have set an exception + } + + res = PyObject_CallMethod(impl->pythonExecutor, + (char*) "killTask", + (char*) "OO", + impl, + taskIdObj); + if (res == NULL) { + cerr << "Failed to call executor's killTask" << endl; + goto cleanup; + } + +cleanup: + if (PyErr_Occurred()) { + PyErr_Print(); + driver->abort(); + } + Py_XDECREF(taskIdObj); + Py_XDECREF(res); +} + + +void ProxyExecutor::frameworkMessage(ExecutorDriver* driver, + const string& data) +{ + InterpreterLock lock; + + PyObject* res = NULL; + + res = PyObject_CallMethod(impl->pythonExecutor, + (char*) "frameworkMessage", + (char*) "Os#", + impl, + data.data(), + data.length()); + if (res == NULL) { + cerr << "Failed to call executor's frameworkMessage" << endl; + goto cleanup; + } + +cleanup: + if (PyErr_Occurred()) { + PyErr_Print(); + driver->abort(); + } + Py_XDECREF(res); +} + + +void ProxyExecutor::shutdown(ExecutorDriver* driver) +{ + InterpreterLock lock; + PyObject* res = PyObject_CallMethod(impl->pythonExecutor, + (char*) "shutdown", + (char*) "O", + impl); + if (res == NULL) { + cerr << "Failed to call executor's shutdown" << endl; + goto cleanup; + } +cleanup: + if (PyErr_Occurred()) { + PyErr_Print(); + driver->abort(); + } + Py_XDECREF(res); +} + + +void ProxyExecutor::error(ExecutorDriver* driver, const string& message) +{ + InterpreterLock lock; + PyObject* res = PyObject_CallMethod(impl->pythonExecutor, + (char*) "error", + (char*) "Os#", + impl, + message.data(), + message.length()); + if (res == NULL) { + cerr << "Failed to call executor's error" << endl; + goto cleanup; + } +cleanup: + if (PyErr_Occurred()) { + PyErr_Print(); + // No need for driver.stop(); it should stop itself + } + Py_XDECREF(res); +} + +} // namespace python { +} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/c5a68be1/src/python/native/src/mesos/native/proxy_executor.hpp ---------------------------------------------------------------------- diff --git a/src/python/native/src/mesos/native/proxy_executor.hpp b/src/python/native/src/mesos/native/proxy_executor.hpp new file mode 100644 index 0000000..eeefc5e --- /dev/null +++ b/src/python/native/src/mesos/native/proxy_executor.hpp @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PROXY_EXECUTOR_HPP +#define PROXY_EXECUTOR_HPP + +// Python.h must be included before standard headers. +// See: http://docs.python.org/2/c-api/intro.html#include-files +#include <Python.h> + +#include <string> +#include <vector> + +#include <mesos/executor.hpp> + +namespace mesos { +namespace python { + +struct MesosExecutorDriverImpl; + +/** + * Proxy Executor implementation that will call into Python. + */ +class ProxyExecutor : public Executor +{ +public: + explicit ProxyExecutor(MesosExecutorDriverImpl *_impl) : impl(_impl) {} + + virtual ~ProxyExecutor() {} + + virtual void registered(ExecutorDriver* driver, + const ExecutorInfo& executorInfo, + const FrameworkInfo& frameworkInfo, + const SlaveInfo& slaveInfo); + virtual void reregistered(ExecutorDriver* driver, const SlaveInfo& slaveInfo); + virtual void disconnected(ExecutorDriver* driver); + virtual void launchTask(ExecutorDriver* driver, const TaskInfo& task); + virtual void killTask(ExecutorDriver* driver, const TaskID& taskId); + virtual void frameworkMessage(ExecutorDriver* driver, + const std::string& data); + virtual void shutdown(ExecutorDriver* driver); + virtual void error(ExecutorDriver* driver, const std::string& message); + +private: + MesosExecutorDriverImpl *impl; +}; + +} // namespace python { +} // namespace mesos { + +#endif // PROXY_EXECUTOR_HPP http://git-wip-us.apache.org/repos/asf/mesos/blob/c5a68be1/src/python/native/src/mesos/native/proxy_scheduler.cpp ---------------------------------------------------------------------- diff --git a/src/python/native/src/mesos/native/proxy_scheduler.cpp b/src/python/native/src/mesos/native/proxy_scheduler.cpp new file mode 100644 index 0000000..95b09cf --- /dev/null +++ b/src/python/native/src/mesos/native/proxy_scheduler.cpp @@ -0,0 +1,386 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Python.h must be included before standard headers. +// See: http://docs.python.org/2/c-api/intro.html#include-files +#include <Python.h> + +#include <iostream> + +#include "proxy_scheduler.hpp" +#include "module.hpp" +#include "mesos_scheduler_driver_impl.hpp" + +using namespace mesos; + +using std::cerr; +using std::endl; +using std::string; +using std::vector; +using std::map; + +namespace mesos { +namespace python { + +void ProxyScheduler::registered(SchedulerDriver* driver, + const FrameworkID& frameworkId, + const MasterInfo& masterInfo) +{ + InterpreterLock lock; + + PyObject* fid = NULL; + PyObject* minfo = NULL; + PyObject* res = NULL; + + fid = createPythonProtobuf(frameworkId, "FrameworkID"); + if (fid == NULL) { + goto cleanup; // createPythonProtobuf will have set an exception + } + + minfo = createPythonProtobuf(masterInfo, "MasterInfo"); + if (minfo == NULL) { + goto cleanup; // createPythonProtobuf will have set an exception + } + + res = PyObject_CallMethod(impl->pythonScheduler, + (char*) "registered", + (char*) "OOO", + impl, + fid, + minfo); + if (res == NULL) { + cerr << "Failed to call scheduler's registered" << endl; + goto cleanup; + } + +cleanup: + if (PyErr_Occurred()) { + PyErr_Print(); + driver->abort(); + } + Py_XDECREF(fid); + Py_XDECREF(minfo); + Py_XDECREF(res); +} + + +void ProxyScheduler::reregistered(SchedulerDriver* driver, + const MasterInfo& masterInfo) +{ + InterpreterLock lock; + + PyObject* minfo = NULL; + PyObject* res = NULL; + + minfo = createPythonProtobuf(masterInfo, "MasterInfo"); + if (minfo == NULL) { + goto cleanup; // createPythonProtobuf will have set an exception + } + + res = PyObject_CallMethod(impl->pythonScheduler, + (char*) "reregistered", + (char*) "OO", + impl, + minfo); + if (res == NULL) { + cerr << "Failed to call scheduler's reregistered" << endl; + goto cleanup; + } + +cleanup: + if (PyErr_Occurred()) { + PyErr_Print(); + driver->abort(); + } + Py_XDECREF(minfo); + Py_XDECREF(res); +} + + +void ProxyScheduler::disconnected(SchedulerDriver* driver) +{ + InterpreterLock lock; + + PyObject* res = NULL; + + res = PyObject_CallMethod(impl->pythonScheduler, + (char*) "disconnected", + (char*) "O", + impl); + if (res == NULL) { + cerr << "Failed to call scheduler's disconnected" << endl; + goto cleanup; + } + +cleanup: + if (PyErr_Occurred()) { + PyErr_Print(); + driver->abort(); + } + Py_XDECREF(res); +} + + +void ProxyScheduler::resourceOffers(SchedulerDriver* driver, + const vector<Offer>& offers) +{ + InterpreterLock lock; + + PyObject* list = NULL; + PyObject* res = NULL; + + list = PyList_New(offers.size()); + if (list == NULL) { + goto cleanup; + } + for (size_t i = 0; i < offers.size(); i++) { + PyObject* offer = createPythonProtobuf(offers[i], "Offer"); + if (offer == NULL) { + goto cleanup; + } + PyList_SetItem(list, i, offer); // Steals the reference to offer + } + + res = PyObject_CallMethod(impl->pythonScheduler, + (char*) "resourceOffers", + (char*) "OO", + impl, + list); + + if (res == NULL) { + cerr << "Failed to call scheduler's resourceOffer" << endl; + goto cleanup; + } + +cleanup: + if (PyErr_Occurred()) { + PyErr_Print(); + driver->abort(); + } + Py_XDECREF(list); + Py_XDECREF(res); +} + + +void ProxyScheduler::offerRescinded(SchedulerDriver* driver, + const OfferID& offerId) +{ + InterpreterLock lock; + + PyObject* oid = NULL; + PyObject* res = NULL; + + oid = createPythonProtobuf(offerId, "OfferID"); + if (oid == NULL) { + goto cleanup; // createPythonProtobuf will have set an exception + } + + res = PyObject_CallMethod(impl->pythonScheduler, + (char*) "offerRescinded", + (char*) "OO", + impl, + oid); + if (res == NULL) { + cerr << "Failed to call scheduler's offerRescinded" << endl; + goto cleanup; + } + +cleanup: + if (PyErr_Occurred()) { + PyErr_Print(); + driver->abort(); + } + Py_XDECREF(oid); + Py_XDECREF(res); +} + + +void ProxyScheduler::statusUpdate(SchedulerDriver* driver, + const TaskStatus& status) +{ + InterpreterLock lock; + + PyObject* stat = NULL; + PyObject* res = NULL; + + stat = createPythonProtobuf(status, "TaskStatus"); + if (stat == NULL) { + goto cleanup; // createPythonProtobuf will have set an exception + } + + res = PyObject_CallMethod(impl->pythonScheduler, + (char*) "statusUpdate", + (char*) "OO", + impl, + stat); + if (res == NULL) { + cerr << "Failed to call scheduler's statusUpdate" << endl; + goto cleanup; + } + +cleanup: + if (PyErr_Occurred()) { + PyErr_Print(); + driver->abort(); + } + Py_XDECREF(stat); + Py_XDECREF(res); +} + + +void ProxyScheduler::frameworkMessage(SchedulerDriver* driver, + const ExecutorID& executorId, + const SlaveID& slaveId, + const string& data) +{ + InterpreterLock lock; + + PyObject* eid = NULL; + PyObject* sid = NULL; + PyObject* res = NULL; + + eid = createPythonProtobuf(executorId, "ExecutorID"); + if (eid == NULL) { + goto cleanup; // createPythonProtobuf will have set an exception + } + + sid = createPythonProtobuf(slaveId, "SlaveID"); + if (sid == NULL) { + goto cleanup; // createPythonProtobuf will have set an exception + } + + res = PyObject_CallMethod(impl->pythonScheduler, + (char*) "frameworkMessage", + (char*) "OOOs#", + impl, + eid, + sid, + data.data(), + data.length()); + if (res == NULL) { + cerr << "Failed to call scheduler's frameworkMessage" << endl; + goto cleanup; + } + +cleanup: + if (PyErr_Occurred()) { + PyErr_Print(); + driver->abort(); + } + Py_XDECREF(eid); + Py_XDECREF(sid); + Py_XDECREF(res); +} + + +void ProxyScheduler::slaveLost(SchedulerDriver* driver, const SlaveID& slaveId) +{ + InterpreterLock lock; + + PyObject* sid = NULL; + PyObject* res = NULL; + + sid = createPythonProtobuf(slaveId, "SlaveID"); + if (sid == NULL) { + goto cleanup; // createPythonProtobuf will have set an exception + } + + res = PyObject_CallMethod(impl->pythonScheduler, + (char*) "slaveLost", + (char*) "OO", + impl, + sid); + if (res == NULL) { + cerr << "Failed to call scheduler's slaveLost" << endl; + goto cleanup; + } + +cleanup: + if (PyErr_Occurred()) { + PyErr_Print(); + driver->abort(); + } + Py_XDECREF(sid); + Py_XDECREF(res); +} + + +void ProxyScheduler::executorLost(SchedulerDriver* driver, + const ExecutorID& executorId, + const SlaveID& slaveId, + int status) +{ + InterpreterLock lock; + + PyObject* executorIdObj = NULL; + PyObject* slaveIdObj = NULL; + PyObject* res = NULL; + + executorIdObj = createPythonProtobuf(executorId, "ExecutorID"); + slaveIdObj = createPythonProtobuf(slaveId, "SlaveID"); + + if (executorIdObj == NULL || slaveIdObj == NULL) { + goto cleanup; // createPythonProtobuf will have set an exception + } + + res = PyObject_CallMethod(impl->pythonScheduler, + (char*) "executorLost", + (char*) "OOOi", + impl, + executorIdObj, + slaveIdObj, + status); + if (res == NULL) { + cerr << "Failed to call scheduler's executorLost" << endl; + goto cleanup; + } + +cleanup: + if (PyErr_Occurred()) { + PyErr_Print(); + driver->abort(); + } + Py_XDECREF(executorIdObj); + Py_XDECREF(slaveIdObj); + Py_XDECREF(res); +} + + +void ProxyScheduler::error(SchedulerDriver* driver, const string& message) +{ + InterpreterLock lock; + PyObject* res = PyObject_CallMethod(impl->pythonScheduler, + (char*) "error", + (char*) "Os#", + impl, + message.data(), + message.length()); + if (res == NULL) { + cerr << "Failed to call scheduler's error" << endl; + goto cleanup; + } +cleanup: + if (PyErr_Occurred()) { + PyErr_Print(); + // No need for driver.stop(); it should stop itself + } + Py_XDECREF(res); +} + +} // namespace python { +} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/c5a68be1/src/python/native/src/mesos/native/proxy_scheduler.hpp ---------------------------------------------------------------------- diff --git a/src/python/native/src/mesos/native/proxy_scheduler.hpp b/src/python/native/src/mesos/native/proxy_scheduler.hpp new file mode 100644 index 0000000..501c574 --- /dev/null +++ b/src/python/native/src/mesos/native/proxy_scheduler.hpp @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PROXY_SCHEDULER_HPP +#define PROXY_SCHEDULER_HPP + +// Python.h must be included before standard headers. +// See: http://docs.python.org/2/c-api/intro.html#include-files +#include <Python.h> + +#include <string> +#include <vector> + +#include <mesos/scheduler.hpp> + +namespace mesos { +namespace python { + +struct MesosSchedulerDriverImpl; + +/** + * Proxy Scheduler implementation that will call into Python. + */ +class ProxyScheduler : public Scheduler +{ +public: + explicit ProxyScheduler(MesosSchedulerDriverImpl* _impl) : impl(_impl) {} + + virtual ~ProxyScheduler() {} + + virtual void registered(SchedulerDriver* driver, + const FrameworkID& frameworkId, + const MasterInfo& masterInfo); + virtual void reregistered(SchedulerDriver* driver, + const MasterInfo& masterInfo); + virtual void disconnected(SchedulerDriver* driver); + virtual void resourceOffers(SchedulerDriver* driver, + const std::vector<Offer>& offers); + virtual void offerRescinded(SchedulerDriver* driver, const OfferID& offerId); + virtual void statusUpdate(SchedulerDriver* driver, const TaskStatus& status); + virtual void frameworkMessage(SchedulerDriver* driver, + const ExecutorID& executorId, + const SlaveID& slaveId, + const std::string& data); + virtual void slaveLost(SchedulerDriver* driver, const SlaveID& slaveId); + virtual void executorLost(SchedulerDriver* driver, + const ExecutorID& executorId, + const SlaveID& slaveId, + int status); + virtual void error(SchedulerDriver* driver, const std::string& message); + +private: + MesosSchedulerDriverImpl* impl; +}; + +} // namespace python { +} // namespace mesos { + +#endif // PROXY_SCHEDULER_HPP http://git-wip-us.apache.org/repos/asf/mesos/blob/c5a68be1/src/python/protocol/setup.py.in ---------------------------------------------------------------------- diff --git a/src/python/protocol/setup.py.in b/src/python/protocol/setup.py.in new file mode 100644 index 0000000..72cb770 --- /dev/null +++ b/src/python/protocol/setup.py.in @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +config = { + 'name': 'mesos.protocol', + 'version': '@PACKAGE_VERSION@', + 'description': 'Mesos protobuf definitions', + 'author': 'Apache Mesos', + 'author_email': '[email protected]', + 'url': 'http://pypi.python.org/pypi/mesos.protocol', + 'namespace_packages': [ 'mesos' ], + 'packages': [ 'mesos', 'mesos.protocol' ], + 'package_dir': { '': 'src' }, + 'install_requires': [ 'protobuf>=2.5.0,<3' ], + 'license': 'Apache 2.0', + 'keywords': 'mesos', + 'classifiers': [ ] +} + +from setuptools import setup + +setup(**config) http://git-wip-us.apache.org/repos/asf/mesos/blob/c5a68be1/src/python/protocol/src/mesos/__init__.py ---------------------------------------------------------------------- diff --git a/src/python/protocol/src/mesos/__init__.py b/src/python/protocol/src/mesos/__init__.py new file mode 100644 index 0000000..f48ad10 --- /dev/null +++ b/src/python/protocol/src/mesos/__init__.py @@ -0,0 +1,6 @@ +# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages +try: + __import__('pkg_resources').declare_namespace(__name__) +except ImportError: + from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) http://git-wip-us.apache.org/repos/asf/mesos/blob/c5a68be1/src/python/protocol/src/mesos/protocol/__init__.py ---------------------------------------------------------------------- diff --git a/src/python/protocol/src/mesos/protocol/__init__.py b/src/python/protocol/src/mesos/protocol/__init__.py new file mode 100644 index 0000000..635f0d9 --- /dev/null +++ b/src/python/protocol/src/mesos/protocol/__init__.py @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. http://git-wip-us.apache.org/repos/asf/mesos/blob/c5a68be1/src/python/setup.py.in ---------------------------------------------------------------------- diff --git a/src/python/setup.py.in b/src/python/setup.py.in index 60add3b..304c4bf 100644 --- a/src/python/setup.py.in +++ b/src/python/setup.py.in @@ -1,148 +1,38 @@ -import sys - -try: - import setuptools as distutils -except ImportError: - print 'Could not find setuptools, bailing.' - sys.exit(1) - -import errno -import glob -import os -import shutil - -from distutils.core import setup, Extension - -abs_top_srcdir = '@abs_top_srcdir@' -abs_top_builddir = '@abs_top_builddir@' - -src_python_dist = os.path.join('src', 'python', 'dist') -src_python_native = os.path.join('src', 'python', 'native') - -leveldb = os.path.join('3rdparty', 'leveldb') -zookeeper = os.path.join('3rdparty', 'zookeeper-3.4.5', 'src', 'c') -libprocess = os.path.join('3rdparty', 'libprocess') - -# Even though a statically compiled libprocess should include glog, -# libev, gperftools, and protobuf before installation this isn't the -# case, so while a libtool managed build will correctly pull in these -# libraries when building the final result, we need to explicitly -# include them here (or more precisely, down where we actually include -# libev.a and libprofiler.a). -glog = os.path.join(libprocess, '3rdparty', 'glog-0.3.3') -libev = os.path.join(libprocess, '3rdparty', 'libev-4.15') -gperftools = os.path.join(libprocess, '3rdparty', 'gperftools-2.0') -protobuf = os.path.join(libprocess, '3rdparty', 'protobuf-2.5.0') - -# We need to execute from the same directory as this script. -os.chdir(os.path.abspath(os.path.dirname(__file__))) - -# Copy the native source files if we are building in a remote build -# directory. This is necessary because the sources list must be -# relative to the directory where this script resides. -# $ cp -rf abs_top_srcdir/native abs_top_builddir/native -if os.path.abspath(abs_top_srcdir) != os.path.abspath(abs_top_builddir): - try: - shutil.rmtree(os.path.join(abs_top_builddir, src_python_native), ignore_errors=True) - shutil.copytree(os.path.join(abs_top_srcdir, src_python_native), - os.path.join(abs_top_builddir, src_python_native)) - except OSError, e: - if e.errno != errno.EEXIST: - raise - - -# Build the list of source files. Note that each source must be -# relative to our current directory (where this script lives). -SOURCES = [ - os.path.join('native', file) - for file in os.listdir(os.path.join(abs_top_srcdir, src_python_native)) - if file.endswith('.cpp') -] - -INCLUDE_DIRS = [ - os.path.join(abs_top_srcdir, 'include'), - os.path.join(abs_top_builddir, 'include'), - # Needed for the *.pb.h protobuf includes. - os.path.join(abs_top_builddir, 'include', 'mesos'), - os.path.join(abs_top_builddir, 'src'), - os.path.join(abs_top_builddir, src_python_native), - os.path.join(abs_top_builddir, protobuf, 'src'), -] - -LIBRARY_DIRS = [] - -EXTRA_OBJECTS = [ - os.path.join(abs_top_builddir, 'src', '.libs', 'libmesos_no_3rdparty.a'), - os.path.join(abs_top_builddir, protobuf, 'src', '.libs', 'libprotobuf.a'), - os.path.join(abs_top_builddir, libprocess, '.libs', 'libprocess.a'), - os.path.join(abs_top_builddir, glog, '.libs', 'libglog.a') -] - -# For leveldb, we need to check for the presence of libleveldb.a, since -# it is possible to disable leveldb inside mesos. -libev = os.path.join(abs_top_builddir, libev, '.libs', 'libev.a') -libleveldb = os.path.join(abs_top_builddir, leveldb, 'libleveldb.a') -libzookeeper = os.path.join( - abs_top_builddir, zookeeper, '.libs', 'libzookeeper_mt.a') - -if os.path.exists(libleveldb): - EXTRA_OBJECTS.append(libleveldb) -else: - EXTRA_OBJECTS.append('-lleveldb') - -if os.path.exists(libzookeeper): - EXTRA_OBJECTS.append(libzookeeper) -else: - EXTRA_OBJECTS.append('-lzookeeper_mt') - -if os.path.exists(libev): - EXTRA_OBJECTS.append(libev) -else: - EXTRA_OBJECTS.append('-lev') - -# For gperftools, we need to check for the presence of libprofiler.a, since -# it is possible to disable perftools inside libprocess. -libprofiler = os.path.join( - abs_top_builddir, gperftools, '.libs', 'libprofiler.a') - -if os.path.exists(libprofiler): - EXTRA_OBJECTS.append(libprofiler) - -EXTRA_LINK_ARGS = [] - -# Add any flags from LDFLAGS. -if 'LDFLAGS' in os.environ: - for flag in os.environ['LDFLAGS'].split(): - EXTRA_LINK_ARGS.append(flag) - -# Add any libraries from LIBS. -if 'LIBS' in os.environ: - for library in os.environ['LIBS'].split(): - EXTRA_LINK_ARGS.append(library) - -DEPENDS = [ - os.path.join(abs_top_srcdir, 'src', 'python', source) - for source in SOURCES -] - -# Note that we add EXTRA_OBJECTS to our dependency list to make sure -# that we rebuild this module when one of them changes (e.g., -# libprocess). -mesos_module = \ - Extension('_mesos', - sources = SOURCES, - include_dirs = INCLUDE_DIRS, - library_dirs = LIBRARY_DIRS, - extra_objects = EXTRA_OBJECTS, - extra_link_args = EXTRA_LINK_ARGS, - depends = EXTRA_OBJECTS, - language = 'c++', - ) - -setup(name = 'mesos', - version = '@PACKAGE_VERSION@', - description = 'Mesos', - package_dir = { '': 'src' }, - packages = ['.'], - install_requires = ['protobuf>=2.5.0'], - ext_modules = [mesos_module]) +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +config = { + 'name': 'mesos', + 'version': '@PACKAGE_VERSION@', + 'description': 'Python bindings for mesos', + 'author': 'Apache Mesos', + 'author_email': '[email protected]', + 'url': 'http://pypi.python.org/pypi/mesos', + 'namespace_packages': [ 'mesos' ], + 'packages': [ 'mesos' ], + 'package_dir': { '': 'src' }, + 'install_requires': [ + 'mesos.interface', + 'mesos.native' + ], + 'license': 'Apache 2.0', + 'keywords': 'mesos', + 'classifiers': [ ] +} + +from setuptools import setup + +setup(**config) http://git-wip-us.apache.org/repos/asf/mesos/blob/c5a68be1/src/python/src/mesos.py ---------------------------------------------------------------------- diff --git a/src/python/src/mesos.py b/src/python/src/mesos.py deleted file mode 100644 index 0152ab4..0000000 --- a/src/python/src/mesos.py +++ /dev/null @@ -1,361 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# See include/mesos/scheduler.hpp, include/mesos/executor.hpp and -# include/mesos/mesos.proto for more information documenting this -# interface. - -"""Python bindings for Mesos.""" - -from __future__ import print_function - -import sys -import _mesos - - -# Alias the implementations from _mesos. - -# TODO(wickman): Make Mesos{Scheduler,Executor}DriverImpl inherit from the -# superclasses defined here. -MesosSchedulerDriver = _mesos.MesosSchedulerDriverImpl -MesosExecutorDriver = _mesos.MesosExecutorDriverImpl - - -__all__ = ( - 'Executor', - 'ExecutorDriver' - 'MesosExecutorDriver', - 'MesosSchedulerDriver', - 'Scheduler', - 'SchedulerDriver', -) - - -class Scheduler(object): - """ - Base class for Mesos schedulers. Users' schedulers should extend this - class to get default implementations of methods they don't override. - """ - - def registered(self, driver, frameworkId, masterInfo): - """ - Invoked when the scheduler successfully registers with a Mesos master. - It is called with the frameworkId, a unique ID generated by the - master, and the masterInfo which is information about the master - itself. - """ - - def reregistered(self, driver, masterInfo): - """ - Invoked when the scheduler re-registers with a newly elected Mesos - master. This is only called when the scheduler has previously been - registered. masterInfo contains information about the newly elected - master. - """ - - def disconnected(self, driver): - """ - Invoked when the scheduler becomes disconnected from the master, e.g. - the master fails and another is taking over. - """ - - def resourceOffers(self, driver, offers): - """ - Invoked when resources have been offered to this framework. A single - offer will only contain resources from a single slave. Resources - associated with an offer will not be re-offered to _this_ framework - until either (a) this framework has rejected those resources (see - SchedulerDriver.launchTasks) or (b) those resources have been - rescinded (see Scheduler.offerRescinded). Note that resources may be - concurrently offered to more than one framework at a time (depending - on the allocator being used). In that case, the first framework to - launch tasks using those resources will be able to use them while the - other frameworks will have those resources rescinded (or if a - framework has already launched tasks with those resources then those - tasks will fail with a TASK_LOST status and a message saying as much). - """ - - def offerRescinded(self, driver, offerId): - """ - Invoked when an offer is no longer valid (e.g., the slave was lost or - another framework used resources in the offer.) If for whatever reason - an offer is never rescinded (e.g., dropped message, failing over - framework, etc.), a framwork that attempts to launch tasks using an - invalid offer will receive TASK_LOST status updats for those tasks - (see Scheduler.resourceOffers). - """ - - def statusUpdate(self, driver, status): - """ - Invoked when the status of a task has changed (e.g., a slave is lost - and so the task is lost, a task finishes and an executor sends a - status update saying so, etc.) Note that returning from this callback - acknowledges receipt of this status update. If for whatever reason - the scheduler aborts during this callback (or the process exits) - another status update will be delivered. Note, however, that this is - currently not true if the slave sending the status update is lost or - fails during that time. - """ - - def frameworkMessage(self, driver, executorId, slaveId, message): - """ - Invoked when an executor sends a message. These messages are best - effort; do not expect a framework message to be retransmitted in any - reliable fashion. - """ - - def slaveLost(self, driver, slaveId): - """ - Invoked when a slave has been determined unreachable (e.g., machine - failure, network partition.) Most frameworks will need to reschedule - any tasks launched on this slave on a new slave. - """ - - def executorLost(self, driver, executorId, slaveId, status): - """ - Invoked when an executor has exited/terminated. Note that any tasks - running will have TASK_LOST status updates automatically generated. - """ - - def error(self, driver, message): - """ - Invoked when there is an unrecoverable error in the scheduler or - scheduler driver. The driver will be aborted BEFORE invoking this - callback. - """ - print("Error from Mesos: %s " % message, file=sys.stderr) - - -class SchedulerDriver(object): - """ - Interface for Mesos scheduler drivers. Users may wish to implement this - class in mock objects for tests. - """ - def start(self): - """ - Starts the scheduler driver. This needs to be called before any other - driver calls are made. - """ - - def stop(self, failover=False): - """ - Stops the scheduler driver. If the 'failover' flag is set to - false then it is expected that this framework will never - reconnect to Mesos. So Mesos will unregister the framework - and shutdown all its tasks and executors. If 'failover' is true, - all executors and tasks will remain running (for some framework - specific failover timeout) allowing the scheduler to reconnect - (possibly in the same process, or from a different process, for - example, on a different machine). - """ - - def abort(self): - """ - Aborts the driver so that no more callbacks can be made to the - scheduler. The semantics of abort and stop have deliberately been - separated so that code can detect an aborted driver (i.e., via the - return status of SchedulerDriver.join), and instantiate and start - another driver if desired (from within the same process.) - """ - - def join(self): - """ - Waits for the driver to be stopped or aborted, possibly blocking the - current thread indefinitely. The return status of this function can - be used to determine if the driver was aborted (see mesos.proto for a - description of Status). - """ - - def run(self): - """ - Starts and immediately joins (i.e., blocks on) the driver. - """ - - def requestResources(self, requests): - """ - Requests resources from Mesos (see mesos.proto for a description of - Request and how, for example, to request resources from specific - slaves.) Any resources available are offered to the framework via - Scheduler.resourceOffers callback, asynchronously. - """ - - def launchTasks(self, offerIds, tasks, filters=None): - """ - Launches the given set of tasks. Any resources remaining (i.e., not - used by the tasks or their executors) will be considered declined. - The specified filters are applied on all unused resources (see - mesos.proto for a description of Filters.) Invoking this function with - an empty collection of tasks declines the offers in entirety (see - Scheduler.declineOffer). Note that passing a single offer is also - supported. - """ - - def killTask(self, taskId): - """ - Kills the specified task. Note that attempting to kill a task is - currently not reliable. If, for example, a scheduler fails over while - it was attempting to kill a task it will need to retry in the future. - Likewise, if unregistered / disconnected, the request will be dropped - dropped (these semantics may be changed in the future). - """ - - def declineOffer(self, offerId, filters=None): - """ - Declines an offer in its entirety and applies the specified - filters on the resources (see mesos.proto for a description of - Filters). Note that this can be done at any time, it is not - necessary to do this within the Scheduler::resourceOffers - callback. - """ - - def reviveOffers(self): - """ - Removes all filters previously set by the framework (via - launchTasks()). This enables the framework to receive offers from - those filtered slaves. - """ - - def sendFrameworkMessage(self, executorId, slaveId, data): - """ - Sends a message from the framework to one of its executors. These - messages are best effort; do not expect a framework message to be - retransmitted in any reliable fashion. - """ - - # TODO(bmahler): Add reconcileTasks! - -class Executor(object): - """ - Base class for Mesos executors. Users' executors should extend this - class to get default implementations of methods they don't override. - """ - - def registered(self, driver, executorInfo, frameworkInfo, slaveInfo): - """ - Invoked once the executor driver has been able to successfully connect - with Mesos. In particular, a scheduler can pass some data to its - executors through the FrameworkInfo.ExecutorInfo's data field. - """ - - def reregistered(self, driver, slaveInfo): - """ - Invoked when the executor re-registers with a restarted slave. - """ - - def disconnected(self, driver): - """ - Invoked when the executor becomes "disconnected" from the slave (e.g., - the slave is being restarted due to an upgrade). - """ - - def launchTask(self, driver, task): - """ - Invoked when a task has been launched on this executor (initiated via - Scheduler.launchTasks). Note that this task can be realized with a - thread, a process, or some simple computation, however, no other - callbacks will be invoked on this executor until this callback has - returned. - """ - - def killTask(self, driver, taskId): - """ - Invoked when a task running within this executor has been killed (via - SchedulerDriver.killTask). Note that no status update will be sent on - behalf of the executor, the executor is responsible for creating a new - TaskStatus (i.e., with TASK_KILLED) and invoking ExecutorDriver's - sendStatusUpdate. - """ - - def frameworkMessage(self, driver, message): - """ - Invoked when a framework message has arrived for this executor. These - messages are best effort; do not expect a framework message to be - retransmitted in any reliable fashion. - """ - - def shutdown(self, driver): - """ - Invoked when the executor should terminate all of its currently - running tasks. Note that after Mesos has determined that an executor - has terminated any tasks that the executor did not send terminal - status updates for (e.g., TASK_KILLED, TASK_FINISHED, TASK_FAILED, - etc) a TASK_LOST status update will be created. - """ - - def error(self, driver, message): - """ - Invoked when a fatal error has occured with the executor and/or - executor driver. The driver will be aborted BEFORE invoking this - callback. - """ - print("Error from Mesos: %s" % message, file=sys.stderr) - - - -class ExecutorDriver(object): - """ - Interface for Mesos executor drivers. Users may wish to extend this - class in mock objects for tests. - """ - def start(self): - """ - Starts the executor driver. This needs to be called before any other - driver calls are made. - """ - - def stop(self): - """ - Stops the executor driver. - """ - - def abort(self): - """ - Aborts the driver so that no more callbacks can be made to the - executor. The semantics of abort and stop have deliberately been - separated so that code can detect an aborted driver (i.e., via the - return status of ExecutorDriver.join), and instantiate and start - another driver if desired (from within the same process, although this - functionality is currently not supported for executors). - """ - - def join(self): - """ - Waits for the driver to be stopped or aborted, possibly blocking the - current thread indefinitely. The return status of this function can - be used to determine if the driver was aborted (see mesos.proto for a - description of Status). - """ - - def run(self): - """ - Starts and immediately joins (i.e., blocks on) the driver. - """ - - def sendStatusUpdate(self, status): - """ - Sends a status update to the framework scheduler, retrying as - necessary until an acknowledgement has been received or the executor - is terminated (in which case, a TASK_LOST status update will be sent). - See Scheduler.statusUpdate for more information about status update - acknowledgements. - """ - - def sendFrameworkMessage(self, data): - """ - Sends a message to the framework scheduler. These messages are best - effort; do not expect a framework message to be retransmitted in any - reliable fashion. - """ http://git-wip-us.apache.org/repos/asf/mesos/blob/c5a68be1/src/python/src/mesos/__init__.py ---------------------------------------------------------------------- diff --git a/src/python/src/mesos/__init__.py b/src/python/src/mesos/__init__.py new file mode 100644 index 0000000..f48ad10 --- /dev/null +++ b/src/python/src/mesos/__init__.py @@ -0,0 +1,6 @@ +# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages +try: + __import__('pkg_resources').declare_namespace(__name__) +except ImportError: + from pkgutil import extend_path + __path__ = extend_path(__path__, __name__)
