Vo Minh Thu (OpenERP) has proposed merging
lp:~openerp-dev/openobject-server/6.1-gunicorn-signaling-vmt into
lp:openobject-server/6.1.
Requested reviews:
OpenERP Core Team (openerp)
For more details, see:
https://code.launchpad.net/~openerp-dev/openobject-server/6.1-gunicorn-signaling-vmt/+merge/95936
--
https://code.launchpad.net/~openerp-dev/openobject-server/6.1-gunicorn-signaling-vmt/+merge/95936
Your team OpenERP R&D Team is subscribed to branch
lp:~openerp-dev/openobject-server/6.1-gunicorn-signaling-vmt.
=== modified file 'gunicorn.conf.py'
--- gunicorn.conf.py 2012-02-10 15:25:21 +0000
+++ gunicorn.conf.py 2012-03-05 16:40:24 +0000
@@ -25,7 +25,6 @@
# Some application-wide initialization is needed.
on_starting = openerp.wsgi.core.on_starting
-when_ready = openerp.wsgi.core.when_ready
pre_request = openerp.wsgi.core.pre_request
post_request = openerp.wsgi.core.post_request
=== modified file 'openerp/__init__.py'
--- openerp/__init__.py 2011-09-27 16:51:33 +0000
+++ openerp/__init__.py 2012-03-05 16:40:24 +0000
@@ -45,5 +45,12 @@
import workflow
import wsgi
+# Is the server running in multi-process mode (e.g. behind Gunicorn).
+# If this is True, the processes have to communicate some events,
+# e.g. database update or cache invalidation. Each process has also
+# its own copy of the data structure and we don't need to care about
+# locks between threads.
+multi_process = False
+
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
=== modified file 'openerp/addons/base/base.sql'
--- openerp/addons/base/base.sql 2012-01-30 09:52:38 +0000
+++ openerp/addons/base/base.sql 2012-03-05 16:40:24 +0000
@@ -347,6 +347,16 @@
res_id integer, primary key(id)
);
+-- Inter-process signaling:
+-- The `base_registry_signaling` sequence indicates the whole registry
+-- must be reloaded.
+-- The `base_cache_signaling sequence` indicates all caches must be
+-- invalidated (i.e. cleared).
+CREATE SEQUENCE base_registry_signaling INCREMENT BY 1 START WITH 1;
+SELECT nextval('base_registry_signaling');
+CREATE SEQUENCE base_cache_signaling INCREMENT BY 1 START WITH 1;
+SELECT nextval('base_cache_signaling');
+
---------------------------------
-- Users
---------------------------------
=== modified file 'openerp/addons/base/ir/ir_ui_menu.py'
--- openerp/addons/base/ir/ir_ui_menu.py 2012-02-10 08:26:37 +0000
+++ openerp/addons/base/ir/ir_ui_menu.py 2012-03-05 16:40:24 +0000
@@ -42,7 +42,7 @@
def __init__(self, *args, **kwargs):
self.cache_lock = threading.RLock()
- self.clear_cache()
+ self._cache = {}
r = super(ir_ui_menu, self).__init__(*args, **kwargs)
self.pool.get('ir.model.access').register_cache_clearing_method(self._name, 'clear_cache')
return r
@@ -50,6 +50,10 @@
def clear_cache(self):
with self.cache_lock:
# radical but this doesn't frequently happen
+ if self._cache:
+ # Normally this is done by openerp.tools.ormcache
+ # but since we do not use it, set it by ourself.
+ self.pool._any_cache_cleared = True
self._cache = {}
def _filter_visible_menus(self, cr, uid, ids, context=None):
=== modified file 'openerp/addons/base/module/module.py'
--- openerp/addons/base/module/module.py 2012-02-14 15:18:46 +0000
+++ openerp/addons/base/module/module.py 2012-03-05 16:40:24 +0000
@@ -30,6 +30,7 @@
import zipfile
import zipimport
+import openerp
import openerp.modules as addons
import pooler
import release
@@ -344,6 +345,7 @@
if to_install_ids:
self.button_install(cr, uid, to_install_ids, context=context)
+ openerp.modules.registry.RegistryManager.signal_registry_change(cr.dbname)
return dict(ACTION_DICT, name=_('Install'))
def button_immediate_install(self, cr, uid, ids, context=None):
=== modified file 'openerp/cron.py'
--- openerp/cron.py 2012-01-24 11:07:30 +0000
+++ openerp/cron.py 2012-03-05 16:40:24 +0000
@@ -204,9 +204,12 @@
_logger.warning("Connection pool size (%s) is set lower than max number of cron threads (%s), "
"this may cause trouble if you reach that number of parallel cron tasks.",
db_maxconn, _thread_slots)
- t = threading.Thread(target=runner, name="openerp.cron.master_thread")
- t.setDaemon(True)
- t.start()
- _logger.debug("Master cron daemon started!")
+ if _thread_slots:
+ t = threading.Thread(target=runner, name="openerp.cron.master_thread")
+ t.setDaemon(True)
+ t.start()
+ _logger.debug("Master cron daemon started!")
+ else:
+ _logger.info("No master cron daemon (0 workers needed).")
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
=== modified file 'openerp/modules/registry.py'
--- openerp/modules/registry.py 2012-01-24 12:42:52 +0000
+++ openerp/modules/registry.py 2012-03-05 16:40:24 +0000
@@ -51,6 +51,18 @@
self.db_name = db_name
self.db = openerp.sql_db.db_connect(db_name)
+ # Inter-process signaling (used only when openerp.multi_process is True):
+ # The `base_registry_signaling` sequence indicates the whole registry
+ # must be reloaded.
+ # The `base_cache_signaling sequence` indicates all caches must be
+ # invalidated (i.e. cleared).
+ self.base_registry_signaling_sequence = 1
+ self.base_cache_signaling_sequence = 1
+
+ # Flag indicating if at least one model cache has been cleared.
+ # Useful only in a multi-process context.
+ self._any_cache_cleared = False
+
cr = self.db.cursor()
has_unaccent = openerp.modules.db.has_unaccent(cr)
if openerp.tools.config['unaccent'] and not has_unaccent:
@@ -114,6 +126,19 @@
"""
for model in self.models.itervalues():
model.clear_caches()
+ # Special case for ir_ui_menu which does not use openerp.tools.ormcache.
+ ir_ui_menu = self.models.get('ir.ui.menu')
+ if ir_ui_menu:
+ ir_ui_menu.clear_cache()
+
+
+ # Useful only in a multi-process context.
+ def reset_any_cache_cleared(self):
+ self._any_cache_cleared = False
+
+ # Useful only in a multi-process context.
+ def any_cache_cleared(self):
+ return self._any_cache_cleared
class RegistryManager(object):
""" Model registries manager.
@@ -215,5 +240,70 @@
if db_name in cls.registries:
cls.registries[db_name].clear_caches()
+ @classmethod
+ def check_registry_signaling(cls, db_name):
+ if openerp.multi_process and db_name in cls.registries:
+ # Check if the model registry must be reloaded (e.g. after the
+ # database has been updated by another process).
+ registry = cls.get(db_name, pooljobs=False)
+ cr = registry.db.cursor()
+ registry_reloaded = False
+ try:
+ cr.execute('SELECT last_value FROM base_registry_signaling')
+ r = cr.fetchone()[0]
+ if registry.base_registry_signaling_sequence != r:
+ _logger.info("Reloading the model registry after database signaling.")
+ # Don't run the cron in the Gunicorn worker.
+ registry = cls.new(db_name, pooljobs=False)
+ registry.base_registry_signaling_sequence = r
+ registry_reloaded = True
+ finally:
+ cr.close()
+
+ # Check if the model caches must be invalidated (e.g. after a write
+ # occured on another process). Don't clear right after a registry
+ # has been reload.
+ cr = openerp.sql_db.db_connect(db_name).cursor()
+ try:
+ cr.execute('SELECT last_value FROM base_cache_signaling')
+ r = cr.fetchone()[0]
+ if registry.base_cache_signaling_sequence != r and not registry_reloaded:
+ _logger.info("Invalidating all model caches after database signaling.")
+ registry.base_cache_signaling_sequence = r
+ registry.clear_caches()
+ registry.reset_any_cache_cleared()
+ finally:
+ cr.close()
+
+ @classmethod
+ def signal_caches_change(cls, db_name):
+ if openerp.multi_process and db_name in cls.registries:
+ # Check the registries if any cache has been cleared and signal it
+ # through the database to other processes.
+ registry = cls.get(db_name, pooljobs=False)
+ if registry.any_cache_cleared():
+ _logger.info("At least one model cache has been cleare, signaling through the database.")
+ cr = registry.db.cursor()
+ r = 1
+ try:
+ cr.execute("select nextval('base_cache_signaling')")
+ r = cr.fetchone()[0]
+ finally:
+ cr.close()
+ registry.base_cache_signaling_sequence = r
+ registry.reset_any_cache_cleared()
+
+ @classmethod
+ def signal_registry_change(cls, db_name):
+ if openerp.multi_process and db_name in cls.registries:
+ registry = cls.get(db_name, pooljobs=False)
+ cr = registry.db.cursor()
+ r = 1
+ try:
+ cr.execute("select nextval('base_registry_signaling')")
+ r = cr.fetchone()[0]
+ finally:
+ cr.close()
+ registry.base_registry_signaling_sequence = r
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
=== modified file 'openerp/osv/orm.py'
--- openerp/osv/orm.py 2012-02-23 17:58:51 +0000
+++ openerp/osv/orm.py 2012-03-05 16:40:24 +0000
@@ -2389,6 +2389,7 @@
try:
getattr(self, '_ormcache')
self._ormcache = {}
+ self.pool._any_cache_cleared = True
except AttributeError:
pass
=== modified file 'openerp/service/web_services.py'
--- openerp/service/web_services.py 2012-02-14 19:30:30 +0000
+++ openerp/service/web_services.py 2012-03-05 16:40:24 +0000
@@ -568,8 +568,10 @@
raise NameError("Method not available %s" % method)
security.check(db,uid,passwd)
assert openerp.osv.osv.service, "The object_proxy class must be started with start_object_proxy."
+ openerp.modules.registry.RegistryManager.check_registry_signaling(db)
fn = getattr(openerp.osv.osv.service, method)
res = fn(db, uid, *params)
+ openerp.modules.registry.RegistryManager.signal_caches_change(db)
return res
@@ -649,8 +651,10 @@
if method not in ['report', 'report_get', 'render_report']:
raise KeyError("Method not supported %s" % method)
security.check(db,uid,passwd)
+ openerp.modules.registry.RegistryManager.check_registry_signaling(db)
fn = getattr(self, 'exp_' + method)
res = fn(db, uid, *params)
+ openerp.modules.registry.RegistryManager.signal_caches_change(db)
return res
def exp_render_report(self, db, uid, object, ids, datas=None, context=None):
=== modified file 'openerp/tools/cache.py'
--- openerp/tools/cache.py 2011-11-22 08:58:48 +0000
+++ openerp/tools/cache.py 2012-03-05 16:40:24 +0000
@@ -57,10 +57,12 @@
try:
key = args[self.skiparg-2:]
del d[key]
+ self2.pool._any_cache_cleared = True
except KeyError:
pass
else:
d.clear()
+ self2.pool._any_cache_cleared = True
class ormcache_multi(ormcache):
def __init__(self, skiparg=2, size=8192, multi=3):
=== modified file 'openerp/wsgi/core.py'
--- openerp/wsgi/core.py 2012-02-21 18:52:47 +0000
+++ openerp/wsgi/core.py 2012-03-05 16:40:24 +0000
@@ -447,7 +447,7 @@
The WSGI server can be shutdown with stop_server() below.
"""
- threading.Thread(target=serve).start()
+ threading.Thread(name='WSGI server', target=serve).start()
def stop_server():
""" Initiate the shutdown of the WSGI server.
@@ -465,7 +465,7 @@
def on_starting(server):
global arbiter_pid
arbiter_pid = os.getpid() # TODO check if this is true even after replacing the executable
- #openerp.tools.cache = kill_workers_cache
+ openerp.multi_process = True # Yay!
openerp.netsvc.init_logger()
openerp.osv.osv.start_object_proxy()
openerp.service.web_services.start_web_services()
@@ -482,11 +482,6 @@
Maybe you forgot to add those addons in your addons_path configuration."""
_logger.exception('Failed to load server-wide module `%s`.%s', m, msg)
-# Install our own signal handler on the master process.
-def when_ready(server):
- # Hijack gunicorn's SIGWINCH handling; we can choose another one.
- signal.signal(signal.SIGWINCH, make_winch_handler(server))
-
# Install limits on virtual memory and CPU time consumption.
def pre_request(worker, req):
import os
@@ -514,30 +509,9 @@
'too high, rebooting the worker.')
worker.alive = False # Commit suicide after the request.
-# Our signal handler will signal a SGIQUIT to all workers.
-def make_winch_handler(server):
- def handle_winch(sig, fram):
- server.kill_workers(signal.SIGQUIT) # This is gunicorn specific.
- return handle_winch
-
# SIGXCPU (exceeded CPU time) signal handler will raise an exception.
def time_expired(n, stack):
_logger.info('CPU time limit exceeded.')
raise Exception('CPU time limit exceeded.') # TODO one of openerp.exception
-# Kill gracefuly the workers (e.g. because we want to clear their cache).
-# This is done by signaling a SIGWINCH to the master process, so it can be
-# called by the workers themselves.
-def kill_workers():
- try:
- os.kill(arbiter_pid, signal.SIGWINCH)
- except OSError, e:
- if e.errno == errno.ESRCH: # no such pid
- return
- raise
-
-class kill_workers_cache(openerp.tools.ormcache):
- def clear(self, dbname, *args, **kwargs):
- kill_workers()
-
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
_______________________________________________
Mailing list: https://launchpad.net/~openerp-dev-gtk
Post to : [email protected]
Unsubscribe : https://launchpad.net/~openerp-dev-gtk
More help : https://help.launchpad.net/ListHelp
