changeset a83631f4ba67 in trytond:default
details: https://hg.tryton.org/trytond?cmd=changeset;node=a83631f4ba67
description:
Make Cache transactional
We implement sync, commit and rollback to follow transaction semantic.
When a transaction clears a cache, the global cache is cleared only
when the
transaction is committed meanwhile the transaction will use a cache for
itself
only.
We remove _nocache option because the sync, commit and rollback operate
only
for the transaction.
issue7975
review53521002
diffstat:
CHANGELOG | 2 +
doc/ref/cache.rst | 12 +-
trytond/admin.py | 2 +-
trytond/cache.py | 135 ++++++++++++++++++++++-----------------
trytond/protocols/dispatcher.py | 2 +-
trytond/tests/test_cache.py | 54 +++++++++++++++-
trytond/tests/test_tryton.py | 19 ++---
trytond/transaction.py | 22 ++---
8 files changed, 155 insertions(+), 93 deletions(-)
diffs (491 lines):
diff -r 9a7247a6df7b -r a83631f4ba67 CHANGELOG
--- a/CHANGELOG Fri Jan 18 11:32:47 2019 +0100
+++ b/CHANGELOG Fri Jan 18 11:57:11 2019 +0100
@@ -1,3 +1,5 @@
+* Remove _nocache on Transaction
+* Make Cache transactional
* Allow to search on key's value of Dict
* Allow to order Dict by key's value
* Do not store keys with null value in Dict
diff -r 9a7247a6df7b -r a83631f4ba67 doc/ref/cache.rst
--- a/doc/ref/cache.rst Fri Jan 18 11:32:47 2019 +0100
+++ b/doc/ref/cache.rst Fri Jan 18 11:57:11 2019 +0100
@@ -34,17 +34,17 @@
Clears all the keys in the cache.
-.. staticmethod:: clean(dbname)
+.. classmethod:: sync(transaction)
-Clean the cache for database `dbname`
+Synchronize cache instances using transaction.
-.. staticmethod:: reset(dbname, name)
+.. classmethod:: commit(transaction)
-Reset the `name` cache for database `dbname`
+Apply cache changes from transaction.
-.. staticmethod:: resets(dbname)
+.. classmethod:: rollback(transaction)
-Resets all the caches stored for database `dbname`
+Remove cache changes from transaction.
.. staticmethod:: drop(dbname)
diff -r 9a7247a6df7b -r a83631f4ba67 trytond/admin.py
--- a/trytond/admin.py Fri Jan 18 11:32:47 2019 +0100
+++ b/trytond/admin.py Fri Jan 18 11:57:11 2019 +0100
@@ -22,7 +22,7 @@
init = {}
for db_name in options.database_names:
init[db_name] = False
- with Transaction().start(db_name, 0, _nocache=True):
+ with Transaction().start(db_name, 0):
database = Database(db_name)
database.connect()
if options.update:
diff -r 9a7247a6df7b -r a83631f4ba67 trytond/cache.py
--- a/trytond/cache.py Fri Jan 18 11:32:47 2019 +0100
+++ b/trytond/cache.py Fri Jan 18 11:57:11 2019 +0100
@@ -1,10 +1,11 @@
# This file is part of Tryton. The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
-from threading import Lock
-from collections import OrderedDict
+from collections import OrderedDict, defaultdict
from datetime import datetime
+from weakref import WeakKeyDictionary
from sql import Table
+from sql.aggregate import Max
from sql.functions import CurrentTimestamp
from trytond.config import config
@@ -25,13 +26,14 @@
class BaseCache(object):
- _cache_instance = []
+ _instances = {}
def __init__(self, name, size_limit=1024, context=True):
self._name = name
self.size_limit = size_limit
self.context = context
- self._cache_instance.append(self)
+ assert self._name not in self._instances
+ self._instances[self._name] = self
def _key(self, key):
if self.context:
@@ -47,16 +49,16 @@
def clear(self):
raise NotImplemented
- @staticmethod
- def clean(dbname):
+ @classmethod
+ def sync(cls, transaction):
raise NotImplemented
- @staticmethod
- def reset(dbname, name):
+ @classmethod
+ def commit(cls, transaction):
raise NotImplemented
- @staticmethod
- def resets(dbname):
+ @classmethod
+ def rollback(cls, transaction):
raise NotImplemented
@classmethod
@@ -68,80 +70,80 @@
"""
A key value LRU cache with size limit.
"""
- _resets = {}
- _resets_lock = Lock()
+ _reset = WeakKeyDictionary()
_clean_last = datetime.now()
def __init__(self, name, size_limit=1024, context=True):
super(MemoryCache, self).__init__(name, size_limit, context)
- self._cache = {}
+ self._database_cache = defaultdict(lambda: LRUDict(size_limit))
+ self._transaction_cache = WeakKeyDictionary()
self._timestamp = None
- self._lock = Lock()
+
+ def _get_cache(self):
+ transaction = Transaction()
+ dbname = transaction.database.name
+ if transaction in self._reset:
+ try:
+ return self._transaction_cache[transaction]
+ except KeyError:
+ self._transaction_cache[transaction] = LRUDict(self.size_limit)
+ return self._transaction_cache[transaction]
+ else:
+ return self._database_cache[dbname]
def get(self, key, default=None):
- dbname = Transaction().database.name
key = self._key(key)
- with self._lock:
- cache = self._cache.setdefault(dbname, LRUDict(self.size_limit))
- try:
- result = cache[key] = cache.pop(key)
- return result
- except (KeyError, TypeError):
- return default
+ cache = self._get_cache()
+ try:
+ result = cache[key] = cache.pop(key)
+ return result
+ except (KeyError, TypeError):
+ return default
def set(self, key, value):
- dbname = Transaction().database.name
key = self._key(key)
- with self._lock:
- cache = self._cache.setdefault(dbname, LRUDict(self.size_limit))
- try:
- cache[key] = value
- except TypeError:
- pass
+ cache = self._get_cache()
+ try:
+ cache[key] = value
+ except TypeError:
+ pass
return value
def clear(self):
- dbname = Transaction().database.name
- Cache.reset(dbname, self._name)
- with self._lock:
- self._cache[dbname] = LRUDict(self.size_limit)
+ transaction = Transaction()
+ self._reset.setdefault(transaction, set()).add(self._name)
+ self._transaction_cache.pop(transaction, None)
@classmethod
- def clean(cls, dbname):
+ def sync(cls, transaction):
if (datetime.now() - cls._clean_last).total_seconds() < _clear_timeout:
return
- with Transaction().new_transaction(_nocache=True) as transaction,\
- transaction.connection.cursor() as cursor:
+ dbname = transaction.database.name
+ with transaction.connection.cursor() as cursor:
table = Table('ir_cache')
cursor.execute(*table.select(table.timestamp, table.name))
timestamps = {}
for timestamp, name in cursor.fetchall():
timestamps[name] = timestamp
- for inst in cls._cache_instance:
- if inst._name in timestamps:
- with inst._lock:
- if (not inst._timestamp
- or timestamps[inst._name] > inst._timestamp):
- inst._timestamp = timestamps[inst._name]
- inst._cache[dbname] = LRUDict(inst.size_limit)
+ for name, timestamp in timestamps.items():
+ try:
+ inst = cls._instances[name]
+ except KeyError:
+ continue
+ if not inst._timestamp or timestamp > inst._timestamp:
+ inst._timestamp = timestamp
+ inst._database_cache[dbname] = LRUDict(inst.size_limit)
cls._clean_last = datetime.now()
@classmethod
- def reset(cls, dbname, name):
- with cls._resets_lock:
- cls._resets.setdefault(dbname, set())
- cls._resets[dbname].add(name)
-
- @classmethod
- def resets(cls, dbname):
+ def commit(cls, transaction):
table = Table('ir_cache')
- resets = cls._resets.setdefault(dbname, set())
- if not resets:
+ reset = cls._reset.setdefault(transaction, set())
+ if not reset:
return
- with Transaction().new_transaction(_nocache=True) as transaction,\
- transaction.connection.cursor() as cursor,\
- cls._resets_lock:
- for name in resets:
+ dbname = transaction.database.name
+ with transaction.connection.cursor() as cursor:
+ for name in reset:
cursor.execute(*table.select(table.name,
where=table.name == name,
limit=1))
@@ -154,12 +156,27 @@
cursor.execute(*table.insert(
[table.timestamp, table.name],
[[CurrentTimestamp(), name]]))
- resets.clear()
+
+ cursor.execute(*table.select(
+ Max(table.timestamp),
+ where=table.name == name))
+ timestamp, = cursor.fetchone()
+
+ inst = cls._instances[name]
+ inst._timestamp = timestamp
+ inst._database_cache[dbname] = LRUDict(inst.size_limit)
+
+ @classmethod
+ def rollback(cls, transaction):
+ try:
+ cls._reset[transaction].clear()
+ except KeyError:
+ pass
@classmethod
def drop(cls, dbname):
- for inst in cls._cache_instance:
- inst._cache.pop(dbname, None)
+ for inst in cls._instances.values():
+ inst._database_cache.pop(dbname, None)
if config.get('cache', 'class'):
diff -r 9a7247a6df7b -r a83631f4ba67 trytond/protocols/dispatcher.py
--- a/trytond/protocols/dispatcher.py Fri Jan 18 11:32:47 2019 +0100
+++ b/trytond/protocols/dispatcher.py Fri Jan 18 11:57:11 2019 +0100
@@ -108,7 +108,7 @@
context = {'_request': request.context}
hostname = get_hostname(request.host)
with Transaction().start(
- None, 0, context=context, close=True, _nocache=True
+ None, 0, context=context, readonly=True, close=True,
) as transaction:
return transaction.database.list(hostname=hostname)
diff -r 9a7247a6df7b -r a83631f4ba67 trytond/tests/test_cache.py
--- a/trytond/tests/test_cache.py Fri Jan 18 11:32:47 2019 +0100
+++ b/trytond/tests/test_cache.py Fri Jan 18 11:57:11 2019 +0100
@@ -3,12 +3,25 @@
import unittest
-from trytond.cache import freeze
+from trytond.cache import freeze, MemoryCache
+from trytond.tests.test_tryton import with_transaction, activate_module
+from trytond.tests.test_tryton import DB_NAME, USER
+from trytond.transaction import Transaction
+
+
+cache = MemoryCache('test.cache')
class CacheTestCase(unittest.TestCase):
"Test Cache"
+ @classmethod
+ def setUpClass(cls):
+ activate_module('tests')
+
+ def tearDown(self):
+ MemoryCache.drop(DB_NAME)
+
def testFreeze(self):
"Test freeze"
self.assertEqual(freeze([1, 2, 3]), (1, 2, 3))
@@ -31,6 +44,45 @@
('string', 'test'),
]))]))]))
+ @with_transaction()
+ def test_memory_cache_set_get(self):
+ "Test MemoryCache set/get"
+ cache.set('foo', 'bar')
+
+ self.assertEqual(cache.get('foo'), 'bar')
+
+ @with_transaction()
+ def test_memory_cache_drop(self):
+ "Test MemoryCache drop"
+ cache.set('foo', 'bar')
+ MemoryCache.drop(DB_NAME)
+
+ self.assertEqual(cache.get('foo'), None)
+
+ def test_memory_cache_transactions(self):
+ "Test MemoryCache with concurrent transactions"
+ transaction1 = Transaction().start(DB_NAME, USER)
+ self.addCleanup(transaction1.stop)
+
+ cache.set('foo', 'bar')
+ self.assertEqual(cache.get('foo'), 'bar')
+
+ transaction2 = transaction1.new_transaction()
+ self.addCleanup(transaction2.stop)
+
+ cache.clear()
+ self.assertEqual(cache.get('foo'), None)
+
+ cache.set('foo', 'baz')
+ self.assertEqual(cache.get('foo'), 'baz')
+
+ Transaction().set_current_transaction(transaction1)
+ self.addCleanup(transaction1.stop)
+ self.assertEqual(cache.get('foo'), 'bar')
+
+ transaction2.commit()
+ self.assertEqual(cache.get('foo'), None)
+
def suite():
func = unittest.TestLoader().loadTestsFromTestCase
diff -r 9a7247a6df7b -r a83631f4ba67 trytond/tests/test_tryton.py
--- a/trytond/tests/test_tryton.py Fri Jan 18 11:32:47 2019 +0100
+++ b/trytond/tests/test_tryton.py Fri Jan 18 11:57:11 2019 +0100
@@ -117,7 +117,7 @@
def _sqlite_copy(file_, restore=False):
import sqlite3 as sqlite
- with Transaction().start(DB_NAME, 0, _nocache=True) as transaction, \
+ with Transaction().start(DB_NAME, 0) as transaction, \
sqlite.connect(file_) as conn2:
conn1 = transaction.connection
# sqlitebck does not work with pysqlite2
@@ -153,8 +153,7 @@
def _pg_restore(cache_file):
with Transaction().start(
- None, 0, close=True, autocommit=True, _nocache=True) \
- as transaction:
+ None, 0, close=True, autocommit=True) as transaction:
transaction.database.create(transaction.connection, DB_NAME)
cmd = ['pg_restore', '-d', DB_NAME]
options, env = _pg_options()
@@ -165,8 +164,7 @@
except OSError:
cache_name, _ = os.path.splitext(os.path.basename(cache_file))
with Transaction().start(
- None, 0, close=True, autocommit=True, _nocache=True) \
- as transaction:
+ None, 0, close=True, autocommit=True) as transaction:
transaction.database.drop(transaction.connection, DB_NAME)
transaction.database.create(
transaction.connection, DB_NAME, cache_name)
@@ -185,8 +183,7 @@
# Ensure any connection is left open
backend.get('Database')(DB_NAME).close()
with Transaction().start(
- None, 0, close=True, autocommit=True, _nocache=True) \
- as transaction:
+ None, 0, close=True, autocommit=True) as transaction:
transaction.database.create(
transaction.connection, cache_name, DB_NAME)
open(cache_file, 'a').close()
@@ -626,11 +623,10 @@
Database = backend.get('Database')
if not db_exist(name):
with Transaction().start(
- None, 0, close=True, autocommit=True, _nocache=True) \
- as transaction:
+ None, 0, close=True, autocommit=True) as transaction:
transaction.database.create(transaction.connection, name)
- with Transaction().start(name, 0, _nocache=True) as transaction,\
+ with Transaction().start(name, 0) as transaction,\
transaction.connection.cursor() as cursor:
Database(name).init()
ir_configuration = Table('ir_configuration')
@@ -660,8 +656,7 @@
database.close()
with Transaction().start(
- None, 0, close=True, autocommit=True, _nocache=True) \
- as transaction:
+ None, 0, close=True, autocommit=True) as transaction:
database.drop(transaction.connection, name)
Pool.stop(name)
Cache.drop(name)
diff -r 9a7247a6df7b -r a83631f4ba67 trytond/transaction.py
--- a/trytond/transaction.py Fri Jan 18 11:32:47 2019 +0100
+++ b/trytond/transaction.py Fri Jan 18 11:57:11 2019 +0100
@@ -78,7 +78,7 @@
LRUDict(config.getint('cache', 'model')))
def start(self, database_name, user, readonly=False, context=None,
- close=False, autocommit=False, _nocache=False):
+ close=False, autocommit=False):
'''
Start transaction
'''
@@ -105,11 +105,10 @@
self.timestamp = {}
self.counter = 0
self._datamanagers = []
- self._nocache = _nocache
- if not _nocache:
+ if database_name:
from trytond.cache import Cache
try:
- Cache.clean(database.name)
+ Cache.sync(self)
except BaseException:
self.stop(False)
raise
@@ -191,14 +190,14 @@
self._local.transactions.append(transaction)
return transaction
- def new_transaction(self, autocommit=False, readonly=False,
- _nocache=False):
+ def new_transaction(self, autocommit=False, readonly=False):
transaction = Transaction(new=True)
return transaction.start(self.database.name, self.user,
context=self.context, close=self.close, readonly=readonly,
- autocommit=autocommit, _nocache=_nocache)
+ autocommit=autocommit)
def commit(self):
+ from trytond.cache import Cache
try:
if self._datamanagers:
for datamanager in self._datamanagers:
@@ -207,6 +206,7 @@
datamanager.commit(self)
for datamanager in self._datamanagers:
datamanager.tpc_vote(self)
+ Cache.commit(self)
self.connection.commit()
except:
self.rollback()
@@ -219,19 +219,15 @@
logger.critical('A datamanager raised an exception in'
' tpc_finish, the data might be inconsistant',
exc_info=True)
- if not self._nocache:
- from trytond.cache import Cache
- Cache.resets(self.database.name)
def rollback(self):
+ from trytond.cache import Cache
for cache in self.cache.values():
cache.clear()
for datamanager in self._datamanagers:
datamanager.tpc_abort(self)
+ Cache.rollback(self)
self.connection.rollback()
- if not self._nocache:
- from trytond.cache import Cache
- Cache.resets(self.database.name)
def join(self, datamanager):
try: