changeset a83631f4ba67 in trytond:default
details: https://hg.tryton.org/trytond?cmd=changeset;node=a83631f4ba67
description:
        Make Cache transactional

        We implement sync, commit and rollback to follow transaction semantic.
        When a transaction clears a cache, the global cache is cleared only 
when the
        transaction is committed meanwhile the transaction will use a cache for 
itself
        only.
        We remove _nocache option because the sync, commit and rollback operate 
only
        for the transaction.

        issue7975
        review53521002
diffstat:

 CHANGELOG                       |    2 +
 doc/ref/cache.rst               |   12 +-
 trytond/admin.py                |    2 +-
 trytond/cache.py                |  135 ++++++++++++++++++++++-----------------
 trytond/protocols/dispatcher.py |    2 +-
 trytond/tests/test_cache.py     |   54 +++++++++++++++-
 trytond/tests/test_tryton.py    |   19 ++---
 trytond/transaction.py          |   22 ++---
 8 files changed, 155 insertions(+), 93 deletions(-)

diffs (491 lines):

diff -r 9a7247a6df7b -r a83631f4ba67 CHANGELOG
--- a/CHANGELOG Fri Jan 18 11:32:47 2019 +0100
+++ b/CHANGELOG Fri Jan 18 11:57:11 2019 +0100
@@ -1,3 +1,5 @@
+* Remove _nocache on Transaction
+* Make Cache transactional
 * Allow to search on key's value of Dict
 * Allow to order Dict by key's value
 * Do not store keys with null value in Dict
diff -r 9a7247a6df7b -r a83631f4ba67 doc/ref/cache.rst
--- a/doc/ref/cache.rst Fri Jan 18 11:32:47 2019 +0100
+++ b/doc/ref/cache.rst Fri Jan 18 11:57:11 2019 +0100
@@ -34,17 +34,17 @@
 
 Clears all the keys in the cache.
 
-.. staticmethod:: clean(dbname)
+.. classmethod:: sync(transaction)
 
-Clean the cache for database `dbname`
+Synchronize cache instances using transaction.
 
-.. staticmethod:: reset(dbname, name)
+.. classmethod:: commit(transaction)
 
-Reset the `name` cache for database `dbname`
+Apply cache changes from transaction.
 
-.. staticmethod:: resets(dbname)
+.. classmethod:: rollback(transaction)
 
-Resets all the caches stored for database `dbname`
+Remove cache changes from transaction.
 
 .. staticmethod:: drop(dbname)
 
diff -r 9a7247a6df7b -r a83631f4ba67 trytond/admin.py
--- a/trytond/admin.py  Fri Jan 18 11:32:47 2019 +0100
+++ b/trytond/admin.py  Fri Jan 18 11:57:11 2019 +0100
@@ -22,7 +22,7 @@
     init = {}
     for db_name in options.database_names:
         init[db_name] = False
-        with Transaction().start(db_name, 0, _nocache=True):
+        with Transaction().start(db_name, 0):
             database = Database(db_name)
             database.connect()
             if options.update:
diff -r 9a7247a6df7b -r a83631f4ba67 trytond/cache.py
--- a/trytond/cache.py  Fri Jan 18 11:32:47 2019 +0100
+++ b/trytond/cache.py  Fri Jan 18 11:57:11 2019 +0100
@@ -1,10 +1,11 @@
 # This file is part of Tryton.  The COPYRIGHT file at the top level of
 # this repository contains the full copyright notices and license terms.
-from threading import Lock
-from collections import OrderedDict
+from collections import OrderedDict, defaultdict
 from datetime import datetime
+from weakref import WeakKeyDictionary
 
 from sql import Table
+from sql.aggregate import Max
 from sql.functions import CurrentTimestamp
 
 from trytond.config import config
@@ -25,13 +26,14 @@
 
 
 class BaseCache(object):
-    _cache_instance = []
+    _instances = {}
 
     def __init__(self, name, size_limit=1024, context=True):
         self._name = name
         self.size_limit = size_limit
         self.context = context
-        self._cache_instance.append(self)
+        assert self._name not in self._instances
+        self._instances[self._name] = self
 
     def _key(self, key):
         if self.context:
@@ -47,16 +49,16 @@
     def clear(self):
         raise NotImplemented
 
-    @staticmethod
-    def clean(dbname):
+    @classmethod
+    def sync(cls, transaction):
         raise NotImplemented
 
-    @staticmethod
-    def reset(dbname, name):
+    @classmethod
+    def commit(cls, transaction):
         raise NotImplemented
 
-    @staticmethod
-    def resets(dbname):
+    @classmethod
+    def rollback(cls, transaction):
         raise NotImplemented
 
     @classmethod
@@ -68,80 +70,80 @@
     """
     A key value LRU cache with size limit.
     """
-    _resets = {}
-    _resets_lock = Lock()
+    _reset = WeakKeyDictionary()
     _clean_last = datetime.now()
 
     def __init__(self, name, size_limit=1024, context=True):
         super(MemoryCache, self).__init__(name, size_limit, context)
-        self._cache = {}
+        self._database_cache = defaultdict(lambda: LRUDict(size_limit))
+        self._transaction_cache = WeakKeyDictionary()
         self._timestamp = None
-        self._lock = Lock()
+
+    def _get_cache(self):
+        transaction = Transaction()
+        dbname = transaction.database.name
+        if transaction in self._reset:
+            try:
+                return self._transaction_cache[transaction]
+            except KeyError:
+                self._transaction_cache[transaction] = LRUDict(self.size_limit)
+                return self._transaction_cache[transaction]
+        else:
+            return self._database_cache[dbname]
 
     def get(self, key, default=None):
-        dbname = Transaction().database.name
         key = self._key(key)
-        with self._lock:
-            cache = self._cache.setdefault(dbname, LRUDict(self.size_limit))
-            try:
-                result = cache[key] = cache.pop(key)
-                return result
-            except (KeyError, TypeError):
-                return default
+        cache = self._get_cache()
+        try:
+            result = cache[key] = cache.pop(key)
+            return result
+        except (KeyError, TypeError):
+            return default
 
     def set(self, key, value):
-        dbname = Transaction().database.name
         key = self._key(key)
-        with self._lock:
-            cache = self._cache.setdefault(dbname, LRUDict(self.size_limit))
-            try:
-                cache[key] = value
-            except TypeError:
-                pass
+        cache = self._get_cache()
+        try:
+            cache[key] = value
+        except TypeError:
+            pass
         return value
 
     def clear(self):
-        dbname = Transaction().database.name
-        Cache.reset(dbname, self._name)
-        with self._lock:
-            self._cache[dbname] = LRUDict(self.size_limit)
+        transaction = Transaction()
+        self._reset.setdefault(transaction, set()).add(self._name)
+        self._transaction_cache.pop(transaction, None)
 
     @classmethod
-    def clean(cls, dbname):
+    def sync(cls, transaction):
         if (datetime.now() - cls._clean_last).total_seconds() < _clear_timeout:
             return
-        with Transaction().new_transaction(_nocache=True) as transaction,\
-                transaction.connection.cursor() as cursor:
+        dbname = transaction.database.name
+        with transaction.connection.cursor() as cursor:
             table = Table('ir_cache')
             cursor.execute(*table.select(table.timestamp, table.name))
             timestamps = {}
             for timestamp, name in cursor.fetchall():
                 timestamps[name] = timestamp
-        for inst in cls._cache_instance:
-            if inst._name in timestamps:
-                with inst._lock:
-                    if (not inst._timestamp
-                            or timestamps[inst._name] > inst._timestamp):
-                        inst._timestamp = timestamps[inst._name]
-                        inst._cache[dbname] = LRUDict(inst.size_limit)
+        for name, timestamp in timestamps.items():
+            try:
+                inst = cls._instances[name]
+            except KeyError:
+                continue
+            if not inst._timestamp or timestamp > inst._timestamp:
+                inst._timestamp = timestamp
+                inst._database_cache[dbname] = LRUDict(inst.size_limit)
         cls._clean_last = datetime.now()
 
     @classmethod
-    def reset(cls, dbname, name):
-        with cls._resets_lock:
-            cls._resets.setdefault(dbname, set())
-            cls._resets[dbname].add(name)
-
-    @classmethod
-    def resets(cls, dbname):
+    def commit(cls, transaction):
         table = Table('ir_cache')
-        resets = cls._resets.setdefault(dbname, set())
-        if not resets:
+        reset = cls._reset.setdefault(transaction, set())
+        if not reset:
             return
-        with Transaction().new_transaction(_nocache=True) as transaction,\
-                transaction.connection.cursor() as cursor,\
-                cls._resets_lock:
-            for name in resets:
+        dbname = transaction.database.name
+        with transaction.connection.cursor() as cursor:
+            for name in reset:
                 cursor.execute(*table.select(table.name,
                         where=table.name == name,
                         limit=1))
@@ -154,12 +156,27 @@
                     cursor.execute(*table.insert(
                             [table.timestamp, table.name],
                             [[CurrentTimestamp(), name]]))
-            resets.clear()
+
+                cursor.execute(*table.select(
+                        Max(table.timestamp),
+                        where=table.name == name))
+                timestamp, = cursor.fetchone()
+
+                inst = cls._instances[name]
+                inst._timestamp = timestamp
+                inst._database_cache[dbname] = LRUDict(inst.size_limit)
+
+    @classmethod
+    def rollback(cls, transaction):
+        try:
+            cls._reset[transaction].clear()
+        except KeyError:
+            pass
 
     @classmethod
     def drop(cls, dbname):
-        for inst in cls._cache_instance:
-            inst._cache.pop(dbname, None)
+        for inst in cls._instances.values():
+            inst._database_cache.pop(dbname, None)
 
 
 if config.get('cache', 'class'):
diff -r 9a7247a6df7b -r a83631f4ba67 trytond/protocols/dispatcher.py
--- a/trytond/protocols/dispatcher.py   Fri Jan 18 11:32:47 2019 +0100
+++ b/trytond/protocols/dispatcher.py   Fri Jan 18 11:57:11 2019 +0100
@@ -108,7 +108,7 @@
     context = {'_request': request.context}
     hostname = get_hostname(request.host)
     with Transaction().start(
-            None, 0, context=context, close=True, _nocache=True
+            None, 0, context=context, readonly=True, close=True,
             ) as transaction:
         return transaction.database.list(hostname=hostname)
 
diff -r 9a7247a6df7b -r a83631f4ba67 trytond/tests/test_cache.py
--- a/trytond/tests/test_cache.py       Fri Jan 18 11:32:47 2019 +0100
+++ b/trytond/tests/test_cache.py       Fri Jan 18 11:57:11 2019 +0100
@@ -3,12 +3,25 @@
 
 import unittest
 
-from trytond.cache import freeze
+from trytond.cache import freeze, MemoryCache
+from trytond.tests.test_tryton import with_transaction, activate_module
+from trytond.tests.test_tryton import DB_NAME, USER
+from trytond.transaction import Transaction
+
+
+cache = MemoryCache('test.cache')
 
 
 class CacheTestCase(unittest.TestCase):
     "Test Cache"
 
+    @classmethod
+    def setUpClass(cls):
+        activate_module('tests')
+
+    def tearDown(self):
+        MemoryCache.drop(DB_NAME)
+
     def testFreeze(self):
         "Test freeze"
         self.assertEqual(freeze([1, 2, 3]), (1, 2, 3))
@@ -31,6 +44,45 @@
                                             ('string', 'test'),
                                             ]))]))]))
 
+    @with_transaction()
+    def test_memory_cache_set_get(self):
+        "Test MemoryCache set/get"
+        cache.set('foo', 'bar')
+
+        self.assertEqual(cache.get('foo'), 'bar')
+
+    @with_transaction()
+    def test_memory_cache_drop(self):
+        "Test MemoryCache drop"
+        cache.set('foo', 'bar')
+        MemoryCache.drop(DB_NAME)
+
+        self.assertEqual(cache.get('foo'), None)
+
+    def test_memory_cache_transactions(self):
+        "Test MemoryCache with concurrent transactions"
+        transaction1 = Transaction().start(DB_NAME, USER)
+        self.addCleanup(transaction1.stop)
+
+        cache.set('foo', 'bar')
+        self.assertEqual(cache.get('foo'), 'bar')
+
+        transaction2 = transaction1.new_transaction()
+        self.addCleanup(transaction2.stop)
+
+        cache.clear()
+        self.assertEqual(cache.get('foo'), None)
+
+        cache.set('foo', 'baz')
+        self.assertEqual(cache.get('foo'), 'baz')
+
+        Transaction().set_current_transaction(transaction1)
+        self.addCleanup(transaction1.stop)
+        self.assertEqual(cache.get('foo'), 'bar')
+
+        transaction2.commit()
+        self.assertEqual(cache.get('foo'), None)
+
 
 def suite():
     func = unittest.TestLoader().loadTestsFromTestCase
diff -r 9a7247a6df7b -r a83631f4ba67 trytond/tests/test_tryton.py
--- a/trytond/tests/test_tryton.py      Fri Jan 18 11:32:47 2019 +0100
+++ b/trytond/tests/test_tryton.py      Fri Jan 18 11:57:11 2019 +0100
@@ -117,7 +117,7 @@
 def _sqlite_copy(file_, restore=False):
     import sqlite3 as sqlite
 
-    with Transaction().start(DB_NAME, 0, _nocache=True) as transaction, \
+    with Transaction().start(DB_NAME, 0) as transaction, \
             sqlite.connect(file_) as conn2:
         conn1 = transaction.connection
         # sqlitebck does not work with pysqlite2
@@ -153,8 +153,7 @@
 
 def _pg_restore(cache_file):
     with Transaction().start(
-            None, 0, close=True, autocommit=True, _nocache=True) \
-            as transaction:
+            None, 0, close=True, autocommit=True) as transaction:
         transaction.database.create(transaction.connection, DB_NAME)
     cmd = ['pg_restore', '-d', DB_NAME]
     options, env = _pg_options()
@@ -165,8 +164,7 @@
     except OSError:
         cache_name, _ = os.path.splitext(os.path.basename(cache_file))
         with Transaction().start(
-                None, 0, close=True, autocommit=True, _nocache=True) \
-                as transaction:
+                None, 0, close=True, autocommit=True) as transaction:
             transaction.database.drop(transaction.connection, DB_NAME)
             transaction.database.create(
                 transaction.connection, DB_NAME, cache_name)
@@ -185,8 +183,7 @@
         # Ensure any connection is left open
         backend.get('Database')(DB_NAME).close()
         with Transaction().start(
-                None, 0, close=True, autocommit=True, _nocache=True) \
-                as transaction:
+                None, 0, close=True, autocommit=True) as transaction:
             transaction.database.create(
                 transaction.connection, cache_name, DB_NAME)
         open(cache_file, 'a').close()
@@ -626,11 +623,10 @@
     Database = backend.get('Database')
     if not db_exist(name):
         with Transaction().start(
-                None, 0, close=True, autocommit=True, _nocache=True) \
-                as transaction:
+                None, 0, close=True, autocommit=True) as transaction:
             transaction.database.create(transaction.connection, name)
 
-        with Transaction().start(name, 0, _nocache=True) as transaction,\
+        with Transaction().start(name, 0) as transaction,\
                 transaction.connection.cursor() as cursor:
             Database(name).init()
             ir_configuration = Table('ir_configuration')
@@ -660,8 +656,7 @@
         database.close()
 
         with Transaction().start(
-                None, 0, close=True, autocommit=True, _nocache=True) \
-                as transaction:
+                None, 0, close=True, autocommit=True) as transaction:
             database.drop(transaction.connection, name)
             Pool.stop(name)
             Cache.drop(name)
diff -r 9a7247a6df7b -r a83631f4ba67 trytond/transaction.py
--- a/trytond/transaction.py    Fri Jan 18 11:32:47 2019 +0100
+++ b/trytond/transaction.py    Fri Jan 18 11:57:11 2019 +0100
@@ -78,7 +78,7 @@
             LRUDict(config.getint('cache', 'model')))
 
     def start(self, database_name, user, readonly=False, context=None,
-            close=False, autocommit=False, _nocache=False):
+            close=False, autocommit=False):
         '''
         Start transaction
         '''
@@ -105,11 +105,10 @@
         self.timestamp = {}
         self.counter = 0
         self._datamanagers = []
-        self._nocache = _nocache
-        if not _nocache:
+        if database_name:
             from trytond.cache import Cache
             try:
-                Cache.clean(database.name)
+                Cache.sync(self)
             except BaseException:
                 self.stop(False)
                 raise
@@ -191,14 +190,14 @@
         self._local.transactions.append(transaction)
         return transaction
 
-    def new_transaction(self, autocommit=False, readonly=False,
-            _nocache=False):
+    def new_transaction(self, autocommit=False, readonly=False):
         transaction = Transaction(new=True)
         return transaction.start(self.database.name, self.user,
             context=self.context, close=self.close, readonly=readonly,
-            autocommit=autocommit, _nocache=_nocache)
+            autocommit=autocommit)
 
     def commit(self):
+        from trytond.cache import Cache
         try:
             if self._datamanagers:
                 for datamanager in self._datamanagers:
@@ -207,6 +206,7 @@
                     datamanager.commit(self)
                 for datamanager in self._datamanagers:
                     datamanager.tpc_vote(self)
+            Cache.commit(self)
             self.connection.commit()
         except:
             self.rollback()
@@ -219,19 +219,15 @@
                 logger.critical('A datamanager raised an exception in'
                     ' tpc_finish, the data might be inconsistant',
                     exc_info=True)
-        if not self._nocache:
-            from trytond.cache import Cache
-            Cache.resets(self.database.name)
 
     def rollback(self):
+        from trytond.cache import Cache
         for cache in self.cache.values():
             cache.clear()
         for datamanager in self._datamanagers:
             datamanager.tpc_abort(self)
+        Cache.rollback(self)
         self.connection.rollback()
-        if not self._nocache:
-            from trytond.cache import Cache
-            Cache.resets(self.database.name)
 
     def join(self, datamanager):
         try:

Reply via email to