This includes an own simple cache implementation and an
interface to a memcache instance.
---
lib/cache.py | 308 +++++++++++++++++++++++++++++++++++++++++
test/ganeti.cache_unittest.py | 104 ++++++++++++++
2 files changed, 412 insertions(+), 0 deletions(-)
create mode 100644 lib/cache.py
create mode 100755 test/ganeti.cache_unittest.py
diff --git a/lib/cache.py b/lib/cache.py
new file mode 100644
index 0000000..7789fb0
--- /dev/null
+++ b/lib/cache.py
@@ -0,0 +1,308 @@
+#
+#
+
+# Copyright (C) 2011 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""This module implements caching."""
+
+
+import threading
+import time
+
+from ganeti import locking
+from ganeti import serializer
+
+try:
+ _has_memcache = True
+ import memcache
+except ImportError:
+ _has_memcache = False
+
+
+_cache = None
+_cache_lock = threading.Lock()
+
+
+def Cache():
+ """Factory method to return appropriate cache.
+
+ """
+ # We need to use the global keyword here
+ global _cache # pylint: disable-msg=W0603
+
+ if not _cache:
+ _cache_lock.acquire()
+ try:
+ if not _cache:
+ if _has_memcache:
+ cache_obj = MemCache()
+ else:
+ cache_obj = SimpleCache()
+ # W0621: Redefine '_cache' from outer scope (used for singleton)
+ _cache = cache_obj # pylint: disable-msg=W0621
+ finally:
+ _cache_lock.release()
+
+ # Make sure we have a fresh internal state and are ready to serve
+ _cache.ResetState()
+ return _cache
+
+
+class CacheBase:
+ """This is the base class for all caches.
+
+ """
+ def __init__(self):
+ """Base init method.
+
+ """
+
+ def Store(self, key, value, ttl=0):
+ """Stores key with value in the cache.
+
+ @param key: The key to associate this cached value
+ @param value: The value to cache
+ @param ttl: TTL in seconds after when this entry is considered outdated
+ @returns True on success, False on failure
+
+ """
+ raise NotImplementedError
+
+ def Get(self, keys):
+ """Retrieve the value from the cache.
+
+ @param keys: The keys to retrieve
+ @returns The list of values
+
+ """
+ raise NotImplementedError
+
+ def Invalidate(self, keys):
+ """Invalidate given keys.
+
+ @param keys: The list of keys to invalidate
+ @returns True on success, False otherwise
+
+ """
+ raise NotImplementedError
+
+ def Flush(self):
+ """Invalidates all of the keys and flushes the cache.
+
+ """
+ raise NotImplementedError
+
+ def ResetState(self):
+ """Used to reset the state of the cache.
+
+ This can be used to reinstantiate connection or any other state refresh
+
+ """
+
+ def Cleanup(self):
+ """Cleanup the cache from expired entries.
+
+ """
+
+
+class SimpleCache(CacheBase):
+ """Implements a very simple, dict base cache.
+
+ """
+ CLEANUP_ROUND = 1800
+
+ def __init__(self, _time_fn=time.time):
+ """Initialize this class.
+
+ @param _time_fn: Function used to return time (unittest only)
+ """
+ CacheBase.__init__(self)
+
+ self._time_fn = _time_fn
+
+ self.cache = {}
+ self.lock = locking.SharedLock("SimpleCache-lock")
+ self.last_cleanup = self._time_fn()
+
+ def _UnlockedCleanup(self):
+ """Does cleanup of the cache.
+
+ """
+ check_time = self._time_fn()
+ if (self.last_cleanup + self.CLEANUP_ROUND) <= check_time:
+ keys = []
+ for key, value in self.cache.items():
+ if not value["ttl"]:
+ continue
+
+ expired = value["timestamp"] + value["ttl"]
+ if expired < check_time:
+ keys.append(key)
+ self._UnlockedInvalidate(keys)
+ self.last_cleanup = check_time
+
+ @locking.ssynchronized("lock")
+ def Cleanup(self):
+ """Cleanup our cache.
+
+ """
+ self._UnlockedCleanup()
+
+ @locking.ssynchronized("lock")
+ def Store(self, key, value, ttl=0):
+ """Stores a value at key in the cache.
+
+ See L{CacheBase} for parameter description
+
+ """
+ assert ttl >= 0
+ self._UnlockedCleanup()
+ val = serializer.Dump(value)
+ cache_val = {
+ "timestamp": self._time_fn(),
+ "ttl": ttl,
+ "value": val
+ }
+ self.cache[key] = cache_val
+ return True
+
+ @locking.ssynchronized("lock", shared=1)
+ def Get(self, keys):
+ """Retrieve the values of keys from cache.
+
+ See L{CacheBase} for parameter description
+
+ """
+ return [self._ExtractValue(key) for key in keys]
+
+ @locking.ssynchronized("lock")
+ def Invalidate(self, keys):
+ """Invalidates value for keys in cache.
+
+ See L{CacheBase} for parameter description
+
+ """
+ self._UnlockedInvalidate(keys)
+ return True
+
+ @locking.ssynchronized("lock")
+ def Flush(self):
+ """Invalidates all keys and values in cache.
+
+ See L{CacheBase} for parameter description
+
+ """
+ self.cache.clear()
+ self.last_cleanup = self._time_fn()
+
+ def _UnlockedInvalidate(self, keys):
+ """Invalidate keys in cache.
+
+ This is the unlocked version, see L{Invalidate} for parameter description
+
+ """
+ for key in keys:
+ if key in self.cache:
+ del self.cache[key]
+
+ def _ExtractValue(self, key):
+ """Extracts just the value for a key.
+
+ This method is taking care if the value did not expire ans returns it
+
+ @param key: The key to look for
+ @returns The value if key is not expired, None otherwise
+
+ """
+ if key in self.cache:
+ cache_val = self.cache[key]
+ if cache_val["ttl"] == 0:
+ return serializer.Load(cache_val["value"])
+ else:
+ expired = cache_val["timestamp"] + cache_val["ttl"]
+
+ if self._time_fn() <= expired:
+ return serializer.Load(cache_val["value"])
+ else:
+ return None
+ else:
+ return None
+
+
+class MemCache(CacheBase):
+ """Implements the memcache cache.
+
+ """
+ def __init__(self, servers=("localhost:11211",)):
+ """Initialize this class.
+
+ @param servers: List of memcache servers in the format <host>:<port>
+
+ """
+ CacheBase.__init__(self)
+
+ self.mc = memcache.Client(servers)
+
+ def Store(self, key, value, ttl=0):
+ """Stores key with value in memcache.
+
+ See L{CacheBase} for parameter description
+
+ """
+ return (self.mc.add(key, value, time=ttl) or
+ self.mc.replace(key, value, time=ttl))
+
+ def Get(self, keys):
+ """Retrieve the list of keys from memcache.
+
+ See L{CacheBase} for parameter description
+
+ """
+ result = self.mc.get_multi(keys)
+ return_result = []
+ for key in keys:
+ if key in result:
+ return_result.append(result[key])
+ else:
+ return_result.append(None)
+ return return_result
+
+ def Invalidate(self, keys):
+ """Invalidates given keys in memcache.
+
+ See L{CacheBase} for parameter description
+
+ """
+ return self.mc.delete_multi(self, keys) == 1
+
+ def Flush(self):
+ """Invalidates all keys in memcache.
+
+ See L{CacheBase} for parameter description
+
+ """
+ self.mc.flush_all()
+
+ def ResetState(self):
+ """Resets the memcache state.
+
+ This basically means try to reconnect to possible memcache servers
+
+ """
+ self.mc.forget_dead_hosts()
diff --git a/test/ganeti.cache_unittest.py b/test/ganeti.cache_unittest.py
new file mode 100755
index 0000000..bf4944c
--- /dev/null
+++ b/test/ganeti.cache_unittest.py
@@ -0,0 +1,104 @@
+#!/usr/bin/python
+#
+
+# Copyright (C) 2011 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+"""Script for testing ganeti.cache"""
+
+import testutils
+import unittest
+
+from ganeti import cache
+
+
+class ReturnStub:
+ def __init__(self, values):
+ self.values = values
+
+ def __call__(self):
+ assert self.values
+ return self.values.pop(0)
+
+
+class SimpleCacheTest(unittest.TestCase):
+ def setUp(self):
+ self.cache = cache.SimpleCache()
+
+ def testNoKey(self):
+ self.assertEqual(self.cache.Get(["i-dont-exist", "neither-do-i", "no"]),
+ [None, None, None])
+
+ def testCache(self):
+ value = 0xc0ffee
+ self.assert_(self.cache.Store("i-exist", value))
+ self.assertEqual(self.cache.Get(["i-exist"]), [value])
+
+ def testMixed(self):
+ value = 0xb4dc0de
+ self.assert_(self.cache.Store("i-exist", value))
+ self.assertEqual(self.cache.Get(["i-exist", "i-dont"]), [value, None])
+
+ def testTtl(self):
+ my_times = ReturnStub([0, 1, 1, 2, 3, 5])
+ ttl_cache = cache.SimpleCache(_time_fn=my_times)
+ self.assert_(ttl_cache.Store("test-expire", 0xdeadbeef, ttl=2))
+
+ # At this point time will return 2, 1 (start) + 2 (ttl) = 3, still valid
+ self.assertEqual(ttl_cache.Get(["test-expire"]), [0xdeadbeef])
+
+ # At this point time will return 3, 1 (start) + 2 (ttl) = 3, still valid
+ self.assertEqual(ttl_cache.Get(["test-expire"]), [0xdeadbeef])
+
+ # We are at 5, < 3, invalid
+ self.assertEqual(ttl_cache.Get(["test-expire"]), [None])
+ self.assertFalse(my_times.values)
+
+ def testCleanup(self):
+ my_times = ReturnStub([0, 1, 1, 2, 2, 3, 3, 5, 5,
+ 21 + cache.SimpleCache.CLEANUP_ROUND,
+ 34 + cache.SimpleCache.CLEANUP_ROUND,
+ 55 + cache.SimpleCache.CLEANUP_ROUND * 2,
+ 89 + cache.SimpleCache.CLEANUP_ROUND * 3])
+ # Index 0
+ ttl_cache = cache.SimpleCache(_time_fn=my_times)
+ # Index 1, 2
+ self.assert_(ttl_cache.Store("foobar", 0x1dea, ttl=6))
+ # Index 3, 4
+ self.assert_(ttl_cache.Store("baz", 0xc0dea55, ttl=11))
+ # Index 6, 7
+ self.assert_(ttl_cache.Store("long-foobar", "pretty long",
+ ttl=(22 + cache.SimpleCache.CLEANUP_ROUND)))
+ # Index 7, 8
+ self.assert_(ttl_cache.Store("foobazbar", "alive forever"))
+
+ self.assertEqual(set(ttl_cache.cache.keys()),
+ set(["foobar", "baz", "long-foobar", "foobazbar"]))
+ ttl_cache.Cleanup()
+ self.assertEqual(set(ttl_cache.cache.keys()),
+ set(["long-foobar", "foobazbar"]))
+ ttl_cache.Cleanup()
+ self.assertEqual(set(ttl_cache.cache.keys()),
+ set(["long-foobar", "foobazbar"]))
+ ttl_cache.Cleanup()
+ self.assertEqual(set(ttl_cache.cache.keys()), set(["foobazbar"]))
+ ttl_cache.Cleanup()
+ self.assertEqual(set(ttl_cache.cache.keys()), set(["foobazbar"]))
+
+
+if __name__ == "__main__":
+ testutils.GanetiTestProgram()
--
1.7.3.1
