Author: chabotc
Date: Tue Jun 10 07:03:44 2008
New Revision: 666115
URL: http://svn.apache.org/viewvc?rev=666115&view=rev
Log:
Added some cache stampeding protection, should make a few things a
lot smoother and faster; Also added a expiration glad to cache::get
as preperation for the refreshInterval in the proxy code
Modified:
incubator/shindig/trunk/php/src/common/Cache.php
incubator/shindig/trunk/php/src/common/CacheFile.php
incubator/shindig/trunk/php/src/common/CacheMemcache.php
Modified: incubator/shindig/trunk/php/src/common/Cache.php
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/php/src/common/Cache.php?rev=666115&r1=666114&r2=666115&view=diff
=
=
=
=
=
=
=
=
======================================================================
--- incubator/shindig/trunk/php/src/common/Cache.php (original)
+++ incubator/shindig/trunk/php/src/common/Cache.php Tue Jun 10
07:03:44 2008
@@ -22,7 +22,7 @@
abstract class Cache {
- abstract function get($key);
+ abstract function get($key, $expiration = false);
abstract function set($key, $value);
Modified: incubator/shindig/trunk/php/src/common/CacheFile.php
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/php/src/common/CacheFile.php?rev=666115&r1=666114&r2=666115&view=diff
=
=
=
=
=
=
=
=
======================================================================
--- incubator/shindig/trunk/php/src/common/CacheFile.php (original)
+++ incubator/shindig/trunk/php/src/common/CacheFile.php Tue Jun 10
07:03:44 2008
@@ -28,15 +28,75 @@
* cache dir, and removes all files that are older then 24 hours (or
whatever your
* config's CACHE_TIME is set too).
*/
+class CacheFile extends Cache {
-//TODO add cache stampeding prevention using file locking mechanisms
+ function isLocked($cacheFile)
+ {
+ // our lock file convention is simple: /the/file/path.lock
+ return file_exists($cacheFile . '.lock');
+ }
+ private function createLock($cacheFile)
+ {
+ $cacheDir = dirname($cacheFile);
+ if (! is_dir($cacheDir)) {
+ if (! @mkdir($cacheDir, 0755, true)) {
+ throw new CacheException("Could not create cache
directory");
+ }
+ }
+ @touch($cacheFile . '.lock');
+ }
-class CacheFile extends Cache {
+ private function removeLock($cacheFile)
+ {
+ // suppress all warnings, if some other process removed it that's
ok too
+ @unlink($cacheFile . '.lock');
+ }
+
+ private function waitForLock($cacheFile)
+ {
+ // 20 x 250 = 5 seconds
+ $tries = 20;
+ $cnt = 0;
+ do {
+ // make sure PHP picks up on file changes. This is an expensive
action but really can't be avoided
+ clearstatcache();
+ // 250 ms is a long time to sleep, but it does stop the server
from burning all resources on polling locks..
+ usleep(250);
+ $cnt ++;
+ } while ($cnt <= $tries && $this->isLocked());
+ if ($this->isLocked()) {
+ // 5 seconds passed, assume the owning process died off and
remove it
+ $this->removeLock($cacheFile);
+ }
+ }
+
+ private function getCacheDir($hash)
+ {
+ // use the first 2 characters of the hash as a directory prefix
+ // this should prevent slowdowns due to huge directory listings
+ // and thus give some basic amount of scalability
+ return Config::get('cache_root') . '/' . substr($hash, 0, 2);
+ }
+
+ private function getCacheFile($hash)
+ {
+ return $this->getCacheDir($hash) . '/' . $hash;
+ }
- function get($key)
+ public function get($key, $expiration = false)
{
+ if (! $expiration) {
+ // if no expiration time was given, fall back on the
global config
+ $expiration = Config::get('cache_time');
+ }
$cacheFile = $this->getCacheFile($key);
+ // See if this cache file is locked, if so we wait upto 5 seconds
for the lock owning process to
+ // complete it's work. If the lock is not released within that
time frame, it's cleaned up.
+ // This should give us a fair amount of 'Cache Stampeding'
protection
+ if ($this->isLocked($cacheFile)) {
+ $this->waitForLock($cacheFile);
+ }
if (file_exists($cacheFile) && is_readable($cacheFile)) {
$now = time();
if (($mtime = filemtime($cacheFile)) !== false && ($now - $mtime)
< Config::get('cache_time')) {
@@ -49,13 +109,14 @@
return false;
}
- function set($key, $value)
+ public function set($key, $value)
{
- // use the first 2 characters of the hash as a directory prefix
- // this should prevent slowdowns due to huge directory listings
- // and thus give some basic amount of scalability
$cacheDir = $this->getCacheDir($key);
$cacheFile = $this->getCacheFile($key);
+ if ($this->isLocked($cacheFile)) {
+ // some other process is writing to this file too, wait until
it's done to prevent hickups
+ $this->waitForLock($cacheFile);
+ }
if (! is_dir($cacheDir)) {
if (! @mkdir($cacheDir, 0755, true)) {
throw new CacheException("Could not create cache
directory");
@@ -64,26 +125,19 @@
// we serialize the whole request object, since we don't only want
the
// responseContent but also the postBody used, headers, size,
etc
$data = serialize($value);
+ $this->createLock($cacheFile);
if (! @file_put_contents($cacheFile, $data)) {
+ $this->removeLock($cacheFile);
throw new CacheException("Could not store data in cache
file");
}
+ $this->removeLock($cacheFile);
}
- function delete($key)
+ public function delete($key)
{
$file = $this->getCacheFile($key);
if (! @unlink($file)) {
throw new CacheException("Cache file could not be
deleted");
}
}
-
- private function getCacheDir($hash)
- {
- return Config::get('cache_root') . '/' . substr($hash, 0, 2);
- }
-
- private function getCacheFile($hash)
- {
- return $this->getCacheDir($hash) . '/' . $hash;
- }
}
\ No newline at end of file
Modified: incubator/shindig/trunk/php/src/common/CacheMemcache.php
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/php/src/common/CacheMemcache.php?rev=666115&r1=666114&r2=666115&view=diff
=
=
=
=
=
=
=
=
======================================================================
--- incubator/shindig/trunk/php/src/common/CacheMemcache.php
(original)
+++ incubator/shindig/trunk/php/src/common/CacheMemcache.php Tue Jun
10 07:03:44 2008
@@ -38,11 +38,44 @@
$this->port = Config::get('cache_port');
}
- public function __destruct()
+ private function isLocked($key)
{
- // if we were connected, close the connection again
- if (is_resource($this->connection)) {
- memcache_close($this->connection);
+ $this->check();
+ if ((@memcache_get($this->connection, $key . '.lock')) ===
false) {
+ return false;
+ }
+ return true;
+ }
+
+ private function createLock($key)
+ {
+ $this->check();
+ // the interesting thing is that this could fail if the lock was
created in the meantime..
+ // but we'll ignore that out of convenience
+ @memcache_add($this->connection, $key . '.lock', '', 0, 5);
+ }
+
+ private function removeLock($key)
+ {
+ $this->check();
+ // suppress all warnings, if some other process removed it that's
ok too
+ @memcache_delete($key . '.lock');
+ }
+
+ private function waitForLock($key)
+ {
+ $this->check();
+ // 20 x 250 = 5 seconds
+ $tries = 20;
+ $cnt = 0;
+ do {
+ // 250 ms is a long time to sleep, but it does stop the server
from burning all resources on polling locks..
+ usleep(250);
+ $cnt ++;
+ } while ($cnt <= $tries && $this->isLocked());
+ if ($this->isLocked()) {
+ // 5 seconds passed, assume the owning process died off and
remove it
+ $this->removeLock($key);
}
}
@@ -50,7 +83,7 @@
// so this potentially saves a lot of overhead
private function connect()
{
- if (! $this->connection = memcache_connect($this->host, $this-
>port)) {
+ if (! $this->connection = @memcache_pconnect($this->host, $this-
>port)) {
throw new CacheException("Couldn't connect to memcache
server");
}
}
@@ -62,34 +95,34 @@
}
}
- // using memcache_add behavior for cache stampeding prevention
- private function add($key, $var, $timeout)
+ public function get($key, $expiration = false)
{
$this->check();
- if (! memcache_add($this->connection, $key, $var, 0, $timeout))
{
- throw new CacheException("Couldn't add to cache");
+ if (!$expiration) {
+ // default to global cache time
+ $expiration = Config::Get('cache_time');
}
- }
-
- public function get($key)
- {
- $this->check();
- if (($ret = memcache_get($this->connection, $key)) === false) {
+ if (($ret = @memcache_get($this->connection, $key)) === false) {
return false;
}
- return $ret;
+ if (time() - $ret['time'] > $expiration) {
+ return false;
+ }
+ return $ret['data'];
}
public function set($key, $value)
{
$this->check();
- if (memcache_set($this->connection, $key, $value, 0,
Config::Get('cache_time')) === false) {
+ // we store it with the cache_time default expiration so objects
will atleast get cleaned eventually.
+ if (memcache_set($this->connection, $key, array('time' => time(),
'data' => $value), 0, Config::Get('cache_time')) === false) {
throw new CacheException("Couldn't store data in
cache");
}
}
- function delete($key)
+ public function delete($key)
{
-
+ $this->check();
+ @memcache_delete($key);
}
}
