hi, i'm using memcache, but not that class, im using other that splits the load in serveral servers.
but still, i can try it and i'll let u know ps: Cache will collapse with classes in many production systems. we should Namespace it. regards ropu On Tue, Jun 10, 2008 at 7:23 AM, Chris Chabot <[EMAIL PROTECTED]> wrote: > Bruno your using the memcache backend on a larger scale then me, could you > give this a spin? > > It should give you a bit more performance and cache stampeding prevention, > which is good mmmkay :) > > I also switched from a connect/disconnect- to a pconnect situation, should > also give a tiny performance benefit (as long as memcache doesn't complain > about the amount of connections) > > Ps, did you do any modifications to this back-end to allow multiple > memcache servers or your just using the default class? > > -- Chris > > > On Jun 10, 2008, at 4:03 PM, [EMAIL PROTECTED] wrote: > > Author: chabotc >> Date: Tue Jun 10 07:03:44 2008 >> New Revision: 666115 >> >> URL: http://svn.apache.org/viewvc?rev=666115&view=rev >> Log: >> Added some cache stampeding protection, should make a few things a lot >> smoother and faster; Also added a expiration glad to cache::get as >> preperation for the refreshInterval in the proxy code >> >> Modified: >> incubator/shindig/trunk/php/src/common/Cache.php >> incubator/shindig/trunk/php/src/common/CacheFile.php >> incubator/shindig/trunk/php/src/common/CacheMemcache.php >> >> Modified: incubator/shindig/trunk/php/src/common/Cache.php >> URL: >> http://svn.apache.org/viewvc/incubator/shindig/trunk/php/src/common/Cache.php?rev=666115&r1=666114&r2=666115&view=diff >> >> ============================================================================== >> --- incubator/shindig/trunk/php/src/common/Cache.php (original) >> +++ incubator/shindig/trunk/php/src/common/Cache.php Tue Jun 10 07:03:44 >> 2008 >> @@ -22,7 +22,7 @@ >> >> abstract class Cache { >> >> - abstract function get($key); >> + abstract function get($key, $expiration = false); >> >> abstract function set($key, $value); >> >> >> Modified: incubator/shindig/trunk/php/src/common/CacheFile.php >> URL: >> http://svn.apache.org/viewvc/incubator/shindig/trunk/php/src/common/CacheFile.php?rev=666115&r1=666114&r2=666115&view=diff >> >> ============================================================================== >> --- incubator/shindig/trunk/php/src/common/CacheFile.php (original) >> +++ incubator/shindig/trunk/php/src/common/CacheFile.php Tue Jun 10 >> 07:03:44 2008 >> @@ -28,15 +28,75 @@ >> * cache dir, and removes all files that are older then 24 hours (or >> whatever your >> * config's CACHE_TIME is set too). >> */ >> +class CacheFile extends Cache { >> >> -//TODO add cache stampeding prevention using file locking mechanisms >> + function isLocked($cacheFile) >> + { >> + // our lock file convention is simple: /the/file/path.lock >> + return file_exists($cacheFile . '.lock'); >> + } >> >> + private function createLock($cacheFile) >> + { >> + $cacheDir = dirname($cacheFile); >> + if (! is_dir($cacheDir)) { >> + if (! @mkdir($cacheDir, 0755, true)) { >> + throw new CacheException("Could not create >> cache directory"); >> + } >> + } >> + @touch($cacheFile . '.lock'); >> + } >> >> -class CacheFile extends Cache { >> + private function removeLock($cacheFile) >> + { >> + // suppress all warnings, if some other process removed it >> that's ok too >> + @unlink($cacheFile . '.lock'); >> + } >> + >> + private function waitForLock($cacheFile) >> + { >> + // 20 x 250 = 5 seconds >> + $tries = 20; >> + $cnt = 0; >> + do { >> + // make sure PHP picks up on file changes. This is >> an expensive action but really can't be avoided >> + clearstatcache(); >> + // 250 ms is a long time to sleep, but it does >> stop the server from burning all resources on polling locks.. >> + usleep(250); >> + $cnt ++; >> + } while ($cnt <= $tries && $this->isLocked()); >> + if ($this->isLocked()) { >> + // 5 seconds passed, assume the owning process >> died off and remove it >> + $this->removeLock($cacheFile); >> + } >> + } >> + >> + private function getCacheDir($hash) >> + { >> + // use the first 2 characters of the hash as a directory >> prefix >> + // this should prevent slowdowns due to huge directory >> listings >> + // and thus give some basic amount of scalability >> + return Config::get('cache_root') . '/' . substr($hash, 0, >> 2); >> + } >> + >> + private function getCacheFile($hash) >> + { >> + return $this->getCacheDir($hash) . '/' . $hash; >> + } >> >> - function get($key) >> + public function get($key, $expiration = false) >> { >> + if (! $expiration) { >> + // if no expiration time was given, fall back on >> the global config >> + $expiration = Config::get('cache_time'); >> + } >> $cacheFile = $this->getCacheFile($key); >> + // See if this cache file is locked, if so we wait upto 5 >> seconds for the lock owning process to >> + // complete it's work. If the lock is not released within >> that time frame, it's cleaned up. >> + // This should give us a fair amount of 'Cache Stampeding' >> protection >> + if ($this->isLocked($cacheFile)) { >> + $this->waitForLock($cacheFile); >> + } >> if (file_exists($cacheFile) && is_readable($cacheFile)) { >> $now = time(); >> if (($mtime = filemtime($cacheFile)) !== false && >> ($now - $mtime) < Config::get('cache_time')) { >> @@ -49,13 +109,14 @@ >> return false; >> } >> >> - function set($key, $value) >> + public function set($key, $value) >> { >> - // use the first 2 characters of the hash as a directory >> prefix >> - // this should prevent slowdowns due to huge directory >> listings >> - // and thus give some basic amount of scalability >> $cacheDir = $this->getCacheDir($key); >> $cacheFile = $this->getCacheFile($key); >> + if ($this->isLocked($cacheFile)) { >> + // some other process is writing to this file too, >> wait until it's done to prevent hickups >> + $this->waitForLock($cacheFile); >> + } >> if (! is_dir($cacheDir)) { >> if (! @mkdir($cacheDir, 0755, true)) { >> throw new CacheException("Could not create >> cache directory"); >> @@ -64,26 +125,19 @@ >> // we serialize the whole request object, since we don't >> only want the >> // responseContent but also the postBody used, headers, >> size, etc >> $data = serialize($value); >> + $this->createLock($cacheFile); >> if (! @file_put_contents($cacheFile, $data)) { >> + $this->removeLock($cacheFile); >> throw new CacheException("Could not store data in >> cache file"); >> } >> + $this->removeLock($cacheFile); >> } >> >> - function delete($key) >> + public function delete($key) >> { >> $file = $this->getCacheFile($key); >> if (! @unlink($file)) { >> throw new CacheException("Cache file could not be >> deleted"); >> } >> } >> - >> - private function getCacheDir($hash) >> - { >> - return Config::get('cache_root') . '/' . substr($hash, 0, >> 2); >> - } >> - >> - private function getCacheFile($hash) >> - { >> - return $this->getCacheDir($hash) . '/' . $hash; >> - } >> } >> \ No newline at end of file >> >> Modified: incubator/shindig/trunk/php/src/common/CacheMemcache.php >> URL: >> http://svn.apache.org/viewvc/incubator/shindig/trunk/php/src/common/CacheMemcache.php?rev=666115&r1=666114&r2=666115&view=diff >> >> ============================================================================== >> --- incubator/shindig/trunk/php/src/common/CacheMemcache.php (original) >> +++ incubator/shindig/trunk/php/src/common/CacheMemcache.php Tue Jun 10 >> 07:03:44 2008 >> @@ -38,11 +38,44 @@ >> $this->port = Config::get('cache_port'); >> } >> >> - public function __destruct() >> + private function isLocked($key) >> { >> - // if we were connected, close the connection again >> - if (is_resource($this->connection)) { >> - memcache_close($this->connection); >> + $this->check(); >> + if ((@memcache_get($this->connection, $key . '.lock')) === >> false) { >> + return false; >> + } >> + return true; >> + } >> + >> + private function createLock($key) >> + { >> + $this->check(); >> + // the interesting thing is that this could fail if the >> lock was created in the meantime.. >> + // but we'll ignore that out of convenience >> + @memcache_add($this->connection, $key . '.lock', '', 0, >> 5); >> + } >> + >> + private function removeLock($key) >> + { >> + $this->check(); >> + // suppress all warnings, if some other process removed it >> that's ok too >> + @memcache_delete($key . '.lock'); >> + } >> + >> + private function waitForLock($key) >> + { >> + $this->check(); >> + // 20 x 250 = 5 seconds >> + $tries = 20; >> + $cnt = 0; >> + do { >> + // 250 ms is a long time to sleep, but it does >> stop the server from burning all resources on polling locks.. >> + usleep(250); >> + $cnt ++; >> + } while ($cnt <= $tries && $this->isLocked()); >> + if ($this->isLocked()) { >> + // 5 seconds passed, assume the owning process >> died off and remove it >> + $this->removeLock($key); >> } >> } >> >> @@ -50,7 +83,7 @@ >> // so this potentially saves a lot of overhead >> private function connect() >> { >> - if (! $this->connection = memcache_connect($this->host, >> $this->port)) { >> + if (! $this->connection = @memcache_pconnect($this->host, >> $this->port)) { >> throw new CacheException("Couldn't connect to >> memcache server"); >> } >> } >> @@ -62,34 +95,34 @@ >> } >> } >> >> - // using memcache_add behavior for cache stampeding prevention >> - private function add($key, $var, $timeout) >> + public function get($key, $expiration = false) >> { >> $this->check(); >> - if (! memcache_add($this->connection, $key, $var, 0, >> $timeout)) { >> - throw new CacheException("Couldn't add to cache"); >> + if (!$expiration) { >> + // default to global cache time >> + $expiration = Config::Get('cache_time'); >> } >> - } >> - >> - public function get($key) >> - { >> - $this->check(); >> - if (($ret = memcache_get($this->connection, $key)) === >> false) { >> + if (($ret = @memcache_get($this->connection, $key)) === >> false) { >> return false; >> } >> - return $ret; >> + if (time() - $ret['time'] > $expiration) { >> + return false; >> + } >> + return $ret['data']; >> } >> >> public function set($key, $value) >> { >> $this->check(); >> - if (memcache_set($this->connection, $key, $value, 0, >> Config::Get('cache_time')) === false) { >> + // we store it with the cache_time default expiration so >> objects will atleast get cleaned eventually. >> + if (memcache_set($this->connection, $key, array('time' => >> time(), 'data' => $value), 0, Config::Get('cache_time')) === false) { >> throw new CacheException("Couldn't store data in >> cache"); >> } >> } >> >> - function delete($key) >> + public function delete($key) >> { >> - >> + $this->check(); >> + @memcache_delete($key); >> } >> } >> >> > -- .-. --- .--. ..- R o p u
