Hi all.
There has been no discussion on this thread for a while. The latest
patch version (attached) fixes all the issues from Amos and Alex I know
of. So I would like to get an approval for the patch or hints at what
else needs to be fixed.
The patch does not change the way unlinkd is started. But after the
changes, unlinkd is started iff it is needed. Whether unlinkd is needed
is determined by both cache dir and DiskIO strategy. Unlinkd may be
started during reconfiguration if a cache dir that needs it was added or
DiskIO strategy changed.
There was discussion about starting unlinkd on demand. But the
consensus seems to be that it Squid needs substantial changes to
minimize fork() performance penalty. So while we want to implement this
long term, for now we should start unlinkd on startup and
reconfiguration.
Please let me know if I missed any comments or issues with the patch.
Regards,
Dmitry
Bug 3150: do not start useless unlinkd.
Unlinkd may be used only by UFS storage. But before the change, Squid
always started unlinkd if it is built, even if it is not needed.
Whether a SwapDir may use unlinkd depends on the SwapDir
implementation and DiskIO strategy it uses. The patch adds
unlinkdUseful() method to SwapDir and DiskIOStrategy to decide if
unlinkd should be started.
After the change, unlinkd may be started during reconfiguration and
unlinkdInit() may be called multiple times.
After the change, unlinkdClose() may be called when unlinkd was never
started. The patch removes a warning which was printed in this case
on Windows.
diff --git src/DiskIO/AIO/AIODiskIOStrategy.cc src/DiskIO/AIO/AIODiskIOStrategy.cc
index 96e11f4..7a94637 100644
--- src/DiskIO/AIO/AIODiskIOStrategy.cc
+++ src/DiskIO/AIO/AIODiskIOStrategy.cc
@@ -83,40 +83,46 @@ AIODiskIOStrategy::newFile (char const *path)
}
return new AIODiskFile (path, this);
}
void
AIODiskIOStrategy::sync()
{
assert(aq.aq_state == AQ_STATE_SETUP);
/*
* Keep calling callback to complete ops until the queue is empty
* We can't quit when callback returns 0 - some calls may not
* return any completed pending events, but they're still pending!
*/
while (aq.aq_numpending)
callback();
}
+bool
+AIODiskIOStrategy::unlinkdUseful() const
+{
+ return false;
+}
+
void
AIODiskIOStrategy::unlinkFile (char const *)
{}
/*
* Note: we grab the state and free the state before calling the callback
* because this allows us to cut down the amount of time it'll take
* to find a free slot (since if we call the callback first, we're going
* to probably be allocated the slot _after_ this one..)
*
* I'll make it much more optimal later.
*/
int
AIODiskIOStrategy::callback()
{
return 0;
int i;
int completed = 0;
int retval, reterr;
FREE *freefunc;
diff --git src/DiskIO/AIO/AIODiskIOStrategy.h src/DiskIO/AIO/AIODiskIOStrategy.h
index 9d461d0..c3c7d34 100644
--- src/DiskIO/AIO/AIODiskIOStrategy.h
+++ src/DiskIO/AIO/AIODiskIOStrategy.h
@@ -35,40 +35,42 @@
#if USE_DISKIO_AIO
#include "DiskIO/DiskIOStrategy.h"
#include "async_io.h"
class AIODiskIOStrategy : public DiskIOStrategy
{
public:
AIODiskIOStrategy();
virtual ~AIODiskIOStrategy();
virtual bool shedLoad();
/* What is the current load? 999 = 99.9% */
virtual int load();
/* Return a handle for performing IO operations */
virtual RefCount<DiskFile> newFile (char const *path);
/* flush all IO operations */
virtual void sync();
+ /** whether the IO Strategy can use unlinkd */
+ virtual bool unlinkdUseful() const;
/* unlink a file by path */
virtual void unlinkFile (char const *);
/* perform any pending callbacks */
virtual int callback();
/* Init per-instance logic */
virtual void init();
/* cachemgr output on the IO instance stats */
virtual void statfs(StoreEntry & sentry)const;
/* module specific options */
virtual ConfigOption *getOptionTree() const;
/* a file descriptor */
int fd;
/* queue of requests */
async_queue_t aq;
int findSlot();
};
diff --git src/DiskIO/Blocking/BlockingIOStrategy.cc src/DiskIO/Blocking/BlockingIOStrategy.cc
index 9a61973..8610037 100644
--- src/DiskIO/Blocking/BlockingIOStrategy.cc
+++ src/DiskIO/Blocking/BlockingIOStrategy.cc
@@ -40,29 +40,35 @@
bool
BlockingIOStrategy::shedLoad()
{
return false;
}
int
BlockingIOStrategy::load()
{
/* Return 999 (99.9%) constant load */
return 999;
}
DiskFile::Pointer
BlockingIOStrategy::newFile (char const *path)
{
return new BlockingFile (path);
}
+bool
+BlockingIOStrategy::unlinkdUseful() const
+{
+ return true;
+}
+
void
BlockingIOStrategy::unlinkFile(char const *path)
{
#if USE_UNLINKD
unlinkdUnlink(path);
#else
::unlink(path);
#endif
}
diff --git src/DiskIO/Blocking/BlockingIOStrategy.h src/DiskIO/Blocking/BlockingIOStrategy.h
index 5dea6e2..8181dd1 100644
--- src/DiskIO/Blocking/BlockingIOStrategy.h
+++ src/DiskIO/Blocking/BlockingIOStrategy.h
@@ -28,24 +28,25 @@
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
*
* Copyright (c) 2003, Robert Collins <[email protected]>
*/
#ifndef SQUID_BLOCKINGIOSTRATEGY_H
#define SQUID_BLOCKINGIOSTRATEGY_H
#include "DiskIO/DiskIOStrategy.h"
class BlockingIOStrategy : public DiskIOStrategy
{
public:
virtual bool shedLoad();
virtual int load();
virtual RefCount<DiskFile> newFile(char const *path);
+ virtual bool unlinkdUseful() const;
virtual void unlinkFile (char const *);
};
#endif /* SQUID_BLOCKINGIOSTRATEGY_H */
diff --git src/DiskIO/DiskDaemon/DiskdIOStrategy.cc src/DiskIO/DiskDaemon/DiskdIOStrategy.cc
index 6d054e3..64c7012 100644
--- src/DiskIO/DiskDaemon/DiskdIOStrategy.cc
+++ src/DiskIO/DiskDaemon/DiskdIOStrategy.cc
@@ -86,40 +86,46 @@ DiskdIOStrategy::load()
void
DiskdIOStrategy::openFailed()
{
diskd_stats.open_fail_queue_len++;
}
DiskFile::Pointer
DiskdIOStrategy::newFile(char const *path)
{
if (shedLoad()) {
openFailed();
return NULL;
}
return new DiskdFile (path, this);
}
DiskdIOStrategy::DiskdIOStrategy() : magic1(64), magic2(72), away(0) , smsgid(-1), rmsgid(-1), wfd(-1) , instanceID(newInstance())
{}
+bool
+DiskdIOStrategy::unlinkdUseful() const
+{
+ return true;
+}
+
void
DiskdIOStrategy::unlinkFile(char const *path)
{
if (shedLoad()) {
/* Damn, we need to issue a sync unlink here :( */
debugs(79, 2, "storeDiskUnlink: Out of queue space, sync unlink");
#if USE_UNLINKD
unlinkdUnlink(path);
#else
unlink(path);
#endif
return;
}
/* We can attempt a diskd unlink */
int x;
diff --git src/DiskIO/DiskDaemon/DiskdIOStrategy.h src/DiskIO/DiskDaemon/DiskdIOStrategy.h
index 161d7ed..650400e 100644
--- src/DiskIO/DiskDaemon/DiskdIOStrategy.h
+++ src/DiskIO/DiskDaemon/DiskdIOStrategy.h
@@ -59,40 +59,41 @@ public:
};
#include "DiskIO/DiskIOStrategy.h"
#include "StoreIOState.h"
class DiskFile;
class DiskdFile;
class ReadRequest;
/// \ingroup diskd
class DiskdIOStrategy : public DiskIOStrategy
{
public:
DiskdIOStrategy();
virtual bool shedLoad();
virtual int load();
virtual RefCount<DiskFile> newFile(char const *path);
+ virtual bool unlinkdUseful() const;
virtual void unlinkFile (char const *);
virtual ConfigOption *getOptionTree() const;
virtual void init();
virtual void sync();
virtual int callback();
virtual void statfs(StoreEntry & sentry)const;
int send(int mtype, int id, DiskdFile *theFile, size_t size, off_t offset, ssize_t shm_offset, RefCountable_ *requestor);
/** public for accessing return address's */
SharedMemory shm;
private:
static size_t newInstance();
static size_t nextInstanceID;
void openFailed();
bool optionQ1Parse(char const *option, const char *value, int reconfiguring);
void optionQ1Dump(StoreEntry * e) const;
bool optionQ2Parse(char const *option, const char *value, int reconfiguring);
void optionQ2Dump(StoreEntry * e) const;
int send(int mtype, int id, RefCount<StoreIOState> sio, size_t size, off_t offset, ssize_t shm_offset);
diff --git src/DiskIO/DiskIOStrategy.h src/DiskIO/DiskIOStrategy.h
index 2ec5617..91990d2 100644
--- src/DiskIO/DiskIOStrategy.h
+++ src/DiskIO/DiskIOStrategy.h
@@ -42,68 +42,73 @@ class DiskFile;
class ConfigOption;
class DiskIOStrategy
{
public:
virtual ~DiskIOStrategy() {}
/** Can the IO Strategy handle more requests ? */
virtual bool shedLoad() = 0;
/** What is the current load? 999 = 99.9% */
virtual int load() = 0;
/** Return a handle for performing IO operations */
virtual RefCount<DiskFile> newFile(char const *path) = 0;
/** flush all IO operations */
virtual void sync() {}
+ /** whether the IO Strategy can use unlinkd */
+ virtual bool unlinkdUseful() const = 0;
+
/** unlink a file by path */
virtual void unlinkFile(char const *) = 0;
/** perform any pending callbacks */
virtual int callback() { return 0; }
/** Init per-instance logic */
virtual void init() {}
/** cachemgr output on the IO instance stats */
virtual void statfs(StoreEntry & sentry)const {}
/** module specific options */
virtual ConfigOption *getOptionTree() const { return NULL;}
};
/* Because we need the DiskFile definition for newFile. */
#include "DiskFile.h"
class SingletonIOStrategy : public DiskIOStrategy
{
public:
SingletonIOStrategy(DiskIOStrategy *anIO) : io(anIO) {}
virtual bool shedLoad() { return io->shedLoad(); }
virtual int load() { return io->load(); }
virtual RefCount<DiskFile> newFile (char const *path) {return io->newFile(path); }
virtual void sync() { io->sync(); }
+ virtual bool unlinkdUseful() const { return io->unlinkdUseful(); }
+
virtual void unlinkFile (char const *path) { io->unlinkFile(path); }
virtual int callback() { return io->callback(); }
virtual void init() { io->init(); }
virtual void statfs(StoreEntry & sentry)const { io->statfs(sentry); }
virtual ConfigOption *getOptionTree() const { return io->getOptionTree(); }
private:
DiskIOStrategy *io;
};
#endif /* SQUID_DISKIOSTRATEGY_H */
diff --git src/DiskIO/DiskThreads/DiskThreadsIOStrategy.cc src/DiskIO/DiskThreads/DiskThreadsIOStrategy.cc
index e7c0fd9..1ba1ecf 100644
--- src/DiskIO/DiskThreads/DiskThreadsIOStrategy.cc
+++ src/DiskIO/DiskThreads/DiskThreadsIOStrategy.cc
@@ -237,26 +237,32 @@ DiskThreadsIOStrategy::load()
if (ql == 0)
loadav = 0;
loadav = ql * 1000 / MAGIC1;
debugs(47, 9, "DiskThreadsIOStrategy::load: load=" << loadav);
return loadav;
}
DiskFile::Pointer
DiskThreadsIOStrategy::newFile (char const *path)
{
if (shedLoad()) {
return NULL;
}
return new DiskThreadsDiskFile (path, this);
}
+bool
+DiskThreadsIOStrategy::unlinkdUseful() const
+{
+ return false;
+}
+
void
DiskThreadsIOStrategy::unlinkFile(char const *path)
{
statCounter.syscalls.disk.unlinks++;
aioUnlink(path, NULL, NULL);
}
diff --git src/DiskIO/DiskThreads/DiskThreadsIOStrategy.h src/DiskIO/DiskThreads/DiskThreadsIOStrategy.h
index 12da076..d7e54b1 100644
--- src/DiskIO/DiskThreads/DiskThreadsIOStrategy.h
+++ src/DiskIO/DiskThreads/DiskThreadsIOStrategy.h
@@ -37,36 +37,37 @@
#ifndef __STORE_DISKTHREADEDIOSTRATEGY_H__
#define __STORE_DISKTHREADEDIOSTRATEGY_H__
#define _AIO_OPEN 0
#define _AIO_READ 1
#define _AIO_WRITE 2
#define _AIO_CLOSE 3
#define _AIO_UNLINK 4
#define _AIO_OPENDIR 5
#define _AIO_STAT 6
#include "DiskIO/DiskIOStrategy.h"
class DiskThreadsIOStrategy : public DiskIOStrategy
{
public:
DiskThreadsIOStrategy();
virtual bool shedLoad();
virtual int load();
virtual RefCount<DiskFile> newFile(char const *path);
+ virtual bool unlinkdUseful() const;
virtual void unlinkFile (char const *);
virtual int callback();
virtual void sync();
virtual void init();
void done();
/* Todo: add access limitations */
bool initialised;
static DiskThreadsIOStrategy Instance;
MemAllocator *squidaio_ctrl_pool;
private:
static void aioStats(StoreEntry * sentry);
void registerWithCacheManager(void);
};
#endif
diff --git src/DiskIO/IpcIo/IpcIoIOStrategy.cc src/DiskIO/IpcIo/IpcIoIOStrategy.cc
index 10a165e..dc3a191 100644
--- src/DiskIO/IpcIo/IpcIoIOStrategy.cc
+++ src/DiskIO/IpcIo/IpcIoIOStrategy.cc
@@ -11,29 +11,35 @@
bool
IpcIoIOStrategy::shedLoad()
{
return false;
}
int
IpcIoIOStrategy::load()
{
/* Return 999 (99.9%) constant load */
return 999;
}
DiskFile::Pointer
IpcIoIOStrategy::newFile (char const *path)
{
return new IpcIoFile (path);
}
+bool
+IpcIoIOStrategy::unlinkdUseful() const
+{
+ return true;
+}
+
void
IpcIoIOStrategy::unlinkFile(char const *path)
{
#if USE_UNLINKD
unlinkdUnlink(path);
#else
::unlink(path);
#endif
}
diff --git src/DiskIO/IpcIo/IpcIoIOStrategy.h src/DiskIO/IpcIo/IpcIoIOStrategy.h
index 3bd2945..447e8e3 100644
--- src/DiskIO/IpcIo/IpcIoIOStrategy.h
+++ src/DiskIO/IpcIo/IpcIoIOStrategy.h
@@ -1,15 +1,16 @@
#ifndef SQUID_IPC_IOIOSTRATEGY_H
#define SQUID_IPC_IOIOSTRATEGY_H
#include "DiskIO/DiskIOStrategy.h"
class IpcIoIOStrategy : public DiskIOStrategy
{
public:
virtual bool shedLoad();
virtual int load();
virtual RefCount<DiskFile> newFile(char const *path);
+ virtual bool unlinkdUseful() const;
virtual void unlinkFile (char const *);
};
#endif /* SQUID_IPC_IOIOSTRATEGY_H */
diff --git src/DiskIO/Mmapped/MmappedIOStrategy.cc src/DiskIO/Mmapped/MmappedIOStrategy.cc
index 6f01590..3f6d681 100644
--- src/DiskIO/Mmapped/MmappedIOStrategy.cc
+++ src/DiskIO/Mmapped/MmappedIOStrategy.cc
@@ -11,29 +11,35 @@
bool
MmappedIOStrategy::shedLoad()
{
return false;
}
int
MmappedIOStrategy::load()
{
/* Return 999 (99.9%) constant load */
return 999;
}
DiskFile::Pointer
MmappedIOStrategy::newFile (char const *path)
{
return new MmappedFile (path);
}
+bool
+MmappedIOStrategy::unlinkdUseful() const
+{
+ return true;
+}
+
void
MmappedIOStrategy::unlinkFile(char const *path)
{
#if USE_UNLINKD
unlinkdUnlink(path);
#else
::unlink(path);
#endif
}
diff --git src/DiskIO/Mmapped/MmappedIOStrategy.h src/DiskIO/Mmapped/MmappedIOStrategy.h
index c295427..2d17524 100644
--- src/DiskIO/Mmapped/MmappedIOStrategy.h
+++ src/DiskIO/Mmapped/MmappedIOStrategy.h
@@ -1,15 +1,16 @@
#ifndef SQUID_MMAPPEDIOSTRATEGY_H
#define SQUID_MMAPPEDIOSTRATEGY_H
#include "DiskIO/DiskIOStrategy.h"
class MmappedIOStrategy : public DiskIOStrategy
{
public:
virtual bool shedLoad();
virtual int load();
virtual RefCount<DiskFile> newFile(char const *path);
+ virtual bool unlinkdUseful() const;
virtual void unlinkFile (char const *);
};
#endif /* SQUID_MMAPPEDIOSTRATEGY_H */
diff --git src/SwapDir.h src/SwapDir.h
index 723443c..00d7a03 100644
--- src/SwapDir.h
+++ src/SwapDir.h
@@ -115,40 +115,42 @@ SQUIDCEXTERN void storeDirLRUDelete(StoreEntry *);
SQUIDCEXTERN void storeDirLRUAdd(StoreEntry *);
SQUIDCEXTERN int storeDirGetBlkSize(const char *path, int *blksize);
SQUIDCEXTERN int storeDirGetUFSStats(const char *, int *, int *, int *, int *);
/// manages a single cache_dir
class SwapDir : public Store
{
public:
typedef RefCount<SwapDir> Pointer;
SwapDir(char const *aType);
virtual ~SwapDir();
virtual void reconfigure() = 0;
char const *type() const;
virtual bool needsDiskStrand() const; ///< needs a dedicated kid process
virtual bool active() const; ///< may be used in this strand
/// whether stat should be reported by this SwapDir
virtual bool doReportStat() const { return active(); }
+ /// whether SwapDir may benefit from unlinkd
+ virtual bool unlinkdUseful() const = 0;
/* official Store interface functions */
virtual void diskFull();
virtual StoreEntry * get(const cache_key *);
virtual void get(String const, STOREGETCLIENT, void * cbdata);
virtual uint64_t maxSize() const { return max_size;}
virtual uint64_t minSize() const;
virtual int64_t maxObjectSize() const { return max_objsize; }
virtual void stat (StoreEntry &anEntry) const;
virtual StoreSearch *search(String const url, HttpRequest *) = 0;
/* migrated from store_dir.cc */
bool objectSizeIsAcceptable(int64_t objsize) const;
diff --git src/fs/coss/CossSwapDir.h src/fs/coss/CossSwapDir.h
index 6929d81..8279d27 100644
--- src/fs/coss/CossSwapDir.h
+++ src/fs/coss/CossSwapDir.h
@@ -18,40 +18,41 @@ class DiskFile;
#endif
/* Note that swap_filen in sio/e are actually disk offsets too! */
/* What we're doing in storeCossAllocate() */
#define COSS_ALLOC_ALLOCATE 1
#define COSS_ALLOC_REALLOC 2
/// \ingroup COSS
class CossSwapDir : public SwapDir, public IORequestor
{
public:
CossSwapDir();
virtual void init();
virtual void create();
virtual void dump(StoreEntry &)const;
~CossSwapDir();
virtual StoreSearch *search(String const url, HttpRequest *);
+ virtual bool unlinkdUseful() const;
virtual void unlink (StoreEntry &);
virtual void statfs (StoreEntry &)const;
virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const;
virtual int callback();
virtual void sync();
virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *);
virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *);
virtual void openLog();
virtual void closeLog();
virtual int writeCleanStart();
virtual void writeCleanDone();
virtual void logEntry(const StoreEntry & e, int op) const;
virtual void parse (int index, char *path);
virtual void reconfigure();
virtual void swappedOut(const StoreEntry &e);
virtual uint64_t currentSize() const { return cur_size; }
virtual uint64_t currentCount() const { return n_disk_objects; }
/* internals */
virtual off_t storeCossFilenoToDiskOffset(sfileno);
virtual sfileno storeCossDiskOffsetToFileno(off_t);
diff --git src/fs/coss/store_io_coss.cc src/fs/coss/store_io_coss.cc
index bb4c09d..8019d13 100644
--- src/fs/coss/store_io_coss.cc
+++ src/fs/coss/store_io_coss.cc
@@ -112,40 +112,47 @@ CossSwapDir::allocate(const StoreEntry * e, int which)
current_membuf = newmb;
}
/*
* If we didn't get a collision, then update the current offset
* and return it
*/
if (coll == 0) {
retofs = current_offset;
current_offset = retofs + allocsize;
/* round up to our blocksize */
current_offset = ((current_offset + blksz_mask) >> blksz_bits ) << blksz_bits;
return storeCossDiskOffsetToFileno(retofs);
} else {
StoreFScoss::GetInstance().stats.alloc.collisions++;
debugs(79, 3, "CossSwapDir::allocate: Collision");
return -1;
}
}
+bool
+CossSwapDir::unlinkdUseful() const
+{
+ // no entry-specific files to unlink
+ return false;
+}
+
void
CossSwapDir::unlink(StoreEntry & e)
{
debugs(79, 3, "storeCossUnlink: offset " << e.swap_filen);
if (e.swap_status == SWAPOUT_DONE && EBIT_TEST(e.flags, ENTRY_VALIDATED)) {
cur_size -= fs.blksize * sizeInBlocks(e.swap_file_sz);
--n_disk_objects;
}
StoreFScoss::GetInstance().stats.unlink.ops++;
StoreFScoss::GetInstance().stats.unlink.success++;
storeCossRemove(this, &e);
}
StoreIOState::Pointer
CossSwapDir::createStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * callback, void *callback_data)
{
CossState *cstate;
StoreIOState::Pointer sio = new CossState(this);
cstate = dynamic_cast<CossState *>(sio.getRaw());
sio->offset_ = 0;
diff --git src/fs/rock/RockSwapDir.cc src/fs/rock/RockSwapDir.cc
index 8924d43..6afea2c 100644
--- src/fs/rock/RockSwapDir.cc
+++ src/fs/rock/RockSwapDir.cc
@@ -727,40 +727,47 @@ Rock::SwapDir::maintain()
void
Rock::SwapDir::reference(StoreEntry &e)
{
debugs(47, 5, HERE << &e << ' ' << e.swap_dirn << ' ' << e.swap_filen);
if (repl && repl->Referenced)
repl->Referenced(repl, &e, &e.repl);
}
bool
Rock::SwapDir::dereference(StoreEntry &e)
{
debugs(47, 5, HERE << &e << ' ' << e.swap_dirn << ' ' << e.swap_filen);
if (repl && repl->Dereferenced)
repl->Dereferenced(repl, &e, &e.repl);
// no need to keep e in the global store_table for us; we have our own map
return false;
}
+bool
+Rock::SwapDir::unlinkdUseful() const
+{
+ // no entry-specific files to unlink
+ return false;
+}
+
void
Rock::SwapDir::unlink(StoreEntry &e)
{
debugs(47, 5, HERE << e);
ignoreReferences(e);
map->free(e.swap_filen);
disconnect(e);
}
void
Rock::SwapDir::trackReferences(StoreEntry &e)
{
debugs(47, 5, HERE << e);
if (repl)
repl->Add(repl, &e, &e.repl);
}
void
Rock::SwapDir::ignoreReferences(StoreEntry &e)
diff --git src/fs/rock/RockSwapDir.h src/fs/rock/RockSwapDir.h
index e2b8e37..b05d1e7 100644
--- src/fs/rock/RockSwapDir.h
+++ src/fs/rock/RockSwapDir.h
@@ -36,40 +36,41 @@ public:
int64_t entryLimitHigh() const { return SwapFilenMax; } ///< Core limit
int64_t entryLimitAllowed() const;
typedef Ipc::StoreMapWithExtras<DbCellHeader> DirMap;
protected:
/* protected ::SwapDir API */
virtual bool needsDiskStrand() const;
virtual void create();
virtual void init();
virtual ConfigOption *getOptionTree() const;
virtual bool allowOptionReconfigure(const char *const option) const;
virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const;
virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *);
virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *);
virtual void maintain();
virtual void diskFull();
virtual void reference(StoreEntry &e);
virtual bool dereference(StoreEntry &e);
+ virtual bool unlinkdUseful() const;
virtual void unlink(StoreEntry &e);
virtual void statfs(StoreEntry &e) const;
/* IORequestor API */
virtual void ioCompletedNotification();
virtual void closeCompleted();
virtual void readCompleted(const char *buf, int len, int errflag, RefCount< ::ReadRequest>);
virtual void writeCompleted(int errflag, size_t len, RefCount< ::WriteRequest>);
virtual void parse(int index, char *path);
void parseSize(const bool reconfiguring); ///< parses anonymous cache_dir size option
void validateOptions(); ///< warns of configuration problems; may quit
bool parseTimeOption(char const *option, const char *value, int reconfiguring);
void dumpTimeOption(StoreEntry * e) const;
bool parseRateOption(char const *option, const char *value, int reconfiguring);
void dumpRateOption(StoreEntry * e) const;
void rebuild(); ///< starts loading and validating stored entry metadata
///< used to add entries successfully loaded during rebuild
bool addEntry(const int fileno, const DbCellHeader &header, const StoreEntry &from);
diff --git src/fs/ufs/store_dir_ufs.cc src/fs/ufs/store_dir_ufs.cc
index 129629c..7ee1df8 100644
--- src/fs/ufs/store_dir_ufs.cc
+++ src/fs/ufs/store_dir_ufs.cc
@@ -1270,40 +1270,47 @@ UFSSwapDir::validFileno(sfileno filn, int flag) const
/*
* UFSSwapDir::unlinkFile
*
* This routine unlinks a file and pulls it out of the bitmap.
* It used to be in commonUfsUnlink(), however an interface change
* forced this bit of code here. Eeek.
*/
void
UFSSwapDir::unlinkFile(sfileno f)
{
debugs(79, 3, "UFSSwapDir::unlinkFile: unlinking fileno " << std::setfill('0') <<
std::hex << std::uppercase << std::setw(8) << f << " '" <<
fullPath(f,NULL) << "'");
/* commonUfsDirMapBitReset(this, f); */
IO->unlinkFile(fullPath(f,NULL));
}
+bool
+UFSSwapDir::unlinkdUseful() const
+{
+ // unlinkd may be useful only in workers
+ return IamWorkerProcess() && IO->io->unlinkdUseful();
+}
+
void
UFSSwapDir::unlink(StoreEntry & e)
{
debugs(79, 3, "storeUfsUnlink: dirno " << index << ", fileno "<<
std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen);
if (e.swap_status == SWAPOUT_DONE && EBIT_TEST(e.flags, ENTRY_VALIDATED)) {
cur_size -= fs.blksize * sizeInBlocks(e.swap_file_sz);
--n_disk_objects;
}
replacementRemove(&e);
mapBitReset(e.swap_filen);
UFSSwapDir::unlinkFile(e.swap_filen);
}
/*
* Add and remove the given StoreEntry from the replacement policy in
* use.
*/
void
diff --git src/fs/ufs/ufscommon.h src/fs/ufs/ufscommon.h
index 7561e8b..3af0619 100644
--- src/fs/ufs/ufscommon.h
+++ src/fs/ufs/ufscommon.h
@@ -42,40 +42,41 @@ class DiskIOModule;
class StoreSearch;
#include "SwapDir.h"
/// \ingroup UFS
class UFSSwapDir : public SwapDir
{
public:
static int IsUFSDir(SwapDir* sd);
static int DirClean(int swap_index);
static int FilenoBelongsHere(int fn, int F0, int F1, int F2);
UFSSwapDir(char const *aType, const char *aModuleType);
virtual void init();
virtual void create();
virtual void dump(StoreEntry &) const;
~UFSSwapDir();
virtual StoreSearch *search(String const url, HttpRequest *);
virtual bool doubleCheck(StoreEntry &);
+ virtual bool unlinkdUseful() const;
virtual void unlink(StoreEntry &);
virtual void statfs(StoreEntry &)const;
virtual void maintain();
virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const;
virtual void reference(StoreEntry &);
virtual bool dereference(StoreEntry &);
virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *);
virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *);
virtual void openLog();
virtual void closeLog();
virtual int writeCleanStart();
virtual void writeCleanDone();
virtual void logEntry(const StoreEntry & e, int op) const;
virtual void parse(int index, char *path);
virtual void reconfigure();
virtual int callback();
virtual void sync();
virtual void swappedOut(const StoreEntry &e);
virtual uint64_t currentSize() const { return cur_size; }
virtual uint64_t currentCount() const { return n_disk_objects; }
diff --git src/main.cc src/main.cc
index 546eb92..56eadc4 100644
--- src/main.cc
+++ src/main.cc
@@ -855,40 +855,45 @@ mainReconfigureFinish(void *)
if (IamPrimaryProcess()) {
#if USE_WCCP
wccpInit();
#endif
#if USE_WCCPv2
wccp2Init();
#endif
}
serverConnectionsOpen();
neighbors_init();
storeDirOpenSwapLogs();
mimeInit(Config.mimeTablePathname);
+#if USE_UNLINKD
+ if (unlinkdNeeded())
+ unlinkdInit();
+#endif
+
#if USE_DELAY_POOLS
Config.ClientDelay.finalize();
#endif
if (Config.onoff.announce) {
if (!eventFind(start_announce, NULL))
eventAdd("start_announce", start_announce, NULL, 3600.0, 1);
} else {
if (eventFind(start_announce, NULL))
eventDelete(start_announce, NULL);
}
writePidFile(); /* write PID file */
debugs(1, 1, "Ready to serve requests.");
reconfiguring = 0;
}
static void
@@ -1068,41 +1073,42 @@ mainInitialize(void)
icapLogOpen();
#endif
#if USE_IDENT
Ident::Init();
#endif
#if SQUID_SNMP
snmpInit();
#endif
#if MALLOC_DBG
malloc_debug(0, malloc_debug_level);
#endif
if (!configured_once) {
#if USE_UNLINKD
- unlinkdInit();
+ if (unlinkdNeeded())
+ unlinkdInit();
#endif
urlInitialize();
statInit();
storeInit();
mainSetCwd();
/* after this point we want to see the mallinfo() output */
do_mallinfo = 1;
mimeInit(Config.mimeTablePathname);
refreshInit();
#if USE_DELAY_POOLS
DelayPools::Init();
#endif
FwdState::initModule();
/* register the modules in the cache manager menus */
cbdataRegisterWithCacheManager();
/* These use separate calls so that the comm loops can eventually
* coexist.
diff --git src/protos.h src/protos.h
index 46cb7b8..00bae33 100644
--- src/protos.h
+++ src/protos.h
@@ -574,40 +574,41 @@ SQUIDCEXTERN int DebugSignal;
/* AYJ debugs function to show locations being reset with memset() */
SQUIDCEXTERN void *xmemset(void *dst, int, size_t);
SQUIDCEXTERN void debug_trap(const char *);
SQUIDCEXTERN void logsFlush(void);
SQUIDCEXTERN const char *checkNullString(const char *p);
SQUIDCEXTERN void squid_getrusage(struct rusage *r);
SQUIDCEXTERN double rusage_cputime(struct rusage *r);
SQUIDCEXTERN int rusage_maxrss(struct rusage *r);
SQUIDCEXTERN int rusage_pagefaults(struct rusage *r);
SQUIDCEXTERN void releaseServerSockets(void);
SQUIDCEXTERN void PrintRusage(void);
SQUIDCEXTERN void dumpMallocStats(void);
#if USE_UNLINKD
+SQUIDCEXTERN bool unlinkdNeeded(void);
SQUIDCEXTERN void unlinkdInit(void);
SQUIDCEXTERN void unlinkdClose(void);
SQUIDCEXTERN void unlinkdUnlink(const char *);
#endif
SQUIDCEXTERN AnyP::ProtocolType urlParseProtocol(const char *, const char *e = NULL);
SQUIDCEXTERN void urlInitialize(void);
SQUIDCEXTERN HttpRequest *urlParse(const HttpRequestMethod&, char *, HttpRequest *request = NULL);
SQUIDCEXTERN const char *urlCanonical(HttpRequest *);
SQUIDCEXTERN char *urlCanonicalClean(const HttpRequest *);
SQUIDCEXTERN const char *urlCanonicalFakeHttps(const HttpRequest * request);
SQUIDCEXTERN bool urlIsRelative(const char *);
SQUIDCEXTERN char *urlMakeAbsolute(const HttpRequest *, const char *);
SQUIDCEXTERN char *urlRInternal(const char *host, unsigned short port, const char *dir, const char *name);
SQUIDCEXTERN char *urlInternal(const char *dir, const char *name);
SQUIDCEXTERN int matchDomainName(const char *host, const char *domain);
SQUIDCEXTERN int urlCheckRequest(const HttpRequest *);
SQUIDCEXTERN int urlDefaultPort(AnyP::ProtocolType p);
SQUIDCEXTERN char *urlHostname(const char *url);
SQUIDCEXTERN void urlExtMethodConfigure(void);
diff --git src/tests/TestSwapDir.cc src/tests/TestSwapDir.cc
index d99d28b..b650b64 100644
--- src/tests/TestSwapDir.cc
+++ src/tests/TestSwapDir.cc
@@ -19,40 +19,46 @@ uint64_t
TestSwapDir::currentCount() const
{
return 2;
}
void
TestSwapDir::stat(StoreEntry &) const
{
const_cast<TestSwapDir *>(this)->statsCalled = true;
}
void
TestSwapDir::reconfigure()
{}
void
TestSwapDir::init()
{}
bool
+TestSwapDir::unlinkdUseful() const
+{
+ return false;
+}
+
+bool
TestSwapDir::canStore(const StoreEntry &, int64_t, int &load) const
{
load = 0;
return true;
}
StoreIOState::Pointer
TestSwapDir::createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *)
{
return NULL;
}
StoreIOState::Pointer
TestSwapDir::openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *)
{
return NULL;
}
void
TestSwapDir::parse(int, char*)
diff --git src/tests/TestSwapDir.h src/tests/TestSwapDir.h
index e5d51cb..ad6f81d 100644
--- src/tests/TestSwapDir.h
+++ src/tests/TestSwapDir.h
@@ -3,30 +3,31 @@
#include "squid.h"
#include "SwapDir.h"
class TestSwapDir : public SwapDir
{
public:
TestSwapDir() : SwapDir("test"), statsCalled (false) {}
bool statsCalled;
virtual uint64_t maxSize() const;
virtual uint64_t currentSize() const;
virtual uint64_t currentCount() const;
virtual void stat(StoreEntry &) const; /* output stats to the provided store entry */
virtual void swappedOut(const StoreEntry &e) {}
virtual void reconfigure();
virtual void init();
+ virtual bool unlinkdUseful() const;
virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const;
virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *);
virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *);
virtual void parse(int, char*);
virtual StoreSearch *search(String, HttpRequest *);
};
typedef RefCount<TestSwapDir> TestSwapDirPointer;
#endif /* TEST_TESTSWAPDIR */
diff --git src/unlinkd.cc src/unlinkd.cc
index c46e1c2..09066fb 100644
--- src/unlinkd.cc
+++ src/unlinkd.cc
@@ -18,40 +18,41 @@
* sources; see the CREDITS file for full details.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
*
*/
#include "squid.h"
#include "SquidTime.h"
+#include "SwapDir.h"
#include "fde.h"
#include "xusleep.h"
/* This code gets linked to Squid */
static int unlinkd_wfd = -1;
static int unlinkd_rfd = -1;
static void * hIpc;
static pid_t pid;
#define UNLINKD_QUEUE_LIMIT 20
void
unlinkdUnlink(const char *path)
{
char buf[MAXPATHLEN];
int l;
int bytes_written;
static int queuelen = 0;
@@ -139,75 +140,90 @@ unlinkdUnlink(const char *path)
statCounter.syscalls.disk.unlinks++;
queuelen++;
}
void
unlinkdClose(void)
#if _SQUID_MSWIN_
{
if (unlinkd_wfd > -1) {
debugs(2, 1, "Closing unlinkd pipe on FD " << unlinkd_wfd);
shutdown(unlinkd_wfd, SD_BOTH);
comm_close(unlinkd_wfd);
if (unlinkd_wfd != unlinkd_rfd)
comm_close(unlinkd_rfd);
unlinkd_wfd = -1;
unlinkd_rfd = -1;
- } else
- debugs(2, 0, "unlinkdClose: WARNING: unlinkd_wfd is " << unlinkd_wfd);
+ }
if (hIpc) {
if (WaitForSingleObject(hIpc, 5000) != WAIT_OBJECT_0) {
getCurrentTime();
debugs(2, 1, "unlinkdClose: WARNING: (unlinkd," << pid << "d) didn't exit in 5 seconds");
}
CloseHandle(hIpc);
}
}
#else
{
if (unlinkd_wfd < 0)
return;
debugs(2, 1, "Closing unlinkd pipe on FD " << unlinkd_wfd);
file_close(unlinkd_wfd);
if (unlinkd_wfd != unlinkd_rfd)
file_close(unlinkd_rfd);
unlinkd_wfd = -1;
unlinkd_rfd = -1;
}
#endif
+bool
+unlinkdNeeded(void)
+{
+ // we should start unlinkd if there are any cache_dirs using it
+ for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
+ const RefCount<SwapDir> sd = Config.cacheSwap.swapDirs[i];
+ if (sd->unlinkdUseful())
+ return true;
+ }
+
+ return false;
+}
+
void
unlinkdInit(void)
{
+ if (unlinkd_wfd >= 0)
+ return; // unlinkd already started
+
const char *args[2];
Ip::Address localhost;
args[0] = "(unlinkd)";
args[1] = NULL;
localhost.SetLocalhost();
pid = ipcCreate(
#if USE_POLL && defined(_SQUID_OSF_)
/* pipes and poll() don't get along on DUNIX -DW */
IPC_STREAM,
#elif defined(_SQUID_MSWIN_)
/* select() will fail on a pipe */
IPC_TCP_SOCKET,
#else
/* We currently need to use FIFO.. see below */
IPC_FIFO,
#endif
Config.Program.unlinkd,
args,
