+---------- On Sep 10, Sean Owen said:
> The problem is, under heavy load, we get into serious lock contention
> problems reading from it.
Are you speculating or have you looked at the mutex statistics?
> 99.99% of the time we are just reading, so
> ideally I'd like to use something akin to a read/write lock, however, the
> overhead of an actual rwlock will probably only make matters worse.
>
> The table is far too large (~50,000 entries)to just load up every time an
> interp is initialized.
We need to know more about your usage pattern. Each NSV is a hash
table. Are you using a lot of NSVs or one NSV with 50,000 keys? Do you
access the NSV once per request, or many times per request?
> Any ideas for me? Is there any way to create a shared read-only
> datastructure that doesn't use thread interlocking? Do I need to code my
> datastructure up in C?
Yes, you'd need to do it in C.
I'd do something like this, but I haven't tested this code:
/*
* An "atomicdata" is a global hash table. One atomicdata in the system
* is considered "active".
*
* Each atomicdata has a reference count. When an atomicdata's reference
* count is zero and it is not the active atomicdata, all resources
* consumed by that atomicdata are freed.
*
* The command "dqd_atomicdata set {k1 v1 k2 v2 ...}" creates a new
* atomicdata with reference count zero. It fills it with the key/value
* pairs k1/v1, k2/v2, etc. Then it atomically makes that new atomicdata
* be the active atomicdata. If there was already an active atomicdata,
* then the command BLOCKS until the old atomicdata's reference count
* is zero and then frees the old atomicdata's resources. Any other
* thread that has a reference to the old atomicdata continues to access
* the old atomicdata (blocking this thread) until it releases its
* reference.
*
* The command "dqd_atomicdata get k" causes the current thread to
* check whether it has a reference to an atomicdata. If it does not,
* then it acquires a reference to the active atomicdata and increments
* the active atomicdata's reference count. Once the thread is certain
* that it has a reference to some atomicdata, it looks up "k" in the
* atomicdata. If "k" is found, then the command returns the value
* associated with "k". Otherwise, it returns an error. In either case,
* the reference to the atomicdata is saved for use by future calls to
* "dqd_atomicdata get" in the same thread.
*
* The command "dqd_atomicdata release" checks whether the current
* thread has a reference to some atomicdata. If it does not, then the
* command simply returns "0". If the thread does have a reference to an
* atomicdata, then the command decrements the atomicdata's reference
* count and forgets the reference.
*
* Each connection thread automatically performs the equivalent of
* "dqd_atomicdata release" after completing each HTTP request. You
* should use "dqd_atomicdata release" in threads that you create and in
* scheduled procs.
*
* A thread must lock a mutex momentarily to acquire a reference to the
* active atomicdata, and to release its reference, and to replace the
* active atomicdata. No locking is required when a thread already has a
* reference to an atomicdata.
*/
#ifndef USE_TCL8X
#define USE_TCL8X
#endif
#include "ns.h"
int Ns_ModuleVersion = 1;
typedef struct AtomicData {
Tcl_HashTable table;
int refCnt;
} AtomicData;
/*
* Threads will access this to get the current table. When we need
* to modify the table, we'll construct a new one from scratch and
* atomically swap them out.
*/
static AtomicData *global_ad;
/* These will protect access to global_ad->refCnt. */
static Ns_Mutex ad_lock;
static Ns_Cond ad_cond;
/* Hold each thread's reference to global_ad. */
static Ns_Tls ad_tls;
static AtomicData *
acquire(void)
{
AtomicData *ad = Ns_TlsGet(&ad_tls);
if (ad == NULL) {
Ns_MutexLock(&ad_lock);
ad = global_ad;
ad->refCnt++;
Ns_MutexUnlock(&ad_lock);
Ns_TlsSet(&ad_tls, ad);
}
return ad;
}
static int
release(void)
{
AtomicData *ad = Ns_TlsGet(&ad_tls);
if (ad == NULL) return 0;
Ns_TlsSet(&ad_tls, NULL);
Ns_MutexLock(&ad_lock);
if (--ad->refCnt == 0)
Ns_CondBroadcast(&ad_cond);
Ns_MutexUnlock(&ad_lock);
return 1;
}
static void
setGlobal(Tcl_HashTable *newTable)
{
AtomicData *ad = (AtomicData *) ns_malloc(sizeof *ad);
AtomicData *old_ad;
Tcl_HashEntry *he;
Tcl_HashSearch search;
ad->table = *newTable;
ad->refCnt = 0;
Ns_MutexLock(&ad_lock);
old_ad = global_ad;
global_ad = ad;
while (old_ad->refCnt != 0) {
Ns_CondWait(&ad_cond, &ad_lock);
}
Ns_MutexUnlock(&ad_lock);
he = Tcl_FirstHashEntry(&old_ad->table, &search);
while (he != NULL) {
ns_free((void *) Tcl_GetHashValue(he));
he = Tcl_NextHashEntry(&search);
}
ns_free((void *) old_ad);
}
static int
setCommand(Tcl_Interp *interp, Tcl_Obj *listObj)
{
int elc;
Tcl_Obj **elv;
Tcl_HashTable ht;
if (Tcl_ListObjGetElements(interp, listObj, &elc, &elv) == TCL_ERROR)
return TCL_ERROR;
if (elc & 1) {
Tcl_SetResult(interp,
"list must contain an even number of elements", TCL_STATIC);
return TCL_ERROR;
}
Tcl_InitHashTable(&ht, TCL_STRING_KEYS);
while (elc > 0) {
int isNew;
Tcl_HashEntry *he = Tcl_CreateHashEntry(&ht,
Tcl_GetString(elv[0]), &isNew);
if (!isNew) ns_free((void *) Tcl_GetHashValue(he));
Tcl_SetHashValue(he, ns_strdup(Tcl_GetString(elv[1])));
elc -= 2;
elv += 2;
}
setGlobal(&ht);
return TCL_OK;
}
static int
getCommand(Tcl_Interp *interp, Tcl_Obj *keyObj)
{
AtomicData *ad;
Tcl_HashEntry *he;
char *key;
ad = acquire();
if (ad == NULL) {
Tcl_SetResult(interp, "no active atomicdata", TCL_STATIC);
return TCL_ERROR;
}
key = Tcl_GetString(keyObj);
he = Tcl_FindHashEntry(&ad->table, key);
if (he == NULL) {
Tcl_AppendResult(interp, "atomicdata does not contain key \"",
key, "\"", NULL);
return TCL_ERROR;
}
Tcl_SetResult(interp, (char *) Tcl_GetHashValue(he), TCL_VOLATILE);
return TCL_OK;
}
static int
releaseCommand(Tcl_Interp *interp)
{
Tcl_SetResult(interp, release() ? "1" : "0", TCL_STATIC);
return TCL_OK;
}
static int
tclCommand(ClientData dummy, Tcl_Interp *interp,
int objc, Tcl_Obj * CONST objv[])
{
if (objc == 2) {
if (STREQ(Tcl_GetString(objv[1]), "release"))
return releaseCommand(interp);
}
else if (objc == 3) {
if (STREQ(Tcl_GetString(objv[1]), "get"))
return getCommand(interp, objv[2]);
else if (STREQ(Tcl_GetString(objv[1]), "set"))
return setCommand(interp, objv[2]);
}
Tcl_SetResult(interp, "syntax error", TCL_STATIC);
return TCL_ERROR;
}
static int
interpInit(Tcl_Interp *interp, void *context)
{
Tcl_CreateObjCommand(interp, "dqd_atomicdata", tclCommand, NULL, NULL);
return NS_OK;
}
int
Ns_ModuleInit(char *server, char *module)
{
Ns_TlsAlloc(&ad_tls, NULL);
Ns_MutexSetName2(&ad_lock, "ad_lock", "");
return NS_OK;
}
