The branch, v3-4-ctdb has been updated
via 224188a5e1388d70049ecc2c30b8b26f1acb84dc (commit)
via 0f9bd4f53c26bbb93e799f275d3af7a725af3251 (commit)
via 7b531a429af4f4579c66cada693f660764bbc56f (commit)
via 089ea402f6b49b05fc3ab8f5699236b0e009a5cb (commit)
via 6d873a37931d9bcd508878d56d86c4e14f8ed2c8 (commit)
via 22c77c7c224381d53067f4fc9feeb058e46ee43a (commit)
via 28c703cac6fbd1850527de5e324deb28082242c2 (commit)
via 7f66849198d39086fbf0c6632fe5c4e7d7496df3 (commit)
via 04045dfe48527bd74d25eb7694baca9e55cd7e60 (commit)
from d6fd406056bc92641a2da3576952e71294f4ffe4 (commit)
http://gitweb.samba.org/?p=obnox/samba-ctdb.git;a=shortlog;h=v3-4-ctdb
- Log -----------------------------------------------------------------
commit 224188a5e1388d70049ecc2c30b8b26f1acb84dc
Author: Volker Lendecke <v...@samba.org>
Date: Tue Jul 14 18:31:28 2009 +0200
Consolidate gencache also every 100 writes in a single process
commit 0f9bd4f53c26bbb93e799f275d3af7a725af3251
Author: Volker Lendecke <v...@samba.org>
Date: Tue Jul 14 11:33:04 2009 +0200
Consolidate string and data_blob routines in gencache
commit 7b531a429af4f4579c66cada693f660764bbc56f
Author: Volker Lendecke <v...@samba.org>
Date: Mon Jul 13 17:04:29 2009 +0200
Make gencache more stable
This provides a compromise between stability and performance: gencache is a
persistent database these days that for performance reasons can not use tdb
transactions for all writes. This patch splits up gencache into gencache.tdb
and gencache_notrans.tdb. gencache_notrans is used with CLEAR_IF_FIRST,
writes
to it don't use transactions. By default every 5 minutes and when a program
exits, all entries from _notrans.tdb are transferred to gencache.tdb in one
transaction.
commit 089ea402f6b49b05fc3ab8f5699236b0e009a5cb
Author: Volker Lendecke <v...@samba.org>
Date: Mon Jul 13 17:03:52 2009 +0200
Add tdb_data_cmp
commit 6d873a37931d9bcd508878d56d86c4e14f8ed2c8
Author: Volker Lendecke <v...@samba.org>
Date: Fri Jul 10 12:24:56 2009 +0200
Remove gencache_init/shutdown
gencache_get/set/del/iterate call gencache_init() internally anyway. And
we've
been very lazy calling gencache_shutdown, so this seems not really required.
commit 22c77c7c224381d53067f4fc9feeb058e46ee43a
Author: Volker Lendecke <v...@samba.org>
Date: Fri Jul 10 12:12:30 2009 +0200
Fix some nonempty blank lines
commit 28c703cac6fbd1850527de5e324deb28082242c2
Author: Volker Lendecke <v...@samba.org>
Date: Fri Jul 10 12:03:35 2009 +0200
Remove gencache_[un]lock_key
commit 7f66849198d39086fbf0c6632fe5c4e7d7496df3
Author: Volker Lendecke <v...@samba.org>
Date: Fri Jul 10 11:00:24 2009 +0200
TDB_CONTEXT -> "struct tdb_context"
commit 04045dfe48527bd74d25eb7694baca9e55cd7e60
Author: Volker Lendecke <v...@samba.org>
Date: Fri Jul 10 10:54:33 2009 +0200
Replace ASSERTs in gencache with "return false"
It's a bit strong to panic here I think.
-----------------------------------------------------------------------
Summary of changes:
source3/include/proto.h | 8 +-
source3/include/util_tdb.h | 2 +
source3/lib/gencache.c | 529 +++++++++++++++++++++++++--------------
source3/lib/netapi/netapi.c | 1 -
source3/lib/util_tdb.c | 19 ++
source3/libads/dns.c | 8 -
source3/libsmb/dsgetdcname.c | 33 +--
source3/libsmb/libsmb_context.c | 1 -
source3/libsmb/namecache.c | 34 ---
source3/libsmb/namequery.c | 12 -
source3/libsmb/trustdom_cache.c | 67 ++----
source3/nmbd/nmbd.c | 2 +
source3/smbd/server.c | 1 +
source3/torture/torture.c | 16 --
source3/utils/net.c | 2 +
source3/utils/net_cache.c | 27 ++-
source3/winbindd/winbindd.c | 2 +
17 files changed, 415 insertions(+), 349 deletions(-)
Changeset truncated at 500 lines:
diff --git a/source3/include/proto.h b/source3/include/proto.h
index 0a37832..60f2513 100644
--- a/source3/include/proto.h
+++ b/source3/include/proto.h
@@ -518,17 +518,15 @@ void pull_file_id_24(char *buf, struct file_id *id);
/* The following definitions come from lib/gencache.c */
-bool gencache_init(void);
-bool gencache_shutdown(void);
bool gencache_set(const char *keystr, const char *value, time_t timeout);
bool gencache_del(const char *keystr);
bool gencache_get(const char *keystr, char **valstr, time_t *timeout);
-bool gencache_get_data_blob(const char *keystr, DATA_BLOB *blob, bool
*expired);
+bool gencache_get_data_blob(const char *keystr, DATA_BLOB *blob,
+ time_t *timeout);
+bool gencache_stabilize(void);
bool gencache_set_data_blob(const char *keystr, const DATA_BLOB *blob, time_t
timeout);
void gencache_iterate(void (*fn)(const char* key, const char *value, time_t
timeout, void* dptr),
void* data, const char* keystr_pattern);
-int gencache_lock_entry( const char *key );
-void gencache_unlock_entry( const char *key );
/* The following definitions come from lib/interface.c */
diff --git a/source3/include/util_tdb.h b/source3/include/util_tdb.h
index c794364..80b9592 100644
--- a/source3/include/util_tdb.h
+++ b/source3/include/util_tdb.h
@@ -59,4 +59,6 @@ struct tdb_wrap *tdb_wrap_open(TALLOC_CTX *mem_ctx,
NTSTATUS map_nt_error_from_tdb(enum TDB_ERROR err);
+int tdb_data_cmp(TDB_DATA t1, TDB_DATA t2);
+
#endif /* __TDBUTIL_H__ */
diff --git a/source3/lib/gencache.c b/source3/lib/gencache.c
index 7f133f2..ee1f4b7 100644
--- a/source3/lib/gencache.c
+++ b/source3/lib/gencache.c
@@ -26,12 +26,13 @@
#define DBGC_CLASS DBGC_TDB
#define TIMEOUT_LEN 12
-#define CACHE_DATA_FMT "%12u/%s"
+#define CACHE_DATA_FMT "%12u/"
#define READ_CACHE_DATA_FMT_TEMPLATE "%%12u/%%%us"
#define BLOB_TYPE "DATA_BLOB"
#define BLOB_TYPE_LEN 9
-static TDB_CONTEXT *cache;
+static struct tdb_context *cache;
+static struct tdb_context *cache_notrans;
/**
* @file gencache.c
@@ -49,9 +50,10 @@ static TDB_CONTEXT *cache;
* false on failure
**/
-bool gencache_init(void)
+static bool gencache_init(void)
{
char* cache_fname = NULL;
+ int open_flags = O_RDWR|O_CREAT;
/* skip file open if it's already opened */
if (cache) return True;
@@ -60,11 +62,12 @@ bool gencache_init(void)
DEBUG(5, ("Opening cache file at %s\n", cache_fname));
- cache = tdb_open_log(cache_fname, 0, TDB_DEFAULT,
- O_RDWR|O_CREAT, 0644);
+ cache = tdb_open_log(cache_fname, 0, TDB_DEFAULT, open_flags, 0644);
if (!cache && (errno == EACCES)) {
- cache = tdb_open_log(cache_fname, 0, TDB_DEFAULT, O_RDONLY,
0644);
+ open_flags = O_RDONLY;
+ cache = tdb_open_log(cache_fname, 0, TDB_DEFAULT, open_flags,
+ 0644);
if (cache) {
DEBUG(5, ("gencache_init: Opening cache file %s
read-only.\n", cache_fname));
}
@@ -74,65 +77,123 @@ bool gencache_init(void)
DEBUG(5, ("Attempt to open gencache.tdb has failed.\n"));
return False;
}
- return True;
-}
+ cache_fname = lock_path("gencache_notrans.tdb");
-/**
- * Cache shutdown function. Closes opened cache tdb file.
- *
- * @return true on successful closing the cache or
- * false on failure during cache shutdown
- **/
+ DEBUG(5, ("Opening cache file at %s\n", cache_fname));
-bool gencache_shutdown(void)
-{
- int ret;
- /* tdb_close routine returns -1 on error */
- if (!cache) return False;
- DEBUG(5, ("Closing cache file\n"));
- ret = tdb_close(cache);
- cache = NULL;
- return ret != -1;
+ cache_notrans = tdb_open_log(cache_fname, 0, TDB_CLEAR_IF_FIRST,
+ open_flags, 0644);
+ if (cache_notrans == NULL) {
+ DEBUG(5, ("Opening %s failed: %s\n", cache_fname,
+ strerror(errno)));
+ tdb_close(cache);
+ return false;
+ }
+
+ return True;
}
+static TDB_DATA last_stabilize_key(void)
+{
+ TDB_DATA result;
+ result.dptr = (uint8_t *)"@LAST_STABILIZED";
+ result.dsize = 17;
+ return result;
+}
/**
* Set an entry in the cache file. If there's no such
* one, then add it.
*
* @param keystr string that represents a key of this entry
- * @param value text representation value being cached
+ * @param blob DATA_BLOB value being cached
* @param timeout time when the value is expired
*
* @retval true when entry is successfuly stored
* @retval false on failure
**/
-bool gencache_set(const char *keystr, const char *value, time_t timeout)
+bool gencache_set_data_blob(const char *keystr, const DATA_BLOB *blob,
+ time_t timeout)
{
int ret;
TDB_DATA databuf;
- char* valstr = NULL;
+ char* val;
+ time_t last_stabilize;
+ static int writecount;
+
+ if (tdb_data_cmp(string_term_tdb_data(keystr),
+ last_stabilize_key()) == 0) {
+ DEBUG(10, ("Can't store %s as a key\n", keystr));
+ return false;
+ }
- /* fail completely if get null pointers passed */
- SMB_ASSERT(keystr && value);
+ if ((keystr == NULL) || (blob == NULL)) {
+ return false;
+ }
if (!gencache_init()) return False;
- if (asprintf(&valstr, CACHE_DATA_FMT, (int)timeout, value) == -1) {
+ val = talloc_asprintf(talloc_tos(), CACHE_DATA_FMT, (int)timeout);
+ if (val == NULL) {
return False;
}
+ val = talloc_realloc(NULL, val, char, talloc_array_length(val)-1);
+ if (val == NULL) {
+ return false;
+ }
+ val = (char *)talloc_append_blob(NULL, val, *blob);
+ if (val == NULL) {
+ return false;
+ }
- databuf = string_term_tdb_data(valstr);
- DEBUG(10, ("Adding cache entry with key = %s; value = %s and timeout ="
- " %s (%d seconds %s)\n", keystr, value,ctime(&timeout),
+ DEBUG(10, ("Adding cache entry with key = %s and timeout ="
+ " %s (%d seconds %s)\n", keystr, ctime(&timeout),
(int)(timeout - time(NULL)),
timeout > time(NULL) ? "ahead" : "in the past"));
- ret = tdb_store_bystring(cache, keystr, databuf, 0);
- SAFE_FREE(valstr);
+ ret = tdb_store_bystring(
+ cache_notrans, keystr,
+ make_tdb_data((uint8_t *)val, talloc_array_length(val)),
+ 0);
+ TALLOC_FREE(val);
+ if (ret != 0) {
+ return false;
+ }
+
+ /*
+ * Every 100 writes within a single process, stabilize the cache with
+ * a transaction. This is done to prevent a single transaction to
+ * become huge and chew lots of memory.
+ */
+ writecount += 1;
+ if (writecount > lp_parm_int(-1, "gencache", "stabilize_count", 100)) {
+ gencache_stabilize();
+ writecount = 0;
+ goto done;
+ }
+
+ /*
+ * Every 5 minutes, call gencache_stabilize() to not let grow
+ * gencache_notrans.tdb too large.
+ */
+
+ last_stabilize = 0;
+ databuf = tdb_fetch(cache_notrans, last_stabilize_key());
+ if ((databuf.dptr != NULL)
+ && (databuf.dptr[databuf.dsize-1] == '\0')) {
+ last_stabilize = atoi((char *)databuf.dptr);
+ SAFE_FREE(databuf.dptr);
+ }
+ if ((last_stabilize
+ + lp_parm_int(-1, "gencache", "stabilize_interval", 300))
+ < time(NULL)) {
+ gencache_stabilize();
+ }
+
+done:
return ret == 0;
}
@@ -147,26 +208,63 @@ bool gencache_set(const char *keystr, const char *value,
time_t timeout)
bool gencache_del(const char *keystr)
{
- int ret;
+ bool exists;
+ bool ret = false;
+ char *value;
- /* fail completely if get null pointers passed */
- SMB_ASSERT(keystr);
+ if (keystr == NULL) {
+ return false;
+ }
if (!gencache_init()) return False;
DEBUG(10, ("Deleting cache entry (key = %s)\n", keystr));
- ret = tdb_delete_bystring(cache, keystr);
- return ret == 0;
+ if (tdb_lock_bystring(cache_notrans, keystr) == -1) {
+ DEBUG(5, ("Could not lock key for %s\n", keystr));
+ return false;
+ }
+
+ /*
+ * We delete an element by setting its timeout to 0. This way we don't
+ * have to do a transaction on gencache.tdb every time we delete an
+ * element.
+ */
+
+ exists = gencache_get(keystr, &value, NULL);
+ if (exists) {
+ SAFE_FREE(value);
+ ret = gencache_set(keystr, "", 0);
+ }
+ tdb_unlock_bystring(cache_notrans, keystr);
+ return ret;
}
+static bool gencache_pull_timeout(char *val, time_t *pres, char **pendptr)
+{
+ time_t res;
+ char *endptr;
+
+ res = strtol(val, &endptr, 10);
+
+ if ((endptr == NULL) || (*endptr != '/')) {
+ DEBUG(2, ("Invalid gencache data format: %s\n", val));
+ return false;
+ }
+ if (pres != NULL) {
+ *pres = res;
+ }
+ if (pendptr != NULL) {
+ *pendptr = endptr;
+ }
+ return true;
+}
/**
* Get existing entry from the cache file.
*
* @param keystr string that represents a key of this entry
- * @param valstr buffer that is allocated and filled with the entry value
- * buffer's disposing must be done outside
+ * @param blob DATA_BLOB that is filled with entry's blob
* @param timeout pointer to a time_t that is filled with entry's
* timeout
*
@@ -174,31 +272,40 @@ bool gencache_del(const char *keystr)
* @retval False for failure
**/
-bool gencache_get(const char *keystr, char **valstr, time_t *timeout)
+bool gencache_get_data_blob(const char *keystr, DATA_BLOB *blob,
+ time_t *timeout)
{
TDB_DATA databuf;
time_t t;
char *endptr;
- /* fail completely if get null pointers passed */
- SMB_ASSERT(keystr);
+ if (keystr == NULL) {
+ return false;
+ }
+
+ if (tdb_data_cmp(string_term_tdb_data(keystr),
+ last_stabilize_key()) == 0) {
+ DEBUG(10, ("Can't get %s as a key\n", keystr));
+ return false;
+ }
if (!gencache_init()) {
return False;
}
- databuf = tdb_fetch_bystring(cache, keystr);
+ databuf = tdb_fetch_bystring(cache_notrans, keystr);
if (databuf.dptr == NULL) {
- DEBUG(10, ("Cache entry with key = %s couldn't be found\n",
+ databuf = tdb_fetch_bystring(cache, keystr);
+ }
+
+ if (databuf.dptr == NULL) {
+ DEBUG(10, ("Cache entry with key = %s couldn't be found \n",
keystr));
return False;
}
- t = strtol((const char *)databuf.dptr, &endptr, 10);
-
- if ((endptr == NULL) || (*endptr != '/')) {
- DEBUG(2, ("Invalid gencache data format: %s\n", databuf.dptr));
+ if (!gencache_pull_timeout((char *)databuf.dptr, &t, &endptr)) {
SAFE_FREE(databuf.dptr);
return False;
}
@@ -207,20 +314,33 @@ bool gencache_get(const char *keystr, char **valstr,
time_t *timeout)
"timeout = %s", t > time(NULL) ? "valid" :
"expired", keystr, endptr+1, ctime(&t)));
+ if (t == 0) {
+ /* Deleted */
+ SAFE_FREE(databuf.dptr);
+ return False;
+ }
+
if (t <= time(NULL)) {
- /* We're expired, delete the entry */
- tdb_delete_bystring(cache, keystr);
+ /*
+ * We're expired, delete the entry. We can't use gencache_del
+ * here, because that uses gencache_get_data_blob for checking
+ * the existence of a record. We know the thing exists and
+ * directly store an empty value with 0 timeout.
+ */
+ gencache_set(keystr, "", 0);
SAFE_FREE(databuf.dptr);
return False;
}
- if (valstr) {
- *valstr = SMB_STRDUP(endptr+1);
- if (*valstr == NULL) {
+ if (blob != NULL) {
+ *blob = data_blob(
+ endptr+1,
+ databuf.dsize - PTR_DIFF(endptr+1, databuf.dptr));
+ if (blob->data == NULL) {
SAFE_FREE(databuf.dptr);
- DEBUG(0, ("strdup failed\n"));
+ DEBUG(0, ("memdup failed\n"));
return False;
}
}
@@ -234,155 +354,192 @@ bool gencache_get(const char *keystr, char **valstr,
time_t *timeout)
return True;
}
+struct stabilize_state {
+ bool written;
+ bool error;
+};
+static int stabilize_fn(struct tdb_context *tdb, TDB_DATA key, TDB_DATA val,
+ void *priv);
+
/**
- * Get existing entry from the cache file.
- *
- * @param keystr string that represents a key of this entry
- * @param blob DATA_BLOB that is filled with entry's blob
- * @param expired pointer to a bool that indicates whether the entry is expired
+ * Stabilize gencache
*
- * @retval true when entry is successfuly fetched
- * @retval False for failure
- **/
+ * Migrate the clear-if-first gencache data to the stable,
+ * transaction-based gencache.tdb
+ */
-bool gencache_get_data_blob(const char *keystr, DATA_BLOB *blob, bool *expired)
+bool gencache_stabilize(void)
{
- TDB_DATA databuf;
- time_t t;
- char *blob_type;
- unsigned char *buf = NULL;
- bool ret = False;
- fstring valstr;
- int buflen = 0, len = 0, blob_len = 0;
- unsigned char *blob_buf = NULL;
-
- /* fail completely if get null pointers passed */
- SMB_ASSERT(keystr);
+ struct stabilize_state state;
+ int res;
+ char *now;
if (!gencache_init()) {
- return False;
+ return false;
}
- databuf = tdb_fetch_bystring(cache, keystr);
- if (!databuf.dptr) {
- DEBUG(10,("Cache entry with key = %s couldn't be found\n",
- keystr));
- return False;
+ res = tdb_transaction_start(cache);
+ if (res == -1) {
+ DEBUG(10, ("Could not start transaction on gencache.tdb: "
+ "%s\n", tdb_errorstr(cache)));
+ return false;
+ }
+ res = tdb_transaction_start(cache_notrans);
+ if (res == -1) {
+ tdb_transaction_cancel(cache);
+ DEBUG(10, ("Could not start transaction on "
+ "gencache_notrans.tdb: %s\n",
+ tdb_errorstr(cache_notrans)));
+ return false;
}
- buf = (unsigned char *)databuf.dptr;
- buflen = databuf.dsize;
+ state.error = false;
+ state.written = false;
- len += tdb_unpack(buf+len, buflen-len, "fB",
- &valstr,
- &blob_len, &blob_buf);
- if (len == -1) {
- goto out;
+ res = tdb_traverse(cache_notrans, stabilize_fn, &state);
+ if ((res == -1) || state.error) {
+ if ((tdb_transaction_cancel(cache_notrans) == -1)
+ || (tdb_transaction_cancel(cache) == -1)) {
+ smb_panic("tdb_transaction_cancel failed\n");
+ }
+ return false;
}
- t = strtol(valstr, &blob_type, 10);
+ if (!state.written) {
+ if ((tdb_transaction_cancel(cache_notrans) == -1)
+ || (tdb_transaction_cancel(cache) == -1)) {
+ smb_panic("tdb_transaction_cancel failed\n");
+ }
+ return true;
+ }
- if (strcmp(blob_type+1, BLOB_TYPE) != 0) {
- goto out;
+ res = tdb_transaction_commit(cache);
+ if (res == -1) {
+ DEBUG(10, ("tdb_transaction_commit on gencache.tdb failed: "
+ "%s\n", tdb_errorstr(cache)));
--
SAMBA-CTDB repository
