diff -Naur pvfs2-export/src/io/trove/trove-dbpf/dbpf-mgmt.c pvfs2-patched/src/io/trove/trove-dbpf/dbpf-mgmt.c --- pvfs2-export/src/io/trove/trove-dbpf/dbpf-mgmt.c 2007-09-12 16:08:09.000000000 -0500 +++ pvfs2-patched/src/io/trove/trove-dbpf/dbpf-mgmt.c 2007-11-28 14:38:19.000000000 -0600 @@ -16,6 +16,7 @@ #include #include #include +#include #include "trove.h" #ifdef HAVE_MALLOC_H @@ -47,6 +48,7 @@ struct dbpf_storage *my_storage_p = NULL; static int db_open_count, db_close_count; +static void unlink_db_cache_files(const char* path); #define COLL_ENV_FLAGS (DB_INIT_MPOOL | DB_CREATE | DB_THREAD) @@ -74,16 +76,33 @@ } /* we start by making sure any old environment remnants are cleaned up */ - ret = db_env_create(&dbenv, 0); - if (ret != 0) + if(my_storage_p->flags & TROVE_DB_CACHE_MMAP) { - gossip_err("dbpf_env_create: could not create " - "any environment handle: %s\n", - db_strerror(ret)); - return 0; + /* mmap case: use env->remove function */ + ret = db_env_create(&dbenv, 0); + if (ret != 0) + { + gossip_err("dbpf_env_create: could not create " + "any environment handle: %s\n", + db_strerror(ret)); + *error = ret; + return NULL; + } + + /* don't check return code here; we don't care if it fails */ + dbenv->remove(dbenv, path, DB_FORCE); + } + else + { + /* shm case */ + /* destroy any old __db.??? files to make sure we don't accidentially + * reuse a shmid and collide with a server process that is already + * running on this node. We don't use env->remove because it could + * interfere with shm regions already allocated by another server + * process + */ + unlink_db_cache_files(path); } - /* don't check return code here; we don't care if it fails */ - dbenv->remove(dbenv, path, DB_FORCE); retry: ret = db_env_create(&dbenv, 0); @@ -134,6 +153,8 @@ else { /* default to using shm style cache */ + + gossip_debug(GOSSIP_TROVE_DEBUG, "dbpf using shm key: %d\n", (646567223+TROVE_shm_key_hint)); ret = dbenv->set_shm_key(dbenv, (646567223+TROVE_shm_key_hint)); @@ -164,13 +185,7 @@ * before using it again */ if (ret == EAGAIN) { - ret = dbenv->remove(dbenv, path, DB_FORCE); - if (ret != 0) - { - gossip_lerr("dbpf_remove(%s): %s\n", path, db_strerror(ret)); - *error = ret; - return NULL; - } + unlink_db_cache_files(path); assert(my_storage_p != NULL); my_storage_p->flags |= TROVE_DB_CACHE_MMAP; goto retry; @@ -180,15 +195,7 @@ * open returns an EINVAL, retry with DB_PRIVATE. */ if(ret == EINVAL) { - - ret = dbenv->remove(dbenv, path, DB_FORCE); - if(ret != 0) - { - gossip_lerr("dbpf_remove(%s): %s\n", path, db_strerror(ret)); - *error = ret; - return NULL; - } - assert(my_storage_p != NULL); + unlink_db_cache_files(path); ret = dbenv->open(dbenv, path, DB_CREATE| DB_THREAD| @@ -212,6 +219,7 @@ return NULL; } } + return dbenv; } @@ -1918,6 +1926,36 @@ return ret; } +static void unlink_db_cache_files(const char* path) +{ + char* db_region_file = NULL; + glob_t pglob; + int ret; + int i; + + db_region_file = malloc(PATH_MAX); + if(!db_region_file) + { + return; + } + + snprintf(db_region_file, PATH_MAX, "%s/__db.???", path); + + ret = glob(db_region_file, 0, NULL, &pglob); + if(ret == 0) + { + for(i=0; i 0) - { - ret = trove_collection_setinfo(0, 0, TROVE_SHM_KEY_HINT, &port_num); - assert(ret == 0); - } + /* help trove chose a differentiating shm key if needed for Berkeley DB */ + shm_key_hint = generate_shm_key_hint(); + gossip_debug(GOSSIP_SERVER_DEBUG, "Server using shm key hint: %d\n", shm_key_hint); + ret = trove_collection_setinfo(0, 0, TROVE_SHM_KEY_HINT, &shm_key_hint); + assert(ret == 0); if(server_config.db_cache_type && (!strcmp(server_config.db_cache_type, "mmap"))) @@ -2010,37 +2006,6 @@ return &server_config; } -/* parse_port_from_host_id() - * - * attempts to parse the port number from a BMI address. Returns port number - * on success, -1 on failure - */ -static int parse_port_from_host_id(char* host_id) -{ - int ret = -1; - int port_num; - char* port_index; - char* colon_index; - - /* see if we have a ://: format */ - port_index = rindex(host_id, ':'); - colon_index = index(host_id, ':'); - /* if so, parse the port number */ - if(port_index && (port_index != colon_index)) - { - port_index++; - ret = sscanf(port_index, "%d", &port_num); - } - - /* report error if we don't find a valid port number in the string */ - if(ret != 1) - { - return(-1); - } - - return(port_num); -} - /* server_op_get_machine() * * looks up the state machine for the op * given and returns it, or @@ -2168,6 +2133,46 @@ return strdup(tmp_alias); } +/* generate_shm_key_hing() + * + * Makes a best effort to produce a unique shm key (for Trove's Berkeley + * DB use) for each server. By default it will base this on the server's + * position in the fs.conf, but it will fall back to using a random number + * + * returns integer key + */ +static int generate_shm_key_hint(void) +{ + int server_index = 1; + struct host_alias_s *cur_alias = NULL; + + PINT_llist *cur = server_config.host_aliases; + + /* iterate through list of aliases in configuration file */ + while(cur) + { + cur_alias = PINT_llist_head(cur); + if(strcmp(cur_alias->bmi_address, server_config.host_id) == 0) + { + /* match */ + /* space the shm keys out by 10 to allow for Berkeley DB using + * using more than one key on each server + */ + return(server_index*10); + } + + server_index++; + cur = PINT_llist_next(cur); + } + + /* If we reach this point, we didn't find this server in the alias list. + * This is not a normal situation, but fall back to using a random + * number for the key just to be safe. + */ + srand((unsigned int)time(NULL)); + return(rand()); +} + /* * Local variables: * c-indent-level: 4