This patch probably has all that's necessary to flip the configuration to use CLD to configure available Chunk nodes and dispose with <StorageNode> forever. However, it did not complete all tests that's necessary to declare victory. So for now we continue to use <StorageNode>. This should be relaiable in Koji, but we have -D in build scripts for now.
The parsing itself uses the same code that our parser of the static configuration uses, although it appears resistant to refactoring without going full tilt to a table-driven system, and so we copy-pasted a bunch from config.c to storparse.c. Signed-Off-By: Pete Zaitcev <[email protected]> diff --git a/server/Makefile.am b/server/Makefile.am index e01f728..f994b36 100644 --- a/server/Makefile.am +++ b/server/Makefile.am @@ -4,7 +4,8 @@ INCLUDES = -I$(top_srcdir)/include @GLIB_CFLAGS@ @CHUNKDC_CFLAGS@ @CLDC_CFLAGS@ sbin_PROGRAMS = tabled tdbadm tabled_SOURCES = tabled.h \ - bucket.c object.c server.c storage.c cldu.c config.c util.c + bucket.c object.c server.c storage.c storparse.c cldu.c \ + config.c util.c tabled_LDADD = ../lib/libhttputil.a ../lib/libtdb.a \ @CHUNKDC_LIBS@ @CLDC_LIBS@ @PCRE_LIBS@ @GLIB_LIBS@ \ @CRYPTO_LIBS@ @DB4_LIBS@ @EVENT_LIBS@ @ARGP_LIBS@ @SSL_LIBS@ diff --git a/server/cldu.c b/server/cldu.c index bbb651d..b275234 100644 --- a/server/cldu.c +++ b/server/cldu.c @@ -5,6 +5,7 @@ #include "tabled-config.h" #include <sys/types.h> #include <sys/socket.h> +#include <sys/time.h> #include <glib.h> #include <syslog.h> #include <string.h> @@ -33,11 +34,14 @@ struct cld_session { bool forced_hosts; /* Administrator overrode default CLD */ bool sess_open; struct cldc_udp *lib; /* library state */ + struct event tm; + int retry_cnt; int actx; /* Active host cldv[actx] */ struct cld_host cldv[N_CLD]; char *thiscell; + char *thishost; struct event ev; /* Associated with fd */ char *cfname; /* /tabled-cell directory */ struct cldc_fh *cfh; /* /tabled-cell directory, keep open for scan */ @@ -49,8 +53,6 @@ struct cld_session { struct cldc_fh *yfh; /* /chunk-cell/NID file */ struct list_head chunks; /* found in xfname, struct chunk_node */ - - void (*state_cb)(enum st_cld); }; static int cldu_set_cldc(struct cld_session *sp, int newactive); @@ -60,6 +62,7 @@ static int cldu_open_f_cb(struct cldc_call_opts *carg, enum cle_err_codes errc); static int cldu_lock_cb(struct cldc_call_opts *carg, enum cle_err_codes errc); static int cldu_put_cb(struct cldc_call_opts *carg, enum cle_err_codes errc); static int cldu_get_1_cb(struct cldc_call_opts *carg, enum cle_err_codes errc); +static void try_open_x(struct cld_session *sp); static int cldu_open_x_cb(struct cldc_call_opts *carg, enum cle_err_codes errc); static int cldu_get_x_cb(struct cldc_call_opts *carg, enum cle_err_codes errc); static int cldu_close_x_cb(struct cldc_call_opts *carg, enum cle_err_codes errc); @@ -70,6 +73,8 @@ static int cldu_close_y_cb(struct cldc_call_opts *carg, enum cle_err_codes errc) static void add_remote(const char *name); static void add_chunk_node(struct cld_session *sp, const char *name); +static struct timeval cldu_retry_delay = { 5, 0 }; + /* * Identify the next host to be tried. * @@ -104,6 +109,9 @@ static int cldu_setcell(struct cld_session *sp, sp->thiscell = strdup(thiscell); if (!sp->thiscell) goto err_oom; + sp->thishost = strdup(thishost); + if (!sp->thishost) + goto err_oom; if (asprintf(&mem, "/tabled-%s", thiscell) == -1) goto err_oom; @@ -124,6 +132,15 @@ err_oom: return 0; } +static void cldu_timer(int fd, short events, void *userdata) +{ + struct cld_session *sp = userdata; + + if (debugging) + applog(LOG_DEBUG, "Trying to open %s\n", sp->xfname); + try_open_x(sp); +} + static void cldu_event(int fd, short events, void *userdata) { struct cld_session *sp = userdata; @@ -451,8 +468,6 @@ static int cldu_put_cb(struct cldc_call_opts *carg, enum cle_err_codes errc) static int cldu_get_1_cb(struct cldc_call_opts *carg, enum cle_err_codes errc) { struct cld_session *sp = carg->private; - struct cldc_call_opts copts; - int rc; const char *ptr; int dir_len; int total_len, rec_len, name_len; @@ -479,7 +494,7 @@ static int cldu_get_1_cb(struct cldc_call_opts *carg, enum cle_err_codes errc) else buf[64] = 0; - if (!strcmp(buf, tabled_srv.ourhost)) { /* use thishost XXX */ + if (!strcmp(buf, sp->thishost)) { if (debugging) applog(LOG_DEBUG, " %s (ourselves)", buf); } else { @@ -492,12 +507,32 @@ static int cldu_get_1_cb(struct cldc_call_opts *carg, enum cle_err_codes errc) dir_len -= total_len; } - if (sp->state_cb) - (*sp->state_cb)(ST_CLD_ACTIVE); - /* - * Now we can collect the Chunk nodes in our cell. + * This will go away with the demise of <StorageNode>. */ + if (tabled_srv.num_stor) { + stor_update_cb(); + return 0; + } + + sp->retry_cnt = 0; + try_open_x(sp); + return 0; +} + +/* + * Open the xfname, so we can collect registered Chunk servers. + */ +static void try_open_x(struct cld_session *sp) +{ + struct cldc_call_opts copts; + int rc; + + if (++sp->retry_cnt >= 5) { + applog(LOG_INFO, "Out of retries for %s, bailing", sp->xfname); + exit(1); + } + memset(&copts, 0, sizeof(copts)); copts.cb = cldu_open_x_cb; copts.private = sp; @@ -507,7 +542,6 @@ static int cldu_get_1_cb(struct cldc_call_opts *carg, enum cle_err_codes errc) if (rc) { applog(LOG_ERR, "cldc_open(%s) call error: %d", sp->xfname, rc); } - return 0; } static int cldu_open_x_cb(struct cldc_call_opts *carg, enum cle_err_codes errc) @@ -517,8 +551,14 @@ static int cldu_open_x_cb(struct cldc_call_opts *carg, enum cle_err_codes errc) int rc; if (errc != CLE_OK) { - applog(LOG_ERR, "CLD open(%s) failed: %d", sp->xfname, errc); - /* XXX recycle, maybe Chunks aren't up yet. */ + if (errc == CLE_INODE_INVAL) { + applog(LOG_ERR, "CLD open(%s) failed: %d, retrying", + sp->xfname, errc); + evtimer_add(&sp->tm, &cldu_retry_delay); + } else { + applog(LOG_ERR, "CLD open(%s) failed: %d", + sp->xfname, errc); + } return 0; } if (sp->xfh == NULL) { @@ -530,7 +570,7 @@ static int cldu_open_x_cb(struct cldc_call_opts *carg, enum cle_err_codes errc) return 0; } - // if (debugging) + if (debugging) applog(LOG_DEBUG, "CLD directory \"%s\" opened", sp->xfname); /* @@ -606,10 +646,12 @@ static int cldu_close_x_cb(struct cldc_call_opts *carg, enum cle_err_codes errc) return 0; } - if (list_empty(&sp->chunks)) - applog(LOG_INFO, "No Chunk nodes found"); - else + if (list_empty(&sp->chunks)) { + applog(LOG_INFO, "%s: No Chunk nodes found", sp->xfname); + try_open_x(sp); + } else { next_chunk(sp); + } return 0; } @@ -698,7 +740,7 @@ static int cldu_get_y_cb(struct cldc_call_opts *carg, enum cle_err_codes errc) if (debugging) applog(LOG_DEBUG, "got %d bytes from %s\n", len, sp->yfname); - stor_add_node(ptr, len); + stor_parse(sp->yfname, ptr, len); close_and_next: memset(&copts, 0, sizeof(copts)); @@ -777,29 +819,31 @@ static void add_chunk_node(struct cld_session *sp, const char *name) static struct cld_session ses; /* - * This initiates our sole session with a CLD instance. + * Global and 1-instance initialization. */ -int cld_begin(const char *thishost, const char *thiscell, - void (*cb)(enum st_cld)) +void cld_init() { - cldc_init(); - INIT_LIST_HEAD(&ses.chunks); - /* - * As long as we permit pre-seeding lists of CLD hosts, - * we cannot wipe our session anymore. Note though, as long - * as cld_end terminates it right, we can call cld_begin again. - */ // memset(&ses, 0, sizeof(struct cld_session)); - ses.state_cb = cb; + INIT_LIST_HEAD(&ses.chunks); +} - if (cldu_setcell(&ses, thiscell, thishost)) { +/* + * This initiates our sole session with a CLD instance. + */ +int cld_begin(const char *thishost, const char *thiscell) +{ + static struct cld_session *sp = &ses; + + evtimer_set(&ses.tm, cldu_timer, &ses); + + if (cldu_setcell(sp, thiscell, thishost)) { /* Already logged error */ goto err_cell; } - if (!ses.forced_hosts) { + if (!sp->forced_hosts) { GList *tmp, *host_list = NULL; int i; @@ -815,9 +859,9 @@ int cld_begin(const char *thishost, const char *thiscell, for (tmp = host_list; tmp; tmp = tmp->next) { struct cldc_host *hp = tmp->data; if (i < N_CLD) { - memcpy(&ses.cldv[i].h, hp, + memcpy(&sp->cldv[i].h, hp, sizeof(struct cldc_host)); - ses.cldv[i].known = 1; + sp->cldv[i].known = 1; i++; } else { free(hp->host); @@ -834,7 +878,7 @@ int cld_begin(const char *thishost, const char *thiscell, * -- Actually, it only works when recovering from CLD failure. * Thereafter, any slave CLD redirects us to the master. */ - if (cldu_set_cldc(&ses, 0)) { + if (cldu_set_cldc(sp, 0)) { /* Already logged error */ goto err_net; } @@ -847,32 +891,6 @@ err_cell: return -1; } -void cld_end(void) -{ - int i; - - if (ses.lib) { - event_del(&ses.ev); - // if (ses.sess_open) /* kill it always, include half-open */ - cldc_kill_sess(ses.lib->sess); - cldc_udp_free(ses.lib); - ses.lib = NULL; - } - - if (!ses.forced_hosts) { - for (i = 0; i < N_CLD; i++) { - if (ses.cldv[i].known) - free(ses.cldv[i].h.host); - } - } - - free(ses.cfname); - free(ses.ffname); - free(ses.xfname); - free(ses.yfname); - free(ses.thiscell); -} - void cldu_add_host(const char *hostname, unsigned int port) { static struct cld_session *sp = &ses; @@ -894,3 +912,41 @@ void cldu_add_host(const char *hostname, unsigned int port) sp->forced_hosts = true; } + +void cld_end(void) +{ + static struct cld_session *sp = &ses; + int i; + + if (sp->lib) { + event_del(&sp->ev); + // if (sp->sess_open) /* kill it always, include half-open */ + cldc_kill_sess(sp->lib->sess); + cldc_udp_free(sp->lib); + sp->lib = NULL; + } + + if (!sp->forced_hosts) { + for (i = 0; i < N_CLD; i++) { + if (sp->cldv[i].known) { + free(sp->cldv[i].h.host); + sp->cldv[i].known = false; + } + } + } + + evtimer_del(&sp->tm); + + free(sp->cfname); + sp->cfname = NULL; + free(sp->ffname); + sp->ffname = NULL; + free(sp->xfname); + sp->xfname = NULL; + free(sp->yfname); + sp->yfname = NULL; + free(sp->thiscell); + sp->thiscell = NULL; + free(sp->thishost); + sp->thishost = NULL; +} diff --git a/server/config.c b/server/config.c index 7ef9c1a..45cee26 100644 --- a/server/config.c +++ b/server/config.c @@ -6,14 +6,11 @@ #include "tabled-config.h" #include <sys/types.h> #include <sys/stat.h> -#include <sys/socket.h> #include <glib.h> #include <syslog.h> #include <string.h> #include <stdio.h> -#include <unistd.h> #include <errno.h> -#include <netdb.h> #include <ctype.h> #include "tabled.h" @@ -24,6 +21,7 @@ struct config_context { struct listen_cfg tmp_listen; bool in_storage; + unsigned int stor_nid; char *stor_port; char *stor_host; @@ -51,6 +49,7 @@ static void cfg_elm_start (GMarkupParseContext *context, else if (!strcmp(element_name, "StorageNode")) { if (!cc->in_storage) { cc->in_storage = true; + cc->stor_nid++; } else { applog(LOG_ERR, "Nested StorageNode in configuration"); } @@ -64,71 +63,10 @@ static void cfg_elm_start (GMarkupParseContext *context, } } -static void cfg_add_storage(const char *hostname, const char *portstr) -{ - struct addrinfo hints; - struct addrinfo *res, *res0; - struct storage_node *sn; - int rc; - - memset(&hints, 0, sizeof(struct addrinfo)); - hints.ai_family = PF_UNSPEC; - hints.ai_socktype = SOCK_DGRAM; - - rc = getaddrinfo(hostname, portstr, &hints, &res0); - if (rc) { - applog(LOG_WARNING, "getaddrinfo(%s:%s) failed: %s", - hostname, portstr, gai_strerror(rc)); - return; - } - - for (res = res0; res; res = res->ai_next) { - if (res->ai_family != AF_INET && res->ai_family != AF_INET6) - continue; - - if (res->ai_addrlen > ADDRSIZE) /* should not happen */ - continue; - - if ((sn = malloc(sizeof(struct storage_node))) == NULL) { - applog(LOG_WARNING, "No core (%ld)", - (long) sizeof(struct storage_node)); - break; - } - memset(sn, 0, sizeof(struct storage_node)); - - memcpy(&sn->addr, res->ai_addr, res->ai_addrlen); - sn->addr_af = res->ai_family; - sn->alen = res->ai_addrlen; - - if ((sn->hostname = strdup(hostname)) == NULL) { - applog(LOG_WARNING, "No core"); - free(sn); - break; - } - - if (debugging) { - char nhost[41]; - char nport[6]; - if (getnameinfo((struct sockaddr *) &sn->addr, sn->alen, - nhost, sizeof(nhost), - nport, sizeof(nport), - NI_NUMERICHOST|NI_NUMERICSERV) == 0) { - applog(LOG_INFO, "Found Chunk host %s port %s", - nhost, nport); - } else { - applog(LOG_INFO, "Found Chunk host"); - } - } - - list_add(&sn->all_link, &tabled_srv.all_stor); - } - - freeaddrinfo(res0); - return; -} - static void cfg_elm_end_storage(struct config_context *cc) { + struct geo dummy_loc; + if (cc->text) { applog(LOG_WARNING, "Extra text in StorageNode element: \"%s\"", cc->text); @@ -146,7 +84,15 @@ static void cfg_elm_end_storage(struct config_context *cc) goto end; } - cfg_add_storage(cc->stor_host, cc->stor_port); + memset(&dummy_loc, 0, sizeof(struct geo)); + stor_add_node(cc->stor_nid, cc->stor_host, cc->stor_port, &dummy_loc); + /* + * We don't call stor_update_cb here because doing it so early + * hangs TDB replication for some reason (and produces a process + * that needs -9 to kill). + * + * Instead, there's a little plug in cldu.c that does it. + */ end: free(cc->stor_host); diff --git a/server/server.c b/server/server.c index 5930712..0398a77 100644 --- a/server/server.c +++ b/server/server.c @@ -113,9 +113,6 @@ struct compiled_pat patterns[] = { { "\\d+\\.\\d+\\.\\d+\\.\\d+" }, }; -static char *state_name_cld[] = { - "Init", "Active" -}; static char *state_name_tdb[ST_TDBNUM] = { "Init", "Open", "Active", "Master", "Slave" }; @@ -383,8 +380,7 @@ static void stats_dump(void) X(event); X(tcp_accept); X(opt_write); - applog(LOG_INFO, "State: CLD %s TDB %s", - state_name_cld[tabled_srv.state_cld], + applog(LOG_INFO, "State: TDB %s", state_name_tdb[tabled_srv.state_tdb]); } @@ -1373,31 +1369,48 @@ static void tdb_state_cb(enum db_event event) } } -static void cld_state_cb(enum st_cld newstate) +/* + * Due to the way storage_node management is tightly woven into the + * server, the management of nodes is not in storage.c, which deals + * with the interface to Chunk and little more. + * + * We don't even bother with registering this callback, just call it by name. + */ +void stor_update_cb(void) { unsigned int env_flags; - if (debugging) { - applog(LOG_DEBUG, "CLD state %s > %s", - state_name_cld[tabled_srv.state_cld], - state_name_cld[newstate]); - } - tabled_srv.state_cld = newstate; - if (newstate == ST_CLD_ACTIVE) { - if (tabled_srv.state_tdb == ST_TDB_INIT) { - tabled_srv.state_tdb = ST_TDB_OPEN; - - env_flags = DB_RECOVER | DB_CREATE | DB_THREAD; - if (tdb_init(&tdb, tabled_srv.tdb_dir, NULL, - env_flags, "tabled", true, - tabled_srv.rep_remotes, - tabled_srv.ourhost, tabled_srv.rep_port, - tdb_state_cb)) { - tabled_srv.state_tdb = ST_TDB_INIT; - applog(LOG_ERR, "Failed to open TDB, limping"); - } + if (debugging) + applog(LOG_DEBUG, "We now have %d storage node(s)", + tabled_srv.num_stor); + if (tabled_srv.num_stor < 1) { + /* + * FIXME In the future, initiate net_close here if net was up. + */ + return; + } + + /* + * We initiate operations even if there's no redundancy in order + * to permit bootstrapping and build-time self-checking. + */ + if (tabled_srv.state_tdb == ST_TDB_INIT) { + tabled_srv.state_tdb = ST_TDB_OPEN; + + env_flags = DB_RECOVER | DB_CREATE | DB_THREAD; + if (tdb_init(&tdb, tabled_srv.tdb_dir, NULL, + env_flags, "tabled", true, + tabled_srv.rep_remotes, + tabled_srv.ourhost, tabled_srv.rep_port, + tdb_state_cb)) { + tabled_srv.state_tdb = ST_TDB_INIT; + applog(LOG_ERR, "Failed to open TDB, limping"); } - /* FIXME re-poke in case of TDB_MASTER, slave list may change */ + } else if (tabled_srv.state_tdb == ST_TDB_MASTER) { + /* + * FIXME This is where we should process redundancy decreases. + */ + ; } } @@ -1571,7 +1584,6 @@ int main (int argc, char *argv[]) int rc = 1; INIT_LIST_HEAD(&tabled_srv.all_stor); - tabled_srv.state_cld = ST_CLD_INIT; tabled_srv.state_tdb = ST_TDB_INIT; /* isspace() and strcasecmp() consistency requires this */ @@ -1579,11 +1591,13 @@ int main (int argc, char *argv[]) compile_patterns(); - cldc_init(); - stc_init(); SSL_library_init(); SSL_load_error_strings(); + stc_init(); + + cld_init(); + /* * parse command line */ @@ -1645,7 +1659,7 @@ int main (int argc, char *argv[]) if (rc) goto err_out_net; - if (cld_begin(tabled_srv.ourhost, NULL, cld_state_cb) != 0) { + if (cld_begin(tabled_srv.ourhost, NULL) != 0) { rc = 1; goto err_cld_session; } diff --git a/server/storage.c b/server/storage.c index 23765b0..380f8b0 100644 --- a/server/storage.c +++ b/server/storage.c @@ -97,7 +97,7 @@ int stor_put_start(struct open_chunk *cep, uint64_t key, uint64_t size) cep->wtogo = size; cep->wkey = key; if (debugging) - applog(LOG_INFO, "stor put %s new for %lld\n", + applog(LOG_INFO, "stor put %s new for %lld", stckey, (long long) size); return 0; @@ -310,12 +310,73 @@ bool stor_obj_test(struct open_chunk *cep, uint64_t key) return true; } -/* - * Add a node using its parameters file (not nul-terminated). - */ -void stor_add_node(const char *data, size_t len) +void stor_add_node(uint32_t nid, const char *hostname, const char *portstr, + struct geo *locp) { - /* P3 */ applog(LOG_INFO, "Adding some node or sumthin."); + struct addrinfo hints; + struct addrinfo *res, *res0; + struct storage_node *sn; + int rc; + + /* XXX FIXME search all_stor for dup NIDs */ + + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = PF_UNSPEC; + hints.ai_socktype = SOCK_DGRAM; + + rc = getaddrinfo(hostname, portstr, &hints, &res0); + if (rc) { + applog(LOG_WARNING, "getaddrinfo(%s:%s) failed: %s", + hostname, portstr, gai_strerror(rc)); + return; + } + + for (res = res0; res; res = res->ai_next) { + if (res->ai_family != AF_INET && res->ai_family != AF_INET6) + continue; + + if (res->ai_addrlen > ADDRSIZE) /* should not happen */ + continue; + + if ((sn = malloc(sizeof(struct storage_node))) == NULL) { + applog(LOG_WARNING, "No core (%ld)", + (long) sizeof(struct storage_node)); + break; + } + memset(sn, 0, sizeof(struct storage_node)); + + sn->id = nid; + + memcpy(&sn->addr, res->ai_addr, res->ai_addrlen); + sn->addr_af = res->ai_family; + sn->alen = res->ai_addrlen; + + if ((sn->hostname = strdup(hostname)) == NULL) { + applog(LOG_WARNING, "No core"); + free(sn); + break; + } + + if (debugging) { + char nhost[41]; + char nport[6]; + if (getnameinfo((struct sockaddr *) &sn->addr, sn->alen, + nhost, sizeof(nhost), + nport, sizeof(nport), + NI_NUMERICHOST|NI_NUMERICSERV) == 0) { + applog(LOG_INFO, "Found Chunk host %s port %s", + nhost, nport); + } else { + applog(LOG_INFO, "Found Chunk host"); + } + } + + list_add(&sn->all_link, &tabled_srv.all_stor); + tabled_srv.num_stor++; + } + + freeaddrinfo(res0); + return; } /* diff --git a/server/storparse.c b/server/storparse.c new file mode 100644 index 0000000..680b047 --- /dev/null +++ b/server/storparse.c @@ -0,0 +1,408 @@ + +/* + * Copyright (c) 2009, Red Hat, Inc. + */ +#define _GNU_SOURCE +#include "tabled-config.h" +#include <sys/types.h> +#include <glib.h> +#include <syslog.h> +#include <string.h> +#include <stdio.h> +#include <ctype.h> +#include "tabled.h" + +struct config_context { + char *text; + + char *fname; + + bool in_chunk; + bool in_chunk_reported; + + bool in_storage; + bool stor_encrypt; + char *stor_port; + char *stor_host; + + bool stor_ok; + char *stor_ok_port; + char *stor_ok_host; + + bool in_geo; + struct geo loc; + + unsigned int nid; +}; + +static void cfg_elm_start (GMarkupParseContext *context, + const gchar *element_name, + const gchar **attribute_names, + const gchar **attribute_values, + gpointer user_data, + GError **error) +{ + struct config_context *cc = user_data; + + if (!strcmp(element_name, "Chunk")) { + if (!cc->in_chunk) + cc->in_chunk = true; + else + applog(LOG_ERR, "%s: Nested Chunk", cc->fname); + } else { + if (!cc->in_chunk) { + /* + * We don't want to flood logs with bogus error + * messages if something benign happens to <Chunk>. + */ + if (!cc->in_chunk_reported) { + applog(LOG_ERR, + "%s: Element %s outside of <Chunk>", + cc->fname); + cc->in_chunk_reported = true; + } + return; + } + } + + if (!strcmp(element_name, "Socket")) { + if (!cc->in_storage) + cc->in_storage = true; + else + applog(LOG_ERR, "%s: Nested Socket", cc->fname); + } + else if (!strcmp(element_name, "Geo")) { + if (!cc->in_geo) + cc->in_geo = true; + else + applog(LOG_ERR, "%s: Nested CLD", cc->fname); + } +} + +static void cfg_elm_end_storage(struct config_context *cc) +{ + if (cc->text) { + applog(LOG_WARNING, "%s: Extra text in Socket element: \"%s\"", + cc->fname, cc->text); + free(cc->text); + cc->text = NULL; + goto end; + } + + if (!cc->stor_host) { + applog(LOG_WARNING, "%s: No host for Socket element", + cc->fname); + goto end; + } + if (!cc->stor_port) { + applog(LOG_WARNING, "%s: No port for Socket element", + cc->fname); + goto end; + } + + /* FIXME Chunkd with SSL needs certs, or else it's security theater. */ + if (cc->stor_encrypt) { + applog(LOG_WARNING, "%s: Good Socket, but " + "SSL access to Chunk is not supported yet", + cc->fname); + goto end; + } + + if (cc->stor_ok) { + applog(LOG_WARNING, "%s: Good Socket, but " + "multihomed Chunk is not supported yet", + cc->fname); + goto end; + } + + cc->stor_ok = true; + cc->stor_ok_host = cc->stor_host; + cc->stor_ok_port = cc->stor_port; + + cc->stor_encrypt = false; + cc->stor_host = NULL; + cc->stor_port = NULL; + return; + +end: + free(cc->stor_host); + cc->stor_host = NULL; + free(cc->stor_port); + cc->stor_port = NULL; +} + +static void cfg_elm_end_geo(struct config_context *cc) +{ + if (cc->text) { + applog(LOG_WARNING, "%s: Extra text in Geo element: \"%s\"", + cc->text); + free(cc->text); + cc->text = NULL; + goto end; + } + + if (!cc->loc.rack) { + applog(LOG_WARNING, "%s: No rack in Geo element", cc->fname); + goto end; + } + + /* do nothing - saving in-place */ + return; + +end: + free(cc->loc.area); + free(cc->loc.zone); + free(cc->loc.rack); + cc->loc.area = NULL; + cc->loc.zone = NULL; + cc->loc.rack = NULL; +} + +static void cfg_elm_end (GMarkupParseContext *context, + const gchar *element_name, + gpointer user_data, + GError **error) +{ + struct config_context *cc = user_data; + long n; + + if (!strcmp(element_name, "NID") && cc->text) { + if (!cc->text) { + applog(LOG_WARNING, "%s: NID element empty", cc->fname); + return; + } + + n = strtol(cc->text, NULL, 10); + if (n == 0 || (n & ~0xffffffff) != 0) + { + applog(LOG_WARNING, + "%s: NID '%s' invalid, ignoring", + cc->fname, cc->text); + free(cc->text); + cc->text = NULL; + return; + } + cc->nid = n; + free(cc->text); + cc->text = NULL; + } + + else if (!strcmp(element_name, "Socket")) { + cfg_elm_end_storage(cc); + cc->in_storage = false; + } + + else if (!strcmp(element_name, "Geo")) { + cfg_elm_end_geo(cc); + cc->in_geo = false; + } + + else if (!strcmp(element_name, "Area")) { + if (!cc->text) + return; + if (cc->text[0] == '-') + return; + + if (cc->in_geo) { + free(cc->loc.area); + cc->loc.area = cc->text; + } else { + applog(LOG_WARNING, + "%s: Area not in Geo", cc->fname); + free(cc->text); + } + cc->text = NULL; + } + + else if (!strcmp(element_name, "Zone")) { + if (!cc->text) + return; + if (cc->text[0] == '-') + return; + + if (cc->in_geo) { + free(cc->loc.zone); + cc->loc.zone = cc->text; + } else { + applog(LOG_WARNING, + "%s: Zone not in Geo", cc->fname); + free(cc->text); + } + cc->text = NULL; + } + + else if (!strcmp(element_name, "Rack")) { + if (!cc->text) + return; + if (cc->text[0] == '-') + return; + + if (cc->in_geo) { + free(cc->loc.rack); + cc->loc.rack = cc->text; + } else { + applog(LOG_WARNING, + "%s: Rack not in Geo", cc->fname); + free(cc->text); + } + cc->text = NULL; + } + + else if (!strcmp(element_name, "Type")) { + if (!cc->text) { + applog(LOG_WARNING, + "%s: Type element empty", cc->fname); + return; + } + + if (!strcmp(cc->text, "chunk")) { + ; + } else if (!strcmp(cc->text, "chunk-ssl")) { + cc->stor_encrypt = true; + } else { + applog(LOG_WARNING, "%s: Type '%s' is invalid", + cc->fname, cc->text); + return; + } + } + + else if (!strcmp(element_name, "Port")) { + if (!cc->text) { + applog(LOG_WARNING, "Port element empty"); + return; + } + + if (cc->in_storage) { + n = strtol(cc->text, NULL, 10); + if (n > 0 && n < 65536) { + free(cc->stor_port); + cc->stor_port = cc->text; + } else { + applog(LOG_WARNING, + "%s: Port '%s' invalid, ignoring", + cc->fname, cc->text); + free(cc->text); + } + cc->text = NULL; + } else { + applog(LOG_WARNING, + "%s: Port element not in Socket"); + } + } + + else if (!strcmp(element_name, "Host")) { + if (!cc->text) { + applog(LOG_WARNING, "Host element empty"); + return; + } + + if (cc->in_storage) { + free(cc->stor_host); + cc->stor_host = cc->text; + cc->text = NULL; + } else { + applog(LOG_WARNING, "%s: Host element not in Socket", + cc->fname); + } + } + + else if (!strcmp(element_name, "Chunk")) { + if (cc->in_chunk) { + cc->in_chunk = false; + } else { + applog(LOG_WARNING, "%s: Unbalanced closing Chunk", + cc->fname); + } + } + + else { + applog(LOG_WARNING, "%s: Unknown element \"%s\"", + cc->fname, element_name); + } + +} + +static bool str_n_isspace(const char *s, size_t n) +{ + char c; + size_t i; + + for (i = 0; i < n; i++) { + c = *s++; + if (!isspace(c)) + return false; + } + return true; +} + +static void cfg_elm_text (GMarkupParseContext *context, + const gchar *text, + gsize text_len, + gpointer user_data, + GError **error) +{ + struct config_context *cc = user_data; + + free(cc->text); + if (str_n_isspace(text, text_len)) + cc->text = NULL; + else + cc->text = g_strndup(text, text_len); +} + +static const GMarkupParser cfg_parse_ops = { + .start_element = cfg_elm_start, + .end_element = cfg_elm_end, + .text = cfg_elm_text, +}; + +void stor_parse(char *fname, const char *text, size_t len) +{ + GMarkupParseContext* parser; + struct config_context ctx; + + memset(&ctx, 0, sizeof(struct config_context)); + ctx.fname = fname; + + parser = g_markup_parse_context_new(&cfg_parse_ops, 0, &ctx, NULL); + if (!parser) { + applog(LOG_ERR, "g_markup_parse_context_new failed"); + return; + } + + if (!g_markup_parse_context_parse(parser, text, len, NULL)) { + applog(LOG_ERR, "Parse failure for Chunk in %s", fname); + g_markup_parse_context_free(parser); + goto out_free_all; + } + + g_markup_parse_context_free(parser); + + if (!ctx.nid) { + applog(LOG_WARNING, "%s: No NID\n", fname); + goto out_free_all; + } + if (!ctx.stor_ok) { + applog(LOG_WARNING, "%s: No useable Socket clause", fname); + goto out_free_all; + } + if (debugging) + applog(LOG_DEBUG, "Adding NID %u host %s port %s", + ctx.nid, ctx.stor_ok_host, ctx.stor_ok_port); + stor_add_node(ctx.nid, ctx.stor_ok_host, ctx.stor_ok_port, &ctx.loc); + stor_update_cb(); + +out_free_all: + free(ctx.text); + + free(ctx.stor_host); + free(ctx.stor_port); + + free(ctx.stor_ok_host); + free(ctx.stor_ok_port); + + free(ctx.loc.area); + free(ctx.loc.zone); + free(ctx.loc.rack); + return; +} diff --git a/server/tabled.h b/server/tabled.h index 357be73..918425a 100644 --- a/server/tabled.h +++ b/server/tabled.h @@ -80,12 +80,21 @@ struct compiled_pat { pcre *re; }; +struct geo { + char *area; + char *zone; /* Building */ + char *rack; +}; + struct storage_node { struct list_head all_link; + uint32_t id; + unsigned alen; int addr_af; struct sockaddr_in6 addr; char *hostname; /* Only used because stc_new is overly smart. */ + int nchu; /* number of open_chunk */ }; @@ -170,10 +179,6 @@ struct client { char req_buf[CLI_REQ_BUF_SZ]; /* input buffer */ }; -enum st_cld { - ST_CLD_INIT, ST_CLD_ACTIVE -}; - enum st_tdb { ST_TDB_INIT, ST_TDB_OPEN, ST_TDB_ACTIVE, ST_TDB_MASTER, ST_TDB_SLAVE, ST_TDBNUM @@ -215,9 +220,9 @@ struct server { GList *sockets; struct list_head all_stor; /* struct storage_node */ + int num_stor; /* number of storage_node's */ uint64_t object_count; - enum st_cld state_cld; enum st_tdb state_tdb, state_tdb_new; enum st_net state_net; @@ -255,9 +260,9 @@ extern void cli_out_end(struct client *cli); extern void cli_in_end(struct client *cli); /* cldc.c */ +extern void cld_init(void); +extern int cld_begin(const char *fqdn, const char *cell); extern void cldu_add_host(const char *host, unsigned int port); -extern int cld_begin(const char *fqdn, const char *cell, - void (*state_cb)(enum st_cld)); extern void cld_end(void); /* util.c */ @@ -295,6 +300,7 @@ extern bool cli_cb_free(struct client *cli, struct client_write *wr, extern bool cli_write_start(struct client *cli); extern int cli_req_avail(struct client *cli); extern void applog(int prio, const char *fmt, ...); +extern void stor_update_cb(void); /* config.c */ extern void read_config(void); @@ -312,7 +318,11 @@ extern bool stor_put_end(struct open_chunk *cep); extern ssize_t stor_get_buf(struct open_chunk *cep, void *data, size_t len); extern int stor_obj_del(struct storage_node *stn, uint64_t key); extern bool stor_obj_test(struct open_chunk *cep, uint64_t key); -extern void stor_add_node(const char *data, size_t len); +extern void stor_add_node(uint32_t nid, const char *hostname, + const char *portstr, struct geo *locp); extern void stor_init(void); +/* storparse.c */ +extern void stor_parse(char *fname, const char *text, size_t len); + #endif /* __TABLED_H__ */ diff --git a/test/chunkd-test.conf b/test/chunkd-test.conf index 9256b9d..119aa00 100644 --- a/test/chunkd-test.conf +++ b/test/chunkd-test.conf @@ -9,7 +9,11 @@ </Listen> <PID>chunkd.pid</PID> <Path>data/chunk</Path> -<NID>1</NID> +<!-- + Excluding NID prevents Chunk from registering with CLD and thus conflicting + with itself as described by <StorageNode> in tabled.conf. + --> +<!-- <NID>1</NID> --> <CLD> <Port>18081</Port> <Host>localhost</Host> diff --git a/test/start-daemon b/test/start-daemon index 590da92..203e392 100755 --- a/test/start-daemon +++ b/test/start-daemon @@ -20,10 +20,10 @@ fi cld -d data/cld -P cld.pid -p 18081 -E chunkd -C $top_srcdir/test/chunkd-test.conf -E -# 3 is enough, but we like to let chunkd to come up eary and register with CLD +# 3 is enough, but we like to let chunkd to come up early and register with CLD sleep 7 -../server/tabled -C $top_srcdir/test/tabled-test.conf -E +../server/tabled -C $top_srcdir/test/tabled-test.conf -E -D sleep 3 -- To unsubscribe from this list: send the line "unsubscribe hail-devel" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
