This unifies the neg and key caches and adds unwindctl status memory.
Please test with your normal workload and report back
unwindctl status
and
unwindctl status memory
after maybe a days usage.
OK?
commit eb2d659a831106f3ff1e62e7cdd8dc8bc13fd9f2
Author: Florian Obser <[email protected]>
Date: Mon Dec 16 18:08:26 2019 +0100
Use neg and key cache from the module environment if initialized so that
unwind can share caches between strategies.
diff --git sbin/unwind/libunbound/validator/validator.c
sbin/unwind/libunbound/validator/validator.c
index fa8d5419a80..c341d017e98 100644
--- sbin/unwind/libunbound/validator/validator.c
+++ sbin/unwind/libunbound/validator/validator.c
@@ -121,6 +121,8 @@ val_apply_cfg(struct module_env* env, struct val_env*
val_env,
log_err("out of memory");
return 0;
}
+ if (env->key_cache)
+ val_env->kcache = env->key_cache;
if(!val_env->kcache)
val_env->kcache = key_cache_create(cfg);
if(!val_env->kcache) {
@@ -146,6 +148,8 @@ val_apply_cfg(struct module_env* env, struct val_env*
val_env,
log_err("validator: cannot apply nsec3 key iterations");
return 0;
}
+ if (env->neg_cache)
+ val_env->neg_cache = env->neg_cache;
if(!val_env->neg_cache)
val_env->neg_cache = val_neg_create(cfg,
val_env->nsec3_maxiter[val_env->nsec3_keyiter_count-1]);
commit a80cf9061e6125eaecae9b7f23a3942e0a4a9618
Author: Florian Obser <[email protected]>
Date: Mon Dec 16 18:12:18 2019 +0100
rework unified cache handling and unify key and neg caches
diff --git sbin/unwind/resolver.c sbin/unwind/resolver.c
index 4a511e030a8..3cdfa27d297 100644
--- sbin/unwind/resolver.c
+++ sbin/unwind/resolver.c
@@ -56,6 +56,9 @@
#include "libunbound/util/module.h"
#include "libunbound/util/regional.h"
#include "libunbound/util/storage/slabhash.h"
+#include "libunbound/validator/validator.h"
+#include "libunbound/validator/val_kcache.h"
+#include "libunbound/validator/val_neg.h"
#include <openssl/crypto.h>
@@ -150,6 +153,7 @@ void ub_resolve_done(void *, int,
void *, int, int, char *,
void asr_resolve_done(struct asr_result *, void *);
void new_resolver(enum uw_resolver_type);
struct uw_resolver *create_resolver(enum uw_resolver_type);
+void setup_unified_caches(void);
void set_unified_cache(struct uw_resolver *);
void free_resolver(struct uw_resolver *);
void set_forwarders(struct uw_resolver *,
@@ -206,9 +210,11 @@ struct event_base *ev_base;
RB_GENERATE(force_tree, force_tree_entry, entry, force_tree_cmp)
-struct alloc_cache unified_cache_alloc;
+int val_id = -1;
struct slabhash *unified_msg_cache;
struct rrset_cache *unified_rrset_cache;
+struct key_cache *unified_key_cache;
+struct val_neg_cache *unified_neg_cache;
static const char * const as112_zones[] = {
/* RFC1918 */
@@ -333,10 +339,10 @@ resolver_sig_handler(int sig, short event, void *arg)
void
resolver(int debug, int verbose)
{
- struct config_file *ub_cfg;
struct event ev_sigint, ev_sigterm;
struct passwd *pw;
struct timeval tv = {DECAY_PERIOD, 0};
+ struct alloc_cache cache_alloc_test;
resolver_conf = config_new_empty();
@@ -390,30 +396,12 @@ resolver(int debug, int verbose)
clock_gettime(CLOCK_MONOTONIC, &last_network_change);
- if ((ub_cfg = config_create_forlib()) == NULL)
- fatal(NULL);
-
- alloc_init(&unified_cache_alloc, NULL, 0);
- if (unified_cache_alloc.max_reg_blocks != 10)
+ alloc_init(&cache_alloc_test, NULL, 0);
+ if (cache_alloc_test.max_reg_blocks != 10)
fatalx("local libunbound/util/alloc.c diff lost");
+ alloc_clear(&cache_alloc_test);
- /*
- * context_finalize() ensures that the cache is sized according to
- * the context's config. If we want to change the cache size we
- * need to reflect it in the config as well otherwise these cache
- * objects get deleted and re-created.
- */
- if ((unified_msg_cache = slabhash_create(ub_cfg->msg_cache_slabs,
- HASH_DEFAULT_STARTARRAY, ub_cfg->msg_cache_size, msgreply_sizefunc,
- query_info_compare, query_entry_delete, reply_info_delete, NULL))
- == NULL)
- fatal(NULL);
-
- if ((unified_rrset_cache = rrset_cache_adjust(NULL, ub_cfg,
- &unified_cache_alloc)) == NULL)
- fatal(NULL);
-
- config_delete(ub_cfg);
+ setup_unified_caches();
TAILQ_INIT(&autoconf_forwarder_list);
TAILQ_INIT(&trust_anchors);
@@ -1106,8 +1094,17 @@ set_unified_cache(struct uw_resolver *res)
res->ctx->env->msg_cache = unified_msg_cache;
res->ctx->env->rrset_cache = unified_rrset_cache;
+ res->ctx->env->key_cache = unified_key_cache;
+ res->ctx->env->neg_cache = unified_neg_cache;
context_finalize(res->ctx);
+
+ if (res->ctx->env->msg_cache != unified_msg_cache ||
+ res->ctx->env->rrset_cache != unified_rrset_cache ||
+ res->ctx->env->key_cache != unified_key_cache ||
+ res->ctx->env->neg_cache != unified_neg_cache)
+ fatalx("failed to set unified caches, libunbound/validator/"
+ "validator.c diff lost");
}
static const struct {
@@ -1286,6 +1283,8 @@ create_resolver(enum uw_resolver_type type)
void
free_resolver(struct uw_resolver *res)
{
+ struct val_env *val_env;
+
if (res == NULL)
return;
@@ -1294,10 +1293,16 @@ free_resolver(struct uw_resolver *res)
else {
evtimer_del(&res->check_ev);
if (res->ctx != NULL) {
- if (res->ctx->env->msg_cache == unified_msg_cache)
+ if (res->ctx->env->msg_cache == unified_msg_cache) {
+ val_env = (struct val_env*)
+ res->ctx->env->modinfo[val_id];
res->ctx->env->msg_cache = NULL;
- if (res->ctx->env->rrset_cache == unified_rrset_cache)
res->ctx->env->rrset_cache = NULL;
+ val_env->kcache = NULL;
+ res->ctx->env->key_cache = NULL;
+ val_env->neg_cache = NULL;
+ res->ctx->env->neg_cache = NULL;
+ }
}
ub_ctx_delete(res->ctx);
asr_resolver_free(res->asr_ctx);
@@ -1305,6 +1310,56 @@ free_resolver(struct uw_resolver *res)
}
}
+void
+setup_unified_caches(void)
+{
+ struct ub_ctx *ctx;
+ struct val_env *val_env;
+ size_t i;
+ int err, j;
+
+ if ((ctx = ub_ctx_create_event(ev_base)) == NULL)
+ fatalx("could not create unbound context");
+ for (i = 0; i < nitems(options); i++) {
+ if ((err = ub_ctx_set_option(ctx, options[i].name,
+ options[i].value)) != 0) {
+ fatalx("error setting %s: %s: %s", options[i].name,
+ options[i].value, ub_strerror(err));
+ }
+ }
+
+ context_finalize(ctx);
+
+ if (ctx->env->msg_cache == NULL || ctx->env->rrset_cache == NULL ||
+ ctx->env->key_cache == NULL || ctx->env->neg_cache == NULL)
+ fatalx("could not setup unified caches");
+
+ unified_msg_cache = ctx->env->msg_cache;
+ unified_rrset_cache = ctx->env->rrset_cache;
+ unified_key_cache = ctx->env->key_cache;
+ unified_neg_cache = ctx->env->neg_cache;
+
+ if (val_id == -1) {
+ for (j = 0; j < ctx->mods.num; j++) {
+ if (strcmp(ctx->mods.mod[j]->name, "validator") == 0) {
+ val_id = j;
+ break;
+ }
+ }
+ if (val_id == -1)
+ fatalx("cannot find validator module");
+ }
+
+ val_env = (struct val_env*)ctx->env->modinfo[val_id];
+ ctx->env->msg_cache = NULL;
+ ctx->env->rrset_cache = NULL;
+ ctx->env->key_cache = NULL;
+ val_env->kcache = NULL;
+ ctx->env->neg_cache = NULL;
+ val_env->neg_cache = NULL;
+ ub_ctx_delete(ctx);
+}
+
void
set_forwarders(struct uw_resolver *res, struct uw_forwarder_head
*uw_forwarder_list, int port_override)
commit bd137095b29134db85897b7facb34b6b39358266
Author: Florian Obser <[email protected]>
Date: Mon Dec 16 18:15:14 2019 +0100
status memory
diff --git sbin/unwind/control.c sbin/unwind/control.c
index 13a6f340a84..34a34f63ee9 100644
--- sbin/unwind/control.c
+++ sbin/unwind/control.c
@@ -274,6 +274,7 @@ control_dispatch_imsg(int fd, short event, void *bula)
break;
case IMSG_CTL_STATUS:
case IMSG_CTL_AUTOCONF:
+ case IMSG_CTL_MEM:
if (IMSG_DATA_SIZE(imsg) != 0)
break;
frontend_imsg_compose_resolver(imsg.hdr.type, 0, NULL,
diff --git sbin/unwind/frontend.c sbin/unwind/frontend.c
index b1792efb668..c99061fe9aa 100644
--- sbin/unwind/frontend.c
+++ sbin/unwind/frontend.c
@@ -484,6 +484,7 @@ frontend_dispatch_resolver(int fd, short event, void *bula)
break;
case IMSG_CTL_RESOLVER_INFO:
case IMSG_CTL_AUTOCONF_RESOLVER_INFO:
+ case IMSG_CTL_MEM_INFO:
case IMSG_CTL_END:
control_imsg_relay(&imsg);
break;
diff --git sbin/unwind/resolver.c sbin/unwind/resolver.c
index 3cdfa27d297..6a5ccd62d1c 100644
--- sbin/unwind/resolver.c
+++ sbin/unwind/resolver.c
@@ -174,6 +174,7 @@ int resolver_cmp(const void *, const void
*);
void restart_ub_resolvers(void);
void show_status(pid_t);
void show_autoconf(pid_t);
+void show_mem(pid_t);
void send_resolver_info(struct uw_resolver *, pid_t);
void send_detailed_resolver_info(struct uw_resolver *,
pid_t);
@@ -513,6 +514,12 @@ resolver_dispatch_frontend(int fd, short event, void *bula)
"%lu", __func__, IMSG_DATA_SIZE(imsg));
show_autoconf(imsg.hdr.pid);
break;
+ case IMSG_CTL_MEM:
+ if (IMSG_DATA_SIZE(imsg) != 0)
+ fatalx("%s: IMSG_CTL_AUTOCONF wrong length: "
+ "%lu", __func__, IMSG_DATA_SIZE(imsg));
+ show_mem(imsg.hdr.pid);
+ break;
case IMSG_NEW_TA:
/* make sure this is a string */
((char *)imsg.data)[IMSG_DATA_SIZE(imsg) - 1] = '\0';
@@ -1697,6 +1704,25 @@ show_autoconf(pid_t pid)
resolver_imsg_compose_frontend(IMSG_CTL_END, pid, NULL, 0);
}
+void
+show_mem(pid_t pid)
+{
+ struct ctl_mem_info cmi;
+
+ memset(&cmi, 0, sizeof(cmi));
+ cmi.msg_cache_used = slabhash_get_mem(unified_msg_cache);
+ cmi.msg_cache_max = slabhash_get_size(unified_msg_cache);
+ cmi.rrset_cache_used = slabhash_get_mem(&unified_rrset_cache->table);
+ cmi.rrset_cache_max = slabhash_get_size(&unified_rrset_cache->table);
+ cmi.key_cache_used = slabhash_get_mem(unified_key_cache->slab);
+ cmi.key_cache_max = slabhash_get_size(unified_key_cache->slab);
+ cmi.neg_cache_used = unified_neg_cache->use;
+ cmi.neg_cache_max = unified_neg_cache->max;
+ resolver_imsg_compose_frontend(IMSG_CTL_MEM_INFO, pid, &cmi,
+ sizeof(cmi));
+
+}
+
void
send_resolver_info(struct uw_resolver *res, pid_t pid)
{
diff --git sbin/unwind/resolver.h sbin/unwind/resolver.h
index 899b493d86c..d605af8af20 100644
--- sbin/unwind/resolver.h
+++ sbin/unwind/resolver.h
@@ -60,6 +60,17 @@ struct ctl_forwarder_info {
int src;
};
+struct ctl_mem_info {
+ size_t msg_cache_used;
+ size_t msg_cache_max;
+ size_t rrset_cache_used;
+ size_t rrset_cache_max;
+ size_t key_cache_used;
+ size_t key_cache_max;
+ size_t neg_cache_used;
+ size_t neg_cache_max;
+};
+
void resolver(int, int);
int resolver_imsg_compose_main(int, pid_t, void *, uint16_t);
int resolver_imsg_compose_frontend(int, pid_t, void *, uint16_t);
diff --git sbin/unwind/unwind.h sbin/unwind/unwind.h
index 7a2fe17d457..087f6fd49ce 100644
--- sbin/unwind/unwind.h
+++ sbin/unwind/unwind.h
@@ -100,6 +100,7 @@ enum imsg_type {
IMSG_CTL_RELOAD,
IMSG_CTL_STATUS,
IMSG_CTL_AUTOCONF,
+ IMSG_CTL_MEM,
IMSG_RECONF_CONF,
IMSG_RECONF_BLOCKLIST_FILE,
IMSG_RECONF_FORWARDER,
@@ -119,6 +120,7 @@ enum imsg_type {
IMSG_ANSWER,
IMSG_CTL_RESOLVER_INFO,
IMSG_CTL_AUTOCONF_RESOLVER_INFO,
+ IMSG_CTL_MEM_INFO,
IMSG_CTL_END,
IMSG_HTTPSOCK,
IMSG_TAFD,
diff --git usr.sbin/unwindctl/parser.c usr.sbin/unwindctl/parser.c
index c091a4efecb..0d35ce29f4c 100644
--- usr.sbin/unwindctl/parser.c
+++ usr.sbin/unwindctl/parser.c
@@ -69,6 +69,7 @@ static const struct token t_log[] = {
static const struct token t_status[] = {
{NOTOKEN, "", NONE, NULL},
{KEYWORD, "autoconf", AUTOCONF, NULL},
+ {KEYWORD, "memory", MEM, NULL},
{ENDTOKEN, "", NONE, NULL}
};
diff --git usr.sbin/unwindctl/parser.h usr.sbin/unwindctl/parser.h
index 7baba5a231a..26fa7a76d35 100644
--- usr.sbin/unwindctl/parser.h
+++ usr.sbin/unwindctl/parser.h
@@ -25,7 +25,8 @@ enum actions {
RELOAD,
PORTAL,
STATUS,
- AUTOCONF
+ AUTOCONF,
+ MEM
};
struct parse_result {
diff --git usr.sbin/unwindctl/unwindctl.8 usr.sbin/unwindctl/unwindctl.8
index c27e407536c..62740a24dd4 100644
--- usr.sbin/unwindctl/unwindctl.8
+++ usr.sbin/unwindctl/unwindctl.8
@@ -60,6 +60,8 @@ Show nameservers learned from
.Xr dhclient 8
or
.Xr slaacd 8 .
+.It Cm status Cm memory
+Show memory consumption.
.El
.Sh FILES
.Bl -tag -width "/dev/unwind.sockXX" -compact
diff --git usr.sbin/unwindctl/unwindctl.c usr.sbin/unwindctl/unwindctl.c
index 836b1224db5..397cdbcea1f 100644
--- usr.sbin/unwindctl/unwindctl.c
+++ usr.sbin/unwindctl/unwindctl.c
@@ -46,6 +46,7 @@
__dead void usage(void);
int show_status_msg(struct imsg *);
int show_autoconf_msg(struct imsg *);
+int show_mem_msg(struct imsg *);
void histogram_header(void);
void print_histogram(const char *name, int64_t[], size_t);
@@ -150,6 +151,9 @@ main(int argc, char *argv[])
case AUTOCONF:
imsg_compose(ibuf, IMSG_CTL_AUTOCONF, 0, 0, -1, NULL, 0);
break;
+ case MEM:
+ imsg_compose(ibuf, IMSG_CTL_MEM, 0, 0, -1, NULL, 0);
+ break;
default:
usage();
}
@@ -177,6 +181,9 @@ main(int argc, char *argv[])
case AUTOCONF:
done = show_autoconf_msg(&imsg);
break;
+ case MEM:
+ done = show_mem_msg(&imsg);
+ break;
default:
break;
}
@@ -319,3 +326,31 @@ print_histogram(const char *name, int64_t histogram[],
size_t n)
printf("%6lld", histogram[i]);
printf("\n");
}
+
+int
+show_mem_msg(struct imsg *imsg)
+{
+ struct ctl_mem_info *cmi;
+
+ switch (imsg->hdr.type) {
+ case IMSG_CTL_MEM_INFO:
+ cmi = imsg->data;
+ printf("msg-cache: %zu / %zu (%.2f%%)\n", cmi->msg_cache_used,
+ cmi->msg_cache_max, 100.0 * cmi->msg_cache_used /
+ cmi->msg_cache_max);
+ printf("rrset-cache: %zu / %zu (%.2f%%)\n",
+ cmi->rrset_cache_used, cmi->rrset_cache_max, 100.0 *
+ cmi->rrset_cache_used / cmi->rrset_cache_max);
+ printf("key-cache: %zu / %zu (%.2f%%)\n", cmi->key_cache_used,
+ cmi->key_cache_max, 100.0 * cmi->key_cache_used /
+ cmi->key_cache_max);
+ printf("neg-cache: %zu / %zu (%.2f%%)\n", cmi->neg_cache_used,
+ cmi->neg_cache_max, 100.0 * cmi->neg_cache_used /
+ cmi->neg_cache_max);
+ break;
+ default:
+ break;
+ }
+
+ return 1;
+}
--
I'm not entirely sure you are real.
