URL: https://github.com/SSSD/sssd/pull/812 Author: jhrozek Title: #812: Implement background refresh for IPA and AD domains and subdomains Action: opened
PR body: """ This PR refactors the existing background refresh task to be more extendable, splits out the account handlers of IPA and AD providers into tevent requests which are finally reused by new ipa_refresh and ad_refresh modules. The refreshes are done in batches so that we don't starve out other requests or even invoke the watchdog. """ To pull the PR as Git branch: git remote add ghsssd https://github.com/SSSD/sssd git fetch ghsssd pull/812/head:pr812 git checkout pr812
From 9812ba6a0074b8c6126374c34d5b8d78376d0db5 Mon Sep 17 00:00:00 2001 From: Jakub Hrozek <jhro...@redhat.com> Date: Wed, 24 Apr 2019 21:09:53 +0200 Subject: [PATCH 1/5] DP: Enable refresh for multiple domains Descend into subdomains on back end refresh and make sure to start from users again. --- src/providers/be_refresh.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/providers/be_refresh.c b/src/providers/be_refresh.c index e8cf5da758..6e590ec03a 100644 --- a/src/providers/be_refresh.c +++ b/src/providers/be_refresh.c @@ -256,7 +256,9 @@ static errno_t be_refresh_step(struct tevent_req *req) /* if not found than continue with next domain */ if (state->index == BE_REFRESH_TYPE_SENTINEL) { - state->domain = get_next_domain(state->domain, 0); + state->domain = get_next_domain(state->domain, + SSS_GND_DESCEND); + state->index = 0; continue; } From 60b656d9e7945074d372d3cc6b40f2a809c0222a Mon Sep 17 00:00:00 2001 From: Jakub Hrozek <jhro...@redhat.com> Date: Wed, 8 May 2019 14:38:44 +0200 Subject: [PATCH 2/5] BE/LDAP: Split out a helper function fro sdap_refresh for later reuse Every refresh request will send the same account_req. Let's split out the function that creates the account_req into a reusable one. Also removes the type string as it was only used in DEBUG messages and there is already a function in the back end API that provides the same functionality. --- src/providers/be_refresh.c | 18 ++++++++++++++++++ src/providers/be_refresh.h | 4 ++++ src/providers/ldap/sdap_refresh.c | 29 +++++------------------------ 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/providers/be_refresh.c b/src/providers/be_refresh.c index 6e590ec03a..bf8b68f0d6 100644 --- a/src/providers/be_refresh.c +++ b/src/providers/be_refresh.c @@ -341,3 +341,21 @@ errno_t be_refresh_recv(struct tevent_req *req) return EOK; } + +struct dp_id_data *be_refresh_acct_req(TALLOC_CTX *mem_ctx, + uint32_t entry_type, + struct sss_domain_info *domain) +{ + struct dp_id_data *account_req; + + account_req = talloc_zero(mem_ctx, struct dp_id_data); + if (account_req == NULL) { + return NULL; + } + + account_req->entry_type = entry_type; + account_req->filter_type = BE_FILTER_NAME; + account_req->extra_value = NULL; + account_req->domain = domain->name; + return account_req; +} diff --git a/src/providers/be_refresh.h b/src/providers/be_refresh.h index 927fa4a33b..26eaf4c62b 100644 --- a/src/providers/be_refresh.h +++ b/src/providers/be_refresh.h @@ -68,4 +68,8 @@ struct tevent_req *be_refresh_send(TALLOC_CTX *mem_ctx, errno_t be_refresh_recv(struct tevent_req *req); +struct dp_id_data *be_refresh_acct_req(TALLOC_CTX *mem_ctx, + uint32_t entry_type, + struct sss_domain_info *domain); + #endif /* _DP_REFRESH_H_ */ diff --git a/src/providers/ldap/sdap_refresh.c b/src/providers/ldap/sdap_refresh.c index 6d6c43e20c..eb7d1a19f9 100644 --- a/src/providers/ldap/sdap_refresh.c +++ b/src/providers/ldap/sdap_refresh.c @@ -30,7 +30,6 @@ struct sdap_refresh_state { struct dp_id_data *account_req; struct sdap_id_ctx *id_ctx; struct sdap_domain *sdom; - const char *type; char **names; size_t index; }; @@ -74,32 +73,12 @@ static struct tevent_req *sdap_refresh_send(TALLOC_CTX *mem_ctx, goto immediately; } - switch (entry_type) { - case BE_REQ_USER: - state->type = "user"; - break; - case BE_REQ_GROUP: - state->type = "group"; - break; - case BE_REQ_NETGROUP: - state->type = "netgroup"; - break; - default: - DEBUG(SSSDBG_CRIT_FAILURE, "Invalid entry type [%d]!\n", entry_type); - } - - state->account_req = talloc_zero(state, struct dp_id_data); + state->account_req = be_refresh_acct_req(state, entry_type, domain); if (state->account_req == NULL) { ret = ENOMEM; goto immediately; } - state->account_req->entry_type = entry_type; - state->account_req->filter_type = BE_FILTER_NAME; - state->account_req->extra_value = NULL; - state->account_req->domain = domain->name; - /* filter will be filled later */ - ret = sdap_refresh_step(req); if (ret == EOK) { DEBUG(SSSDBG_TRACE_FUNC, "Nothing to refresh\n"); @@ -143,7 +122,8 @@ static errno_t sdap_refresh_step(struct tevent_req *req) } DEBUG(SSSDBG_TRACE_FUNC, "Issuing refresh of %s %s\n", - state->type, state->account_req->filter_value); + be_req2str(state->account_req->entry_type), + state->account_req->filter_value); subreq = sdap_handle_acct_req_send(state, state->be_ctx, state->account_req, state->id_ctx, @@ -178,7 +158,8 @@ static void sdap_refresh_done(struct tevent_req *subreq) talloc_zfree(subreq); if (ret != EOK) { DEBUG(SSSDBG_CRIT_FAILURE, "Unable to refresh %s [dp_error: %d, " - "sdap_ret: %d, errno: %d]: %s\n", state->type, + "sdap_ret: %d, errno: %d]: %s\n", + be_req2str(state->account_req->entry_type), dp_error, sdap_ret, ret, err_msg); goto done; } From 35ff275c11d4937c10c99c3df19845be2c1adecb Mon Sep 17 00:00:00 2001 From: Jakub Hrozek <jhro...@redhat.com> Date: Wed, 8 May 2019 23:16:07 +0200 Subject: [PATCH 3/5] BE: Send refresh requests in batches As we extend the background refresh into larger domains, the amount of data that SSSD refreshes on the background might be larger. And refreshing all expired entries in a single request might block sssd_be for a long time, either triggering the watchdog or starving other legitimate requests. Therefore the background refresh will be done in batches of 200 entries. The first batch of every type (up to 200 users, up to 200 groups, ...) will be scheduled imediatelly and subsequent batches with a 0.5 second delay. --- src/providers/be_refresh.c | 129 +++++++++++++++++++++++++++++++------ src/util/util.h | 8 +++ 2 files changed, 119 insertions(+), 18 deletions(-) diff --git a/src/providers/be_refresh.c b/src/providers/be_refresh.c index bf8b68f0d6..ad1241ba32 100644 --- a/src/providers/be_refresh.c +++ b/src/providers/be_refresh.c @@ -183,8 +183,21 @@ struct be_refresh_state { struct sss_domain_info *domain; enum be_refresh_type index; time_t period; + + char **refresh_values; + size_t refresh_val_size; + size_t refresh_index; + + size_t batch_size; + char **refresh_batch; }; +static errno_t be_refresh_batch_step(struct tevent_req *req, + uint32_t msec_delay); +static void be_refresh_batch_step_wakeup(struct tevent_context *ev, + struct tevent_timer *tt, + struct timeval tv, + void *pvt); static errno_t be_refresh_step(struct tevent_req *req); static void be_refresh_done(struct tevent_req *subreq); @@ -215,6 +228,13 @@ struct tevent_req *be_refresh_send(TALLOC_CTX *mem_ctx, goto immediately; } + state->batch_size = 200; + state->refresh_batch = talloc_zero_array(state, char *, state->batch_size+1); + if (state->refresh_batch == NULL) { + ret = ENOMEM; + goto immediately; + } + ret = be_refresh_step(req); if (ret == EOK) { goto immediately; @@ -240,8 +260,6 @@ struct tevent_req *be_refresh_send(TALLOC_CTX *mem_ctx, static errno_t be_refresh_step(struct tevent_req *req) { struct be_refresh_state *state = NULL; - struct tevent_req *subreq = NULL; - char **values = NULL; errno_t ret; state = tevent_req_data(req, struct be_refresh_state); @@ -268,42 +286,101 @@ static errno_t be_refresh_step(struct tevent_req *req) goto done; } + talloc_zfree(state->refresh_values); ret = be_refresh_get_values(state, state->index, state->domain, - state->period, &values); + state->period, &state->refresh_values); if (ret != EOK) { DEBUG(SSSDBG_CRIT_FAILURE, "Unable to obtain DN list [%d]: %s\n", ret, sss_strerror(ret)); goto done; } - DEBUG(SSSDBG_TRACE_FUNC, "Refreshing %s in domain %s\n", - state->cb->name, state->domain->name); + for (state->refresh_val_size = 0; + state->refresh_values[state->refresh_val_size] != NULL; + state->refresh_val_size++); + + DEBUG(SSSDBG_TRACE_FUNC, "Refreshing %zu %s in domain %s\n", + state->refresh_val_size, state->cb->name, state->domain->name); - subreq = state->cb->send_fn(state, state->ev, state->be_ctx, - state->domain, values, state->cb->pvt); - if (subreq == NULL) { - ret = ENOMEM; + ret = be_refresh_batch_step(req, 0); + if (ret == EOK) { + state->index++; + continue; + } else if (ret != EAGAIN) { goto done; } - - /* make the list disappear with subreq */ - talloc_steal(subreq, values); - - tevent_req_set_callback(subreq, be_refresh_done, req); + /* EAGAIN only, refreshing something.. */ state->index++; - ret = EAGAIN; goto done; } ret = EOK; done: - if (ret != EOK && ret != EAGAIN) { - talloc_free(values); + return ret; +} + +static errno_t be_refresh_batch_step(struct tevent_req *req, + uint32_t msec_delay) +{ + struct be_refresh_state *state = tevent_req_data(req, struct be_refresh_state); + struct timeval tv; + struct tevent_timer *timeout = NULL; + + size_t remaining; + size_t batch_size; + + memset(state->refresh_batch, 0, sizeof(char *) * state->batch_size); + + if (state->refresh_index >= state->refresh_val_size) { + DEBUG(SSSDBG_FUNC_DATA, "The batch is done\n"); + return EOK; } - return ret; + remaining = state->refresh_val_size - state->refresh_index; + batch_size = MIN(remaining, state->batch_size); + DEBUG(SSSDBG_FUNC_DATA, + "This batch will refresh %zu entries (so far %zu/%zu)\n", + batch_size, state->refresh_index, state->refresh_val_size); + + for (size_t i = 0; i < batch_size; i++) { + state->refresh_batch[i] = state->refresh_values[state->refresh_index]; + state->refresh_index++; + } + + tv = tevent_timeval_current_ofs(0, msec_delay * 1000); + timeout = tevent_add_timer(state->be_ctx->ev, req, tv, + be_refresh_batch_step_wakeup, req); + if (timeout == NULL) { + return ENOMEM; + } + + return EAGAIN; +} + +static void be_refresh_batch_step_wakeup(struct tevent_context *ev, + struct tevent_timer *tt, + struct timeval tv, + void *pvt) +{ + struct tevent_req *req; + struct tevent_req *subreq = NULL; + struct be_refresh_state *state = NULL; + + req = talloc_get_type(pvt, struct tevent_req); + state = tevent_req_data(req, struct be_refresh_state); + + DEBUG(SSSDBG_TRACE_INTERNAL, "Issuing refresh\n"); + subreq = state->cb->send_fn(state, state->ev, state->be_ctx, + state->domain, + state->refresh_batch, + state->cb->pvt); + if (subreq == NULL) { + tevent_req_error(req, ENOMEM); + return; + } + tevent_req_set_callback(subreq, be_refresh_done, req); } static void be_refresh_done(struct tevent_req *subreq) @@ -321,8 +398,24 @@ static void be_refresh_done(struct tevent_req *subreq) goto done; } + ret = be_refresh_batch_step(req, 500); + if (ret == EAGAIN) { + DEBUG(SSSDBG_TRACE_INTERNAL, + "Another batch in this step in progress\n"); + return; + } else if (ret != EOK) { + DEBUG(SSSDBG_OP_FAILURE, + "be_refresh_batch_step failed [%d]: %s\n", + ret, sss_strerror(ret)); + goto done; + } + + DEBUG(SSSDBG_TRACE_INTERNAL, "All batches in this step refreshed\n"); + + /* Proceed to the next step */ ret = be_refresh_step(req); if (ret == EAGAIN) { + DEBUG(SSSDBG_TRACE_INTERNAL, "Another step in progress\n"); return; } diff --git a/src/util/util.h b/src/util/util.h index c818576f6c..62c51b25aa 100644 --- a/src/util/util.h +++ b/src/util/util.h @@ -66,6 +66,14 @@ #define NULL 0 #endif +#ifndef MIN +#define MIN(a, b) (((a) < (b)) ? (a) : (b)) +#endif + +#ifndef MAX +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) +#endif + #define SSSD_MAIN_OPTS SSSD_DEBUG_OPTS #define SSSD_SERVER_OPTS(uid, gid) \ From aef983c4949bdd4d462d3828e7649f26a5b7d36d Mon Sep 17 00:00:00 2001 From: Jakub Hrozek <jhro...@redhat.com> Date: Wed, 24 Apr 2019 20:52:11 +0200 Subject: [PATCH 4/5] AD: Implement background refresh for AD domains Split out the actual useful functionality from the AD account handler into a tevent request. This tevent request is then subsequently used by a new ad_refresh module. --- Makefile.am | 5 +- src/providers/ad/ad_common.h | 3 + src/providers/ad/ad_id.c | 137 ++++++++++++++---- src/providers/ad/ad_id.h | 10 ++ src/providers/ad/ad_init.c | 2 +- src/providers/ad/ad_refresh.c | 259 ++++++++++++++++++++++++++++++++++ 6 files changed, 385 insertions(+), 31 deletions(-) create mode 100644 src/providers/ad/ad_refresh.c diff --git a/Makefile.am b/Makefile.am index 9614334643..f09991e97d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -4500,7 +4500,10 @@ libsss_ad_la_SOURCES = \ src/providers/ad/ad_gpo_ndr.c \ src/providers/ad/ad_srv.c \ src/providers/ad/ad_subdomains.c \ - src/providers/ad/ad_domain_info.c + src/providers/ad/ad_domain_info.c \ + src/providers/ad/ad_refresh.c \ + $(NULL) + if BUILD_SUDO libsss_ad_la_SOURCES += \ diff --git a/src/providers/ad/ad_common.h b/src/providers/ad/ad_common.h index 39be3028fb..e4d62e852b 100644 --- a/src/providers/ad/ad_common.h +++ b/src/providers/ad/ad_common.h @@ -219,4 +219,7 @@ errno_t netlogon_get_domain_info(TALLOC_CTX *mem_ctx, char **_site, char **_forest); +errno_t ad_refresh_init(struct be_refresh_ctx *refresh_ctx, + struct ad_id_ctx *id_ctx); + #endif /* AD_COMMON_H_ */ diff --git a/src/providers/ad/ad_id.c b/src/providers/ad/ad_id.c index c3bda16626..b57d78207c 100644 --- a/src/providers/ad/ad_id.c +++ b/src/providers/ad/ad_id.c @@ -360,44 +360,36 @@ get_conn_list(TALLOC_CTX *mem_ctx, struct ad_id_ctx *ad_ctx, return clist; } -struct ad_account_info_handler_state { - struct sss_domain_info *domain; - struct dp_reply_std reply; +struct ad_account_info_state { + const char *err_msg; + int dp_error; }; -static void ad_account_info_handler_done(struct tevent_req *subreq); +static void ad_account_info_done(struct tevent_req *subreq); struct tevent_req * -ad_account_info_handler_send(TALLOC_CTX *mem_ctx, - struct ad_id_ctx *id_ctx, - struct dp_id_data *data, - struct dp_req_params *params) +ad_account_info_send(TALLOC_CTX *mem_ctx, + struct be_ctx *be_ctx, + struct ad_id_ctx *id_ctx, + struct dp_id_data *data) { - struct ad_account_info_handler_state *state; - struct sdap_id_conn_ctx **clist; - struct sdap_id_ctx *sdap_id_ctx; - struct sss_domain_info *domain; + struct sss_domain_info *domain = NULL; + struct ad_account_info_state *state = NULL; + struct tevent_req *req = NULL; + struct tevent_req *subreq = NULL; + struct sdap_id_conn_ctx **clist = NULL; + struct sdap_id_ctx *sdap_id_ctx = NULL; struct sdap_domain *sdom; - struct tevent_req *subreq; - struct tevent_req *req; - struct be_ctx *be_ctx; errno_t ret; - sdap_id_ctx = id_ctx->sdap_id_ctx; - be_ctx = params->be_ctx; - req = tevent_req_create(mem_ctx, &state, - struct ad_account_info_handler_state); + struct ad_account_info_state); if (req == NULL) { DEBUG(SSSDBG_CRIT_FAILURE, "tevent_req_create() failed\n"); return NULL; } - if (sdap_is_enum_request(data)) { - DEBUG(SSSDBG_TRACE_LIBS, "Skipping enumeration on demand\n"); - ret = EOK; - goto immediately; - } + sdap_id_ctx = id_ctx->sdap_id_ctx; domain = be_ctx->domain; if (strcasecmp(data->domain, be_ctx->domain->name) != 0) { @@ -406,6 +398,7 @@ ad_account_info_handler_send(TALLOC_CTX *mem_ctx, } if (domain == NULL) { + DEBUG(SSSDBG_CRIT_FAILURE, "Unknown domain\n"); ret = EINVAL; goto immediately; } @@ -413,6 +406,7 @@ ad_account_info_handler_send(TALLOC_CTX *mem_ctx, /* Determine whether to connect to GC, LDAP or try both. */ clist = get_conn_list(state, id_ctx, domain, data); if (clist == NULL) { + DEBUG(SSSDBG_CRIT_FAILURE, "Cannot create conn list\n"); ret = EIO; goto immediately; } @@ -423,14 +417,99 @@ ad_account_info_handler_send(TALLOC_CTX *mem_ctx, goto immediately; } - state->domain = sdom->dom; - subreq = ad_handle_acct_info_send(state, data, sdap_id_ctx, id_ctx->ad_options, sdom, clist); if (subreq == NULL) { ret = ENOMEM; goto immediately; } + tevent_req_set_callback(subreq, ad_account_info_done, req); + return req; + +immediately: + tevent_req_error(req, ret); + tevent_req_post(req, be_ctx->ev); + return req; +} + +static void ad_account_info_done(struct tevent_req *subreq) +{ + struct ad_account_info_state *state = NULL; + struct tevent_req *req = NULL; + errno_t ret; + + req = tevent_req_callback_data(subreq, struct tevent_req); + state = tevent_req_data(req, struct ad_account_info_state); + + ret = ad_handle_acct_info_recv(subreq, &state->dp_error, &state->err_msg); + if (ret != EOK) { + DEBUG(SSSDBG_OP_FAILURE, + "ad_handle_acct_info_recv failed [%d]: %s\n", + ret, sss_strerror(ret)); + /* The caller wouldn't fail either, just report the error up */ + } + talloc_zfree(subreq); + tevent_req_done(req); +} + +errno_t ad_account_info_recv(struct tevent_req *req, + int *_dp_error, + const char **_err_msg) +{ + struct ad_account_info_state *state = NULL; + + state = tevent_req_data(req, struct ad_account_info_state); + + TEVENT_REQ_RETURN_ON_ERROR(req); + + if (_err_msg != NULL) { + *_err_msg = state->err_msg; + } + + if (_dp_error) { + *_dp_error = state->dp_error; + } + + return EOK; +} + +struct ad_account_info_handler_state { + struct sss_domain_info *domain; + struct dp_reply_std reply; +}; + +static void ad_account_info_handler_done(struct tevent_req *subreq); + +struct tevent_req * +ad_account_info_handler_send(TALLOC_CTX *mem_ctx, + struct ad_id_ctx *id_ctx, + struct dp_id_data *data, + struct dp_req_params *params) +{ + struct ad_account_info_handler_state *state; + struct tevent_req *subreq; + struct tevent_req *req; + errno_t ret; + + + req = tevent_req_create(mem_ctx, &state, + struct ad_account_info_handler_state); + if (req == NULL) { + DEBUG(SSSDBG_CRIT_FAILURE, "tevent_req_create() failed\n"); + return NULL; + } + + if (sdap_is_enum_request(data)) { + DEBUG(SSSDBG_TRACE_LIBS, "Skipping enumeration on demand\n"); + ret = EOK; + goto immediately; + } + + subreq = ad_account_info_send(state, params->be_ctx, id_ctx, data); + if (subreq == NULL) { + ret = ENOMEM; + goto immediately; + } tevent_req_set_callback(subreq, ad_account_info_handler_done, req); @@ -457,7 +536,7 @@ static void ad_account_info_handler_done(struct tevent_req *subreq) req = tevent_req_callback_data(subreq, struct tevent_req); state = tevent_req_data(req, struct ad_account_info_handler_state); - ret = ad_handle_acct_info_recv(subreq, &dp_error, &err_msg); + ret = ad_account_info_recv(subreq, &dp_error, &err_msg); talloc_zfree(subreq); /* TODO For backward compatibility we always return EOK to DP now. */ @@ -466,8 +545,8 @@ static void ad_account_info_handler_done(struct tevent_req *subreq) } errno_t ad_account_info_handler_recv(TALLOC_CTX *mem_ctx, - struct tevent_req *req, - struct dp_reply_std *data) + struct tevent_req *req, + struct dp_reply_std *data) { struct ad_account_info_handler_state *state = NULL; diff --git a/src/providers/ad/ad_id.h b/src/providers/ad/ad_id.h index 5154393c5f..19cc54eec9 100644 --- a/src/providers/ad/ad_id.h +++ b/src/providers/ad/ad_id.h @@ -33,6 +33,16 @@ errno_t ad_account_info_handler_recv(TALLOC_CTX *mem_ctx, struct tevent_req *req, struct dp_reply_std *data); +struct tevent_req * +ad_account_info_send(TALLOC_CTX *mem_ctx, + struct be_ctx *be_ctx, + struct ad_id_ctx *id_ctx, + struct dp_id_data *data); + +errno_t ad_account_info_recv(struct tevent_req *req, + int *_dp_error, + const char **_err_msg); + struct tevent_req * ad_handle_acct_info_send(TALLOC_CTX *mem_ctx, struct dp_id_data *ar, diff --git a/src/providers/ad/ad_init.c b/src/providers/ad/ad_init.c index 4f1bd6dd16..b292d9809c 100644 --- a/src/providers/ad/ad_init.c +++ b/src/providers/ad/ad_init.c @@ -408,7 +408,7 @@ static errno_t ad_init_misc(struct be_ctx *be_ctx, return ret; } - ret = sdap_refresh_init(be_ctx->refresh_ctx, sdap_id_ctx); + ret = ad_refresh_init(be_ctx->refresh_ctx, ad_id_ctx); if (ret != EOK && ret != EEXIST) { DEBUG(SSSDBG_MINOR_FAILURE, "Periodical refresh " "will not work [%d]: %s\n", ret, sss_strerror(ret)); diff --git a/src/providers/ad/ad_refresh.c b/src/providers/ad/ad_refresh.c new file mode 100644 index 0000000000..0013be6b25 --- /dev/null +++ b/src/providers/ad/ad_refresh.c @@ -0,0 +1,259 @@ +/* + Copyright (C) 2019 Red Hat + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +#include <talloc.h> +#include <tevent.h> + +#include "providers/ad/ad_common.h" +#include "providers/ad/ad_id.h" + +struct ad_refresh_state { + struct tevent_context *ev; + struct be_ctx *be_ctx; + struct dp_id_data *account_req; + struct ad_id_ctx *id_ctx; + char **names; + size_t index; +}; + +static errno_t ad_refresh_step(struct tevent_req *req); +static void ad_refresh_done(struct tevent_req *subreq); + +static struct tevent_req *ad_refresh_send(TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct be_ctx *be_ctx, + struct sss_domain_info *domain, + int entry_type, + char **names, + void *pvt) +{ + struct ad_refresh_state *state = NULL; + struct tevent_req *req = NULL; + errno_t ret; + + req = tevent_req_create(mem_ctx, &state, + struct ad_refresh_state); + if (req == NULL) { + DEBUG(SSSDBG_CRIT_FAILURE, "tevent_req_create() failed\n"); + return NULL; + } + + if (names == NULL) { + ret = EOK; + goto immediately; + } + + state->ev = ev; + state->be_ctx = be_ctx; + state->id_ctx = talloc_get_type(pvt, struct ad_id_ctx); + state->names = names; + state->index = 0; + + state->account_req = be_refresh_acct_req(state, entry_type, domain); + if (state->account_req == NULL) { + ret = ENOMEM; + goto immediately; + } + + ret = ad_refresh_step(req); + if (ret == EOK) { + DEBUG(SSSDBG_TRACE_FUNC, "Nothing to refresh\n"); + goto immediately; + } else if (ret != EAGAIN) { + DEBUG(SSSDBG_CRIT_FAILURE, "ad_refresh_step() failed " + "[%d]: %s\n", ret, sss_strerror(ret)); + goto immediately; + } + + return req; + +immediately: + if (ret == EOK) { + tevent_req_done(req); + } else { + tevent_req_error(req, ret); + } + tevent_req_post(req, ev); + + return req; +} + +static errno_t ad_refresh_step(struct tevent_req *req) +{ + struct ad_refresh_state *state = NULL; + struct tevent_req *subreq = NULL; + errno_t ret; + + state = tevent_req_data(req, struct ad_refresh_state); + + if (state->names == NULL) { + ret = EOK; + goto done; + } + + state->account_req->filter_value = state->names[state->index]; + if (state->account_req->filter_value == NULL) { + ret = EOK; + goto done; + } + + DEBUG(SSSDBG_TRACE_FUNC, "Issuing refresh of %s %s\n", + be_req2str(state->account_req->entry_type), + state->account_req->filter_value); + + subreq = ad_account_info_send(state, state->be_ctx, state->id_ctx, + state->account_req); + if (subreq == NULL) { + ret = ENOMEM; + goto done; + } + + tevent_req_set_callback(subreq, ad_refresh_done, req); + + state->index++; + ret = EAGAIN; + +done: + return ret; +} + +static void ad_refresh_done(struct tevent_req *subreq) +{ + struct ad_refresh_state *state = NULL; + struct tevent_req *req = NULL; + const char *err_msg = NULL; + errno_t dp_error; + errno_t ret; + + req = tevent_req_callback_data(subreq, struct tevent_req); + state = tevent_req_data(req, struct ad_refresh_state); + + ret = ad_account_info_recv(subreq, &dp_error, &err_msg); + talloc_zfree(subreq); + if (ret != EOK) { + DEBUG(SSSDBG_CRIT_FAILURE, "Unable to refresh %s [dp_error: %d, " + "errno: %d]: %s\n", be_req2str(state->account_req->entry_type), + dp_error, ret, err_msg); + goto done; + } + + ret = ad_refresh_step(req); + if (ret == EAGAIN) { + return; + } + +done: + if (ret != EOK) { + tevent_req_error(req, ret); + return; + } + + tevent_req_done(req); +} + +static errno_t ad_refresh_recv(struct tevent_req *req) +{ + TEVENT_REQ_RETURN_ON_ERROR(req); + + return EOK; +} + +static struct tevent_req * +ad_refresh_users_send(TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct be_ctx *be_ctx, + struct sss_domain_info *domain, + char **names, + void *pvt) +{ + return ad_refresh_send(mem_ctx, ev, be_ctx, domain, + BE_REQ_USER, names, pvt); +} + +static errno_t ad_refresh_users_recv(struct tevent_req *req) +{ + return ad_refresh_recv(req); +} + +static struct tevent_req * +ad_refresh_groups_send(TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct be_ctx *be_ctx, + struct sss_domain_info *domain, + char **names, + void *pvt) +{ + return ad_refresh_send(mem_ctx, ev, be_ctx, domain, + BE_REQ_GROUP, names, pvt); +} + +static errno_t ad_refresh_groups_recv(struct tevent_req *req) +{ + return ad_refresh_recv(req); +} + +static struct tevent_req * +ad_refresh_netgroups_send(TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct be_ctx *be_ctx, + struct sss_domain_info *domain, + char **names, + void *pvt) +{ + return ad_refresh_send(mem_ctx, ev, be_ctx, domain, + BE_REQ_NETGROUP, names, pvt); +} + +static errno_t ad_refresh_netgroups_recv(struct tevent_req *req) +{ + return ad_refresh_recv(req); +} + +errno_t ad_refresh_init(struct be_refresh_ctx *refresh_ctx, + struct ad_id_ctx *id_ctx) +{ + errno_t ret; + + ret = be_refresh_add_cb(refresh_ctx, BE_REFRESH_TYPE_USERS, + ad_refresh_users_send, + ad_refresh_users_recv, + id_ctx); + if (ret != EOK && ret != EEXIST) { + DEBUG(SSSDBG_MINOR_FAILURE, "Periodical refresh of users " + "will not work [%d]: %s\n", ret, strerror(ret)); + } + + ret = be_refresh_add_cb(refresh_ctx, BE_REFRESH_TYPE_GROUPS, + ad_refresh_groups_send, + ad_refresh_groups_recv, + id_ctx); + if (ret != EOK && ret != EEXIST) { + DEBUG(SSSDBG_MINOR_FAILURE, "Periodical refresh of groups " + "will not work [%d]: %s\n", ret, strerror(ret)); + } + + ret = be_refresh_add_cb(refresh_ctx, BE_REFRESH_TYPE_NETGROUPS, + ad_refresh_netgroups_send, + ad_refresh_netgroups_recv, + id_ctx); + if (ret != EOK && ret != EEXIST) { + DEBUG(SSSDBG_MINOR_FAILURE, "Periodical refresh of netgroups " + "will not work [%d]: %s\n", ret, strerror(ret)); + } + + return ret; +} From 9b1e1075bdf0a3b1be4267885b4b01e733bc4697 Mon Sep 17 00:00:00 2001 From: Jakub Hrozek <jhro...@redhat.com> Date: Wed, 8 May 2019 14:39:23 +0200 Subject: [PATCH 5/5] IPA: Implement background refresh for IPA domains Split out the actual useful functionality from the IPA account lookup handler into a tevent request. This tevent request is then used in a new ipa_refresh module. --- Makefile.am | 1 + src/providers/ipa/ipa_common.h | 3 + src/providers/ipa/ipa_id.c | 137 +++++++++++++---- src/providers/ipa/ipa_id.h | 8 + src/providers/ipa/ipa_init.c | 2 +- src/providers/ipa/ipa_refresh.c | 254 ++++++++++++++++++++++++++++++++ 6 files changed, 373 insertions(+), 32 deletions(-) create mode 100644 src/providers/ipa/ipa_refresh.c diff --git a/Makefile.am b/Makefile.am index f09991e97d..c44ae2bf96 100644 --- a/Makefile.am +++ b/Makefile.am @@ -4428,6 +4428,7 @@ libsss_ipa_la_SOURCES = \ src/providers/ipa/ipa_srv.c \ src/providers/ipa/ipa_idmap.c \ src/providers/ipa/ipa_dn.c \ + src/providers/ipa/ipa_refresh.c \ src/providers/ad/ad_opts.c \ src/providers/ad/ad_common.c \ src/providers/ad/ad_dyndns.c \ diff --git a/src/providers/ipa/ipa_common.h b/src/providers/ipa/ipa_common.h index 31e671eb50..fc11d8109e 100644 --- a/src/providers/ipa/ipa_common.h +++ b/src/providers/ipa/ipa_common.h @@ -301,4 +301,7 @@ errno_t ipa_get_host_attrs(struct dp_option *ipa_options, struct sysdb_attrs **hosts, struct sysdb_attrs **_ipa_host); +errno_t ipa_refresh_init(struct be_refresh_ctx *refresh_ctx, + struct ipa_id_ctx *id_ctx); + #endif /* _IPA_COMMON_H_ */ diff --git a/src/providers/ipa/ipa_id.c b/src/providers/ipa/ipa_id.c index e644af5ffc..d3d14c6a11 100644 --- a/src/providers/ipa/ipa_id.c +++ b/src/providers/ipa/ipa_id.c @@ -1344,43 +1344,39 @@ ipa_decide_account_info_type(struct dp_id_data *data, struct be_ctx *be_ctx) return IPA_ACCOUNT_INFO_OTHER; } -struct ipa_account_info_handler_state { +struct ipa_account_info_state { enum ipa_account_info_type type; - struct dp_reply_std reply; + + const char *err_msg; + int dp_error; }; -static void ipa_account_info_handler_done(struct tevent_req *subreq); +static void ipa_account_info_done(struct tevent_req *subreq); struct tevent_req * -ipa_account_info_handler_send(TALLOC_CTX *mem_ctx, - struct ipa_id_ctx *id_ctx, - struct dp_id_data *data, - struct dp_req_params *params) +ipa_account_info_send(TALLOC_CTX *mem_ctx, + struct be_ctx *be_ctx, + struct ipa_id_ctx *id_ctx, + struct dp_id_data *data) { - struct ipa_account_info_handler_state *state; + struct ipa_account_info_state *state = NULL; + struct tevent_req *req = NULL; struct tevent_req *subreq = NULL; - struct tevent_req *req; errno_t ret; req = tevent_req_create(mem_ctx, &state, - struct ipa_account_info_handler_state); + struct ipa_account_info_state); if (req == NULL) { DEBUG(SSSDBG_CRIT_FAILURE, "tevent_req_create() failed\n"); return NULL; } - state->type = ipa_decide_account_info_type(data, params->be_ctx); - - if (sdap_is_enum_request(data)) { - DEBUG(SSSDBG_TRACE_LIBS, "Skipping enumeration on demand\n"); - ret = EOK; - goto immediately; - } + state->type = ipa_decide_account_info_type(data, be_ctx); switch (state->type) { case IPA_ACCOUNT_INFO_SUBDOMAIN: /* Subdomain lookups are handled differently on server and client. */ - subreq = ipa_subdomain_account_send(state, params->ev, id_ctx, data); + subreq = ipa_subdomain_account_send(state, be_ctx->ev, id_ctx, data); break; case IPA_ACCOUNT_INFO_NETGROUP: if (data->filter_type != BE_FILTER_NAME) { @@ -1388,11 +1384,11 @@ ipa_account_info_handler_send(TALLOC_CTX *mem_ctx, goto immediately; } - subreq = ipa_id_get_netgroup_send(state, params->ev, id_ctx, + subreq = ipa_id_get_netgroup_send(state, be_ctx->ev, id_ctx, data->filter_value); break; case IPA_ACCOUNT_INFO_OTHER: - subreq = ipa_id_get_account_info_send(state, params->ev, id_ctx, data); + subreq = ipa_id_get_account_info_send(state, be_ctx->ev, id_ctx, data); break; } @@ -1400,7 +1396,96 @@ ipa_account_info_handler_send(TALLOC_CTX *mem_ctx, ret = ENOMEM; goto immediately; } + tevent_req_set_callback(subreq, ipa_account_info_done, req); + return req; + +immediately: + tevent_req_error(req, ret); + tevent_req_post(req, be_ctx->ev); + return req; +} + +static void ipa_account_info_done(struct tevent_req *subreq) +{ + struct ipa_account_info_state *state = NULL; + struct tevent_req *req = NULL; + errno_t ret; + + req = tevent_req_callback_data(subreq, struct tevent_req); + state = tevent_req_data(req, struct ipa_account_info_state); + + switch (state->type) { + case IPA_ACCOUNT_INFO_SUBDOMAIN: + ret = ipa_subdomain_account_recv(subreq, &state->dp_error); + break; + case IPA_ACCOUNT_INFO_NETGROUP: + ret = ipa_id_get_netgroup_recv(subreq, &state->dp_error); + break; + case IPA_ACCOUNT_INFO_OTHER: + ret = ipa_id_get_account_info_recv(subreq, &state->dp_error); + break; + } + talloc_zfree(subreq); + + if (ret != EOK) { + tevent_req_error(req, ret); + return; + } + + tevent_req_done(req); +} + +errno_t ipa_account_info_recv(struct tevent_req *req, + int *_dp_error) +{ + struct ipa_account_info_state *state = NULL; + + state = tevent_req_data(req, struct ipa_account_info_state); + + /* Fail the request after collecting the dp_error */ + if (_dp_error) { + *_dp_error = state->dp_error; + } + + TEVENT_REQ_RETURN_ON_ERROR(req); + return EOK; +} + +struct ipa_account_info_handler_state { + struct dp_reply_std reply; +}; + +static void ipa_account_info_handler_done(struct tevent_req *subreq); + +struct tevent_req * +ipa_account_info_handler_send(TALLOC_CTX *mem_ctx, + struct ipa_id_ctx *id_ctx, + struct dp_id_data *data, + struct dp_req_params *params) +{ + struct ipa_account_info_handler_state *state; + struct tevent_req *subreq = NULL; + struct tevent_req *req; + errno_t ret; + + req = tevent_req_create(mem_ctx, &state, + struct ipa_account_info_handler_state); + if (req == NULL) { + DEBUG(SSSDBG_CRIT_FAILURE, "tevent_req_create() failed\n"); + return NULL; + } + + if (sdap_is_enum_request(data)) { + DEBUG(SSSDBG_TRACE_LIBS, "Skipping enumeration on demand\n"); + ret = EOK; + goto immediately; + } + subreq = ipa_account_info_send(state, params->be_ctx, id_ctx, data); + if (subreq == NULL) { + ret = ENOMEM; + goto immediately; + } tevent_req_set_callback(subreq, ipa_account_info_handler_done, req); return req; @@ -1425,17 +1510,7 @@ static void ipa_account_info_handler_done(struct tevent_req *subreq) req = tevent_req_callback_data(subreq, struct tevent_req); state = tevent_req_data(req, struct ipa_account_info_handler_state); - switch (state->type) { - case IPA_ACCOUNT_INFO_SUBDOMAIN: - ret = ipa_subdomain_account_recv(subreq, &dp_error); - break; - case IPA_ACCOUNT_INFO_NETGROUP: - ret = ipa_id_get_netgroup_recv(subreq, &dp_error); - break; - case IPA_ACCOUNT_INFO_OTHER: - ret = ipa_id_get_account_info_recv(subreq, &dp_error); - break; - } + ret = ipa_account_info_recv(subreq, &dp_error); talloc_zfree(subreq); /* TODO For backward compatibility we always return EOK to DP now. */ diff --git a/src/providers/ipa/ipa_id.h b/src/providers/ipa/ipa_id.h index 4b25498821..fe9acfeef0 100644 --- a/src/providers/ipa/ipa_id.h +++ b/src/providers/ipa/ipa_id.h @@ -33,6 +33,14 @@ #define IPA_DEFAULT_VIEW_NAME "Default Trust View" +struct tevent_req * +ipa_account_info_send(TALLOC_CTX *mem_ctx, + struct be_ctx *be_ctx, + struct ipa_id_ctx *id_ctx, + struct dp_id_data *data); +errno_t ipa_account_info_recv(struct tevent_req *req, + int *_dp_error); + struct tevent_req * ipa_account_info_handler_send(TALLOC_CTX *mem_ctx, struct ipa_id_ctx *id_ctx, diff --git a/src/providers/ipa/ipa_init.c b/src/providers/ipa/ipa_init.c index 6818e21714..7e6c5fd863 100644 --- a/src/providers/ipa/ipa_init.c +++ b/src/providers/ipa/ipa_init.c @@ -594,7 +594,7 @@ static errno_t ipa_init_misc(struct be_ctx *be_ctx, } } - ret = sdap_refresh_init(be_ctx->refresh_ctx, sdap_id_ctx); + ret = ipa_refresh_init(be_ctx->refresh_ctx, ipa_id_ctx); if (ret != EOK && ret != EEXIST) { DEBUG(SSSDBG_MINOR_FAILURE, "Periodical refresh " "will not work [%d]: %s\n", ret, sss_strerror(ret)); diff --git a/src/providers/ipa/ipa_refresh.c b/src/providers/ipa/ipa_refresh.c new file mode 100644 index 0000000000..6be9b8eb68 --- /dev/null +++ b/src/providers/ipa/ipa_refresh.c @@ -0,0 +1,254 @@ +/* + Copyright (C) 2019 Red Hat + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +#include <talloc.h> +#include <tevent.h> + +#include "providers/ipa/ipa_common.h" +#include "providers/ipa/ipa_id.h" + +struct ipa_refresh_state { + struct tevent_context *ev; + struct be_ctx *be_ctx; + struct dp_id_data *account_req; + struct ipa_id_ctx *id_ctx; + char **names; + size_t index; +}; + +static errno_t ipa_refresh_step(struct tevent_req *req); +static void ipa_refresh_done(struct tevent_req *subreq); + +static struct tevent_req *ipa_refresh_send(TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct be_ctx *be_ctx, + struct sss_domain_info *domain, + int entry_type, + char **names, + void *pvt) +{ + struct ipa_refresh_state *state = NULL; + struct tevent_req *req = NULL; + errno_t ret; + + req = tevent_req_create(mem_ctx, &state, + struct ipa_refresh_state); + if (req == NULL) { + DEBUG(SSSDBG_CRIT_FAILURE, "tevent_req_create() failed\n"); + return NULL; + } + + if (names == NULL) { + ret = EOK; + goto immediately; + } + + state->ev = ev; + state->be_ctx = be_ctx; + state->id_ctx = talloc_get_type(pvt, struct ipa_id_ctx); + state->names = names; + state->index = 0; + + state->account_req = be_refresh_acct_req(state, entry_type, domain); + if (state->account_req == NULL) { + ret = ENOMEM; + goto immediately; + } + + ret = ipa_refresh_step(req); + if (ret == EOK) { + DEBUG(SSSDBG_TRACE_FUNC, "Nothing to refresh\n"); + goto immediately; + } else if (ret != EAGAIN) { + DEBUG(SSSDBG_CRIT_FAILURE, "ipa_refresh_step() failed " + "[%d]: %s\n", ret, sss_strerror(ret)); + goto immediately; + } + + return req; + +immediately: + if (ret == EOK) { + tevent_req_done(req); + } else { + tevent_req_error(req, ret); + } + tevent_req_post(req, ev); + + return req; +} + +static errno_t ipa_refresh_step(struct tevent_req *req) +{ + struct ipa_refresh_state *state = NULL; + struct tevent_req *subreq = NULL; + errno_t ret; + + state = tevent_req_data(req, struct ipa_refresh_state); + + if (state->names == NULL) { + ret = EOK; + goto done; + } + + state->account_req->filter_value = state->names[state->index]; + if (state->account_req->filter_value == NULL) { + ret = EOK; + goto done; + } + + subreq = ipa_account_info_send(state, state->be_ctx, state->id_ctx, + state->account_req); + if (subreq == NULL) { + ret = ENOMEM; + goto done; + } + + tevent_req_set_callback(subreq, ipa_refresh_done, req); + + state->index++; + ret = EAGAIN; + +done: + return ret; +} + +static void ipa_refresh_done(struct tevent_req *subreq) +{ + struct ipa_refresh_state *state = NULL; + struct tevent_req *req = NULL; + errno_t dp_error; + errno_t ret; + + req = tevent_req_callback_data(subreq, struct tevent_req); + state = tevent_req_data(req, struct ipa_refresh_state); + + ret = ipa_account_info_recv(subreq, &dp_error); + talloc_zfree(subreq); + if (ret != EOK) { + DEBUG(SSSDBG_CRIT_FAILURE, "Unable to refresh %s [dp_error: %d, " + "errno: %d]\n", be_req2str(state->account_req->entry_type), + dp_error, ret); + goto done; + } + + ret = ipa_refresh_step(req); + if (ret == EAGAIN) { + return; + } + +done: + if (ret != EOK) { + tevent_req_error(req, ret); + return; + } + + tevent_req_done(req); +} + +static errno_t ipa_refresh_recv(struct tevent_req *req) +{ + TEVENT_REQ_RETURN_ON_ERROR(req); + + return EOK; +} + +static struct tevent_req * +ipa_refresh_users_send(TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct be_ctx *be_ctx, + struct sss_domain_info *domain, + char **names, + void *pvt) +{ + return ipa_refresh_send(mem_ctx, ev, be_ctx, domain, + BE_REQ_USER, names, pvt); +} + +static errno_t ipa_refresh_users_recv(struct tevent_req *req) +{ + return ipa_refresh_recv(req); +} + +static struct tevent_req * +ipa_refresh_groups_send(TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct be_ctx *be_ctx, + struct sss_domain_info *domain, + char **names, + void *pvt) +{ + return ipa_refresh_send(mem_ctx, ev, be_ctx, domain, + BE_REQ_GROUP, names, pvt); +} + +static errno_t ipa_refresh_groups_recv(struct tevent_req *req) +{ + return ipa_refresh_recv(req); +} + +static struct tevent_req * +ipa_refresh_netgroups_send(TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct be_ctx *be_ctx, + struct sss_domain_info *domain, + char **names, + void *pvt) +{ + return ipa_refresh_send(mem_ctx, ev, be_ctx, domain, + BE_REQ_NETGROUP, names, pvt); +} + +static errno_t ipa_refresh_netgroups_recv(struct tevent_req *req) +{ + return ipa_refresh_recv(req); +} + +errno_t ipa_refresh_init(struct be_refresh_ctx *refresh_ctx, + struct ipa_id_ctx *id_ctx) +{ + errno_t ret; + + ret = be_refresh_add_cb(refresh_ctx, BE_REFRESH_TYPE_USERS, + ipa_refresh_users_send, + ipa_refresh_users_recv, + id_ctx); + if (ret != EOK && ret != EEXIST) { + DEBUG(SSSDBG_MINOR_FAILURE, "Periodical refresh of users " + "will not work [%d]: %s\n", ret, strerror(ret)); + } + + ret = be_refresh_add_cb(refresh_ctx, BE_REFRESH_TYPE_GROUPS, + ipa_refresh_groups_send, + ipa_refresh_groups_recv, + id_ctx); + if (ret != EOK && ret != EEXIST) { + DEBUG(SSSDBG_MINOR_FAILURE, "Periodical refresh of groups " + "will not work [%d]: %s\n", ret, strerror(ret)); + } + + ret = be_refresh_add_cb(refresh_ctx, BE_REFRESH_TYPE_NETGROUPS, + ipa_refresh_netgroups_send, + ipa_refresh_netgroups_recv, + id_ctx); + if (ret != EOK && ret != EEXIST) { + DEBUG(SSSDBG_MINOR_FAILURE, "Periodical refresh of netgroups " + "will not work [%d]: %s\n", ret, strerror(ret)); + } + + return ret; +}
_______________________________________________ sssd-devel mailing list -- sssd-devel@lists.fedorahosted.org To unsubscribe send an email to sssd-devel-le...@lists.fedorahosted.org Fedora Code of Conduct: https://getfedora.org/code-of-conduct.html List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines List Archives: https://lists.fedorahosted.org/archives/list/sssd-devel@lists.fedorahosted.org