The branch, master has been updated
via 22f1c4005ca paged results: testing suite for new paged results
module
via 975807001e9 paged results: new paged results module using GUID list
via 96c03c75329 vlv: tests for delete, add, and modify cases
from ba016939aa9 s3-vfs-fruit: add close call
https://git.samba.org/?p=samba.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit 22f1c4005caae5c5a376fd180be98b1be0db2afc
Author: Aaron Haslett <[email protected]>
Date: Mon Nov 12 14:35:40 2018 +1300
paged results: testing suite for new paged results module
Testing the new GUID list based paged results module
Signed-off-by: Aaron Haslett <[email protected]>
Reviewed-by: Gary Lockyer <[email protected]>
Reviewed-by: Andrew Bartlett <[email protected]>
Autobuild-User(master): Gary Lockyer <[email protected]>
Autobuild-Date(master): Fri Dec 21 11:10:30 CET 2018 on sn-devel-144
commit 975807001e91f75906ccc8a01bc4093aea46e9eb
Author: Aaron Haslett <[email protected]>
Date: Mon Nov 12 14:30:55 2018 +1300
paged results: new paged results module using GUID list
Replacing paged results module to use GUID list instead of storing
result list in memory, in order to improve memory performance.
Signed-off-by: Aaron Haslett <[email protected]>
Reviewed-by: Gary Lockyer <[email protected]>
Reviewed-by: Andrew Bartlett <[email protected]>
commit 96c03c75329e03bdbb1049522207cfca76d4489d
Author: Aaron Haslett <[email protected]>
Date: Mon Nov 12 14:15:08 2018 +1300
vlv: tests for delete, add, and modify cases
More vlv testing for cases involving modifying, deleting, and adding records
while observing the effect on already initialised views.
Signed-off-by: Aaron Haslett <[email protected]>
Reviewed-by: Gary Lockyer <[email protected]>
Reviewed-by: Andrew Bartlett <[email protected]>
-----------------------------------------------------------------------
Summary of changes:
lib/ldb/modules/paged_results.c | 445 ------------
lib/ldb/wscript | 9 -
python/samba/tests/samba3sam.py | 2 +-
selftest/knownfail.d/vlv | 2 +
source4/dsdb/samdb/ldb_modules/paged_results.c | 795 +++++++++++++++++++++
source4/dsdb/samdb/ldb_modules/samba_dsdb.c | 2 +-
.../dsdb/samdb/ldb_modules/wscript_build_server | 9 +
source4/dsdb/tests/python/vlv.py | 529 +++++++++++++-
8 files changed, 1334 insertions(+), 459 deletions(-)
delete mode 100644 lib/ldb/modules/paged_results.c
create mode 100644 selftest/knownfail.d/vlv
create mode 100644 source4/dsdb/samdb/ldb_modules/paged_results.c
Changeset truncated at 500 lines:
diff --git a/lib/ldb/modules/paged_results.c b/lib/ldb/modules/paged_results.c
deleted file mode 100644
index ecb22271d28..00000000000
--- a/lib/ldb/modules/paged_results.c
+++ /dev/null
@@ -1,445 +0,0 @@
-/*
- ldb database library
-
- Copyright (C) Simo Sorce 2005-2008
-
- ** NOTE! The following LGPL license applies to the ldb
- ** library. This does NOT imply that all of Samba is released
- ** under the LGPL
-
- This library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 3 of the License, or (at your option) any later version.
-
- This library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with this library; if not, see <http://www.gnu.org/licenses/>.
-*/
-
-/*
- * Name: paged_result
- *
- * Component: ldb paged results control module
- *
- * Description: this module caches a complete search and sends back
- * results in chunks as asked by the client
- *
- * Author: Simo Sorce
- */
-
-#include "replace.h"
-#include "system/filesys.h"
-#include "system/time.h"
-#include "dlinklist.h"
-#include <assert.h>
-#include "ldb_module.h"
-
-struct message_store {
- /* keep the whole ldb_reply as an optimization
- * instead of freeing and talloc-ing the container
- * on each result */
- struct ldb_reply *r;
- struct message_store *next;
-};
-
-struct private_data;
-
-struct results_store {
- struct results_store *prev, *next;
-
- struct private_data *priv;
-
- char *cookie;
- time_t timestamp;
-
- struct message_store *first;
- struct message_store *last;
- int num_entries;
-
- struct message_store *first_ref;
- struct message_store *last_ref;
-
- struct ldb_control **controls;
-};
-
-struct private_data {
- uint32_t next_free_id;
- size_t num_stores;
- struct results_store *store;
-
-};
-
-static int store_destructor(struct results_store *del)
-{
- struct private_data *priv = del->priv;
- DLIST_REMOVE(priv->store, del);
-
- assert(priv->num_stores > 0);
- priv->num_stores -= 1;
-
- return 0;
-}
-
-static struct results_store *new_store(struct private_data *priv)
-{
- struct results_store *newr;
- uint32_t new_id = priv->next_free_id++;
-
- /* TODO: we should have a limit on the number of
- * outstanding paged searches
- */
-
- newr = talloc(priv, struct results_store);
- if (!newr) return NULL;
-
- newr->priv = priv;
-
- newr->cookie = talloc_asprintf(newr, "%d", new_id);
- if (!newr->cookie) {
- talloc_free(newr);
- return NULL;
- }
-
- newr->timestamp = time(NULL);
-
- newr->first = NULL;
- newr->num_entries = 0;
- newr->first_ref = NULL;
- newr->controls = NULL;
-
- DLIST_ADD(priv->store, newr);
-
- assert(priv->num_stores < SIZE_MAX);
- priv->num_stores += 1;
-
- talloc_set_destructor(newr, store_destructor);
-
- if (priv->num_stores > 10) {
- struct results_store *last;
- /*
- * 10 is the default for MaxResultSetsPerConn --
- * possibly need to parameterize it.
- */
- last = DLIST_TAIL(priv->store);
- TALLOC_FREE(last);
- }
-
- return newr;
-}
-
-struct paged_context {
- struct ldb_module *module;
- struct ldb_request *req;
-
- struct results_store *store;
- int size;
- struct ldb_control **controls;
-};
-
-static int paged_results(struct paged_context *ac)
-{
- struct ldb_paged_control *paged;
- struct message_store *msg;
- unsigned int i, num_ctrls;
- int ret;
-
- if (ac->store == NULL) {
- return LDB_ERR_OPERATIONS_ERROR;
- }
-
- while (ac->store->num_entries > 0 && ac->size > 0) {
- msg = ac->store->first;
- ret = ldb_module_send_entry(ac->req, msg->r->message,
msg->r->controls);
- if (ret != LDB_SUCCESS) {
- return ret;
- }
-
- ac->store->first = msg->next;
- talloc_free(msg);
- ac->store->num_entries--;
- ac->size--;
- }
-
- while (ac->store->first_ref != NULL) {
- msg = ac->store->first_ref;
- ret = ldb_module_send_referral(ac->req, msg->r->referral);
- if (ret != LDB_SUCCESS) {
- return ret;
- }
-
- ac->store->first_ref = msg->next;
- talloc_free(msg);
- }
-
- /* return result done */
- num_ctrls = 1;
- i = 0;
-
- if (ac->store->controls != NULL) {
- while (ac->store->controls[i]) i++; /* counting */
-
- num_ctrls += i;
- }
-
- ac->controls = talloc_array(ac, struct ldb_control *, num_ctrls +1);
- if (ac->controls == NULL) {
- return LDB_ERR_OPERATIONS_ERROR;
- }
- ac->controls[num_ctrls] = NULL;
-
- for (i = 0; i < (num_ctrls -1); i++) {
- ac->controls[i] = talloc_reference(ac->controls,
ac->store->controls[i]);
- }
-
- ac->controls[i] = talloc(ac->controls, struct ldb_control);
- if (ac->controls[i] == NULL) {
- return LDB_ERR_OPERATIONS_ERROR;
- }
-
- ac->controls[i]->oid = talloc_strdup(ac->controls[i],
- LDB_CONTROL_PAGED_RESULTS_OID);
- if (ac->controls[i]->oid == NULL) {
- return LDB_ERR_OPERATIONS_ERROR;
- }
-
- ac->controls[i]->critical = 0;
-
- paged = talloc(ac->controls[i], struct ldb_paged_control);
- if (paged == NULL) {
- return LDB_ERR_OPERATIONS_ERROR;
- }
-
- ac->controls[i]->data = paged;
-
- if (ac->size > 0) {
- paged->size = 0;
- paged->cookie = NULL;
- paged->cookie_len = 0;
- } else {
- paged->size = ac->store->num_entries;
- paged->cookie = talloc_strdup(paged, ac->store->cookie);
- paged->cookie_len = strlen(paged->cookie) + 1;
- }
-
- return LDB_SUCCESS;
-}
-
-static int paged_search_callback(struct ldb_request *req, struct ldb_reply
*ares)
-{
- struct paged_context *ac ;
- struct message_store *msg_store;
- int ret;
-
- ac = talloc_get_type(req->context, struct paged_context);
-
- if (!ares) {
- return ldb_module_done(ac->req, NULL, NULL,
- LDB_ERR_OPERATIONS_ERROR);
- }
- if (ares->error != LDB_SUCCESS) {
- return ldb_module_done(ac->req, ares->controls,
- ares->response, ares->error);
- }
-
- switch (ares->type) {
- case LDB_REPLY_ENTRY:
- msg_store = talloc(ac->store, struct message_store);
- if (msg_store == NULL) {
- return ldb_module_done(ac->req, NULL, NULL,
- LDB_ERR_OPERATIONS_ERROR);
- }
- msg_store->next = NULL;
- msg_store->r = talloc_steal(msg_store, ares);
-
- if (ac->store->first == NULL) {
- ac->store->first = msg_store;
- } else {
- ac->store->last->next = msg_store;
- }
- ac->store->last = msg_store;
-
- ac->store->num_entries++;
-
- break;
-
- case LDB_REPLY_REFERRAL:
- msg_store = talloc(ac->store, struct message_store);
- if (msg_store == NULL) {
- return ldb_module_done(ac->req, NULL, NULL,
- LDB_ERR_OPERATIONS_ERROR);
- }
- msg_store->next = NULL;
- msg_store->r = talloc_steal(msg_store, ares);
-
- if (ac->store->first_ref == NULL) {
- ac->store->first_ref = msg_store;
- } else {
- ac->store->last_ref->next = msg_store;
- }
- ac->store->last_ref = msg_store;
-
- break;
-
- case LDB_REPLY_DONE:
- ac->store->controls = talloc_move(ac->store, &ares->controls);
- ret = paged_results(ac);
- return ldb_module_done(ac->req, ac->controls,
- ares->response, ret);
- }
-
- return LDB_SUCCESS;
-}
-
-static int paged_search(struct ldb_module *module, struct ldb_request *req)
-{
- struct ldb_context *ldb;
- struct ldb_control *control;
- struct private_data *private_data;
- struct ldb_paged_control *paged_ctrl;
- struct ldb_control **saved_controls;
- struct ldb_request *search_req;
- struct paged_context *ac;
- int ret;
-
- ldb = ldb_module_get_ctx(module);
-
- /* check if there's a paged request control */
- control = ldb_request_get_control(req, LDB_CONTROL_PAGED_RESULTS_OID);
- if (control == NULL) {
- /* not found go on */
- return ldb_next_request(module, req);
- }
-
- paged_ctrl = talloc_get_type(control->data, struct ldb_paged_control);
- if (!paged_ctrl) {
- return LDB_ERR_PROTOCOL_ERROR;
- }
-
- private_data = talloc_get_type(ldb_module_get_private(module),
- struct private_data);
-
- ac = talloc_zero(req, struct paged_context);
- if (ac == NULL) {
- ldb_set_errstring(ldb, "Out of Memory");
- return LDB_ERR_OPERATIONS_ERROR;
- }
-
- ac->module = module;
- ac->req = req;
- ac->size = paged_ctrl->size;
- if (ac->size < 0) {
- /* apparently some clients send more than 2^31. This
- violates the ldap standard, but we need to cope */
- ac->size = 0x7FFFFFFF;
- }
-
- /* check if it is a continuation search the store */
- if (paged_ctrl->cookie_len == 0) {
- if (paged_ctrl->size == 0) {
- return LDB_ERR_OPERATIONS_ERROR;
- }
-
- ac->store = new_store(private_data);
- if (ac->store == NULL) {
- return LDB_ERR_OPERATIONS_ERROR;
- }
-
- ret = ldb_build_search_req_ex(&search_req, ldb, ac,
- req->op.search.base,
- req->op.search.scope,
- req->op.search.tree,
- req->op.search.attrs,
- req->controls,
- ac,
- paged_search_callback,
- req);
- if (ret != LDB_SUCCESS) {
- return ret;
- }
-
- /* save it locally and remove it from the list */
- /* we do not need to replace them later as we
- * are keeping the original req intact */
- if (!ldb_save_controls(control, search_req, &saved_controls)) {
- return LDB_ERR_OPERATIONS_ERROR;
- }
-
- return ldb_next_request(module, search_req);
-
- } else {
- struct results_store *current = NULL;
-
- /* TODO: age out old outstanding requests */
- for (current = private_data->store; current; current =
current->next) {
- if (strcmp(current->cookie, paged_ctrl->cookie) == 0) {
- current->timestamp = time(NULL);
- break;
- }
- }
- if (current == NULL) {
- return LDB_ERR_UNWILLING_TO_PERFORM;
- }
-
- DLIST_PROMOTE(private_data->store, current);
-
- ac->store = current;
-
- /* check if it is an abandon */
- if (ac->size == 0) {
- return ldb_module_done(req, NULL, NULL,
- LDB_SUCCESS);
- }
-
- ret = paged_results(ac);
- if (ret != LDB_SUCCESS) {
- return ldb_module_done(req, NULL, NULL, ret);
- }
- return ldb_module_done(req, ac->controls, NULL,
- LDB_SUCCESS);
- }
-}
-
-static int paged_request_init(struct ldb_module *module)
-{
- struct ldb_context *ldb;
- struct private_data *data;
- int ret;
-
- ldb = ldb_module_get_ctx(module);
-
- data = talloc(module, struct private_data);
- if (data == NULL) {
- return LDB_ERR_OTHER;
- }
-
- data->next_free_id = 1;
- data->num_stores = 0;
- data->store = NULL;
- ldb_module_set_private(module, data);
-
- ret = ldb_mod_register_control(module, LDB_CONTROL_PAGED_RESULTS_OID);
- if (ret != LDB_SUCCESS) {
- ldb_debug(ldb, LDB_DEBUG_WARNING,
- "paged_results:"
- "Unable to register control with rootdse!");
- }
-
- return ldb_next_init(module);
-}
-
-static const struct ldb_module_ops ldb_paged_results_module_ops = {
- .name = "paged_results",
- .search = paged_search,
- .init_context = paged_request_init
-};
-
-int ldb_paged_results_init(const char *version)
-{
- LDB_MODULE_CHECK_VERSION(version);
- return ldb_register_module(&ldb_paged_results_module_ops);
-}
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 037245f0877..0389451371a 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -327,15 +327,6 @@ def build(bld):
public_headers_install=not private_library)
t.env.LDB_VERSION = VERSION
-
- bld.SAMBA_MODULE('ldb_paged_results',
- 'modules/paged_results.c',
- init_function='ldb_paged_results_init',
- module_init_name='ldb_init_module',
- internal_module=False,
- deps='ldb',
- subsystem='ldb')
-
bld.SAMBA_MODULE('ldb_asq',
'modules/asq.c',
init_function='ldb_asq_init',
diff --git a/python/samba/tests/samba3sam.py b/python/samba/tests/samba3sam.py
index 0ed268a6032..591cfebbdde 100644
--- a/python/samba/tests/samba3sam.py
+++ b/python/samba/tests/samba3sam.py
@@ -55,7 +55,7 @@ class MapBaseTestCase(TestCaseInTempDir):
"@TO": "sambaDomainName=TESTS," + s3.basedn})
ldb.add({"dn": "@MODULES",
- "@LIST":
"rootdse,paged_results,server_sort,asq,samldb,password_hash,operational,objectguid,rdn_name,samba3sam,samba3sid,show_deleted_ignore,dsdb_flags_ignore,partition"})
+ "@LIST":
"rootdse,dsdb_paged_results,server_sort,asq,samldb,password_hash,operational,objectguid,rdn_name,samba3sam,samba3sid,show_deleted_ignore,dsdb_flags_ignore,partition"})
ldb.add({"dn": "@PARTITION",
"partition": ["%s" % (s4.basedn_casefold),
diff --git a/selftest/knownfail.d/vlv b/selftest/knownfail.d/vlv
new file mode 100644
index 00000000000..f187a2ed55e
--- /dev/null
+++ b/selftest/knownfail.d/vlv
@@ -0,0 +1,2 @@
+samba4.ldap.vlv.python.*__main__.VLVTests.test_vlv_change_search_expr
+samba4.ldap.vlv.python.*__main__.PagedResultsTests.test_paged_cant_change_controls_data
diff --git a/source4/dsdb/samdb/ldb_modules/paged_results.c
b/source4/dsdb/samdb/ldb_modules/paged_results.c
new file mode 100644
index 00000000000..78ad44f6601
--- /dev/null
+++ b/source4/dsdb/samdb/ldb_modules/paged_results.c
@@ -0,0 +1,795 @@
+/*
+ ldb database library
--
Samba Shared Repository
