The branch, master has been updated
via fdb16dd lib: talloc: tests - add test_pthread_talloc_passing()
testing talloc in a pthread environment.
via 61e0a67 lib: docs: talloc: Add a threads tutorial and samples
showing how to use talloc with threads.
via 9139caa dsdb: fix error message in tombstone_reanimation test.
via 06a410d dsdb: fix error message in sam test
from 74e52a4 lib: Remove ccan
https://git.samba.org/?p=samba.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit fdb16dd247797eabfdda49d6b6fb40a4642c0664
Author: Jeremy Allison <[email protected]>
Date: Mon Mar 16 12:18:17 2015 -0700
lib: talloc: tests - add test_pthread_talloc_passing() testing talloc in a
pthread environment.
Signed-off-by: Jeremy Allison <[email protected]>
Reviewed-by: Simo <[email protected]>
Reviewed-by: "Stefan (metze) Metzmacher" <[email protected]>
Autobuild-User(master): Jeremy Allison <[email protected]>
Autobuild-Date(master): Tue Mar 17 19:23:29 CET 2015 on sn-devel-104
commit 61e0a67c552d28c7544399fe07f381eab673e726
Author: Jeremy Allison <[email protected]>
Date: Mon Mar 16 12:17:40 2015 -0700
lib: docs: talloc: Add a threads tutorial and samples showing how to use
talloc with threads.
Signed-off-by: Jeremy Allison <[email protected]>
Reviewed-by: Simo <[email protected]>
commit 9139caa57a4247ecaa71d6f86db2f79ec20f8140
Author: Michael Adam <[email protected]>
Date: Mon Mar 16 18:45:52 2015 +0100
dsdb: fix error message in tombstone_reanimation test.
Signed-off-by: Michael Adam <[email protected]>
Reviewed-by: Jeremy Allison <[email protected]>
commit 06a410dfb137bf9fc775ba8ffb3e42a506b495a9
Author: Michael Adam <[email protected]>
Date: Mon Mar 16 18:40:53 2015 +0100
dsdb: fix error message in sam test
Signed-off-by: Michael Adam <[email protected]>
Reviewed-by: Jeremy Allison <[email protected]>
-----------------------------------------------------------------------
Summary of changes:
lib/talloc/doc/mainpage.dox | 3 +-
lib/talloc/doc/tutorial_introduction.dox | 4 +-
lib/talloc/doc/tutorial_threads.dox | 203 +++++++++++++++++++++
lib/talloc/talloc_guide.txt | 3 +-
lib/talloc/testsuite.c | 153 ++++++++++++++++
lib/talloc/wscript | 6 +-
source4/dsdb/tests/python/sam.py | 2 +-
source4/dsdb/tests/python/tombstone_reanimation.py | 6 +-
8 files changed, 372 insertions(+), 8 deletions(-)
create mode 100644 lib/talloc/doc/tutorial_threads.dox
Changeset truncated at 500 lines:
diff --git a/lib/talloc/doc/mainpage.dox b/lib/talloc/doc/mainpage.dox
index 3b56898..ece6ccb 100644
--- a/lib/talloc/doc/mainpage.dox
+++ b/lib/talloc/doc/mainpage.dox
@@ -102,7 +102,8 @@
* - when using talloc_enable_leak_report(), giving directly NULL as a parent
* context implicitly refers to a hidden "null context" global variable, so
* this should not be used in a multi-threaded environment without proper
- * synchronization.
+ * synchronization. In threaded code turn off null tracking using
+ * talloc_disable_null_tracking().
* - the context returned by talloc_autofree_context() is also global so
* shouldn't be used by several threads simultaneously without
* synchronization.
diff --git a/lib/talloc/doc/tutorial_introduction.dox
b/lib/talloc/doc/tutorial_introduction.dox
index 02777b9..418c38b 100644
--- a/lib/talloc/doc/tutorial_introduction.dox
+++ b/lib/talloc/doc/tutorial_introduction.dox
@@ -40,4 +40,6 @@ recursively frees all of its descendants as well.
@subpage libtalloc_bestpractices
-*/
\ No newline at end of file
+@subpage libtalloc_threads
+
+*/
diff --git a/lib/talloc/doc/tutorial_threads.dox
b/lib/talloc/doc/tutorial_threads.dox
new file mode 100644
index 0000000..111bbf5
--- /dev/null
+++ b/lib/talloc/doc/tutorial_threads.dox
@@ -0,0 +1,203 @@
+/**
+@page libtalloc_threads Chapter 8: Using threads with talloc
+
+@section Talloc and thread safety
+
+The talloc library is not internally thread-safe, in that accesses
+to variables on a talloc context are not controlled by mutexes or
+other thread-safe primitives.
+
+However, so long as talloc_disable_null_tracking() is called from
+the main thread to disable global variable access within talloc,
+then each thread can safely use its own top level talloc context
+allocated off the NULL context.
+
+For example:
+
+@code
+static void *thread_fn(void *arg)
+{
+ const char *ctx_name = (const char *)arg;
+ /*
+ * Create a new top level talloc hierarchy in
+ * this thread.
+ */
+ void *top_ctx = talloc_named_const(NULL, 0, "top");
+ if (top_ctx == NULL) {
+ return NULL;
+ }
+ sub_ctx = talloc_named_const(top_ctx, 100, ctx_name);
+ if (sub_ctx == NULL) {
+ return NULL;
+ }
+
+ /*
+ * Do more processing/talloc calls on top_ctx
+ * and its children.
+ */
+ ......
+
+ talloc_free(top_ctx);
+ return value;
+}
+@endcode
+
+is a perfectly safe use of talloc within a thread.
+
+The problem comes when one thread wishes to move some
+memory allocated on its local top level talloc context
+to another thread. Care must be taken to add data access
+exclusion to prevent memory corruption. One method would
+be to lock a mutex before any talloc call on each thread,
+but this would push the burden of total talloc thread-safety
+on the poor user of the library.
+
+A much easier way to transfer talloced memory between
+threads is by the use of an intermediate, mutex locked,
+intermediate variable.
+
+An example of this is below - taken from test code inside
+the talloc testsuite.
+
+The main thread creates 1000 sub-threads, and then accepts
+the transfer of some thread-talloc'ed memory onto its top
+level context from each thread in turn.
+
+A pthread mutex and condition variable are used to
+synchronize the transfer via the intermediate_ptr
+variable.
+
+@code
+/* Required sync variables. */
+static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t condvar = PTHREAD_COND_INITIALIZER;
+
+/* Intermediate talloc pointer for transfer. */
+static void *intermediate_ptr;
+
+/* Subthread. */
+static void *thread_fn(void *arg)
+{
+ int ret;
+ const char *ctx_name = (const char *)arg;
+ void *sub_ctx = NULL;
+ /*
+ * Do stuff that creates a new talloc hierarchy in
+ * this thread.
+ */
+ void *top_ctx = talloc_named_const(NULL, 0, "top");
+ if (top_ctx == NULL) {
+ return NULL;
+ }
+ sub_ctx = talloc_named_const(top_ctx, 100, ctx_name);
+ if (sub_ctx == NULL) {
+ return NULL;
+ }
+
+ /*
+ * Now transfer a pointer from our hierarchy
+ * onto the intermediate ptr.
+ */
+ ret = pthread_mutex_lock(&mtx);
+ if (ret != 0) {
+ talloc_free(top_ctx);
+ return NULL;
+ }
+
+ /* Wait for intermediate_ptr to be free. */
+ while (intermediate_ptr != NULL) {
+ ret = pthread_cond_wait(&condvar, &mtx);
+ if (ret != 0) {
+ talloc_free(top_ctx);
+ return NULL;
+ }
+ }
+
+ /* and move our memory onto it from our toplevel hierarchy. */
+ intermediate_ptr = talloc_move(NULL, &sub_ctx);
+
+ /* Tell the main thread it's ready for pickup. */
+ pthread_cond_broadcast(&condvar);
+ pthread_mutex_unlock(&mtx);
+
+ talloc_free(top_ctx);
+ return NULL;
+}
+
+/* Main thread. */
+
+#define NUM_THREADS 1000
+
+static bool test_pthread_talloc_passing(void)
+{
+ int i;
+ int ret;
+ char str_array[NUM_THREADS][20];
+ pthread_t thread_id;
+ void *mem_ctx;
+
+ /*
+ * Important ! Null tracking breaks threaded talloc.
+ * It *must* be turned off.
+ */
+ talloc_disable_null_tracking();
+
+ /* Main thread toplevel context. */
+ mem_ctx = talloc_named_const(NULL, 0, "toplevel");
+ if (mem_ctx == NULL) {
+ return false;
+ }
+
+ /*
+ * Spin off NUM_THREADS threads.
+ * They will use their own toplevel contexts.
+ */
+ for (i = 0; i < NUM_THREADS; i++) {
+ (void)snprintf(str_array[i],
+ 20,
+ "thread:%d",
+ i);
+ if (str_array[i] == NULL) {
+ return false;
+ }
+ ret = pthread_create(&thread_id,
+ NULL,
+ thread_fn,
+ str_array[i]);
+ if (ret != 0) {
+ return false;
+ }
+ }
+
+ /* Now wait for NUM_THREADS transfers of the talloc'ed memory. */
+ for (i = 0; i < NUM_THREADS; i++) {
+ ret = pthread_mutex_lock(&mtx);
+ if (ret != 0) {
+ talloc_free(mem_ctx);
+ return false;
+ }
+
+ /* Wait for intermediate_ptr to have our data. */
+ while (intermediate_ptr == NULL) {
+ ret = pthread_cond_wait(&condvar, &mtx);
+ if (ret != 0) {
+ talloc_free(mem_ctx);
+ return false;
+ }
+ }
+
+ /* and move it onto our toplevel hierarchy. */
+ (void)talloc_move(mem_ctx, &intermediate_ptr);
+
+ /* Tell the sub-threads we're ready for another. */
+ pthread_cond_broadcast(&condvar);
+ pthread_mutex_unlock(&mtx);
+ }
+
+ /* Dump the hierarchy. */
+ talloc_report(mem_ctx, stdout);
+ talloc_free(mem_ctx);
+ return true;
+}
+@endcode
+*/
diff --git a/lib/talloc/talloc_guide.txt b/lib/talloc/talloc_guide.txt
index 16afc9b..95f7f29 100644
--- a/lib/talloc/talloc_guide.txt
+++ b/lib/talloc/talloc_guide.txt
@@ -69,7 +69,8 @@ order to be safe. In particular:
- when using talloc_enable_leak_report(), giving directly NULL as a
parent context implicitly refers to a hidden "null context" global
variable, so this should not be used in a multi-threaded environment
-without proper synchronization ;
+without proper synchronization. In threaded code turn off null tracking using
+talloc_disable_null_tracking(). ;
- the context returned by talloc_autofree_context() is also global so
shouldn't be used by several threads simultaneously without
synchronization.
diff --git a/lib/talloc/testsuite.c b/lib/talloc/testsuite.c
index eb3e13d..6d0fe94 100644
--- a/lib/talloc/testsuite.c
+++ b/lib/talloc/testsuite.c
@@ -27,6 +27,10 @@
#include "system/time.h"
#include <talloc.h>
+#ifdef HAVE_PTHREAD
+#include <pthread.h>
+#endif
+
#include "talloc_testsuite.h"
static struct timeval timeval_current(void)
@@ -1701,6 +1705,151 @@ static bool test_memlimit(void)
return true;
}
+#ifdef HAVE_PTHREAD
+
+#define NUM_THREADS 100
+
+/* Sync variables. */
+static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t condvar = PTHREAD_COND_INITIALIZER;
+static void *intermediate_ptr;
+
+/* Subthread. */
+static void *thread_fn(void *arg)
+{
+ int ret;
+ const char *ctx_name = (const char *)arg;
+ void *sub_ctx = NULL;
+ /*
+ * Do stuff that creates a new talloc hierarchy in
+ * this thread.
+ */
+ void *top_ctx = talloc_named_const(NULL, 0, "top");
+ if (top_ctx == NULL) {
+ return NULL;
+ }
+ sub_ctx = talloc_named_const(top_ctx, 100, ctx_name);
+ if (sub_ctx == NULL) {
+ return NULL;
+ }
+
+ /*
+ * Now transfer a pointer from our hierarchy
+ * onto the intermediate ptr.
+ */
+ ret = pthread_mutex_lock(&mtx);
+ if (ret != 0) {
+ talloc_free(top_ctx);
+ return NULL;
+ }
+ /* Wait for intermediate_ptr to be free. */
+ while (intermediate_ptr != NULL) {
+ ret = pthread_cond_wait(&condvar, &mtx);
+ if (ret != 0) {
+ talloc_free(top_ctx);
+ return NULL;
+ }
+ }
+
+ /* and move our memory onto it from our toplevel hierarchy. */
+ intermediate_ptr = talloc_move(NULL, &sub_ctx);
+
+ /* Tell the main thread it's ready for pickup. */
+ pthread_cond_broadcast(&condvar);
+ pthread_mutex_unlock(&mtx);
+
+ talloc_free(top_ctx);
+ return NULL;
+}
+
+/* Main thread. */
+static bool test_pthread_talloc_passing(void)
+{
+ int i;
+ int ret;
+ char str_array[NUM_THREADS][20];
+ pthread_t thread_id;
+ void *mem_ctx;
+
+ /*
+ * Important ! Null tracking breaks threaded talloc.
+ * It *must* be turned off.
+ */
+ talloc_disable_null_tracking();
+
+ printf("test: pthread_talloc_passing\n# PTHREAD TALLOC PASSING\n");
+
+ /* Main thread toplevel context. */
+ mem_ctx = talloc_named_const(NULL, 0, "toplevel");
+ if (mem_ctx == NULL) {
+ printf("failed to create toplevel context\n");
+ return false;
+ }
+
+ /*
+ * Spin off NUM_THREADS threads.
+ * They will use their own toplevel contexts.
+ */
+ for (i = 0; i < NUM_THREADS; i++) {
+ (void)snprintf(str_array[i],
+ 20,
+ "thread:%d",
+ i);
+ if (str_array[i] == NULL) {
+ printf("snprintf %d failed\n", i);
+ return false;
+ }
+ ret = pthread_create(&thread_id,
+ NULL,
+ thread_fn,
+ str_array[i]);
+ if (ret != 0) {
+ printf("failed to create thread %d (%d)\n", i, ret);
+ return false;
+ }
+ }
+
+ printf("Created %d threads\n", NUM_THREADS);
+
+ /* Now wait for NUM_THREADS transfers of the talloc'ed memory. */
+ for (i = 0; i < NUM_THREADS; i++) {
+ ret = pthread_mutex_lock(&mtx);
+ if (ret != 0) {
+ printf("pthread_mutex_lock %d failed (%d)\n", i, ret);
+ talloc_free(mem_ctx);
+ return false;
+ }
+
+ /* Wait for intermediate_ptr to have our data. */
+ while (intermediate_ptr == NULL) {
+ ret = pthread_cond_wait(&condvar, &mtx);
+ if (ret != 0) {
+ printf("pthread_cond_wait %d failed (%d)\n", i,
+ ret);
+ talloc_free(mem_ctx);
+ return false;
+ }
+ }
+
+ /* and move it onto our toplevel hierarchy. */
+ (void)talloc_move(mem_ctx, &intermediate_ptr);
+
+ /* Tell the sub-threads we're ready for another. */
+ pthread_cond_broadcast(&condvar);
+ pthread_mutex_unlock(&mtx);
+ }
+
+ CHECK_SIZE("pthread_talloc_passing", mem_ctx, NUM_THREADS * 100);
+#if 1
+ /* Dump the hierarchy. */
+ talloc_report(mem_ctx, stdout);
+#endif
+ talloc_free(mem_ctx);
+ printf("success: pthread_talloc_passing\n");
+ return true;
+}
+#endif
+
static void test_reset(void)
{
talloc_set_log_fn(test_log_stdout);
@@ -1771,6 +1920,10 @@ bool torture_local_talloc(struct torture_context *tctx)
ret &= test_free_children();
test_reset();
ret &= test_memlimit();
+#ifdef HAVE_PTHREAD
+ test_reset();
+ ret &= test_pthread_talloc_passing();
+#endif
if (ret) {
diff --git a/lib/talloc/wscript b/lib/talloc/wscript
index 97c52c3..9efc895 100644
--- a/lib/talloc/wscript
+++ b/lib/talloc/wscript
@@ -89,9 +89,13 @@ def build(bld):
public_headers=[],
enabled=bld.env.TALLOC_COMPAT1)
+ testsuite_deps = 'talloc'
+ if bld.CONFIG_SET('HAVE_PTHREAD'):
+ testsuite_deps += ' pthread'
+
bld.SAMBA_BINARY('talloc_testsuite',
'testsuite_main.c testsuite.c',
- deps='talloc',
+ testsuite_deps,
install=False)
else:
diff --git a/source4/dsdb/tests/python/sam.py b/source4/dsdb/tests/python/sam.py
index 620c96c..d5c27da 100755
--- a/source4/dsdb/tests/python/sam.py
+++ b/source4/dsdb/tests/python/sam.py
@@ -2906,7 +2906,7 @@ class SamTests(samba.tests.TestCase):
actual_names = set(user_obj.keys())
# Samba does not use 'dSCorePropagationData', so skip it
actual_names -= set(['dSCorePropagationData'])
- self.assertEqual(set(expected_attrs.keys()), actual_names, "Actual
object does not has expected attributes")
+ self.assertEqual(set(expected_attrs.keys()), actual_names, "Actual
object does not have expected attributes")
# check attribute values
for name in expected_attrs.keys():
actual_val = user_obj.get(name)
diff --git a/source4/dsdb/tests/python/tombstone_reanimation.py
b/source4/dsdb/tests/python/tombstone_reanimation.py
index 6407279..ad780fd 100644
--- a/source4/dsdb/tests/python/tombstone_reanimation.py
+++ b/source4/dsdb/tests/python/tombstone_reanimation.py
@@ -83,7 +83,7 @@ class
RestoredObjectAttributesBaseTestCase(samba.tests.TestCase):
return self.search_dn(msg['dn'])
def assertAttributesEqual(self, obj_orig, attrs_orig, obj_restored,
attrs_rest):
- self.assertEqual(attrs_orig, attrs_rest, "Actual object does not has
expected attributes")
+ self.assertEqual(attrs_orig, attrs_rest, "Actual object does not have
expected attributes")
# remove volatile attributes, they can't be equal
attrs_orig -= set(["uSNChanged", "dSCorePropagationData",
"whenChanged"])
for attr in attrs_orig:
@@ -115,7 +115,7 @@ class
RestoredObjectAttributesBaseTestCase(samba.tests.TestCase):
actual_names = set(obj_msg.keys())
# Samba does not use 'dSCorePropagationData', so skip it
actual_names -= set(['dSCorePropagationData'])
- self.assertEqual(set(attr_expected.keys()), actual_names, "Actual
object does not has expected attributes")
+ self.assertEqual(set(attr_expected.keys()), actual_names, "Actual
object does not have expected attributes")
for name in attr_expected.keys():
expected_val = attr_expected[name]
actual_val = obj_msg.get(name)
@@ -359,7 +359,7 @@ class
RestoreUserObjectTestCase(RestoredObjectAttributesBaseTestCase):
# windows restore more attributes that originally we have
orig_attrs.update(['adminCount', 'operatorCount', 'lastKnownParent'])
rest_attrs = set(obj_restore.keys())
- self.assertEqual(orig_attrs, rest_attrs, "Actual object does not has
expected attributes")
+ self.assertEqual(orig_attrs, rest_attrs, "Actual object does not have
expected attributes")
self.assertAttributesExists(self._expected_user_attributes(username,
usr_dn, "Person"), obj_restore)
--
Samba Shared Repository
