The branch, master has been updated via 04126d5 lib/util: Fix format strings and argument data types via b077969 torture: Fix format-nonliteral warning via c1a791e passdb: Fix format-nonliteral warning via 908c068 wibindd: Fix format-nonliteral warning via 5855b03 regedit: Fix format-nonliteral warning via 44baf81 s3-libnet: Fix format-nonliteral warning via 2d47c2b s3-netapi: Fix format-nonliteral warning via 4b4d364 s3-include: Fix format-nonliteral warning via e9b8751 s3-lib: Fix format-nonliteral warning via a47e953 ldb: Fix format-nonliteral warning via 258cf9a lib/util: Fix format-nonliteral warning via b891fea tdb: Fix format-nonliteral warning via 5fb54d4 talloc: Fix format-nonliteral warning via d123085 s3-lib: Remove unused function sprintf_append via a343fbc lib: Remove unused source3/lib/asys via 353cf16 vfs: Remove link to asys_ via 2d484dc vfs: Convert vfs_fsync_send to pthreadpool_tevent via c780adb vfs: Convert vfs_write_send to pthreadpool_tevent via 995d0a2 vfs: Convert vfs_pread_send to pthreadpool_tevent via afb86d3 smbd: Add pthreadpool_tevent to smbd_server_connection via 2e0990f smbtorture3: Add LOCAL-PTHREADPOOL-TEVENT via bbcf220 lib: Add pthreadpool_tevent via d7e5128 lib: add job data to to callback via 5593467 lib: Move pipe signalling to pthreadpool_pipe.c via e47bf91 lib: Use pthreadpool_pipe instead of pthreadpool via fccb5fb lib: Add pthreadpool_pipe via 6935fdd tevent: version 0.9.30 via c7e2cb8 tevent: Use eventfd for signal/thread wakeup via f40c171 tevent: Move a variable declaration into a while block via 2d55883 tevent: Move rundown of the event pipe via 9d2ae47 tevent: Simple test for threaded immediates via 80cd6b7 tevent: reorder tevent_context for cache locality via 110f925 lib: enable threaded immediates in source3 via f6aaece tevent: Add threaded immediate activation via 8a9b8ac tevent: Move the async wakeup pipe to common via c4ef0c8 tevent: Fix a typo via 4771d07 libreplace: Ask for eventfd(2) from 17dfd8b ctdb-tests: Reimplement daemons_stop() using ctdbd_wrapper
https://git.samba.org/?p=samba.git;a=shortlog;h=master - Log ----------------------------------------------------------------- commit 04126d54d5430c5e2fdbbfea58d76b253b81a407 Author: Amitay Isaacs <ami...@gmail.com> Date: Mon Aug 22 02:53:00 2016 +1000 lib/util: Fix format strings and argument data types BUG: https://bugzilla.samba.org/show_bug.cgi?id=12168 Signed-off-by: Amitay Isaacs <ami...@gmail.com> Reviewed-by: Jeremy Allison <j...@samba.org> Autobuild-User(master): Jeremy Allison <j...@samba.org> Autobuild-Date(master): Wed Aug 24 05:32:15 CEST 2016 on sn-devel-144 commit b077969a72a8010b7fac37102eb1d0c1a2111fc4 Author: Amitay Isaacs <ami...@gmail.com> Date: Mon Aug 8 17:59:08 2016 +1000 torture: Fix format-nonliteral warning BUG: https://bugzilla.samba.org/show_bug.cgi?id=12168 Signed-off-by: Amitay Isaacs <ami...@gmail.com> Reviewed-by: Andreas Schneider <a...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit c1a791e17c6e94c7d33351c420f238b06461a044 Author: Amitay Isaacs <ami...@gmail.com> Date: Mon Aug 8 17:40:51 2016 +1000 passdb: Fix format-nonliteral warning BUG: https://bugzilla.samba.org/show_bug.cgi?id=12168 Signed-off-by: Amitay Isaacs <ami...@gmail.com> Reviewed-by: Andreas Schneider <a...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit 908c068979dc69ae4664a9fa31e2f91ed115b84a Author: Amitay Isaacs <ami...@gmail.com> Date: Mon Aug 8 14:22:30 2016 +1000 wibindd: Fix format-nonliteral warning BUG: https://bugzilla.samba.org/show_bug.cgi?id=12168 Signed-off-by: Amitay Isaacs <ami...@gmail.com> Reviewed-by: Andreas Schneider <a...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit 5855b039ddedd39a81eaf419d771b32e02668cd3 Author: Amitay Isaacs <ami...@gmail.com> Date: Mon Aug 8 14:21:14 2016 +1000 regedit: Fix format-nonliteral warning BUG: https://bugzilla.samba.org/show_bug.cgi?id=12168 Signed-off-by: Amitay Isaacs <ami...@gmail.com> Reviewed-by: Andreas Schneider <a...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit 44baf81e8342349a9131acc5d784e65247ac32a6 Author: Amitay Isaacs <ami...@gmail.com> Date: Mon Aug 8 14:18:31 2016 +1000 s3-libnet: Fix format-nonliteral warning BUG: https://bugzilla.samba.org/show_bug.cgi?id=12168 Signed-off-by: Amitay Isaacs <ami...@gmail.com> Reviewed-by: Andreas Schneider <a...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit 2d47c2b7a15ce228a201dfda63455acbff981680 Author: Amitay Isaacs <ami...@gmail.com> Date: Mon Aug 8 14:17:18 2016 +1000 s3-netapi: Fix format-nonliteral warning BUG: https://bugzilla.samba.org/show_bug.cgi?id=12168 Signed-off-by: Amitay Isaacs <ami...@gmail.com> Reviewed-by: Andreas Schneider <a...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit 4b4d3643555a68f585e38e37604ed4ecae9137f5 Author: Amitay Isaacs <ami...@gmail.com> Date: Mon Aug 8 14:16:29 2016 +1000 s3-include: Fix format-nonliteral warning BUG: https://bugzilla.samba.org/show_bug.cgi?id=12168 Signed-off-by: Amitay Isaacs <ami...@gmail.com> Reviewed-by: Andreas Schneider <a...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit e9b8751b587c1eabc4c1a7a7d75e3c4498c6ffc4 Author: Amitay Isaacs <ami...@gmail.com> Date: Sun Aug 7 18:14:31 2016 +1000 s3-lib: Fix format-nonliteral warning BUG: https://bugzilla.samba.org/show_bug.cgi?id=12168 Signed-off-by: Amitay Isaacs <ami...@gmail.com> Reviewed-by: Andreas Schneider <a...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit a47e95337e96f3e62cde41680d94268f7ce58c6f Author: Amitay Isaacs <ami...@gmail.com> Date: Wed Aug 3 00:44:24 2016 +1000 ldb: Fix format-nonliteral warning BUG: https://bugzilla.samba.org/show_bug.cgi?id=12168 Signed-off-by: Amitay Isaacs <ami...@gmail.com> Reviewed-by: Andreas Schneider <a...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit 258cf9a170f331992c916569d3b5b52b5d731226 Author: Amitay Isaacs <ami...@gmail.com> Date: Tue Aug 2 18:11:17 2016 +1000 lib/util: Fix format-nonliteral warning BUG: https://bugzilla.samba.org/show_bug.cgi?id=12168 Signed-off-by: Amitay Isaacs <ami...@gmail.com> Reviewed-by: Andreas Schneider <a...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit b891feaf2b1db064a725e205c861dc6d0c83aca0 Author: Amitay Isaacs <ami...@gmail.com> Date: Tue Aug 2 18:17:34 2016 +1000 tdb: Fix format-nonliteral warning BUG: https://bugzilla.samba.org/show_bug.cgi?id=12168 Signed-off-by: Amitay Isaacs <ami...@gmail.com> Reviewed-by: Andreas Schneider <a...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit 5fb54d4d814288b796a075e01c054f607d6ce571 Author: Amitay Isaacs <ami...@gmail.com> Date: Tue Aug 2 18:05:14 2016 +1000 talloc: Fix format-nonliteral warning BUG: https://bugzilla.samba.org/show_bug.cgi?id=12168 Signed-off-by: Amitay Isaacs <ami...@gmail.com> Reviewed-by: Andreas Schneider <a...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit d123085f695745cc58f3d6882bfc7ce0893d8b84 Author: Amitay Isaacs <ami...@gmail.com> Date: Tue Aug 23 15:05:08 2016 +1000 s3-lib: Remove unused function sprintf_append BUG: https://bugzilla.samba.org/show_bug.cgi?id=12168 Signed-off-by: Amitay Isaacs <ami...@gmail.com> Reviewed-by: Jeremy Allison <j...@samba.org> commit a343fbc6f1dc18ed2499535c76e4dd79b01d29b2 Author: Volker Lendecke <v...@samba.org> Date: Sun Aug 7 16:03:00 2016 +0200 lib: Remove unused source3/lib/asys Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit 353cf16d0127479cec04176d089785233b9f096b Author: Volker Lendecke <v...@samba.org> Date: Sun Aug 7 15:59:10 2016 +0200 vfs: Remove link to asys_ No longer needed after conversion to pthreadpool_tevent Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit 2d484dcd19a581d704ecc7ab69a0e22dd22238a8 Author: Volker Lendecke <v...@samba.org> Date: Sun Aug 7 15:53:12 2016 +0200 vfs: Convert vfs_fsync_send to pthreadpool_tevent Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit c780adb365fb44f1bd011244fcfa657acc18a5d5 Author: Volker Lendecke <v...@samba.org> Date: Sun Aug 7 15:44:52 2016 +0200 vfs: Convert vfs_write_send to pthreadpool_tevent Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit 995d0a207d891f36a5a0c761101c26e05aeb3fb0 Author: Volker Lendecke <v...@samba.org> Date: Sat Aug 6 21:58:09 2016 +0200 vfs: Convert vfs_pread_send to pthreadpool_tevent Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit afb86d3450c80c5774ae9ccefa939ed148b4b45c Author: Volker Lendecke <v...@samba.org> Date: Mon Aug 8 15:07:30 2016 +0200 smbd: Add pthreadpool_tevent to smbd_server_connection Prerequisite to convert the vfs _send/recv functions Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit 2e0990fef2b85ad14318290f03c2f8502c081ae7 Author: Volker Lendecke <v...@samba.org> Date: Mon Aug 8 15:04:39 2016 +0200 smbtorture3: Add LOCAL-PTHREADPOOL-TEVENT Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit bbcf2204b550255216f6860a57e71ee0c229b2d9 Author: Volker Lendecke <v...@samba.org> Date: Mon Aug 8 15:02:36 2016 +0200 lib: Add pthreadpool_tevent This is a replacement for fncall.[ch] without having to go through a pipe for job completion signalling Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit d7e51286e7ab08f3b5f5ba243b3b88e23f3fe360 Author: Volker Lendecke <v...@samba.org> Date: Sun Jul 31 08:57:35 2016 +0200 lib: add job data to to callback The pthreadpool_tevent wrapper will need this Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit 5593467eb1a838ac542fa0fca6a5e2b989bbde31 Author: Volker Lendecke <v...@samba.org> Date: Mon Aug 15 13:59:12 2016 +0200 lib: Move pipe signalling to pthreadpool_pipe.c Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit e47bf91fcb75841dbc05a42fff14c55f21c55777 Author: Volker Lendecke <v...@samba.org> Date: Mon Aug 15 13:57:20 2016 +0200 lib: Use pthreadpool_pipe instead of pthreadpool Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit fccb5fb20ef543c861dcc2ad42e42fdbaa21d9de Author: Volker Lendecke <v...@samba.org> Date: Sat Jul 30 10:20:08 2016 +0200 lib: Add pthreadpool_pipe First step to separate the signalling mechanism from the core pthreadpool code. A later patch will add a pthreadpool that directly indicates job completion via tevent_threaded_schedule_immediate. Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit 6935fdd07e9d876692c3de08fd6e10bf32b949cc Author: Stefan Metzmacher <me...@samba.org> Date: Wed Aug 17 10:08:57 2016 +0200 tevent: version 0.9.30 * add tevent_threaded_context_create() and tevent_threaded_schedule_immediate() They add a way to pass the thread result from a helper thread into the main event loop. Signed-off-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit c7e2cb839df13bcbd14a05a39559366813b8c566 Author: Volker Lendecke <v...@samba.org> Date: Fri Aug 12 16:32:33 2016 +0200 tevent: Use eventfd for signal/thread wakeup According to the manpage, eventfd is cheaper than a pipe. At least, we can save a file descriptor and space for it in struct tevent_context :-) Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit f40c171743d011e70e9674af312046034278b1e1 Author: Volker Lendecke <v...@samba.org> Date: Fri Aug 12 16:07:07 2016 +0200 tevent: Move a variable declaration into a while block Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit 2d55883e28e74b8e1d576e4a94a22d95a7b78618 Author: Volker Lendecke <v...@samba.org> Date: Fri Aug 12 16:00:56 2016 +0200 tevent: Move rundown of the event pipe Purely cosmetic change: This moves closing the signal/thread event pipe to where it's opened. This prepares the eventfd support, making the "magic" for eventfd more obvious. Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit 9d2ae4785d81d2679c41a99ed80032bb055bfdf4 Author: Volker Lendecke <v...@samba.org> Date: Mon Aug 8 12:53:08 2016 +0200 tevent: Simple test for threaded immediates Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit 80cd6b75288583086c7855a6c37be868d5e2cfb7 Author: Volker Lendecke <v...@samba.org> Date: Mon Aug 8 08:56:23 2016 +0200 tevent: reorder tevent_context for cache locality No functionality change. This just looks better in objdump --disassemble :-) Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit 110f9258ddf995a334280c42b98e6f15d4d947d8 Author: Volker Lendecke <v...@samba.org> Date: Mon Aug 8 12:51:56 2016 +0200 lib: enable threaded immediates in source3 Logically this belongs into the previous patch, but tevent deserves isolated patches :-) Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit f6aaece578a53a539a3fe8bf3aeb800967615e2d Author: Volker Lendecke <v...@samba.org> Date: Mon Aug 8 11:26:37 2016 +0200 tevent: Add threaded immediate activation This is infrastructure to improve our async r/w result handling and latency. The pthreadpool signalling goes through a pipe. This has downsides: The main event loop has to go through a read on the pipe before it can ship the result. Also, it is not guaranteed by poll/epoll that the pthreadpool signal pipe is handled with top priority. When an async pread/pwrite has finished, we should immediately ship the result to the client, not waiting for anything else. This patch enables tevent_immediate structs as job signalling. This means a busy main tevent loop will handle the threaded job completion before any timed or file descriptor events. Opposite to Jeremy's tevent_thread_proxy this is done by a modification of the main event loop by looking at a linked list under a central mutex. Regarding performance: In a later commit I've created a test that does nothing but fire one immediate over and over again. If you add a phread_mutex_lock and unlock pair in the immediate handler, you lose roughly 25% of rounds per second, so it is measurable. It is questionable that will be measurable in the real world, but to counter concerns activation of immediates needs to go through a new struct tevent_threaded_context. Only if such a tevent_threaded_context exists for a tevent context, the main loop takes the hit to look at the mutex'ed list of finished jobs. This patch by design does not care about talloc hierarchies. The idea is that the main thread owning the tevent context creates a chunk of memory and prepares the tevent_immediate indication job completion. The main thread hands the memory chunk together with the immediate as a job description over to a helper thread. The helper thread does its job and upon completion calls tevent_threaded_schedule_immediate with the already-prepared immediate. From that point on memory ownership is again transferred to the main thread. Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit 8a9b8ac72461657a3a5de2fd5232eaa9861bf696 Author: Volker Lendecke <v...@samba.org> Date: Fri Jul 29 08:53:59 2016 +0200 tevent: Move the async wakeup pipe to common Signalling the main event loop will also happen from threads soon, and that will use the same mechanism. This also keeps the pipe open after the last signal handler is removed. Threaded jobs will come and go very frequently, and always setting up and tearing down the pipe for each job will be expensive. Also, this is "just" two file descriptors, and with eventfd just one. Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit c4ef0c8f3ecc4266fed3af8537ba2998730b3750 Author: Volker Lendecke <v...@samba.org> Date: Mon Aug 15 10:33:09 2016 +0200 tevent: Fix a typo Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> commit 4771d0792e4343cc8631a36247a2f3c7c31d58b3 Author: Volker Lendecke <v...@samba.org> Date: Fri Aug 12 16:40:05 2016 +0200 libreplace: Ask for eventfd(2) This will be used in tevent soon Signed-off-by: Volker Lendecke <v...@samba.org> Reviewed-by: Stefan Metzmacher <me...@samba.org> Reviewed-by: Jeremy Allison <j...@samba.org> ----------------------------------------------------------------------- Summary of changes: lib/ldb/tools/ldbdump.c | 3 + lib/ldb/tools/ldbutil.h | 2 +- lib/replace/wscript | 3 + lib/talloc/talloc.c | 8 +- lib/tdb/tools/tdbdump.c | 3 + .../ABI/{tevent-0.9.26.sigs => tevent-0.9.30.sigs} | 6 + lib/tevent/testsuite.c | 99 ++++++ lib/tevent/tevent.c | 283 ++++++++++++++- lib/tevent/tevent.h | 74 ++++ lib/tevent/tevent_epoll.c | 4 + lib/tevent/tevent_internal.h | 45 ++- lib/tevent/tevent_poll.c | 9 +- lib/tevent/tevent_port.c | 4 + lib/tevent/tevent_select.c | 4 + lib/tevent/tevent_signal.c | 60 +--- lib/tevent/tevent_threads.c | 121 ++++++- lib/tevent/wscript | 8 +- lib/util/dprintf.c | 3 + lib/util/talloc_report.c | 14 +- source3/include/proto.h | 11 +- source3/include/tldap_util.h | 7 +- source3/lib/asys/asys.c | 342 ------------------- source3/lib/asys/asys.h | 155 --------- source3/lib/asys/tests.c | 90 ----- source3/lib/asys/wscript_build | 10 - source3/lib/cbuf.h | 2 +- source3/lib/events.c | 4 + source3/lib/fncall.c | 16 +- source3/lib/netapi/netapi_private.h | 4 +- source3/lib/pthreadpool/pthreadpool.c | 89 ++--- source3/lib/pthreadpool/pthreadpool.h | 37 +- source3/lib/pthreadpool/pthreadpool_pipe.c | 182 ++++++++++ .../lib/pthreadpool/pthreadpool_pipe.h | 33 +- source3/lib/pthreadpool/pthreadpool_sync.c | 149 +------- source3/lib/pthreadpool/pthreadpool_tevent.c | 240 +++++++++++++ .../pthreadpool_tevent.h} | 25 +- source3/lib/pthreadpool/tests.c | 94 ++--- source3/lib/pthreadpool/wscript_build | 14 +- source3/lib/unix_msg/unix_msg.c | 22 +- source3/lib/util_str.c | 59 ---- source3/lib/util_tdb.c | 3 + source3/libnet/libnet_join.c | 10 + source3/libnet/libnet_samsync_ldif.c | 3 + source3/modules/vfs_aio_pthread.c | 22 +- source3/modules/vfs_default.c | 379 ++++++++++++++------- source3/passdb/pdb_samba_dsdb.c | 27 +- source3/selftest/tests.py | 2 +- source3/smbd/globals.h | 8 +- source3/torture/bench_pthreadpool.c | 22 +- source3/torture/proto.h | 1 + source3/torture/t_strappend.c | 49 --- source3/torture/test_pthreadpool_tevent.c | 82 +++++ source3/torture/torture.c | 2 +- source3/utils/regedit.c | 2 +- source3/utils/regedit_dialog.c | 6 + source3/utils/regedit_dialog.h | 18 +- source3/winbindd/idmap_tdb2.c | 4 + source3/wscript_build | 4 +- source4/torture/dns/dlz_bind9.c | 3 + 59 files changed, 1687 insertions(+), 1298 deletions(-) copy lib/tevent/ABI/{tevent-0.9.26.sigs => tevent-0.9.30.sigs} (93%) delete mode 100644 source3/lib/asys/asys.c delete mode 100644 source3/lib/asys/asys.h delete mode 100644 source3/lib/asys/tests.c delete mode 100644 source3/lib/asys/wscript_build create mode 100644 source3/lib/pthreadpool/pthreadpool_pipe.c copy lib/poll_funcs/poll_funcs_tevent.h => source3/lib/pthreadpool/pthreadpool_pipe.h (52%) create mode 100644 source3/lib/pthreadpool/pthreadpool_tevent.c copy source3/lib/{server_id_watch.h => pthreadpool/pthreadpool_tevent.h} (60%) delete mode 100644 source3/torture/t_strappend.c create mode 100644 source3/torture/test_pthreadpool_tevent.c Changeset truncated at 500 lines: diff --git a/lib/ldb/tools/ldbdump.c b/lib/ldb/tools/ldbdump.c index 3197d19..33f853d 100644 --- a/lib/ldb/tools/ldbdump.c +++ b/lib/ldb/tools/ldbdump.c @@ -110,6 +110,9 @@ static int traverse_fn(TDB_CONTEXT *tdb, TDB_DATA key, TDB_DATA _dbuf, void *sta } static void log_stderr(struct tdb_context *tdb, enum tdb_debug_level level, + const char *fmt, ...) PRINTF_ATTRIBUTE(3,4); + +static void log_stderr(struct tdb_context *tdb, enum tdb_debug_level level, const char *fmt, ...) { va_list ap; diff --git a/lib/ldb/tools/ldbutil.h b/lib/ldb/tools/ldbutil.h index f8d3f3a..6723863 100644 --- a/lib/ldb/tools/ldbutil.h +++ b/lib/ldb/tools/ldbutil.h @@ -43,4 +43,4 @@ int ldb_search_ctrl(struct ldb_context *ldb, TALLOC_CTX *mem_ctx, struct ldb_result **result, struct ldb_dn *base, enum ldb_scope scope, const char * const *attrs, struct ldb_control **controls, - const char *exp_fmt, ...); + const char *exp_fmt, ...) PRINTF_ATTRIBUTE(8,9); diff --git a/lib/replace/wscript b/lib/replace/wscript index 145300d..1dfd902 100644 --- a/lib/replace/wscript +++ b/lib/replace/wscript @@ -483,6 +483,9 @@ removeea setea if conf.CONFIG_SET('HAVE_PORT_CREATE') and conf.CONFIG_SET('HAVE_PORT_H'): conf.DEFINE('HAVE_SOLARIS_PORTS', 1) + if conf.CHECK_FUNCS('eventfd', headers='sys/eventfd.h'): + conf.DEFINE('HAVE_EVENTFD', 1) + conf.CHECK_HEADERS('poll.h') conf.CHECK_FUNCS('poll') diff --git a/lib/talloc/talloc.c b/lib/talloc/talloc.c index 09318e9..927b7f5 100644 --- a/lib/talloc/talloc.c +++ b/lib/talloc/talloc.c @@ -2476,8 +2476,12 @@ _PUBLIC_ char *talloc_strndup_append_buffer(char *s, const char *a, size_t n) #endif static struct talloc_chunk *_vasprintf_tc(const void *t, - const char *fmt, - va_list ap) + const char *fmt, + va_list ap) PRINTF_ATTRIBUTE(2,0); + +static struct talloc_chunk *_vasprintf_tc(const void *t, + const char *fmt, + va_list ap) { int len; char *ret; diff --git a/lib/tdb/tools/tdbdump.c b/lib/tdb/tools/tdbdump.c index 9a0a7fe..8cf3146 100644 --- a/lib/tdb/tools/tdbdump.c +++ b/lib/tdb/tools/tdbdump.c @@ -52,6 +52,9 @@ static int traverse_fn(TDB_CONTEXT *tdb, TDB_DATA key, TDB_DATA dbuf, void *stat } static void log_stderr(struct tdb_context *tdb, enum tdb_debug_level level, + const char *fmt, ...) PRINTF_ATTRIBUTE(3,4); + +static void log_stderr(struct tdb_context *tdb, enum tdb_debug_level level, const char *fmt, ...) { va_list ap; diff --git a/lib/tevent/ABI/tevent-0.9.26.sigs b/lib/tevent/ABI/tevent-0.9.30.sigs similarity index 93% copy from lib/tevent/ABI/tevent-0.9.26.sigs copy to lib/tevent/ABI/tevent-0.9.30.sigs index 1357751..9b8bfa1 100644 --- a/lib/tevent/ABI/tevent-0.9.26.sigs +++ b/lib/tevent/ABI/tevent-0.9.30.sigs @@ -16,6 +16,7 @@ _tevent_req_nomem: bool (const void *, struct tevent_req *, const char *) _tevent_req_notify_callback: void (struct tevent_req *, const char *) _tevent_req_oom: void (struct tevent_req *, const char *) _tevent_schedule_immediate: void (struct tevent_immediate *, struct tevent_context *, tevent_immediate_handler_t, void *, const char *, const char *) +_tevent_threaded_schedule_immediate: void (struct tevent_threaded_context *, struct tevent_immediate *, tevent_immediate_handler_t, void *, const char *, const char *) tevent_backend_list: const char **(TALLOC_CTX *) tevent_cleanup_pending_signal_handlers: void (struct tevent_signal *) tevent_common_add_fd: struct tevent_fd *(struct tevent_context *, TALLOC_CTX *, int, uint16_t, tevent_fd_handler_t, void *, const char *, const char *) @@ -28,10 +29,14 @@ tevent_common_fd_destructor: int (struct tevent_fd *) tevent_common_fd_get_flags: uint16_t (struct tevent_fd *) tevent_common_fd_set_close_fn: void (struct tevent_fd *, tevent_fd_close_fn_t) tevent_common_fd_set_flags: void (struct tevent_fd *, uint16_t) +tevent_common_have_events: bool (struct tevent_context *) tevent_common_loop_immediate: bool (struct tevent_context *) tevent_common_loop_timer_delay: struct timeval (struct tevent_context *) tevent_common_loop_wait: int (struct tevent_context *, const char *) tevent_common_schedule_immediate: void (struct tevent_immediate *, struct tevent_context *, tevent_immediate_handler_t, void *, const char *, const char *) +tevent_common_threaded_activate_immediate: void (struct tevent_context *) +tevent_common_wakeup: int (struct tevent_context *) +tevent_common_wakeup_init: int (struct tevent_context *) tevent_context_init: struct tevent_context *(TALLOC_CTX *) tevent_context_init_byname: struct tevent_context *(TALLOC_CTX *, const char *) tevent_context_init_ops: struct tevent_context *(TALLOC_CTX *, const struct tevent_ops *, void *) @@ -77,6 +82,7 @@ tevent_set_trace_callback: void (struct tevent_context *, tevent_trace_callback_ tevent_signal_support: bool (struct tevent_context *) tevent_thread_proxy_create: struct tevent_thread_proxy *(struct tevent_context *) tevent_thread_proxy_schedule: void (struct tevent_thread_proxy *, struct tevent_immediate **, tevent_immediate_handler_t, void *) +tevent_threaded_context_create: struct tevent_threaded_context *(TALLOC_CTX *, struct tevent_context *) tevent_timeval_add: struct timeval (const struct timeval *, uint32_t, uint32_t) tevent_timeval_compare: int (const struct timeval *, const struct timeval *) tevent_timeval_current: struct timeval (void) diff --git a/lib/tevent/testsuite.c b/lib/tevent/testsuite.c index b37c7b1..4783ab4 100644 --- a/lib/tevent/testsuite.c +++ b/lib/tevent/testsuite.c @@ -1148,6 +1148,101 @@ static bool test_multi_tevent_threaded_1(struct torture_context *test, talloc_free(master_ev); return true; } + +struct threaded_test_2 { + struct tevent_threaded_context *tctx; + struct tevent_immediate *im; + pthread_t thread_id; +}; + +static void master_callback_2(struct tevent_context *ev, + struct tevent_immediate *im, + void *private_data); + +static void *thread_fn_2(void *private_data) +{ + struct threaded_test_2 *state = private_data; + + state->thread_id = pthread_self(); + + usleep(random() % 7000); + + tevent_threaded_schedule_immediate( + state->tctx, state->im, master_callback_2, state); + + return NULL; +} + +static void master_callback_2(struct tevent_context *ev, + struct tevent_immediate *im, + void *private_data) +{ + struct threaded_test_2 *state = private_data; + int i; + + for (i = 0; i < NUM_TEVENT_THREADS; i++) { + if (pthread_equal(state->thread_id, thread_map[i])) { + break; + } + } + torture_comment(thread_test_ctx, + "Callback_2 %u from thread %u\n", + thread_counter, + i); + thread_counter++; +} + +static bool test_multi_tevent_threaded_2(struct torture_context *test, + const void *test_data) +{ + unsigned i; + + struct tevent_context *ev; + struct tevent_threaded_context *tctx; + int ret; + + thread_test_ctx = test; + thread_counter = 0; + + ev = tevent_context_init(test); + torture_assert(test, ev != NULL, "tevent_context_init failed"); + + tctx = tevent_threaded_context_create(ev, ev); + torture_assert(test, tctx != NULL, + "tevent_threaded_context_create failed"); + + for (i=0; i<NUM_TEVENT_THREADS; i++) { + struct threaded_test_2 *state; + + state = talloc(ev, struct threaded_test_2); + torture_assert(test, state != NULL, "talloc failed"); + + state->tctx = tctx; + state->im = tevent_create_immediate(state); + torture_assert(test, state->im != NULL, + "tevent_create_immediate failed"); + + ret = pthread_create(&thread_map[i], NULL, thread_fn_2, state); + torture_assert(test, ret == 0, "pthread_create failed"); + } + + while (thread_counter < NUM_TEVENT_THREADS) { + ret = tevent_loop_once(ev); + torture_assert(test, ret == 0, "tevent_loop_once failed"); + } + + /* Wait for all the threads to finish - join 'em. */ + for (i = 0; i < NUM_TEVENT_THREADS; i++) { + void *retval; + ret = pthread_join(thread_map[i], &retval); + torture_assert(test, ret == 0, "pthread_join failed"); + /* Free the child thread event context. */ + } + + talloc_free(tctx); + talloc_free(ev); + return true; +} #endif struct torture_suite *torture_local_event(TALLOC_CTX *mem_ctx) @@ -1190,6 +1285,10 @@ struct torture_suite *torture_local_event(TALLOC_CTX *mem_ctx) test_multi_tevent_threaded_1, NULL); + torture_suite_add_simple_tcase_const(suite, "multi_tevent_threaded_2", + test_multi_tevent_threaded_2, + NULL); + #endif return suite; diff --git a/lib/tevent/tevent.c b/lib/tevent/tevent.c index 843cf05..331be0e 100644 --- a/lib/tevent/tevent.c +++ b/lib/tevent/tevent.c @@ -59,10 +59,18 @@ */ #include "replace.h" #include "system/filesys.h" +#ifdef HAVE_PTHREAD +#include "system/threads.h" +#endif #define TEVENT_DEPRECATED 1 #include "tevent.h" #include "tevent_internal.h" #include "tevent_util.h" +#ifdef HAVE_EVENTFD +#include <sys/eventfd.h> +#endif + +static void tevent_abort(struct tevent_context *ev, const char *reason); struct tevent_ops_list { struct tevent_ops_list *next, *prev; @@ -173,6 +181,93 @@ const char **tevent_backend_list(TALLOC_CTX *mem_ctx) return list; } +static void tevent_common_wakeup_fini(struct tevent_context *ev); + +#ifdef HAVE_PTHREAD + +static pthread_mutex_t tevent_contexts_mutex = PTHREAD_MUTEX_INITIALIZER; +static struct tevent_context *tevent_contexts = NULL; +static pthread_once_t tevent_atfork_initialized = PTHREAD_ONCE_INIT; + +static void tevent_atfork_prepare(void) +{ + struct tevent_context *ev; + int ret; + + ret = pthread_mutex_lock(&tevent_contexts_mutex); + if (ret != 0) { + abort(); + } + + for (ev = tevent_contexts; ev != NULL; ev = ev->next) { + ret = pthread_mutex_lock(&ev->scheduled_mutex); + if (ret != 0) { + tevent_abort(ev, "pthread_mutex_lock failed"); + } + } +} + +static void tevent_atfork_parent(void) +{ + struct tevent_context *ev; + int ret; + + for (ev = DLIST_TAIL(tevent_contexts); ev != NULL; + ev = DLIST_PREV(ev)) { + ret = pthread_mutex_unlock(&ev->scheduled_mutex); + if (ret != 0) { + tevent_abort(ev, "pthread_mutex_unlock failed"); + } + } + + ret = pthread_mutex_unlock(&tevent_contexts_mutex); + if (ret != 0) { + abort(); + } +} + +static void tevent_atfork_child(void) +{ + struct tevent_context *ev; + int ret; + + for (ev = DLIST_TAIL(tevent_contexts); ev != NULL; + ev = DLIST_PREV(ev)) { + struct tevent_threaded_context *tctx; + + for (tctx = ev->threaded_contexts; tctx != NULL; + tctx = tctx->next) { + tctx->event_ctx = NULL; + } + + ev->threaded_contexts = NULL; + + ret = pthread_mutex_unlock(&ev->scheduled_mutex); + if (ret != 0) { + tevent_abort(ev, "pthread_mutex_unlock failed"); + } + } + + ret = pthread_mutex_unlock(&tevent_contexts_mutex); + if (ret != 0) { + abort(); + } +} + +static void tevent_prep_atfork(void) +{ + int ret; + + ret = pthread_atfork(tevent_atfork_prepare, + tevent_atfork_parent, + tevent_atfork_child); + if (ret != 0) { + abort(); + } +} + +#endif + int tevent_common_context_destructor(struct tevent_context *ev) { struct tevent_fd *fd, *fn; @@ -180,13 +275,35 @@ int tevent_common_context_destructor(struct tevent_context *ev) struct tevent_immediate *ie, *in; struct tevent_signal *se, *sn; - if (ev->pipe_fde) { - talloc_free(ev->pipe_fde); - close(ev->pipe_fds[0]); - close(ev->pipe_fds[1]); - ev->pipe_fde = NULL; +#ifdef HAVE_PTHREAD + int ret; + + ret = pthread_mutex_lock(&tevent_contexts_mutex); + if (ret != 0) { + abort(); } + DLIST_REMOVE(tevent_contexts, ev); + + ret = pthread_mutex_unlock(&tevent_contexts_mutex); + if (ret != 0) { + abort(); + } +#endif + + if (ev->threaded_contexts != NULL) { + /* + * Threaded contexts are indicators that threads are + * about to send us immediates via + * tevent_threaded_schedule_immediate. The caller + * needs to make sure that the tevent context lives + * long enough to receive immediates from all threads. + */ + tevent_abort(ev, "threaded contexts exist"); + } + + tevent_common_wakeup_fini(ev); + for (fd = ev->fd_events; fd; fd = fn) { fn = fd->next; fd->event_ctx = NULL; @@ -255,6 +372,36 @@ struct tevent_context *tevent_context_init_ops(TALLOC_CTX *mem_ctx, ev = talloc_zero(mem_ctx, struct tevent_context); if (!ev) return NULL; +#ifdef HAVE_PTHREAD + + ret = pthread_once(&tevent_atfork_initialized, tevent_prep_atfork); + if (ret != 0) { + talloc_free(ev); + return NULL; + } + + ret = pthread_mutex_init(&ev->scheduled_mutex, NULL); + if (ret != 0) { + talloc_free(ev); + return NULL; + } + + ret = pthread_mutex_lock(&tevent_contexts_mutex); + if (ret != 0) { + pthread_mutex_destroy(&ev->scheduled_mutex); + talloc_free(ev); + return NULL; + } + + DLIST_ADD(tevent_contexts, ev); + + ret = pthread_mutex_unlock(&tevent_contexts_mutex); + if (ret != 0) { + abort(); + } + +#endif + talloc_set_destructor(ev, tevent_common_context_destructor); ev->ops = ops; @@ -620,6 +767,28 @@ done: return ret; } +bool tevent_common_have_events(struct tevent_context *ev) +{ + if (ev->fd_events != NULL) { + if (ev->fd_events != ev->wakeup_fde) { + return true; + } + if (ev->fd_events->next != NULL) { + return true; + } + + /* + * At this point we just have the wakeup pipe event as + * the only fd_event. That one does not count as a + * regular event, so look at the other event types. + */ + } + + return ((ev->timer_events != NULL) || + (ev->immediate_events != NULL) || + (ev->signal_events != NULL)); +} + /* return on failure or (with 0) if all fd events are removed */ @@ -629,10 +798,7 @@ int tevent_common_loop_wait(struct tevent_context *ev, /* * loop as long as we have events pending */ - while (ev->fd_events || - ev->timer_events || - ev->immediate_events || - ev->signal_events) { + while (tevent_common_have_events(ev)) { int ret; ret = _tevent_loop_once(ev, location); if (ret != 0) { @@ -670,3 +836,102 @@ int tevent_re_initialise(struct tevent_context *ev) return ev->ops->context_init(ev); } + +static void wakeup_pipe_handler(struct tevent_context *ev, + struct tevent_fd *fde, + uint16_t flags, void *_private) +{ + ssize_t ret; + + do { + /* + * This is the boilerplate for eventfd, but it works + * for pipes too. And as we don't care about the data + * we read, we're fine. + */ + uint64_t val; + ret = read(fde->fd, &val, sizeof(val)); + } while (ret == -1 && errno == EINTR); +} + +/* + * Initialize the wakeup pipe and pipe fde + */ + +int tevent_common_wakeup_init(struct tevent_context *ev) +{ + int ret; + + if (ev->wakeup_fde != NULL) { + return 0; + } + +#ifdef HAVE_EVENTFD + ret = eventfd(0, EFD_NONBLOCK); + if (ret == -1) { + return errno; -- Samba Shared Repository