The branch, v3-3-stable has been updated via 518b825b570d1678d4221bd0ecb9c45a7f651813 (commit) via 54ca4cc63c5364bcfb48a1a31812b4b1fc51ac44 (commit) via 765e6313408fda8943ae2ca48e4ad2f61a7714f6 (commit) via 6aa55d24912f6631d1c708051b725fce7f8d74f2 (commit) via 2a86f7546872cea3a7164dab273b8716ed26005a (commit) from a5aa69f9d1a6499e9eea1b4821bdfbc87597a603 (commit)
http://gitweb.samba.org/?p=samba.git;a=shortlog;h=v3-3-stable - Log ----------------------------------------------------------------- commit 518b825b570d1678d4221bd0ecb9c45a7f651813 Author: Karolin Seeger <ksee...@samba.org> Date: Wed Mar 11 08:59:10 2009 +0100 WHATSNEW: Update changes. Karolin (cherry picked from commit eeaf0cb1572fff27df732714ddc192db2c8f9aff) commit 54ca4cc63c5364bcfb48a1a31812b4b1fc51ac44 Author: Stefan Metzmacher <me...@samba.org> Date: Tue Mar 10 16:53:06 2009 -0700 s3:signing: the seqnum should only be decremented by 1 for ntcancel requests [MS-SMB] 3.3.5.1 Receiving Any Message says that the seqnum is incremented by only for ntcancel requests for any other request it's by incremented by 2, even if it doesn't expect a response. metze (cherry picked from commit 0999366b6b36f3084870af0375d686b0cbaae698) commit 765e6313408fda8943ae2ca48e4ad2f61a7714f6 Author: Volker Lendecke <v...@sernet.de> Date: Sun Nov 23 08:01:33 2008 +0100 Add a vfs_preopen module to hide fs latencies (cherry picked from commit c5e1900879411cf240ddb4c1dcb11e645cbcb03e) commit 6aa55d24912f6631d1c708051b725fce7f8d74f2 Author: Volker Lendecke <v...@samba.org> Date: Sat Feb 14 13:18:28 2009 +0100 Don't log NDR_PRINT_DEBUG at level 0, this always ends up in syslog I think we eventually need to pass down the debuglevel here. (cherry picked from commit 20086d0f4ac318f245bb48fc45999c9104ddccef) commit 2a86f7546872cea3a7164dab273b8716ed26005a Author: Volker Lendecke <v...@samba.org> Date: Tue Mar 10 14:06:05 2009 +0100 Fix a valgrind error / segfault in dns_register_smbd() (cherry picked from commit 4624451b7b78695b3f84e7fff91018ee9df0e589) ----------------------------------------------------------------------- Summary of changes: WHATSNEW.txt | 9 +- docs-xml/manpages-3/vfs_preopen.8.xml | 115 +++++++++ source/Makefile.in | 5 + source/configure.in | 3 +- source/include/proto.h | 2 +- source/librpc/ndr/ndr.c | 4 +- source/libsmb/smb_signing.c | 6 +- source/modules/vfs_preopen.c | 456 +++++++++++++++++++++++++++++++++ source/smbd/aio.c | 8 +- source/smbd/dnsregister.c | 3 +- source/smbd/nttrans.c | 2 +- 11 files changed, 599 insertions(+), 14 deletions(-) create mode 100644 docs-xml/manpages-3/vfs_preopen.8.xml create mode 100644 source/modules/vfs_preopen.c Changeset truncated at 500 lines: diff --git a/WHATSNEW.txt b/WHATSNEW.txt index 06b9624..da078b7 100644 --- a/WHATSNEW.txt +++ b/WHATSNEW.txt @@ -1,6 +1,6 @@ ============================= Release Notes for Samba 3.3.2 - March, 09 2009 + March, 12 2009 ============================= This is the latest bugfix release release of the Samba 3.3 series. @@ -8,6 +8,7 @@ This is the latest bugfix release release of the Samba 3.3 series. Major enhancements in Samba 3.3.2 include: * Fix "force group" (bug #6155). + * Fix saving of files on Samba share using MS Office 2007 (bug #6160). * Fix guest authentication in setups with "security = share" and "guest ok = yes" when Winbind is running. * Fix corruptions of source path in tar mode of smbclient (bug #6161). @@ -26,7 +27,7 @@ o Jeremy Allison <j...@samba.org> * BUG 6082: Fix renaming and deleting of directories using Windows clients. * BUG 6154: Make ZFS honor admin users. * BUG 6155: Fix "force group". - * BUG 6160: Fix saving of files on Samba share using MS Office 2007 + * BUG 6160: Fix saving of files on Samba share using MS Office 2007. * BUG 6161: Fix corruptions of source path in tar mode of smbclient. * Fix some NetBSD warnings. * Fix bug in processing of open modes in POSIX open. @@ -57,10 +58,14 @@ o Björn Jacke <b...@sernet.de> o Volker Lendecke <v...@samba.org> * BUG 6124: Fix the build on IRIX. + * Add a vfs_preopen module to hide fs latencies. + * Don't log NDR_PRINT_DEBUG at level 0, this always ends up in syslog. + * Fix a valgrind error / segfault in dns_register_smbd(). o Stefan Metzmacher <me...@samba.org> * Fix build on SLES8. + * Decremented by 1 for ntcancel requests. o Tim Prouty <tpro...@samba.org> diff --git a/docs-xml/manpages-3/vfs_preopen.8.xml b/docs-xml/manpages-3/vfs_preopen.8.xml new file mode 100644 index 0000000..a84d472 --- /dev/null +++ b/docs-xml/manpages-3/vfs_preopen.8.xml @@ -0,0 +1,115 @@ +<?xml version="1.0" encoding="iso-8859-1"?> +<!DOCTYPE refentry PUBLIC "-//Samba-Team//DTD DocBook V4.2-Based Variant V1.0//EN" "http://www.samba.org/samba/DTD/samba-doc"> +<refentry id="vfs_preopen.8"> + +<refmeta> + <refentrytitle>vfs_preopen</refentrytitle> + <manvolnum>8</manvolnum> + <refmiscinfo class="source">Samba</refmiscinfo> + <refmiscinfo class="manual">System Administration tools</refmiscinfo> + <refmiscinfo class="version">3.3</refmiscinfo> +</refmeta> + +<refnamediv> + <refname>vfs_preopen</refname> + <refpurpose>Hide read latencies for applications reading numbered files</refpurpose> +</refnamediv> + +<refsynopsisdiv> + <cmdsynopsis> + <command>vfs objects = preopen</command> + </cmdsynopsis> +</refsynopsisdiv> + +<refsect1> + <title>DESCRIPTION</title> + + <para>This VFS module is part of the + <citerefentry><refentrytitle>samba</refentrytitle> + <manvolnum>7</manvolnum></citerefentry> suite.</para> + + <para>This module assists applications that want to read numbered + files in sequence with very strict latency requirements. One area + where this happens in video streaming applications that want to read + one file per frame.</para> + + <para>When you use this module, a number of helper processes is + started that speculatively open files and read a number of bytes to + prime the file system cache, so that later on when the real + application's request comes along, no disk access is necessary.</para> + + <para>This module is stackable.</para> + +</refsect1> + + +<refsect1> + <title>OPTIONS</title> + + <variablelist> + + <varlistentry> + <term>preopen:names = /pattern/</term> + <listitem> + <para> + preopen:names specifies the file name pattern which should + trigger the preopen helpers to do their work. We assume that + the files are numbered incrementally. So if your file names + are numbered FRAME00000.frm FRAME00001.frm and so on you would + list them as <command>preopen:names=/FRAME*.frm/</command> + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term>preopen:num_bytes = BYTES</term> + <listitem> + <para> + Specifies the number of bytes the helpers should speculatively + read, defaults to 1. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term>preopen:helpers = NUM-PROCS</term> + <listitem> + <para> + Number of forked helper processes, defaults to 1. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term>preopen:queuelen = NUM-FILES</term> + <listitem> + <para> + Number of files that should be speculatively opened. Defaults + to the 10 subsequent files. + </para> + </listitem> + </varlistentry> + + </variablelist> +</refsect1> + +<refsect1> + <title>VERSION</title> + <para>This man page is correct for version 3.3 of the Samba suite. + </para> +</refsect1> + +<refsect1> + <title>AUTHOR</title> + + <para>The original Samba software and related utilities + were created by Andrew Tridgell. Samba is now developed + by the Samba Team as an Open Source project similar + to the way the Linux kernel is developed.</para> + + <para>The PREOPEN VFS module was created with contributions from + Volker Lendecke and the developers at IBM. + </para> +</refsect1> + +</refentry> diff --git a/source/Makefile.in b/source/Makefile.in index f7b4431..e160a99 100644 --- a/source/Makefile.in +++ b/source/Makefile.in @@ -630,6 +630,7 @@ VFS_READAHEAD_OBJ = modules/vfs_readahead.o VFS_TSMSM_OBJ = modules/vfs_tsmsm.o VFS_FILEID_OBJ = modules/vfs_fileid.o VFS_AIO_FORK_OBJ = modules/vfs_aio_fork.o +VFS_PREOPEN_OBJ = modules/vfs_preopen.o VFS_SYNCOPS_OBJ = modules/vfs_syncops.o VFS_ACL_XATTR_OBJ = modules/vfs_acl_xattr.o VFS_ACL_TDB_OBJ = modules/vfs_acl_tdb.o @@ -2449,6 +2450,10 @@ bin/scrip...@shlibext@: $(BINARY_PREREQS) libgpo/gpext/scripts.o @echo "Building plugin $@" @$(SHLD_MODULE) libgpo/gpext/scripts.o +bin/preop...@shlibext@: $(BINARY_PREREQS) $(VFS_PREOPEN_OBJ) + @echo "Building plugin $@" + @$(SHLD_MODULE) $(VFS_PREOPEN_OBJ) + bin/smb_traffic_analyz...@shlibext@: $(BINARY_PREREQS) $(VFS_SMB_TRAFFIC_ANALYZER_OBJ) @echo "Building plugin $@" @$(SHLD_MODULE) $(VFS_SMB_TRAFFIC_ANALYZER_OBJ) diff --git a/source/configure.in b/source/configure.in index cfb97b5..fee3ac1 100644 --- a/source/configure.in +++ b/source/configure.in @@ -409,7 +409,7 @@ dnl These have to be built static: default_static_modules="pdb_smbpasswd pdb_tdbsam rpc_lsarpc rpc_samr rpc_winreg rpc_initshutdown rpc_dssetup rpc_wkssvc rpc_svcctl2 rpc_ntsvcs2 rpc_netlogon rpc_netdfs rpc_srvsvc rpc_spoolss rpc_eventlog2 auth_sam auth_unix auth_winbind auth_server auth_domain auth_builtin vfs_default nss_info_template" dnl These are preferably build shared, and static if dlopen() is not available -default_shared_modules="vfs_recycle vfs_audit vfs_extd_audit vfs_full_audit vfs_netatalk vfs_fake_perms vfs_default_quota vfs_readonly vfs_cap vfs_expand_msdfs vfs_shadow_copy vfs_shadow_copy2 charset_CP850 charset_CP437 auth_script vfs_readahead vfs_xattr_tdb vfs_streams_xattr vfs_streams_depot vfs_acl_xattr vfs_acl_tdb vfs_smb_traffic_analyzer" +default_shared_modules="vfs_recycle vfs_audit vfs_extd_audit vfs_full_audit vfs_netatalk vfs_fake_perms vfs_default_quota vfs_readonly vfs_cap vfs_expand_msdfs vfs_shadow_copy vfs_shadow_copy2 charset_CP850 charset_CP437 auth_script vfs_readahead vfs_xattr_tdb vfs_streams_xattr vfs_streams_depot vfs_acl_xattr vfs_acl_tdb vfs_smb_traffic_analyzer vfs_preopen" if test "x$developer" = xyes; then default_static_modules="$default_static_modules rpc_rpcecho" @@ -6394,6 +6394,7 @@ SMB_MODULE(vfs_readahead, \$(VFS_READAHEAD_OBJ), "bin/readahead.$SHLIBEXT", VFS) SMB_MODULE(vfs_tsmsm, \$(VFS_TSMSM_OBJ), "bin/tsmsm.$SHLIBEXT", VFS) SMB_MODULE(vfs_fileid, \$(VFS_FILEID_OBJ), "bin/fileid.$SHLIBEXT", VFS) SMB_MODULE(vfs_aio_fork, \$(VFS_AIO_FORK_OBJ), "bin/aio_fork.$SHLIBEXT", VFS) +SMB_MODULE(vfs_preopen, \$(VFS_PREOPEN_OBJ), "bin/preopen.$SHLIBEXT", VFS) SMB_MODULE(vfs_syncops, \$(VFS_SYNCOPS_OBJ), "bin/syncops.$SHLIBEXT", VFS) SMB_MODULE(vfs_zfsacl, \$(VFS_ZFSACL_OBJ), "bin/zfsacl.$SHLIBEXT", VFS) SMB_MODULE(vfs_notify_fam, \$(VFS_NOTIFY_FAM_OBJ), "bin/notify_fam.$SHLIBEXT", VFS) diff --git a/source/include/proto.h b/source/include/proto.h index c3df0ae..51405f0 100644 --- a/source/include/proto.h +++ b/source/include/proto.h @@ -4942,7 +4942,7 @@ bool srv_oplock_set_signing(bool onoff); bool srv_check_sign_mac(const char *inbuf, bool must_be_ok); void srv_calculate_sign_mac(char *outbuf); void srv_defer_sign_response(uint16 mid); -void srv_cancel_sign_response(uint16 mid); +void srv_cancel_sign_response(uint16 mid, bool cancel); void srv_set_signing_negotiated(void); bool srv_is_signing_active(void); bool srv_is_signing_negotiated(void); diff --git a/source/librpc/ndr/ndr.c b/source/librpc/ndr/ndr.c index d94d12e..84ce4bd 100644 --- a/source/librpc/ndr/ndr.c +++ b/source/librpc/ndr/ndr.c @@ -187,10 +187,10 @@ _PUBLIC_ void ndr_print_debug_helper(struct ndr_print *ndr, const char *format, } for (i=0;i<ndr->depth;i++) { - DEBUGADD(0,(" ")); + DEBUGADD(1,(" ")); } - DEBUGADD(0,("%s\n", s)); + DEBUGADD(1,("%s\n", s)); free(s); } diff --git a/source/libsmb/smb_signing.c b/source/libsmb/smb_signing.c index ea1eb05..55b30d4 100644 --- a/source/libsmb/smb_signing.c +++ b/source/libsmb/smb_signing.c @@ -865,7 +865,7 @@ void srv_defer_sign_response(uint16 mid) cancelled by mid. This should never find one.... ************************************************************/ -void srv_cancel_sign_response(uint16 mid) +void srv_cancel_sign_response(uint16 mid, bool cancel) { struct smb_basic_signing_context *data; uint32 dummy_seq; @@ -884,7 +884,9 @@ void srv_cancel_sign_response(uint16 mid) ; /* cancel doesn't send a reply so doesn't burn a sequence number. */ - data->send_seq_num -= 1; + if (cancel) { + data->send_seq_num -= 1; + } } /*********************************************************** diff --git a/source/modules/vfs_preopen.c b/source/modules/vfs_preopen.c new file mode 100644 index 0000000..25b9e7f --- /dev/null +++ b/source/modules/vfs_preopen.c @@ -0,0 +1,456 @@ +/* + * Force a readahead of files by opening them and reading the first bytes + * + * Copyright (C) Volker Lendecke 2008 + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include "includes.h" + +struct preopen_state; + +struct preopen_helper { + struct preopen_state *state; + struct fd_event *fde; + pid_t pid; + int fd; + bool busy; +}; + +struct preopen_state { + int num_helpers; + struct preopen_helper *helpers; + + size_t to_read; /* How many bytes to read in children? */ + int queue_max; + + char *template_fname; /* Filename to be sent to children */ + size_t number_start; /* start offset into "template_fname" */ + int num_digits; /* How many digits is the number long? */ + + int fnum_sent; /* last fname sent to children */ + + int fnum_queue_end; /* last fname to be sent, based on + * last open call + preopen:queuelen + */ + + name_compare_entry *preopen_names; +}; + +static void preopen_helper_destroy(struct preopen_helper *c) +{ + int status; + close(c->fd); + c->fd = -1; + kill(c->pid, SIGKILL); + waitpid(c->pid, &status, 0); + c->busy = true; +} + +static void preopen_queue_run(struct preopen_state *state) +{ + char *pdelimiter; + char delimiter; + + pdelimiter = state->template_fname + state->number_start + + state->num_digits; + delimiter = *pdelimiter; + + while (state->fnum_sent < state->fnum_queue_end) { + + ssize_t written; + size_t to_write; + int helper; + + for (helper=0; helper<state->num_helpers; helper++) { + if (state->helpers[helper].busy) { + continue; + } + break; + } + if (helper == state->num_helpers) { + /* everyone is busy */ + return; + } + + snprintf(state->template_fname + state->number_start, + state->num_digits + 1, + "%.*lu", state->num_digits, + (long unsigned int)(state->fnum_sent + 1)); + *pdelimiter = delimiter; + + to_write = talloc_get_size(state->template_fname); + written = write_data(state->helpers[helper].fd, + state->template_fname, to_write); + state->helpers[helper].busy = true; + + if (written != to_write) { + preopen_helper_destroy(&state->helpers[helper]); + } + state->fnum_sent += 1; + } +} + +static void preopen_helper_readable(struct event_context *ev, + struct fd_event *fde, uint16_t flags, + void *priv) +{ + struct preopen_helper *helper = (struct preopen_helper *)priv; + struct preopen_state *state = helper->state; + ssize_t nread; + char c; + + if ((flags & EVENT_FD_READ) == 0) { + return; + } + + nread = read(helper->fd, &c, 1); + if (nread <= 0) { + preopen_helper_destroy(helper); + return; + } + + helper->busy = false; + + preopen_queue_run(state); +} + +static int preopen_helpers_destructor(struct preopen_state *c) +{ + int i; + + for (i=0; i<c->num_helpers; i++) { + if (c->helpers[i].fd == -1) { + continue; + } + preopen_helper_destroy(&c->helpers[i]); + } + + return 0; +} + +static bool preopen_helper_open_one(int sock_fd, char **pnamebuf, + size_t to_read, void *filebuf) +{ + char *namebuf = *pnamebuf; + ssize_t nwritten, nread; + char c = 0; + int fd; + + nread = 0; + + while ((nread == 0) || (namebuf[nread-1] != '\0')) { + ssize_t thistime; + + thistime = read(sock_fd, namebuf + nread, + talloc_get_size(namebuf) - nread); + if (thistime <= 0) { + return false; + } + + nread += thistime; + + if (nread == talloc_get_size(namebuf)) { + namebuf = TALLOC_REALLOC_ARRAY( + NULL, namebuf, char, + talloc_get_size(namebuf) * 2); + if (namebuf == NULL) { + return false; + } + *pnamebuf = namebuf; + } + } + + fd = open(namebuf, O_RDONLY); + if (fd == -1) { + goto done; + } + nread = read(fd, filebuf, to_read); + close(fd); + + done: + nwritten = write(sock_fd, &c, 1); + return true; +} + +static bool preopen_helper(int fd, size_t to_read) +{ + char *namebuf; + void *readbuf; + + namebuf = TALLOC_ARRAY(NULL, char, 1024); + if (namebuf == NULL) { + return false; + } + + readbuf = talloc_size(NULL, to_read); + if (readbuf == NULL) { + TALLOC_FREE(namebuf); + return false; + } + + while (preopen_helper_open_one(fd, &namebuf, to_read, readbuf)) { + ; + } + + TALLOC_FREE(readbuf); + TALLOC_FREE(namebuf); + return false; +} + +static NTSTATUS preopen_init_helper(struct preopen_helper *h) +{ + int fdpair[2]; + NTSTATUS status; + + if (socketpair(AF_UNIX, SOCK_STREAM, 0, fdpair) == -1) { + status = map_nt_error_from_unix(errno); + DEBUG(10, ("socketpair() failed: %s\n", strerror(errno))); + return status; + } + + h->pid = sys_fork(); + + if (h->pid == -1) { + return map_nt_error_from_unix(errno); + } + + if (h->pid == 0) { -- Samba Shared Repository