The branch, v4-17-stable has been updated via 8e1f74303ee VERSION: Disable GIT_SNAPSHOT for the 4.17.0rc2 release. via 9e75207d331 WHATSNEW: Add release notes for Samba 4.17.0rc2. via 8b6cea8105c WHATSNEW: SMB Server performance improvements via c027512a612 s3:vfs.h: add comment about VFS_OPEN_HOW_RESOLVE_NO_SYMLINKS via ff46ee6ad51 s3: smbd: Add IS_VETO_PATH checks to openat_pathref_fsp_case_insensitive(). via 9e32b03e1ee s3: smbd: Add IS_VETO_PATH check to openat_pathref_dirfsp_nosymlink(). via 80c090c87b2 s3: tests: Add samba3.blackbox.test_veto_files. via 912ee2c92d4 selftest/Samba3: let nt4_dc* use vfs_default:VFS_OPEN_HOW_RESOLVE_NO_SYMLINKS=no via 783e6e7520c vfs_default: Use openat2(RESOLVE_NO_SYMLINKS) if available via 3ec21a8dd98 vfs_default: prepare O_PATH usage with openat2() via 40476e83899 s3:smbd: let openat_pathref_dirfsp_nosymlink() try VFS_OPEN_HOW_RESOLVE_NO_SYMLINKS first via 5d703111ff2 s3:smbd: let openat_pathref_dirfsp_nosymlink() handle ELOOP similar to ENOTDIR via 4ec4806b35a s3:smbd: let openat_pathref_dirfsp_nosymlink() do a verification loop against . and .. first via 25071a1f4ee vfs: define VFS_OPEN_HOW_RESOLVE_NO_SYMLINKS via b71871a193d lib/replace: let DISABLE_OPATH also undef __NR_openat2 via 4b1f56aa04a lib/replace: add fallback defines for __NR_openat2 via dd18624395d lib/replace: use syscall(__NR_openat2) if available via b9a1441238f lib/replace: always include <sys/syscall.h> in replace.c if available via 5326bbac232 lib/replace: add a replacement for openat2() that returns ENOSYS via cc9caffa60e vfs_btrfs: fix include order, includes.h or replace.h should be first via c8c2cbca60b vfs_io_uring: hide a possible definition of struct open_how in liburing/compat.h via efb488977f5 wafsamba: allow cflags for CHECK_TYPE[_IN]() via c2a69553872 s3:tests: add a lot more tests to test_symlink_traversal_smb2.sh via 7b4e11f1554 s3:utils: Fix NULL check via b8a5f41b790 s3:util: Initialize json_object structures so we can call json_free() via fc3f035e368 s3: smbd: Remove unix_convert() and associated functions. via 0ffe593bdab s3: smbd: Remove the old dfs_path_lookup() code. via 37ce01d6ed2 s3: smbd: Switch get_referred_path() over to use the new dfs_path_lookup(). via bd5c6755581 s3: smbd: Add new version of dfs_path_lookup() that uses filename_convert_dirfsp(). via 161324f5758 s3: smbd: Remove dfs_redirect(). via 38740ceea80 s3: smbd: Remove call to dfs_redirect() from filename_convert_dirfsp_nosymlink(). via 66bc141ddfb s3: smbd: Remove call to dfs_redirect() from filename_convert_smb1_search_path(). via d0a9046c80e s3: smbd: In filename_convert_dirfsp_nosymlink(), cope with an MS-DFS link as the terminal component. via 879b42bd6f4 s3: smbd: In filename_convert_dirfsp_nosymlink(), allow a NT_STATUS_PATH_NOT_COVERED error to be returned. via 5f68afbd016 s3: smbd: Allow openat_pathref_dirfsp_nosymlink() to return NT_STATUS_PATH_NOT_COVERED for a DFS link on a DFS share. via 7e9fb8e9fbe s3: smbd: In get create_junction(), make sure check_path_syntax() is called on returned reqpath. via 9a9b953a9d7 s3: smbd: In get referred_path(), make sure check_path_syntax() is called on returned reqpath. via d1ba2845a2a s3: smbd: Add dfs_filename_convert(). Simple wrapper around parse_dfs_path(). via c0f9b5f41e4 s3: smbd: Use helper function msdfs_servicename_matches_connection() in dfs_redirect(). via 74dc7cb556a s3: smbd: Use helper function msdfs_servicename_matches_connection() in parse_dfs_path(). via 0dd880abd96 s3: smbd: Add helper function msdfs_servicename_matches_connection(). via 8ce26e1e4be s3: smbd: Remove definition of struct dfs_path. via 274c8a06b48 s3: smbd: Remove use of 'struct dfs_path'. Not needed for a (hostname, servicename, path) tuple. via 3a944329c31 s3: smbd: Add TALLOC_CTX * parameter to parse_dfs_path(). via 8031584e1eb s3: smbd: Ensure smb2_file_rename_information() uses the SMB2 pathname parsers, not the SMB1 parsers. via 8d09dc16912 s3: smbd: Make sure we have identical check_path_syntax logic in smbd_smb2_create_durable_lease_check(), as for smb2_create. via c940c9eae94 s3: smbd: In smbd_smb2_create_send() call the helper function check_path_syntax_smb2(). via deb009404a5 s3: smbd: Add helper function check_path_syntax_smb2(). via 33d00d7e881 s3: smbd: Add new function check_path_syntax_smb2_msdfs() for SMB2 MSDFS paths. via 496b9b45c38 s3: smbd: Fix cosmetic bug logging pathnames from Linux kernel clients using SMB1 DFS calls. via adcf069e71e s4:torture/smb2: add smb2.bench.echo via 76672394ba5 s4:torture/smb2: teach smb2.bench.path-contention-shared about --option="torture:qdepth=4" via ca8fab6e92f s4:param: add --option="libsmb:client_guid=6112f7d3-9528-4a2a-8861-0ca129aae6c4" support... via 691d0fad1c3 s3:g_lock: use TDB_VOLATILE to avoid fcntl locks via c12a8d50837 smbd: avoid calling SMB_VFS_FGET_NT_ACL() if do_not_check_mask already covers all via 851d7768c30 s3:include: remove unused update_stat_ex_file_id() prototype via f82ef749180 smbstatus: Fix the 32-bit build on FreeBSD via 7b338dc6f57 smbd: Use dirfsp where we have it via d6c44a93f08 s3:tests: let smbstatus json tests fail if jq is not installed via fd61f48e0c1 manpages: add smbstatus option --json with sample output via d6afd0d9417 WHATSNEW: announce new smbstatus json support via a8ddc56e71c WHATSNEW: add section for new smbconf python api via 4cbef001b52 VERSION: Bump version up to 4.17.0rc2... from abc2296a670 VERSION: Disable GIT_SNAPSHOT for the Samba 4.17.0rc1 release.
https://git.samba.org/?p=samba.git;a=shortlog;h=v4-17-stable - Log ----------------------------------------------------------------- ----------------------------------------------------------------------- Summary of changes: VERSION | 2 +- WHATSNEW.txt | 64 +- buildtools/wafsamba/samba_autoconf.py | 7 +- docs-xml/manpages/smbstatus.1.xml | 264 ++++ lib/replace/replace.c | 52 +- lib/replace/system/filesys.h | 35 + lib/replace/wscript | 1 + selftest/target/Samba3.pm | 6 + source3/include/msdfs.h | 7 - source3/include/proto.h | 1 - source3/include/vfs.h | 4 + source3/lib/g_lock.c | 2 +- source3/modules/vfs_btrfs.c | 4 +- source3/modules/vfs_default.c | 71 +- source3/modules/vfs_io_uring.c | 18 + source3/script/tests/test_smbstatus.sh | 22 - .../script/tests/test_symlink_traversal_smb2.sh | 116 ++ source3/script/tests/test_veto_files.sh | 201 +++ source3/selftest/tests.py | 4 + source3/smbd/filename.c | 1639 ++------------------ source3/smbd/files.c | 141 +- source3/smbd/msdfs.c | 714 +++++---- source3/smbd/open.c | 71 +- source3/smbd/proto.h | 25 +- source3/smbd/smb2_create.c | 21 +- source3/smbd/smb2_reply.c | 68 + source3/smbd/smb2_trans2.c | 30 +- source3/utils/status_json.c | 43 +- source3/wscript | 13 + source4/param/loadparm.c | 11 +- source4/torture/smb2/create.c | 591 +++++-- 31 files changed, 2183 insertions(+), 2065 deletions(-) create mode 100755 source3/script/tests/test_veto_files.sh Changeset truncated at 500 lines: diff --git a/VERSION b/VERSION index 5e355c77c6c..47a2a58ee76 100644 --- a/VERSION +++ b/VERSION @@ -87,7 +87,7 @@ SAMBA_VERSION_PRE_RELEASE= # e.g. SAMBA_VERSION_RC_RELEASE=1 # # -> "3.0.0rc1" # ######################################################## -SAMBA_VERSION_RC_RELEASE=1 +SAMBA_VERSION_RC_RELEASE=2 ######################################################## # To mark SVN snapshots this should be set to 'yes' # diff --git a/WHATSNEW.txt b/WHATSNEW.txt index d39748f0587..988c4c66ed8 100644 --- a/WHATSNEW.txt +++ b/WHATSNEW.txt @@ -1,7 +1,7 @@ Release Announcements ===================== -This is the first release candidate of Samba 4.17. This is *not* +This is the second release candidate of Samba 4.17. This is *not* intended for production environments and is designed for testing purposes only. Please report any defects via the Samba bug reporting system at https://bugzilla.samba.org/. @@ -16,6 +16,27 @@ UPGRADING NEW FEATURES/CHANGES ==================== +SMB Server performance improvements +----------------------------------- + +The security improvements in recent releases +(4.13, 4.14, 4.15, 4.16), mainly as protection against symlink races, +caused performance regressions for meta data heavy workloads. + +With 4.17 the situation improved a lot again: + +- Pathnames given by a client are devided into dirname and basename. + The amount of syscalls to validate dirnames is reduced to 2 syscalls + (openat, close) per component. On modern Linux kernels (>= 5.6) smbd + makes use of the openat2() syscall with RESOLVE_NO_SYMLINKS, + in order to just use 2 syscalls (openat2, close) for the whole dirname. + +- Contended path based operations used to generate a lot of unsolicited + wakeup events causing thundering herd problems, which lead to masive + latencies for some clients. These events are now avoided in order + to provide stable latencies and much higher throughput of open/close + operations. + Configure without the SMB1 Server --------------------------------- @@ -147,6 +168,25 @@ can cause the Kerberos password salt to change. This means that after *both* an account rename and a password change, only the current password will be recognised for password history purposes. +Python API for smbconf +---------------------- + +Samba's smbconf library provides a generic frontend to various +configuration backends (plain text file, registry) as a C library. A +new Python wrapper, importable as 'samba.smbconf' is available. An +additional module, 'samba.samba3.smbconf', is also available to enable +registry backend support. These libraries allow Python programs to +read, and optionally write, Samba configuration natively. + +JSON support for smbstatus +-------------------------- + +It is now possible to print detailed information in JSON format in +the smbstatus program using the new option --json. The JSON output +covers all the existing text output including sessions, connections, +open files, byte-range locks, notifies and profile data with all +low-level information maintained by Samba in the respective databases. + REMOVED FEATURES ================ @@ -165,6 +205,28 @@ smb.conf changes dns port New default 53 nt hash store New parameter always +CHANGES SINCE 4.17.0rc1 +======================= + +o Jeremy Allison <j...@samba.org> + * BUG 15143: New filename parser doesn't check veto files smb.conf parameter. + * BUG 15144: 4.17.rc1 still uses symlink-race prone unix_convert() + * BUG 15146: Backport fileserver related changed to 4.17.0rc2 + +o Jule Anger <jan...@samba.org> + * BUG 15147: Manpage for smbstatus json is missing + +o Volker Lendecke <v...@samba.org> + * BUG 15146: Backport fileserver related changed to 4.17.0rc2 + +o Stefan Metzmacher <me...@samba.org> + * BUG 15125: Performance regression on contended path based operations + * BUG 15146: Backport fileserver related changed to 4.17.0rc2 + +o Andreas Schneider <a...@samba.org> + * BUG 15140: Fix issues found by coverity in smbstatus json code + * BUG 15146: Backport fileserver related changed to 4.17.0rc2 + KNOWN ISSUES ============ diff --git a/buildtools/wafsamba/samba_autoconf.py b/buildtools/wafsamba/samba_autoconf.py index 9db53e40724..3ca2f334190 100644 --- a/buildtools/wafsamba/samba_autoconf.py +++ b/buildtools/wafsamba/samba_autoconf.py @@ -146,7 +146,7 @@ def header_list(conf, headers=None, lib=None): @conf -def CHECK_TYPE(conf, t, alternate=None, headers=None, define=None, lib=None, msg=None): +def CHECK_TYPE(conf, t, alternate=None, headers=None, define=None, lib=None, msg=None, cflags=''): '''check for a single type''' if define is None: define = 'HAVE_' + t.upper().replace(' ', '_') @@ -158,6 +158,7 @@ def CHECK_TYPE(conf, t, alternate=None, headers=None, define=None, lib=None, msg headers=headers, local_include=False, msg=msg, + cflags=cflags, lib=lib, link=False) if not ret and alternate: @@ -177,9 +178,9 @@ def CHECK_TYPES(conf, list, headers=None, define=None, alternate=None, lib=None) @conf -def CHECK_TYPE_IN(conf, t, headers=None, alternate=None, define=None): +def CHECK_TYPE_IN(conf, t, headers=None, alternate=None, define=None, cflags=''): '''check for a single type with a header''' - return CHECK_TYPE(conf, t, headers=headers, alternate=alternate, define=define) + return CHECK_TYPE(conf, t, headers=headers, alternate=alternate, define=define, cflags=cflags) @conf diff --git a/docs-xml/manpages/smbstatus.1.xml b/docs-xml/manpages/smbstatus.1.xml index e0945761b28..63ae4641bf1 100644 --- a/docs-xml/manpages/smbstatus.1.xml +++ b/docs-xml/manpages/smbstatus.1.xml @@ -31,6 +31,7 @@ <arg choice="opt">-B|--byterange</arg> <arg choice="opt">-n|--numeric</arg> <arg choice="opt">-f|--fast</arg> + <arg choice="opt">-j|--json</arg> <arg choice="opt">--resolve-uids</arg> <arg choice="opt">-?|--help</arg> <arg choice="opt">--usage</arg> @@ -136,6 +137,269 @@ </listitem> </varlistentry> + <varlistentry> + <term>-j|--json</term> + <listitem><para>Output more detailed information in JSON format instead + of human readable. + The output has the following format: + <programlisting> +{ + "timestamp": "2022-04-15T18:25:15.364891+0200", + "version": "4.17.0pre1-GIT-a0f12b9c80b", + "smb_conf": "/opt/samba/etc/smb.conf", + "sessions": { + "3639217376": { + "session_id": "3639217376", + "server_id": { + "pid": "69650", + "task_id": "0", + "vnn": "4294967295", + "unique_id": "10756714984493602300" + }, + "uid": 1000, + "gid": 1000, + "username": "johndoe", + "groupname": "johndoe", + "remote_machine": "127.0.0.1", + "hostname": "ipv4:127.0.0.1:59944", + "session_dialect": "SMB3_11", + "encryption": { + "cipher": "", + "degree": "none" + }, + "signing": { + "cipher": "AES-128-GMAC", + "degree": "partial" + } + } + }, + "tcons": { + "3813255619": { + "service": "sharename", + "server_id": { + "pid": "69650", + "task_id": "0", + "vnn": "4294967295", + "unique_id": "10756714984493602300" + }, + "tcon_id": "3813255619", + "session_id": "3639217376", + "machine": "127.0.0.1", + "connected_at": "2022-04-15T17:30:37+0200", + "encryption": { + "cipher": "AES-128-GMAC", + "degree": "full" + }, + "signing": { + "cipher": "", + "degree": "none" + } + } + }, + "open_files": { + "/home/johndoe/testfolder/sample": { + "service_path": "/home/johndoe/testfolder", + "filename": "sample", + "fileid": { + "devid": 59, + "inode": 11404245, + "extid": 0 + }, + "num_pending_deletes": 0, + "opens": { + "56839/2": { + "server_id": { + "pid": "69650", + "task_id": "0", + "vnn": "4294967295", + "unique_id": "10756714984493602300" + }, + "uid": 1000, + "share_file_id": 2, + "sharemode": { + "hex": "0x00000003", + "NONE": false, + "READ": true, + "WRITE": true, + "DELETE": false, + "text": "RW" + }, + "access_mask": { + "hex": "0x00000003", + "READ_DATA": true, + "WRITE_DATA": true, + "APPEND_DATA": false, + "READ_EA": false, + "WRITE_EA": false, + "EXECUTE": false, + "READ_ATTRIBUTES": false, + "WRITE_ATTRIBUTES": false, + "DELETE_CHILD": false, + "DELETE": false, + "READ_CONTROL": false, + "WRITE_DAC": false, + "SYNCHRONIZE": false, + "ACCESS_SYSTEM_SECURITY": false, + "text": "RW" + }, + "caching": { + "READ": false, + "WRITE": false, + "HANDLE": false, + "hex": "0x00000000", + "text": "" + }, + "oplock": {}, + "lease": {}, + "opened_at": "2022-04-15T17:30:38+0200" + } + } + } + } +} + </programlisting> </para> + + <para>If oplocks are used: + <programlisting> + "oplock": { + "EXCLUSIVE": false, + "BATCH": false, + "LEVEL_II": true, + "LEASE": false, + "text": "LEVEL_II" + } + </programlisting> </para> + + <para>If leases are used: + + <programlisting> + "lease": { + "lease_key": "29316055-f55c-de10-c813-af7bf5a430bb", + "hex": "0x00000005", + "READ": true, + "WRITE": true, + "HANDLE": false, + "text": "RW" + } + </programlisting> </para> + + <para>With byte-range locks (-B, --byterange): + <programlisting> + "byte_range_locks": { + "/home/johndoe/testfolder/sample": { + "fileid": { + "devid": 59, + "inode": 11404245, + "extid": 0 + }, + "file_name": "sample", + "share_path": "/home/johndoe/testfolder", + "locks": [ + { + "server_id": { + "pid": "69650", + "task_id": "0", + "vnn": "4294967295", + "unique_id": "10756714984493602300" + }, + "type": "R", + "flavour": "Posix", + "start": 0, + "size": 16 + } + ] + } + </programlisting> </para> + + <para> With notifies (-N, --notify): + <programlisting> + "notify": { + "77247": { + "server_id": { + "pid": "69650", + "task_id": "0", + "vnn": "4294967295", + "unique_id": "10756714984493602300" + }, + "path": "/home/johndoe/testfolder/testdir", + "filter": 4095, + "subdir_filter": 4095, + "creation_time": "1970-01-01T01:00:14.326582+01:00" + } + } + </programlisting> </para> + + <para> For profiling (-P, --profile): + <programlisting> +{ + "timestamp": "2022-04-15T18:40:43.112216+0200", + "version": "4.17.0pre1-GIT-a0f12b9c80b", + "smb_conf": "/opt/samba/etc/smb.conf", + "SMBD loop": { + "connect": { + "count": 2 + }, + "disconnect": { + "count": 1 + }, + ... + }, + "System Calls": { + "syscall_opendir": { + "count": 0, + "time": 0 + }, + ... + }, + "ACL Calls": { + "get_nt_acl": { + "count": 0, + "time": 0 + }, + ... + }, + "Stat Cache": { + "statcache_lookups": { + "count": 2 + }, + ... + }, + "SMB Calls": { + "SMBmkdir": { + "count": 0, + "time": 0 + }, + ... + }, + "Trans2 Calls": { + "Trans2_open": { + "count": 0, + "time": 0 + }, + ... + }, + "NT Transact Calls": { + "NT_transact_create": { + "count": 0, + "time": 0 + }, + ... + }, + "SMB2 Calls": { + "smb2_negprot": { + "count": 2, + "time": 3060, + "idle": 0, + "inbytes": 452, + "outbytes": 568 + }, + ... + } +} + </programlisting> </para> + </listitem> + </varlistentry> + &popt.autohelp; &cmdline.common.samba.client; </variablelist> diff --git a/lib/replace/replace.c b/lib/replace/replace.c index 0652cb4e6d6..cbf372e494f 100644 --- a/lib/replace/replace.c +++ b/lib/replace/replace.c @@ -33,6 +33,10 @@ #include "system/locale.h" #include "system/wait.h" +#ifdef HAVE_SYS_SYSCALL_H +#include <sys/syscall.h> +#endif + #ifdef _WIN32 #define mkdir(d,m) _mkdir(d) #endif @@ -1058,9 +1062,6 @@ const char *rep_getprogname(void) #endif /* HAVE_GETPROGNAME */ #ifndef HAVE_COPY_FILE_RANGE -# ifdef HAVE_SYSCALL_COPY_FILE_RANGE -# include <sys/syscall.h> -# endif ssize_t rep_copy_file_range(int fd_in, loff_t *off_in, int fd_out, @@ -1081,3 +1082,48 @@ ssize_t rep_copy_file_range(int fd_in, return -1; } #endif /* HAVE_COPY_FILE_RANGE */ + +#ifndef HAVE_OPENAT2 + +/* fallback known wellknown __NR_openat2 values */ +#ifndef __NR_openat2 +# if defined(LINUX) && defined(HAVE_SYS_SYSCALL_H) +# if defined(__i386__) +# define __NR_openat2 437 +# elif defined(__x86_64__) && defined(__LP64__) +# define __NR_openat2 437 /* 437 0x1B5 */ +# elif defined(__x86_64__) && defined(__ILP32__) +# define __NR_openat2 1073742261 /* 1073742261 0x400001B5 */ +# elif defined(__aarch64__) +# define __NR_openat2 437 +# elif defined(__arm__) +# define __NR_openat2 437 +# elif defined(__sparc__) +# define __NR_openat2 437 +# endif +# endif /* defined(LINUX) && defined(HAVE_SYS_SYSCALL_H) */ +#endif /* !__NR_openat2 */ + +#ifdef DISABLE_OPATH +/* + * systems without O_PATH also don't have openat2, + * so make sure we at a realistic combination. + */ +#undef __NR_openat2 +#endif /* DISABLE_OPATH */ + +long rep_openat2(int dirfd, const char *pathname, + struct open_how *how, size_t size) +{ +#ifdef __NR_openat2 + return syscall(__NR_openat2, + dirfd, + pathname, + how, + size); +#else + errno = ENOSYS; + return -1; +#endif +} +#endif /* !HAVE_OPENAT2 */ diff --git a/lib/replace/system/filesys.h b/lib/replace/system/filesys.h index bb9482c69af..8005b18780f 100644 -- Samba Shared Repository