Hello community, here is the log from the commit of package python-pyftpdlib for openSUSE:Leap:15.2 checked in at 2020-03-20 05:15:04 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Leap:15.2/python-pyftpdlib (Old) and /work/SRC/openSUSE:Leap:15.2/.python-pyftpdlib.new.3160 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-pyftpdlib" Fri Mar 20 05:15:04 2020 rev:5 rq:786357 version:1.5.6 Changes: -------- --- /work/SRC/openSUSE:Leap:15.2/python-pyftpdlib/python-pyftpdlib.changes 2020-03-02 13:20:50.446098230 +0100 +++ /work/SRC/openSUSE:Leap:15.2/.python-pyftpdlib.new.3160/python-pyftpdlib.changes 2020-03-20 05:15:12.178559997 +0100 @@ -1,0 +2,8 @@ +Wed Mar 18 09:21:28 UTC 2020 - Tomáš Chvátal <tchva...@suse.com> + +- Update to 1.5.6: + - #467: added pre-fork concurrency model, spawn()ing worker processes to split + load. + - #520: directory LISTing is now 3.7x times faster. + +------------------------------------------------------------------- Old: ---- pyftpdlib-1.5.5.tar.gz New: ---- pyftpdlib-1.5.6.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-pyftpdlib.spec ++++++ --- /var/tmp/diff_new_pack.H4v6Lq/_old 2020-03-20 05:15:12.526560231 +0100 +++ /var/tmp/diff_new_pack.H4v6Lq/_new 2020-03-20 05:15:12.530560233 +0100 @@ -1,7 +1,7 @@ # # spec file for package python-pyftpdlib # -# Copyright (c) 2019 SUSE LINUX GmbH, Nuernberg, Germany. +# Copyright (c) 2020 SUSE LLC # Copyright (c) 2016 LISA GmbH, Bingen, Germany. # # All modifications and additions to the file contributed by third parties @@ -21,7 +21,7 @@ # Tests randomly fail: https://github.com/giampaolo/pyftpdlib/issues/386 %bcond_with test Name: python-pyftpdlib -Version: 1.5.5 +Version: 1.5.6 Release: 0 Summary: Asynchronous FTP server library for Python License: MIT @@ -57,6 +57,7 @@ %install %python_install +%python_expand rm -r %{buildroot}%{$python_sitelib}/pyftpdlib/test %python_expand %fdupes %{buildroot}%{$python_sitelib} %if %{with test} ++++++ pyftpdlib-1.5.5.tar.gz -> pyftpdlib-1.5.6.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyftpdlib-1.5.5/HISTORY.rst new/pyftpdlib-1.5.6/HISTORY.rst --- old/pyftpdlib-1.5.5/HISTORY.rst 2019-04-04 11:10:16.000000000 +0200 +++ new/pyftpdlib-1.5.6/HISTORY.rst 2020-02-16 16:39:33.000000000 +0100 @@ -1,5 +1,14 @@ Bug tracker at https://github.com/giampaolo/pyftpdlib/issues +Version: 1.5.6 - 2020-02-16 +=========================== + +**Enhancements** + +- #467: added pre-fork concurrency model, spawn()ing worker processes to split + load. +- #520: directory LISTing is now 3.7x times faster. + Version: 1.5.5 - 2019-04-04 =========================== diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyftpdlib-1.5.5/LICENSE new/pyftpdlib-1.5.6/LICENSE --- old/pyftpdlib-1.5.5/LICENSE 2017-12-30 09:44:57.000000000 +0100 +++ new/pyftpdlib-1.5.6/LICENSE 2019-10-22 06:45:18.000000000 +0200 @@ -1,22 +1,21 @@ -====================================================================== -Copyright (C) 2007-2016 Giampaolo Rodola' <g.rod...@gmail.com> +MIT License - All Rights Reserved +Copyright (c) 2007 Giampaolo Rodola' -Permission to use, copy, modify, and distribute this software and -its documentation for any purpose and without fee is hereby -granted, provided that the above copyright notice appear in all -copies and that both that copyright notice and this permission -notice appear in supporting documentation, and that the name of -Giampaolo Rodola' not be used in advertising or publicity pertaining to -distribution of the software without specific, written prior -permission. +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: -Giampaolo Rodola' DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN -NO EVENT Giampaolo Rodola' BE LIABLE FOR ANY SPECIAL, INDIRECT OR -CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -====================================================================== +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyftpdlib-1.5.5/Makefile new/pyftpdlib-1.5.6/Makefile --- old/pyftpdlib-1.5.5/Makefile 2019-03-27 17:32:12.000000000 +0100 +++ new/pyftpdlib-1.5.6/Makefile 2019-10-24 10:26:36.000000000 +0200 @@ -6,6 +6,7 @@ TSCRIPT = pyftpdlib/test/runner.py ARGS = DEV_DEPS = \ + cffi \ check-manifest \ coverage \ flake8 \ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyftpdlib-1.5.5/PKG-INFO new/pyftpdlib-1.5.6/PKG-INFO --- old/pyftpdlib-1.5.5/PKG-INFO 2019-04-04 11:11:03.000000000 +0200 +++ new/pyftpdlib-1.5.6/PKG-INFO 2020-02-16 16:43:47.000000000 +0100 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: pyftpdlib -Version: 1.5.5 +Version: 1.5.6 Summary: Very fast asynchronous FTP server library Home-page: https://github.com/giampaolo/pyftpdlib/ Author: Giampaolo Rodola' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyftpdlib-1.5.5/docs/api.rst new/pyftpdlib-1.5.6/docs/api.rst --- old/pyftpdlib-1.5.5/docs/api.rst 2018-05-15 17:45:22.000000000 +0200 +++ new/pyftpdlib-1.5.6/docs/api.rst 2019-10-24 10:26:36.000000000 +0200 @@ -379,13 +379,41 @@ Number of maximum connections accepted for the same IP address (default ``0`` == no limit). - .. method:: serve_forever(timeout=None, blocking=True, handle_exit=True) + .. method:: serve_forever(timeout=None, blocking=True, handle_exit=True, worker_processes=1) Starts the asynchronous IO loop. - *Changed in version 1.0.0: no longer a classmethod; 'use_poll' and 'count' - *parameters were removed. 'blocking' and 'handle_exit' parameters were - *added* + - (float) timeout: the timeout passed to the underlying IO + loop expressed in seconds. + + - (bool) blocking: if False loop once and then return the + timeout of the next scheduled call next to expire soonest + (if any). + + - (bool) handle_exit: when True catches ``KeyboardInterrupt`` and + ``SystemExit`` exceptions (caused by SIGTERM / SIGINT signals) and + gracefully exits after cleaning up resources. Also, logs server start and + stop. + + - (int) worker_processes: pre-fork a certain number of child + processes before starting. See: :ref:`pre-fork-model`. + Each child process will keep using a 1-thread, async + concurrency model, handling multiple concurrent connections. + If the number is None or <= 0 the number of usable cores + available on this machine is detected and used. + It is a good idea to use this option in case the app risks + blocking for too long on a single function call (e.g. + hard-disk is slow, long DB query on auth etc.). + By splitting the work load over multiple processes the delay + introduced by a blocking function call is amortized and divided + by the number of worker processes. + + *Changed in version 1.0.0*: no longer a classmethod + + *Changed in version 1.0.0*: 'use_poll' and 'count' parameters were removed + + *Changed in version 1.0.0*: 'blocking' and 'handle_exit' parameters were + added .. method:: close() @@ -648,7 +676,7 @@ A modified version of base :class:`pyftpdlib.servers.FTPServer` class which spawns a thread every time a new connection is established. Differently from base FTPServer class, the handler will be free to block without hanging the - whole IO loop. + whole IO loop. See :ref:`changing-the-concurrency-model`. *New in version 1.0.0* @@ -660,7 +688,7 @@ A modified version of base :class:`pyftpdlib.servers.FTPServer` class which spawns a process every time a new connection is established. Differently from base FTPServer class, the handler will be free to block without hanging the - whole IO loop. + whole IO loop. See :ref:`changing-the-concurrency-model`. *New in version 1.0.0* diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyftpdlib-1.5.5/docs/faqs.rst new/pyftpdlib-1.5.6/docs/faqs.rst --- old/pyftpdlib-1.5.5/docs/faqs.rst 2017-12-30 09:44:57.000000000 +0100 +++ new/pyftpdlib-1.5.6/docs/faqs.rst 2019-10-22 06:45:18.000000000 +0200 @@ -90,13 +90,7 @@ Which Python versions are compatible? ------------------------------------- -From *2.6* to *3.4*. -Python 2.4 and 2.5 support has been removed starting from version 0.6.0. -The latest version supporting Python 2.3 is -`pyftpdlib 1.4.0 <https://pypi.python.org/packages/source/p/pyftpdlib/pyftpdlib-1.4.0.tar.gz>`__. -Python 2.3 support has been removed starting from version 0.6.0. The latest -version supporting Python 2.3 is -`pyftpdlib 0.5.2 <https://pypi.python.org/packages/source/p/pyftpdlib/pyftpdlib-0.5.2.tar.gz>`__. +From *2.6* to *3.X*. On which platforms can pyftpdlib be used? ----------------------------------------- diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyftpdlib-1.5.5/docs/tutorial.rst new/pyftpdlib-1.5.6/docs/tutorial.rst --- old/pyftpdlib-1.5.5/docs/tutorial.rst 2017-12-30 09:44:57.000000000 +0100 +++ new/pyftpdlib-1.5.6/docs/tutorial.rst 2019-10-24 12:14:43.000000000 +0200 @@ -281,11 +281,12 @@ if __name__ == "__main__": main() +.. _changing-the-concurrency-model: Changing the concurrency model ============================== -By nature pyftpdlib is asynchronous. This means it uses a single process/thread +By nature pyftpdlib is asynchronous. That means it uses a single process/thread to handle multiple client connections and file transfers. This is why it is so fast, lightweight and scalable (see `benchmarks <benchmarks.html>`__). The async model has one big drawback though: the code cannot contain instructions @@ -297,15 +298,16 @@ filesystem (say a network filesystem such as samba). If the filesystem is slow (say, a ``open(file, 'r').read(8192)`` takes 2 secs to complete) then you are stuck. -Starting from version 1.0.0 pyftpdlib supports 2 new classes which changes the -default concurrency model by introducing multiple threads or processes. In -technical terms this means that every time a client connects a separate -thread/process is spawned and internally it will run its own IO loop. In -practical terms this means that you can block as long as you want. +Starting from version 1.0.0 pyftpdlib can change the concurrency model by using +multiple processes or threads instead. +In technical (internal) terms that means that every time a client connects a +separate thread/process is spawned and internally it will run its own IO loop. +In practical terms this means that you can block as long as you want. Changing the concurrency module is easy: you just need to import a substitute for `FTPServer <api.html#pyftpdlib.servers.FTPServer>`__. class: -Thread-based example: +Multiple threads +^^^^^^^^^^^^^^^^ .. code-block:: python @@ -313,7 +315,6 @@ from pyftpdlib.servers import ThreadedFTPServer # <- from pyftpdlib.authorizers import DummyAuthorizer - def main(): authorizer = DummyAuthorizer() authorizer.add_user('user', '12345', '.') @@ -326,7 +327,8 @@ main() -Multiple process example: +Multiple processes +^^^^^^^^^^^^^^^^^^ .. code-block:: python @@ -334,7 +336,6 @@ from pyftpdlib.servers import MultiprocessFTPServer # <- from pyftpdlib.authorizers import DummyAuthorizer - def main(): authorizer = DummyAuthorizer() authorizer.add_user('user', '12345', '.') @@ -346,7 +347,44 @@ if __name__ == "__main__": main() +.. _pre-fork: + +Pre-fork +^^^^^^^^ + +There also exists a third option (UNIX only): the pre-fork model. +Pre-fork means that a certain number of worker processes are ``spawn()``ed +before starting the server. +Each worker process will keep using a 1-thread, async concurrency model, +handling multiple concurrent connections, but the workload is split. +This way the delay introduced by a blocking function call is amortized and +divided by the number of workers, and thus also the disk I/O latency is +minimized. +Every time a new connection comes in, the parent process will automatically +delegate the connection to one of the subprocesses, so from the app standpoint +this is completely transparent. +As a general rule, it is always a good idea to use this model in production. +The optimal value depends on many factors including (but not limited to) the +number of CPU cores, the number of hard disk drives that store data, and load +pattern. When one is in doubt, setting it to the number of available CPU cores +would be a good start. + +.. code-block:: python + + from pyftpdlib.handlers import FTPHandler + from pyftpdlib.servers import FTPServer + from pyftpdlib.authorizers import DummyAuthorizer + + def main(): + authorizer = DummyAuthorizer() + authorizer.add_user('user', '12345', '.') + handler = FTPHandler + handler.authorizer = authorizer + server = FTPServer(('', 2121), handler) + server.serve_forever(worker_processes=4) # <- + if __name__ == "__main__": + main() Throttle bandwidth ================== @@ -369,7 +407,6 @@ from pyftpdlib.servers import FTPServer from pyftpdlib.authorizers import DummyAuthorizer - def main(): authorizer = DummyAuthorizer() authorizer.add_user('user', '12345', os.getcwd(), perm='elradfmwMT') @@ -421,7 +458,6 @@ from pyftpdlib.authorizers import DummyAuthorizer from pyftpdlib.handlers import TLS_FTPHandler - def main(): authorizer = DummyAuthorizer() authorizer.add_user('user', '12345', '.', perm='elradfmwMT') diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib/__init__.py new/pyftpdlib-1.5.6/pyftpdlib/__init__.py --- old/pyftpdlib-1.5.5/pyftpdlib/__init__.py 2019-03-27 21:53:41.000000000 +0100 +++ new/pyftpdlib-1.5.6/pyftpdlib/__init__.py 2019-10-24 10:28:08.000000000 +0200 @@ -68,6 +68,6 @@ """ -__ver__ = '1.5.5' +__ver__ = '1.5.6' __author__ = "Giampaolo Rodola' <g.rod...@gmail.com>" __web__ = 'https://github.com/giampaolo/pyftpdlib/' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib/_compat.py new/pyftpdlib-1.5.6/pyftpdlib/_compat.py --- old/pyftpdlib-1.5.5/pyftpdlib/_compat.py 2017-12-30 09:44:57.000000000 +0100 +++ new/pyftpdlib-1.5.6/pyftpdlib/_compat.py 2019-10-24 10:26:36.000000000 +0200 @@ -20,6 +20,7 @@ getcwdu = os.getcwd unicode = str xrange = range + long = int else: def u(s): return unicode(s) @@ -30,6 +31,7 @@ getcwdu = os.getcwdu unicode = unicode xrange = xrange + long = long # removed in 3.0, reintroduced in 3.2 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib/filesystems.py new/pyftpdlib-1.5.6/pyftpdlib/filesystems.py --- old/pyftpdlib-1.5.5/pyftpdlib/filesystems.py 2018-08-29 12:40:33.000000000 +0200 +++ new/pyftpdlib-1.5.6/pyftpdlib/filesystems.py 2020-02-16 16:39:33.000000000 +0100 @@ -35,6 +35,22 @@ 7: 'Jul', 8: 'Aug', 9: 'Sep', 10: 'Oct', 11: 'Nov', 12: 'Dec'} +def _memoize(fun): + """A simple memoize decorator for functions supporting (hashable) + positional arguments. + """ + def wrapper(*args, **kwargs): + key = (args, frozenset(sorted(kwargs.items()))) + try: + return cache[key] + except KeyError: + ret = cache[key] = fun(*args, **kwargs) + return ret + + cache = {} + return wrapper + + # =================================================================== # --- custom exceptions # =================================================================== @@ -406,6 +422,14 @@ drwxrwxrwx 1 owner group 0 Aug 31 18:50 e-books -rw-rw-rw- 1 owner group 380 Sep 02 3:40 module.py """ + @_memoize + def get_user_by_uid(uid): + return self.get_user_by_uid(uid) + + @_memoize + def get_group_by_gid(gid): + return self.get_group_by_gid(gid) + assert isinstance(basedir, unicode), basedir if self.cmd_channel.use_gmt_times: timefunc = time.gmtime @@ -441,8 +465,8 @@ if not nlinks: # non-posix system, let's use a bogus value nlinks = 1 size = st.st_size # file size - uname = self.get_user_by_uid(st.st_uid) - gname = self.get_group_by_gid(st.st_gid) + uname = get_user_by_uid(st.st_uid) + gname = get_group_by_gid(st.st_gid) mtime = timefunc(st.st_mtime) # if modification time > 6 months shows "month year" # else "month hh:mm"; this matches proftpd format, see: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib/handlers.py new/pyftpdlib-1.5.6/pyftpdlib/handlers.py --- old/pyftpdlib-1.5.5/pyftpdlib/handlers.py 2019-04-04 01:41:30.000000000 +0200 +++ new/pyftpdlib-1.5.6/pyftpdlib/handlers.py 2019-10-24 10:26:36.000000000 +0200 @@ -54,6 +54,7 @@ from .log import debug from .log import logger + CR_BYTE = ord('\r') diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib/log.py new/pyftpdlib-1.5.6/pyftpdlib/log.py --- old/pyftpdlib-1.5.5/pyftpdlib/log.py 2018-05-15 15:36:28.000000000 +0200 +++ new/pyftpdlib-1.5.6/pyftpdlib/log.py 2019-10-24 10:26:36.000000000 +0200 @@ -40,6 +40,7 @@ # configurable options LEVEL = logging.INFO PREFIX = '[%(levelname)1.1s %(asctime)s]' +PREFIX_MPROC = '[%(levelname)1.1s %(asctime)s %(process)s]' COLOURED = _stderr_supports_color() TIME_FORMAT = "%Y-%m-%d %H:%M:%S" @@ -53,6 +54,7 @@ * Timestamps on every log line. * Robust against str/bytes encoding problems. """ + PREFIX = PREFIX def __init__(self, *args, **kwargs): logging.Formatter.__init__(self, *args, **kwargs) @@ -90,7 +92,7 @@ record.asctime = time.strftime(TIME_FORMAT, self.converter(record.created)) - prefix = PREFIX % record.__dict__ + prefix = self.PREFIX % record.__dict__ if self._coloured: prefix = (self._colors.get(record.levelno, self._normal) + prefix + self._normal) @@ -143,14 +145,16 @@ # TODO: write tests def config_logging(level=LEVEL, prefix=PREFIX, other_loggers=None): # Little speed up - if "%(process)d" not in prefix: + if "(process)" not in prefix: logging.logProcesses = False if "%(processName)s" not in prefix: logging.logMultiprocessing = False if "%(thread)d" not in prefix and "%(threadName)s" not in prefix: logging.logThreads = False handler = logging.StreamHandler() - handler.setFormatter(LogFormatter()) + formatter = LogFormatter() + formatter.PREFIX = prefix + handler.setFormatter(formatter) loggers = [logging.getLogger('pyftpdlib')] if other_loggers is not None: loggers.extend(other_loggers) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib/prefork.py new/pyftpdlib-1.5.6/pyftpdlib/prefork.py --- old/pyftpdlib-1.5.5/pyftpdlib/prefork.py 1970-01-01 01:00:00.000000000 +0100 +++ new/pyftpdlib-1.5.6/pyftpdlib/prefork.py 2019-10-24 10:26:36.000000000 +0200 @@ -0,0 +1,125 @@ +# Copyright (C) 2007 Giampaolo Rodola' <g.rod...@gmail.com>. +# Use of this source code is governed by MIT license that can be +# found in the LICENSE file. + +"""Process utils.""" + +import errno +import os +import sys +import time +from binascii import hexlify +try: + import multiprocessing +except ImportError: + multiprocessing = None + +from ._compat import long +from .log import logger + + +_task_id = None + + +def cpu_count(): + """Returns the number of processors on this machine.""" + if multiprocessing is None: + return 1 + try: + return multiprocessing.cpu_count() + except NotImplementedError: + pass + try: + return os.sysconf("SC_NPROCESSORS_CONF") + except (AttributeError, ValueError): + pass + return 1 + + +def _reseed_random(): + if 'random' not in sys.modules: + return + import random + # If os.urandom is available, this method does the same thing as + # random.seed (at least as of python 2.6). If os.urandom is not + # available, we mix in the pid in addition to a timestamp. + try: + seed = long(hexlify(os.urandom(16)), 16) + except NotImplementedError: + seed = int(time.time() * 1000) ^ os.getpid() + random.seed(seed) + + +def fork_processes(number, max_restarts=100): + """Starts multiple worker processes. + + If *number* is None or <= 0, we detect the number of cores available + on this machine and fork that number of child processes. + If *number* is given and > 0, we fork that specific number of + sub-processes. + + Since we use processes and not threads, there is no shared memory + between any server code. + + In each child process, *fork_processes* returns its *task id*, a + number between 0 and *number*. Processes that exit abnormally + (due to a signal or non-zero exit status) are restarted with the + same id (up to *max_restarts* times). In the parent process, + *fork_processes* returns None if all child processes have exited + normally, but will otherwise only exit by throwing an exception. + """ + global _task_id + assert _task_id is None + if number is None or number <= 0: + number = cpu_count() + logger.info("starting %d pre-fork processes", number) + children = {} + + def start_child(i): + pid = os.fork() + if pid == 0: + # child process + _reseed_random() + global _task_id + _task_id = i + return i + else: + children[pid] = i + return None + + for i in range(number): + id = start_child(i) + if id is not None: + return id + num_restarts = 0 + while children: + try: + pid, status = os.wait() + except OSError as e: + if e.errno == errno.EINTR: + continue + raise + if pid not in children: + continue + id = children.pop(pid) + if os.WIFSIGNALED(status): + logger.warning("child %d (pid %d) killed by signal %d, restarting", + id, pid, os.WTERMSIG(status)) + elif os.WEXITSTATUS(status) != 0: + logger.warning( + "child %d (pid %d) exited with status %d, restarting", + id, pid, os.WEXITSTATUS(status)) + else: + logger.info("child %d (pid %d) exited normally", id, pid) + continue + num_restarts += 1 + if num_restarts > max_restarts: + raise RuntimeError("Too many child restarts, giving up") + new_id = start_child(id) + if new_id is not None: + return new_id + # All child processes exited cleanly, so exit the master process + # instead of just returning to right after the call to + # fork_processes (which will probably just start up another IOLoop + # unless the caller checks the return value). + sys.exit(0) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib/servers.py new/pyftpdlib-1.5.6/pyftpdlib/servers.py --- old/pyftpdlib-1.5.5/pyftpdlib/servers.py 2019-04-04 01:42:29.000000000 +0200 +++ new/pyftpdlib-1.5.6/pyftpdlib/servers.py 2019-10-24 10:26:36.000000000 +0200 @@ -48,6 +48,10 @@ from .log import debug from .log import is_logging_configured from .log import logger +from .log import PREFIX +from .log import PREFIX_MPROC +from .prefork import fork_processes + __all__ = ['FTPServer', 'ThreadedFTPServer'] _BSD = 'bsd' in sys.platform @@ -135,7 +139,7 @@ else: return self._map_len() <= self.max_cons - def _log_start(self): + def _log_start(self, prefork=False): def get_fqname(obj): try: return obj.__module__ + "." + obj.__class__.__name__ @@ -149,29 +153,25 @@ # If we get to this point it means the user hasn't # configured any logger. We want logging to be on # by default (stderr). - config_logging() + config_logging(prefix=PREFIX_MPROC if prefork else PREFIX) if self.handler.passive_ports: pasv_ports = "%s->%s" % (self.handler.passive_ports[0], self.handler.passive_ports[-1]) else: pasv_ports = None - addr = self.address - if hasattr(self.handler, 'ssl_protocol'): - proto = "FTP+SSL" - else: - proto = "FTP" - logger.info(">>> starting %s server on %s:%s, pid=%i <<<" - % (proto, addr[0], addr[1], os.getpid())) + model = 'prefork + ' if prefork else '' if ('ThreadedFTPServer' in __all__ and issubclass(self.__class__, ThreadedFTPServer)): - logger.info("concurrency model: multi-thread") + model += 'multi-thread' elif ('MultiprocessFTPServer' in __all__ and issubclass(self.__class__, MultiprocessFTPServer)): - logger.info("concurrency model: multi-process") + model += 'multi-process' elif issubclass(self.__class__, FTPServer): - logger.info("concurrency model: async") - + model += 'async' + else: + model += 'unknown (custom class)' + logger.info("concurrency model: " + model) logger.info("masquerade (NAT) address: %s", self.handler.masquerade_address) logger.info("passive ports: %s", pasv_ports) @@ -191,7 +191,8 @@ if getattr(self.handler, 'keyfile', None): logger.debug("SSL keyfile: %r", self.handler.keyfile) - def serve_forever(self, timeout=None, blocking=True, handle_exit=True): + def serve_forever(self, timeout=None, blocking=True, handle_exit=True, + worker_processes=1): """Start serving. - (float) timeout: the timeout passed to the underlying IO @@ -205,11 +206,41 @@ SystemExit exceptions (generally caused by SIGTERM / SIGINT signals) and gracefully exits after cleaning up resources. Also, logs server start and stop. + + - (int) worker_processes: pre-fork a certain number of child + processes before starting. + Each child process will keep using a 1-thread, async + concurrency model, handling multiple concurrent connections. + If the number is None or <= 0 the number of usable cores + available on this machine is detected and used. + It is a good idea to use this option in case the app risks + blocking for too long on a single function call (e.g. + hard-disk is slow, long DB query on auth etc.). + By splitting the work load over multiple processes the delay + introduced by a blocking function call is amortized and divided + by the number of worker processes. """ - if handle_exit: - log = handle_exit and blocking + log = handle_exit and blocking + + # + if worker_processes != 1 and os.name == 'posix': + if not blocking: + raise ValueError( + "'worker_processes' and 'blocking' are mutually exclusive") + if log: + self._log_start(prefork=True) + fork_processes(worker_processes) + else: if log: self._log_start() + + # + proto = "FTP+SSL" if hasattr(self.handler, 'ssl_protocol') else "FTP" + logger.info(">>> starting %s server on %s:%s, pid=%i <<<" + % (proto, self.address[0], self.address[1], os.getpid())) + + # + if handle_exit: try: self.ioloop.loop(timeout, blocking) except (KeyboardInterrupt, SystemExit): @@ -217,9 +248,8 @@ if blocking: if log: logger.info( - ">>> shutting down FTP server (%s active socket " - "fds) <<<", - self._map_len()) + ">>> shutting down FTP server, %s socket(s), pid=%i " + "<<<", self._map_len(), os.getpid()) self.close_all() else: self.ioloop.loop(timeout, blocking) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib/test/__init__.py new/pyftpdlib-1.5.6/pyftpdlib/test/__init__.py --- old/pyftpdlib-1.5.5/pyftpdlib/test/__init__.py 2018-05-15 14:14:07.000000000 +0200 +++ new/pyftpdlib-1.5.6/pyftpdlib/test/__init__.py 2020-01-25 17:17:26.000000000 +0100 @@ -2,6 +2,7 @@ # Use of this source code is governed by MIT license that can be # found in the LICENSE file. +from __future__ import print_function import contextlib import errno import functools @@ -22,6 +23,7 @@ from pyftpdlib._compat import getcwdu from pyftpdlib._compat import u from pyftpdlib.authorizers import DummyAuthorizer +from pyftpdlib.handlers import _import_sendfile from pyftpdlib.handlers import FTPHandler from pyftpdlib.ioloop import IOLoop from pyftpdlib.servers import FTPServer @@ -36,10 +38,7 @@ if not hasattr(unittest.TestCase, "assertRaisesRegex"): unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp -if os.name == 'posix': - import sendfile -else: - sendfile = None +sendfile = _import_sendfile() # Attempt to use IP rather than hostname (test suite will run a lot faster) @@ -255,7 +254,11 @@ assert len(ts) == 1, ts p = psutil.Process() children = p.children() - assert not children, children + if children: + for p in children: + p.kill() + p.wait(1) + assert not children, children cons = [x for x in p.connections('tcp') if x.status != psutil.CONN_CLOSE_WAIT] assert not cons, cons diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib/test/test_functional.py new/pyftpdlib-1.5.6/pyftpdlib/test/test_functional.py --- old/pyftpdlib-1.5.5/pyftpdlib/test/test_functional.py 2019-03-27 17:31:43.000000000 +0100 +++ new/pyftpdlib-1.5.6/pyftpdlib/test/test_functional.py 2019-10-24 10:26:36.000000000 +0200 @@ -23,6 +23,7 @@ from pyftpdlib._compat import PY3 from pyftpdlib._compat import u from pyftpdlib.filesystems import AbstractedFS +from pyftpdlib.handlers import _import_sendfile from pyftpdlib.handlers import DTPHandler from pyftpdlib.handlers import FTPHandler from pyftpdlib.handlers import SUPPORTS_HYBRID_IPV6 @@ -66,10 +67,8 @@ import ssl -if POSIX: - import sendfile -else: - sendfile = None + +sendfile = _import_sendfile() class TestFtpAuthentication(unittest.TestCase): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib.egg-info/PKG-INFO new/pyftpdlib-1.5.6/pyftpdlib.egg-info/PKG-INFO --- old/pyftpdlib-1.5.5/pyftpdlib.egg-info/PKG-INFO 2019-04-04 11:11:03.000000000 +0200 +++ new/pyftpdlib-1.5.6/pyftpdlib.egg-info/PKG-INFO 2020-02-16 16:43:47.000000000 +0100 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: pyftpdlib -Version: 1.5.5 +Version: 1.5.6 Summary: Very fast asynchronous FTP server library Home-page: https://github.com/giampaolo/pyftpdlib/ Author: Giampaolo Rodola' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib.egg-info/SOURCES.txt new/pyftpdlib-1.5.6/pyftpdlib.egg-info/SOURCES.txt --- old/pyftpdlib-1.5.5/pyftpdlib.egg-info/SOURCES.txt 2019-04-04 11:11:03.000000000 +0200 +++ new/pyftpdlib-1.5.6/pyftpdlib.egg-info/SOURCES.txt 2020-02-16 16:43:47.000000000 +0100 @@ -42,6 +42,7 @@ pyftpdlib/handlers.py pyftpdlib/ioloop.py pyftpdlib/log.py +pyftpdlib/prefork.py pyftpdlib/servers.py pyftpdlib.egg-info/PKG-INFO pyftpdlib.egg-info/SOURCES.txt