Hello community,
here is the log from the commit of package python-pyftpdlib for
openSUSE:Leap:15.2 checked in at 2020-03-20 05:15:04
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Leap:15.2/python-pyftpdlib (Old)
and /work/SRC/openSUSE:Leap:15.2/.python-pyftpdlib.new.3160 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-pyftpdlib"
Fri Mar 20 05:15:04 2020 rev:5 rq:786357 version:1.5.6
Changes:
--------
--- /work/SRC/openSUSE:Leap:15.2/python-pyftpdlib/python-pyftpdlib.changes
2020-03-02 13:20:50.446098230 +0100
+++
/work/SRC/openSUSE:Leap:15.2/.python-pyftpdlib.new.3160/python-pyftpdlib.changes
2020-03-20 05:15:12.178559997 +0100
@@ -1,0 +2,8 @@
+Wed Mar 18 09:21:28 UTC 2020 - Tomáš Chvátal <[email protected]>
+
+- Update to 1.5.6:
+ - #467: added pre-fork concurrency model, spawn()ing worker processes to
split
+ load.
+ - #520: directory LISTing is now 3.7x times faster.
+
+-------------------------------------------------------------------
Old:
----
pyftpdlib-1.5.5.tar.gz
New:
----
pyftpdlib-1.5.6.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python-pyftpdlib.spec ++++++
--- /var/tmp/diff_new_pack.H4v6Lq/_old 2020-03-20 05:15:12.526560231 +0100
+++ /var/tmp/diff_new_pack.H4v6Lq/_new 2020-03-20 05:15:12.530560233 +0100
@@ -1,7 +1,7 @@
#
# spec file for package python-pyftpdlib
#
-# Copyright (c) 2019 SUSE LINUX GmbH, Nuernberg, Germany.
+# Copyright (c) 2020 SUSE LLC
# Copyright (c) 2016 LISA GmbH, Bingen, Germany.
#
# All modifications and additions to the file contributed by third parties
@@ -21,7 +21,7 @@
# Tests randomly fail: https://github.com/giampaolo/pyftpdlib/issues/386
%bcond_with test
Name: python-pyftpdlib
-Version: 1.5.5
+Version: 1.5.6
Release: 0
Summary: Asynchronous FTP server library for Python
License: MIT
@@ -57,6 +57,7 @@
%install
%python_install
+%python_expand rm -r %{buildroot}%{$python_sitelib}/pyftpdlib/test
%python_expand %fdupes %{buildroot}%{$python_sitelib}
%if %{with test}
++++++ pyftpdlib-1.5.5.tar.gz -> pyftpdlib-1.5.6.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/pyftpdlib-1.5.5/HISTORY.rst
new/pyftpdlib-1.5.6/HISTORY.rst
--- old/pyftpdlib-1.5.5/HISTORY.rst 2019-04-04 11:10:16.000000000 +0200
+++ new/pyftpdlib-1.5.6/HISTORY.rst 2020-02-16 16:39:33.000000000 +0100
@@ -1,5 +1,14 @@
Bug tracker at https://github.com/giampaolo/pyftpdlib/issues
+Version: 1.5.6 - 2020-02-16
+===========================
+
+**Enhancements**
+
+- #467: added pre-fork concurrency model, spawn()ing worker processes to split
+ load.
+- #520: directory LISTing is now 3.7x times faster.
+
Version: 1.5.5 - 2019-04-04
===========================
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/pyftpdlib-1.5.5/LICENSE new/pyftpdlib-1.5.6/LICENSE
--- old/pyftpdlib-1.5.5/LICENSE 2017-12-30 09:44:57.000000000 +0100
+++ new/pyftpdlib-1.5.6/LICENSE 2019-10-22 06:45:18.000000000 +0200
@@ -1,22 +1,21 @@
-======================================================================
-Copyright (C) 2007-2016 Giampaolo Rodola' <[email protected]>
+MIT License
- All Rights Reserved
+Copyright (c) 2007 Giampaolo Rodola'
-Permission to use, copy, modify, and distribute this software and
-its documentation for any purpose and without fee is hereby
-granted, provided that the above copyright notice appear in all
-copies and that both that copyright notice and this permission
-notice appear in supporting documentation, and that the name of
-Giampaolo Rodola' not be used in advertising or publicity pertaining to
-distribution of the software without specific, written prior
-permission.
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
-Giampaolo Rodola' DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
-INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
-NO EVENT Giampaolo Rodola' BE LIABLE FOR ANY SPECIAL, INDIRECT OR
-CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
-OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
-NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
-CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-======================================================================
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/pyftpdlib-1.5.5/Makefile new/pyftpdlib-1.5.6/Makefile
--- old/pyftpdlib-1.5.5/Makefile 2019-03-27 17:32:12.000000000 +0100
+++ new/pyftpdlib-1.5.6/Makefile 2019-10-24 10:26:36.000000000 +0200
@@ -6,6 +6,7 @@
TSCRIPT = pyftpdlib/test/runner.py
ARGS =
DEV_DEPS = \
+ cffi \
check-manifest \
coverage \
flake8 \
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/pyftpdlib-1.5.5/PKG-INFO new/pyftpdlib-1.5.6/PKG-INFO
--- old/pyftpdlib-1.5.5/PKG-INFO 2019-04-04 11:11:03.000000000 +0200
+++ new/pyftpdlib-1.5.6/PKG-INFO 2020-02-16 16:43:47.000000000 +0100
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: pyftpdlib
-Version: 1.5.5
+Version: 1.5.6
Summary: Very fast asynchronous FTP server library
Home-page: https://github.com/giampaolo/pyftpdlib/
Author: Giampaolo Rodola'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/pyftpdlib-1.5.5/docs/api.rst
new/pyftpdlib-1.5.6/docs/api.rst
--- old/pyftpdlib-1.5.5/docs/api.rst 2018-05-15 17:45:22.000000000 +0200
+++ new/pyftpdlib-1.5.6/docs/api.rst 2019-10-24 10:26:36.000000000 +0200
@@ -379,13 +379,41 @@
Number of maximum connections accepted for the same IP address (default
``0`` == no limit).
- .. method:: serve_forever(timeout=None, blocking=True, handle_exit=True)
+ .. method:: serve_forever(timeout=None, blocking=True, handle_exit=True,
worker_processes=1)
Starts the asynchronous IO loop.
- *Changed in version 1.0.0: no longer a classmethod; 'use_poll' and 'count'
- *parameters were removed. 'blocking' and 'handle_exit' parameters were
- *added*
+ - (float) timeout: the timeout passed to the underlying IO
+ loop expressed in seconds.
+
+ - (bool) blocking: if False loop once and then return the
+ timeout of the next scheduled call next to expire soonest
+ (if any).
+
+ - (bool) handle_exit: when True catches ``KeyboardInterrupt`` and
+ ``SystemExit`` exceptions (caused by SIGTERM / SIGINT signals) and
+ gracefully exits after cleaning up resources. Also, logs server start and
+ stop.
+
+ - (int) worker_processes: pre-fork a certain number of child
+ processes before starting. See: :ref:`pre-fork-model`.
+ Each child process will keep using a 1-thread, async
+ concurrency model, handling multiple concurrent connections.
+ If the number is None or <= 0 the number of usable cores
+ available on this machine is detected and used.
+ It is a good idea to use this option in case the app risks
+ blocking for too long on a single function call (e.g.
+ hard-disk is slow, long DB query on auth etc.).
+ By splitting the work load over multiple processes the delay
+ introduced by a blocking function call is amortized and divided
+ by the number of worker processes.
+
+ *Changed in version 1.0.0*: no longer a classmethod
+
+ *Changed in version 1.0.0*: 'use_poll' and 'count' parameters were removed
+
+ *Changed in version 1.0.0*: 'blocking' and 'handle_exit' parameters were
+ added
.. method:: close()
@@ -648,7 +676,7 @@
A modified version of base :class:`pyftpdlib.servers.FTPServer` class which
spawns a thread every time a new connection is established. Differently from
base FTPServer class, the handler will be free to block without hanging the
- whole IO loop.
+ whole IO loop. See :ref:`changing-the-concurrency-model`.
*New in version 1.0.0*
@@ -660,7 +688,7 @@
A modified version of base :class:`pyftpdlib.servers.FTPServer` class which
spawns a process every time a new connection is established. Differently from
base FTPServer class, the handler will be free to block without hanging the
- whole IO loop.
+ whole IO loop. See :ref:`changing-the-concurrency-model`.
*New in version 1.0.0*
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/pyftpdlib-1.5.5/docs/faqs.rst
new/pyftpdlib-1.5.6/docs/faqs.rst
--- old/pyftpdlib-1.5.5/docs/faqs.rst 2017-12-30 09:44:57.000000000 +0100
+++ new/pyftpdlib-1.5.6/docs/faqs.rst 2019-10-22 06:45:18.000000000 +0200
@@ -90,13 +90,7 @@
Which Python versions are compatible?
-------------------------------------
-From *2.6* to *3.4*.
-Python 2.4 and 2.5 support has been removed starting from version 0.6.0.
-The latest version supporting Python 2.3 is
-`pyftpdlib 1.4.0
<https://pypi.python.org/packages/source/p/pyftpdlib/pyftpdlib-1.4.0.tar.gz>`__.
-Python 2.3 support has been removed starting from version 0.6.0. The latest
-version supporting Python 2.3 is
-`pyftpdlib 0.5.2
<https://pypi.python.org/packages/source/p/pyftpdlib/pyftpdlib-0.5.2.tar.gz>`__.
+From *2.6* to *3.X*.
On which platforms can pyftpdlib be used?
-----------------------------------------
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/pyftpdlib-1.5.5/docs/tutorial.rst
new/pyftpdlib-1.5.6/docs/tutorial.rst
--- old/pyftpdlib-1.5.5/docs/tutorial.rst 2017-12-30 09:44:57.000000000
+0100
+++ new/pyftpdlib-1.5.6/docs/tutorial.rst 2019-10-24 12:14:43.000000000
+0200
@@ -281,11 +281,12 @@
if __name__ == "__main__":
main()
+.. _changing-the-concurrency-model:
Changing the concurrency model
==============================
-By nature pyftpdlib is asynchronous. This means it uses a single process/thread
+By nature pyftpdlib is asynchronous. That means it uses a single process/thread
to handle multiple client connections and file transfers. This is why it is so
fast, lightweight and scalable (see `benchmarks <benchmarks.html>`__). The
async model has one big drawback though: the code cannot contain instructions
@@ -297,15 +298,16 @@
filesystem (say a network filesystem such as samba). If the filesystem is slow
(say, a ``open(file, 'r').read(8192)`` takes 2 secs to complete) then you are
stuck.
-Starting from version 1.0.0 pyftpdlib supports 2 new classes which changes the
-default concurrency model by introducing multiple threads or processes. In
-technical terms this means that every time a client connects a separate
-thread/process is spawned and internally it will run its own IO loop. In
-practical terms this means that you can block as long as you want.
+Starting from version 1.0.0 pyftpdlib can change the concurrency model by using
+multiple processes or threads instead.
+In technical (internal) terms that means that every time a client connects a
+separate thread/process is spawned and internally it will run its own IO loop.
+In practical terms this means that you can block as long as you want.
Changing the concurrency module is easy: you just need to import a substitute
for `FTPServer <api.html#pyftpdlib.servers.FTPServer>`__. class:
-Thread-based example:
+Multiple threads
+^^^^^^^^^^^^^^^^
.. code-block:: python
@@ -313,7 +315,6 @@
from pyftpdlib.servers import ThreadedFTPServer # <-
from pyftpdlib.authorizers import DummyAuthorizer
-
def main():
authorizer = DummyAuthorizer()
authorizer.add_user('user', '12345', '.')
@@ -326,7 +327,8 @@
main()
-Multiple process example:
+Multiple processes
+^^^^^^^^^^^^^^^^^^
.. code-block:: python
@@ -334,7 +336,6 @@
from pyftpdlib.servers import MultiprocessFTPServer # <-
from pyftpdlib.authorizers import DummyAuthorizer
-
def main():
authorizer = DummyAuthorizer()
authorizer.add_user('user', '12345', '.')
@@ -346,7 +347,44 @@
if __name__ == "__main__":
main()
+.. _pre-fork:
+
+Pre-fork
+^^^^^^^^
+
+There also exists a third option (UNIX only): the pre-fork model.
+Pre-fork means that a certain number of worker processes are ``spawn()``ed
+before starting the server.
+Each worker process will keep using a 1-thread, async concurrency model,
+handling multiple concurrent connections, but the workload is split.
+This way the delay introduced by a blocking function call is amortized and
+divided by the number of workers, and thus also the disk I/O latency is
+minimized.
+Every time a new connection comes in, the parent process will automatically
+delegate the connection to one of the subprocesses, so from the app standpoint
+this is completely transparent.
+As a general rule, it is always a good idea to use this model in production.
+The optimal value depends on many factors including (but not limited to) the
+number of CPU cores, the number of hard disk drives that store data, and load
+pattern. When one is in doubt, setting it to the number of available CPU cores
+would be a good start.
+
+.. code-block:: python
+
+ from pyftpdlib.handlers import FTPHandler
+ from pyftpdlib.servers import FTPServer
+ from pyftpdlib.authorizers import DummyAuthorizer
+
+ def main():
+ authorizer = DummyAuthorizer()
+ authorizer.add_user('user', '12345', '.')
+ handler = FTPHandler
+ handler.authorizer = authorizer
+ server = FTPServer(('', 2121), handler)
+ server.serve_forever(worker_processes=4) # <-
+ if __name__ == "__main__":
+ main()
Throttle bandwidth
==================
@@ -369,7 +407,6 @@
from pyftpdlib.servers import FTPServer
from pyftpdlib.authorizers import DummyAuthorizer
-
def main():
authorizer = DummyAuthorizer()
authorizer.add_user('user', '12345', os.getcwd(), perm='elradfmwMT')
@@ -421,7 +458,6 @@
from pyftpdlib.authorizers import DummyAuthorizer
from pyftpdlib.handlers import TLS_FTPHandler
-
def main():
authorizer = DummyAuthorizer()
authorizer.add_user('user', '12345', '.', perm='elradfmwMT')
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib/__init__.py
new/pyftpdlib-1.5.6/pyftpdlib/__init__.py
--- old/pyftpdlib-1.5.5/pyftpdlib/__init__.py 2019-03-27 21:53:41.000000000
+0100
+++ new/pyftpdlib-1.5.6/pyftpdlib/__init__.py 2019-10-24 10:28:08.000000000
+0200
@@ -68,6 +68,6 @@
"""
-__ver__ = '1.5.5'
+__ver__ = '1.5.6'
__author__ = "Giampaolo Rodola' <[email protected]>"
__web__ = 'https://github.com/giampaolo/pyftpdlib/'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib/_compat.py
new/pyftpdlib-1.5.6/pyftpdlib/_compat.py
--- old/pyftpdlib-1.5.5/pyftpdlib/_compat.py 2017-12-30 09:44:57.000000000
+0100
+++ new/pyftpdlib-1.5.6/pyftpdlib/_compat.py 2019-10-24 10:26:36.000000000
+0200
@@ -20,6 +20,7 @@
getcwdu = os.getcwd
unicode = str
xrange = range
+ long = int
else:
def u(s):
return unicode(s)
@@ -30,6 +31,7 @@
getcwdu = os.getcwdu
unicode = unicode
xrange = xrange
+ long = long
# removed in 3.0, reintroduced in 3.2
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib/filesystems.py
new/pyftpdlib-1.5.6/pyftpdlib/filesystems.py
--- old/pyftpdlib-1.5.5/pyftpdlib/filesystems.py 2018-08-29
12:40:33.000000000 +0200
+++ new/pyftpdlib-1.5.6/pyftpdlib/filesystems.py 2020-02-16
16:39:33.000000000 +0100
@@ -35,6 +35,22 @@
7: 'Jul', 8: 'Aug', 9: 'Sep', 10: 'Oct', 11: 'Nov', 12: 'Dec'}
+def _memoize(fun):
+ """A simple memoize decorator for functions supporting (hashable)
+ positional arguments.
+ """
+ def wrapper(*args, **kwargs):
+ key = (args, frozenset(sorted(kwargs.items())))
+ try:
+ return cache[key]
+ except KeyError:
+ ret = cache[key] = fun(*args, **kwargs)
+ return ret
+
+ cache = {}
+ return wrapper
+
+
# ===================================================================
# --- custom exceptions
# ===================================================================
@@ -406,6 +422,14 @@
drwxrwxrwx 1 owner group 0 Aug 31 18:50 e-books
-rw-rw-rw- 1 owner group 380 Sep 02 3:40 module.py
"""
+ @_memoize
+ def get_user_by_uid(uid):
+ return self.get_user_by_uid(uid)
+
+ @_memoize
+ def get_group_by_gid(gid):
+ return self.get_group_by_gid(gid)
+
assert isinstance(basedir, unicode), basedir
if self.cmd_channel.use_gmt_times:
timefunc = time.gmtime
@@ -441,8 +465,8 @@
if not nlinks: # non-posix system, let's use a bogus value
nlinks = 1
size = st.st_size # file size
- uname = self.get_user_by_uid(st.st_uid)
- gname = self.get_group_by_gid(st.st_gid)
+ uname = get_user_by_uid(st.st_uid)
+ gname = get_group_by_gid(st.st_gid)
mtime = timefunc(st.st_mtime)
# if modification time > 6 months shows "month year"
# else "month hh:mm"; this matches proftpd format, see:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib/handlers.py
new/pyftpdlib-1.5.6/pyftpdlib/handlers.py
--- old/pyftpdlib-1.5.5/pyftpdlib/handlers.py 2019-04-04 01:41:30.000000000
+0200
+++ new/pyftpdlib-1.5.6/pyftpdlib/handlers.py 2019-10-24 10:26:36.000000000
+0200
@@ -54,6 +54,7 @@
from .log import debug
from .log import logger
+
CR_BYTE = ord('\r')
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib/log.py
new/pyftpdlib-1.5.6/pyftpdlib/log.py
--- old/pyftpdlib-1.5.5/pyftpdlib/log.py 2018-05-15 15:36:28.000000000
+0200
+++ new/pyftpdlib-1.5.6/pyftpdlib/log.py 2019-10-24 10:26:36.000000000
+0200
@@ -40,6 +40,7 @@
# configurable options
LEVEL = logging.INFO
PREFIX = '[%(levelname)1.1s %(asctime)s]'
+PREFIX_MPROC = '[%(levelname)1.1s %(asctime)s %(process)s]'
COLOURED = _stderr_supports_color()
TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
@@ -53,6 +54,7 @@
* Timestamps on every log line.
* Robust against str/bytes encoding problems.
"""
+ PREFIX = PREFIX
def __init__(self, *args, **kwargs):
logging.Formatter.__init__(self, *args, **kwargs)
@@ -90,7 +92,7 @@
record.asctime = time.strftime(TIME_FORMAT,
self.converter(record.created))
- prefix = PREFIX % record.__dict__
+ prefix = self.PREFIX % record.__dict__
if self._coloured:
prefix = (self._colors.get(record.levelno, self._normal) +
prefix + self._normal)
@@ -143,14 +145,16 @@
# TODO: write tests
def config_logging(level=LEVEL, prefix=PREFIX, other_loggers=None):
# Little speed up
- if "%(process)d" not in prefix:
+ if "(process)" not in prefix:
logging.logProcesses = False
if "%(processName)s" not in prefix:
logging.logMultiprocessing = False
if "%(thread)d" not in prefix and "%(threadName)s" not in prefix:
logging.logThreads = False
handler = logging.StreamHandler()
- handler.setFormatter(LogFormatter())
+ formatter = LogFormatter()
+ formatter.PREFIX = prefix
+ handler.setFormatter(formatter)
loggers = [logging.getLogger('pyftpdlib')]
if other_loggers is not None:
loggers.extend(other_loggers)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib/prefork.py
new/pyftpdlib-1.5.6/pyftpdlib/prefork.py
--- old/pyftpdlib-1.5.5/pyftpdlib/prefork.py 1970-01-01 01:00:00.000000000
+0100
+++ new/pyftpdlib-1.5.6/pyftpdlib/prefork.py 2019-10-24 10:26:36.000000000
+0200
@@ -0,0 +1,125 @@
+# Copyright (C) 2007 Giampaolo Rodola' <[email protected]>.
+# Use of this source code is governed by MIT license that can be
+# found in the LICENSE file.
+
+"""Process utils."""
+
+import errno
+import os
+import sys
+import time
+from binascii import hexlify
+try:
+ import multiprocessing
+except ImportError:
+ multiprocessing = None
+
+from ._compat import long
+from .log import logger
+
+
+_task_id = None
+
+
+def cpu_count():
+ """Returns the number of processors on this machine."""
+ if multiprocessing is None:
+ return 1
+ try:
+ return multiprocessing.cpu_count()
+ except NotImplementedError:
+ pass
+ try:
+ return os.sysconf("SC_NPROCESSORS_CONF")
+ except (AttributeError, ValueError):
+ pass
+ return 1
+
+
+def _reseed_random():
+ if 'random' not in sys.modules:
+ return
+ import random
+ # If os.urandom is available, this method does the same thing as
+ # random.seed (at least as of python 2.6). If os.urandom is not
+ # available, we mix in the pid in addition to a timestamp.
+ try:
+ seed = long(hexlify(os.urandom(16)), 16)
+ except NotImplementedError:
+ seed = int(time.time() * 1000) ^ os.getpid()
+ random.seed(seed)
+
+
+def fork_processes(number, max_restarts=100):
+ """Starts multiple worker processes.
+
+ If *number* is None or <= 0, we detect the number of cores available
+ on this machine and fork that number of child processes.
+ If *number* is given and > 0, we fork that specific number of
+ sub-processes.
+
+ Since we use processes and not threads, there is no shared memory
+ between any server code.
+
+ In each child process, *fork_processes* returns its *task id*, a
+ number between 0 and *number*. Processes that exit abnormally
+ (due to a signal or non-zero exit status) are restarted with the
+ same id (up to *max_restarts* times). In the parent process,
+ *fork_processes* returns None if all child processes have exited
+ normally, but will otherwise only exit by throwing an exception.
+ """
+ global _task_id
+ assert _task_id is None
+ if number is None or number <= 0:
+ number = cpu_count()
+ logger.info("starting %d pre-fork processes", number)
+ children = {}
+
+ def start_child(i):
+ pid = os.fork()
+ if pid == 0:
+ # child process
+ _reseed_random()
+ global _task_id
+ _task_id = i
+ return i
+ else:
+ children[pid] = i
+ return None
+
+ for i in range(number):
+ id = start_child(i)
+ if id is not None:
+ return id
+ num_restarts = 0
+ while children:
+ try:
+ pid, status = os.wait()
+ except OSError as e:
+ if e.errno == errno.EINTR:
+ continue
+ raise
+ if pid not in children:
+ continue
+ id = children.pop(pid)
+ if os.WIFSIGNALED(status):
+ logger.warning("child %d (pid %d) killed by signal %d, restarting",
+ id, pid, os.WTERMSIG(status))
+ elif os.WEXITSTATUS(status) != 0:
+ logger.warning(
+ "child %d (pid %d) exited with status %d, restarting",
+ id, pid, os.WEXITSTATUS(status))
+ else:
+ logger.info("child %d (pid %d) exited normally", id, pid)
+ continue
+ num_restarts += 1
+ if num_restarts > max_restarts:
+ raise RuntimeError("Too many child restarts, giving up")
+ new_id = start_child(id)
+ if new_id is not None:
+ return new_id
+ # All child processes exited cleanly, so exit the master process
+ # instead of just returning to right after the call to
+ # fork_processes (which will probably just start up another IOLoop
+ # unless the caller checks the return value).
+ sys.exit(0)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib/servers.py
new/pyftpdlib-1.5.6/pyftpdlib/servers.py
--- old/pyftpdlib-1.5.5/pyftpdlib/servers.py 2019-04-04 01:42:29.000000000
+0200
+++ new/pyftpdlib-1.5.6/pyftpdlib/servers.py 2019-10-24 10:26:36.000000000
+0200
@@ -48,6 +48,10 @@
from .log import debug
from .log import is_logging_configured
from .log import logger
+from .log import PREFIX
+from .log import PREFIX_MPROC
+from .prefork import fork_processes
+
__all__ = ['FTPServer', 'ThreadedFTPServer']
_BSD = 'bsd' in sys.platform
@@ -135,7 +139,7 @@
else:
return self._map_len() <= self.max_cons
- def _log_start(self):
+ def _log_start(self, prefork=False):
def get_fqname(obj):
try:
return obj.__module__ + "." + obj.__class__.__name__
@@ -149,29 +153,25 @@
# If we get to this point it means the user hasn't
# configured any logger. We want logging to be on
# by default (stderr).
- config_logging()
+ config_logging(prefix=PREFIX_MPROC if prefork else PREFIX)
if self.handler.passive_ports:
pasv_ports = "%s->%s" % (self.handler.passive_ports[0],
self.handler.passive_ports[-1])
else:
pasv_ports = None
- addr = self.address
- if hasattr(self.handler, 'ssl_protocol'):
- proto = "FTP+SSL"
- else:
- proto = "FTP"
- logger.info(">>> starting %s server on %s:%s, pid=%i <<<"
- % (proto, addr[0], addr[1], os.getpid()))
+ model = 'prefork + ' if prefork else ''
if ('ThreadedFTPServer' in __all__ and
issubclass(self.__class__, ThreadedFTPServer)):
- logger.info("concurrency model: multi-thread")
+ model += 'multi-thread'
elif ('MultiprocessFTPServer' in __all__ and
issubclass(self.__class__, MultiprocessFTPServer)):
- logger.info("concurrency model: multi-process")
+ model += 'multi-process'
elif issubclass(self.__class__, FTPServer):
- logger.info("concurrency model: async")
-
+ model += 'async'
+ else:
+ model += 'unknown (custom class)'
+ logger.info("concurrency model: " + model)
logger.info("masquerade (NAT) address: %s",
self.handler.masquerade_address)
logger.info("passive ports: %s", pasv_ports)
@@ -191,7 +191,8 @@
if getattr(self.handler, 'keyfile', None):
logger.debug("SSL keyfile: %r", self.handler.keyfile)
- def serve_forever(self, timeout=None, blocking=True, handle_exit=True):
+ def serve_forever(self, timeout=None, blocking=True, handle_exit=True,
+ worker_processes=1):
"""Start serving.
- (float) timeout: the timeout passed to the underlying IO
@@ -205,11 +206,41 @@
SystemExit exceptions (generally caused by SIGTERM / SIGINT
signals) and gracefully exits after cleaning up resources.
Also, logs server start and stop.
+
+ - (int) worker_processes: pre-fork a certain number of child
+ processes before starting.
+ Each child process will keep using a 1-thread, async
+ concurrency model, handling multiple concurrent connections.
+ If the number is None or <= 0 the number of usable cores
+ available on this machine is detected and used.
+ It is a good idea to use this option in case the app risks
+ blocking for too long on a single function call (e.g.
+ hard-disk is slow, long DB query on auth etc.).
+ By splitting the work load over multiple processes the delay
+ introduced by a blocking function call is amortized and divided
+ by the number of worker processes.
"""
- if handle_exit:
- log = handle_exit and blocking
+ log = handle_exit and blocking
+
+ #
+ if worker_processes != 1 and os.name == 'posix':
+ if not blocking:
+ raise ValueError(
+ "'worker_processes' and 'blocking' are mutually exclusive")
+ if log:
+ self._log_start(prefork=True)
+ fork_processes(worker_processes)
+ else:
if log:
self._log_start()
+
+ #
+ proto = "FTP+SSL" if hasattr(self.handler, 'ssl_protocol') else "FTP"
+ logger.info(">>> starting %s server on %s:%s, pid=%i <<<"
+ % (proto, self.address[0], self.address[1], os.getpid()))
+
+ #
+ if handle_exit:
try:
self.ioloop.loop(timeout, blocking)
except (KeyboardInterrupt, SystemExit):
@@ -217,9 +248,8 @@
if blocking:
if log:
logger.info(
- ">>> shutting down FTP server (%s active socket "
- "fds) <<<",
- self._map_len())
+ ">>> shutting down FTP server, %s socket(s), pid=%i "
+ "<<<", self._map_len(), os.getpid())
self.close_all()
else:
self.ioloop.loop(timeout, blocking)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib/test/__init__.py
new/pyftpdlib-1.5.6/pyftpdlib/test/__init__.py
--- old/pyftpdlib-1.5.5/pyftpdlib/test/__init__.py 2018-05-15
14:14:07.000000000 +0200
+++ new/pyftpdlib-1.5.6/pyftpdlib/test/__init__.py 2020-01-25
17:17:26.000000000 +0100
@@ -2,6 +2,7 @@
# Use of this source code is governed by MIT license that can be
# found in the LICENSE file.
+from __future__ import print_function
import contextlib
import errno
import functools
@@ -22,6 +23,7 @@
from pyftpdlib._compat import getcwdu
from pyftpdlib._compat import u
from pyftpdlib.authorizers import DummyAuthorizer
+from pyftpdlib.handlers import _import_sendfile
from pyftpdlib.handlers import FTPHandler
from pyftpdlib.ioloop import IOLoop
from pyftpdlib.servers import FTPServer
@@ -36,10 +38,7 @@
if not hasattr(unittest.TestCase, "assertRaisesRegex"):
unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
-if os.name == 'posix':
- import sendfile
-else:
- sendfile = None
+sendfile = _import_sendfile()
# Attempt to use IP rather than hostname (test suite will run a lot faster)
@@ -255,7 +254,11 @@
assert len(ts) == 1, ts
p = psutil.Process()
children = p.children()
- assert not children, children
+ if children:
+ for p in children:
+ p.kill()
+ p.wait(1)
+ assert not children, children
cons = [x for x in p.connections('tcp')
if x.status != psutil.CONN_CLOSE_WAIT]
assert not cons, cons
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib/test/test_functional.py
new/pyftpdlib-1.5.6/pyftpdlib/test/test_functional.py
--- old/pyftpdlib-1.5.5/pyftpdlib/test/test_functional.py 2019-03-27
17:31:43.000000000 +0100
+++ new/pyftpdlib-1.5.6/pyftpdlib/test/test_functional.py 2019-10-24
10:26:36.000000000 +0200
@@ -23,6 +23,7 @@
from pyftpdlib._compat import PY3
from pyftpdlib._compat import u
from pyftpdlib.filesystems import AbstractedFS
+from pyftpdlib.handlers import _import_sendfile
from pyftpdlib.handlers import DTPHandler
from pyftpdlib.handlers import FTPHandler
from pyftpdlib.handlers import SUPPORTS_HYBRID_IPV6
@@ -66,10 +67,8 @@
import ssl
-if POSIX:
- import sendfile
-else:
- sendfile = None
+
+sendfile = _import_sendfile()
class TestFtpAuthentication(unittest.TestCase):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib.egg-info/PKG-INFO
new/pyftpdlib-1.5.6/pyftpdlib.egg-info/PKG-INFO
--- old/pyftpdlib-1.5.5/pyftpdlib.egg-info/PKG-INFO 2019-04-04
11:11:03.000000000 +0200
+++ new/pyftpdlib-1.5.6/pyftpdlib.egg-info/PKG-INFO 2020-02-16
16:43:47.000000000 +0100
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: pyftpdlib
-Version: 1.5.5
+Version: 1.5.6
Summary: Very fast asynchronous FTP server library
Home-page: https://github.com/giampaolo/pyftpdlib/
Author: Giampaolo Rodola'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/pyftpdlib-1.5.5/pyftpdlib.egg-info/SOURCES.txt
new/pyftpdlib-1.5.6/pyftpdlib.egg-info/SOURCES.txt
--- old/pyftpdlib-1.5.5/pyftpdlib.egg-info/SOURCES.txt 2019-04-04
11:11:03.000000000 +0200
+++ new/pyftpdlib-1.5.6/pyftpdlib.egg-info/SOURCES.txt 2020-02-16
16:43:47.000000000 +0100
@@ -42,6 +42,7 @@
pyftpdlib/handlers.py
pyftpdlib/ioloop.py
pyftpdlib/log.py
+pyftpdlib/prefork.py
pyftpdlib/servers.py
pyftpdlib.egg-info/PKG-INFO
pyftpdlib.egg-info/SOURCES.txt
