This is an automated email from the ASF dual-hosted git repository. not-in-ldap pushed a commit to branch aevri/win32_minimal_seemstowork_20190829 in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 00ed4dbb100493fe3ab172b6eff35e7fa6ecef09 Author: Angelos Evripiotis <[email protected]> AuthorDate: Tue Jun 25 15:46:37 2019 +0100 WIP: mv multiprocessing os-specifics to platform --- src/buildstream/_platform/darwin.py | 12 ++++++++++++ src/buildstream/_platform/platform.py | 4 ++++ src/buildstream/_platform/win32.py | 11 +++++++++-- src/buildstream/_scheduler/jobs/job.py | 5 +++-- src/buildstream/_scheduler/scheduler.py | 4 ---- 5 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/buildstream/_platform/darwin.py b/src/buildstream/_platform/darwin.py index 86820a3..4efa53a 100644 --- a/src/buildstream/_platform/darwin.py +++ b/src/buildstream/_platform/darwin.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see <http://www.gnu.org/licenses/>. +import multiprocessing import os import resource @@ -28,6 +29,12 @@ class Darwin(Platform): # This value comes from OPEN_MAX in syslimits.h OPEN_MAX = 10240 + def __init__(self): + super().__init__() + if None is multiprocessing.get_start_method(allow_none=True): + multiprocessing.set_start_method('spawn') + self._manager = None + def create_sandbox(self, *args, **kwargs): kwargs['dummy_reason'] = \ "OSXFUSE is not supported and there are no supported sandbox " + \ @@ -62,3 +69,8 @@ class Darwin(Platform): old_soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) soft_limit = min(max(self.OPEN_MAX, old_soft_limit), hard_limit) resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit)) + + def make_queue(self): + if self._manager is None: + self._manager = multiprocessing.Manager() + return self._manager.Queue() diff --git a/src/buildstream/_platform/platform.py b/src/buildstream/_platform/platform.py index 42c379e..d0a21a1 100644 --- a/src/buildstream/_platform/platform.py +++ b/src/buildstream/_platform/platform.py @@ -17,6 +17,7 @@ # Authors: # Tristan Maat <[email protected]> +import multiprocessing import os import platform import sys @@ -133,6 +134,9 @@ class Platform(): uname_machine = platform.uname().machine return Platform.canonicalize_arch(uname_machine) + def make_queue(self): + return multiprocessing.Queue() + ################################################################## # Sandbox functions # ################################################################## diff --git a/src/buildstream/_platform/win32.py b/src/buildstream/_platform/win32.py index db70e4a..8844973 100644 --- a/src/buildstream/_platform/win32.py +++ b/src/buildstream/_platform/win32.py @@ -1,4 +1,4 @@ -import os +import multiprocessing from .._exceptions import PlatformError from ..sandbox import SandboxNone @@ -9,8 +9,10 @@ from . import Platform class Win32(Platform): def __init__(self): - super().__init__() + if None is multiprocessing.get_start_method(allow_none=True): + multiprocessing.set_start_method('spawn') + self._manager = None def create_sandbox(self, *args, **kwargs): kwargs['dummy_reason'] = \ @@ -29,3 +31,8 @@ class Win32(Platform): def maximize_open_file_limit(self): pass + + def make_queue(self): + if self._manager is None: + self._manager = multiprocessing.Manager() + return self._manager.Queue() diff --git a/src/buildstream/_scheduler/jobs/job.py b/src/buildstream/_scheduler/jobs/job.py index 9864e4a..f192886 100644 --- a/src/buildstream/_scheduler/jobs/job.py +++ b/src/buildstream/_scheduler/jobs/job.py @@ -33,6 +33,7 @@ import traceback from ..._exceptions import ImplError, BstError, set_last_task_error, SkipJob from ..._message import Message, MessageType, unconditional_messages from ... import _signals, utils +from ..._platform import Platform from .jobpickler import pickle_child_job, unpickle_child_job @@ -184,8 +185,8 @@ class Job(): # Starts the job. # def start(self): - - self._queue = self._scheduler.manager.Queue() + platform = Platform.get_platform() + self._queue = platform.make_queue() self._tries += 1 self._parent_start_listening() diff --git a/src/buildstream/_scheduler/scheduler.py b/src/buildstream/_scheduler/scheduler.py index 7f876b5..86510c6 100644 --- a/src/buildstream/_scheduler/scheduler.py +++ b/src/buildstream/_scheduler/scheduler.py @@ -72,10 +72,6 @@ class Scheduler(): interrupt_callback=None, ticker_callback=None): - import multiprocessing - multiprocessing.set_start_method('spawn') - self.manager = multiprocessing.Manager() - # # Public members #
