#4652: [with code, needs testing] make distutils compile Cython extensions in
parallel
------------------------+---------------------------------------------------
Reporter: craigcitro | Owner: craigcitro
Type: defect | Status: new
Priority: major | Milestone: sage-3.2.2
Component: build | Resolution:
Keywords: |
------------------------+---------------------------------------------------
Comment (by was):
Just out of curiosity, did you try to get this to work without modifying
build_ext.py at all, but by simply changing stuff in that module at
runtime, like I illustrate in #3765?
Here's the diff of what you did:
{{{
--- build_ext.py.orig 2008-11-29 10:32:57.000000000 -0800
+++ build_ext.py 2008-11-29 03:38:35.000000000 -0800
@@ -409,13 +409,57 @@
# get_outputs ()
def build_extensions(self):
+
# First, sanity-check the 'extensions' list
self.check_extensions_list(self.extensions)
- for ext in self.extensions:
- self.build_extension(ext)
+ if not os.environ.has_key('MAKE'):
+ ncpus = 1
+ else:
+ MAKE = os.environ['MAKE']
+ z = [w[2:] for w in MAKE.split() if w.startswith('-j')]
+ if len(z) == 0: # no command line option
+ ncpus = 1
+ else:
+ # Determine number of cpus from command line argument.
+ # Also, use the OS to cap the number of cpus, in case
+ # user annoyingly makes a typo and asks to use 10000
+ # cpus at once.
+ try:
+ ncpus = int(z[0])
+ n = 2*number_of_cpus()
+ if n: # prevent dumb typos.
+ ncpus = min(ncpus, n)
+ except ValueError:
+ ncpus = 1
+
+ import time
+ t = time.time()
+
+ # See if we're trying out the experimental parallel build
+ # code.
+ if ncpus > 1 and os.environ.has_key('SAGE_PARALLEL_DIST'):
+ extensions_to_compile = []
+ for ext in self.extensions:
+ need_to_compile, p = self.prepare_extension(ext)
+ if need_to_compile:
+ extensions_to_compile.append(p)
+
+ if extensions_to_compile:
+ from processing import Pool
+ p = Pool(min(ncpus, len(extensions_to_compile)))
+ for r in p.imap(self.build_extension,
extensions_to_compile):
+ pass
+
+ else:
+ for ext in self.extensions:
+ need_to_compile, p = self.prepare_extension(ext)
+ if need_to_compile:
+ self.build_extension(p)
+
+ print "Total time spent compiling C/C++ extensions: ",
time.time() - t, "seconds."
- def build_extension(self, ext):
+ def prepare_extension(self, ext):
sources = ext.sources
if sources is None or type(sources) not in (ListType, TupleType):
raise DistutilsSetupError, \
@@ -443,9 +487,16 @@
depends = sources + ext.depends
if not (self.force or newer_group(depends, ext_filename,
'newer')):
log.debug("skipping '%s' extension (up-to-date)", ext.name)
- return
+ need_to_compile = False
else:
log.info("building '%s' extension", ext.name)
+ need_to_compile = True
+
+ return need_to_compile, (sources, ext, ext_filename)
+
+ def build_extension(self, p):
+
+ sources, ext, ext_filename = p
# First, scan the sources for SWIG definition files (.i), run
# SWIG on 'em to create .c files, and modify the sources list
@@ -715,3 +766,23 @@
return ext.libraries
# class build_ext
+
+
+
+def number_of_cpus():
+ """
+ Try to determine the number of cpu's on this system.
+ If successful return that number. Otherwise return 0
+ to indicate failure.
+
+ OUTPUT:
+ int
+ """
+ if hasattr(os, "sysconf") and
os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"): # Linux and Unix
+ n = os.sysconf("SC_NPROCESSORS_ONLN")
+ if isinstance(n, int) and n > 0:
+ return n
+ try:
+ return int(os.popen2("sysctl -n hw.ncpu")[1].read().strip())
+ except:
+ return 0
}}}
--
Ticket URL: <http://trac.sagemath.org/sage_trac/ticket/4652#comment:1>
Sage <http://sagemath.org/>
Sage - Open Source Mathematical Software: Building the Car Instead of
Reinventing the Wheel
--~--~---------~--~----~------------~-------~--~----~
You received this message because you are subscribed to the Google Groups
"sage-trac" group.
To post to this group, send email to [email protected]
To unsubscribe from this group, send email to [EMAIL PROTECTED]
For more options, visit this group at
http://groups.google.com/group/sage-trac?hl=en
-~----------~----~----~----~------~----~------~--~---
