This looks like a great change. Why not delete the bat file? Reviewed-by Jaben Carsey: <[email protected]>
Jaben > On Aug 28, 2018, at 8:28 AM, Liming Gao <[email protected]> wrote: > > From: Dongao Guo <[email protected]> > > Add NmakeSubdirs.py to replace NmakeSubdirs.bat in VS Makefile. This script > will > invoke nmake in multi thread mode. It can save more than half time of > BaseTools > C clean build. > GCC make supports multiple thread in make phase. So, GNNmakefile doesn't need > apply > this script. > > single task or job=1: > just single thread and invoke subprocess,subprocess will use > system.stdout to print output. > multi task: > thread number is logic cpu count.All subprocess output will pass to > python script by PIPE and then script print it to system.stdout. > > Contributed-under: TianoCore Contribution Agreement 1.1 > Signed-off-by: Dongao Guo<[email protected]> > Reviewed-by: Liming Gao <[email protected]> > Test-by: Liming Gao <[email protected]> > --- > BaseTools/Makefile | 12 +- > BaseTools/Source/C/Makefile | 14 +-- > BaseTools/Source/C/Makefiles/NmakeSubdirs.py | 169 +++++++++++++++++++++++++++ > 3 files changed, 182 insertions(+), 13 deletions(-) > create mode 100644 BaseTools/Source/C/Makefiles/NmakeSubdirs.py > > diff --git a/BaseTools/Makefile b/BaseTools/Makefile > index 3736d85..b98cd85 100644 > --- a/BaseTools/Makefile > +++ b/BaseTools/Makefile > @@ -1,7 +1,7 @@ > ## @file > # Windows makefile for Base Tools project build. > # > -# Copyright (c) 2007 - 2017, Intel Corporation. All rights reserved.<BR> > +# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR> > # This program and the accompanying materials > # are licensed and made available under the terms and conditions of the BSD > License > # which accompanies this distribution. The full text of the license may be > found at > @@ -20,19 +20,19 @@ SUBDIRS = $(BASE_TOOLS_PATH)\Source\C > $(BASE_TOOLS_PATH)\Source\Python > all: c python > > c : > - @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat all > $(BASE_TOOLS_PATH)\Source\C > + @$(PYTHON_HOME)\python.exe > $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py all > $(BASE_TOOLS_PATH)\Source\C > > python: > - @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat all > $(BASE_TOOLS_PATH)\Source\Python > + @$(PYTHON_HOME)\python.exe > $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py all > $(BASE_TOOLS_PATH)\Source\Python > > subdirs: $(SUBDIRS) > - @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat all $** > + @$(PYTHON_HOME)\python.exe > $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py all $** > > .PHONY: clean > clean: > - @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat clean $(SUBDIRS) > + $(PYTHON_HOME)\python.exe > $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py clean $(SUBDIRS) > > .PHONY: cleanall > cleanall: > - @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat cleanall $(SUBDIRS) > + $(PYTHON_HOME)\python.exe > $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py cleanall $(SUBDIRS) > > diff --git a/BaseTools/Source/C/Makefile b/BaseTools/Source/C/Makefile > index 5428180..1246d23 100644 > --- a/BaseTools/Source/C/Makefile > +++ b/BaseTools/Source/C/Makefile > @@ -1,7 +1,7 @@ > ## @file > # Windows makefile for C tools build. > # > -# Copyright (c) 2009 - 2017, Intel Corporation. All rights reserved.<BR> > +# Copyright (c) 2009 - 2018, Intel Corporation. All rights reserved.<BR> > # This program and the accompanying materials > # are licensed and made available under the terms and conditions of the BSD > License > # which accompanies this distribution. The full text of the license may be > found at > @@ -16,7 +16,7 @@ HOST_ARCH = IA32 > > LIBRARIES = Common > APPLICATIONS = \ > - BootSectImage \ > + VfrCompile \ > BrotliCompress \ > EfiLdrImage \ > EfiRom \ > @@ -32,7 +32,7 @@ APPLICATIONS = \ > Split \ > TianoCompress \ > VolInfo \ > - VfrCompile \ > + BootSectImage \ > DevicePath > > all: libs apps install > @@ -43,7 +43,7 @@ libs: $(LIBRARIES) > @echo # Build libraries > @echo ###################### > @if not exist $(LIB_PATH) mkdir $(LIB_PATH) > - @Makefiles\NmakeSubdirs.bat all $** > + @$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py all $** > > apps: $(APPLICATIONS) > @echo. > @@ -51,7 +51,7 @@ apps: $(APPLICATIONS) > @echo # Build executables > @echo ###################### > @if not exist $(BIN_PATH) mkdir $(BIN_PATH) > - @Makefiles\NmakeSubdirs.bat all $** > + @$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py all $** > > install: $(LIB_PATH) $(BIN_PATH) > @echo. > @@ -65,11 +65,11 @@ install: $(LIB_PATH) $(BIN_PATH) > > .PHONY: clean > clean: > - @Makefiles\NmakeSubdirs.bat clean $(LIBRARIES) $(APPLICATIONS) > + @$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py clean $(LIBRARIES) > $(APPLICATIONS) > > .PHONY: cleanall > cleanall: > - @Makefiles\NmakeSubdirs.bat cleanall $(LIBRARIES) $(APPLICATIONS) > + @$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py cleanall $(LIBRARIES) > $(APPLICATIONS) > > !INCLUDE Makefiles\ms.rule > > diff --git a/BaseTools/Source/C/Makefiles/NmakeSubdirs.py > b/BaseTools/Source/C/Makefiles/NmakeSubdirs.py > new file mode 100644 > index 0000000..29bb5df > --- /dev/null > +++ b/BaseTools/Source/C/Makefiles/NmakeSubdirs.py > @@ -0,0 +1,169 @@ > +# @file NmakeSubdirs.py > +# This script support parallel build for nmake in windows environment. > +# It supports Python2.x and Python3.x both. > +# > +# Copyright (c) 2018, Intel Corporation. All rights reserved.<BR> > +# > +# This program and the accompanying materials > +# are licensed and made available under the terms and conditions of the BSD > License > +# which accompanies this distribution. The full text of the license may be > found at > +# http://opensource.org/licenses/bsd-license.php > +# > +# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, > +# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR > IMPLIED. > +# > + > +# > +# Import Modules > +# > + > +from __future__ import print_function > +import argparse > +import threading > +import time > +import os > +import subprocess > +import multiprocessing > +import copy > +import sys > +__prog__ = 'NmakeSubdirs' > +__version__ = '%s Version %s' % (__prog__, '0.10 ') > +__copyright__ = 'Copyright (c) 2018, Intel Corporation. All rights > reserved.' > +__description__ = 'Replace for NmakeSubdirs.bat in windows ,support parallel > build for nmake.\n' > + > +cpu_count = multiprocessing.cpu_count() > +output_lock = threading.Lock() > +def RunCommand(WorkDir=None, *Args, **kwargs): > + if WorkDir is None: > + WorkDir = os.curdir > + if "stderr" not in kwargs: > + kwargs["stderr"] = subprocess.STDOUT > + if "stdout" not in kwargs: > + kwargs["stdout"] = subprocess.PIPE > + p = subprocess.Popen(Args, cwd=WorkDir, stderr=kwargs["stderr"], > stdout=kwargs["stdout"]) > + stdout, stderr = p.communicate() > + message = "" > + if stdout is not None: > + message = stdout.decode() #for compatibility in python 2 and 3 > + > + if p.returncode != 0: > + raise RuntimeError("Error while execute command \'{0}\' in direcotry > {1}\n{2}".format(" ".join(Args), WorkDir, message)) > + > + output_lock.acquire(True) > + print("execute command \"{0}\" in directory {1}".format(" ".join(Args), > WorkDir)) > + print(message) > + output_lock.release() > + > + return p.returncode, stdout > + > +class TaskUnit(object): > + def __init__(self, func, args, kwargs): > + self.func = func > + self.args = args > + self.kwargs = kwargs > + > + def __eq__(self, other): > + return id(self).__eq__(id(other)) > + > + def run(self): > + return self.func(*self.args, **self.kwargs) > + > + def __str__(self): > + para = list(self.args) > + para.extend("{0}={1}".format(k, v)for k, v in self.kwargs.items()) > + > + return "{0}({1})".format(self.func.__name__, ",".join(para)) > + > +class ThreadControl(object): > + > + def __init__(self, maxthread): > + self._processNum = maxthread > + self.pending = [] > + self.running = [] > + self.pendingLock = threading.Lock() > + self.runningLock = threading.Lock() > + self.error = False > + self.errorLock = threading.Lock() > + self.errorMsg = "errorMsg" > + > + def addTask(self, func, *args, **kwargs): > + self.pending.append(TaskUnit(func, args, kwargs)) > + > + def waitComplete(self): > + self._schedule.join() > + > + def startSchedule(self): > + self._schedule = threading.Thread(target=self.Schedule) > + self._schedule.start() > + > + def Schedule(self): > + for i in range(self._processNum): > + task = threading.Thread(target=self.startTask) > + task.daemon = False > + self.running.append(task) > + > + self.runningLock.acquire(True) > + for thread in self.running: > + thread.start() > + self.runningLock.release() > + > + while len(self.running) > 0: > + time.sleep(0.1) > + if self.error: > + print("subprocess not exit sucessfully") > + print(self.errorMsg) > + > + def startTask(self): > + while True: > + if self.error: > + break > + self.pendingLock.acquire(True) > + if len(self.pending) == 0: > + self.pendingLock.release() > + break > + task = self.pending.pop(0) > + self.pendingLock.release() > + try: > + task.run() > + except RuntimeError as e: > + if self.error: break > + self.errorLock.acquire(True) > + self.error = True > + self.errorMsg = str(e) > + time.sleep(0.1) > + self.errorLock.release() > + break > + > + self.runningLock.acquire(True) > + self.running.remove(threading.currentThread()) > + self.runningLock.release() > + > +def Run(): > + curdir = os.path.abspath(os.curdir) > + if len(args.subdirs) == 1: > + args.jobs = 1 > + if args.jobs == 1: > + try: > + for dir in args.subdirs: > + RunCommand(os.path.join(curdir, dir), "nmake", args.target, > stdout=sys.stdout, stderr=subprocess.STDOUT) > + except RuntimeError: > + exit(1) > + else: > + controller = ThreadControl(args.jobs) > + for dir in args.subdirs: > + controller.addTask(RunCommand, os.path.join(curdir, dir), > "nmake", args.target) > + controller.startSchedule() > + controller.waitComplete() > + if controller.error: > + exit(1) > + > +if __name__ == "__main__": > + parser = argparse.ArgumentParser(prog=__prog__, > description=__description__ + __copyright__, conflict_handler='resolve') > + > + parser.add_argument("target", help="the target for nmake") > + parser.add_argument("subdirs", nargs="+", help="the relative dir path of > makefile") > + parser.add_argument("--jobs", type=int, dest="jobs", default=cpu_count, > help="thread number") > + parser.add_argument('--version', action='version', version=__version__) > + args = parser.parse_args() > + Run() > + > -- > 2.10.0.windows.1 > > _______________________________________________ > edk2-devel mailing list > [email protected] > https://lists.01.org/mailman/listinfo/edk2-devel _______________________________________________ edk2-devel mailing list [email protected] https://lists.01.org/mailman/listinfo/edk2-devel
