Jaben: NmakeSubdirs.bat will be removed once NmakeSubdirs.py is stable. Now, bat is kept for a while.
Thanks Liming > -----Original Message----- > From: Carsey, Jaben > Sent: Wednesday, August 29, 2018 12:10 AM > To: Gao, Liming <liming....@intel.com> > Cc: edk2-devel@lists.01.org; Guo, Dongao <dongao....@intel.com> > Subject: Re: [edk2] [Patch] BaseTools: Support multi thread build Basetool on > Windows > > This looks like a great change. Why not delete the bat file? > > Reviewed-by Jaben Carsey: <jaben.caspr...@intel.com> > > Jaben > > > On Aug 28, 2018, at 8:28 AM, Liming Gao <liming....@intel.com> wrote: > > > > From: Dongao Guo <dongao....@intel.com> > > > > Add NmakeSubdirs.py to replace NmakeSubdirs.bat in VS Makefile. This script > > will > > invoke nmake in multi thread mode. It can save more than half time of > > BaseTools > > C clean build. > > GCC make supports multiple thread in make phase. So, GNNmakefile doesn't > > need apply > > this script. > > > > single task or job=1: > > just single thread and invoke subprocess,subprocess will use > > system.stdout to print output. > > multi task: > > thread number is logic cpu count.All subprocess output will pass to > > python script by PIPE and then script print it to system.stdout. > > > > Contributed-under: TianoCore Contribution Agreement 1.1 > > Signed-off-by: Dongao Guo<dongao....@intel.com> > > Reviewed-by: Liming Gao <liming....@intel.com> > > Test-by: Liming Gao <liming....@intel.com> > > --- > > BaseTools/Makefile | 12 +- > > BaseTools/Source/C/Makefile | 14 +-- > > BaseTools/Source/C/Makefiles/NmakeSubdirs.py | 169 > > +++++++++++++++++++++++++++ > > 3 files changed, 182 insertions(+), 13 deletions(-) > > create mode 100644 BaseTools/Source/C/Makefiles/NmakeSubdirs.py > > > > diff --git a/BaseTools/Makefile b/BaseTools/Makefile > > index 3736d85..b98cd85 100644 > > --- a/BaseTools/Makefile > > +++ b/BaseTools/Makefile > > @@ -1,7 +1,7 @@ > > ## @file > > # Windows makefile for Base Tools project build. > > # > > -# Copyright (c) 2007 - 2017, Intel Corporation. All rights reserved.<BR> > > +# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR> > > # This program and the accompanying materials > > # are licensed and made available under the terms and conditions of the BSD > > License > > # which accompanies this distribution. The full text of the license may be > > found at > > @@ -20,19 +20,19 @@ SUBDIRS = $(BASE_TOOLS_PATH)\Source\C > > $(BASE_TOOLS_PATH)\Source\Python > > all: c python > > > > c : > > - @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat all > > $(BASE_TOOLS_PATH)\Source\C > > + @$(PYTHON_HOME)\python.exe > > $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py all > $(BASE_TOOLS_PATH)\Source\C > > > > python: > > - @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat all > > $(BASE_TOOLS_PATH)\Source\Python > > + @$(PYTHON_HOME)\python.exe > > $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py all > $(BASE_TOOLS_PATH)\Source\Python > > > > subdirs: $(SUBDIRS) > > - @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat all $** > > + @$(PYTHON_HOME)\python.exe > > $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py all $** > > > > .PHONY: clean > > clean: > > - @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat clean $(SUBDIRS) > > + $(PYTHON_HOME)\python.exe > > $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py clean $(SUBDIRS) > > > > .PHONY: cleanall > > cleanall: > > - @$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat cleanall > > $(SUBDIRS) > > + $(PYTHON_HOME)\python.exe > > $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py cleanall $(SUBDIRS) > > > > diff --git a/BaseTools/Source/C/Makefile b/BaseTools/Source/C/Makefile > > index 5428180..1246d23 100644 > > --- a/BaseTools/Source/C/Makefile > > +++ b/BaseTools/Source/C/Makefile > > @@ -1,7 +1,7 @@ > > ## @file > > # Windows makefile for C tools build. > > # > > -# Copyright (c) 2009 - 2017, Intel Corporation. All rights reserved.<BR> > > +# Copyright (c) 2009 - 2018, Intel Corporation. All rights reserved.<BR> > > # This program and the accompanying materials > > # are licensed and made available under the terms and conditions of the BSD > > License > > # which accompanies this distribution. The full text of the license may be > > found at > > @@ -16,7 +16,7 @@ HOST_ARCH = IA32 > > > > LIBRARIES = Common > > APPLICATIONS = \ > > - BootSectImage \ > > + VfrCompile \ > > BrotliCompress \ > > EfiLdrImage \ > > EfiRom \ > > @@ -32,7 +32,7 @@ APPLICATIONS = \ > > Split \ > > TianoCompress \ > > VolInfo \ > > - VfrCompile \ > > + BootSectImage \ > > DevicePath > > > > all: libs apps install > > @@ -43,7 +43,7 @@ libs: $(LIBRARIES) > > @echo # Build libraries > > @echo ###################### > > @if not exist $(LIB_PATH) mkdir $(LIB_PATH) > > - @Makefiles\NmakeSubdirs.bat all $** > > + @$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py all $** > > > > apps: $(APPLICATIONS) > > @echo. > > @@ -51,7 +51,7 @@ apps: $(APPLICATIONS) > > @echo # Build executables > > @echo ###################### > > @if not exist $(BIN_PATH) mkdir $(BIN_PATH) > > - @Makefiles\NmakeSubdirs.bat all $** > > + @$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py all $** > > > > install: $(LIB_PATH) $(BIN_PATH) > > @echo. > > @@ -65,11 +65,11 @@ install: $(LIB_PATH) $(BIN_PATH) > > > > .PHONY: clean > > clean: > > - @Makefiles\NmakeSubdirs.bat clean $(LIBRARIES) $(APPLICATIONS) > > + @$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py clean $(LIBRARIES) > > $(APPLICATIONS) > > > > .PHONY: cleanall > > cleanall: > > - @Makefiles\NmakeSubdirs.bat cleanall $(LIBRARIES) $(APPLICATIONS) > > + @$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py cleanall > > $(LIBRARIES) $(APPLICATIONS) > > > > !INCLUDE Makefiles\ms.rule > > > > diff --git a/BaseTools/Source/C/Makefiles/NmakeSubdirs.py > > b/BaseTools/Source/C/Makefiles/NmakeSubdirs.py > > new file mode 100644 > > index 0000000..29bb5df > > --- /dev/null > > +++ b/BaseTools/Source/C/Makefiles/NmakeSubdirs.py > > @@ -0,0 +1,169 @@ > > +# @file NmakeSubdirs.py > > +# This script support parallel build for nmake in windows environment. > > +# It supports Python2.x and Python3.x both. > > +# > > +# Copyright (c) 2018, Intel Corporation. All rights reserved.<BR> > > +# > > +# This program and the accompanying materials > > +# are licensed and made available under the terms and conditions of the > > BSD License > > +# which accompanies this distribution. The full text of the license may > > be found at > > +# http://opensource.org/licenses/bsd-license.php > > +# > > +# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, > > +# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR > > IMPLIED. > > +# > > + > > +# > > +# Import Modules > > +# > > + > > +from __future__ import print_function > > +import argparse > > +import threading > > +import time > > +import os > > +import subprocess > > +import multiprocessing > > +import copy > > +import sys > > +__prog__ = 'NmakeSubdirs' > > +__version__ = '%s Version %s' % (__prog__, '0.10 ') > > +__copyright__ = 'Copyright (c) 2018, Intel Corporation. All rights > > reserved.' > > +__description__ = 'Replace for NmakeSubdirs.bat in windows ,support > > parallel build for nmake.\n' > > + > > +cpu_count = multiprocessing.cpu_count() > > +output_lock = threading.Lock() > > +def RunCommand(WorkDir=None, *Args, **kwargs): > > + if WorkDir is None: > > + WorkDir = os.curdir > > + if "stderr" not in kwargs: > > + kwargs["stderr"] = subprocess.STDOUT > > + if "stdout" not in kwargs: > > + kwargs["stdout"] = subprocess.PIPE > > + p = subprocess.Popen(Args, cwd=WorkDir, stderr=kwargs["stderr"], > > stdout=kwargs["stdout"]) > > + stdout, stderr = p.communicate() > > + message = "" > > + if stdout is not None: > > + message = stdout.decode() #for compatibility in python 2 and 3 > > + > > + if p.returncode != 0: > > + raise RuntimeError("Error while execute command \'{0}\' in > > direcotry {1}\n{2}".format(" ".join(Args), WorkDir, message)) > > + > > + output_lock.acquire(True) > > + print("execute command \"{0}\" in directory {1}".format(" > > ".join(Args), WorkDir)) > > + print(message) > > + output_lock.release() > > + > > + return p.returncode, stdout > > + > > +class TaskUnit(object): > > + def __init__(self, func, args, kwargs): > > + self.func = func > > + self.args = args > > + self.kwargs = kwargs > > + > > + def __eq__(self, other): > > + return id(self).__eq__(id(other)) > > + > > + def run(self): > > + return self.func(*self.args, **self.kwargs) > > + > > + def __str__(self): > > + para = list(self.args) > > + para.extend("{0}={1}".format(k, v)for k, v in self.kwargs.items()) > > + > > + return "{0}({1})".format(self.func.__name__, ",".join(para)) > > + > > +class ThreadControl(object): > > + > > + def __init__(self, maxthread): > > + self._processNum = maxthread > > + self.pending = [] > > + self.running = [] > > + self.pendingLock = threading.Lock() > > + self.runningLock = threading.Lock() > > + self.error = False > > + self.errorLock = threading.Lock() > > + self.errorMsg = "errorMsg" > > + > > + def addTask(self, func, *args, **kwargs): > > + self.pending.append(TaskUnit(func, args, kwargs)) > > + > > + def waitComplete(self): > > + self._schedule.join() > > + > > + def startSchedule(self): > > + self._schedule = threading.Thread(target=self.Schedule) > > + self._schedule.start() > > + > > + def Schedule(self): > > + for i in range(self._processNum): > > + task = threading.Thread(target=self.startTask) > > + task.daemon = False > > + self.running.append(task) > > + > > + self.runningLock.acquire(True) > > + for thread in self.running: > > + thread.start() > > + self.runningLock.release() > > + > > + while len(self.running) > 0: > > + time.sleep(0.1) > > + if self.error: > > + print("subprocess not exit sucessfully") > > + print(self.errorMsg) > > + > > + def startTask(self): > > + while True: > > + if self.error: > > + break > > + self.pendingLock.acquire(True) > > + if len(self.pending) == 0: > > + self.pendingLock.release() > > + break > > + task = self.pending.pop(0) > > + self.pendingLock.release() > > + try: > > + task.run() > > + except RuntimeError as e: > > + if self.error: break > > + self.errorLock.acquire(True) > > + self.error = True > > + self.errorMsg = str(e) > > + time.sleep(0.1) > > + self.errorLock.release() > > + break > > + > > + self.runningLock.acquire(True) > > + self.running.remove(threading.currentThread()) > > + self.runningLock.release() > > + > > +def Run(): > > + curdir = os.path.abspath(os.curdir) > > + if len(args.subdirs) == 1: > > + args.jobs = 1 > > + if args.jobs == 1: > > + try: > > + for dir in args.subdirs: > > + RunCommand(os.path.join(curdir, dir), "nmake", > > args.target, stdout=sys.stdout, stderr=subprocess.STDOUT) > > + except RuntimeError: > > + exit(1) > > + else: > > + controller = ThreadControl(args.jobs) > > + for dir in args.subdirs: > > + controller.addTask(RunCommand, os.path.join(curdir, dir), > > "nmake", args.target) > > + controller.startSchedule() > > + controller.waitComplete() > > + if controller.error: > > + exit(1) > > + > > +if __name__ == "__main__": > > + parser = argparse.ArgumentParser(prog=__prog__, > > description=__description__ + __copyright__, conflict_handler='resolve') > > + > > + parser.add_argument("target", help="the target for nmake") > > + parser.add_argument("subdirs", nargs="+", help="the relative dir path > > of makefile") > > + parser.add_argument("--jobs", type=int, dest="jobs", > > default=cpu_count, help="thread number") > > + parser.add_argument('--version', action='version', version=__version__) > > + args = parser.parse_args() > > + Run() > > + > > -- > > 2.10.0.windows.1 > > > > _______________________________________________ > > edk2-devel mailing list > > edk2-devel@lists.01.org > > https://lists.01.org/mailman/listinfo/edk2-devel _______________________________________________ edk2-devel mailing list edk2-devel@lists.01.org https://lists.01.org/mailman/listinfo/edk2-devel
