Reviewed-by: Liming Gao <[email protected]>
> -----Original Message----- > From: edk2-devel [mailto:[email protected]] On Behalf Of > Yonghong Zhu > Sent: Wednesday, March 16, 2016 11:09 AM > To: [email protected] > Subject: [edk2] [Patch] BaseTools: Fix nmake failure due to command-line > length limitation > > NMAKE is limited to command-line length of 4096 characters. Due to the > large number of /I directives specified on command line (one per include > directory), the path length of WORKSPACE is multiplied by the number of > /I directives and can exceed the limit. > This patch: > 1. Add new build option -l, --cmd-len to set the maximum command line > length, default value is 4096. > 2. Generate the response file only if the command line length exceed its > maximum characters (default is 4096) when build the module. Cover > PP_FLAGS, CC_FLAGS, VFRPP_FLAGS, APP_FLAGS, ASLPP_FLAGS, > ASLCC_FLAGS and > ASM_FLAGS. > 3. The content of the response file is combine from the FLAGS option and > INC option. > 4. When build failure, it would print out the response file's file > location and its content. > > Contributed-under: TianoCore Contribution Agreement 1.0 > Signed-off-by: Yonghong Zhu <[email protected]> > --- > BaseTools/Source/Python/AutoGen/AutoGen.py | 9 +++ > BaseTools/Source/Python/AutoGen/GenMake.py | 114 > ++++++++++++++++++++++++++- > BaseTools/Source/Python/Common/GlobalData.py | 2 +- > BaseTools/Source/Python/build/build.py | 12 +++ > 4 files changed, 135 insertions(+), 2 deletions(-) > > diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py > b/BaseTools/Source/Python/AutoGen/AutoGen.py > index c7aa84f..4934c57 100644 > --- a/BaseTools/Source/Python/AutoGen/AutoGen.py > +++ b/BaseTools/Source/Python/AutoGen/AutoGen.py > @@ -2376,10 +2376,11 @@ class ModuleAutoGen(AutoGen): > self._OutputDir = None > self._DebugDir = None > self._MakeFileDir = None > > self._IncludePathList = None > + self._IncludePathLength = 0 > self._AutoGenFileList = None > self._UnicodeFileList = None > self._SourceFileList = None > self._ObjectFileList = None > self._BinaryFileList = None > @@ -3222,10 +3223,17 @@ class ModuleAutoGen(AutoGen): > for Inc in Package.Includes: > if Inc not in self._IncludePathList: > self._IncludePathList.append(str(Inc)) > return self._IncludePathList > > + def _GetIncludePathLength(self): > + self._IncludePathLength = 0 > + if self._IncludePathList: > + for inc in self._IncludePathList: > + self._IncludePathLength += len(' ' + inc) > + return self._IncludePathLength > + > ## Get HII EX PCDs which maybe used by VFR > # > # efivarstore used by VFR may relate with HII EX PCDs > # Get the variable name and GUID from efivarstore and HII EX PCD > # List the HII EX PCDs in As Built INF if both name and GUID match. > @@ -3814,10 +3822,11 @@ class ModuleAutoGen(AutoGen): > DebugDir = property(_GetDebugDir) > MakeFileDir = property(_GetMakeFileDir) > CustomMakefile = property(_GetCustomMakefile) > > IncludePathList = property(_GetIncludePathList) > + IncludePathLength = property(_GetIncludePathLength) > AutoGenFileList = property(_GetAutoGenFileList) > UnicodeFileList = property(_GetUnicodeFileList) > SourceFileList = property(_GetSourceFileList) > BinaryFileList = property(_GetBinaryFiles) # FileType : [File List] > Targets = property(_GetTargets) > diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py > b/BaseTools/Source/Python/AutoGen/GenMake.py > index ec24c70..5543aaa 100644 > --- a/BaseTools/Source/Python/AutoGen/GenMake.py > +++ b/BaseTools/Source/Python/AutoGen/GenMake.py > @@ -1,9 +1,9 @@ > ## @file > # Create makefile for MS nmake and GNU make > # > -# Copyright (c) 2007 - 2014, Intel Corporation. All rights reserved.<BR> > +# Copyright (c) 2007 - 2016, Intel Corporation. All rights reserved.<BR> > # This program and the accompanying materials > # are licensed and made available under the terms and conditions of the BSD > License > # which accompanies this distribution. The full text of the license may be > found at > # http://opensource.org/licenses/bsd-license.php > # > @@ -495,10 +495,26 @@ cleanlib: > if Attr == "FLAGS": > Value = RemoveDupOption(Value, IncPrefix, > self._AutoGenObject.IncludePathList) > ToolsDef.append("%s_%s = %s" % (Tool, Attr, Value)) > ToolsDef.append("") > > + # generate the Response file and Response flag > + RespDict = self.CommandExceedLimit() > + RespFileList = os.path.join(self._AutoGenObject.OutputDir, > 'respfilelist.txt') > + if RespDict: > + RespFileListContent = '' > + for Resp in RespDict.keys(): > + RespFile = os.path.join(self._AutoGenObject.OutputDir, > str(Resp).lower() + '.txt') > + SaveFileOnChange(RespFile, RespDict[Resp], False) > + ToolsDef.append("%s = %s" % (Resp, '@' + RespFile)) > + RespFileListContent += '@' + RespFile + os.linesep > + RespFileListContent += RespDict[Resp] + os.linesep > + SaveFileOnChange(RespFileList, RespFileListContent, False) > + else: > + if os.path.exists(RespFileList): > + os.remove(RespFileList) > + > # convert source files and binary files to build targets > self.ResultFileList = [str(T.Target) for T in > self._AutoGenObject.CodaTargetList] > if len(self.ResultFileList) == 0 and > len(self._AutoGenObject.SourceFileList) <> 0: > EdkLogger.error("build", AUTOGEN_ERROR, "Nothing to build", > ExtraData="[%s]" % str(self._AutoGenObject)) > @@ -618,10 +634,106 @@ cleanlib: > "backward_compatible_target": BcTargetList, > } > > return MakefileTemplateDict > > + def CommandExceedLimit(self): > + FlagDict = { > + 'CC' : { 'Macro' : '$(CC_FLAGS)', 'Value' : > False}, > + 'PP' : { 'Macro' : '$(PP_FLAGS)', 'Value' : > False}, > + 'APP' : { 'Macro' : '$(APP_FLAGS)', 'Value' : > False}, > + 'ASLPP' : { 'Macro' : '$(ASLPP_FLAGS)', 'Value' : > False}, > + 'VFRPP' : { 'Macro' : '$(VFRPP_FLAGS)', 'Value' : > False}, > + 'ASM' : { 'Macro' : '$(ASM_FLAGS)', 'Value' : > False}, > + 'ASLCC' : { 'Macro' : '$(ASLCC_FLAGS)', 'Value' : > False}, > + } > + > + RespDict = {} > + FileTypeList = [] > + IncPrefix = self._INC_FLAG_[self._AutoGenObject.ToolChainFamily] > + > + # base on the source files to decide the file type > + for File in self._AutoGenObject.SourceFileList: > + for type in self._AutoGenObject.FileTypes: > + if File in self._AutoGenObject.FileTypes[type]: > + if type not in FileTypeList: > + FileTypeList.append(type) > + > + # calculate the command-line length > + if FileTypeList: > + for type in FileTypeList: > + BuildTargets = > self._AutoGenObject.BuildRules[type].BuildTargets > + for Target in BuildTargets: > + CommandList = BuildTargets[Target].Commands > + for SingleCommand in CommandList: > + Tool = '' > + SingleCommandLength = len(SingleCommand) > + SingleCommandList = SingleCommand.split() > + if len(SingleCommandList) > 0: > + for Flag in FlagDict.keys(): > + if '$('+ Flag +')' in SingleCommandList[0]: > + Tool = Flag > + break > + if Tool: > + SingleCommandLength += > len(self._AutoGenObject._BuildOption[Tool]['PATH']) > + for item in SingleCommandList[1:]: > + if FlagDict[Tool]['Macro'] in item: > + Str = > self._AutoGenObject._BuildOption[Tool]['FLAGS'] > + while(Str.find('$(') != -1): > + for macro in > self._AutoGenObject.Macros.keys(): > + MacroName = '$('+ macro + ')' > + if (Str.find(MacroName) != -1): > + Str = Str.replace(MacroName, > self._AutoGenObject.Macros[macro]) > + break > + else: > + EdkLogger.error("build", > AUTOGEN_ERROR, "Not > supported macro is found in make command : %s" % Str, ExtraData="[%s]" % > str(self._AutoGenObject)) > + SingleCommandLength += len(Str) > + elif '$(INC)' in item: > + SingleCommandLength += > self._AutoGenObject.IncludePathLength + len(IncPrefix) * > len(self._AutoGenObject._IncludePathList) > + elif item.find('$(') != -1: > + Str = item > + for Option in > self._AutoGenObject.BuildOption.keys(): > + for Attr in > self._AutoGenObject.BuildOption[Option]: > + if Str.find(Option + '_' + Attr) > != -1: > + Str = Str.replace('$(' + > Option + '_' + Attr + ')', > self._AutoGenObject.BuildOption[Option][Attr]) > + while(Str.find('$(') != -1): > + for macro in > self._AutoGenObject.Macros.keys(): > + MacroName = '$('+ macro + ')' > + if (Str.find(MacroName) != -1): > + Str = Str.replace(MacroName, > self._AutoGenObject.Macros[macro]) > + break > + else: > + EdkLogger.error("build", > AUTOGEN_ERROR, "Not > supported macro is found in make command : %s" % Str, ExtraData="[%s]" % > str(self._AutoGenObject)) > + > + SingleCommandLength += len(Str) > + > + if SingleCommandLength > > GlobalData.gCommandMaxLength: > + FlagDict[Tool]['Value'] = True > + > + # generate the response file content by combine the FLAGS and > INC > + for Flag in FlagDict.keys(): > + if FlagDict[Flag]['Value']: > + Key = Flag + '_RESP' > + RespMacro = FlagDict[Flag]['Macro'].replace('FLAGS', > 'RESP') > + Value = > self._AutoGenObject.BuildOption[Flag]['FLAGS'] > + for inc in self._AutoGenObject._IncludePathList: > + Value += ' ' + IncPrefix + inc > + while (Value.find('$(') != -1): > + for macro in self._AutoGenObject.Macros.keys(): > + MacroName = '$('+ macro + ')' > + if (Value.find(MacroName) != -1): > + Value = Value.replace(MacroName, > self._AutoGenObject.Macros[macro]) > + break > + else: > + EdkLogger.error("build", AUTOGEN_ERROR, "Not > supported macro is found in make command : %s" % Str, ExtraData="[%s]" % > str(self._AutoGenObject)) > + RespDict[Key] = Value > + for Target in BuildTargets: > + for i, SingleCommand in > enumerate(BuildTargets[Target].Commands): > + if FlagDict[Flag]['Macro'] in SingleCommand: > + BuildTargets[Target].Commands[i] = > SingleCommand.replace('$(INC)','').replace(FlagDict[Flag]['Macro'], > RespMacro) > + return RespDict > + > def ProcessBuildTargetList(self): > # > # Search dependency file list for each source file > # > ForceIncludedFile = [] > diff --git a/BaseTools/Source/Python/Common/GlobalData.py > b/BaseTools/Source/Python/Common/GlobalData.py > index 4234bf5..8f544bd 100644 > --- a/BaseTools/Source/Python/Common/GlobalData.py > +++ b/BaseTools/Source/Python/Common/GlobalData.py > @@ -32,11 +32,11 @@ gPlatformPcds = {} > gPlatformOtherPcds = {} > gActivePlatform = None > gCommandLineDefines = {} > gEdkGlobal = {} > gOverrideDir = {} > - > +gCommandMaxLength = 4096 > # for debug trace purpose when problem occurs > gProcessingFile = '' > gBuildingModule = '' > > ## Regular expression for matching macro used in DSC/DEC/INF file inclusion > diff --git a/BaseTools/Source/Python/build/build.py > b/BaseTools/Source/Python/build/build.py > index 5619638..1607928 100644 > --- a/BaseTools/Source/Python/build/build.py > +++ b/BaseTools/Source/Python/build/build.py > @@ -302,10 +302,18 @@ def LaunchCommand(Command, WorkingDir): > > # check the return code of the program > if Proc.returncode != 0: > if type(Command) != type(""): > Command = " ".join(Command) > + # print out the Response file and its content when make failure > + RespFile = os.path.join(WorkingDir, 'OUTPUT', 'respfilelist.txt') > + if os.path.isfile(RespFile): > + f = open(RespFile) > + RespContent = f.read() > + f.close() > + EdkLogger.info(RespContent) > + > EdkLogger.error("build", COMMAND_FAILURE, ExtraData="%s [%s]" % > (Command, WorkingDir)) > > ## The smallest unit that can be built in multi-thread build mode > # > # This is the base class of build unit. The "Obj" parameter must provide > @@ -773,10 +781,13 @@ class Build(): > self.Platform = None > self.LoadFixAddress = 0 > self.UniFlag = BuildOptions.Flag > self.BuildModules = [] > > + if BuildOptions.CommandLength: > + GlobalData.gCommandMaxLength = BuildOptions.CommandLength > + > # print dot character during doing some time-consuming work > self.Progress = Utils.Progressor() > > self.InitBuild() > > @@ -1929,10 +1940,11 @@ def MyOptionParser(): > Parser.add_option("-N", "--no-cache", action="store_true", > dest="DisableCache", default=False, help="Disable build cache mechanism") > Parser.add_option("--conf", action="store", type="string", > dest="ConfDirectory", help="Specify the customized Conf directory.") > Parser.add_option("--check-usage", action="store_true", > dest="CheckUsage", default=False, help="Check usage content of entries > listed in INF file.") > Parser.add_option("--ignore-sources", action="store_true", > dest="IgnoreSources", default=False, help="Focus to a binary build and > ignore all source files") > Parser.add_option("--pcd", action="append", dest="OptionPcd", > help="Set PCD value by command line. Format: 'PcdName=Value' ") > + Parser.add_option("-l", "--cmd-len", action="store", type="int", > dest="CommandLength", help="Specify the maximum line length of build > command. Default is 4096.") > > (Opt, Args) = Parser.parse_args() > return (Opt, Args) > > ## Tool entrance method > -- > 2.6.1.windows.1 > > _______________________________________________ > edk2-devel mailing list > [email protected] > https://lists.01.org/mailman/listinfo/edk2-devel _______________________________________________ edk2-devel mailing list [email protected] https://lists.01.org/mailman/listinfo/edk2-devel
