Sorry for the incomplete mail. I verified this patch on Ovmf, Emulator, MinPlatforms and Intel Server Platform.
Thanks, Bob -----Original Message----- From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Bob Feng Sent: Thursday, September 5, 2019 10:04 PM To: Gao, Liming <liming....@intel.com>; devel@edk2.groups.io Cc: Shi, Steven <steven....@intel.com> Subject: Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen Liming, Yes, this change only impact -u option. I separate _MultiThreadBuildPlatform function into two functions and create 2 new functions specially for IgnoreAutoGen, so the there looks big change for build.py in this patch. I verified this patch on Ovmf, E -Bob -----Original Message----- From: Gao, Liming Sent: Thursday, September 5, 2019 9:12 PM To: devel@edk2.groups.io; Feng, Bob C <bob.c.f...@intel.com> Cc: Shi, Steven <steven....@intel.com> Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen Bob: Does this change only impact -u option behavior? Thanks Liming > -----Original Message----- > From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of > Bob Feng > Sent: Thursday, September 5, 2019 5:05 PM > To: devel@edk2.groups.io > Cc: Gao, Liming <liming....@intel.com>; Shi, Steven > <steven....@intel.com>; Feng, Bob C <bob.c.f...@intel.com> > Subject: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for > IgnoreAutoGen > > https://bugzilla.tianocore.org/show_bug.cgi?id=2080 > > This patch is to improve build -u option to re-use > GlobalVar_<platformguid>_<arch>.bin file which is introduced by > multiple-process-autogen feature. > > Cc: Liming Gao <liming....@intel.com> > Cc: Steven Shi <steven....@intel.com> > Signed-off-by: Bob Feng <bob.c.f...@intel.com> > --- > .../Source/Python/AutoGen/AutoGenWorker.py | 2 +- > .../Source/Python/AutoGen/ModuleAutoGen.py | 4 +- > .../Python/AutoGen/ModuleAutoGenHelper.py | 29 ++ > .../Source/Python/AutoGen/PlatformAutoGen.py | 2 + > .../Source/Python/AutoGen/WorkspaceAutoGen.py | 1 - > BaseTools/Source/Python/build/build.py | 355 +++++++++++------- > 6 files changed, 255 insertions(+), 138 deletions(-) > > diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > index 2e68538b1cb4..f488ae9d5f80 100755 > --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py > @@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process): > Ma.ReferenceModules = > Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] > if GlobalData.gBinCacheSource and CommandTarget in [None, > "", "all"]: > Ma.GenModuleFilesHash(GlobalData.gCacheIR) > Ma.GenPreMakefileHash(GlobalData.gCacheIR) > if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): > - continue > + continue > > Ma.CreateCodeFile(False) > > Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, > Ma.Arch),[])) > > if GlobalData.gBinCacheSource and CommandTarget in [None, > "", "all"]: > diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py > b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py > index 3bb7e91154ac..5f28681e3146 100755 > --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py > +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py > @@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen): > not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: > self.GenModuleFilesHash(gDict) > > if not (self.MetaFile.Path, self.Arch) in gDict or \ > not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: > - EdkLogger.quiet("[cache warning]: Cannot generate > ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, > self.Arch)) > - return > + EdkLogger.quiet("[cache warning]: Cannot generate > + ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, > self.Arch)) > + return > > # Initialze hash object > m = hashlib.md5() > > # Add Platform level hash > diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py > b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py > index c7591253debd..9dd93b9beb12 100644 > --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py > +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py > @@ -234,22 +234,51 @@ class AutoGenInfo(object): > # > #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in > multiple thread # class WorkSpaceInfo(AutoGenInfo): > def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch): > + if not hasattr(self, "_Init"): > + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch) > + self._Init = True > + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch): > self._SrcTimeStamp = 0 > self.Db = BuildDB > self.BuildDatabase = self.Db.BuildObject > self.Target = Target > self.ToolChain = ToolChain > self.WorkspaceDir = Workspace > self.ActivePlatform = MetaFile > self.ArchList = Arch > + self.AutoGenObjectList = [] > + @property > + def BuildDir(self): > + return self.AutoGenObjectList[0].BuildDir > > + @property > + def Name(self): > + return self.AutoGenObjectList[0].Platform.PlatformName > + > + @property > + def FlashDefinition(self): > + return self.AutoGenObjectList[0].Platform.FlashDefinition > + @property > + def GenFdsCommandDict(self): > + FdsCommandDict = > self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict") > + if FdsCommandDict: > + return FdsCommandDict > + return {} > + > + @cached_property > + def FvDir(self): > + return os.path.join(self.BuildDir, TAB_FV_DIRECTORY) > > class PlatformInfo(AutoGenInfo): > def __init__(self, Workspace, MetaFile, Target, ToolChain, > Arch,DataPipe): > + if not hasattr(self, "_Init"): > + self.do_init(Workspace, MetaFile, Target, ToolChain, > Arch,DataPipe) > + self._Init = True > + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch,DataPipe): > self.Wa = Workspace > self.WorkspaceDir = self.Wa.WorkspaceDir > self.MetaFile = MetaFile > self.Arch = Arch > self.Target = Target > diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py > b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py > index 565424a95ead..1e17b6687129 100644 > --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py > +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py > @@ -186,11 +186,13 @@ class PlatformAutoGen(AutoGen): > key = (Ma.MetaFile.File, self.Arch) > if key in FfsCommand: > Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key]) > else: > Ma.CreateMakeFile(CreateModuleMakeFile) > + self.CreateLibModuelDirs() > > + def CreateLibModuelDirs(self): > # no need to create makefile for the platform more than once > if self.IsMakeFileCreated: > return > > # create library/module build dirs for platform diff --git > a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py > b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py > index 24942674721f..f9d2c216169c 100644 > --- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py > +++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py > @@ -126,11 +126,10 @@ class WorkspaceAutoGen(AutoGen): > self._CheckPcdDefineAndType() > > self.CreateBuildOptionsFile() > self.CreatePcdTokenNumberFile() > self.CreateModuleHashInfo() > - GlobalData.gAutoGenPhase = False > > # > # Merge Arch > # > def MergeArch(self): > diff --git a/BaseTools/Source/Python/build/build.py > b/BaseTools/Source/Python/build/build.py > index 0406ac314b65..db57c3e526da 100755 > --- a/BaseTools/Source/Python/build/build.py > +++ b/BaseTools/Source/Python/build/build.py > @@ -52,10 +52,13 @@ from PatchPcdValue.PatchPcdValue import > PatchBinaryFile > > import Common.GlobalData as GlobalData from GenFds.GenFds import > GenFds, GenFdsApi import multiprocessing as mp from multiprocessing > import Manager > +from AutoGen.DataPipe import MemoryDataPipe from > +AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo from > +GenFds.FdfParser import FdfParser > > > ## standard targets of build command > gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', > 'fds', 'clean', 'cleanall', 'cleanlib', 'run'] > > @@ -837,11 +840,11 @@ class Build(): > for PcdMa in PcdMaList: > if GlobalData.gBinCacheSource and self.Target in [None, > "", "all"]: > PcdMa.GenModuleFilesHash(share_data) > PcdMa.GenPreMakefileHash(share_data) > if PcdMa.CanSkipbyPreMakefileCache(share_data): > - continue > + continue > > PcdMa.CreateCodeFile(False) > PcdMa.CreateMakeFile(False,GenFfsList = > DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, > PcdMa.Arch),[])) > > if GlobalData.gBinCacheSource and self.Target in [None, > "", "all"]: > @@ -1218,18 +1221,28 @@ class Build(): > mqueue = mp.Queue() > for m in AutoGenObject.GetAllModuleInfo: > mqueue.put(m) > > AutoGenObject.DataPipe.DataContainer = {"CommandTarget": > self.Target} > + AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": > AutoGenObject.Workspace._SrcTimeStamp} > + AutoGenObject.CreateLibModuelDirs() > + AutoGenObject.DataPipe.DataContainer = > {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList} > + AutoGenObject.DataPipe.DataContainer = > {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList} > + AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": > + AutoGenObject.Workspace.GenFdsCommandDict} > self.Progress.Start("Generating makefile and code") > data_pipe_file = os.path.join(AutoGenObject.BuildDir, > "GlobalVar_%s_%s.bin" % > (str(AutoGenObject.Guid),AutoGenObject.Arch)) > AutoGenObject.DataPipe.dump(data_pipe_file) > autogen_rt,errorcode = self.StartAutoGen(mqueue, > AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, > GlobalData.gCacheIR) > + AutoGenIdFile = > os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") > + with open(AutoGenIdFile,"w") as fw: > + fw.write("Arch=%s\n" % > "|".join((AutoGenObject.Workspace.ArchList))) > + fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir) > + fw.write("PlatformGuid=%s\n" % > + str(AutoGenObject.Guid)) > self.Progress.Stop("done!") > if not autogen_rt: > self.AutoGenMgr.TerminateWorkers() > - self.AutoGenMgr.join(0.1) > + self.AutoGenMgr.join(1) > raise FatalError(errorcode) > AutoGenObject.CreateCodeFile(False) > AutoGenObject.CreateMakeFile(False) > else: > # always recreate top/platform makefile when clean, just > in case of inconsistency @@ -1778,10 +1791,11 @@ class Build(): > self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList) > # > # Save MAP buffer into MAP file. > # > self._SaveMapFile (MapBuffer, Wa) > + self.CreateGuidedSectionToolsFile(Wa) > > ## Build active module for different build targets, different tool > chains and different archs > # > def _BuildModule(self): > for BuildTarget in self.BuildTargetList: > @@ -1852,13 +1866,13 @@ class Build(): > > if GlobalData.gBinCacheSource and self.Target in > [None, "", "all"]: > Ma.GenModuleFilesHash(GlobalData.gCacheIR) > Ma.GenPreMakefileHash(GlobalData.gCacheIR) > if > Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): > - self.HashSkipModules.append(Ma) > - EdkLogger.quiet("cache hit: %s[%s]" % > (Ma.MetaFile.Path, Ma.Arch)) > - continue > + self.HashSkipModules.append(Ma) > + EdkLogger.quiet("cache hit: %s[%s]" % > (Ma.MetaFile.Path, Ma.Arch)) > + continue > > # Not to auto-gen for targets 'clean', > 'cleanlib', 'cleanall', 'run', 'fds' > if self.Target not in ['clean', 'cleanlib', > 'cleanall', 'run', 'fds']: > # for target which must generate AutoGen > code and makefile > if not self.SkipAutoGen or self.Target == > 'genc': > @@ -1987,125 +2001,221 @@ class Build(): > GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, > ArchList, GlobalData) > for Cmd in GenFfsDict: > tmpInf, tmpArch = GenFfsDict[Cmd] > CmdSetDict[tmpInf, tmpArch].add(Cmd) > return CmdSetDict > + def VerifyAutoGenFiles(self): > + AutoGenIdFile = > os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") > + try: > + with open(AutoGenIdFile) as fd: > + lines = fd.readlines() > + except: > + return None > + for line in lines: > + if "Arch" in line: > + ArchList = line.strip().split("=")[1].split("|") > + if "BuildDir" in line: > + BuildDir = line.split("=")[1].strip() > + if "PlatformGuid" in line: > + PlatformGuid = line.split("=")[1].strip() > + GlobalVarList = [] > + for arch in ArchList: > + global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % > (str(PlatformGuid),arch)) > + if not os.path.exists(global_var): > + return None > + GlobalVarList.append(global_var) > + for global_var in GlobalVarList: > + data_pipe = MemoryDataPipe() > + data_pipe.load(global_var) > + target = data_pipe.Get("P_Info").get("Target") > + toolchain = data_pipe.Get("P_Info").get("ToolChain") > + archlist = data_pipe.Get("P_Info").get("ArchList") > + Arch = data_pipe.Get("P_Info").get("Arch") > + active_p = data_pipe.Get("P_Info").get("ActivePlatform") > + workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir") > + PackagesPath = os.getenv("PACKAGES_PATH") > + mws.setWs(workspacedir, PackagesPath) > + LibraryBuildDirectoryList = > data_pipe.Get("LibraryBuildDirectoryList") > + ModuleBuildDirectoryList = > + data_pipe.Get("ModuleBuildDirectoryList") > + > + for m_build_dir in LibraryBuildDirectoryList: > + if not > os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])): > + return None > + for m_build_dir in ModuleBuildDirectoryList: > + if not > os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])): > + return None > + Wa = WorkSpaceInfo( > + workspacedir,active_p,target,toolchain,archlist > + ) > + Pa = PlatformInfo(Wa, active_p, target, toolchain, > Arch,data_pipe) > + Wa.AutoGenObjectList.append(Pa) > + return Wa > + def SetupMakeSetting(self,Wa): > + BuildModules = [] > + for Pa in Wa.AutoGenObjectList: > + for m in Pa._MbList: > + ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, > Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe) > + BuildModules.append(ma) > + fdf_file = Wa.FlashDefinition > + if fdf_file: > + Fdf = FdfParser(fdf_file.Path) > + Fdf.ParseFile() > + GlobalData.gFdfParser = Fdf > + if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict: > + FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName] > + for FdRegion in FdDict.RegionList: > + if str(FdRegion.RegionType) is 'FILE' and > self.Platform.VpdToolGuid in str(FdRegion.RegionDataList): > + if int(FdRegion.Offset) % 8 != 0: > + EdkLogger.error("build", FORMAT_INVALID, > + 'The VPD Base Address %s must be 8-byte aligned.' % > (FdRegion.Offset)) > + Wa.FdfProfile = Fdf.Profile > + self.Fdf = Fdf > + else: > + self.Fdf = None > + return BuildModules > > ## Build a platform in multi-thread mode > # > - def _MultiThreadBuildPlatform(self): > - SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE > auto-generated\n', False) > - for BuildTarget in self.BuildTargetList: > - GlobalData.gGlobalDefines['TARGET'] = BuildTarget > - index = 0 > - for ToolChain in self.ToolChainList: > - WorkspaceAutoGenTime = time.time() > - GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain > - GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain > - GlobalData.gGlobalDefines['FAMILY'] = > self.ToolChainFamily[index] > - index += 1 > - Wa = WorkspaceAutoGen( > - self.WorkspaceDir, > - self.PlatformFile, > - BuildTarget, > - ToolChain, > - self.ArchList, > - self.BuildDatabase, > - self.TargetTxt, > - self.ToolDef, > - self.Fdf, > - self.FdList, > - self.FvList, > - self.CapList, > - self.SkuId, > - self.UniFlag, > - self.Progress > - ) > - self.Fdf = Wa.FdfFile > - self.LoadFixAddress = Wa.Platform.LoadFixAddress > - self.BuildReport.AddPlatformReport(Wa) > - Wa.CreateMakeFile(False) > + def PerformAutoGen(self,BuildTarget,ToolChain): > + WorkspaceAutoGenTime = time.time() > + Wa = WorkspaceAutoGen( > + self.WorkspaceDir, > + self.PlatformFile, > + BuildTarget, > + ToolChain, > + self.ArchList, > + self.BuildDatabase, > + self.TargetTxt, > + self.ToolDef, > + self.Fdf, > + self.FdList, > + self.FvList, > + self.CapList, > + self.SkuId, > + self.UniFlag, > + self.Progress > + ) > + self.Fdf = Wa.FdfFile > + self.LoadFixAddress = Wa.Platform.LoadFixAddress > + self.BuildReport.AddPlatformReport(Wa) > + Wa.CreateMakeFile(False) > > # Add ffs build to makefile > - CmdListDict = {} > - if GlobalData.gEnableGenfdsMultiThread and self.Fdf: > - CmdListDict = self._GenFfsCmd(Wa.ArchList) > + CmdListDict = {} > + if GlobalData.gEnableGenfdsMultiThread and self.Fdf: > + CmdListDict = self._GenFfsCmd(Wa.ArchList) > > - # Add Platform and Package level hash in share_data for > module hash calculation later > - if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: > - GlobalData.gCacheIR[('PlatformHash')] = > GlobalData.gPlatformHash > - for PkgName in GlobalData.gPackageHash.keys(): > - GlobalData.gCacheIR[(PkgName, 'PackageHash')] = > GlobalData.gPackageHash[PkgName] > + # Add Platform and Package level hash in share_data for module hash > calculation later > + if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: > + GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash > + for PkgName in GlobalData.gPackageHash.keys(): > + GlobalData.gCacheIR[(PkgName, 'PackageHash')] = > + GlobalData.gPackageHash[PkgName] > > - # multi-thread exit flag > - ExitFlag = threading.Event() > - ExitFlag.clear() > - self.AutoGenTime += int(round((time.time() - > WorkspaceAutoGenTime))) > - self.BuildModules = [] > - TotalModules = [] > - for Arch in Wa.ArchList: > - PcdMaList = [] > - AutoGenStart = time.time() > - GlobalData.gGlobalDefines['ARCH'] = Arch > - Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, > ToolChain, Arch) > - if Pa is None: > - continue > - ModuleList = [] > - for Inf in Pa.Platform.Modules: > - ModuleList.append(Inf) > + self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) > + BuildModules = [] > + TotalModules = [] > + for Arch in Wa.ArchList: > + PcdMaList = [] > + AutoGenStart = time.time() > + GlobalData.gGlobalDefines['ARCH'] = Arch > + Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, > ToolChain, Arch) > + if Pa is None: > + continue > + ModuleList = [] > + for Inf in Pa.Platform.Modules: > + ModuleList.append(Inf) > # Add the INF only list in FDF > - if GlobalData.gFdfParser is not None: > - for InfName in GlobalData.gFdfParser.Profile.InfList: > - Inf = PathClass(NormPath(InfName), > self.WorkspaceDir, Arch) > - if Inf in Pa.Platform.Modules: > - continue > - ModuleList.append(Inf) > - Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} > - Pa.DataPipe.DataContainer = {"Workspace_timestamp": > Wa._SrcTimeStamp} > - Pa.DataPipe.DataContainer = {"CommandTarget": > self.Target} > - for Module in ModuleList: > + if GlobalData.gFdfParser is not None: > + for InfName in GlobalData.gFdfParser.Profile.InfList: > + Inf = PathClass(NormPath(InfName), self.WorkspaceDir, > Arch) > + if Inf in Pa.Platform.Modules: > + continue > + ModuleList.append(Inf) > + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} > + Pa.DataPipe.DataContainer = {"Workspace_timestamp": > Wa._SrcTimeStamp} > + Pa.DataPipe.DataContainer = {"CommandTarget": self.Target} > + Pa.CreateLibModuelDirs() > + Pa.DataPipe.DataContainer = > {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList} > + Pa.DataPipe.DataContainer = > {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList} > + Pa.DataPipe.DataContainer = {"FdsCommandDict": > Wa.GenFdsCommandDict} > + ModuleCodaFile = {} > + for ma in Pa.ModuleAutoGenList: > + > + ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFil > + e.Path)] = [item.Target for item in > ma.CodaTargetList] > + Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile} > + for Module in ModuleList: > # Get ModuleAutoGen object to generate C code file > and makefile > - Ma = ModuleAutoGen(Wa, Module, BuildTarget, > ToolChain, Arch, self.PlatformFile,Pa.DataPipe) > + Ma = ModuleAutoGen(Wa, Module, BuildTarget, > + ToolChain, Arch, self.PlatformFile,Pa.DataPipe) > > - if Ma is None: > + if Ma is None: > + continue > + if Ma.PcdIsDriver: > + Ma.PlatformInfo = Pa > + Ma.Workspace = Wa > + PcdMaList.append(Ma) > + TotalModules.append(Ma) > + # Initialize all modules in tracking to 'FAIL' > + GlobalData.gModuleBuildTracking[Ma] = 'FAIL' > + > + > + mqueue = mp.Queue() > + for m in Pa.GetAllModuleInfo: > + mqueue.put(m) > + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" > % (str(Pa.Guid),Pa.Arch)) > + Pa.DataPipe.dump(data_pipe_file) > + > + autogen_rt, errorcode = self.StartAutoGen(mqueue, > + Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR) > + > + # Skip cache hit modules > + if GlobalData.gBinCacheSource: > + for Ma in TotalModules: > + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR > and \ > + GlobalData.gCacheIR[(Ma.MetaFile.Path, > Ma.Arch)].PreMakeCacheHit: > + self.HashSkipModules.append(Ma) > + continue > + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR > and \ > + GlobalData.gCacheIR[(Ma.MetaFile.Path, > Ma.Arch)].MakeCacheHit: > + self.HashSkipModules.append(Ma) > continue > - if Ma.PcdIsDriver: > - Ma.PlatformInfo = Pa > - Ma.Workspace = Wa > - PcdMaList.append(Ma) > - TotalModules.append(Ma) > - # Initialize all modules in tracking to 'FAIL' > - GlobalData.gModuleBuildTracking[Ma] = 'FAIL' > + BuildModules.append(Ma) > + else: > + BuildModules.extend(TotalModules) > > - mqueue = mp.Queue() > - for m in Pa.GetAllModuleInfo: > - mqueue.put(m) > - data_pipe_file = os.path.join(Pa.BuildDir, > "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) > - Pa.DataPipe.dump(data_pipe_file) > - autogen_rt, errorcode = self.StartAutoGen(mqueue, > Pa.DataPipe, self.SkipAutoGen, PcdMaList, > GlobalData.gCacheIR) > + if not autogen_rt: > + self.AutoGenMgr.TerminateWorkers() > + self.AutoGenMgr.join(1) > + raise FatalError(errorcode) > + self.AutoGenTime += int(round((time.time() - AutoGenStart))) > + AutoGenIdFile = > os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") > + with open(AutoGenIdFile,"w") as fw: > + fw.write("Arch=%s\n" % "|".join((Wa.ArchList))) > + fw.write("BuildDir=%s\n" % Wa.BuildDir) > + fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid)) > + self.Progress.Stop("done!") > + return Wa, BuildModules > > - # Skip cache hit modules > - if GlobalData.gBinCacheSource: > - for Ma in TotalModules: > - if (Ma.MetaFile.Path, Ma.Arch) in > GlobalData.gCacheIR and \ > - GlobalData.gCacheIR[(Ma.MetaFile.Path, > Ma.Arch)].PreMakeCacheHit: > - self.HashSkipModules.append(Ma) > - continue > - if (Ma.MetaFile.Path, Ma.Arch) in > GlobalData.gCacheIR and \ > - GlobalData.gCacheIR[(Ma.MetaFile.Path, > Ma.Arch)].MakeCacheHit: > - self.HashSkipModules.append(Ma) > - continue > - self.BuildModules.append(Ma) > + def _MultiThreadBuildPlatform(self): > + SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE > auto-generated\n', False) > + for BuildTarget in self.BuildTargetList: > + GlobalData.gGlobalDefines['TARGET'] = BuildTarget > + index = 0 > + for ToolChain in self.ToolChainList: > + GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain > + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain > + GlobalData.gGlobalDefines['FAMILY'] = > self.ToolChainFamily[index] > + index += 1 > + ExitFlag = threading.Event() > + ExitFlag.clear() > + if self.SkipAutoGen: > + Wa = self.VerifyAutoGenFiles() > + if Wa is None: > + Wa, self.BuildModules = > + self.PerformAutoGen(BuildTarget,ToolChain) > else: > - self.BuildModules.extend(TotalModules) > - > - if not autogen_rt: > - self.AutoGenMgr.TerminateWorkers() > - self.AutoGenMgr.join(0.1) > - raise FatalError(errorcode) > - self.AutoGenTime += int(round((time.time() - AutoGenStart))) > - self.Progress.Stop("done!") > + GlobalData.gAutoGenPhase = True > + self.BuildModules = self.SetupMakeSetting(Wa) > + else: > + Wa, self.BuildModules = > self.PerformAutoGen(BuildTarget,ToolChain) > + Pa = Wa.AutoGenObjectList[0] > + GlobalData.gAutoGenPhase = False > > if GlobalData.gBinCacheSource: > EdkLogger.quiet("Total cache hit driver num: %s, > cache miss driver num: %s" % (len(set(self.HashSkipModules)), > len(set(self.BuildModules)))) > CacheHitMa = set() > CacheNotHitMa = set() @@ -2136,18 +2246,17 @@ > class Build(): > # Start task scheduler > if not BuildTask.IsOnGoing(): > > BuildTask.StartScheduler(self.ThreadNumber, ExitFlag) > > # in case there's an interruption. we need a full > version of makefile for platform > - Pa.CreateMakeFile(False) > + > if BuildTask.HasError(): > self.invalidateHash() > EdkLogger.error("build", BUILD_ERROR, "Failed to > build module", ExtraData=GlobalData.gBuildingModule) > self.MakeTime += int(round((time.time() - > MakeStart))) > > MakeContiue = time.time() > - > # > # > # All modules have been put in build tasks queue. Tell task > scheduler > # to exit if all tasks are completed > # > @@ -2175,17 +2284,12 @@ class Build(): > if (Arch == 'IA32' or Arch == 'ARM') and > self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= > 0x100000000: > EdkLogger.error("build", > PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than > or equal to 4G for the platorm with IA32 or ARM arch modules") > # > # Get Module List > # > - ModuleList = {} > - for Pa in Wa.AutoGenObjectList: > - for Ma in Pa.ModuleAutoGenList: > - if Ma is None: > - continue > - if not Ma.IsLibrary: > - ModuleList[Ma.Guid.upper()] = Ma > + ModuleList = {ma.Guid.upper():ma for ma in > + self.BuildModules} > + > # > # Rebase module to the preferred memory address before > GenFds > # > MapBuffer = [] > if self.LoadFixAddress != 0: > @@ -2206,33 +2310,17 @@ class Build(): > self.GenFdsTime += int(round((time.time() - > GenFdsStart))) > # > # Save MAP buffer into MAP file. > # > self._SaveMapFile(MapBuffer, Wa) > + self.CreateGuidedSectionToolsFile(Wa) > self.invalidateHash() > - > ## Generate GuidedSectionTools.txt in the FV directories. > # > - def CreateGuidedSectionToolsFile(self): > + def CreateGuidedSectionToolsFile(self,Wa): > for BuildTarget in self.BuildTargetList: > for ToolChain in self.ToolChainList: > - Wa = WorkspaceAutoGen( > - self.WorkspaceDir, > - self.PlatformFile, > - BuildTarget, > - ToolChain, > - self.ArchList, > - self.BuildDatabase, > - self.TargetTxt, > - self.ToolDef, > - self.Fdf, > - self.FdList, > - self.FvList, > - self.CapList, > - self.SkuId, > - self.UniFlag > - ) > FvDir = Wa.FvDir > if not os.path.exists(FvDir): > continue > > for Arch in self.ArchList: > @@ -2285,11 +2373,10 @@ class Build(): > if not self.SpawnMode or self.Target not in ["", "all"]: > self.SpawnMode = False > self._BuildPlatform() > else: > self._MultiThreadBuildPlatform() > - self.CreateGuidedSectionToolsFile() > else: > self.SpawnMode = False > self._BuildModule() > > if self.Target == 'cleanall': > -- > 2.20.1.windows.1 > > > -=-=-=-=-=-=-=-=-=-=-=- Groups.io Links: You receive all messages sent to this group. View/Reply Online (#46901): https://edk2.groups.io/g/devel/message/46901 Mute This Topic: https://groups.io/mt/33151466/21656 Group Owner: devel+ow...@edk2.groups.io Unsubscribe: https://edk2.groups.io/g/devel/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-