Liming Gao <liming....@intel.com> >-----Original Message----- >From: Feng, Bob C >Sent: Thursday, September 05, 2019 10:06 PM >To: devel@edk2.groups.io; Feng, Bob C <bob.c.f...@intel.com>; Gao, Liming ><liming....@intel.com> >Cc: Shi, Steven <steven....@intel.com> >Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for >IgnoreAutoGen > >Sorry for the incomplete mail. > >I verified this patch on Ovmf, Emulator, MinPlatforms and Intel Server >Platform. > >Thanks, >Bob > >-----Original Message----- >From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Bob Feng >Sent: Thursday, September 5, 2019 10:04 PM >To: Gao, Liming <liming....@intel.com>; devel@edk2.groups.io >Cc: Shi, Steven <steven....@intel.com> >Subject: Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for >IgnoreAutoGen > >Liming, > >Yes, this change only impact -u option. > >I separate _MultiThreadBuildPlatform function into two functions and create 2 >new functions specially for IgnoreAutoGen, so the there looks big change for >build.py in this patch. I verified this patch on Ovmf, E > >-Bob > >-----Original Message----- >From: Gao, Liming >Sent: Thursday, September 5, 2019 9:12 PM >To: devel@edk2.groups.io; Feng, Bob C <bob.c.f...@intel.com> >Cc: Shi, Steven <steven....@intel.com> >Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for >IgnoreAutoGen > >Bob: > Does this change only impact -u option behavior? > >Thanks >Liming >> -----Original Message----- >> From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of >> Bob Feng >> Sent: Thursday, September 5, 2019 5:05 PM >> To: devel@edk2.groups.io >> Cc: Gao, Liming <liming....@intel.com>; Shi, Steven >> <steven....@intel.com>; Feng, Bob C <bob.c.f...@intel.com> >> Subject: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for >> IgnoreAutoGen >> >> https://bugzilla.tianocore.org/show_bug.cgi?id=2080 >> >> This patch is to improve build -u option to re-use >> GlobalVar_<platformguid>_<arch>.bin file which is introduced by >> multiple-process-autogen feature. >> >> Cc: Liming Gao <liming....@intel.com> >> Cc: Steven Shi <steven....@intel.com> >> Signed-off-by: Bob Feng <bob.c.f...@intel.com> >> --- >> .../Source/Python/AutoGen/AutoGenWorker.py | 2 +- >> .../Source/Python/AutoGen/ModuleAutoGen.py | 4 +- >> .../Python/AutoGen/ModuleAutoGenHelper.py | 29 ++ >> .../Source/Python/AutoGen/PlatformAutoGen.py | 2 + >> .../Source/Python/AutoGen/WorkspaceAutoGen.py | 1 - >> BaseTools/Source/Python/build/build.py | 355 +++++++++++------- >> 6 files changed, 255 insertions(+), 138 deletions(-) >> >> diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py >> b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py >> index 2e68538b1cb4..f488ae9d5f80 100755 >> --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py >> +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py >> @@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process): >> Ma.ReferenceModules = >Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] >> if GlobalData.gBinCacheSource and CommandTarget in [None, >> "", >"all"]: >> Ma.GenModuleFilesHash(GlobalData.gCacheIR) >> Ma.GenPreMakefileHash(GlobalData.gCacheIR) >> if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): >> - continue >> + continue >> >> Ma.CreateCodeFile(False) >> >> Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, >> Ma.Arch),[])) >> >> if GlobalData.gBinCacheSource and CommandTarget in [None, >> "", >"all"]: >> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py >> b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py >> index 3bb7e91154ac..5f28681e3146 100755 >> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py >> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py >> @@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen): >> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: >> self.GenModuleFilesHash(gDict) >> >> if not (self.MetaFile.Path, self.Arch) in gDict or \ >> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: >> - EdkLogger.quiet("[cache warning]: Cannot generate >ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, >> self.Arch)) >> - return >> + EdkLogger.quiet("[cache warning]: Cannot generate >> + ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, >> self.Arch)) >> + return >> >> # Initialze hash object >> m = hashlib.md5() >> >> # Add Platform level hash >> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py >> b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py >> index c7591253debd..9dd93b9beb12 100644 >> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py >> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py >> @@ -234,22 +234,51 @@ class AutoGenInfo(object): >> # >> #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in >> multiple thread # class WorkSpaceInfo(AutoGenInfo): >> def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch): >> + if not hasattr(self, "_Init"): >> + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch) >> + self._Init = True >> + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch): >> self._SrcTimeStamp = 0 >> self.Db = BuildDB >> self.BuildDatabase = self.Db.BuildObject >> self.Target = Target >> self.ToolChain = ToolChain >> self.WorkspaceDir = Workspace >> self.ActivePlatform = MetaFile >> self.ArchList = Arch >> + self.AutoGenObjectList = [] >> + @property >> + def BuildDir(self): >> + return self.AutoGenObjectList[0].BuildDir >> >> + @property >> + def Name(self): >> + return self.AutoGenObjectList[0].Platform.PlatformName >> + >> + @property >> + def FlashDefinition(self): >> + return self.AutoGenObjectList[0].Platform.FlashDefinition >> + @property >> + def GenFdsCommandDict(self): >> + FdsCommandDict = >self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict") >> + if FdsCommandDict: >> + return FdsCommandDict >> + return {} >> + >> + @cached_property >> + def FvDir(self): >> + return os.path.join(self.BuildDir, TAB_FV_DIRECTORY) >> >> class PlatformInfo(AutoGenInfo): >> def __init__(self, Workspace, MetaFile, Target, ToolChain, >> Arch,DataPipe): >> + if not hasattr(self, "_Init"): >> + self.do_init(Workspace, MetaFile, Target, ToolChain, >> Arch,DataPipe) >> + self._Init = True >> + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch,DataPipe): >> self.Wa = Workspace >> self.WorkspaceDir = self.Wa.WorkspaceDir >> self.MetaFile = MetaFile >> self.Arch = Arch >> self.Target = Target >> diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py >> b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py >> index 565424a95ead..1e17b6687129 100644 >> --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py >> +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py >> @@ -186,11 +186,13 @@ class PlatformAutoGen(AutoGen): >> key = (Ma.MetaFile.File, self.Arch) >> if key in FfsCommand: >> Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key]) >> else: >> Ma.CreateMakeFile(CreateModuleMakeFile) >> + self.CreateLibModuelDirs() >> >> + def CreateLibModuelDirs(self): >> # no need to create makefile for the platform more than once >> if self.IsMakeFileCreated: >> return >> >> # create library/module build dirs for platform diff --git >> a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py >> b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py >> index 24942674721f..f9d2c216169c 100644 >> --- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py >> +++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py >> @@ -126,11 +126,10 @@ class WorkspaceAutoGen(AutoGen): >> self._CheckPcdDefineAndType() >> >> self.CreateBuildOptionsFile() >> self.CreatePcdTokenNumberFile() >> self.CreateModuleHashInfo() >> - GlobalData.gAutoGenPhase = False >> >> # >> # Merge Arch >> # >> def MergeArch(self): >> diff --git a/BaseTools/Source/Python/build/build.py >> b/BaseTools/Source/Python/build/build.py >> index 0406ac314b65..db57c3e526da 100755 >> --- a/BaseTools/Source/Python/build/build.py >> +++ b/BaseTools/Source/Python/build/build.py >> @@ -52,10 +52,13 @@ from PatchPcdValue.PatchPcdValue import >> PatchBinaryFile >> >> import Common.GlobalData as GlobalData from GenFds.GenFds import >> GenFds, GenFdsApi import multiprocessing as mp from multiprocessing >> import Manager >> +from AutoGen.DataPipe import MemoryDataPipe from >> +AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo >from >> +GenFds.FdfParser import FdfParser >> >> >> ## standard targets of build command >> gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', >> 'fds', 'clean', 'cleanall', 'cleanlib', 'run'] >> >> @@ -837,11 +840,11 @@ class Build(): >> for PcdMa in PcdMaList: >> if GlobalData.gBinCacheSource and self.Target in [None, >> "", "all"]: >> PcdMa.GenModuleFilesHash(share_data) >> PcdMa.GenPreMakefileHash(share_data) >> if PcdMa.CanSkipbyPreMakefileCache(share_data): >> - continue >> + continue >> >> PcdMa.CreateCodeFile(False) >> PcdMa.CreateMakeFile(False,GenFfsList = >> DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, >> PcdMa.Arch),[])) >> >> if GlobalData.gBinCacheSource and self.Target in [None, >> "", "all"]: >> @@ -1218,18 +1221,28 @@ class Build(): >> mqueue = mp.Queue() >> for m in AutoGenObject.GetAllModuleInfo: >> mqueue.put(m) >> >> AutoGenObject.DataPipe.DataContainer = {"CommandTarget": >> self.Target} >> + AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": >AutoGenObject.Workspace._SrcTimeStamp} >> + AutoGenObject.CreateLibModuelDirs() >> + AutoGenObject.DataPipe.DataContainer = >{"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList} >> + AutoGenObject.DataPipe.DataContainer = >{"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList} >> + AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": >> + AutoGenObject.Workspace.GenFdsCommandDict} >> self.Progress.Start("Generating makefile and code") >> data_pipe_file = os.path.join(AutoGenObject.BuildDir, >> "GlobalVar_%s_%s.bin" % >> (str(AutoGenObject.Guid),AutoGenObject.Arch)) >> AutoGenObject.DataPipe.dump(data_pipe_file) >> autogen_rt,errorcode = self.StartAutoGen(mqueue, >> AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, >> GlobalData.gCacheIR) >> + AutoGenIdFile = >os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") >> + with open(AutoGenIdFile,"w") as fw: >> + fw.write("Arch=%s\n" % >"|".join((AutoGenObject.Workspace.ArchList))) >> + fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir) >> + fw.write("PlatformGuid=%s\n" % >> + str(AutoGenObject.Guid)) >> self.Progress.Stop("done!") >> if not autogen_rt: >> self.AutoGenMgr.TerminateWorkers() >> - self.AutoGenMgr.join(0.1) >> + self.AutoGenMgr.join(1) >> raise FatalError(errorcode) >> AutoGenObject.CreateCodeFile(False) >> AutoGenObject.CreateMakeFile(False) >> else: >> # always recreate top/platform makefile when clean, just >> in case of inconsistency @@ -1778,10 +1791,11 @@ class Build(): >> self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList) >> # >> # Save MAP buffer into MAP file. >> # >> self._SaveMapFile (MapBuffer, Wa) >> + self.CreateGuidedSectionToolsFile(Wa) >> >> ## Build active module for different build targets, different tool >> chains >and different archs >> # >> def _BuildModule(self): >> for BuildTarget in self.BuildTargetList: >> @@ -1852,13 +1866,13 @@ class Build(): >> >> if GlobalData.gBinCacheSource and self.Target >> in [None, "", >"all"]: >> Ma.GenModuleFilesHash(GlobalData.gCacheIR) >> Ma.GenPreMakefileHash(GlobalData.gCacheIR) >> if >> Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): >> - self.HashSkipModules.append(Ma) >> - EdkLogger.quiet("cache hit: %s[%s]" % >> (Ma.MetaFile.Path, >Ma.Arch)) >> - continue >> + self.HashSkipModules.append(Ma) >> + EdkLogger.quiet("cache hit: %s[%s]" % >(Ma.MetaFile.Path, Ma.Arch)) >> + continue >> >> # Not to auto-gen for targets 'clean', >> 'cleanlib', 'cleanall', 'run', >'fds' >> if self.Target not in ['clean', 'cleanlib', >> 'cleanall', 'run', 'fds']: >> # for target which must generate AutoGen >> code and >makefile >> if not self.SkipAutoGen or self.Target == >> 'genc': >> @@ -1987,125 +2001,221 @@ class Build(): >> GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, >ArchList, GlobalData) >> for Cmd in GenFfsDict: >> tmpInf, tmpArch = GenFfsDict[Cmd] >> CmdSetDict[tmpInf, tmpArch].add(Cmd) >> return CmdSetDict >> + def VerifyAutoGenFiles(self): >> + AutoGenIdFile = >os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") >> + try: >> + with open(AutoGenIdFile) as fd: >> + lines = fd.readlines() >> + except: >> + return None >> + for line in lines: >> + if "Arch" in line: >> + ArchList = line.strip().split("=")[1].split("|") >> + if "BuildDir" in line: >> + BuildDir = line.split("=")[1].strip() >> + if "PlatformGuid" in line: >> + PlatformGuid = line.split("=")[1].strip() >> + GlobalVarList = [] >> + for arch in ArchList: >> + global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % >(str(PlatformGuid),arch)) >> + if not os.path.exists(global_var): >> + return None >> + GlobalVarList.append(global_var) >> + for global_var in GlobalVarList: >> + data_pipe = MemoryDataPipe() >> + data_pipe.load(global_var) >> + target = data_pipe.Get("P_Info").get("Target") >> + toolchain = data_pipe.Get("P_Info").get("ToolChain") >> + archlist = data_pipe.Get("P_Info").get("ArchList") >> + Arch = data_pipe.Get("P_Info").get("Arch") >> + active_p = data_pipe.Get("P_Info").get("ActivePlatform") >> + workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir") >> + PackagesPath = os.getenv("PACKAGES_PATH") >> + mws.setWs(workspacedir, PackagesPath) >> + LibraryBuildDirectoryList = >> data_pipe.Get("LibraryBuildDirectoryList") >> + ModuleBuildDirectoryList = >> + data_pipe.Get("ModuleBuildDirectoryList") >> + >> + for m_build_dir in LibraryBuildDirectoryList: >> + if not >os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[Ge >nMake.gMakeType])): >> + return None >> + for m_build_dir in ModuleBuildDirectoryList: >> + if not >os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[Ge >nMake.gMakeType])): >> + return None >> + Wa = WorkSpaceInfo( >> + workspacedir,active_p,target,toolchain,archlist >> + ) >> + Pa = PlatformInfo(Wa, active_p, target, toolchain, >> Arch,data_pipe) >> + Wa.AutoGenObjectList.append(Pa) >> + return Wa >> + def SetupMakeSetting(self,Wa): >> + BuildModules = [] >> + for Pa in Wa.AutoGenObjectList: >> + for m in Pa._MbList: >> + ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, >Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe) >> + BuildModules.append(ma) >> + fdf_file = Wa.FlashDefinition >> + if fdf_file: >> + Fdf = FdfParser(fdf_file.Path) >> + Fdf.ParseFile() >> + GlobalData.gFdfParser = Fdf >> + if Fdf.CurrentFdName and Fdf.CurrentFdName in >> Fdf.Profile.FdDict: >> + FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName] >> + for FdRegion in FdDict.RegionList: >> + if str(FdRegion.RegionType) is 'FILE' and >self.Platform.VpdToolGuid in str(FdRegion.RegionDataList): >> + if int(FdRegion.Offset) % 8 != 0: >> + EdkLogger.error("build", FORMAT_INVALID, >> + 'The VPD Base Address %s must be 8-byte aligned.' % >> (FdRegion.Offset)) >> + Wa.FdfProfile = Fdf.Profile >> + self.Fdf = Fdf >> + else: >> + self.Fdf = None >> + return BuildModules >> >> ## Build a platform in multi-thread mode >> # >> - def _MultiThreadBuildPlatform(self): >> - SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE >auto-generated\n', False) >> - for BuildTarget in self.BuildTargetList: >> - GlobalData.gGlobalDefines['TARGET'] = BuildTarget >> - index = 0 >> - for ToolChain in self.ToolChainList: >> - WorkspaceAutoGenTime = time.time() >> - GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain >> - GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain >> - GlobalData.gGlobalDefines['FAMILY'] = >> self.ToolChainFamily[index] >> - index += 1 >> - Wa = WorkspaceAutoGen( >> - self.WorkspaceDir, >> - self.PlatformFile, >> - BuildTarget, >> - ToolChain, >> - self.ArchList, >> - self.BuildDatabase, >> - self.TargetTxt, >> - self.ToolDef, >> - self.Fdf, >> - self.FdList, >> - self.FvList, >> - self.CapList, >> - self.SkuId, >> - self.UniFlag, >> - self.Progress >> - ) >> - self.Fdf = Wa.FdfFile >> - self.LoadFixAddress = Wa.Platform.LoadFixAddress >> - self.BuildReport.AddPlatformReport(Wa) >> - Wa.CreateMakeFile(False) >> + def PerformAutoGen(self,BuildTarget,ToolChain): >> + WorkspaceAutoGenTime = time.time() >> + Wa = WorkspaceAutoGen( >> + self.WorkspaceDir, >> + self.PlatformFile, >> + BuildTarget, >> + ToolChain, >> + self.ArchList, >> + self.BuildDatabase, >> + self.TargetTxt, >> + self.ToolDef, >> + self.Fdf, >> + self.FdList, >> + self.FvList, >> + self.CapList, >> + self.SkuId, >> + self.UniFlag, >> + self.Progress >> + ) >> + self.Fdf = Wa.FdfFile >> + self.LoadFixAddress = Wa.Platform.LoadFixAddress >> + self.BuildReport.AddPlatformReport(Wa) >> + Wa.CreateMakeFile(False) >> >> # Add ffs build to makefile >> - CmdListDict = {} >> - if GlobalData.gEnableGenfdsMultiThread and self.Fdf: >> - CmdListDict = self._GenFfsCmd(Wa.ArchList) >> + CmdListDict = {} >> + if GlobalData.gEnableGenfdsMultiThread and self.Fdf: >> + CmdListDict = self._GenFfsCmd(Wa.ArchList) >> >> - # Add Platform and Package level hash in share_data for >> module >hash calculation later >> - if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: >> - GlobalData.gCacheIR[('PlatformHash')] = >GlobalData.gPlatformHash >> - for PkgName in GlobalData.gPackageHash.keys(): >> - GlobalData.gCacheIR[(PkgName, 'PackageHash')] = >GlobalData.gPackageHash[PkgName] >> + # Add Platform and Package level hash in share_data for module hash >calculation later >> + if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: >> + GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash >> + for PkgName in GlobalData.gPackageHash.keys(): >> + GlobalData.gCacheIR[(PkgName, 'PackageHash')] = >> + GlobalData.gPackageHash[PkgName] >> >> - # multi-thread exit flag >> - ExitFlag = threading.Event() >> - ExitFlag.clear() >> - self.AutoGenTime += int(round((time.time() - >WorkspaceAutoGenTime))) >> - self.BuildModules = [] >> - TotalModules = [] >> - for Arch in Wa.ArchList: >> - PcdMaList = [] >> - AutoGenStart = time.time() >> - GlobalData.gGlobalDefines['ARCH'] = Arch >> - Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, >ToolChain, Arch) >> - if Pa is None: >> - continue >> - ModuleList = [] >> - for Inf in Pa.Platform.Modules: >> - ModuleList.append(Inf) >> + self.AutoGenTime += int(round((time.time() - >WorkspaceAutoGenTime))) >> + BuildModules = [] >> + TotalModules = [] >> + for Arch in Wa.ArchList: >> + PcdMaList = [] >> + AutoGenStart = time.time() >> + GlobalData.gGlobalDefines['ARCH'] = Arch >> + Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, >> ToolChain, >Arch) >> + if Pa is None: >> + continue >> + ModuleList = [] >> + for Inf in Pa.Platform.Modules: >> + ModuleList.append(Inf) >> # Add the INF only list in FDF >> - if GlobalData.gFdfParser is not None: >> - for InfName in >> GlobalData.gFdfParser.Profile.InfList: >> - Inf = PathClass(NormPath(InfName), >> self.WorkspaceDir, Arch) >> - if Inf in Pa.Platform.Modules: >> - continue >> - ModuleList.append(Inf) >> - Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} >> - Pa.DataPipe.DataContainer = {"Workspace_timestamp": >Wa._SrcTimeStamp} >> - Pa.DataPipe.DataContainer = {"CommandTarget": >> self.Target} >> - for Module in ModuleList: >> + if GlobalData.gFdfParser is not None: >> + for InfName in GlobalData.gFdfParser.Profile.InfList: >> + Inf = PathClass(NormPath(InfName), self.WorkspaceDir, >> Arch) >> + if Inf in Pa.Platform.Modules: >> + continue >> + ModuleList.append(Inf) >> + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} >> + Pa.DataPipe.DataContainer = {"Workspace_timestamp": >Wa._SrcTimeStamp} >> + Pa.DataPipe.DataContainer = {"CommandTarget": self.Target} >> + Pa.CreateLibModuelDirs() >> + Pa.DataPipe.DataContainer = >{"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList} >> + Pa.DataPipe.DataContainer = >{"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList} >> + Pa.DataPipe.DataContainer = {"FdsCommandDict": >Wa.GenFdsCommandDict} >> + ModuleCodaFile = {} >> + for ma in Pa.ModuleAutoGenList: >> + >> + ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFil >> + e.Path)] = [item.Target for item in >> ma.CodaTargetList] >> + Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile} >> + for Module in ModuleList: >> # Get ModuleAutoGen object to generate C code file >> and >makefile >> - Ma = ModuleAutoGen(Wa, Module, BuildTarget, >> ToolChain, >Arch, self.PlatformFile,Pa.DataPipe) >> + Ma = ModuleAutoGen(Wa, Module, BuildTarget, >> + ToolChain, Arch, self.PlatformFile,Pa.DataPipe) >> >> - if Ma is None: >> + if Ma is None: >> + continue >> + if Ma.PcdIsDriver: >> + Ma.PlatformInfo = Pa >> + Ma.Workspace = Wa >> + PcdMaList.append(Ma) >> + TotalModules.append(Ma) >> + # Initialize all modules in tracking to 'FAIL' >> + GlobalData.gModuleBuildTracking[Ma] = 'FAIL' >> + >> + >> + mqueue = mp.Queue() >> + for m in Pa.GetAllModuleInfo: >> + mqueue.put(m) >> + data_pipe_file = os.path.join(Pa.BuildDir, >> "GlobalVar_%s_%s.bin" % >(str(Pa.Guid),Pa.Arch)) >> + Pa.DataPipe.dump(data_pipe_file) >> + >> + autogen_rt, errorcode = self.StartAutoGen(mqueue, >> + Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR) >> + >> + # Skip cache hit modules >> + if GlobalData.gBinCacheSource: >> + for Ma in TotalModules: >> + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR >> and \ >> + GlobalData.gCacheIR[(Ma.MetaFile.Path, >Ma.Arch)].PreMakeCacheHit: >> + self.HashSkipModules.append(Ma) >> + continue >> + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR >> and \ >> + GlobalData.gCacheIR[(Ma.MetaFile.Path, >Ma.Arch)].MakeCacheHit: >> + self.HashSkipModules.append(Ma) >> continue >> - if Ma.PcdIsDriver: >> - Ma.PlatformInfo = Pa >> - Ma.Workspace = Wa >> - PcdMaList.append(Ma) >> - TotalModules.append(Ma) >> - # Initialize all modules in tracking to 'FAIL' >> - GlobalData.gModuleBuildTracking[Ma] = 'FAIL' >> + BuildModules.append(Ma) >> + else: >> + BuildModules.extend(TotalModules) >> >> - mqueue = mp.Queue() >> - for m in Pa.GetAllModuleInfo: >> - mqueue.put(m) >> - data_pipe_file = os.path.join(Pa.BuildDir, >"GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) >> - Pa.DataPipe.dump(data_pipe_file) >> - autogen_rt, errorcode = self.StartAutoGen(mqueue, >> Pa.DataPipe, >self.SkipAutoGen, PcdMaList, >> GlobalData.gCacheIR) >> + if not autogen_rt: >> + self.AutoGenMgr.TerminateWorkers() >> + self.AutoGenMgr.join(1) >> + raise FatalError(errorcode) >> + self.AutoGenTime += int(round((time.time() - AutoGenStart))) >> + AutoGenIdFile = >os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") >> + with open(AutoGenIdFile,"w") as fw: >> + fw.write("Arch=%s\n" % "|".join((Wa.ArchList))) >> + fw.write("BuildDir=%s\n" % Wa.BuildDir) >> + fw.write("PlatformGuid=%s\n" % >str(Wa.AutoGenObjectList[0].Guid)) >> + self.Progress.Stop("done!") >> + return Wa, BuildModules >> >> - # Skip cache hit modules >> - if GlobalData.gBinCacheSource: >> - for Ma in TotalModules: >> - if (Ma.MetaFile.Path, Ma.Arch) in >> GlobalData.gCacheIR and \ >> - GlobalData.gCacheIR[(Ma.MetaFile.Path, >Ma.Arch)].PreMakeCacheHit: >> - self.HashSkipModules.append(Ma) >> - continue >> - if (Ma.MetaFile.Path, Ma.Arch) in >> GlobalData.gCacheIR and \ >> - GlobalData.gCacheIR[(Ma.MetaFile.Path, >Ma.Arch)].MakeCacheHit: >> - self.HashSkipModules.append(Ma) >> - continue >> - self.BuildModules.append(Ma) >> + def _MultiThreadBuildPlatform(self): >> + SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE >auto-generated\n', False) >> + for BuildTarget in self.BuildTargetList: >> + GlobalData.gGlobalDefines['TARGET'] = BuildTarget >> + index = 0 >> + for ToolChain in self.ToolChainList: >> + GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain >> + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain >> + GlobalData.gGlobalDefines['FAMILY'] = >> self.ToolChainFamily[index] >> + index += 1 >> + ExitFlag = threading.Event() >> + ExitFlag.clear() >> + if self.SkipAutoGen: >> + Wa = self.VerifyAutoGenFiles() >> + if Wa is None: >> + Wa, self.BuildModules = >> + self.PerformAutoGen(BuildTarget,ToolChain) >> else: >> - self.BuildModules.extend(TotalModules) >> - >> - if not autogen_rt: >> - self.AutoGenMgr.TerminateWorkers() >> - self.AutoGenMgr.join(0.1) >> - raise FatalError(errorcode) >> - self.AutoGenTime += int(round((time.time() - AutoGenStart))) >> - self.Progress.Stop("done!") >> + GlobalData.gAutoGenPhase = True >> + self.BuildModules = self.SetupMakeSetting(Wa) >> + else: >> + Wa, self.BuildModules = >self.PerformAutoGen(BuildTarget,ToolChain) >> + Pa = Wa.AutoGenObjectList[0] >> + GlobalData.gAutoGenPhase = False >> >> if GlobalData.gBinCacheSource: >> EdkLogger.quiet("Total cache hit driver num: %s, >> cache miss driver num: %s" % (len(set(self.HashSkipModules)), >> len(set(self.BuildModules)))) >> CacheHitMa = set() >> CacheNotHitMa = set() @@ -2136,18 +2246,17 @@ >> class Build(): >> # Start task scheduler >> if not BuildTask.IsOnGoing(): >> >> BuildTask.StartScheduler(self.ThreadNumber, ExitFlag) >> >> # in case there's an interruption. we need a full >> version of >makefile for platform >> - Pa.CreateMakeFile(False) >> + >> if BuildTask.HasError(): >> self.invalidateHash() >> EdkLogger.error("build", BUILD_ERROR, "Failed to >> build >module", ExtraData=GlobalData.gBuildingModule) >> self.MakeTime += int(round((time.time() - >> MakeStart))) >> >> MakeContiue = time.time() >> - >> # >> # >> # All modules have been put in build tasks queue. Tell task >scheduler >> # to exit if all tasks are completed >> # >> @@ -2175,17 +2284,12 @@ class Build(): >> if (Arch == 'IA32' or Arch == 'ARM') and >> self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= >0x100000000: >> EdkLogger.error("build", >> PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to >larger than or equal to 4G for the platorm with IA32 or ARM arch modules") >> # >> # Get Module List >> # >> - ModuleList = {} >> - for Pa in Wa.AutoGenObjectList: >> - for Ma in Pa.ModuleAutoGenList: >> - if Ma is None: >> - continue >> - if not Ma.IsLibrary: >> - ModuleList[Ma.Guid.upper()] = Ma >> + ModuleList = {ma.Guid.upper():ma for ma in >> + self.BuildModules} >> + >> # >> # Rebase module to the preferred memory address before >GenFds >> # >> MapBuffer = [] >> if self.LoadFixAddress != 0: >> @@ -2206,33 +2310,17 @@ class Build(): >> self.GenFdsTime += int(round((time.time() - >> GenFdsStart))) >> # >> # Save MAP buffer into MAP file. >> # >> self._SaveMapFile(MapBuffer, Wa) >> + self.CreateGuidedSectionToolsFile(Wa) >> self.invalidateHash() >> - >> ## Generate GuidedSectionTools.txt in the FV directories. >> # >> - def CreateGuidedSectionToolsFile(self): >> + def CreateGuidedSectionToolsFile(self,Wa): >> for BuildTarget in self.BuildTargetList: >> for ToolChain in self.ToolChainList: >> - Wa = WorkspaceAutoGen( >> - self.WorkspaceDir, >> - self.PlatformFile, >> - BuildTarget, >> - ToolChain, >> - self.ArchList, >> - self.BuildDatabase, >> - self.TargetTxt, >> - self.ToolDef, >> - self.Fdf, >> - self.FdList, >> - self.FvList, >> - self.CapList, >> - self.SkuId, >> - self.UniFlag >> - ) >> FvDir = Wa.FvDir >> if not os.path.exists(FvDir): >> continue >> >> for Arch in self.ArchList: >> @@ -2285,11 +2373,10 @@ class Build(): >> if not self.SpawnMode or self.Target not in ["", "all"]: >> self.SpawnMode = False >> self._BuildPlatform() >> else: >> self._MultiThreadBuildPlatform() >> - self.CreateGuidedSectionToolsFile() >> else: >> self.SpawnMode = False >> self._BuildModule() >> >> if self.Target == 'cleanall': >> -- >> 2.20.1.windows.1 >> >> >> > > >
-=-=-=-=-=-=-=-=-=-=-=- Groups.io Links: You receive all messages sent to this group. View/Reply Online (#47023): https://edk2.groups.io/g/devel/message/47023 Mute This Topic: https://groups.io/mt/33151466/21656 Group Owner: devel+ow...@edk2.groups.io Unsubscribe: https://edk2.groups.io/g/devel/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
