Reviewed-by: Liming Gao <[email protected]> >-----Original Message----- >From: Zhu, Yonghong >Sent: Monday, September 04, 2017 4:44 PM >To: [email protected] >Cc: Gao, Liming <[email protected]> >Subject: [Patch] BaseTools: add support for BIOS build with binary cache > >Add three new options: >--hash enables hash-based caching during build process. when --hash is >enabled, build tool will base on the module hash value to do the >incremental build, without --hash, build tool will base on the >timestamp to do the incremental build. --hash option use md5 method to >get every hash value, DSC/FDF, tools_def.txt, build_rule.txt and build >command are calculated as global hash value, Package DEC and its >include header files are calculated as package hash value, Module >source files and its INF file are calculated as module hash value. >Library hash value will combine the global hash value and its dependent >package hash value. Driver hash value will combine the global hash >value, its dependent package hash value and its linked library hash >value. >When --hash and --bindest are specified, build tool will copy the >generated binary files for each module into the directory specified by >bindest at the build phase. Bindest caches all generated binary files. >When --hash and --binsource are specified, build tool will try to get >the binary files from the binary source directory at the build phase. >If the cached binary has the same hash value, it will be directly used. >Otherwise, build tool will compile the source files and generate the >binary files. > >Cc: Liming Gao <[email protected]> >Contributed-under: TianoCore Contribution Agreement 1.1 >Signed-off-by: Yonghong Zhu <[email protected]> >--- > BaseTools/Source/Python/AutoGen/AutoGen.py | 161 >+++++++++++++++++++++++++-- > BaseTools/Source/Python/Common/GlobalData.py | 7 ++ > BaseTools/Source/Python/build/build.py | 30 +++++ > 3 files changed, 189 insertions(+), 9 deletions(-) > >diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py >b/BaseTools/Source/Python/AutoGen/AutoGen.py >index 70e2e62..246bfec 100644 >--- a/BaseTools/Source/Python/AutoGen/AutoGen.py >+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py >@@ -41,10 +41,11 @@ import Common.VpdInfoFile as VpdInfoFile > from GenPcdDb import CreatePcdDatabaseCode > from Workspace.MetaFileCommentParser import UsageList > from Common.MultipleWorkspace import MultipleWorkspace as mws > import InfSectionParser > import datetime >+import hashlib > > ## Regular expression for splitting Dependency Expression string into tokens > gDepexTokenPattern = re.compile("(\(|\)|\w+| \S+\.inf)") > > # >@@ -263,10 +264,14 @@ class WorkspaceAutoGen(AutoGen): > self.FdfFile = FlashDefinitionFile > self.FdTargetList = Fds > self.FvTargetList = Fvs > self.CapTargetList = Caps > self.AutoGenObjectList = [] >+ self._BuildDir = None >+ self._FvDir = None >+ self._MakeFileDir = None >+ self._BuildCommand = None > > # there's many relative directory operations, so ... > os.chdir(self.WorkspaceDir) > > # >@@ -642,10 +647,18 @@ class WorkspaceAutoGen(AutoGen): > # > Pa.CollectPlatformDynamicPcds() > Pa.CollectFixedAtBuildPcds() > self.AutoGenObjectList.append(Pa) > >+ # >+ # Generate Package level hash value >+ # >+ GlobalData.gPackageHash[Arch] = {} >+ if GlobalData.gUseHashCache: >+ for Pkg in Pkgs: >+ self._GenPkgLevelHash(Pkg) >+ > # > # Check PCDs token value conflict in each DEC file. > # > self._CheckAllPcdsTokenValueConflict() > >@@ -655,15 +668,10 @@ class WorkspaceAutoGen(AutoGen): > self._CheckPcdDefineAndType() > > # if self.FdfFile: > # self._CheckDuplicateInFV(Fdf) > >- self._BuildDir = None >- self._FvDir = None >- self._MakeFileDir = None >- self._BuildCommand = None >- > # > # Create BuildOptions Macro & PCD metafile, also add the Active > Platform >and FDF file. > # > content = 'gCommandLineDefines: ' > content += str(GlobalData.gCommandLineDefines) >@@ -675,10 +683,14 @@ class WorkspaceAutoGen(AutoGen): > content += str(self.Platform) > content += os.linesep > if self.FdfFile: > content += 'Flash Image Definition: ' > content += str(self.FdfFile) >+ content += os.linesep >+ if GlobalData.gBinCacheDest: >+ content += 'Cache of .efi location: ' >+ content += str(GlobalData.gBinCacheDest) > SaveFileOnChange(os.path.join(self.BuildDir, 'BuildOptions'), content, >False) > > # > # Create PcdToken Number file for Dynamic/DynamicEx Pcd. > # >@@ -704,10 +716,22 @@ class WorkspaceAutoGen(AutoGen): > for f in AllWorkSpaceMetaFiles: > if os.stat(f)[8] > SrcTimeStamp: > SrcTimeStamp = os.stat(f)[8] > self._SrcTimeStamp = SrcTimeStamp > >+ if GlobalData.gUseHashCache: >+ m = hashlib.md5() >+ for files in AllWorkSpaceMetaFiles: >+ if files.endswith('.dec'): >+ continue >+ f = open(files, 'r') >+ Content = f.read() >+ f.close() >+ m.update(Content) >+ SaveFileOnChange(os.path.join(self.BuildDir, 'AutoGen.hash'), >m.hexdigest(), True) >+ GlobalData.gPlatformHash = m.hexdigest() >+ > # > # Write metafile list to build directory > # > AutoGenFilePath = os.path.join(self.BuildDir, 'AutoGen') > if os.path.exists (AutoGenFilePath): >@@ -717,10 +741,33 @@ class WorkspaceAutoGen(AutoGen): > with open(os.path.join(self.BuildDir, 'AutoGen'), 'w+') as file: > for f in AllWorkSpaceMetaFiles: > print >> file, f > return True > >+ def _GenPkgLevelHash(self, Pkg): >+ PkgDir = os.path.join(self.BuildDir, Pkg.Arch, Pkg.PackageName) >+ CreateDirectory(PkgDir) >+ HashFile = os.path.join(PkgDir, Pkg.PackageName + '.hash') >+ m = hashlib.md5() >+ # Get .dec file's hash value >+ f = open(Pkg.MetaFile.Path, 'r') >+ Content = f.read() >+ f.close() >+ m.update(Content) >+ # Get include files hash value >+ if Pkg.Includes: >+ for inc in Pkg.Includes: >+ for Root, Dirs, Files in os.walk(str(inc)): >+ for File in Files: >+ File_Path = os.path.join(Root, File) >+ f = open(File_Path, 'r') >+ Content = f.read() >+ f.close() >+ m.update(Content) >+ SaveFileOnChange(HashFile, m.hexdigest(), True) >+ if Pkg.PackageName not in GlobalData.gPackageHash[Pkg.Arch]: >+ GlobalData.gPackageHash[Pkg.Arch][Pkg.PackageName] = >m.hexdigest() > > def _GetMetaFiles(self, Target, Toolchain, Arch): > AllWorkSpaceMetaFiles = set() > # > # add fdf >@@ -954,11 +1001,12 @@ class WorkspaceAutoGen(AutoGen): > self._FvDir = path.join(self.BuildDir, 'FV') > return self._FvDir > > ## Return the directory to store all intermediate and final files built > def _GetBuildDir(self): >- return self.AutoGenObjectList[0].BuildDir >+ if self._BuildDir == None: >+ return self.AutoGenObjectList[0].BuildDir > > ## Return the build output directory platform specifies > def _GetOutputDir(self): > return self.Platform.OutputDirectory > >@@ -3898,37 +3946,45 @@ class ModuleAutoGen(AutoGen): > AsBuiltInfDict['module_uefi_specification_version'] += >[self.Specification['UEFI_SPECIFICATION_VERSION']] > if 'PI_SPECIFICATION_VERSION' in self.Specification: > AsBuiltInfDict['module_pi_specification_version'] += >[self.Specification['PI_SPECIFICATION_VERSION']] > > OutputDir = self.OutputDir.replace('\\', '/').strip('/') >- >+ self.OutputFile = [] > for Item in self.CodaTargetList: > File = Item.Target.Path.replace('\\', > '/').strip('/').replace(OutputDir, >'').strip('/') >+ if File not in self.OutputFile: >+ self.OutputFile.append(File) > if Item.Target.Ext.lower() == '.aml': > AsBuiltInfDict['binary_item'] += ['ASL|' + File] > elif Item.Target.Ext.lower() == '.acpi': > AsBuiltInfDict['binary_item'] += ['ACPI|' + File] > elif Item.Target.Ext.lower() == '.efi': > AsBuiltInfDict['binary_item'] += ['PE32|' + self.Name + > '.efi'] > else: > AsBuiltInfDict['binary_item'] += ['BIN|' + File] > if self.DepexGenerated: >+ if self.Name + '.depex' not in self.OutputFile: >+ self.OutputFile.append(self.Name + '.depex') > if self.ModuleType in ['PEIM']: > AsBuiltInfDict['binary_item'] += ['PEI_DEPEX|' + self.Name + > '.depex'] > if self.ModuleType in ['DXE_DRIVER', 'DXE_RUNTIME_DRIVER', >'DXE_SAL_DRIVER', 'UEFI_DRIVER']: > AsBuiltInfDict['binary_item'] += ['DXE_DEPEX|' + self.Name + > '.depex'] > if self.ModuleType in ['DXE_SMM_DRIVER']: > AsBuiltInfDict['binary_item'] += ['SMM_DEPEX|' + self.Name + >'.depex'] > > Bin = self._GenOffsetBin() > if Bin: > AsBuiltInfDict['binary_item'] += ['BIN|%s' % Bin] >+ if Bin not in self.OutputFile: >+ self.OutputFile.append(Bin) > > for Root, Dirs, Files in os.walk(OutputDir): > for File in Files: > if File.lower().endswith('.pdb'): > AsBuiltInfDict['binary_item'] += ['DISPOSABLE|' + File] >+ if File not in self.OutputFile: >+ self.OutputFile.append(File) > HeaderComments = self.Module.HeaderComments > StartPos = 0 > for Index in range(len(HeaderComments)): > if HeaderComments[Index].find('@BinaryHeader') != -1: > HeaderComments[Index] = >HeaderComments[Index].replace('@BinaryHeader', '@file') >@@ -4105,11 +4161,52 @@ class ModuleAutoGen(AutoGen): > AsBuiltInf.Append(gAsBuiltInfHeaderString.Replace(AsBuiltInfDict)) > > SaveFileOnChange(os.path.join(self.OutputDir, self.Name + '.inf'), >str(AsBuiltInf), False) > > self.IsAsBuiltInfCreated = True >- >+ if GlobalData.gBinCacheDest: >+ self.CopyModuleToCache() >+ >+ def CopyModuleToCache(self): >+ FileDir = path.join(GlobalData.gBinCacheDest, self.Arch, >self.SourceDir, >self.MetaFile.BaseName) >+ CreateDirectory (FileDir) >+ HashFile = path.join(self.BuildDir, self.Name + '.hash') >+ ModuleFile = path.join(self.OutputDir, self.Name + '.inf') >+ if os.path.exists(HashFile): >+ shutil.copy2(HashFile, FileDir) >+ if os.path.exists(ModuleFile): >+ shutil.copy2(ModuleFile, FileDir) >+ if self.OutputFile: >+ for File in self.OutputFile: >+ if not os.path.isabs(File): >+ File = os.path.join(self.OutputDir, File) >+ if os.path.exists(File): >+ shutil.copy2(File, FileDir) >+ >+ def AttemptModuleCacheCopy(self): >+ if self.IsBinaryModule: >+ return False >+ FileDir = path.join(GlobalData.gBinCacheSource, self.Arch, >self.SourceDir, >self.MetaFile.BaseName) >+ HashFile = path.join(FileDir, self.Name + '.hash') >+ if os.path.exists(HashFile): >+ f = open(HashFile, 'r') >+ CacheHash = f.read() >+ f.close() >+ if GlobalData.gModuleHash[self.Arch][self.Name]: >+ if CacheHash == GlobalData.gModuleHash[self.Arch][self.Name]: >+ for root, dir, files in os.walk(FileDir): >+ for f in files: >+ if self.Name + '.hash' in f: >+ shutil.copy2(HashFile, self.BuildDir) >+ else: >+ File = path.join(root, f) >+ shutil.copy2(File, self.OutputDir) >+ if self.Name == "PcdPeim" or self.Name == "PcdDxe": >+ CreatePcdDatabaseCode(self, TemplateString(), >TemplateString()) >+ return True >+ return False >+ > ## Create makefile for the module and its dependent libraries > # > # @param CreateLibraryMakeFile Flag indicating if or not the > makefiles >of > # dependent libraries will be > created > # >@@ -4233,12 +4330,58 @@ class ModuleAutoGen(AutoGen): > self._LibraryAutoGenList.append(La) > for Lib in La.CodaTargetList: > self._ApplyBuildRule(Lib.Target, TAB_UNKNOWN_FILE) > return self._LibraryAutoGenList > >+ def GenModuleHash(self): >+ if self.Arch not in GlobalData.gModuleHash: >+ GlobalData.gModuleHash[self.Arch] = {} >+ m = hashlib.md5() >+ # Add Platform level hash >+ m.update(GlobalData.gPlatformHash) >+ # Add Package level hash >+ if self.DependentPackageList: >+ for Pkg in self.DependentPackageList: >+ if Pkg.PackageName in GlobalData.gPackageHash[self.Arch]: >+ >m.update(GlobalData.gPackageHash[self.Arch][Pkg.PackageName]) >+ >+ # Add Library hash >+ if self.LibraryAutoGenList: >+ for Lib in self.LibraryAutoGenList: >+ if Lib.Name not in GlobalData.gModuleHash[self.Arch]: >+ Lib.GenModuleHash() >+ m.update(GlobalData.gModuleHash[self.Arch][Lib.Name]) >+ >+ # Add Module self >+ f = open(str(self.MetaFile), 'r') >+ Content = f.read() >+ f.close() >+ m.update(Content) >+ # Add Module's source files >+ if self.SourceFileList: >+ for File in self.SourceFileList: >+ f = open(str(File), 'r') >+ Content = f.read() >+ f.close() >+ m.update(Content) >+ >+ ModuleHashFile = path.join(self.BuildDir, self.Name + ".hash") >+ if self.Name not in GlobalData.gModuleHash[self.Arch]: >+ GlobalData.gModuleHash[self.Arch][self.Name] = m.hexdigest() >+ if GlobalData.gBinCacheSource: >+ CacheValid = self.AttemptModuleCacheCopy() >+ if CacheValid: >+ return False >+ return SaveFileOnChange(ModuleHashFile, m.hexdigest(), True) >+ >+ ## Decide whether we can skip the ModuleAutoGen process >+ def CanSkipbyHash(self): >+ if GlobalData.gUseHashCache: >+ return not self.GenModuleHash() >+ > ## Decide whether we can skip the ModuleAutoGen process >- # If any source file is newer than the modeule than we cannot skip >+ # If any source file is newer than the module than we cannot skip > # > def CanSkip(self): > if not os.path.exists(self.GetTimeStampPath()): > return False > #last creation time of the module >diff --git a/BaseTools/Source/Python/Common/GlobalData.py >b/BaseTools/Source/Python/Common/GlobalData.py >index 667877e..45e7ea0 100644 >--- a/BaseTools/Source/Python/Common/GlobalData.py >+++ b/BaseTools/Source/Python/Common/GlobalData.py >@@ -85,5 +85,12 @@ BuildOptionPcd = [] > # > MixedPcd = {} > > # Pcd name for the Pcd which used in the Conditional directives > gConditionalPcds = [] >+ >+gUseHashCache = None >+gBinCacheDest = None >+gBinCacheSource = None >+gPlatformHash = None >+gPackageHash = {} >+gModuleHash = {} >diff --git a/BaseTools/Source/Python/build/build.py >b/BaseTools/Source/Python/build/build.py >index 7436453..bb34b87 100644 >--- a/BaseTools/Source/Python/build/build.py >+++ b/BaseTools/Source/Python/build/build.py >@@ -764,10 +764,34 @@ class Build(): > self.TargetTxt = TargetTxtClassObject() > self.ToolDef = ToolDefClassObject() > GlobalData.BuildOptionPcd = BuildOptions.OptionPcd > #Set global flag for build mode > GlobalData.gIgnoreSource = BuildOptions.IgnoreSources >+ GlobalData.gUseHashCache = BuildOptions.UseHashCache >+ GlobalData.gBinCacheDest = BuildOptions.BinCacheDest >+ GlobalData.gBinCacheSource = BuildOptions.BinCacheSource >+ >+ if GlobalData.gBinCacheDest and not GlobalData.gUseHashCache: >+ EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="-- >bindest must be used together with --hashcache.") >+ >+ if GlobalData.gBinCacheSource and not GlobalData.gUseHashCache: >+ EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="-- >binsource must be used together with --hashcache.") >+ >+ if GlobalData.gBinCacheDest and GlobalData.gBinCacheSource: >+ EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="-- >bindest can not be used together with --binsource.") >+ >+ if GlobalData.gBinCacheSource: >+ BinCacheSource = os.path.normpath(GlobalData.gBinCacheSource) >+ if not os.path.isabs(BinCacheSource): >+ BinCacheSource = mws.join(self.WorkspaceDir, BinCacheSource) >+ GlobalData.gBinCacheSource = BinCacheSource >+ >+ if GlobalData.gBinCacheDest: >+ BinCacheDest = os.path.normpath(GlobalData.gBinCacheDest) >+ if not os.path.isabs(BinCacheDest): >+ BinCacheDest = mws.join(self.WorkspaceDir, BinCacheDest) >+ GlobalData.gBinCacheDest = BinCacheDest > > if self.ConfDirectory: > # Get alternate Conf location, if it is absolute, then just use > the absolute >directory name > ConfDirectoryPath = os.path.normpath(self.ConfDirectory) > >@@ -1908,10 +1932,13 @@ class Build(): > # Get ModuleAutoGen object to generate C code file and >makefile > Ma = ModuleAutoGen(Wa, Module, BuildTarget, > ToolChain, Arch, >self.PlatformFile) > > if Ma == None: > continue >+ if Ma.CanSkipbyHash(): >+ continue >+ > # Not to auto-gen for targets 'clean', 'cleanlib', > 'cleanall', 'run', 'fds' > if self.Target not in ['clean', 'cleanlib', > 'cleanall', 'run', 'fds']: > # for target which must generate AutoGen code and > makefile > if not self.SkipAutoGen or self.Target == 'genc': > Ma.CreateCodeFile(True) >@@ -2213,10 +2240,13 @@ def MyOptionParser(): > Parser.add_option("--conf", action="store", type="string", >dest="ConfDirectory", help="Specify the customized Conf directory.") > Parser.add_option("--check-usage", action="store_true", >dest="CheckUsage", default=False, help="Check usage content of entries >listed in INF file.") > Parser.add_option("--ignore-sources", action="store_true", >dest="IgnoreSources", default=False, help="Focus to a binary build and ignore >all source files") > Parser.add_option("--pcd", action="append", dest="OptionPcd", help="Set >PCD value by command line. Format: \"PcdName=Value\" ") > Parser.add_option("-l", "--cmd-len", action="store", type="int", >dest="CommandLength", help="Specify the maximum line length of build >command. Default is 4096.") >+ Parser.add_option("--hash", action="store_true", dest="UseHashCache", >default=False, help="Enable hash-based caching during build process.") >+ Parser.add_option("--bindest", action="store", type="string", >dest="BinCacheDest", help="Generate a cache of binary files in the specified >directory.") >+ Parser.add_option("--binsource", action="store", type="string", >dest="BinCacheSource", help="Consume a cache of binary files from the >specified directory.") > > (Opt, Args) = Parser.parse_args() > return (Opt, Args) > > ## Tool entrance method >-- >2.6.1.windows.1
_______________________________________________ edk2-devel mailing list [email protected] https://lists.01.org/mailman/listinfo/edk2-devel
